Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Pull crypto updates from Herbert Xu:
 "Algorithms:
   - add private key generation to ecdh

  Drivers:
   - add generic gcm(aes) to aesni-intel
   - add SafeXcel EIP197 crypto engine driver
   - add ecb(aes), cfb(aes) and ecb(des3_ede) to cavium
   - add support for CNN55XX adapters in cavium
   - add ctr mode to chcr
   - add support for gcm(aes) to omap"

* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (140 commits)
  crypto: testmgr - Reenable sha1/aes in FIPS mode
  crypto: ccp - Release locks before returning
  crypto: cavium/nitrox - dma_mapping_error() returns bool
  crypto: doc - fix typo in docs
  Documentation/bindings: Document the SafeXel cryptographic engine driver
  crypto: caam - fix gfp allocation flags (part II)
  crypto: caam - fix gfp allocation flags (part I)
  crypto: drbg - Fixes panic in wait_for_completion call
  crypto: caam - make of_device_ids const.
  crypto: vmx - remove unnecessary check
  crypto: n2 - make of_device_ids const
  crypto: inside-secure - use the base_end pointer in ring rollback
  crypto: inside-secure - increase the batch size
  crypto: inside-secure - only dequeue when needed
  crypto: inside-secure - get the backlog before dequeueing the request
  crypto: inside-secure - stop requeueing failed requests
  crypto: inside-secure - use one queue per hw ring
  crypto: inside-secure - update the context and request later
  crypto: inside-secure - align the cipher and hash send functions
  crypto: inside-secure - optimize DSE bufferability control
  ...

127 files changed, 12841 insertions, 1535 deletions

diff --git a/Documentation/crypto/api-samples.rst b/Documentation/crypto/api-samples.rst
index d021fd96a76d..2531948db89f 100644
--- a/Documentation/crypto/api-samples.rst
+++ b/Documentation/crypto/api-samples.rst
@@ -155,9 +155,9 @@ Code Example For Use of Operational State Memory With SHASH
         char ctx[];
     };
 
-    static struct sdesc init_sdesc(struct crypto_shash *alg)
+    static struct sdesc *init_sdesc(struct crypto_shash *alg)
     {
-        struct sdesc sdesc;
+        struct sdesc *sdesc;
         int size;
 
         size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);
@@ -169,15 +169,16 @@ Code Example For Use of Operational State Memory With SHASH
         return sdesc;
     }
 
-    static int calc_hash(struct crypto_shashalg,
-                 const unsigned chardata, unsigned int datalen,
-                 unsigned chardigest) {
-        struct sdesc sdesc;
+    static int calc_hash(struct crypto_shash *alg,
+                 const unsigned char *data, unsigned int datalen,
+                 unsigned char *digest)
+    {
+        struct sdesc *sdesc;
         int ret;
 
         sdesc = init_sdesc(alg);
         if (IS_ERR(sdesc)) {
-            pr_info("trusted_key: can't alloc %s\n", hash_alg);
+            pr_info("can't alloc sdesc\n");
             return PTR_ERR(sdesc);
         }
 
@@ -186,6 +187,23 @@ Code Example For Use of Operational State Memory With SHASH
         return ret;
     }
 
+    static int test_hash(const unsigned char *data, unsigned int datalen,
+                 unsigned char *digest)
+    {
+        struct crypto_shash *alg;
+        char *hash_alg_name = "sha1-padlock-nano";
+        int ret;
+
+        alg = crypto_alloc_shash(hash_alg_name, CRYPTO_ALG_TYPE_SHASH, 0);
+        if (IS_ERR(alg)) {
+                pr_info("can't alloc alg %s\n", hash_alg_name);
+                return PTR_ERR(alg);
+        }
+        ret = calc_hash(alg, data, datalen, digest);
+        crypto_free_shash(alg);
+        return ret;
+    }
+
 
 Code Example For Random Number Generator Usage
 ----------------------------------------------
@@ -195,8 +213,8 @@ Code Example For Random Number Generator Usage
 
     static int get_random_numbers(u8 *buf, unsigned int len)
     {
-        struct crypto_rngrng = NULL;
-        chardrbg = "drbg_nopr_sha256"; /* Hash DRBG with SHA-256, no PR */
+        struct crypto_rng *rng = NULL;
+        char *drbg = "drbg_nopr_sha256"; /* Hash DRBG with SHA-256, no PR */
         int ret;
 
         if (!buf || !len) {
@@ -207,7 +225,7 @@ Code Example For Random Number Generator Usage
         rng = crypto_alloc_rng(drbg, 0, 0);
         if (IS_ERR(rng)) {
             pr_debug("could not allocate RNG handle for %s\n", drbg);
-            return -PTR_ERR(rng);
+            return PTR_ERR(rng);
         }
 
         ret = crypto_rng_get_bytes(rng, buf, len);
diff --git a/Documentation/crypto/userspace-if.rst b/Documentation/crypto/userspace-if.rst
index de5a72e32bc9..ff86befa61e0 100644
--- a/Documentation/crypto/userspace-if.rst
+++ b/Documentation/crypto/userspace-if.rst
@@ -327,7 +327,7 @@ boundary. Non-aligned data can be used as well, but may require more
 operations of the kernel which would defeat the speed gains obtained
 from the zero-copy interface.
 
-The system-interent limit for the size of one zero-copy operation is 16
+The system-inherent limit for the size of one zero-copy operation is 16
 pages. If more data is to be sent to AF_ALG, user space must slice the
 input into segments with a maximum size of 16 pages.
 
diff --git a/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt b/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt
new file mode 100644
index 000000000000..f69773f4252b
--- /dev/null
+++ b/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt
@@ -0,0 +1,29 @@
+Inside Secure SafeXcel cryptographic engine
+
+Required properties:
+- compatible: Should be "inside-secure,safexcel-eip197".
+- reg: Base physical address of the engine and length of memory mapped region.
+- interrupts: Interrupt numbers for the rings and engine.
+- interrupt-names: Should be "ring0", "ring1", "ring2", "ring3", "eip", "mem".
+
+Optional properties:
+- clocks: Reference to the crypto engine clock.
+- dma-mask: The address mask limitation. Defaults to 64.
+
+Example:
+
+        crypto: crypto@800000 {
+                compatible = "inside-secure,safexcel-eip197";
+                reg = <0x800000 0x200000>;
+                interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>,
+                             <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
+                             <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>,
+                             <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>,
+                             <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
+                             <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
+                interrupt-names = "mem", "ring0", "ring1", "ring2", "ring3",
+                                  "eip";
+                clocks = <&cpm_syscon0 1 26>;
+                dma-mask = <0xff 0xffffffff>;
+                status = "disabled";
+        };
diff --git a/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt b/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt
index c204725e5873..450da3661cad 100644
--- a/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt
+++ b/Documentation/devicetree/bindings/crypto/mediatek-crypto.txt
@@ -6,8 +6,7 @@ Required properties:
 - interrupts: Should contain the five crypto engines interrupts in numeric
         order. These are global system and four descriptor rings.
 - clocks: the clock used by the core
-- clock-names: the names of the clock listed in the clocks property. These are
-        "ethif", "cryp"
+- clock-names: Must contain "cryp".
 - power-domains: Must contain a reference to the PM domain.
 
 
@@ -20,8 +19,7 @@ Example:
                              <GIC_SPI 84 IRQ_TYPE_LEVEL_LOW>,
                              <GIC_SPI 91 IRQ_TYPE_LEVEL_LOW>,
                              <GIC_SPI 97 IRQ_TYPE_LEVEL_LOW>;
-                clocks = <&topckgen CLK_TOP_ETHIF_SEL>,
-                         <&ethsys CLK_ETHSYS_CRYPTO>;
-                clock-names = "ethif","cryp";
+                clocks = <&ethsys CLK_ETHSYS_CRYPTO>;
+                clock-names = "cryp";
                 power-domains = <&scpsys MT2701_POWER_DOMAIN_ETH>;
         };
diff --git a/Documentation/devicetree/bindings/rng/mtk-rng.txt b/Documentation/devicetree/bindings/rng/mtk-rng.txt
index a6d62a2abd39..366b99bff8cd 100644
--- a/Documentation/devicetree/bindings/rng/mtk-rng.txt
+++ b/Documentation/devicetree/bindings/rng/mtk-rng.txt
@@ -2,7 +2,9 @@ Device-Tree bindings for Mediatek random number generator
 found in Mediatek SoC family
 
 Required properties:
-- compatible        : Should be "mediatek,mt7623-rng"
+- compatible        : Should be
+                        "mediatek,mt7622-rng",  "mediatek,mt7623-rng" : for MT7622
+                        "mediatek,mt7623-rng" : for MT7623
 - clocks            : list of clock specifiers, corresponding to
                       entries in clock-names property;
 - clock-names       : Should contain "rng" entries;
diff --git a/Documentation/devicetree/bindings/rng/timeriomem_rng.txt b/Documentation/devicetree/bindings/rng/timeriomem_rng.txt
index 6616d15866a3..214940093b55 100644
--- a/Documentation/devicetree/bindings/rng/timeriomem_rng.txt
+++ b/Documentation/devicetree/bindings/rng/timeriomem_rng.txt
@@ -5,6 +5,13 @@ Required properties:
 - reg : base address to sample from
 - period : wait time in microseconds to use between samples
 
+Optional properties:
+- quality : estimated number of bits of true entropy per 1024 bits read from the
+            rng.  Defaults to zero which causes the kernel's default quality to
+            be used instead.  Note that the default quality is usually zero
+            which disables using this rng to automatically fill the kernel's
+            entropy pool.
+
 N.B. currently 'reg' must be four bytes wide and aligned
 
 Example:
diff --git a/MAINTAINERS b/MAINTAINERS
index 61d0ac6f535c..df62ee151b62 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -3746,6 +3746,13 @@ S:	Supported
 F:      drivers/infiniband/hw/cxgb4/
 F:      include/uapi/rdma/cxgb4-abi.h
 
+CXGB4 CRYPTO DRIVER (chcr)
+M:      Harsh Jain <harsh@chelsio.com>
+L:      linux-crypto@vger.kernel.org
+W:      http://www.chelsio.com
+S:      Supported
+F:      drivers/crypto/chelsio
+
 CXGB4VF ETHERNET DRIVER (CXGB4VF)
 M:      Casey Leedom <leedom@chelsio.com>
 L:      netdev@vger.kernel.org
@@ -6647,6 +6654,12 @@ F:	Documentation/input/multi-touch-protocol.rst
 F:      drivers/input/input-mt.c
 K:      \b(ABS|SYN)_MT_
 
+INSIDE SECURE CRYPTO DRIVER
+M:      Antoine Tenart <antoine.tenart@free-electrons.com>
+F:      drivers/crypto/inside-secure/
+S:      Maintained
+L:      linux-crypto@vger.kernel.org
+
 INTEL ASoC BDW/HSW DRIVERS
 M:      Jie Yang <yang.jie@linux.intel.com>
 L:      alsa-devel@alsa-project.org (moderated for non-subscribers)
@@ -8306,6 +8319,11 @@ L:	linux-wireless@vger.kernel.org
 S:      Maintained
 F:      drivers/net/wireless/mediatek/mt7601u/
 
+MEDIATEK RANDOM NUMBER GENERATOR SUPPORT
+M:      Sean Wang <sean.wang@mediatek.com>
+S:      Maintained
+F:      drivers/char/hw_random/mtk-rng.c
+
 MEGACHIPS STDPXXXX-GE-B850V3-FW LVDS/DP++ BRIDGES
 M:      Peter Senna Tschudin <peter.senna@collabora.com>
 M:      Martin Donnelly <martin.donnelly@ge.com>
diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c
index 883b84d828c5..0f966a8ca1ce 100644
--- a/arch/arm/crypto/aes-ce-glue.c
+++ b/arch/arm/crypto/aes-ce-glue.c
@@ -14,6 +14,7 @@
 #include <crypto/aes.h>
 #include <crypto/internal/simd.h>
 #include <crypto/internal/skcipher.h>
+#include <linux/cpufeature.h>
 #include <linux/module.h>
 #include <crypto/xts.h>
 
@@ -425,9 +426,6 @@ static int __init aes_init(void)
         int err;
         int i;
 
-        if (!(elf_hwcap2 & HWCAP2_AES))
-                return -ENODEV;
-
         err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
         if (err)
                 return err;
@@ -451,5 +449,5 @@ unregister_simds:
         return err;
 }
 
-module_init(aes_init);
+module_cpu_feature_match(AES, aes_init);
 module_exit(aes_exit);
diff --git a/arch/arm/crypto/crc32-ce-glue.c b/arch/arm/crypto/crc32-ce-glue.c
index e1566bec1016..1b0e0e86ee9c 100644
--- a/arch/arm/crypto/crc32-ce-glue.c
+++ b/arch/arm/crypto/crc32-ce-glue.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation.
  */
 
+#include <linux/cpufeature.h>
 #include <linux/crc32.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -233,6 +234,11 @@ static void __exit crc32_pmull_mod_exit(void)
                                   ARRAY_SIZE(crc32_pmull_algs));
 }
 
+static const struct cpu_feature crc32_cpu_feature[] = {
+        { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
+};
+MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
+
 module_init(crc32_pmull_mod_init);
 module_exit(crc32_pmull_mod_exit);
 
diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c
index 7546b3c02466..6bac8bea9f1e 100644
--- a/arch/arm/crypto/ghash-ce-glue.c
+++ b/arch/arm/crypto/ghash-ce-glue.c
@@ -15,6 +15,7 @@
 #include <crypto/cryptd.h>
 #include <crypto/internal/hash.h>
 #include <crypto/gf128mul.h>
+#include <linux/cpufeature.h>
 #include <linux/crypto.h>
 #include <linux/module.h>
 
@@ -311,9 +312,6 @@ static int __init ghash_ce_mod_init(void)
 {
         int err;
 
-        if (!(elf_hwcap2 & HWCAP2_PMULL))
-                return -ENODEV;
-
         err = crypto_register_shash(&ghash_alg);
         if (err)
                 return err;
@@ -334,5 +332,5 @@ static void __exit ghash_ce_mod_exit(void)
         crypto_unregister_shash(&ghash_alg);
 }
 
-module_init(ghash_ce_mod_init);
+module_cpu_feature_match(PMULL, ghash_ce_mod_init);
 module_exit(ghash_ce_mod_exit);
diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c
index 80bc2fcd241a..555f72b5e659 100644
--- a/arch/arm/crypto/sha1-ce-glue.c
+++ b/arch/arm/crypto/sha1-ce-glue.c
@@ -11,6 +11,7 @@
 #include <crypto/internal/hash.h>
 #include <crypto/sha.h>
 #include <crypto/sha1_base.h>
+#include <linux/cpufeature.h>
 #include <linux/crypto.h>
 #include <linux/module.h>
 
@@ -82,8 +83,6 @@ static struct shash_alg alg = {
 
 static int __init sha1_ce_mod_init(void)
 {
-        if (!(elf_hwcap2 & HWCAP2_SHA1))
-                return -ENODEV;
         return crypto_register_shash(&alg);
 }
 
@@ -92,5 +91,5 @@ static void __exit sha1_ce_mod_fini(void)
         crypto_unregister_shash(&alg);
 }
 
-module_init(sha1_ce_mod_init);
+module_cpu_feature_match(SHA1, sha1_ce_mod_init);
 module_exit(sha1_ce_mod_fini);
diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c
index 0755b2d657f3..df4dcef054ae 100644
--- a/arch/arm/crypto/sha2-ce-glue.c
+++ b/arch/arm/crypto/sha2-ce-glue.c
@@ -11,6 +11,7 @@
 #include <crypto/internal/hash.h>
 #include <crypto/sha.h>
 #include <crypto/sha256_base.h>
+#include <linux/cpufeature.h>
 #include <linux/crypto.h>
 #include <linux/module.h>
 
@@ -100,8 +101,6 @@ static struct shash_alg algs[] = { {
 
 static int __init sha2_ce_mod_init(void)
 {
-        if (!(elf_hwcap2 & HWCAP2_SHA2))
-                return -ENODEV;
         return crypto_register_shashes(algs, ARRAY_SIZE(algs));
 }
 
@@ -110,5 +109,5 @@ static void __exit sha2_ce_mod_fini(void)
         crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
 }
 
-module_init(sha2_ce_mod_init);
+module_cpu_feature_match(SHA2, sha2_ce_mod_init);
 module_exit(sha2_ce_mod_fini);
diff --git a/arch/arm64/crypto/sha1-ce-core.S b/arch/arm64/crypto/sha1-ce-core.S
index c98e7e849f06..8550408735a0 100644
--- a/arch/arm64/crypto/sha1-ce-core.S
+++ b/arch/arm64/crypto/sha1-ce-core.S
@@ -82,7 +82,8 @@ ENTRY(sha1_ce_transform)
         ldr             dgb, [x0, #16]
 
         /* load sha1_ce_state::finalize */
-        ldr             w4, [x0, #:lo12:sha1_ce_offsetof_finalize]
+        ldr_l           w4, sha1_ce_offsetof_finalize, x4
+        ldr             w4, [x0, x4]
 
         /* load input */
 0:      ld1             {v8.4s-v11.4s}, [x1], #64
@@ -132,7 +133,8 @@ CPU_LE(	rev32		v11.16b, v11.16b	)
          * the padding is handled by the C code in that case.
          */
         cbz             x4, 3f
-        ldr             x4, [x0, #:lo12:sha1_ce_offsetof_count]
+        ldr_l           w4, sha1_ce_offsetof_count, x4
+        ldr             x4, [x0, x4]
         movi            v9.2d, #0
         mov             x8, #0x80000000
         movi            v10.2d, #0
diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c
index aefda9868627..ea319c055f5d 100644
--- a/arch/arm64/crypto/sha1-ce-glue.c
+++ b/arch/arm64/crypto/sha1-ce-glue.c
@@ -17,9 +17,6 @@
 #include <linux/crypto.h>
 #include <linux/module.h>
 
-#define ASM_EXPORT(sym, val) \
-        asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val));
-
 MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions");
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 MODULE_LICENSE("GPL v2");
@@ -32,6 +29,9 @@ struct sha1_ce_state {
 asmlinkage void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src,
                                   int blocks);
 
+const u32 sha1_ce_offsetof_count = offsetof(struct sha1_ce_state, sst.count);
+const u32 sha1_ce_offsetof_finalize = offsetof(struct sha1_ce_state, finalize);
+
 static int sha1_ce_update(struct shash_desc *desc, const u8 *data,
                           unsigned int len)
 {
@@ -52,11 +52,6 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data,
         struct sha1_ce_state *sctx = shash_desc_ctx(desc);
         bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE);
 
-        ASM_EXPORT(sha1_ce_offsetof_count,
-                   offsetof(struct sha1_ce_state, sst.count));
-        ASM_EXPORT(sha1_ce_offsetof_finalize,
-                   offsetof(struct sha1_ce_state, finalize));
-
         /*
          * Allow the asm code to perform the finalization if there is no
          * partial data and the input is a round multiple of the block size.
diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/crypto/sha2-ce-core.S
index 01cfee066837..679c6c002f4f 100644
--- a/arch/arm64/crypto/sha2-ce-core.S
+++ b/arch/arm64/crypto/sha2-ce-core.S
@@ -88,7 +88,8 @@ ENTRY(sha2_ce_transform)
         ld1             {dgav.4s, dgbv.4s}, [x0]
 
         /* load sha256_ce_state::finalize */
-        ldr             w4, [x0, #:lo12:sha256_ce_offsetof_finalize]
+        ldr_l           w4, sha256_ce_offsetof_finalize, x4
+        ldr             w4, [x0, x4]
 
         /* load input */
 0:      ld1             {v16.4s-v19.4s}, [x1], #64
@@ -136,7 +137,8 @@ CPU_LE(	rev32		v19.16b, v19.16b	)
          * the padding is handled by the C code in that case.
          */
         cbz             x4, 3f
-        ldr             x4, [x0, #:lo12:sha256_ce_offsetof_count]
+        ldr_l           w4, sha256_ce_offsetof_count, x4
+        ldr             x4, [x0, x4]
         movi            v17.2d, #0
         mov             x8, #0x80000000
         movi            v18.2d, #0
diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c
index 7cd587564a41..0ed9486f75dd 100644
--- a/arch/arm64/crypto/sha2-ce-glue.c
+++ b/arch/arm64/crypto/sha2-ce-glue.c
@@ -17,9 +17,6 @@
 #include <linux/crypto.h>
 #include <linux/module.h>
 
-#define ASM_EXPORT(sym, val) \
-        asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val));
-
 MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions");
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 MODULE_LICENSE("GPL v2");
@@ -32,6 +29,11 @@ struct sha256_ce_state {
 asmlinkage void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src,
                                   int blocks);
 
+const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state,
+                                              sst.count);
+const u32 sha256_ce_offsetof_finalize = offsetof(struct sha256_ce_state,
+                                                 finalize);
+
 static int sha256_ce_update(struct shash_desc *desc, const u8 *data,
                             unsigned int len)
 {
@@ -52,11 +54,6 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data,
         struct sha256_ce_state *sctx = shash_desc_ctx(desc);
         bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE);
 
-        ASM_EXPORT(sha256_ce_offsetof_count,
-                   offsetof(struct sha256_ce_state, sst.count));
-        ASM_EXPORT(sha256_ce_offsetof_finalize,
-                   offsetof(struct sha256_ce_state, finalize));
-
         /*
          * Allow the asm code to perform the finalization if there is no
          * partial data and the input is a round multiple of the block size.
diff --git a/arch/x86/crypto/aes-x86_64-asm_64.S b/arch/x86/crypto/aes-x86_64-asm_64.S
index 910565547163..8739cf7795de 100644
--- a/arch/x86/crypto/aes-x86_64-asm_64.S
+++ b/arch/x86/crypto/aes-x86_64-asm_64.S
@@ -42,17 +42,15 @@
 #define R5E     %esi
 #define R6      %rdi
 #define R6E     %edi
-#define R7      %rbp
-#define R7E     %ebp
+#define R7      %r9     /* don't use %rbp; it breaks stack traces */
+#define R7E     %r9d
 #define R8      %r8
-#define R9      %r9
 #define R10     %r10
 #define R11     %r11
 
-#define prologue(FUNC,KEY,B128,B192,r1,r2,r3,r4,r5,r6,r7,r8,r9,r10,r11) \
+#define prologue(FUNC,KEY,B128,B192,r1,r2,r5,r6,r7,r8,r9,r10,r11) \
         ENTRY(FUNC);                    \
         movq    r1,r2;                  \
-        movq    r3,r4;                  \
         leaq    KEY+48(r8),r9;          \
         movq    r10,r11;                \
         movl    (r7),r5 ## E;           \
@@ -70,9 +68,8 @@
         je      B192;                   \
         leaq    32(r9),r9;
 
-#define epilogue(FUNC,r1,r2,r3,r4,r5,r6,r7,r8,r9) \
+#define epilogue(FUNC,r1,r2,r5,r6,r7,r8,r9) \
         movq    r1,r2;                  \
-        movq    r3,r4;                  \
         movl    r5 ## E,(r9);           \
         movl    r6 ## E,4(r9);          \
         movl    r7 ## E,8(r9);          \
@@ -88,12 +85,12 @@
         movl    TAB(,r6,4),r6 ## E;     \
         roll    $16,r2 ## E;            \
         shrl    $16,r4 ## E;            \
-        movzbl  r4 ## H,r7 ## E;        \
-        movzbl  r4 ## L,r4 ## E;        \
+        movzbl  r4 ## L,r7 ## E;        \
+        movzbl  r4 ## H,r4 ## E;        \
         xorl    OFFSET(r8),ra ## E;     \
         xorl    OFFSET+4(r8),rb ## E;   \
-        xorl    TAB+3072(,r7,4),r5 ## E;\
-        xorl    TAB+2048(,r4,4),r6 ## E;\
+        xorl    TAB+3072(,r4,4),r5 ## E;\
+        xorl    TAB+2048(,r7,4),r6 ## E;\
         movzbl  r1 ## L,r7 ## E;        \
         movzbl  r1 ## H,r4 ## E;        \
         movl    TAB+1024(,r4,4),r4 ## E;\
@@ -101,19 +98,19 @@
         roll    $16,r1 ## E;            \
         shrl    $16,r3 ## E;            \
         xorl    TAB(,r7,4),r5 ## E;     \
-        movzbl  r3 ## H,r7 ## E;        \
-        movzbl  r3 ## L,r3 ## E;        \
-        xorl    TAB+3072(,r7,4),r4 ## E;\
-        xorl    TAB+2048(,r3,4),r5 ## E;\
-        movzbl  r1 ## H,r7 ## E;        \
-        movzbl  r1 ## L,r3 ## E;        \
+        movzbl  r3 ## L,r7 ## E;        \
+        movzbl  r3 ## H,r3 ## E;        \
+        xorl    TAB+3072(,r3,4),r4 ## E;\
+        xorl    TAB+2048(,r7,4),r5 ## E;\
+        movzbl  r1 ## L,r7 ## E;        \
+        movzbl  r1 ## H,r3 ## E;        \
         shrl    $16,r1 ## E;            \
-        xorl    TAB+3072(,r7,4),r6 ## E;\
-        movl    TAB+2048(,r3,4),r3 ## E;\
-        movzbl  r1 ## H,r7 ## E;        \
-        movzbl  r1 ## L,r1 ## E;        \
-        xorl    TAB+1024(,r7,4),r6 ## E;\
-        xorl    TAB(,r1,4),r3 ## E;     \
+        xorl    TAB+3072(,r3,4),r6 ## E;\
+        movl    TAB+2048(,r7,4),r3 ## E;\
+        movzbl  r1 ## L,r7 ## E;        \
+        movzbl  r1 ## H,r1 ## E;        \
+        xorl    TAB+1024(,r1,4),r6 ## E;\
+        xorl    TAB(,r7,4),r3 ## E;     \
         movzbl  r2 ## H,r1 ## E;        \
         movzbl  r2 ## L,r7 ## E;        \
         shrl    $16,r2 ## E;            \
@@ -131,9 +128,9 @@
         movl    r4 ## E,r2 ## E;
 
 #define entry(FUNC,KEY,B128,B192) \
-        prologue(FUNC,KEY,B128,B192,R2,R8,R7,R9,R1,R3,R4,R6,R10,R5,R11)
+        prologue(FUNC,KEY,B128,B192,R2,R8,R1,R3,R4,R6,R10,R5,R11)
 
-#define return(FUNC) epilogue(FUNC,R8,R2,R9,R7,R5,R6,R3,R4,R11)
+#define return(FUNC) epilogue(FUNC,R8,R2,R5,R6,R3,R4,R11)
 
 #define encrypt_round(TAB,OFFSET) \
         round(TAB,OFFSET,R1,R2,R3,R4,R5,R6,R7,R10,R5,R6,R3,R4) \
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 3c465184ff8a..16627fec80b2 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -89,6 +89,29 @@ SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100
 ALL_F:      .octa 0xffffffffffffffffffffffffffffffff
             .octa 0x00000000000000000000000000000000
 
+.section .rodata
+.align 16
+.type aad_shift_arr, @object
+.size aad_shift_arr, 272
+aad_shift_arr:
+        .octa     0xffffffffffffffffffffffffffffffff
+        .octa     0xffffffffffffffffffffffffffffff0C
+        .octa     0xffffffffffffffffffffffffffff0D0C
+        .octa     0xffffffffffffffffffffffffff0E0D0C
+        .octa     0xffffffffffffffffffffffff0F0E0D0C
+        .octa     0xffffffffffffffffffffff0C0B0A0908
+        .octa     0xffffffffffffffffffff0D0C0B0A0908
+        .octa     0xffffffffffffffffff0E0D0C0B0A0908
+        .octa     0xffffffffffffffff0F0E0D0C0B0A0908
+        .octa     0xffffffffffffff0C0B0A090807060504
+        .octa     0xffffffffffff0D0C0B0A090807060504
+        .octa     0xffffffffff0E0D0C0B0A090807060504
+        .octa     0xffffffff0F0E0D0C0B0A090807060504
+        .octa     0xffffff0C0B0A09080706050403020100
+        .octa     0xffff0D0C0B0A09080706050403020100
+        .octa     0xff0E0D0C0B0A09080706050403020100
+        .octa     0x0F0E0D0C0B0A09080706050403020100
+
 
 .text
 
@@ -252,32 +275,66 @@ XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
         mov        arg8, %r12           # %r12 = aadLen
         mov        %r12, %r11
         pxor       %xmm\i, %xmm\i
+        pxor       \XMM2, \XMM2
 
-_get_AAD_loop\num_initial_blocks\operation:
-        movd       (%r10), \TMP1
-        pslldq     $12, \TMP1
-        psrldq     $4, %xmm\i
+        cmp        $16, %r11
+        jl         _get_AAD_rest8\num_initial_blocks\operation
+_get_AAD_blocks\num_initial_blocks\operation:
+        movdqu     (%r10), %xmm\i
+        PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data
+        pxor       %xmm\i, \XMM2
+        GHASH_MUL  \XMM2, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        add        $16, %r10
+        sub        $16, %r12
+        sub        $16, %r11
+        cmp        $16, %r11
+        jge        _get_AAD_blocks\num_initial_blocks\operation
+
+        movdqu     \XMM2, %xmm\i
+        cmp        $0, %r11
+        je         _get_AAD_done\num_initial_blocks\operation
+
+        pxor       %xmm\i,%xmm\i
+
+        /* read the last <16B of AAD. since we have at least 4B of
+        data right after the AAD (the ICV, and maybe some CT), we can
+        read 4B/8B blocks safely, and then get rid of the extra stuff */
+_get_AAD_rest8\num_initial_blocks\operation:
+        cmp        $4, %r11
+        jle        _get_AAD_rest4\num_initial_blocks\operation
+        movq       (%r10), \TMP1
+        add        $8, %r10
+        sub        $8, %r11
+        pslldq     $8, \TMP1
+        psrldq     $8, %xmm\i
         pxor       \TMP1, %xmm\i
+        jmp        _get_AAD_rest8\num_initial_blocks\operation
+_get_AAD_rest4\num_initial_blocks\operation:
+        cmp        $0, %r11
+        jle        _get_AAD_rest0\num_initial_blocks\operation
+        mov        (%r10), %eax
+        movq       %rax, \TMP1
         add        $4, %r10
-        sub        $4, %r12
-        jne        _get_AAD_loop\num_initial_blocks\operation
-
-        cmp        $16, %r11
-        je         _get_AAD_loop2_done\num_initial_blocks\operation
-
-        mov        $16, %r12
-_get_AAD_loop2\num_initial_blocks\operation:
+        sub        $4, %r10
+        pslldq     $12, \TMP1
         psrldq     $4, %xmm\i
-        sub        $4, %r12
-        cmp        %r11, %r12
-        jne        _get_AAD_loop2\num_initial_blocks\operation
-
-_get_AAD_loop2_done\num_initial_blocks\operation:
+        pxor       \TMP1, %xmm\i
+_get_AAD_rest0\num_initial_blocks\operation:
+        /* finalize: shift out the extra bytes we read, and align
+        left. since pslldq can only shift by an immediate, we use
+        vpshufb and an array of shuffle masks */
+        movq       %r12, %r11
+        salq       $4, %r11
+        movdqu     aad_shift_arr(%r11), \TMP1
+        PSHUFB_XMM \TMP1, %xmm\i
+_get_AAD_rest_final\num_initial_blocks\operation:
         PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
+        pxor       \XMM2, %xmm\i
+        GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 
+_get_AAD_done\num_initial_blocks\operation:
         xor        %r11, %r11 # initialise the data pointer offset as zero
-
-        # start AES for num_initial_blocks blocks
+        # start AES for num_initial_blocks blocks
 
         mov        %arg5, %rax                      # %rax = *Y0
         movdqu     (%rax), \XMM0                    # XMM0 = Y0
@@ -322,7 +379,7 @@ aes_loop_initial_dec\num_initial_blocks:
                 # prepare plaintext/ciphertext for GHASH computation
 .endr
 .endif
-        GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+
         # apply GHASH on num_initial_blocks blocks
 
 .if \i == 5
@@ -477,28 +534,66 @@ XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
         mov        arg8, %r12           # %r12 = aadLen
         mov        %r12, %r11
         pxor       %xmm\i, %xmm\i
-_get_AAD_loop\num_initial_blocks\operation:
-        movd       (%r10), \TMP1
-        pslldq     $12, \TMP1
-        psrldq     $4, %xmm\i
+        pxor       \XMM2, \XMM2
+
+        cmp        $16, %r11
+        jl         _get_AAD_rest8\num_initial_blocks\operation
+_get_AAD_blocks\num_initial_blocks\operation:
+        movdqu     (%r10), %xmm\i
+        PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
+        pxor       %xmm\i, \XMM2
+        GHASH_MUL  \XMM2, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+        add        $16, %r10
+        sub        $16, %r12
+        sub        $16, %r11
+        cmp        $16, %r11
+        jge        _get_AAD_blocks\num_initial_blocks\operation
+
+        movdqu     \XMM2, %xmm\i
+        cmp        $0, %r11
+        je         _get_AAD_done\num_initial_blocks\operation
+
+        pxor       %xmm\i,%xmm\i
+
+        /* read the last <16B of AAD. since we have at least 4B of
+        data right after the AAD (the ICV, and maybe some PT), we can
+        read 4B/8B blocks safely, and then get rid of the extra stuff */
+_get_AAD_rest8\num_initial_blocks\operation:
+        cmp        $4, %r11
+        jle        _get_AAD_rest4\num_initial_blocks\operation
+        movq       (%r10), \TMP1
+        add        $8, %r10
+        sub        $8, %r11
+        pslldq     $8, \TMP1
+        psrldq     $8, %xmm\i
         pxor       \TMP1, %xmm\i
+        jmp        _get_AAD_rest8\num_initial_blocks\operation
+_get_AAD_rest4\num_initial_blocks\operation:
+        cmp        $0, %r11
+        jle        _get_AAD_rest0\num_initial_blocks\operation
+        mov        (%r10), %eax
+        movq       %rax, \TMP1
         add        $4, %r10
-        sub        $4, %r12
-        jne        _get_AAD_loop\num_initial_blocks\operation
-        cmp        $16, %r11
-        je         _get_AAD_loop2_done\num_initial_blocks\operation
-        mov        $16, %r12
-_get_AAD_loop2\num_initial_blocks\operation:
+        sub        $4, %r10
+        pslldq     $12, \TMP1
         psrldq     $4, %xmm\i
-        sub        $4, %r12
-        cmp        %r11, %r12
-        jne        _get_AAD_loop2\num_initial_blocks\operation
-_get_AAD_loop2_done\num_initial_blocks\operation:
+        pxor       \TMP1, %xmm\i
+_get_AAD_rest0\num_initial_blocks\operation:
+        /* finalize: shift out the extra bytes we read, and align
+        left. since pslldq can only shift by an immediate, we use
+        vpshufb and an array of shuffle masks */
+        movq       %r12, %r11
+        salq       $4, %r11
+        movdqu     aad_shift_arr(%r11), \TMP1
+        PSHUFB_XMM \TMP1, %xmm\i
+_get_AAD_rest_final\num_initial_blocks\operation:
         PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
+        pxor       \XMM2, %xmm\i
+        GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 
+_get_AAD_done\num_initial_blocks\operation:
         xor        %r11, %r11 # initialise the data pointer offset as zero
-
-        # start AES for num_initial_blocks blocks
+        # start AES for num_initial_blocks blocks
 
         mov        %arg5, %rax                      # %rax = *Y0
         movdqu     (%rax), \XMM0                    # XMM0 = Y0
@@ -543,7 +638,7 @@ aes_loop_initial_enc\num_initial_blocks:
                 # prepare plaintext/ciphertext for GHASH computation
 .endr
 .endif
-        GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
+
         # apply GHASH on num_initial_blocks blocks
 
 .if \i == 5
@@ -1454,18 +1549,35 @@ _return_T_decrypt:
         mov     arg10, %r11               # %r11 = auth_tag_len
         cmp     $16, %r11
         je      _T_16_decrypt
-        cmp     $12, %r11
-        je      _T_12_decrypt
+        cmp     $8, %r11
+        jl      _T_4_decrypt
 _T_8_decrypt:
         MOVQ_R64_XMM    %xmm0, %rax
         mov     %rax, (%r10)
-        jmp     _return_T_done_decrypt
-_T_12_decrypt:
-        MOVQ_R64_XMM    %xmm0, %rax
-        mov     %rax, (%r10)
+        add     $8, %r10
+        sub     $8, %r11
         psrldq  $8, %xmm0
+        cmp     $0, %r11
+        je      _return_T_done_decrypt
+_T_4_decrypt:
+        movd    %xmm0, %eax
+        mov     %eax, (%r10)
+        add     $4, %r10
+        sub     $4, %r11
+        psrldq  $4, %xmm0
+        cmp     $0, %r11
+        je      _return_T_done_decrypt
+_T_123_decrypt:
         movd    %xmm0, %eax
-        mov     %eax, 8(%r10)
+        cmp     $2, %r11
+        jl      _T_1_decrypt
+        mov     %ax, (%r10)
+        cmp     $2, %r11
+        je      _return_T_done_decrypt
+        add     $2, %r10
+        sar     $16, %eax
+_T_1_decrypt:
+        mov     %al, (%r10)
         jmp     _return_T_done_decrypt
 _T_16_decrypt:
         movdqu  %xmm0, (%r10)
@@ -1718,18 +1830,35 @@ _return_T_encrypt:
         mov     arg10, %r11                    # %r11 = auth_tag_len
         cmp     $16, %r11
         je      _T_16_encrypt
-        cmp     $12, %r11
-        je      _T_12_encrypt
+        cmp     $8, %r11
+        jl      _T_4_encrypt
 _T_8_encrypt:
         MOVQ_R64_XMM    %xmm0, %rax
         mov     %rax, (%r10)
-        jmp     _return_T_done_encrypt
-_T_12_encrypt:
-        MOVQ_R64_XMM    %xmm0, %rax
-        mov     %rax, (%r10)
+        add     $8, %r10
+        sub     $8, %r11
         psrldq  $8, %xmm0
+        cmp     $0, %r11
+        je      _return_T_done_encrypt
+_T_4_encrypt:
+        movd    %xmm0, %eax
+        mov     %eax, (%r10)
+        add     $4, %r10
+        sub     $4, %r11
+        psrldq  $4, %xmm0
+        cmp     $0, %r11
+        je      _return_T_done_encrypt
+_T_123_encrypt:
         movd    %xmm0, %eax
-        mov     %eax, 8(%r10)
+        cmp     $2, %r11
+        jl      _T_1_encrypt
+        mov     %ax, (%r10)
+        cmp     $2, %r11
+        je      _return_T_done_encrypt
+        add     $2, %r10
+        sar     $16, %eax
+_T_1_encrypt:
+        mov     %al, (%r10)
         jmp     _return_T_done_encrypt
 _T_16_encrypt:
         movdqu  %xmm0, (%r10)
diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S
index d664382c6e56..faecb1518bf8 100644
--- a/arch/x86/crypto/aesni-intel_avx-x86_64.S
+++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S
@@ -155,6 +155,30 @@ SHIFT_MASK:      .octa     0x0f0e0d0c0b0a09080706050403020100
 ALL_F:           .octa     0xffffffffffffffffffffffffffffffff
                  .octa     0x00000000000000000000000000000000
 
+.section .rodata
+.align 16
+.type aad_shift_arr, @object
+.size aad_shift_arr, 272
+aad_shift_arr:
+        .octa     0xffffffffffffffffffffffffffffffff
+        .octa     0xffffffffffffffffffffffffffffff0C
+        .octa     0xffffffffffffffffffffffffffff0D0C
+        .octa     0xffffffffffffffffffffffffff0E0D0C
+        .octa     0xffffffffffffffffffffffff0F0E0D0C
+        .octa     0xffffffffffffffffffffff0C0B0A0908
+        .octa     0xffffffffffffffffffff0D0C0B0A0908
+        .octa     0xffffffffffffffffff0E0D0C0B0A0908
+        .octa     0xffffffffffffffff0F0E0D0C0B0A0908
+        .octa     0xffffffffffffff0C0B0A090807060504
+        .octa     0xffffffffffff0D0C0B0A090807060504
+        .octa     0xffffffffff0E0D0C0B0A090807060504
+        .octa     0xffffffff0F0E0D0C0B0A090807060504
+        .octa     0xffffff0C0B0A09080706050403020100
+        .octa     0xffff0D0C0B0A09080706050403020100
+        .octa     0xff0E0D0C0B0A09080706050403020100
+        .octa     0x0F0E0D0C0B0A09080706050403020100
+
+
 .text
 
 
@@ -372,41 +396,72 @@ VARIABLE_OFFSET = 16*8
 
 .macro INITIAL_BLOCKS_AVX num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC
         i = (8-\num_initial_blocks)
+        j = 0
         setreg
 
-        mov     arg6, %r10                      # r10 = AAD
-        mov     arg7, %r12                      # r12 = aadLen
-
-
-        mov     %r12, %r11
-
-        vpxor   reg_i, reg_i, reg_i
-_get_AAD_loop\@:
-        vmovd   (%r10), \T1
-        vpslldq $12, \T1, \T1
-        vpsrldq $4, reg_i, reg_i
-        vpxor   \T1, reg_i, reg_i
-
-        add     $4, %r10
-        sub     $4, %r12
-        jg      _get_AAD_loop\@
-
-
-        cmp     $16, %r11
-        je      _get_AAD_loop2_done\@
-        mov     $16, %r12
-
-_get_AAD_loop2\@:
-        vpsrldq $4, reg_i, reg_i
-        sub     $4, %r12
-        cmp     %r11, %r12
-        jg      _get_AAD_loop2\@
-
-_get_AAD_loop2_done\@:
-
-        #byte-reflect the AAD data
-        vpshufb SHUF_MASK(%rip), reg_i, reg_i
-
+        mov     arg6, %r10                      # r10 = AAD
+        mov     arg7, %r12                      # r12 = aadLen
+
+
+        mov     %r12, %r11
+
+        vpxor   reg_j, reg_j, reg_j
+        vpxor   reg_i, reg_i, reg_i
+        cmp     $16, %r11
+        jl      _get_AAD_rest8\@
+_get_AAD_blocks\@:
+        vmovdqu (%r10), reg_i
+        vpshufb SHUF_MASK(%rip), reg_i, reg_i
+        vpxor   reg_i, reg_j, reg_j
+        GHASH_MUL_AVX       reg_j, \T2, \T1, \T3, \T4, \T5, \T6
+        add     $16, %r10
+        sub     $16, %r12
+        sub     $16, %r11
+        cmp     $16, %r11
+        jge     _get_AAD_blocks\@
+        vmovdqu reg_j, reg_i
+        cmp     $0, %r11
+        je      _get_AAD_done\@
+
+        vpxor   reg_i, reg_i, reg_i
+
+        /* read the last <16B of AAD. since we have at least 4B of
+        data right after the AAD (the ICV, and maybe some CT), we can
+        read 4B/8B blocks safely, and then get rid of the extra stuff */
+_get_AAD_rest8\@:
+        cmp     $4, %r11
+        jle     _get_AAD_rest4\@
+        movq    (%r10), \T1
+        add     $8, %r10
+        sub     $8, %r11
+        vpslldq $8, \T1, \T1
+        vpsrldq $8, reg_i, reg_i
+        vpxor   \T1, reg_i, reg_i
+        jmp     _get_AAD_rest8\@
+_get_AAD_rest4\@:
+        cmp     $0, %r11
+        jle      _get_AAD_rest0\@
+        mov     (%r10), %eax
+        movq    %rax, \T1
+        add     $4, %r10
+        sub     $4, %r11
+        vpslldq $12, \T1, \T1
+        vpsrldq $4, reg_i, reg_i
+        vpxor   \T1, reg_i, reg_i
+_get_AAD_rest0\@:
+        /* finalize: shift out the extra bytes we read, and align
+        left. since pslldq can only shift by an immediate, we use
+        vpshufb and an array of shuffle masks */
+        movq    %r12, %r11
+        salq    $4, %r11
+        movdqu  aad_shift_arr(%r11), \T1
+        vpshufb \T1, reg_i, reg_i
+_get_AAD_rest_final\@:
+        vpshufb SHUF_MASK(%rip), reg_i, reg_i
+        vpxor   reg_j, reg_i, reg_i
+        GHASH_MUL_AVX       reg_i, \T2, \T1, \T3, \T4, \T5, \T6
+
+_get_AAD_done\@:
         # initialize the data pointer offset as zero
         xor     %r11, %r11
 
@@ -480,7 +535,6 @@ _get_AAD_loop2_done\@:
         i = (8-\num_initial_blocks)
         j = (9-\num_initial_blocks)
         setreg
-        GHASH_MUL_AVX       reg_i, \T2, \T1, \T3, \T4, \T5, \T6
 
 .rep \num_initial_blocks
         vpxor    reg_i, reg_j, reg_j
@@ -1427,19 +1481,36 @@ _return_T\@:
         cmp     $16, %r11
         je      _T_16\@
 
-        cmp     $12, %r11
-        je      _T_12\@
+        cmp     $8, %r11
+        jl      _T_4\@
 
 _T_8\@:
         vmovq   %xmm9, %rax
         mov     %rax, (%r10)
-        jmp     _return_T_done\@
-_T_12\@:
-        vmovq   %xmm9, %rax
-        mov     %rax, (%r10)
+        add     $8, %r10
+        sub     $8, %r11
         vpsrldq $8, %xmm9, %xmm9
+        cmp     $0, %r11
+        je     _return_T_done\@
+_T_4\@:
         vmovd   %xmm9, %eax
-        mov     %eax, 8(%r10)
+        mov     %eax, (%r10)
+        add     $4, %r10
+        sub     $4, %r11
+        vpsrldq     $4, %xmm9, %xmm9
+        cmp     $0, %r11
+        je     _return_T_done\@
+_T_123\@:
+        vmovd     %xmm9, %eax
+        cmp     $2, %r11
+        jl     _T_1\@
+        mov     %ax, (%r10)
+        cmp     $2, %r11
+        je     _return_T_done\@
+        add     $2, %r10
+        sar     $16, %eax
+_T_1\@:
+        mov     %al, (%r10)
         jmp     _return_T_done\@
 
 _T_16\@:
@@ -1631,41 +1702,73 @@ ENDPROC(aesni_gcm_dec_avx_gen2)
 
 .macro INITIAL_BLOCKS_AVX2 num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC VER
         i = (8-\num_initial_blocks)
+        j = 0
         setreg
 
-        mov     arg6, %r10                       # r10 = AAD
-        mov     arg7, %r12                       # r12 = aadLen
-
-
-        mov     %r12, %r11
-
-        vpxor   reg_i, reg_i, reg_i
-_get_AAD_loop\@:
-        vmovd   (%r10), \T1
-        vpslldq $12, \T1, \T1
-        vpsrldq $4, reg_i, reg_i
-        vpxor   \T1, reg_i, reg_i
-
-        add     $4, %r10
-        sub     $4, %r12
-        jg      _get_AAD_loop\@
-
-
-        cmp     $16, %r11
-        je      _get_AAD_loop2_done\@
-        mov     $16, %r12
-
-_get_AAD_loop2\@:
-        vpsrldq $4, reg_i, reg_i
-        sub     $4, %r12
-        cmp     %r11, %r12
-        jg      _get_AAD_loop2\@
-
-_get_AAD_loop2_done\@:
-
-        #byte-reflect the AAD data
-        vpshufb SHUF_MASK(%rip), reg_i, reg_i
-
+        mov     arg6, %r10                       # r10 = AAD
+        mov     arg7, %r12                       # r12 = aadLen
+
+
+        mov     %r12, %r11
+
+        vpxor   reg_j, reg_j, reg_j
+        vpxor   reg_i, reg_i, reg_i
+
+        cmp     $16, %r11
+        jl      _get_AAD_rest8\@
+_get_AAD_blocks\@:
+        vmovdqu (%r10), reg_i
+        vpshufb SHUF_MASK(%rip), reg_i, reg_i
+        vpxor   reg_i, reg_j, reg_j
+        GHASH_MUL_AVX2      reg_j, \T2, \T1, \T3, \T4, \T5, \T6
+        add     $16, %r10
+        sub     $16, %r12
+        sub     $16, %r11
+        cmp     $16, %r11
+        jge     _get_AAD_blocks\@
+        vmovdqu reg_j, reg_i
+        cmp     $0, %r11
+        je      _get_AAD_done\@
+
+        vpxor   reg_i, reg_i, reg_i
+
+        /* read the last <16B of AAD. since we have at least 4B of
+        data right after the AAD (the ICV, and maybe some CT), we can
+        read 4B/8B blocks safely, and then get rid of the extra stuff */
+_get_AAD_rest8\@:
+        cmp     $4, %r11
+        jle     _get_AAD_rest4\@
+        movq    (%r10), \T1
+        add     $8, %r10
+        sub     $8, %r11
+        vpslldq $8, \T1, \T1
+        vpsrldq $8, reg_i, reg_i
+        vpxor   \T1, reg_i, reg_i
+        jmp     _get_AAD_rest8\@
+_get_AAD_rest4\@:
+        cmp     $0, %r11
+        jle     _get_AAD_rest0\@
+        mov     (%r10), %eax
+        movq    %rax, \T1
+        add     $4, %r10
+        sub     $4, %r11
+        vpslldq $12, \T1, \T1
+        vpsrldq $4, reg_i, reg_i
+        vpxor   \T1, reg_i, reg_i
+_get_AAD_rest0\@:
+        /* finalize: shift out the extra bytes we read, and align
+        left. since pslldq can only shift by an immediate, we use
+        vpshufb and an array of shuffle masks */
+        movq    %r12, %r11
+        salq    $4, %r11
+        movdqu  aad_shift_arr(%r11), \T1
+        vpshufb \T1, reg_i, reg_i
+_get_AAD_rest_final\@:
+        vpshufb SHUF_MASK(%rip), reg_i, reg_i
+        vpxor   reg_j, reg_i, reg_i
+        GHASH_MUL_AVX2      reg_i, \T2, \T1, \T3, \T4, \T5, \T6
+
+_get_AAD_done\@:
         # initialize the data pointer offset as zero
         xor     %r11, %r11
 
@@ -1740,7 +1843,6 @@ _get_AAD_loop2_done\@:
         i = (8-\num_initial_blocks)
         j = (9-\num_initial_blocks)
         setreg
-        GHASH_MUL_AVX2       reg_i, \T2, \T1, \T3, \T4, \T5, \T6
 
 .rep \num_initial_blocks
         vpxor    reg_i, reg_j, reg_j
@@ -2702,19 +2804,36 @@ _return_T\@:
         cmp     $16, %r11
         je      _T_16\@
 
-        cmp     $12, %r11
-        je      _T_12\@
+        cmp     $8, %r11
+        jl      _T_4\@
 
 _T_8\@:
         vmovq   %xmm9, %rax
         mov     %rax, (%r10)
-        jmp     _return_T_done\@
-_T_12\@:
-        vmovq   %xmm9, %rax
-        mov     %rax, (%r10)
+        add     $8, %r10
+        sub     $8, %r11
         vpsrldq $8, %xmm9, %xmm9
+        cmp     $0, %r11
+        je     _return_T_done\@
+_T_4\@:
         vmovd   %xmm9, %eax
-        mov     %eax, 8(%r10)
+        mov     %eax, (%r10)
+        add     $4, %r10
+        sub     $4, %r11
+        vpsrldq     $4, %xmm9, %xmm9
+        cmp     $0, %r11
+        je     _return_T_done\@
+_T_123\@:
+        vmovd     %xmm9, %eax
+        cmp     $2, %r11
+        jl     _T_1\@
+        mov     %ax, (%r10)
+        cmp     $2, %r11
+        je     _return_T_done\@
+        add     $2, %r10
+        sar     $16, %eax
+_T_1\@:
+        mov     %al, (%r10)
         jmp     _return_T_done\@
 
 _T_16\@:
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 93de8ea51548..4a55cdcdc008 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -61,6 +61,11 @@ struct aesni_rfc4106_gcm_ctx {
         u8 nonce[4];
 };
 
+struct generic_gcmaes_ctx {
+        u8 hash_subkey[16] AESNI_ALIGN_ATTR;
+        struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
+};
+
 struct aesni_xts_ctx {
         u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
         u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
@@ -102,13 +107,11 @@ asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
  * const u8 *in, Plaintext input
  * unsigned long plaintext_len, Length of data in bytes for encryption.
- * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
- *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
- *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
+ * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
+ *         16-byte aligned pointer.
  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
  * const u8 *aad, Additional Authentication Data (AAD)
- * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
- *          is going to be 8 or 12 bytes
+ * unsigned long aad_len, Length of AAD in bytes.
  * u8 *auth_tag, Authenticated Tag output.
  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
  *          Valid values are 16 (most likely), 12 or 8.
@@ -123,9 +126,8 @@ asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
  * u8 *out, Plaintext output. Decrypt in-place is allowed.
  * const u8 *in, Ciphertext input
  * unsigned long ciphertext_len, Length of data in bytes for decryption.
- * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
- *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
- *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
+ * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
+ *         16-byte aligned pointer.
  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
  * const u8 *aad, Additional Authentication Data (AAD)
  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
@@ -275,6 +277,16 @@ aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
                 align = 1;
         return PTR_ALIGN(crypto_aead_ctx(tfm), align);
 }
+
+static inline struct
+generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
+{
+        unsigned long align = AESNI_ALIGN;
+
+        if (align <= crypto_tfm_ctx_alignment())
+                align = 1;
+        return PTR_ALIGN(crypto_aead_ctx(tfm), align);
+}
 #endif
 
 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
@@ -712,32 +724,34 @@ static int rfc4106_set_authsize(struct crypto_aead *parent,
         return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
 }
 
-static int helper_rfc4106_encrypt(struct aead_request *req)
+static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
+                                       unsigned int authsize)
+{
+        switch (authsize) {
+        case 4:
+        case 8:
+        case 12:
+        case 13:
+        case 14:
+        case 15:
+        case 16:
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
+                          u8 *hash_subkey, u8 *iv, void *aes_ctx)
 {
         u8 one_entry_in_sg = 0;
         u8 *src, *dst, *assoc;
-        __be32 counter = cpu_to_be32(1);
         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
-        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
-        void *aes_ctx = &(ctx->aes_key_expanded);
         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
-        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
         struct scatter_walk src_sg_walk;
         struct scatter_walk dst_sg_walk = {};
-        unsigned int i;
-
-        /* Assuming we are supporting rfc4106 64-bit extended */
-        /* sequence numbers We need to have the AAD length equal */
-        /* to 16 or 20 bytes */
-        if (unlikely(req->assoclen != 16 && req->assoclen != 20))
-                return -EINVAL;
-
-        /* IV below built */
-        for (i = 0; i < 4; i++)
-                *(iv+i) = ctx->nonce[i];
-        for (i = 0; i < 8; i++)
-                *(iv+4+i) = req->iv[i];
-        *((__be32 *)(iv+12)) = counter;
 
         if (sg_is_last(req->src) &&
             (!PageHighMem(sg_page(req->src)) ||
@@ -768,7 +782,7 @@ static int helper_rfc4106_encrypt(struct aead_request *req)
 
         kernel_fpu_begin();
         aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv,
-                          ctx->hash_subkey, assoc, req->assoclen - 8,
+                          hash_subkey, assoc, assoclen,
                           dst + req->cryptlen, auth_tag_len);
         kernel_fpu_end();
 
@@ -791,37 +805,20 @@ static int helper_rfc4106_encrypt(struct aead_request *req)
         return 0;
 }
 
-static int helper_rfc4106_decrypt(struct aead_request *req)
+static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
+                          u8 *hash_subkey, u8 *iv, void *aes_ctx)
 {
         u8 one_entry_in_sg = 0;
         u8 *src, *dst, *assoc;
         unsigned long tempCipherLen = 0;
-        __be32 counter = cpu_to_be32(1);
-        int retval = 0;
         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
-        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
-        void *aes_ctx = &(ctx->aes_key_expanded);
         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
-        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
         u8 authTag[16];
         struct scatter_walk src_sg_walk;
         struct scatter_walk dst_sg_walk = {};
-        unsigned int i;
-
-        if (unlikely(req->assoclen != 16 && req->assoclen != 20))
-                return -EINVAL;
-
-        /* Assuming we are supporting rfc4106 64-bit extended */
-        /* sequence numbers We need to have the AAD length */
-        /* equal to 16 or 20 bytes */
+        int retval = 0;
 
         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
-        /* IV below built */
-        for (i = 0; i < 4; i++)
-                *(iv+i) = ctx->nonce[i];
-        for (i = 0; i < 8; i++)
-                *(iv+4+i) = req->iv[i];
-        *((__be32 *)(iv+12)) = counter;
 
         if (sg_is_last(req->src) &&
             (!PageHighMem(sg_page(req->src)) ||
@@ -838,7 +835,6 @@ static int helper_rfc4106_decrypt(struct aead_request *req)
                         scatterwalk_start(&dst_sg_walk, req->dst);
                         dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
                 }
-
         } else {
                 /* Allocate memory for src, dst, assoc */
                 assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
@@ -850,9 +846,10 @@ static int helper_rfc4106_decrypt(struct aead_request *req)
                 dst = src;
         }
 
+
         kernel_fpu_begin();
         aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
-                          ctx->hash_subkey, assoc, req->assoclen - 8,
+                          hash_subkey, assoc, assoclen,
                           authTag, auth_tag_len);
         kernel_fpu_end();
 
@@ -875,6 +872,60 @@ static int helper_rfc4106_decrypt(struct aead_request *req)
                 kfree(assoc);
         }
         return retval;
+
+}
+
+static int helper_rfc4106_encrypt(struct aead_request *req)
+{
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        void *aes_ctx = &(ctx->aes_key_expanded);
+        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
+        unsigned int i;
+        __be32 counter = cpu_to_be32(1);
+
+        /* Assuming we are supporting rfc4106 64-bit extended */
+        /* sequence numbers We need to have the AAD length equal */
+        /* to 16 or 20 bytes */
+        if (unlikely(req->assoclen != 16 && req->assoclen != 20))
+                return -EINVAL;
+
+        /* IV below built */
+        for (i = 0; i < 4; i++)
+                *(iv+i) = ctx->nonce[i];
+        for (i = 0; i < 8; i++)
+                *(iv+4+i) = req->iv[i];
+        *((__be32 *)(iv+12)) = counter;
+
+        return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
+                              aes_ctx);
+}
+
+static int helper_rfc4106_decrypt(struct aead_request *req)
+{
+        __be32 counter = cpu_to_be32(1);
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        void *aes_ctx = &(ctx->aes_key_expanded);
+        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
+        unsigned int i;
+
+        if (unlikely(req->assoclen != 16 && req->assoclen != 20))
+                return -EINVAL;
+
+        /* Assuming we are supporting rfc4106 64-bit extended */
+        /* sequence numbers We need to have the AAD length */
+        /* equal to 16 or 20 bytes */
+
+        /* IV below built */
+        for (i = 0; i < 4; i++)
+                *(iv+i) = ctx->nonce[i];
+        for (i = 0; i < 8; i++)
+                *(iv+4+i) = req->iv[i];
+        *((__be32 *)(iv+12)) = counter;
+
+        return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
+                              aes_ctx);
 }
 
 static int rfc4106_encrypt(struct aead_request *req)
@@ -1035,6 +1086,46 @@ struct {
 };
 
 #ifdef CONFIG_X86_64
+static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
+                                  unsigned int key_len)
+{
+        struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead);
+
+        return aes_set_key_common(crypto_aead_tfm(aead),
+                                  &ctx->aes_key_expanded, key, key_len) ?:
+               rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
+}
+
+static int generic_gcmaes_encrypt(struct aead_request *req)
+{
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
+        void *aes_ctx = &(ctx->aes_key_expanded);
+        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
+        __be32 counter = cpu_to_be32(1);
+
+        memcpy(iv, req->iv, 12);
+        *((__be32 *)(iv+12)) = counter;
+
+        return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
+                              aes_ctx);
+}
+
+static int generic_gcmaes_decrypt(struct aead_request *req)
+{
+        __be32 counter = cpu_to_be32(1);
+        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+        void *aes_ctx = &(ctx->aes_key_expanded);
+        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
+
+        memcpy(iv, req->iv, 12);
+        *((__be32 *)(iv+12)) = counter;
+
+        return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
+                              aes_ctx);
+}
+
 static struct aead_alg aesni_aead_algs[] = { {
         .setkey                 = common_rfc4106_set_key,
         .setauthsize            = common_rfc4106_set_authsize,
@@ -1069,6 +1160,23 @@ static struct aead_alg aesni_aead_algs[] = { {
                 .cra_ctxsize            = sizeof(struct cryptd_aead *),
                 .cra_module             = THIS_MODULE,
         },
+}, {
+        .setkey                 = generic_gcmaes_set_key,
+        .setauthsize            = generic_gcmaes_set_authsize,
+        .encrypt                = generic_gcmaes_encrypt,
+        .decrypt                = generic_gcmaes_decrypt,
+        .ivsize                 = 12,
+        .maxauthsize            = 16,
+        .base = {
+                .cra_name               = "gcm(aes)",
+                .cra_driver_name        = "generic-gcm-aesni",
+                .cra_priority           = 400,
+                .cra_flags              = CRYPTO_ALG_ASYNC,
+                .cra_blocksize          = 1,
+                .cra_ctxsize            = sizeof(struct generic_gcmaes_ctx),
+                .cra_alignmask          = AESNI_ALIGN - 1,
+                .cra_module             = THIS_MODULE,
+        },
 } };
 #else
 static struct aead_alg aesni_aead_algs[0];
diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c
index 24ac9fad832d..d61e57960fe0 100644
--- a/arch/x86/crypto/glue_helper.c
+++ b/arch/x86/crypto/glue_helper.c
@@ -176,9 +176,6 @@ __glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
                                 src -= 1;
                                 dst -= 1;
                         } while (nbytes >= func_bytes);
-
-                        if (nbytes < bsize)
-                                goto done;
                 }
         }
 
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb.c b/arch/x86/crypto/sha512-mb/sha512_mb.c
index 2dd3674b5a1e..458409b7568d 100644
--- a/arch/x86/crypto/sha512-mb/sha512_mb.c
+++ b/arch/x86/crypto/sha512-mb/sha512_mb.c
@@ -269,19 +269,19 @@ static struct sha512_hash_ctx
                  * LAST
                  */
                 ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
-                return ctx;
+                goto unlock;
         }
 
         if (ctx->status & HASH_CTX_STS_PROCESSING) {
                 /* Cannot submit to a currently processing job. */
                 ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING;
-                return ctx;
+                goto unlock;
         }
 
         if ((ctx->status & HASH_CTX_STS_COMPLETE) && !(flags & HASH_FIRST)) {
                 /* Cannot update a finished job. */
                 ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED;
-                return ctx;
+                goto unlock;
         }
 
 
@@ -363,6 +363,7 @@ static struct sha512_hash_ctx
         }
 
         ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
+unlock:
         spin_unlock_irqrestore(&cstate->work_lock, irqflags);
         return ctx;
 }
diff --git a/crypto/Kconfig b/crypto/Kconfig
index aac4bc90a138..caa770e535a2 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -130,6 +130,7 @@ config CRYPTO_DH
 config CRYPTO_ECDH
         tristate "ECDH algorithm"
         select CRYTPO_KPP
+        select CRYPTO_RNG_DEFAULT
         help
           Generic implementation of the ECDH algorithm
 
diff --git a/crypto/Makefile b/crypto/Makefile
index 8a44057240d5..d41f0331b085 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -33,10 +33,6 @@ obj-$(CONFIG_CRYPTO_KPP2) += kpp.o
 dh_generic-y := dh.o
 dh_generic-y += dh_helper.o
 obj-$(CONFIG_CRYPTO_DH) += dh_generic.o
-ecdh_generic-y := ecc.o
-ecdh_generic-y += ecdh.o
-ecdh_generic-y += ecdh_helper.o
-obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o
 
 $(obj)/rsapubkey-asn1.o: $(obj)/rsapubkey-asn1.c $(obj)/rsapubkey-asn1.h
 $(obj)/rsaprivkey-asn1.o: $(obj)/rsaprivkey-asn1.c $(obj)/rsaprivkey-asn1.h
@@ -138,6 +134,11 @@ obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o
 obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o
 obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o
 
+ecdh_generic-y := ecc.o
+ecdh_generic-y += ecdh.o
+ecdh_generic-y += ecdh_helper.o
+obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o
+
 #
 # generic algorithms and the async_tx api
 #
diff --git a/crypto/aes_ti.c b/crypto/aes_ti.c
index 92644fd1ac19..03023b2290e8 100644
--- a/crypto/aes_ti.c
+++ b/crypto/aes_ti.c
@@ -114,7 +114,7 @@ static u32 mix_columns(u32 x)
          * | 0x2 0x3 0x1 0x1 |   | x[0] |
          * | 0x1 0x2 0x3 0x1 |   | x[1] |
          * | 0x1 0x1 0x2 0x3 | x | x[2] |
-         * | 0x3 0x1 0x1 0x3 |   | x[3] |
+         * | 0x3 0x1 0x1 0x2 |   | x[3] |
          */
         u32 y = mul_by_x(x) ^ ror32(x, 16);
 
diff --git a/crypto/algapi.c b/crypto/algapi.c
index 9eed4ef9c971..e4cc7615a139 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -260,7 +260,7 @@ void crypto_alg_tested(const char *name, int err)
                         goto found;
         }
 
-        printk(KERN_ERR "alg: Unexpected test result for %s: %d\n", name, err);
+        pr_err("alg: Unexpected test result for %s: %d\n", name, err);
         goto unlock;
 
 found:
diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c
index 727bd5c3569e..61e7c4e02fd2 100644
--- a/crypto/crypto_engine.c
+++ b/crypto/crypto_engine.c
@@ -70,7 +70,7 @@ static void crypto_pump_requests(struct crypto_engine *engine,
 
                 if (engine->unprepare_crypt_hardware &&
                     engine->unprepare_crypt_hardware(engine))
-                        pr_err("failed to unprepare crypt hardware\n");
+                        dev_err(engine->dev, "failed to unprepare crypt hardware\n");
 
                 spin_lock_irqsave(&engine->queue_lock, flags);
                 engine->idling = false;
@@ -99,7 +99,7 @@ static void crypto_pump_requests(struct crypto_engine *engine,
         if (!was_busy && engine->prepare_crypt_hardware) {
                 ret = engine->prepare_crypt_hardware(engine);
                 if (ret) {
-                        pr_err("failed to prepare crypt hardware\n");
+                        dev_err(engine->dev, "failed to prepare crypt hardware\n");
                         goto req_err;
                 }
         }
@@ -110,14 +110,15 @@ static void crypto_pump_requests(struct crypto_engine *engine,
                 if (engine->prepare_hash_request) {
                         ret = engine->prepare_hash_request(engine, hreq);
                         if (ret) {
-                                pr_err("failed to prepare request: %d\n", ret);
+                                dev_err(engine->dev, "failed to prepare request: %d\n",
+                                        ret);
                                 goto req_err;
                         }
                         engine->cur_req_prepared = true;
                 }
                 ret = engine->hash_one_request(engine, hreq);
                 if (ret) {
-                        pr_err("failed to hash one request from queue\n");
+                        dev_err(engine->dev, "failed to hash one request from queue\n");
                         goto req_err;
                 }
                 return;
@@ -126,19 +127,20 @@ static void crypto_pump_requests(struct crypto_engine *engine,
                 if (engine->prepare_cipher_request) {
                         ret = engine->prepare_cipher_request(engine, breq);
                         if (ret) {
-                                pr_err("failed to prepare request: %d\n", ret);
+                                dev_err(engine->dev, "failed to prepare request: %d\n",
+                                        ret);
                                 goto req_err;
                         }
                         engine->cur_req_prepared = true;
                 }
                 ret = engine->cipher_one_request(engine, breq);
                 if (ret) {
-                        pr_err("failed to cipher one request from queue\n");
+                        dev_err(engine->dev, "failed to cipher one request from queue\n");
                         goto req_err;
                 }
                 return;
         default:
-                pr_err("failed to prepare request of unknown type\n");
+                dev_err(engine->dev, "failed to prepare request of unknown type\n");
                 return;
         }
 
@@ -275,7 +277,7 @@ void crypto_finalize_cipher_request(struct crypto_engine *engine,
                     engine->unprepare_cipher_request) {
                         ret = engine->unprepare_cipher_request(engine, req);
                         if (ret)
-                                pr_err("failed to unprepare request\n");
+                                dev_err(engine->dev, "failed to unprepare request\n");
                 }
                 spin_lock_irqsave(&engine->queue_lock, flags);
                 engine->cur_req = NULL;
@@ -312,7 +314,7 @@ void crypto_finalize_hash_request(struct crypto_engine *engine,
                     engine->unprepare_hash_request) {
                         ret = engine->unprepare_hash_request(engine, req);
                         if (ret)
-                                pr_err("failed to unprepare request\n");
+                                dev_err(engine->dev, "failed to unprepare request\n");
                 }
                 spin_lock_irqsave(&engine->queue_lock, flags);
                 engine->cur_req = NULL;
@@ -384,7 +386,7 @@ int crypto_engine_stop(struct crypto_engine *engine)
         spin_unlock_irqrestore(&engine->queue_lock, flags);
 
         if (ret)
-                pr_warn("could not stop engine\n");
+                dev_warn(engine->dev, "could not stop engine\n");
 
         return ret;
 }
@@ -411,6 +413,7 @@ struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
         if (!engine)
                 return NULL;
 
+        engine->dev = dev;
         engine->rt = rt;
         engine->running = false;
         engine->busy = false;
diff --git a/crypto/dh.c b/crypto/dh.c
index 87e3542cf1b8..b1032a5c1bfa 100644
--- a/crypto/dh.c
+++ b/crypto/dh.c
@@ -4,9 +4,9 @@
  * Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
  *
  * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public Licence
+ * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
- * 2 of the Licence, or (at your option) any later version.
+ * 2 of the License, or (at your option) any later version.
  */
 
 #include <linux/module.h>
@@ -85,6 +85,9 @@ static int dh_set_secret(struct crypto_kpp *tfm, const void *buf,
         struct dh_ctx *ctx = dh_get_ctx(tfm);
         struct dh params;
 
+        /* Free the old MPI key if any */
+        dh_free_ctx(ctx);
+
         if (crypto_dh_decode_key(buf, len, &params) < 0)
                 return -EINVAL;
 
@@ -144,7 +147,7 @@ err_free_val:
         return ret;
 }
 
-static int dh_max_size(struct crypto_kpp *tfm)
+static unsigned int dh_max_size(struct crypto_kpp *tfm)
 {
         struct dh_ctx *ctx = dh_get_ctx(tfm);
 
diff --git a/crypto/dh_helper.c b/crypto/dh_helper.c
index 02db76b20d00..8ba8a3f82620 100644
--- a/crypto/dh_helper.c
+++ b/crypto/dh_helper.c
@@ -3,9 +3,9 @@
  * Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
  *
  * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public Licence
+ * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
- * 2 of the Licence, or (at your option) any later version.
+ * 2 of the License, or (at your option) any later version.
  */
 #include <linux/kernel.h>
 #include <linux/export.h>
diff --git a/crypto/drbg.c b/crypto/drbg.c
index cdb27ac4b226..633a88e93ab0 100644
--- a/crypto/drbg.c
+++ b/crypto/drbg.c
@@ -1691,6 +1691,7 @@ static int drbg_init_sym_kernel(struct drbg_state *drbg)
                 return PTR_ERR(sk_tfm);
         }
         drbg->ctr_handle = sk_tfm;
+        init_completion(&drbg->ctr_completion);
 
         req = skcipher_request_alloc(sk_tfm, GFP_KERNEL);
         if (!req) {
diff --git a/crypto/ecc.c b/crypto/ecc.c
index 414c78a9c214..633a9bcdc574 100644
--- a/crypto/ecc.c
+++ b/crypto/ecc.c
@@ -29,6 +29,7 @@
 #include <linux/swab.h>
 #include <linux/fips.h>
 #include <crypto/ecdh.h>
+#include <crypto/rng.h>
 
 #include "ecc.h"
 #include "ecc_curve_defs.h"
@@ -904,7 +905,7 @@ static inline void ecc_swap_digits(const u64 *in, u64 *out,
 }
 
 int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
-                     const u8 *private_key, unsigned int private_key_len)
+                     const u64 *private_key, unsigned int private_key_len)
 {
         int nbytes;
         const struct ecc_curve *curve = ecc_get_curve(curve_id);
@@ -917,24 +918,77 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
         if (private_key_len != nbytes)
                 return -EINVAL;
 
-        if (vli_is_zero((const u64 *)&private_key[0], ndigits))
+        if (vli_is_zero(private_key, ndigits))
                 return -EINVAL;
 
         /* Make sure the private key is in the range [1, n-1]. */
-        if (vli_cmp(curve->n, (const u64 *)&private_key[0], ndigits) != 1)
+        if (vli_cmp(curve->n, private_key, ndigits) != 1)
                 return -EINVAL;
 
         return 0;
 }
 
-int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits,
-                      const u8 *private_key, unsigned int private_key_len,
-                      u8 *public_key, unsigned int public_key_len)
+/*
+ * ECC private keys are generated using the method of extra random bits,
+ * equivalent to that described in FIPS 186-4, Appendix B.4.1.
+ *
+ * d = (c mod(n–1)) + 1    where c is a string of random bits, 64 bits longer
+ *                         than requested
+ * 0 <= c mod(n-1) <= n-2  and implies that
+ * 1 <= d <= n-1
+ *
+ * This method generates a private key uniformly distributed in the range
+ * [1, n-1].
+ */
+int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits, u64 *privkey)
+{
+        const struct ecc_curve *curve = ecc_get_curve(curve_id);
+        u64 priv[ndigits];
+        unsigned int nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
+        unsigned int nbits = vli_num_bits(curve->n, ndigits);
+        int err;
+
+        /* Check that N is included in Table 1 of FIPS 186-4, section 6.1.1 */
+        if (nbits < 160)
+                return -EINVAL;
+
+        /*
+         * FIPS 186-4 recommends that the private key should be obtained from a
+         * RBG with a security strength equal to or greater than the security
+         * strength associated with N.
+         *
+         * The maximum security strength identified by NIST SP800-57pt1r4 for
+         * ECC is 256 (N >= 512).
+         *
+         * This condition is met by the default RNG because it selects a favored
+         * DRBG with a security strength of 256.
+         */
+        if (crypto_get_default_rng())
+                err = -EFAULT;
+
+        err = crypto_rng_get_bytes(crypto_default_rng, (u8 *)priv, nbytes);
+        crypto_put_default_rng();
+        if (err)
+                return err;
+
+        if (vli_is_zero(priv, ndigits))
+                return -EINVAL;
+
+        /* Make sure the private key is in the range [1, n-1]. */
+        if (vli_cmp(curve->n, priv, ndigits) != 1)
+                return -EINVAL;
+
+        ecc_swap_digits(priv, privkey, ndigits);
+
+        return 0;
+}
+
+int ecc_make_pub_key(unsigned int curve_id, unsigned int ndigits,
+                     const u64 *private_key, u64 *public_key)
 {
         int ret = 0;
         struct ecc_point *pk;
         u64 priv[ndigits];
-        unsigned int nbytes;
         const struct ecc_curve *curve = ecc_get_curve(curve_id);
 
         if (!private_key || !curve) {
@@ -942,7 +996,7 @@ int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits,
                 goto out;
         }
 
-        ecc_swap_digits((const u64 *)private_key, priv, ndigits);
+        ecc_swap_digits(private_key, priv, ndigits);
 
         pk = ecc_alloc_point(ndigits);
         if (!pk) {
@@ -956,9 +1010,8 @@ int ecdh_make_pub_key(unsigned int curve_id, unsigned int ndigits,
                 goto err_free_point;
         }
 
-        nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
-        ecc_swap_digits(pk->x, (u64 *)public_key, ndigits);
-        ecc_swap_digits(pk->y, (u64 *)&public_key[nbytes], ndigits);
+        ecc_swap_digits(pk->x, public_key, ndigits);
+        ecc_swap_digits(pk->y, &public_key[ndigits], ndigits);
 
 err_free_point:
         ecc_free_point(pk);
@@ -967,9 +1020,8 @@ out:
 }
 
 int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
-                       const u8 *private_key, unsigned int private_key_len,
-                       const u8 *public_key, unsigned int public_key_len,
-                       u8 *secret, unsigned int secret_len)
+                              const u64 *private_key, const u64 *public_key,
+                              u64 *secret)
 {
         int ret = 0;
         struct ecc_point *product, *pk;
@@ -999,13 +1051,13 @@ int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
                 goto err_alloc_product;
         }
 
-        ecc_swap_digits((const u64 *)public_key, pk->x, ndigits);
-        ecc_swap_digits((const u64 *)&public_key[nbytes], pk->y, ndigits);
-        ecc_swap_digits((const u64 *)private_key, priv, ndigits);
+        ecc_swap_digits(public_key, pk->x, ndigits);
+        ecc_swap_digits(&public_key[ndigits], pk->y, ndigits);
+        ecc_swap_digits(private_key, priv, ndigits);
 
         ecc_point_mult(product, pk, priv, rand_z, curve->p, ndigits);
 
-        ecc_swap_digits(product->x, (u64 *)secret, ndigits);
+        ecc_swap_digits(product->x, secret, ndigits);
 
         if (ecc_point_is_zero(product))
                 ret = -EFAULT;
diff --git a/crypto/ecc.h b/crypto/ecc.h
index 663d598c7406..e4fd4492c765 100644
--- a/crypto/ecc.h
+++ b/crypto/ecc.h
@@ -34,41 +34,51 @@
  * ecc_is_key_valid() - Validate a given ECDH private key
  *
  * @curve_id:           id representing the curve to use
- * @ndigits:            curve number of digits
+ * @ndigits:            curve's number of digits
  * @private_key:        private key to be used for the given curve
- * @private_key_len:    private key len
+ * @private_key_len:    private key length
  *
  * Returns 0 if the key is acceptable, a negative value otherwise
  */
 int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
-                     const u8 *private_key, unsigned int private_key_len);
+                     const u64 *private_key, unsigned int private_key_len);
+
+/**
+ * ecc_gen_privkey() -  Generates an ECC private key.
+ * The private key is a random integer in the range 0 < random < n, where n is a
+ * prime that is the order of the cyclic subgroup generated by the distinguished
+ * point G.
+ * @curve_id:           id representing the curve to use
+ * @ndigits:            curve number of digits
+ * @private_key:        buffer for storing the generated private key
+ *
+ * Returns 0 if the private key was generated successfully, a negative value
+ * if an error occurred.
+ */
+int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits, u64 *privkey);
 
 /**
- * ecdh_make_pub_key() - Compute an ECC public key
+ * ecc_make_pub_key() - Compute an ECC public key
  *
  * @curve_id:           id representing the curve to use
+ * @ndigits:            curve's number of digits
  * @private_key:        pregenerated private key for the given curve
- * @private_key_len:    length of private_key
- * @public_key:         buffer for storing the public key generated
- * @public_key_len:     length of the public_key buffer
+ * @public_key:         buffer for storing the generated public key
  *
  * Returns 0 if the public key was generated successfully, a negative value
  * if an error occurred.
  */
-int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits,
-                      const u8 *private_key, unsigned int private_key_len,
-                      u8 *public_key, unsigned int public_key_len);
+int ecc_make_pub_key(const unsigned int curve_id, unsigned int ndigits,
+                     const u64 *private_key, u64 *public_key);
 
 /**
  * crypto_ecdh_shared_secret() - Compute a shared secret
  *
  * @curve_id:           id representing the curve to use
+ * @ndigits:            curve's number of digits
  * @private_key:        private key of part A
- * @private_key_len:    length of private_key
  * @public_key:         public key of counterpart B
- * @public_key_len:     length of public_key
  * @secret:             buffer for storing the calculated shared secret
- * @secret_len:         length of the secret buffer
  *
  * Note: It is recommended that you hash the result of crypto_ecdh_shared_secret
  * before using it for symmetric encryption or HMAC.
@@ -77,7 +87,6 @@ int ecdh_make_pub_key(const unsigned int curve_id, unsigned int ndigits,
  * if an error occurred.
  */
 int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
-                       const u8 *private_key, unsigned int private_key_len,
-                       const u8 *public_key, unsigned int public_key_len,
-                       u8 *secret, unsigned int secret_len);
+                              const u64 *private_key, const u64 *public_key,
+                              u64 *secret);
 #endif
diff --git a/crypto/ecdh.c b/crypto/ecdh.c
index 63ca33771e4e..61c7708905d0 100644
--- a/crypto/ecdh.c
+++ b/crypto/ecdh.c
@@ -4,9 +4,9 @@
  * Authors: Salvator Benedetto <salvatore.benedetto@intel.com>
  *
  * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public Licence
+ * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
- * 2 of the Licence, or (at your option) any later version.
+ * 2 of the License, or (at your option) any later version.
  */
 
 #include <linux/module.h>
@@ -55,8 +55,12 @@ static int ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
         ctx->curve_id = params.curve_id;
         ctx->ndigits = ndigits;
 
+        if (!params.key || !params.key_size)
+                return ecc_gen_privkey(ctx->curve_id, ctx->ndigits,
+                                       ctx->private_key);
+
         if (ecc_is_key_valid(ctx->curve_id, ctx->ndigits,
-                             (const u8 *)params.key, params.key_size) < 0)
+                             (const u64 *)params.key, params.key_size) < 0)
                 return -EINVAL;
 
         memcpy(ctx->private_key, params.key, params.key_size);
@@ -81,16 +85,14 @@ static int ecdh_compute_value(struct kpp_request *req)
                         return -EINVAL;
 
                 ret = crypto_ecdh_shared_secret(ctx->curve_id, ctx->ndigits,
-                                         (const u8 *)ctx->private_key, nbytes,
-                                         (const u8 *)ctx->public_key, 2 * nbytes,
-                                         (u8 *)ctx->shared_secret, nbytes);
+                                                ctx->private_key,
+                                                ctx->public_key,
+                                                ctx->shared_secret);
 
                 buf = ctx->shared_secret;
         } else {
-                ret = ecdh_make_pub_key(ctx->curve_id, ctx->ndigits,
-                                        (const u8 *)ctx->private_key, nbytes,
-                                        (u8 *)ctx->public_key,
-                                        sizeof(ctx->public_key));
+                ret = ecc_make_pub_key(ctx->curve_id, ctx->ndigits,
+                                       ctx->private_key, ctx->public_key);
                 buf = ctx->public_key;
                 /* Public part is a point thus it has both coordinates */
                 nbytes *= 2;
@@ -106,13 +108,12 @@ static int ecdh_compute_value(struct kpp_request *req)
         return ret;
 }
 
-static int ecdh_max_size(struct crypto_kpp *tfm)
+static unsigned int ecdh_max_size(struct crypto_kpp *tfm)
 {
         struct ecdh_ctx *ctx = ecdh_get_ctx(tfm);
-        int nbytes = ctx->ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
 
-        /* Public key is made of two coordinates */
-        return 2 * nbytes;
+        /* Public key is made of two coordinates, add one to the left shift */
+        return ctx->ndigits << (ECC_DIGITS_TO_BYTES_SHIFT + 1);
 }
 
 static void no_exit_tfm(struct crypto_kpp *tfm)
diff --git a/crypto/ecdh_helper.c b/crypto/ecdh_helper.c
index 3cd8a2414e60..f05bea5fd257 100644
--- a/crypto/ecdh_helper.c
+++ b/crypto/ecdh_helper.c
@@ -3,9 +3,9 @@
  * Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
  *
  * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public Licence
+ * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
- * 2 of the Licence, or (at your option) any later version.
+ * 2 of the License, or (at your option) any later version.
  */
 #include <linux/kernel.h>
 #include <linux/export.h>
diff --git a/crypto/hmac.c b/crypto/hmac.c
index 72e38c098bb3..92871dc2a63e 100644
--- a/crypto/hmac.c
+++ b/crypto/hmac.c
@@ -16,6 +16,7 @@
  *
  */
 
+#include <crypto/hmac.h>
 #include <crypto/internal/hash.h>
 #include <crypto/scatterwalk.h>
 #include <linux/err.h>
@@ -74,8 +75,8 @@ static int hmac_setkey(struct crypto_shash *parent,
         memcpy(opad, ipad, bs);
 
         for (i = 0; i < bs; i++) {
-                ipad[i] ^= 0x36;
-                opad[i] ^= 0x5c;
+                ipad[i] ^= HMAC_IPAD_VALUE;
+                opad[i] ^= HMAC_OPAD_VALUE;
         }
 
         return crypto_shash_init(shash) ?:
diff --git a/crypto/rng.c b/crypto/rng.c
index f46dac5288b9..5e8469244960 100644
--- a/crypto/rng.c
+++ b/crypto/rng.c
@@ -33,11 +33,6 @@ struct crypto_rng *crypto_default_rng;
 EXPORT_SYMBOL_GPL(crypto_default_rng);
 static int crypto_default_rng_refcnt;
 
-static inline struct crypto_rng *__crypto_rng_cast(struct crypto_tfm *tfm)
-{
-        return container_of(tfm, struct crypto_rng, base);
-}
-
 int crypto_rng_reset(struct crypto_rng *tfm, const u8 *seed, unsigned int slen)
 {
         u8 *buf = NULL;
diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c
index 8baab4307f7b..407c64bdcdd9 100644
--- a/crypto/rsa-pkcs1pad.c
+++ b/crypto/rsa-pkcs1pad.c
@@ -120,9 +120,6 @@ static int pkcs1pad_set_pub_key(struct crypto_akcipher *tfm, const void *key,
 
         /* Find out new modulus size from rsa implementation */
         err = crypto_akcipher_maxsize(ctx->child);
-        if (err < 0)
-                return err;
-
         if (err > PAGE_SIZE)
                 return -ENOTSUPP;
 
@@ -144,9 +141,6 @@ static int pkcs1pad_set_priv_key(struct crypto_akcipher *tfm, const void *key,
 
         /* Find out new modulus size from rsa implementation */
         err = crypto_akcipher_maxsize(ctx->child);
-        if (err < 0)
-                return err;
-
         if (err > PAGE_SIZE)
                 return -ENOTSUPP;
 
@@ -154,7 +148,7 @@ static int pkcs1pad_set_priv_key(struct crypto_akcipher *tfm, const void *key,
         return 0;
 }
 
-static int pkcs1pad_get_max_size(struct crypto_akcipher *tfm)
+static unsigned int pkcs1pad_get_max_size(struct crypto_akcipher *tfm)
 {
         struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm);
 
@@ -164,7 +158,7 @@ static int pkcs1pad_get_max_size(struct crypto_akcipher *tfm)
          * decrypt/verify.
          */
 
-        return ctx->key_size ?: -EINVAL;
+        return ctx->key_size;
 }
 
 static void pkcs1pad_sg_set_buf(struct scatterlist *sg, void *buf, size_t len,
@@ -496,7 +490,7 @@ static int pkcs1pad_verify_complete(struct akcipher_request *req, int err)
                 goto done;
         pos++;
 
-        if (memcmp(out_buf + pos, digest_info->data, digest_info->size))
+        if (crypto_memneq(out_buf + pos, digest_info->data, digest_info->size))
                 goto done;
 
         pos += digest_info->size;
diff --git a/crypto/rsa.c b/crypto/rsa.c
index 4c280b6a3ea9..b067f3a93880 100644
--- a/crypto/rsa.c
+++ b/crypto/rsa.c
@@ -337,11 +337,11 @@ err:
         return -ENOMEM;
 }
 
-static int rsa_max_size(struct crypto_akcipher *tfm)
+static unsigned int rsa_max_size(struct crypto_akcipher *tfm)
 {
         struct rsa_mpi_key *pkey = akcipher_tfm_ctx(tfm);
 
-        return pkey->n ? mpi_get_size(pkey->n) : -EINVAL;
+        return mpi_get_size(pkey->n);
 }
 
 static void rsa_exit_tfm(struct crypto_akcipher *tfm)
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index 9a11f3c2bf98..0dd6a432d6ca 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -138,8 +138,6 @@ static int test_aead_cycles(struct aead_request *req, int enc, int blen)
         int ret = 0;
         int i;
 
-        local_irq_disable();
-
         /* Warm-up run. */
         for (i = 0; i < 4; i++) {
                 if (enc)
@@ -169,8 +167,6 @@ static int test_aead_cycles(struct aead_request *req, int enc, int blen)
         }
 
 out:
-        local_irq_enable();
-
         if (ret == 0)
                 printk("1 operation in %lu cycles (%d bytes)\n",
                        (cycles + 4) / 8, blen);
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 6f5f3ed8376c..7125ba3880af 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -218,14 +218,14 @@ static int ahash_partial_update(struct ahash_request **preq,
                         crypto_ahash_reqtfm(req));
         state = kmalloc(statesize + sizeof(guard), GFP_KERNEL);
         if (!state) {
-                pr_err("alt: hash: Failed to alloc state for %s\n", algo);
+                pr_err("alg: hash: Failed to alloc state for %s\n", algo);
                 goto out_nostate;
         }
         memcpy(state + statesize, guard, sizeof(guard));
         ret = crypto_ahash_export(req, state);
         WARN_ON(memcmp(state + statesize, guard, sizeof(guard)));
         if (ret) {
-                pr_err("alt: hash: Failed to export() for %s\n", algo);
+                pr_err("alg: hash: Failed to export() for %s\n", algo);
                 goto out;
         }
         ahash_request_free(req);
@@ -344,19 +344,19 @@ static int __test_hash(struct crypto_ahash *tfm,
                 } else {
                         ret = wait_async_op(&tresult, crypto_ahash_init(req));
                         if (ret) {
-                                pr_err("alt: hash: init failed on test %d "
+                                pr_err("alg: hash: init failed on test %d "
                                        "for %s: ret=%d\n", j, algo, -ret);
                                 goto out;
                         }
                         ret = wait_async_op(&tresult, crypto_ahash_update(req));
                         if (ret) {
-                                pr_err("alt: hash: update failed on test %d "
+                                pr_err("alg: hash: update failed on test %d "
                                        "for %s: ret=%d\n", j, algo, -ret);
                                 goto out;
                         }
                         ret = wait_async_op(&tresult, crypto_ahash_final(req));
                         if (ret) {
-                                pr_err("alt: hash: final failed on test %d "
+                                pr_err("alg: hash: final failed on test %d "
                                        "for %s: ret=%d\n", j, algo, -ret);
                                 goto out;
                         }
@@ -488,13 +488,13 @@ static int __test_hash(struct crypto_ahash *tfm,
                 ahash_request_set_crypt(req, sg, result, template[i].tap[0]);
                 ret = wait_async_op(&tresult, crypto_ahash_init(req));
                 if (ret) {
-                        pr_err("alt: hash: init failed on test %d for %s: ret=%d\n",
+                        pr_err("alg: hash: init failed on test %d for %s: ret=%d\n",
                                 j, algo, -ret);
                         goto out;
                 }
                 ret = wait_async_op(&tresult, crypto_ahash_update(req));
                 if (ret) {
-                        pr_err("alt: hash: update failed on test %d for %s: ret=%d\n",
+                        pr_err("alg: hash: update failed on test %d for %s: ret=%d\n",
                                 j, algo, -ret);
                         goto out;
                 }
@@ -505,7 +505,7 @@ static int __test_hash(struct crypto_ahash *tfm,
                                 hash_buff, k, temp, &sg[0], algo, result,
                                 &tresult);
                         if (ret) {
-                                pr_err("hash: partial update failed on test %d for %s: ret=%d\n",
+                                pr_err("alg: hash: partial update failed on test %d for %s: ret=%d\n",
                                         j, algo, -ret);
                                 goto out_noreq;
                         }
@@ -513,7 +513,7 @@ static int __test_hash(struct crypto_ahash *tfm,
                 }
                 ret = wait_async_op(&tresult, crypto_ahash_final(req));
                 if (ret) {
-                        pr_err("alt: hash: final failed on test %d for %s: ret=%d\n",
+                        pr_err("alg: hash: final failed on test %d for %s: ret=%d\n",
                                 j, algo, -ret);
                         goto out;
                 }
@@ -1997,6 +1997,9 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec,
         struct kpp_request *req;
         void *input_buf = NULL;
         void *output_buf = NULL;
+        void *a_public = NULL;
+        void *a_ss = NULL;
+        void *shared_secret = NULL;
         struct tcrypt_result result;
         unsigned int out_len_max;
         int err = -ENOMEM;
@@ -2026,20 +2029,31 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec,
         kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
                                  tcrypt_complete, &result);
 
-        /* Compute public key */
+        /* Compute party A's public key */
         err = wait_async_op(&result, crypto_kpp_generate_public_key(req));
         if (err) {
-                pr_err("alg: %s: generate public key test failed. err %d\n",
+                pr_err("alg: %s: Party A: generate public key test failed. err %d\n",
                        alg, err);
                 goto free_output;
         }
-        /* Verify calculated public key */
-        if (memcmp(vec->expected_a_public, sg_virt(req->dst),
-                   vec->expected_a_public_size)) {
-                pr_err("alg: %s: generate public key test failed. Invalid output\n",
-                       alg);
-                err = -EINVAL;
-                goto free_output;
+
+        if (vec->genkey) {
+                /* Save party A's public key */
+                a_public = kzalloc(out_len_max, GFP_KERNEL);
+                if (!a_public) {
+                        err = -ENOMEM;
+                        goto free_output;
+                }
+                memcpy(a_public, sg_virt(req->dst), out_len_max);
+        } else {
+                /* Verify calculated public key */
+                if (memcmp(vec->expected_a_public, sg_virt(req->dst),
+                           vec->expected_a_public_size)) {
+                        pr_err("alg: %s: Party A: generate public key test failed. Invalid output\n",
+                               alg);
+                        err = -EINVAL;
+                        goto free_output;
+                }
         }
 
         /* Calculate shared secret key by using counter part (b) public key. */
@@ -2058,15 +2072,53 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec,
                                  tcrypt_complete, &result);
         err = wait_async_op(&result, crypto_kpp_compute_shared_secret(req));
         if (err) {
-                pr_err("alg: %s: compute shard secret test failed. err %d\n",
+                pr_err("alg: %s: Party A: compute shared secret test failed. err %d\n",
                        alg, err);
                 goto free_all;
         }
+
+        if (vec->genkey) {
+                /* Save the shared secret obtained by party A */
+                a_ss = kzalloc(vec->expected_ss_size, GFP_KERNEL);
+                if (!a_ss) {
+                        err = -ENOMEM;
+                        goto free_all;
+                }
+                memcpy(a_ss, sg_virt(req->dst), vec->expected_ss_size);
+
+                /*
+                 * Calculate party B's shared secret by using party A's
+                 * public key.
+                 */
+                err = crypto_kpp_set_secret(tfm, vec->b_secret,
+                                            vec->b_secret_size);
+                if (err < 0)
+                        goto free_all;
+
+                sg_init_one(&src, a_public, vec->expected_a_public_size);
+                sg_init_one(&dst, output_buf, out_len_max);
+                kpp_request_set_input(req, &src, vec->expected_a_public_size);
+                kpp_request_set_output(req, &dst, out_len_max);
+                kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+                                         tcrypt_complete, &result);
+                err = wait_async_op(&result,
+                                    crypto_kpp_compute_shared_secret(req));
+                if (err) {
+                        pr_err("alg: %s: Party B: compute shared secret failed. err %d\n",
+                               alg, err);
+                        goto free_all;
+                }
+
+                shared_secret = a_ss;
+        } else {
+                shared_secret = (void *)vec->expected_ss;
+        }
+
         /*
          * verify shared secret from which the user will derive
          * secret key by executing whatever hash it has chosen
          */
-        if (memcmp(vec->expected_ss, sg_virt(req->dst),
+        if (memcmp(shared_secret, sg_virt(req->dst),
                    vec->expected_ss_size)) {
                 pr_err("alg: %s: compute shared secret test failed. Invalid output\n",
                        alg);
@@ -2074,8 +2126,10 @@ static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec,
         }
 
 free_all:
+        kfree(a_ss);
         kfree(input_buf);
 free_output:
+        kfree(a_public);
         kfree(output_buf);
 free_req:
         kpp_request_free(req);
@@ -2168,8 +2222,11 @@ static int test_akcipher_one(struct crypto_akcipher *tfm,
         akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
                                       tcrypt_complete, &result);
 
-        /* Run RSA encrypt - c = m^e mod n;*/
-        err = wait_async_op(&result, crypto_akcipher_encrypt(req));
+        err = wait_async_op(&result, vecs->siggen_sigver_test ?
+                                     /* Run asymmetric signature generation */
+                                     crypto_akcipher_sign(req) :
+                                     /* Run asymmetric encrypt */
+                                     crypto_akcipher_encrypt(req));
         if (err) {
                 pr_err("alg: akcipher: encrypt test failed. err %d\n", err);
                 goto free_all;
@@ -2207,8 +2264,11 @@ static int test_akcipher_one(struct crypto_akcipher *tfm,
         init_completion(&result.completion);
         akcipher_request_set_crypt(req, &src, &dst, vecs->c_size, out_len_max);
 
-        /* Run RSA decrypt - m = c^d mod n;*/
-        err = wait_async_op(&result, crypto_akcipher_decrypt(req));
+        err = wait_async_op(&result, vecs->siggen_sigver_test ?
+                                     /* Run asymmetric signature verification */
+                                     crypto_akcipher_verify(req) :
+                                     /* Run asymmetric decrypt */
+                                     crypto_akcipher_decrypt(req));
         if (err) {
                 pr_err("alg: akcipher: decrypt test failed. err %d\n", err);
                 goto free_all;
@@ -2306,6 +2366,7 @@ static const struct alg_test_desc alg_test_descs[] = {
         }, {
                 .alg = "authenc(hmac(sha1),cbc(aes))",
                 .test = alg_test_aead,
+                .fips_allowed = 1,
                 .suite = {
                         .aead = {
                                 .enc = __VECS(hmac_sha1_aes_cbc_enc_tv_temp)
@@ -3255,6 +3316,25 @@ static const struct alg_test_desc alg_test_descs[] = {
                         }
                 }
         }, {
+                .alg = "pkcs1pad(rsa,sha224)",
+                .test = alg_test_null,
+                .fips_allowed = 1,
+        }, {
+                .alg = "pkcs1pad(rsa,sha256)",
+                .test = alg_test_akcipher,
+                .fips_allowed = 1,
+                .suite = {
+                        .akcipher = __VECS(pkcs1pad_rsa_tv_template)
+                }
+        }, {
+                .alg = "pkcs1pad(rsa,sha384)",
+                .test = alg_test_null,
+                .fips_allowed = 1,
+        }, {
+                .alg = "pkcs1pad(rsa,sha512)",
+                .test = alg_test_null,
+                .fips_allowed = 1,
+        }, {
                 .alg = "poly1305",
                 .test = alg_test_hash,
                 .suite = {
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 429357339dcc..6ceb0e2758bb 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -133,17 +133,21 @@ struct akcipher_testvec {
         unsigned int m_size;
         unsigned int c_size;
         bool public_key_vec;
+        bool siggen_sigver_test;
 };
 
 struct kpp_testvec {
         const unsigned char *secret;
+        const unsigned char *b_secret;
         const unsigned char *b_public;
         const unsigned char *expected_a_public;
         const unsigned char *expected_ss;
         unsigned short secret_size;
+        unsigned short b_secret_size;
         unsigned short b_public_size;
         unsigned short expected_a_public_size;
         unsigned short expected_ss_size;
+        bool genkey;
 };
 
 static const char zeroed_string[48];
@@ -538,6 +542,101 @@ static const struct akcipher_testvec rsa_tv_template[] = {
         }
 };
 
+/*
+ * PKCS#1 RSA test vectors. Obtained from CAVS testing.
+ */
+static const struct akcipher_testvec pkcs1pad_rsa_tv_template[] = {
+        {
+        .key =
+        "\x30\x82\x03\x1f\x02\x01\x10\x02\x82\x01\x01\x00\xd7\x1e\x77\x82"
+        "\x8c\x92\x31\xe7\x69\x02\xa2\xd5\x5c\x78\xde\xa2\x0c\x8f\xfe\x28"
+        "\x59\x31\xdf\x40\x9c\x60\x61\x06\xb9\x2f\x62\x40\x80\x76\xcb\x67"
+        "\x4a\xb5\x59\x56\x69\x17\x07\xfa\xf9\x4c\xbd\x6c\x37\x7a\x46\x7d"
+        "\x70\xa7\x67\x22\xb3\x4d\x7a\x94\xc3\xba\x4b\x7c\x4b\xa9\x32\x7c"
+        "\xb7\x38\x95\x45\x64\xa4\x05\xa8\x9f\x12\x7c\x4e\xc6\xc8\x2d\x40"
+        "\x06\x30\xf4\x60\xa6\x91\xbb\x9b\xca\x04\x79\x11\x13\x75\xf0\xae"
+        "\xd3\x51\x89\xc5\x74\xb9\xaa\x3f\xb6\x83\xe4\x78\x6b\xcd\xf9\x5c"
+        "\x4c\x85\xea\x52\x3b\x51\x93\xfc\x14\x6b\x33\x5d\x30\x70\xfa\x50"
+        "\x1b\x1b\x38\x81\x13\x8d\xf7\xa5\x0c\xc0\x8e\xf9\x63\x52\x18\x4e"
+        "\xa9\xf9\xf8\x5c\x5d\xcd\x7a\x0d\xd4\x8e\x7b\xee\x91\x7b\xad\x7d"
+        "\xb4\x92\xd5\xab\x16\x3b\x0a\x8a\xce\x8e\xde\x47\x1a\x17\x01\x86"
+        "\x7b\xab\x99\xf1\x4b\x0c\x3a\x0d\x82\x47\xc1\x91\x8c\xbb\x2e\x22"
+        "\x9e\x49\x63\x6e\x02\xc1\xc9\x3a\x9b\xa5\x22\x1b\x07\x95\xd6\x10"
+        "\x02\x50\xfd\xfd\xd1\x9b\xbe\xab\xc2\xc0\x74\xd7\xec\x00\xfb\x11"
+        "\x71\xcb\x7a\xdc\x81\x79\x9f\x86\x68\x46\x63\x82\x4d\xb7\xf1\xe6"
+        "\x16\x6f\x42\x63\xf4\x94\xa0\xca\x33\xcc\x75\x13\x02\x82\x01\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+        "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x01"
+        "\x02\x82\x01\x00\x62\xb5\x60\x31\x4f\x3f\x66\x16\xc1\x60\xac\x47"
+        "\x2a\xff\x6b\x69\x00\x4a\xb2\x5c\xe1\x50\xb9\x18\x74\xa8\xe4\xdc"
+        "\xa8\xec\xcd\x30\xbb\xc1\xc6\xe3\xc6\xac\x20\x2a\x3e\x5e\x8b\x12"
+        "\xe6\x82\x08\x09\x38\x0b\xab\x7c\xb3\xcc\x9c\xce\x97\x67\xdd\xef"
+        "\x95\x40\x4e\x92\xe2\x44\xe9\x1d\xc1\x14\xfd\xa9\xb1\xdc\x71\x9c"
+        "\x46\x21\xbd\x58\x88\x6e\x22\x15\x56\xc1\xef\xe0\xc9\x8d\xe5\x80"
+        "\x3e\xda\x7e\x93\x0f\x52\xf6\xf5\xc1\x91\x90\x9e\x42\x49\x4f\x8d"
+        "\x9c\xba\x38\x83\xe9\x33\xc2\x50\x4f\xec\xc2\xf0\xa8\xb7\x6e\x28"
+        "\x25\x56\x6b\x62\x67\xfe\x08\xf1\x56\xe5\x6f\x0e\x99\xf1\xe5\x95"
+        "\x7b\xef\xeb\x0a\x2c\x92\x97\x57\x23\x33\x36\x07\xdd\xfb\xae\xf1"
+        "\xb1\xd8\x33\xb7\x96\x71\x42\x36\xc5\xa4\xa9\x19\x4b\x1b\x52\x4c"
+        "\x50\x69\x91\xf0\x0e\xfa\x80\x37\x4b\xb5\xd0\x2f\xb7\x44\x0d\xd4"
+        "\xf8\x39\x8d\xab\x71\x67\x59\x05\x88\x3d\xeb\x48\x48\x33\x88\x4e"
+        "\xfe\xf8\x27\x1b\xd6\x55\x60\x5e\x48\xb7\x6d\x9a\xa8\x37\xf9\x7a"
+        "\xde\x1b\xcd\x5d\x1a\x30\xd4\xe9\x9e\x5b\x3c\x15\xf8\x9c\x1f\xda"
+        "\xd1\x86\x48\x55\xce\x83\xee\x8e\x51\xc7\xde\x32\x12\x47\x7d\x46"
+        "\xb8\x35\xdf\x41\x02\x01\x30\x02\x01\x30\x02\x01\x30\x02\x01\x30"
+        "\x02\x01\x30",
+        .key_len = 804,
+        /*
+         * m is SHA256 hash of following message:
+         * "\x49\x41\xbe\x0a\x0c\xc9\xf6\x35\x51\xe4\x27\x56\x13\x71\x4b\xd0"
+         * "\x36\x92\x84\x89\x1b\xf8\x56\x4a\x72\x61\x14\x69\x4f\x5e\x98\xa5"
+         * "\x80\x5a\x37\x51\x1f\xd8\xf5\xb5\x63\xfc\xf4\xb1\xbb\x4d\x33\xa3"
+         * "\x1e\xb9\x75\x8b\x9c\xda\x7e\x6d\x3a\x77\x85\xf7\xfc\x4e\xe7\x64"
+         * "\x43\x10\x19\xa0\x59\xae\xe0\xad\x4b\xd3\xc4\x45\xf7\xb1\xc2\xc1"
+         * "\x65\x01\x41\x39\x5b\x45\x47\xed\x2b\x51\xed\xe3\xd0\x09\x10\xd2"
+         * "\x39\x6c\x4a\x3f\xe5\xd2\x20\xe6\xb0\x71\x7d\x5b\xed\x26\x60\xf1"
+         * "\xb4\x73\xd1\xdb\x7d\xc4\x19\x91\xee\xf6\x32\x76\xf2\x19\x7d\xb7"
+         */
+        .m =
+        "\x3e\xc8\xa1\x26\x20\x54\x44\x52\x48\x0d\xe5\x66\xf3\xb3\xf5\x04"
+        "\xbe\x10\xa8\x48\x94\x22\x2d\xdd\xba\x7a\xb4\x76\x8d\x79\x98\x89",
+        .m_size = 32,
+        .c =
+        "\xc7\xa3\x98\xeb\x43\xd1\x08\xc2\x3d\x78\x45\x04\x70\xc9\x01\xee"
+        "\xf8\x85\x37\x7c\x0b\xf9\x19\x70\x5c\x45\x7b\x2f\x3a\x0b\xb7\x8b"
+        "\xc4\x0d\x7b\x3a\x64\x0b\x0f\xdb\x78\xa9\x0b\xfd\x8d\x82\xa4\x86"
+        "\x39\xbf\x21\xb8\x84\xc4\xce\x9f\xc2\xe8\xb6\x61\x46\x17\xb9\x4e"
+        "\x0b\x57\x05\xb4\x4f\xf9\x9c\x93\x2d\x9b\xd5\x48\x1d\x80\x12\xef"
+        "\x3a\x77\x7f\xbc\xb5\x8e\x2b\x6b\x7c\xfc\x9f\x8c\x9d\xa2\xc4\x85"
+        "\xb0\x87\xe9\x17\x9b\xb6\x23\x62\xd2\xa9\x9f\x57\xe8\xf7\x04\x45"
+        "\x24\x3a\x45\xeb\xeb\x6a\x08\x8e\xaf\xc8\xa0\x84\xbc\x5d\x13\x38"
+        "\xf5\x17\x8c\xa3\x96\x9b\xa9\x38\x8d\xf0\x35\xad\x32\x8a\x72\x5b"
+        "\xdf\x21\xab\x4b\x0e\xa8\x29\xbb\x61\x54\xbf\x05\xdb\x84\x84\xde"
+        "\xdd\x16\x36\x31\xda\xf3\x42\x6d\x7a\x90\x22\x9b\x11\x29\xa6\xf8"
+        "\x30\x61\xda\xd3\x8b\x54\x1e\x42\xd1\x47\x1d\x6f\xd1\xcd\x42\x0b"
+        "\xd1\xe4\x15\x85\x7e\x08\xd6\x59\x64\x4c\x01\x34\x91\x92\x26\xe8"
+        "\xb0\x25\x8c\xf8\xf4\xfa\x8b\xc9\x31\x33\x76\x72\xfb\x64\x92\x9f"
+        "\xda\x62\x8d\xe1\x2a\x71\x91\x43\x40\x61\x3c\x5a\xbe\x86\xfc\x5b"
+        "\xe6\xf9\xa9\x16\x31\x1f\xaf\x25\x6d\xc2\x4a\x23\x6e\x63\x02\xa2",
+        .c_size = 256,
+        .siggen_sigver_test = true,
+        }
+};
+
 static const struct kpp_testvec dh_tv_template[] = {
         {
         .secret =
@@ -840,6 +939,50 @@ static const struct kpp_testvec ecdh_tv_template[] = {
         .b_public_size = 64,
         .expected_a_public_size = 64,
         .expected_ss_size = 32
+        }, {
+        .secret =
+#ifdef __LITTLE_ENDIAN
+        "\x02\x00" /* type */
+        "\x08\x00" /* len */
+        "\x02\x00" /* curve_id */
+        "\x00\x00", /* key_size */
+#else
+        "\x00\x02" /* type */
+        "\x00\x08" /* len */
+        "\x00\x02" /* curve_id */
+        "\x00\x00", /* key_size */
+#endif
+        .b_secret =
+#ifdef __LITTLE_ENDIAN
+        "\x02\x00" /* type */
+        "\x28\x00" /* len */
+        "\x02\x00" /* curve_id */
+        "\x20\x00" /* key_size */
+#else
+        "\x00\x02" /* type */
+        "\x00\x28" /* len */
+        "\x00\x02" /* curve_id */
+        "\x00\x20" /* key_size */
+#endif
+        "\x24\xd1\x21\xeb\xe5\xcf\x2d\x83"
+        "\xf6\x62\x1b\x6e\x43\x84\x3a\xa3"
+        "\x8b\xe0\x86\xc3\x20\x19\xda\x92"
+        "\x50\x53\x03\xe1\xc0\xea\xb8\x82",
+        .b_public =
+        "\x1a\x7f\xeb\x52\x00\xbd\x3c\x31"
+        "\x7d\xb6\x70\xc1\x86\xa6\xc7\xc4"
+        "\x3b\xc5\x5f\x6c\x6f\x58\x3c\xf5"
+        "\xb6\x63\x82\x77\x33\x24\xa1\x5f"
+        "\x6a\xca\x43\x6f\xf7\x7e\xff\x02"
+        "\x37\x08\xcc\x40\x5e\x7a\xfd\x6a"
+        "\x6a\x02\x6e\x41\x87\x68\x38\x77"
+        "\xfa\xa9\x44\x43\x2d\xef\x09\xdf",
+        .secret_size = 8,
+        .b_secret_size = 40,
+        .b_public_size = 64,
+        .expected_a_public_size = 64,
+        .expected_ss_size = 32,
+        .genkey = true,
         }
 };
 
diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c
index df8eb54fd5a3..8da7bcf54105 100644
--- a/drivers/char/hw_random/mtk-rng.c
+++ b/drivers/char/hw_random/mtk-rng.c
@@ -25,6 +25,10 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+/* Runtime PM autosuspend timeout: */
+#define RNG_AUTOSUSPEND_TIMEOUT         100
 
 #define USEC_POLL                       2
 #define TIMEOUT_POLL                    20
@@ -90,6 +94,8 @@ static int mtk_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait)
         struct mtk_rng *priv = to_mtk_rng(rng);
         int retval = 0;
 
+        pm_runtime_get_sync((struct device *)priv->rng.priv);
+
         while (max >= sizeof(u32)) {
                 if (!mtk_rng_wait_ready(rng, wait))
                         break;
@@ -100,6 +106,9 @@ static int mtk_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait)
                 max -= sizeof(u32);
         }
 
+        pm_runtime_mark_last_busy((struct device *)priv->rng.priv);
+        pm_runtime_put_sync_autosuspend((struct device *)priv->rng.priv);
+
         return retval || !wait ? retval : -EIO;
 }
 
@@ -120,9 +129,12 @@ static int mtk_rng_probe(struct platform_device *pdev)
                 return -ENOMEM;
 
         priv->rng.name = pdev->name;
+#ifndef CONFIG_PM
         priv->rng.init = mtk_rng_init;
         priv->rng.cleanup = mtk_rng_cleanup;
+#endif
         priv->rng.read = mtk_rng_read;
+        priv->rng.priv = (unsigned long)&pdev->dev;
 
         priv->clk = devm_clk_get(&pdev->dev, "rng");
         if (IS_ERR(priv->clk)) {
@@ -142,11 +154,40 @@ static int mtk_rng_probe(struct platform_device *pdev)
                 return ret;
         }
 
+        dev_set_drvdata(&pdev->dev, priv);
+        pm_runtime_set_autosuspend_delay(&pdev->dev, RNG_AUTOSUSPEND_TIMEOUT);
+        pm_runtime_use_autosuspend(&pdev->dev);
+        pm_runtime_enable(&pdev->dev);
+
         dev_info(&pdev->dev, "registered RNG driver\n");
 
         return 0;
 }
 
+#ifdef CONFIG_PM
+static int mtk_rng_runtime_suspend(struct device *dev)
+{
+        struct mtk_rng *priv = dev_get_drvdata(dev);
+
+        mtk_rng_cleanup(&priv->rng);
+
+        return 0;
+}
+
+static int mtk_rng_runtime_resume(struct device *dev)
+{
+        struct mtk_rng *priv = dev_get_drvdata(dev);
+
+        return mtk_rng_init(&priv->rng);
+}
+
+static UNIVERSAL_DEV_PM_OPS(mtk_rng_pm_ops, mtk_rng_runtime_suspend,
+                            mtk_rng_runtime_resume, NULL);
+#define MTK_RNG_PM_OPS (&mtk_rng_pm_ops)
+#else   /* CONFIG_PM */
+#define MTK_RNG_PM_OPS NULL
+#endif  /* CONFIG_PM */
+
 static const struct of_device_id mtk_rng_match[] = {
         { .compatible = "mediatek,mt7623-rng" },
         {},
@@ -157,6 +198,7 @@ static struct platform_driver mtk_rng_driver = {
         .probe          = mtk_rng_probe,
         .driver = {
                 .name = MTK_RNG_DEV,
+                .pm = MTK_RNG_PM_OPS,
                 .of_match_table = mtk_rng_match,
         },
 };
diff --git a/drivers/char/hw_random/omap3-rom-rng.c b/drivers/char/hw_random/omap3-rom-rng.c
index 37a58d78aab3..38b719017186 100644
--- a/drivers/char/hw_random/omap3-rom-rng.c
+++ b/drivers/char/hw_random/omap3-rom-rng.c
@@ -53,7 +53,10 @@ static int omap3_rom_rng_get_random(void *buf, unsigned int count)
 
         cancel_delayed_work_sync(&idle_work);
         if (rng_idle) {
-                clk_prepare_enable(rng_clk);
+                r = clk_prepare_enable(rng_clk);
+                if (r)
+                        return r;
+
                 r = omap3_rom_rng_call(0, 0, RNG_GEN_PRNG_HW_INIT);
                 if (r != 0) {
                         clk_disable_unprepare(rng_clk);
@@ -88,6 +91,8 @@ static struct hwrng omap3_rom_rng_ops = {
 
 static int omap3_rom_rng_probe(struct platform_device *pdev)
 {
+        int ret = 0;
+
         pr_info("initializing\n");
 
         omap3_rom_rng_call = pdev->dev.platform_data;
@@ -104,7 +109,9 @@ static int omap3_rom_rng_probe(struct platform_device *pdev)
         }
 
         /* Leave the RNG in reset state. */
-        clk_prepare_enable(rng_clk);
+        ret = clk_prepare_enable(rng_clk);
+        if (ret)
+                return ret;
         omap3_rom_rng_idle(0);
 
         return hwrng_register(&omap3_rom_rng_ops);
diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c
index a0faa5f05deb..03ff5483d865 100644
--- a/drivers/char/hw_random/timeriomem-rng.c
+++ b/drivers/char/hw_random/timeriomem-rng.c
@@ -151,8 +151,15 @@ static int timeriomem_rng_probe(struct platform_device *pdev)
                         dev_err(&pdev->dev, "missing period\n");
                         return -EINVAL;
                 }
+
+                if (!of_property_read_u32(pdev->dev.of_node,
+                                                "quality", &i))
+                        priv->rng_ops.quality = i;
+                else
+                        priv->rng_ops.quality = 0;
         } else {
                 period = pdata->period;
+                priv->rng_ops.quality = pdata->quality;
         }
 
         priv->period = ns_to_ktime(period * NSEC_PER_USEC);
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 9c7951bb05ac..193204dfbf3a 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -327,6 +327,15 @@ config HW_RANDOM_PPC4XX
          This option provides the kernel-side support for the TRNG hardware
          found in the security function of some PowerPC 4xx SoCs.
 
+config CRYPTO_DEV_OMAP
+        tristate "Support for OMAP crypto HW accelerators"
+        depends on ARCH_OMAP2PLUS
+        help
+          OMAP processors have various crypto HW accelerators. Select this if
+          you want to use the OMAP modules for any of the crypto algorithms.
+
+if CRYPTO_DEV_OMAP
+
 config CRYPTO_DEV_OMAP_SHAM
         tristate "Support for OMAP MD5/SHA1/SHA2 hw accelerator"
         depends on ARCH_OMAP2PLUS
@@ -348,6 +357,7 @@ config CRYPTO_DEV_OMAP_AES
         select CRYPTO_CBC
         select CRYPTO_ECB
         select CRYPTO_CTR
+        select CRYPTO_AEAD
         help
           OMAP processors have AES module accelerator. Select this if you
           want to use the OMAP module for AES algorithms.
@@ -364,6 +374,8 @@ config CRYPTO_DEV_OMAP_DES
           the ECB and CBC modes of operation are supported by the driver. Also
           accesses made on unaligned boundaries are supported.
 
+endif # CRYPTO_DEV_OMAP
+
 config CRYPTO_DEV_PICOXCELL
         tristate "Support for picoXcell IPSEC and Layer2 crypto engines"
         depends on (ARCH_PICOXCELL || COMPILE_TEST) && HAVE_CLK
@@ -542,6 +554,7 @@ config CRYPTO_DEV_MXS_DCP
 
 source "drivers/crypto/qat/Kconfig"
 source "drivers/crypto/cavium/cpt/Kconfig"
+source "drivers/crypto/cavium/nitrox/Kconfig"
 
 config CRYPTO_DEV_CAVIUM_ZIP
         tristate "Cavium ZIP driver"
@@ -656,4 +669,21 @@ config CRYPTO_DEV_BCM_SPU
 
 source "drivers/crypto/stm32/Kconfig"
 
+config CRYPTO_DEV_SAFEXCEL
+        tristate "Inside Secure's SafeXcel cryptographic engine driver"
+        depends on HAS_DMA && OF
+        depends on (ARM64 && ARCH_MVEBU) || (COMPILE_TEST && 64BIT)
+        select CRYPTO_AES
+        select CRYPTO_BLKCIPHER
+        select CRYPTO_HASH
+        select CRYPTO_HMAC
+        select CRYPTO_SHA1
+        select CRYPTO_SHA256
+        select CRYPTO_SHA512
+        help
+          This driver interfaces with the SafeXcel EIP-197 cryptographic engine
+          designed by Inside Secure. Select this if you want to use CBC/ECB
+          chain mode, AES cipher mode and SHA1/SHA224/SHA256/SHA512 hash
+          algorithms.
+
 endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 463f33592d93..2c555a3393b2 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -6,6 +6,7 @@ obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/
 obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/
 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chelsio/
 obj-$(CONFIG_CRYPTO_DEV_CPT) += cavium/cpt/
+obj-$(CONFIG_CRYPTO_DEV_NITROX) += cavium/nitrox/
 obj-$(CONFIG_CRYPTO_DEV_EXYNOS_RNG) += exynos-rng.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/
 obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
@@ -20,7 +21,9 @@ obj-$(CONFIG_CRYPTO_DEV_MXC_SCC) += mxc-scc.o
 obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o
 n2_crypto-y := n2_core.o n2_asm.o
 obj-$(CONFIG_CRYPTO_DEV_NX) += nx/
-obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes.o
+obj-$(CONFIG_CRYPTO_DEV_OMAP) += omap-crypto.o
+obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes-driver.o
+omap-aes-driver-objs := omap-aes.o omap-aes-gcm.o
 obj-$(CONFIG_CRYPTO_DEV_OMAP_DES) += omap-des.o
 obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
@@ -39,3 +42,4 @@ obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/
 obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/
 obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/
 obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/
+obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index fdc83a2281ca..65dc78b91dea 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -1179,6 +1179,7 @@ static int crypto4xx_probe(struct platform_device *ofdev)
         dev_set_drvdata(dev, core_dev);
         core_dev->ofdev = ofdev;
         core_dev->dev = kzalloc(sizeof(struct crypto4xx_device), GFP_KERNEL);
+        rc = -ENOMEM;
         if (!core_dev->dev)
                 goto err_alloc_dev;
 
diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c
index cc0d5b98006e..9cfd36c1bcb6 100644
--- a/drivers/crypto/bcm/cipher.c
+++ b/drivers/crypto/bcm/cipher.c
@@ -36,6 +36,7 @@
 #include <crypto/internal/aead.h>
 #include <crypto/aes.h>
 #include <crypto/des.h>
+#include <crypto/hmac.h>
 #include <crypto/sha.h>
 #include <crypto/md5.h>
 #include <crypto/authenc.h>
@@ -2510,8 +2511,8 @@ static int ahash_hmac_setkey(struct crypto_ahash *ahash, const u8 *key,
                 memcpy(ctx->opad, ctx->ipad, blocksize);
 
                 for (index = 0; index < blocksize; index++) {
-                        ctx->ipad[index] ^= 0x36;
-                        ctx->opad[index] ^= 0x5c;
+                        ctx->ipad[index] ^= HMAC_IPAD_VALUE;
+                        ctx->opad[index] ^= HMAC_OPAD_VALUE;
                 }
 
                 flow_dump("  ipad: ", ctx->ipad, blocksize);
@@ -2638,7 +2639,7 @@ static int aead_need_fallback(struct aead_request *req)
             (spu->spu_type == SPU_TYPE_SPUM) &&
             (ctx->digestsize != 8) && (ctx->digestsize != 12) &&
             (ctx->digestsize != 16)) {
-                flow_log("%s() AES CCM needs fallbck for digest size %d\n",
+                flow_log("%s() AES CCM needs fallback for digest size %d\n",
                          __func__, ctx->digestsize);
                 return 1;
         }
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 398807d1b77e..fde399c88779 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -1187,8 +1187,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         struct crypto_aead *aead = crypto_aead_reqtfm(req);
         struct caam_ctx *ctx = crypto_aead_ctx(aead);
         struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
         struct aead_edesc *edesc;
         int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
@@ -1475,8 +1475,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
         struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                                          CRYPTO_TFM_REQ_MAY_SLEEP)) ?
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                        GFP_KERNEL : GFP_ATOMIC;
         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
         struct ablkcipher_edesc *edesc;
@@ -1681,8 +1680,7 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
         struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                                          CRYPTO_TFM_REQ_MAY_SLEEP)) ?
+        gfp_t flags = (req->base.flags &  CRYPTO_TFM_REQ_MAY_SLEEP) ?
                        GFP_KERNEL : GFP_ATOMIC;
         int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents;
         struct ablkcipher_edesc *edesc;
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index ea0e5b8b9171..78c4c0485c58 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -555,8 +555,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
                                                  typeof(*alg), aead);
         struct device *qidev = ctx->qidev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
         struct aead_edesc *edesc;
         dma_addr_t qm_sg_dma, iv_dma = 0;
@@ -808,8 +808,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
         struct device *qidev = ctx->qidev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                                          CRYPTO_TFM_REQ_MAY_SLEEP)) ?
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                        GFP_KERNEL : GFP_ATOMIC;
         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
         struct ablkcipher_edesc *edesc;
@@ -953,8 +952,7 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
         struct device *qidev = ctx->qidev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                                          CRYPTO_TFM_REQ_MAY_SLEEP)) ?
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                        GFP_KERNEL : GFP_ATOMIC;
         int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents;
         struct ablkcipher_edesc *edesc;
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index da4f94eab3da..7c44c90ad593 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -719,8 +719,8 @@ static int ahash_update_ctx(struct ahash_request *req)
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
         struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
         u8 *buf = current_buf(state);
         int *buflen = current_buflen(state);
         u8 *next_buf = alt_buf(state);
@@ -849,8 +849,8 @@ static int ahash_final_ctx(struct ahash_request *req)
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
         struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
         int buflen = *current_buflen(state);
         u32 *desc;
         int sec4_sg_bytes, sec4_sg_src_index;
@@ -926,8 +926,8 @@ static int ahash_finup_ctx(struct ahash_request *req)
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
         struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
         int buflen = *current_buflen(state);
         u32 *desc;
         int sec4_sg_src_index;
@@ -1013,8 +1013,8 @@ static int ahash_digest(struct ahash_request *req)
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
         struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
         u32 *desc;
         int digestsize = crypto_ahash_digestsize(ahash);
         int src_nents, mapped_nents;
@@ -1093,8 +1093,8 @@ static int ahash_final_no_ctx(struct ahash_request *req)
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
         struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
         u8 *buf = current_buf(state);
         int buflen = *current_buflen(state);
         u32 *desc;
@@ -1154,8 +1154,8 @@ static int ahash_update_no_ctx(struct ahash_request *req)
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
         struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
         u8 *buf = current_buf(state);
         int *buflen = current_buflen(state);
         u8 *next_buf = alt_buf(state);
@@ -1280,8 +1280,8 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
         struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
         int buflen = *current_buflen(state);
         u32 *desc;
         int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
@@ -1370,8 +1370,8 @@ static int ahash_update_first(struct ahash_request *req)
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
         struct device *jrdev = ctx->jrdev;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
         u8 *next_buf = alt_buf(state);
         int *next_buflen = alt_buflen(state);
         int to_hash;
diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c
index 49cbdcba7883..7a897209f181 100644
--- a/drivers/crypto/caam/caampkc.c
+++ b/drivers/crypto/caam/caampkc.c
@@ -18,6 +18,10 @@
 #define DESC_RSA_PUB_LEN        (2 * CAAM_CMD_SZ + sizeof(struct rsa_pub_pdb))
 #define DESC_RSA_PRIV_F1_LEN    (2 * CAAM_CMD_SZ + \
                                  sizeof(struct rsa_priv_f1_pdb))
+#define DESC_RSA_PRIV_F2_LEN    (2 * CAAM_CMD_SZ + \
+                                 sizeof(struct rsa_priv_f2_pdb))
+#define DESC_RSA_PRIV_F3_LEN    (2 * CAAM_CMD_SZ + \
+                                 sizeof(struct rsa_priv_f3_pdb))
 
 static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc,
                          struct akcipher_request *req)
@@ -54,6 +58,42 @@ static void rsa_priv_f1_unmap(struct device *dev, struct rsa_edesc *edesc,
         dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE);
 }
 
+static void rsa_priv_f2_unmap(struct device *dev, struct rsa_edesc *edesc,
+                              struct akcipher_request *req)
+{
+        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
+        struct caam_rsa_key *key = &ctx->key;
+        struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2;
+        size_t p_sz = key->p_sz;
+        size_t q_sz = key->p_sz;
+
+        dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE);
+        dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE);
+        dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE);
+        dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE);
+        dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_TO_DEVICE);
+}
+
+static void rsa_priv_f3_unmap(struct device *dev, struct rsa_edesc *edesc,
+                              struct akcipher_request *req)
+{
+        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
+        struct caam_rsa_key *key = &ctx->key;
+        struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3;
+        size_t p_sz = key->p_sz;
+        size_t q_sz = key->p_sz;
+
+        dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE);
+        dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE);
+        dma_unmap_single(dev, pdb->dp_dma, p_sz, DMA_TO_DEVICE);
+        dma_unmap_single(dev, pdb->dq_dma, q_sz, DMA_TO_DEVICE);
+        dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE);
+        dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE);
+        dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_TO_DEVICE);
+}
+
 /* RSA Job Completion handler */
 static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context)
 {
@@ -90,6 +130,42 @@ static void rsa_priv_f1_done(struct device *dev, u32 *desc, u32 err,
         akcipher_request_complete(req, err);
 }
 
+static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err,
+                             void *context)
+{
+        struct akcipher_request *req = context;
+        struct rsa_edesc *edesc;
+
+        if (err)
+                caam_jr_strstatus(dev, err);
+
+        edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
+
+        rsa_priv_f2_unmap(dev, edesc, req);
+        rsa_io_unmap(dev, edesc, req);
+        kfree(edesc);
+
+        akcipher_request_complete(req, err);
+}
+
+static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err,
+                             void *context)
+{
+        struct akcipher_request *req = context;
+        struct rsa_edesc *edesc;
+
+        if (err)
+                caam_jr_strstatus(dev, err);
+
+        edesc = container_of(desc, struct rsa_edesc, hw_desc[0]);
+
+        rsa_priv_f3_unmap(dev, edesc, req);
+        rsa_io_unmap(dev, edesc, req);
+        kfree(edesc);
+
+        akcipher_request_complete(req, err);
+}
+
 static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
                                          size_t desclen)
 {
@@ -97,8 +173,8 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
         struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
         struct device *dev = ctx->dev;
         struct rsa_edesc *edesc;
-        gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
-                       CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+        gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                       GFP_KERNEL : GFP_ATOMIC;
         int sgc;
         int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
         int src_nents, dst_nents;
@@ -258,6 +334,172 @@ static int set_rsa_priv_f1_pdb(struct akcipher_request *req,
         return 0;
 }
 
+static int set_rsa_priv_f2_pdb(struct akcipher_request *req,
+                               struct rsa_edesc *edesc)
+{
+        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
+        struct caam_rsa_key *key = &ctx->key;
+        struct device *dev = ctx->dev;
+        struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2;
+        int sec4_sg_index = 0;
+        size_t p_sz = key->p_sz;
+        size_t q_sz = key->p_sz;
+
+        pdb->d_dma = dma_map_single(dev, key->d, key->d_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->d_dma)) {
+                dev_err(dev, "Unable to map RSA private exponent memory\n");
+                return -ENOMEM;
+        }
+
+        pdb->p_dma = dma_map_single(dev, key->p, p_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->p_dma)) {
+                dev_err(dev, "Unable to map RSA prime factor p memory\n");
+                goto unmap_d;
+        }
+
+        pdb->q_dma = dma_map_single(dev, key->q, q_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->q_dma)) {
+                dev_err(dev, "Unable to map RSA prime factor q memory\n");
+                goto unmap_p;
+        }
+
+        pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->tmp1_dma)) {
+                dev_err(dev, "Unable to map RSA tmp1 memory\n");
+                goto unmap_q;
+        }
+
+        pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->tmp2_dma)) {
+                dev_err(dev, "Unable to map RSA tmp2 memory\n");
+                goto unmap_tmp1;
+        }
+
+        if (edesc->src_nents > 1) {
+                pdb->sgf |= RSA_PRIV_PDB_SGF_G;
+                pdb->g_dma = edesc->sec4_sg_dma;
+                sec4_sg_index += edesc->src_nents;
+        } else {
+                pdb->g_dma = sg_dma_address(req->src);
+        }
+
+        if (edesc->dst_nents > 1) {
+                pdb->sgf |= RSA_PRIV_PDB_SGF_F;
+                pdb->f_dma = edesc->sec4_sg_dma +
+                             sec4_sg_index * sizeof(struct sec4_sg_entry);
+        } else {
+                pdb->f_dma = sg_dma_address(req->dst);
+        }
+
+        pdb->sgf |= (key->d_sz << RSA_PDB_D_SHIFT) | key->n_sz;
+        pdb->p_q_len = (q_sz << RSA_PDB_Q_SHIFT) | p_sz;
+
+        return 0;
+
+unmap_tmp1:
+        dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE);
+unmap_q:
+        dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE);
+unmap_p:
+        dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE);
+unmap_d:
+        dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE);
+
+        return -ENOMEM;
+}
+
+static int set_rsa_priv_f3_pdb(struct akcipher_request *req,
+                               struct rsa_edesc *edesc)
+{
+        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
+        struct caam_rsa_key *key = &ctx->key;
+        struct device *dev = ctx->dev;
+        struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3;
+        int sec4_sg_index = 0;
+        size_t p_sz = key->p_sz;
+        size_t q_sz = key->p_sz;
+
+        pdb->p_dma = dma_map_single(dev, key->p, p_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->p_dma)) {
+                dev_err(dev, "Unable to map RSA prime factor p memory\n");
+                return -ENOMEM;
+        }
+
+        pdb->q_dma = dma_map_single(dev, key->q, q_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->q_dma)) {
+                dev_err(dev, "Unable to map RSA prime factor q memory\n");
+                goto unmap_p;
+        }
+
+        pdb->dp_dma = dma_map_single(dev, key->dp, p_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->dp_dma)) {
+                dev_err(dev, "Unable to map RSA exponent dp memory\n");
+                goto unmap_q;
+        }
+
+        pdb->dq_dma = dma_map_single(dev, key->dq, q_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->dq_dma)) {
+                dev_err(dev, "Unable to map RSA exponent dq memory\n");
+                goto unmap_dp;
+        }
+
+        pdb->c_dma = dma_map_single(dev, key->qinv, p_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->c_dma)) {
+                dev_err(dev, "Unable to map RSA CRT coefficient qinv memory\n");
+                goto unmap_dq;
+        }
+
+        pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->tmp1_dma)) {
+                dev_err(dev, "Unable to map RSA tmp1 memory\n");
+                goto unmap_qinv;
+        }
+
+        pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_TO_DEVICE);
+        if (dma_mapping_error(dev, pdb->tmp2_dma)) {
+                dev_err(dev, "Unable to map RSA tmp2 memory\n");
+                goto unmap_tmp1;
+        }
+
+        if (edesc->src_nents > 1) {
+                pdb->sgf |= RSA_PRIV_PDB_SGF_G;
+                pdb->g_dma = edesc->sec4_sg_dma;
+                sec4_sg_index += edesc->src_nents;
+        } else {
+                pdb->g_dma = sg_dma_address(req->src);
+        }
+
+        if (edesc->dst_nents > 1) {
+                pdb->sgf |= RSA_PRIV_PDB_SGF_F;
+                pdb->f_dma = edesc->sec4_sg_dma +
+                             sec4_sg_index * sizeof(struct sec4_sg_entry);
+        } else {
+                pdb->f_dma = sg_dma_address(req->dst);
+        }
+
+        pdb->sgf |= key->n_sz;
+        pdb->p_q_len = (q_sz << RSA_PDB_Q_SHIFT) | p_sz;
+
+        return 0;
+
+unmap_tmp1:
+        dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE);
+unmap_qinv:
+        dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE);
+unmap_dq:
+        dma_unmap_single(dev, pdb->dq_dma, q_sz, DMA_TO_DEVICE);
+unmap_dp:
+        dma_unmap_single(dev, pdb->dp_dma, p_sz, DMA_TO_DEVICE);
+unmap_q:
+        dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE);
+unmap_p:
+        dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE);
+
+        return -ENOMEM;
+}
+
 static int caam_rsa_enc(struct akcipher_request *req)
 {
         struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
@@ -301,24 +543,14 @@ init_fail:
         return ret;
 }
 
-static int caam_rsa_dec(struct akcipher_request *req)
+static int caam_rsa_dec_priv_f1(struct akcipher_request *req)
 {
         struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
         struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
-        struct caam_rsa_key *key = &ctx->key;
         struct device *jrdev = ctx->dev;
         struct rsa_edesc *edesc;
         int ret;
 
-        if (unlikely(!key->n || !key->d))
-                return -EINVAL;
-
-        if (req->dst_len < key->n_sz) {
-                req->dst_len = key->n_sz;
-                dev_err(jrdev, "Output buffer length less than parameter n\n");
-                return -EOVERFLOW;
-        }
-
         /* Allocate extended descriptor */
         edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F1_LEN);
         if (IS_ERR(edesc))
@@ -344,17 +576,147 @@ init_fail:
         return ret;
 }
 
+static int caam_rsa_dec_priv_f2(struct akcipher_request *req)
+{
+        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
+        struct device *jrdev = ctx->dev;
+        struct rsa_edesc *edesc;
+        int ret;
+
+        /* Allocate extended descriptor */
+        edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F2_LEN);
+        if (IS_ERR(edesc))
+                return PTR_ERR(edesc);
+
+        /* Set RSA Decrypt Protocol Data Block - Private Key Form #2 */
+        ret = set_rsa_priv_f2_pdb(req, edesc);
+        if (ret)
+                goto init_fail;
+
+        /* Initialize Job Descriptor */
+        init_rsa_priv_f2_desc(edesc->hw_desc, &edesc->pdb.priv_f2);
+
+        ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f2_done, req);
+        if (!ret)
+                return -EINPROGRESS;
+
+        rsa_priv_f2_unmap(jrdev, edesc, req);
+
+init_fail:
+        rsa_io_unmap(jrdev, edesc, req);
+        kfree(edesc);
+        return ret;
+}
+
+static int caam_rsa_dec_priv_f3(struct akcipher_request *req)
+{
+        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
+        struct device *jrdev = ctx->dev;
+        struct rsa_edesc *edesc;
+        int ret;
+
+        /* Allocate extended descriptor */
+        edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F3_LEN);
+        if (IS_ERR(edesc))
+                return PTR_ERR(edesc);
+
+        /* Set RSA Decrypt Protocol Data Block - Private Key Form #3 */
+        ret = set_rsa_priv_f3_pdb(req, edesc);
+        if (ret)
+                goto init_fail;
+
+        /* Initialize Job Descriptor */
+        init_rsa_priv_f3_desc(edesc->hw_desc, &edesc->pdb.priv_f3);
+
+        ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f3_done, req);
+        if (!ret)
+                return -EINPROGRESS;
+
+        rsa_priv_f3_unmap(jrdev, edesc, req);
+
+init_fail:
+        rsa_io_unmap(jrdev, edesc, req);
+        kfree(edesc);
+        return ret;
+}
+
+static int caam_rsa_dec(struct akcipher_request *req)
+{
+        struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
+        struct caam_rsa_key *key = &ctx->key;
+        int ret;
+
+        if (unlikely(!key->n || !key->d))
+                return -EINVAL;
+
+        if (req->dst_len < key->n_sz) {
+                req->dst_len = key->n_sz;
+                dev_err(ctx->dev, "Output buffer length less than parameter n\n");
+                return -EOVERFLOW;
+        }
+
+        if (key->priv_form == FORM3)
+                ret = caam_rsa_dec_priv_f3(req);
+        else if (key->priv_form == FORM2)
+                ret = caam_rsa_dec_priv_f2(req);
+        else
+                ret = caam_rsa_dec_priv_f1(req);
+
+        return ret;
+}
+
 static void caam_rsa_free_key(struct caam_rsa_key *key)
 {
         kzfree(key->d);
+        kzfree(key->p);
+        kzfree(key->q);
+        kzfree(key->dp);
+        kzfree(key->dq);
+        kzfree(key->qinv);
+        kzfree(key->tmp1);
+        kzfree(key->tmp2);
         kfree(key->e);
         kfree(key->n);
-        key->d = NULL;
-        key->e = NULL;
-        key->n = NULL;
-        key->d_sz = 0;
-        key->e_sz = 0;
-        key->n_sz = 0;
+        memset(key, 0, sizeof(*key));
+}
+
+static void caam_rsa_drop_leading_zeros(const u8 **ptr, size_t *nbytes)
+{
+        while (!**ptr && *nbytes) {
+                (*ptr)++;
+                (*nbytes)--;
+        }
+}
+
+/**
+ * caam_read_rsa_crt - Used for reading dP, dQ, qInv CRT members.
+ * dP, dQ and qInv could decode to less than corresponding p, q length, as the
+ * BER-encoding requires that the minimum number of bytes be used to encode the
+ * integer. dP, dQ, qInv decoded values have to be zero-padded to appropriate
+ * length.
+ *
+ * @ptr   : pointer to {dP, dQ, qInv} CRT member
+ * @nbytes: length in bytes of {dP, dQ, qInv} CRT member
+ * @dstlen: length in bytes of corresponding p or q prime factor
+ */
+static u8 *caam_read_rsa_crt(const u8 *ptr, size_t nbytes, size_t dstlen)
+{
+        u8 *dst;
+
+        caam_rsa_drop_leading_zeros(&ptr, &nbytes);
+        if (!nbytes)
+                return NULL;
+
+        dst = kzalloc(dstlen, GFP_DMA | GFP_KERNEL);
+        if (!dst)
+                return NULL;
+
+        memcpy(dst + (dstlen - nbytes), ptr, nbytes);
+
+        return dst;
 }
 
 /**
@@ -370,10 +732,9 @@ static inline u8 *caam_read_raw_data(const u8 *buf, size_t *nbytes)
 {
         u8 *val;
 
-        while (!*buf && *nbytes) {
-                buf++;
-                (*nbytes)--;
-        }
+        caam_rsa_drop_leading_zeros(&buf, nbytes);
+        if (!*nbytes)
+                return NULL;
 
         val = kzalloc(*nbytes, GFP_DMA | GFP_KERNEL);
         if (!val)
@@ -437,6 +798,64 @@ err:
         return -ENOMEM;
 }
 
+static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx,
+                                       struct rsa_key *raw_key)
+{
+        struct caam_rsa_key *rsa_key = &ctx->key;
+        size_t p_sz = raw_key->p_sz;
+        size_t q_sz = raw_key->q_sz;
+
+        rsa_key->p = caam_read_raw_data(raw_key->p, &p_sz);
+        if (!rsa_key->p)
+                return;
+        rsa_key->p_sz = p_sz;
+
+        rsa_key->q = caam_read_raw_data(raw_key->q, &q_sz);
+        if (!rsa_key->q)
+                goto free_p;
+        rsa_key->q_sz = q_sz;
+
+        rsa_key->tmp1 = kzalloc(raw_key->p_sz, GFP_DMA | GFP_KERNEL);
+        if (!rsa_key->tmp1)
+                goto free_q;
+
+        rsa_key->tmp2 = kzalloc(raw_key->q_sz, GFP_DMA | GFP_KERNEL);
+        if (!rsa_key->tmp2)
+                goto free_tmp1;
+
+        rsa_key->priv_form = FORM2;
+
+        rsa_key->dp = caam_read_rsa_crt(raw_key->dp, raw_key->dp_sz, p_sz);
+        if (!rsa_key->dp)
+                goto free_tmp2;
+
+        rsa_key->dq = caam_read_rsa_crt(raw_key->dq, raw_key->dq_sz, q_sz);
+        if (!rsa_key->dq)
+                goto free_dp;
+
+        rsa_key->qinv = caam_read_rsa_crt(raw_key->qinv, raw_key->qinv_sz,
+                                          q_sz);
+        if (!rsa_key->qinv)
+                goto free_dq;
+
+        rsa_key->priv_form = FORM3;
+
+        return;
+
+free_dq:
+        kzfree(rsa_key->dq);
+free_dp:
+        kzfree(rsa_key->dp);
+free_tmp2:
+        kzfree(rsa_key->tmp2);
+free_tmp1:
+        kzfree(rsa_key->tmp1);
+free_q:
+        kzfree(rsa_key->q);
+free_p:
+        kzfree(rsa_key->p);
+}
+
 static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
                                  unsigned int keylen)
 {
@@ -483,6 +902,8 @@ static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
         memcpy(rsa_key->d, raw_key.d, raw_key.d_sz);
         memcpy(rsa_key->e, raw_key.e, raw_key.e_sz);
 
+        caam_rsa_set_priv_key_form(ctx, &raw_key);
+
         return 0;
 
 err:
@@ -490,12 +911,11 @@ err:
         return -ENOMEM;
 }
 
-static int caam_rsa_max_size(struct crypto_akcipher *tfm)
+static unsigned int caam_rsa_max_size(struct crypto_akcipher *tfm)
 {
         struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
-        struct caam_rsa_key *key = &ctx->key;
 
-        return (key->n) ? key->n_sz : -EINVAL;
+        return ctx->key.n_sz;
 }
 
 /* Per session pkc's driver context creation function */
diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h
index f595d159b112..87ab75e9df43 100644
--- a/drivers/crypto/caam/caampkc.h
+++ b/drivers/crypto/caam/caampkc.h
@@ -13,21 +13,75 @@
 #include "pdb.h"
 
 /**
+ * caam_priv_key_form - CAAM RSA private key representation
+ * CAAM RSA private key may have either of three forms.
+ *
+ * 1. The first representation consists of the pair (n, d), where the
+ *    components have the following meanings:
+ *        n      the RSA modulus
+ *        d      the RSA private exponent
+ *
+ * 2. The second representation consists of the triplet (p, q, d), where the
+ *    components have the following meanings:
+ *        p      the first prime factor of the RSA modulus n
+ *        q      the second prime factor of the RSA modulus n
+ *        d      the RSA private exponent
+ *
+ * 3. The third representation consists of the quintuple (p, q, dP, dQ, qInv),
+ *    where the components have the following meanings:
+ *        p      the first prime factor of the RSA modulus n
+ *        q      the second prime factor of the RSA modulus n
+ *        dP     the first factors's CRT exponent
+ *        dQ     the second factors's CRT exponent
+ *        qInv   the (first) CRT coefficient
+ *
+ * The benefit of using the third or the second key form is lower computational
+ * cost for the decryption and signature operations.
+ */
+enum caam_priv_key_form {
+        FORM1,
+        FORM2,
+        FORM3
+};
+
+/**
  * caam_rsa_key - CAAM RSA key structure. Keys are allocated in DMA zone.
  * @n           : RSA modulus raw byte stream
  * @e           : RSA public exponent raw byte stream
  * @d           : RSA private exponent raw byte stream
+ * @p           : RSA prime factor p of RSA modulus n
+ * @q           : RSA prime factor q of RSA modulus n
+ * @dp          : RSA CRT exponent of p
+ * @dp          : RSA CRT exponent of q
+ * @qinv        : RSA CRT coefficient
+ * @tmp1        : CAAM uses this temporary buffer as internal state buffer.
+ *                It is assumed to be as long as p.
+ * @tmp2        : CAAM uses this temporary buffer as internal state buffer.
+ *                It is assumed to be as long as q.
  * @n_sz        : length in bytes of RSA modulus n
  * @e_sz        : length in bytes of RSA public exponent
  * @d_sz        : length in bytes of RSA private exponent
+ * @p_sz        : length in bytes of RSA prime factor p of RSA modulus n
+ * @q_sz        : length in bytes of RSA prime factor q of RSA modulus n
+ * @priv_form   : CAAM RSA private key representation
  */
 struct caam_rsa_key {
         u8 *n;
         u8 *e;
         u8 *d;
+        u8 *p;
+        u8 *q;
+        u8 *dp;
+        u8 *dq;
+        u8 *qinv;
+        u8 *tmp1;
+        u8 *tmp2;
         size_t n_sz;
         size_t e_sz;
         size_t d_sz;
+        size_t p_sz;
+        size_t q_sz;
+        enum caam_priv_key_form priv_form;
 };
 
 /**
@@ -59,6 +113,8 @@ struct rsa_edesc {
         union {
                 struct rsa_pub_pdb pub;
                 struct rsa_priv_f1_pdb priv_f1;
+                struct rsa_priv_f2_pdb priv_f2;
+                struct rsa_priv_f3_pdb priv_f3;
         } pdb;
         u32 hw_desc[];
 };
@@ -66,5 +122,7 @@ struct rsa_edesc {
 /* Descriptor construction primitives. */
 void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb);
 void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb);
+void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb);
+void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb);
 
 #endif
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index 27631000b9f8..1ccfb317d468 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -536,7 +536,7 @@ static int caam_jr_probe(struct platform_device *pdev)
         return 0;
 }
 
-static struct of_device_id caam_jr_match[] = {
+static const struct of_device_id caam_jr_match[] = {
         {
                 .compatible = "fsl,sec-v4.0-job-ring",
         },
diff --git a/drivers/crypto/caam/pdb.h b/drivers/crypto/caam/pdb.h
index aaa00dd1c601..31e59963f4d2 100644
--- a/drivers/crypto/caam/pdb.h
+++ b/drivers/crypto/caam/pdb.h
@@ -483,6 +483,8 @@ struct dsa_verify_pdb {
 #define RSA_PDB_E_MASK          (0xFFF << RSA_PDB_E_SHIFT)
 #define RSA_PDB_D_SHIFT         12
 #define RSA_PDB_D_MASK          (0xFFF << RSA_PDB_D_SHIFT)
+#define RSA_PDB_Q_SHIFT         12
+#define RSA_PDB_Q_MASK          (0xFFF << RSA_PDB_Q_SHIFT)
 
 #define RSA_PDB_SGF_F           (0x8 << RSA_PDB_SGF_SHIFT)
 #define RSA_PDB_SGF_G           (0x4 << RSA_PDB_SGF_SHIFT)
@@ -490,6 +492,8 @@ struct dsa_verify_pdb {
 #define RSA_PRIV_PDB_SGF_G      (0x8 << RSA_PDB_SGF_SHIFT)
 
 #define RSA_PRIV_KEY_FRM_1      0
+#define RSA_PRIV_KEY_FRM_2      1
+#define RSA_PRIV_KEY_FRM_3      2
 
 /**
  * RSA Encrypt Protocol Data Block
@@ -525,4 +529,62 @@ struct rsa_priv_f1_pdb {
         dma_addr_t      d_dma;
 } __packed;
 
+/**
+ * RSA Decrypt PDB - Private Key Form #2
+ * @sgf     : scatter-gather field
+ * @g_dma   : dma address of encrypted input data
+ * @f_dma   : dma address of output data
+ * @d_dma   : dma address of RSA private exponent
+ * @p_dma   : dma address of RSA prime factor p of RSA modulus n
+ * @q_dma   : dma address of RSA prime factor q of RSA modulus n
+ * @tmp1_dma: dma address of temporary buffer. CAAM uses this temporary buffer
+ *            as internal state buffer. It is assumed to be as long as p.
+ * @tmp2_dma: dma address of temporary buffer. CAAM uses this temporary buffer
+ *            as internal state buffer. It is assumed to be as long as q.
+ * @p_q_len : length in bytes of first two prime factors of the RSA modulus n
+ */
+struct rsa_priv_f2_pdb {
+        u32             sgf;
+        dma_addr_t      g_dma;
+        dma_addr_t      f_dma;
+        dma_addr_t      d_dma;
+        dma_addr_t      p_dma;
+        dma_addr_t      q_dma;
+        dma_addr_t      tmp1_dma;
+        dma_addr_t      tmp2_dma;
+        u32             p_q_len;
+} __packed;
+
+/**
+ * RSA Decrypt PDB - Private Key Form #3
+ * This is the RSA Chinese Reminder Theorem (CRT) form for two prime factors of
+ * the RSA modulus.
+ * @sgf     : scatter-gather field
+ * @g_dma   : dma address of encrypted input data
+ * @f_dma   : dma address of output data
+ * @c_dma   : dma address of RSA CRT coefficient
+ * @p_dma   : dma address of RSA prime factor p of RSA modulus n
+ * @q_dma   : dma address of RSA prime factor q of RSA modulus n
+ * @dp_dma  : dma address of RSA CRT exponent of RSA prime factor p
+ * @dp_dma  : dma address of RSA CRT exponent of RSA prime factor q
+ * @tmp1_dma: dma address of temporary buffer. CAAM uses this temporary buffer
+ *            as internal state buffer. It is assumed to be as long as p.
+ * @tmp2_dma: dma address of temporary buffer. CAAM uses this temporary buffer
+ *            as internal state buffer. It is assumed to be as long as q.
+ * @p_q_len : length in bytes of first two prime factors of the RSA modulus n
+ */
+struct rsa_priv_f3_pdb {
+        u32             sgf;
+        dma_addr_t      g_dma;
+        dma_addr_t      f_dma;
+        dma_addr_t      c_dma;
+        dma_addr_t      p_dma;
+        dma_addr_t      q_dma;
+        dma_addr_t      dp_dma;
+        dma_addr_t      dq_dma;
+        dma_addr_t      tmp1_dma;
+        dma_addr_t      tmp2_dma;
+        u32             p_q_len;
+} __packed;
+
 #endif
diff --git a/drivers/crypto/caam/pkc_desc.c b/drivers/crypto/caam/pkc_desc.c
index 4e4183e615ea..9e2ce6fe2e43 100644
--- a/drivers/crypto/caam/pkc_desc.c
+++ b/drivers/crypto/caam/pkc_desc.c
@@ -34,3 +34,39 @@ void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb)
         append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY |
                          RSA_PRIV_KEY_FRM_1);
 }
+
+/* Descriptor for RSA Private operation - Private Key Form #2 */
+void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb)
+{
+        init_job_desc_pdb(desc, 0, sizeof(*pdb));
+        append_cmd(desc, pdb->sgf);
+        append_ptr(desc, pdb->g_dma);
+        append_ptr(desc, pdb->f_dma);
+        append_ptr(desc, pdb->d_dma);
+        append_ptr(desc, pdb->p_dma);
+        append_ptr(desc, pdb->q_dma);
+        append_ptr(desc, pdb->tmp1_dma);
+        append_ptr(desc, pdb->tmp2_dma);
+        append_cmd(desc, pdb->p_q_len);
+        append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY |
+                         RSA_PRIV_KEY_FRM_2);
+}
+
+/* Descriptor for RSA Private operation - Private Key Form #3 */
+void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb)
+{
+        init_job_desc_pdb(desc, 0, sizeof(*pdb));
+        append_cmd(desc, pdb->sgf);
+        append_ptr(desc, pdb->g_dma);
+        append_ptr(desc, pdb->f_dma);
+        append_ptr(desc, pdb->c_dma);
+        append_ptr(desc, pdb->p_dma);
+        append_ptr(desc, pdb->q_dma);
+        append_ptr(desc, pdb->dp_dma);
+        append_ptr(desc, pdb->dq_dma);
+        append_ptr(desc, pdb->tmp1_dma);
+        append_ptr(desc, pdb->tmp2_dma);
+        append_cmd(desc, pdb->p_q_len);
+        append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY |
+                         RSA_PRIV_KEY_FRM_3);
+}
diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c
index cc853f913d4b..1b220f3ed017 100644
--- a/drivers/crypto/cavium/cpt/cptvf_algs.c
+++ b/drivers/crypto/cavium/cpt/cptvf_algs.c
@@ -98,7 +98,6 @@ static inline void update_output_data(struct cpt_request_info *req_info,
 }
 
 static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc,
-                                 u32 cipher_type, u32 aes_key_type,
                                  u32 *argcnt)
 {
         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
@@ -124,11 +123,11 @@ static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc,
         req_info->req.param1 = req->nbytes; /* Encryption Data length */
         req_info->req.param2 = 0; /*Auth data length */
 
-        fctx->enc.enc_ctrl.e.enc_cipher = cipher_type;
-        fctx->enc.enc_ctrl.e.aes_key = aes_key_type;
+        fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
+        fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
         fctx->enc.enc_ctrl.e.iv_source = FROM_DPTR;
 
-        if (cipher_type == AES_XTS)
+        if (ctx->cipher_type == AES_XTS)
                 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
         else
                 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
@@ -154,14 +153,13 @@ static inline u32 create_ctx_hdr(struct ablkcipher_request *req, u32 enc,
 }
 
 static inline u32 create_input_list(struct ablkcipher_request  *req, u32 enc,
-                                    u32 cipher_type, u32 aes_key_type,
                                     u32 enc_iv_len)
 {
         struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
         struct cpt_request_info *req_info = &rctx->cpt_req;
         u32 argcnt =  0;
 
-        create_ctx_hdr(req, enc, cipher_type, aes_key_type, &argcnt);
+        create_ctx_hdr(req, enc, &argcnt);
         update_input_iv(req_info, req->info, enc_iv_len, &argcnt);
         update_input_data(req_info, req->src, req->nbytes, &argcnt);
         req_info->incnt = argcnt;
@@ -177,7 +175,6 @@ static inline void store_cb_info(struct ablkcipher_request *req,
 }
 
 static inline void create_output_list(struct ablkcipher_request *req,
-                                      u32 cipher_type,
                                       u32 enc_iv_len)
 {
         struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
@@ -197,12 +194,9 @@ static inline void create_output_list(struct ablkcipher_request *req,
         req_info->outcnt = argcnt;
 }
 
-static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc,
-                              u32 cipher_type)
+static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc)
 {
         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
-        struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-        u32 key_type = AES_128_BIT;
         struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
         u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
         struct fc_context *fctx = &rctx->fctx;
@@ -210,36 +204,10 @@ static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc,
         void *cdev = NULL;
         int status;
 
-        switch (ctx->key_len) {
-        case 16:
-                key_type = AES_128_BIT;
-                break;
-        case 24:
-                key_type = AES_192_BIT;
-                break;
-        case 32:
-                if (cipher_type == AES_XTS)
-                        key_type = AES_128_BIT;
-                else
-                        key_type = AES_256_BIT;
-                break;
-        case 64:
-                if (cipher_type == AES_XTS)
-                        key_type = AES_256_BIT;
-                else
-                        return -EINVAL;
-                break;
-        default:
-                return -EINVAL;
-        }
-
-        if (cipher_type == DES3_CBC)
-                key_type = 0;
-
         memset(req_info, 0, sizeof(struct cpt_request_info));
         memset(fctx, 0, sizeof(struct fc_context));
-        create_input_list(req, enc, cipher_type, key_type, enc_iv_len);
-        create_output_list(req, cipher_type, enc_iv_len);
+        create_input_list(req, enc, enc_iv_len);
+        create_output_list(req, enc_iv_len);
         store_cb_info(req, req_info);
         cdev = dev_handle.cdev[smp_processor_id()];
         status = cptvf_do_request(cdev, req_info);
@@ -254,34 +222,14 @@ static inline int cvm_enc_dec(struct ablkcipher_request *req, u32 enc,
                 return -EINPROGRESS;
 }
 
-int cvm_des3_encrypt_cbc(struct ablkcipher_request *req)
+int cvm_encrypt(struct ablkcipher_request *req)
 {
-        return cvm_enc_dec(req, true, DES3_CBC);
+        return cvm_enc_dec(req, true);
 }
 
-int cvm_des3_decrypt_cbc(struct ablkcipher_request *req)
+int cvm_decrypt(struct ablkcipher_request *req)
 {
-        return cvm_enc_dec(req, false, DES3_CBC);
-}
-
-int cvm_aes_encrypt_xts(struct ablkcipher_request *req)
-{
-        return cvm_enc_dec(req, true, AES_XTS);
-}
-
-int cvm_aes_decrypt_xts(struct ablkcipher_request *req)
-{
-        return cvm_enc_dec(req, false, AES_XTS);
-}
-
-int cvm_aes_encrypt_cbc(struct ablkcipher_request *req)
-{
-        return cvm_enc_dec(req, true, AES_CBC);
-}
-
-int cvm_aes_decrypt_cbc(struct ablkcipher_request *req)
-{
-        return cvm_enc_dec(req, false, AES_CBC);
+        return cvm_enc_dec(req, false);
 }
 
 int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
@@ -299,24 +247,93 @@ int cvm_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
         ctx->key_len = keylen;
         memcpy(ctx->enc_key, key1, keylen / 2);
         memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
+        ctx->cipher_type = AES_XTS;
+        switch (ctx->key_len) {
+        case 32:
+                ctx->key_type = AES_128_BIT;
+                break;
+        case 64:
+                ctx->key_type = AES_256_BIT;
+                break;
+        default:
+                return -EINVAL;
+        }
 
         return 0;
 }
 
-int cvm_enc_dec_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
-                       u32 keylen)
+static int cvm_validate_keylen(struct cvm_enc_ctx *ctx, u32 keylen)
+{
+        if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
+                ctx->key_len = keylen;
+                switch (ctx->key_len) {
+                case 16:
+                        ctx->key_type = AES_128_BIT;
+                        break;
+                case 24:
+                        ctx->key_type = AES_192_BIT;
+                        break;
+                case 32:
+                        ctx->key_type = AES_256_BIT;
+                        break;
+                default:
+                        return -EINVAL;
+                }
+
+                if (ctx->cipher_type == DES3_CBC)
+                        ctx->key_type = 0;
+
+                return 0;
+        }
+
+        return -EINVAL;
+}
+
+static int cvm_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+                      u32 keylen, u8 cipher_type)
 {
         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
         struct cvm_enc_ctx *ctx = crypto_tfm_ctx(tfm);
 
-        if ((keylen == 16) || (keylen == 24) || (keylen == 32)) {
-                ctx->key_len = keylen;
+        ctx->cipher_type = cipher_type;
+        if (!cvm_validate_keylen(ctx, keylen)) {
                 memcpy(ctx->enc_key, key, keylen);
                 return 0;
+        } else {
+                crypto_ablkcipher_set_flags(cipher,
+                                            CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
         }
-        crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+}
 
-        return -EINVAL;
+static int cvm_cbc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+                              u32 keylen)
+{
+        return cvm_setkey(cipher, key, keylen, AES_CBC);
+}
+
+static int cvm_ecb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+                              u32 keylen)
+{
+        return cvm_setkey(cipher, key, keylen, AES_ECB);
+}
+
+static int cvm_cfb_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+                              u32 keylen)
+{
+        return cvm_setkey(cipher, key, keylen, AES_CFB);
+}
+
+static int cvm_cbc_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+                               u32 keylen)
+{
+        return cvm_setkey(cipher, key, keylen, DES3_CBC);
+}
+
+static int cvm_ecb_des3_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+                               u32 keylen)
+{
+        return cvm_setkey(cipher, key, keylen, DES3_ECB);
 }
 
 int cvm_enc_dec_init(struct crypto_tfm *tfm)
@@ -349,8 +366,8 @@ struct crypto_alg algs[] = { {
                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
                         .setkey = cvm_xts_setkey,
-                        .encrypt = cvm_aes_encrypt_xts,
-                        .decrypt = cvm_aes_decrypt_xts,
+                        .encrypt = cvm_encrypt,
+                        .decrypt = cvm_decrypt,
                 },
         },
         .cra_init = cvm_enc_dec_init,
@@ -369,9 +386,51 @@ struct crypto_alg algs[] = { {
                         .ivsize = AES_BLOCK_SIZE,
                         .min_keysize = AES_MIN_KEY_SIZE,
                         .max_keysize = AES_MAX_KEY_SIZE,
-                        .setkey = cvm_enc_dec_setkey,
-                        .encrypt = cvm_aes_encrypt_cbc,
-                        .decrypt = cvm_aes_decrypt_cbc,
+                        .setkey = cvm_cbc_aes_setkey,
+                        .encrypt = cvm_encrypt,
+                        .decrypt = cvm_decrypt,
+                },
+        },
+        .cra_init = cvm_enc_dec_init,
+        .cra_module = THIS_MODULE,
+}, {
+        .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+        .cra_blocksize = AES_BLOCK_SIZE,
+        .cra_ctxsize = sizeof(struct cvm_enc_ctx),
+        .cra_alignmask = 7,
+        .cra_priority = 4001,
+        .cra_name = "ecb(aes)",
+        .cra_driver_name = "cavium-ecb-aes",
+        .cra_type = &crypto_ablkcipher_type,
+        .cra_u = {
+                .ablkcipher = {
+                        .ivsize = AES_BLOCK_SIZE,
+                        .min_keysize = AES_MIN_KEY_SIZE,
+                        .max_keysize = AES_MAX_KEY_SIZE,
+                        .setkey = cvm_ecb_aes_setkey,
+                        .encrypt = cvm_encrypt,
+                        .decrypt = cvm_decrypt,
+                },
+        },
+        .cra_init = cvm_enc_dec_init,
+        .cra_module = THIS_MODULE,
+}, {
+        .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+        .cra_blocksize = AES_BLOCK_SIZE,
+        .cra_ctxsize = sizeof(struct cvm_enc_ctx),
+        .cra_alignmask = 7,
+        .cra_priority = 4001,
+        .cra_name = "cfb(aes)",
+        .cra_driver_name = "cavium-cfb-aes",
+        .cra_type = &crypto_ablkcipher_type,
+        .cra_u = {
+                .ablkcipher = {
+                        .ivsize = AES_BLOCK_SIZE,
+                        .min_keysize = AES_MIN_KEY_SIZE,
+                        .max_keysize = AES_MAX_KEY_SIZE,
+                        .setkey = cvm_cfb_aes_setkey,
+                        .encrypt = cvm_encrypt,
+                        .decrypt = cvm_decrypt,
                 },
         },
         .cra_init = cvm_enc_dec_init,
@@ -390,9 +449,30 @@ struct crypto_alg algs[] = { {
                         .min_keysize = DES3_EDE_KEY_SIZE,
                         .max_keysize = DES3_EDE_KEY_SIZE,
                         .ivsize = DES_BLOCK_SIZE,
-                        .setkey = cvm_enc_dec_setkey,
-                        .encrypt = cvm_des3_encrypt_cbc,
-                        .decrypt = cvm_des3_decrypt_cbc,
+                        .setkey = cvm_cbc_des3_setkey,
+                        .encrypt = cvm_encrypt,
+                        .decrypt = cvm_decrypt,
+                },
+        },
+        .cra_init = cvm_enc_dec_init,
+        .cra_module = THIS_MODULE,
+}, {
+        .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+        .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+        .cra_ctxsize = sizeof(struct cvm_des3_ctx),
+        .cra_alignmask = 7,
+        .cra_priority = 4001,
+        .cra_name = "ecb(des3_ede)",
+        .cra_driver_name = "cavium-ecb-des3_ede",
+        .cra_type = &crypto_ablkcipher_type,
+        .cra_u = {
+                .ablkcipher = {
+                        .min_keysize = DES3_EDE_KEY_SIZE,
+                        .max_keysize = DES3_EDE_KEY_SIZE,
+                        .ivsize = DES_BLOCK_SIZE,
+                        .setkey = cvm_ecb_des3_setkey,
+                        .encrypt = cvm_encrypt,
+                        .decrypt = cvm_decrypt,
                 },
         },
         .cra_init = cvm_enc_dec_init,
diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.h b/drivers/crypto/cavium/cpt/cptvf_algs.h
index a12050d11b0c..902f25751123 100644
--- a/drivers/crypto/cavium/cpt/cptvf_algs.h
+++ b/drivers/crypto/cavium/cpt/cptvf_algs.h
@@ -77,6 +77,11 @@ union encr_ctrl {
         } e;
 };
 
+struct cvm_cipher {
+        const char *name;
+        u8 value;
+};
+
 struct enc_context {
         union encr_ctrl enc_ctrl;
         u8 encr_key[32];
@@ -96,6 +101,8 @@ struct fc_context {
 struct cvm_enc_ctx {
         u32 key_len;
         u8 enc_key[MAX_KEY_SIZE];
+        u8 cipher_type:4;
+        u8 key_type:2;
 };
 
 struct cvm_des3_ctx {
diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c
index 6ffc740c7431..5c796ed55eba 100644
--- a/drivers/crypto/cavium/cpt/cptvf_main.c
+++ b/drivers/crypto/cavium/cpt/cptvf_main.c
@@ -525,7 +525,7 @@ static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq)
         intr = cptvf_read_vf_misc_intr_status(cptvf);
         /*Check for MISC interrupt types*/
         if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
-                dev_err(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
+                dev_dbg(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
                         intr, cptvf->vfid);
                 cptvf_handle_mbox_intr(cptvf);
                 cptvf_clear_mbox_intr(cptvf);
diff --git a/drivers/crypto/cavium/nitrox/Kconfig b/drivers/crypto/cavium/nitrox/Kconfig
new file mode 100644
index 000000000000..731e6a57218a
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/Kconfig
@@ -0,0 +1,21 @@
+#
+# Cavium NITROX Crypto Device configuration
+#
+config CRYPTO_DEV_NITROX
+        tristate
+        select CRYPTO_BLKCIPHER
+        select CRYPTO_AES
+        select CRYPTO_DES
+        select FW_LOADER
+
+config CRYPTO_DEV_NITROX_CNN55XX
+        tristate "Support for Cavium CNN55XX driver"
+        depends on PCI_MSI && 64BIT
+        select CRYPTO_DEV_NITROX
+        default m
+        help
+          Support for Cavium NITROX family CNN55XX driver
+          for accelerating crypto workloads.
+
+          To compile this as a module, choose M here: the module
+          will be called n5pf.
diff --git a/drivers/crypto/cavium/nitrox/Makefile b/drivers/crypto/cavium/nitrox/Makefile
new file mode 100644
index 000000000000..5af2e4368267
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/Makefile
@@ -0,0 +1,8 @@
+obj-$(CONFIG_CRYPTO_DEV_NITROX_CNN55XX) += n5pf.o
+
+n5pf-objs := nitrox_main.o \
+        nitrox_isr.o \
+        nitrox_lib.o \
+        nitrox_hal.o \
+        nitrox_reqmgr.o \
+        nitrox_algs.o
diff --git a/drivers/crypto/cavium/nitrox/nitrox_algs.c b/drivers/crypto/cavium/nitrox/nitrox_algs.c
new file mode 100644
index 000000000000..ce330278ef8a
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_algs.c
@@ -0,0 +1,457 @@
+#include <linux/crypto.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/printk.h>
+
+#include <crypto/aes.h>
+#include <crypto/skcipher.h>
+#include <crypto/ctr.h>
+#include <crypto/des.h>
+#include <crypto/xts.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_common.h"
+#include "nitrox_req.h"
+
+#define PRIO 4001
+
+struct nitrox_cipher {
+        const char *name;
+        enum flexi_cipher value;
+};
+
+/**
+ * supported cipher list
+ */
+static const struct nitrox_cipher flexi_cipher_table[] = {
+        { "null",               CIPHER_NULL },
+        { "cbc(des3_ede)",      CIPHER_3DES_CBC },
+        { "ecb(des3_ede)",      CIPHER_3DES_ECB },
+        { "cbc(aes)",           CIPHER_AES_CBC },
+        { "ecb(aes)",           CIPHER_AES_ECB },
+        { "cfb(aes)",           CIPHER_AES_CFB },
+        { "rfc3686(ctr(aes))",  CIPHER_AES_CTR },
+        { "xts(aes)",           CIPHER_AES_XTS },
+        { "cts(cbc(aes))",      CIPHER_AES_CBC_CTS },
+        { NULL,                 CIPHER_INVALID }
+};
+
+static enum flexi_cipher flexi_cipher_type(const char *name)
+{
+        const struct nitrox_cipher *cipher = flexi_cipher_table;
+
+        while (cipher->name) {
+                if (!strcmp(cipher->name, name))
+                        break;
+                cipher++;
+        }
+        return cipher->value;
+}
+
+static int flexi_aes_keylen(int keylen)
+{
+        int aes_keylen;
+
+        switch (keylen) {
+        case AES_KEYSIZE_128:
+                aes_keylen = 1;
+                break;
+        case AES_KEYSIZE_192:
+                aes_keylen = 2;
+                break;
+        case AES_KEYSIZE_256:
+                aes_keylen = 3;
+                break;
+        default:
+                aes_keylen = -EINVAL;
+                break;
+        }
+        return aes_keylen;
+}
+
+static int nitrox_skcipher_init(struct crypto_skcipher *tfm)
+{
+        struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
+        void *fctx;
+
+        /* get the first device */
+        nctx->ndev = nitrox_get_first_device();
+        if (!nctx->ndev)
+                return -ENODEV;
+
+        /* allocate nitrox crypto context */
+        fctx = crypto_alloc_context(nctx->ndev);
+        if (!fctx) {
+                nitrox_put_device(nctx->ndev);
+                return -ENOMEM;
+        }
+        nctx->u.ctx_handle = (uintptr_t)fctx;
+        crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) +
+                                    sizeof(struct nitrox_kcrypt_request));
+        return 0;
+}
+
+static void nitrox_skcipher_exit(struct crypto_skcipher *tfm)
+{
+        struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm);
+
+        /* free the nitrox crypto context */
+        if (nctx->u.ctx_handle) {
+                struct flexi_crypto_context *fctx = nctx->u.fctx;
+
+                memset(&fctx->crypto, 0, sizeof(struct crypto_keys));
+                memset(&fctx->auth, 0, sizeof(struct auth_keys));
+                crypto_free_context((void *)fctx);
+        }
+        nitrox_put_device(nctx->ndev);
+
+        nctx->u.ctx_handle = 0;
+        nctx->ndev = NULL;
+}
+
+static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher,
+                                         int aes_keylen, const u8 *key,
+                                         unsigned int keylen)
+{
+        struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
+        struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);
+        struct flexi_crypto_context *fctx;
+        enum flexi_cipher cipher_type;
+        const char *name;
+
+        name = crypto_tfm_alg_name(tfm);
+        cipher_type = flexi_cipher_type(name);
+        if (unlikely(cipher_type == CIPHER_INVALID)) {
+                pr_err("unsupported cipher: %s\n", name);
+                return -EINVAL;
+        }
+
+        /* fill crypto context */
+        fctx = nctx->u.fctx;
+        fctx->flags = 0;
+        fctx->w0.cipher_type = cipher_type;
+        fctx->w0.aes_keylen = aes_keylen;
+        fctx->w0.iv_source = IV_FROM_DPTR;
+        fctx->flags = cpu_to_be64(*(u64 *)&fctx->w0);
+        /* copy the key to context */
+        memcpy(fctx->crypto.u.key, key, keylen);
+
+        return 0;
+}
+
+static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key,
+                             unsigned int keylen)
+{
+        int aes_keylen;
+
+        aes_keylen = flexi_aes_keylen(keylen);
+        if (aes_keylen < 0) {
+                crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+        return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
+}
+
+static void nitrox_skcipher_callback(struct skcipher_request *skreq,
+                                     int err)
+{
+        if (err) {
+                pr_err_ratelimited("request failed status 0x%0x\n", err);
+                err = -EINVAL;
+        }
+        skcipher_request_complete(skreq, err);
+}
+
+static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc)
+{
+        struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq);
+        struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher);
+        struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+        int ivsize = crypto_skcipher_ivsize(cipher);
+        struct se_crypto_request *creq;
+
+        creq = &nkreq->creq;
+        creq->flags = skreq->base.flags;
+        creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
+                     GFP_KERNEL : GFP_ATOMIC;
+
+        /* fill the request */
+        creq->ctrl.value = 0;
+        creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
+        creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT);
+        /* param0: length of the data to be encrypted */
+        creq->gph.param0 = cpu_to_be16(skreq->cryptlen);
+        creq->gph.param1 = 0;
+        /* param2: encryption data offset */
+        creq->gph.param2 = cpu_to_be16(ivsize);
+        creq->gph.param3 = 0;
+
+        creq->ctx_handle = nctx->u.ctx_handle;
+        creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
+
+        /* copy the iv */
+        memcpy(creq->iv, skreq->iv, ivsize);
+        creq->ivsize = ivsize;
+        creq->src = skreq->src;
+        creq->dst = skreq->dst;
+
+        nkreq->nctx = nctx;
+        nkreq->skreq = skreq;
+
+        /* send the crypto request */
+        return nitrox_process_se_request(nctx->ndev, creq,
+                                         nitrox_skcipher_callback, skreq);
+}
+
+static int nitrox_aes_encrypt(struct skcipher_request *skreq)
+{
+        return nitrox_skcipher_crypt(skreq, true);
+}
+
+static int nitrox_aes_decrypt(struct skcipher_request *skreq)
+{
+        return nitrox_skcipher_crypt(skreq, false);
+}
+
+static int nitrox_3des_setkey(struct crypto_skcipher *cipher,
+                              const u8 *key, unsigned int keylen)
+{
+        if (keylen != DES3_EDE_KEY_SIZE) {
+                crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+
+        return nitrox_skcipher_setkey(cipher, 0, key, keylen);
+}
+
+static int nitrox_3des_encrypt(struct skcipher_request *skreq)
+{
+        return nitrox_skcipher_crypt(skreq, true);
+}
+
+static int nitrox_3des_decrypt(struct skcipher_request *skreq)
+{
+        return nitrox_skcipher_crypt(skreq, false);
+}
+
+static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher,
+                                 const u8 *key, unsigned int keylen)
+{
+        struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
+        struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);
+        struct flexi_crypto_context *fctx;
+        int aes_keylen, ret;
+
+        ret = xts_check_key(tfm, key, keylen);
+        if (ret)
+                return ret;
+
+        keylen /= 2;
+
+        aes_keylen = flexi_aes_keylen(keylen);
+        if (aes_keylen < 0) {
+                crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+
+        fctx = nctx->u.fctx;
+        /* copy KEY2 */
+        memcpy(fctx->auth.u.key2, (key + keylen), keylen);
+
+        return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
+}
+
+static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher,
+                                         const u8 *key, unsigned int keylen)
+{
+        struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher);
+        struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm);
+        struct flexi_crypto_context *fctx;
+        int aes_keylen;
+
+        if (keylen < CTR_RFC3686_NONCE_SIZE)
+                return -EINVAL;
+
+        fctx = nctx->u.fctx;
+
+        memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE),
+               CTR_RFC3686_NONCE_SIZE);
+
+        keylen -= CTR_RFC3686_NONCE_SIZE;
+
+        aes_keylen = flexi_aes_keylen(keylen);
+        if (aes_keylen < 0) {
+                crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+        return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
+}
+
+static struct skcipher_alg nitrox_skciphers[] = { {
+        .base = {
+                .cra_name = "cbc(aes)",
+                .cra_driver_name = "n5_cbc(aes)",
+                .cra_priority = PRIO,
+                .cra_flags = CRYPTO_ALG_ASYNC,
+                .cra_blocksize = AES_BLOCK_SIZE,
+                .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+                .cra_alignmask = 0,
+                .cra_module = THIS_MODULE,
+        },
+        .min_keysize = AES_MIN_KEY_SIZE,
+        .max_keysize = AES_MAX_KEY_SIZE,
+        .ivsize = AES_BLOCK_SIZE,
+        .setkey = nitrox_aes_setkey,
+        .encrypt = nitrox_aes_encrypt,
+        .decrypt = nitrox_aes_decrypt,
+        .init = nitrox_skcipher_init,
+        .exit = nitrox_skcipher_exit,
+}, {
+        .base = {
+                .cra_name = "ecb(aes)",
+                .cra_driver_name = "n5_ecb(aes)",
+                .cra_priority = PRIO,
+                .cra_flags = CRYPTO_ALG_ASYNC,
+                .cra_blocksize = AES_BLOCK_SIZE,
+                .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+                .cra_alignmask = 0,
+                .cra_module = THIS_MODULE,
+        },
+        .min_keysize = AES_MIN_KEY_SIZE,
+        .max_keysize = AES_MAX_KEY_SIZE,
+        .ivsize = AES_BLOCK_SIZE,
+        .setkey = nitrox_aes_setkey,
+        .encrypt = nitrox_aes_encrypt,
+        .decrypt = nitrox_aes_decrypt,
+        .init = nitrox_skcipher_init,
+        .exit = nitrox_skcipher_exit,
+}, {
+        .base = {
+                .cra_name = "cfb(aes)",
+                .cra_driver_name = "n5_cfb(aes)",
+                .cra_priority = PRIO,
+                .cra_flags = CRYPTO_ALG_ASYNC,
+                .cra_blocksize = AES_BLOCK_SIZE,
+                .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+                .cra_alignmask = 0,
+                .cra_module = THIS_MODULE,
+        },
+        .min_keysize = AES_MIN_KEY_SIZE,
+        .max_keysize = AES_MAX_KEY_SIZE,
+        .ivsize = AES_BLOCK_SIZE,
+        .setkey = nitrox_aes_setkey,
+        .encrypt = nitrox_aes_encrypt,
+        .decrypt = nitrox_aes_decrypt,
+        .init = nitrox_skcipher_init,
+        .exit = nitrox_skcipher_exit,
+}, {
+        .base = {
+                .cra_name = "xts(aes)",
+                .cra_driver_name = "n5_xts(aes)",
+                .cra_priority = PRIO,
+                .cra_flags = CRYPTO_ALG_ASYNC,
+                .cra_blocksize = AES_BLOCK_SIZE,
+                .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+                .cra_alignmask = 0,
+                .cra_module = THIS_MODULE,
+        },
+        .min_keysize = 2 * AES_MIN_KEY_SIZE,
+        .max_keysize = 2 * AES_MAX_KEY_SIZE,
+        .ivsize = AES_BLOCK_SIZE,
+        .setkey = nitrox_aes_xts_setkey,
+        .encrypt = nitrox_aes_encrypt,
+        .decrypt = nitrox_aes_decrypt,
+        .init = nitrox_skcipher_init,
+        .exit = nitrox_skcipher_exit,
+}, {
+        .base = {
+                .cra_name = "rfc3686(ctr(aes))",
+                .cra_driver_name = "n5_rfc3686(ctr(aes))",
+                .cra_priority = PRIO,
+                .cra_flags = CRYPTO_ALG_ASYNC,
+                .cra_blocksize = 1,
+                .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+                .cra_alignmask = 0,
+                .cra_module = THIS_MODULE,
+        },
+        .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+        .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+        .ivsize = CTR_RFC3686_IV_SIZE,
+        .init = nitrox_skcipher_init,
+        .exit = nitrox_skcipher_exit,
+        .setkey = nitrox_aes_ctr_rfc3686_setkey,
+        .encrypt = nitrox_aes_encrypt,
+        .decrypt = nitrox_aes_decrypt,
+}, {
+        .base = {
+                .cra_name = "cts(cbc(aes))",
+                .cra_driver_name = "n5_cts(cbc(aes))",
+                .cra_priority = PRIO,
+                .cra_flags = CRYPTO_ALG_ASYNC,
+                .cra_blocksize = AES_BLOCK_SIZE,
+                .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+                .cra_alignmask = 0,
+                .cra_type = &crypto_ablkcipher_type,
+                .cra_module = THIS_MODULE,
+        },
+        .min_keysize = AES_MIN_KEY_SIZE,
+        .max_keysize = AES_MAX_KEY_SIZE,
+        .ivsize = AES_BLOCK_SIZE,
+        .setkey = nitrox_aes_setkey,
+        .encrypt = nitrox_aes_encrypt,
+        .decrypt = nitrox_aes_decrypt,
+        .init = nitrox_skcipher_init,
+        .exit = nitrox_skcipher_exit,
+}, {
+        .base = {
+                .cra_name = "cbc(des3_ede)",
+                .cra_driver_name = "n5_cbc(des3_ede)",
+                .cra_priority = PRIO,
+                .cra_flags = CRYPTO_ALG_ASYNC,
+                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+                .cra_alignmask = 0,
+                .cra_module = THIS_MODULE,
+        },
+        .min_keysize = DES3_EDE_KEY_SIZE,
+        .max_keysize = DES3_EDE_KEY_SIZE,
+        .ivsize = DES3_EDE_BLOCK_SIZE,
+        .setkey = nitrox_3des_setkey,
+        .encrypt = nitrox_3des_encrypt,
+        .decrypt = nitrox_3des_decrypt,
+        .init = nitrox_skcipher_init,
+        .exit = nitrox_skcipher_exit,
+}, {
+        .base = {
+                .cra_name = "ecb(des3_ede)",
+                .cra_driver_name = "n5_ecb(des3_ede)",
+                .cra_priority = PRIO,
+                .cra_flags = CRYPTO_ALG_ASYNC,
+                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                .cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
+                .cra_alignmask = 0,
+                .cra_module = THIS_MODULE,
+        },
+        .min_keysize = DES3_EDE_KEY_SIZE,
+        .max_keysize = DES3_EDE_KEY_SIZE,
+        .ivsize = DES3_EDE_BLOCK_SIZE,
+        .setkey = nitrox_3des_setkey,
+        .encrypt = nitrox_3des_encrypt,
+        .decrypt = nitrox_3des_decrypt,
+        .init = nitrox_skcipher_init,
+        .exit = nitrox_skcipher_exit,
+}
+
+};
+
+int nitrox_crypto_register(void)
+{
+        return crypto_register_skciphers(nitrox_skciphers,
+                                         ARRAY_SIZE(nitrox_skciphers));
+}
+
+void nitrox_crypto_unregister(void)
+{
+        crypto_unregister_skciphers(nitrox_skciphers,
+                                    ARRAY_SIZE(nitrox_skciphers));
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_common.h b/drivers/crypto/cavium/nitrox/nitrox_common.h
new file mode 100644
index 000000000000..4888c7823a5f
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_common.h
@@ -0,0 +1,42 @@
+#ifndef __NITROX_COMMON_H
+#define __NITROX_COMMON_H
+
+#include "nitrox_dev.h"
+#include "nitrox_req.h"
+
+int nitrox_crypto_register(void);
+void nitrox_crypto_unregister(void);
+void *crypto_alloc_context(struct nitrox_device *ndev);
+void crypto_free_context(void *ctx);
+struct nitrox_device *nitrox_get_first_device(void);
+void nitrox_put_device(struct nitrox_device *ndev);
+
+void nitrox_pf_cleanup_isr(struct nitrox_device *ndev);
+int nitrox_pf_init_isr(struct nitrox_device *ndev);
+
+int nitrox_common_sw_init(struct nitrox_device *ndev);
+void nitrox_common_sw_cleanup(struct nitrox_device *ndev);
+
+void pkt_slc_resp_handler(unsigned long data);
+int nitrox_process_se_request(struct nitrox_device *ndev,
+                              struct se_crypto_request *req,
+                              completion_t cb,
+                              struct skcipher_request *skreq);
+void backlog_qflush_work(struct work_struct *work);
+
+void nitrox_config_emu_unit(struct nitrox_device *ndev);
+void nitrox_config_pkt_input_rings(struct nitrox_device *ndev);
+void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev);
+void nitrox_config_vfmode(struct nitrox_device *ndev, int mode);
+void nitrox_config_nps_unit(struct nitrox_device *ndev);
+void nitrox_config_pom_unit(struct nitrox_device *ndev);
+void nitrox_config_rand_unit(struct nitrox_device *ndev);
+void nitrox_config_efl_unit(struct nitrox_device *ndev);
+void nitrox_config_bmi_unit(struct nitrox_device *ndev);
+void nitrox_config_bmo_unit(struct nitrox_device *ndev);
+void nitrox_config_lbc_unit(struct nitrox_device *ndev);
+void invalidate_lbc(struct nitrox_device *ndev);
+void enable_pkt_input_ring(struct nitrox_device *ndev, int ring);
+void enable_pkt_solicit_port(struct nitrox_device *ndev, int port);
+
+#endif /* __NITROX_COMMON_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h
new file mode 100644
index 000000000000..30b04c4c6076
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h
@@ -0,0 +1,1084 @@
+#ifndef __NITROX_CSR_H
+#define __NITROX_CSR_H
+
+#include <asm/byteorder.h>
+#include <linux/types.h>
+
+/* EMU clusters */
+#define NR_CLUSTERS             4
+#define AE_CORES_PER_CLUSTER    20
+#define SE_CORES_PER_CLUSTER    16
+
+/* BIST registers */
+#define EMU_BIST_STATUSX(_i)    (0x1402700 + ((_i) * 0x40000))
+#define UCD_BIST_STATUS         0x12C0070
+#define NPS_CORE_BIST_REG       0x10000E8
+#define NPS_CORE_NPC_BIST_REG   0x1000128
+#define NPS_PKT_SLC_BIST_REG    0x1040088
+#define NPS_PKT_IN_BIST_REG     0x1040100
+#define POM_BIST_REG            0x11C0100
+#define BMI_BIST_REG            0x1140080
+#define EFL_CORE_BIST_REGX(_i)  (0x1240100 + ((_i) * 0x400))
+#define EFL_TOP_BIST_STAT       0x1241090
+#define BMO_BIST_REG            0x1180080
+#define LBC_BIST_STATUS         0x1200020
+#define PEM_BIST_STATUSX(_i)    (0x1080468 | ((_i) << 18))
+
+/* EMU registers */
+#define EMU_SE_ENABLEX(_i)      (0x1400000 + ((_i) * 0x40000))
+#define EMU_AE_ENABLEX(_i)      (0x1400008 + ((_i) * 0x40000))
+#define EMU_WD_INT_ENA_W1SX(_i) (0x1402318 + ((_i) * 0x40000))
+#define EMU_GE_INT_ENA_W1SX(_i) (0x1402518 + ((_i) * 0x40000))
+#define EMU_FUSE_MAPX(_i)       (0x1402708 + ((_i) * 0x40000))
+
+/* UCD registers */
+#define UCD_UCODE_LOAD_BLOCK_NUM        0x12C0010
+#define UCD_UCODE_LOAD_IDX_DATAX(_i)    (0x12C0018 + ((_i) * 0x20))
+#define UCD_SE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0000 + ((_i) * 0x1000))
+
+/* NPS core registers */
+#define NPS_CORE_GBL_VFCFG      0x1000000
+#define NPS_CORE_CONTROL        0x1000008
+#define NPS_CORE_INT_ACTIVE     0x1000080
+#define NPS_CORE_INT            0x10000A0
+#define NPS_CORE_INT_ENA_W1S    0x10000B8
+#define NPS_STATS_PKT_DMA_RD_CNT        0x1000180
+#define NPS_STATS_PKT_DMA_WR_CNT        0x1000190
+
+/* NPS packet registers */
+#define NPS_PKT_INT                             0x1040018
+#define NPS_PKT_IN_RERR_HI              0x1040108
+#define NPS_PKT_IN_RERR_HI_ENA_W1S      0x1040120
+#define NPS_PKT_IN_RERR_LO              0x1040128
+#define NPS_PKT_IN_RERR_LO_ENA_W1S      0x1040140
+#define NPS_PKT_IN_ERR_TYPE             0x1040148
+#define NPS_PKT_IN_ERR_TYPE_ENA_W1S     0x1040160
+#define NPS_PKT_IN_INSTR_CTLX(_i)       (0x10060 + ((_i) * 0x40000))
+#define NPS_PKT_IN_INSTR_BADDRX(_i)     (0x10068 + ((_i) * 0x40000))
+#define NPS_PKT_IN_INSTR_RSIZEX(_i)     (0x10070 + ((_i) * 0x40000))
+#define NPS_PKT_IN_DONE_CNTSX(_i)       (0x10080 + ((_i) * 0x40000))
+#define NPS_PKT_IN_INSTR_BAOFF_DBELLX(_i)       (0x10078 + ((_i) * 0x40000))
+#define NPS_PKT_IN_INT_LEVELSX(_i)              (0x10088 + ((_i) * 0x40000))
+
+#define NPS_PKT_SLC_RERR_HI             0x1040208
+#define NPS_PKT_SLC_RERR_HI_ENA_W1S     0x1040220
+#define NPS_PKT_SLC_RERR_LO             0x1040228
+#define NPS_PKT_SLC_RERR_LO_ENA_W1S     0x1040240
+#define NPS_PKT_SLC_ERR_TYPE            0x1040248
+#define NPS_PKT_SLC_ERR_TYPE_ENA_W1S    0x1040260
+#define NPS_PKT_SLC_CTLX(_i)            (0x10000 + ((_i) * 0x40000))
+#define NPS_PKT_SLC_CNTSX(_i)           (0x10008 + ((_i) * 0x40000))
+#define NPS_PKT_SLC_INT_LEVELSX(_i)     (0x10010 + ((_i) * 0x40000))
+
+/* POM registers */
+#define POM_INT_ENA_W1S         0x11C0018
+#define POM_GRP_EXECMASKX(_i)   (0x11C1100 | ((_i) * 8))
+#define POM_INT         0x11C0000
+#define POM_PERF_CTL    0x11CC400
+
+/* BMI registers */
+#define BMI_INT         0x1140000
+#define BMI_CTL         0x1140020
+#define BMI_INT_ENA_W1S 0x1140018
+#define BMI_NPS_PKT_CNT 0x1140070
+
+/* EFL registers */
+#define EFL_CORE_INT_ENA_W1SX(_i)               (0x1240018 + ((_i) * 0x400))
+#define EFL_CORE_VF_ERR_INT0X(_i)               (0x1240050 + ((_i) * 0x400))
+#define EFL_CORE_VF_ERR_INT0_ENA_W1SX(_i)       (0x1240068 + ((_i) * 0x400))
+#define EFL_CORE_VF_ERR_INT1X(_i)               (0x1240070 + ((_i) * 0x400))
+#define EFL_CORE_VF_ERR_INT1_ENA_W1SX(_i)       (0x1240088 + ((_i) * 0x400))
+#define EFL_CORE_SE_ERR_INTX(_i)                (0x12400A0 + ((_i) * 0x400))
+#define EFL_RNM_CTL_STATUS                      0x1241800
+#define EFL_CORE_INTX(_i)                       (0x1240000 + ((_i) * 0x400))
+
+/* BMO registers */
+#define BMO_CTL2                0x1180028
+#define BMO_NPS_SLC_PKT_CNT     0x1180078
+
+/* LBC registers */
+#define LBC_INT                 0x1200000
+#define LBC_INVAL_CTL           0x1201010
+#define LBC_PLM_VF1_64_INT      0x1202008
+#define LBC_INVAL_STATUS        0x1202010
+#define LBC_INT_ENA_W1S         0x1203000
+#define LBC_PLM_VF1_64_INT_ENA_W1S      0x1205008
+#define LBC_PLM_VF65_128_INT            0x1206008
+#define LBC_ELM_VF1_64_INT              0x1208000
+#define LBC_PLM_VF65_128_INT_ENA_W1S    0x1209008
+#define LBC_ELM_VF1_64_INT_ENA_W1S      0x120B000
+#define LBC_ELM_VF65_128_INT            0x120C000
+#define LBC_ELM_VF65_128_INT_ENA_W1S    0x120F000
+
+/* PEM registers */
+#define PEM0_INT 0x1080428
+
+/**
+ * struct emu_fuse_map - EMU Fuse Map Registers
+ * @ae_fuse: Fuse settings for AE 19..0
+ * @se_fuse: Fuse settings for SE 15..0
+ *
+ * A set bit indicates the unit is fuse disabled.
+ */
+union emu_fuse_map {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 valid : 1;
+                u64 raz_52_62 : 11;
+                u64 ae_fuse : 20;
+                u64 raz_16_31 : 16;
+                u64 se_fuse : 16;
+#else
+                u64 se_fuse : 16;
+                u64 raz_16_31 : 16;
+                u64 ae_fuse : 20;
+                u64 raz_52_62 : 11;
+                u64 valid : 1;
+#endif
+        } s;
+};
+
+/**
+ * struct emu_se_enable - Symmetric Engine Enable Registers
+ * @enable: Individual enables for each of the clusters
+ *   16 symmetric engines.
+ */
+union emu_se_enable {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz : 48;
+                u64 enable : 16;
+#else
+                u64 enable : 16;
+                u64 raz : 48;
+#endif
+        } s;
+};
+
+/**
+ * struct emu_ae_enable - EMU Asymmetric engines.
+ * @enable: Individual enables for each of the cluster's
+ *   20 Asymmetric Engines.
+ */
+union emu_ae_enable {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz : 44;
+                u64 enable : 20;
+#else
+                u64 enable : 20;
+                u64 raz : 44;
+#endif
+        } s;
+};
+
+/**
+ * struct emu_wd_int_ena_w1s - EMU Interrupt Enable Registers
+ * @ae_wd: Reads or sets enable for EMU(0..3)_WD_INT[AE_WD]
+ * @se_wd: Reads or sets enable for EMU(0..3)_WD_INT[SE_WD]
+ */
+union emu_wd_int_ena_w1s {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz2 : 12;
+                u64 ae_wd : 20;
+                u64 raz1 : 16;
+                u64 se_wd : 16;
+#else
+                u64 se_wd : 16;
+                u64 raz1 : 16;
+                u64 ae_wd : 20;
+                u64 raz2 : 12;
+#endif
+        } s;
+};
+
+/**
+ * struct emu_ge_int_ena_w1s - EMU Interrupt Enable set registers
+ * @ae_ge: Reads or sets enable for EMU(0..3)_GE_INT[AE_GE]
+ * @se_ge: Reads or sets enable for EMU(0..3)_GE_INT[SE_GE]
+ */
+union emu_ge_int_ena_w1s {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz_52_63 : 12;
+                u64 ae_ge : 20;
+                u64 raz_16_31: 16;
+                u64 se_ge : 16;
+#else
+                u64 se_ge : 16;
+                u64 raz_16_31: 16;
+                u64 ae_ge : 20;
+                u64 raz_52_63 : 12;
+#endif
+        } s;
+};
+
+/**
+ * struct nps_pkt_slc_ctl - Solicited Packet Out Control Registers
+ * @rh: Indicates whether to remove or include the response header
+ *   1 = Include, 0 = Remove
+ * @z: If set, 8 trailing 0x00 bytes will be added to the end of the
+ *   outgoing packet.
+ * @enb: Enable for this port.
+ */
+union nps_pkt_slc_ctl {
+        u64 value;
+        struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+                u64 raz : 61;
+                u64 rh : 1;
+                u64 z : 1;
+                u64 enb : 1;
+#else
+                u64 enb : 1;
+                u64 z : 1;
+                u64 rh : 1;
+                u64 raz : 61;
+#endif
+        } s;
+};
+
+/**
+ * struct nps_pkt_slc_cnts - Solicited Packet Out Count Registers
+ * @slc_int: Returns a 1 when:
+ *   NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or
+ *   NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET].
+ *   To clear the bit, the CNTS register must be written to clear.
+ * @in_int: Returns a 1 when:
+ *   NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT].
+ *   To clear the bit, the DONE_CNTS register must be written to clear.
+ * @mbox_int: Returns a 1 when:
+ *   NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. To clear the bit,
+ *   write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] with 1.
+ * @timer: Timer, incremented every 2048 coprocessor clock cycles
+ *   when [CNT] is not zero. The hardware clears both [TIMER] and
+ *   [INT] when [CNT] goes to 0.
+ * @cnt: Packet counter. Hardware adds to [CNT] as it sends packets out.
+ *   On a write to this CSR, hardware subtracts the amount written to the
+ *   [CNT] field from [CNT].
+ */
+union nps_pkt_slc_cnts {
+        u64 value;
+        struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+                u64 slc_int : 1;
+                u64 uns_int : 1;
+                u64 in_int : 1;
+                u64 mbox_int : 1;
+                u64 resend : 1;
+                u64 raz : 5;
+                u64 timer : 22;
+                u64 cnt : 32;
+#else
+                u64 cnt : 32;
+                u64 timer : 22;
+                u64 raz : 5;
+                u64 resend : 1;
+                u64 mbox_int : 1;
+                u64 in_int : 1;
+                u64 uns_int : 1;
+                u64 slc_int : 1;
+#endif
+        } s;
+};
+
+/**
+ * struct nps_pkt_slc_int_levels - Solicited Packet Out Interrupt Levels
+ *   Registers.
+ * @bmode: Determines whether NPS_PKT_SLC_CNTS[CNT] is a byte or
+ *   packet counter.
+ * @timet: Output port counter time interrupt threshold.
+ * @cnt: Output port counter interrupt threshold.
+ */
+union nps_pkt_slc_int_levels {
+        u64 value;
+        struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+                u64 bmode : 1;
+                u64 raz : 9;
+                u64 timet : 22;
+                u64 cnt : 32;
+#else
+                u64 cnt : 32;
+                u64 timet : 22;
+                u64 raz : 9;
+                u64 bmode : 1;
+#endif
+        } s;
+};
+
+/**
+ * struct nps_pkt_inst - NPS Packet Interrupt Register
+ * @in_err: Set when any NPS_PKT_IN_RERR_HI/LO bit and
+ *    corresponding NPS_PKT_IN_RERR_*_ENA_* bit are bot set.
+ * @uns_err: Set when any NSP_PKT_UNS_RERR_HI/LO bit and
+ *    corresponding NPS_PKT_UNS_RERR_*_ENA_* bit are both set.
+ * @slc_er: Set when any NSP_PKT_SLC_RERR_HI/LO bit and
+ *    corresponding NPS_PKT_SLC_RERR_*_ENA_* bit are both set.
+ */
+union nps_pkt_int {
+        u64 value;
+        struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+                u64 raz : 54;
+                u64 uns_wto : 1;
+                u64 in_err : 1;
+                u64 uns_err : 1;
+                u64 slc_err : 1;
+                u64 in_dbe : 1;
+                u64 in_sbe : 1;
+                u64 uns_dbe : 1;
+                u64 uns_sbe : 1;
+                u64 slc_dbe : 1;
+                u64 slc_sbe : 1;
+#else
+                u64 slc_sbe : 1;
+                u64 slc_dbe : 1;
+                u64 uns_sbe : 1;
+                u64 uns_dbe : 1;
+                u64 in_sbe : 1;
+                u64 in_dbe : 1;
+                u64 slc_err : 1;
+                u64 uns_err : 1;
+                u64 in_err : 1;
+                u64 uns_wto : 1;
+                u64 raz : 54;
+#endif
+        } s;
+};
+
+/**
+ * struct nps_pkt_in_done_cnts - Input instruction ring counts registers
+ * @slc_cnt: Returns a 1 when:
+ *    NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or
+ *    NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SCL(i)_INT_LEVELS[TIMET]
+ *    To clear the bit, the CNTS register must be
+ *    written to clear the underlying condition
+ * @uns_int: Return a 1 when:
+ *    NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT], or
+ *    NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET]
+ *    To clear the bit, the CNTS register must be
+ *    written to clear the underlying condition
+ * @in_int: Returns a 1 when:
+ *    NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT]
+ *    To clear the bit, the DONE_CNTS register
+ *    must be written to clear the underlying condition
+ * @mbox_int: Returns a 1 when:
+ *    NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set.
+ *    To clear the bit, write NPS_PKT_MBOX_PF_VF(i)_INT[INTR]
+ *    with 1.
+ * @resend: A write of 1 will resend an MSI-X interrupt message if any
+ *    of the following conditions are true for this ring "i".
+ *    NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT]
+ *    NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET]
+ *    NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT]
+ *    NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET]
+ *    NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT]
+ *    NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set
+ * @cnt: Packet counter. Hardware adds to [CNT] as it reads
+ *    packets. On a write to this CSR, hardware substracts the
+ *    amount written to the [CNT] field from [CNT], which will
+ *    clear PKT_IN(i)_INT_STATUS[INTR] if [CNT] becomes <=
+ *    NPS_PKT_IN(i)_INT_LEVELS[CNT]. This register should be
+ *    cleared before enabling a ring by reading the current
+ *    value and writing it back.
+ */
+union nps_pkt_in_done_cnts {
+        u64 value;
+        struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+                u64 slc_int : 1;
+                u64 uns_int : 1;
+                u64 in_int : 1;
+                u64 mbox_int : 1;
+                u64 resend : 1;
+                u64 raz : 27;
+                u64 cnt : 32;
+#else
+                u64 cnt : 32;
+                u64 raz : 27;
+                u64 resend : 1;
+                u64 mbox_int : 1;
+                u64 in_int : 1;
+                u64 uns_int : 1;
+                u64 slc_int : 1;
+#endif
+        } s;
+};
+
+/**
+ * struct nps_pkt_in_instr_ctl - Input Instruction Ring Control Registers.
+ * @is64b: If 1, the ring uses 64-byte instructions. If 0, the
+ *   ring uses 32-byte instructions.
+ * @enb: Enable for the input ring.
+ */
+union nps_pkt_in_instr_ctl {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz : 62;
+                u64 is64b : 1;
+                u64 enb : 1;
+#else
+                u64 enb : 1;
+                u64 is64b : 1;
+                u64 raz : 62;
+#endif
+        } s;
+};
+
+/**
+ * struct nps_pkt_in_instr_rsize - Input instruction ring size registers
+ * @rsize: Ring size (number of instructions)
+ */
+union nps_pkt_in_instr_rsize {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz : 32;
+                u64 rsize : 32;
+#else
+                u64 rsize : 32;
+                u64 raz : 32;
+#endif
+        } s;
+};
+
+/**
+ * struct nps_pkt_in_instr_baoff_dbell - Input instruction ring
+ *   base address offset and doorbell registers
+ * @aoff: Address offset. The offset from the NPS_PKT_IN_INSTR_BADDR
+ *   where the next pointer is read.
+ * @dbell: Pointer list doorbell count. Write operations to this field
+ *   increments the present value here. Read operations return the
+ *   present value.
+ */
+union nps_pkt_in_instr_baoff_dbell {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 aoff : 32;
+                u64 dbell : 32;
+#else
+                u64 dbell : 32;
+                u64 aoff : 32;
+#endif
+        } s;
+};
+
+/**
+ * struct nps_core_int_ena_w1s - NPS core interrupt enable set register
+ * @host_nps_wr_err: Reads or sets enable for
+ *   NPS_CORE_INT[HOST_NPS_WR_ERR].
+ * @npco_dma_malform: Reads or sets enable for
+ *   NPS_CORE_INT[NPCO_DMA_MALFORM].
+ * @exec_wr_timeout: Reads or sets enable for
+ *   NPS_CORE_INT[EXEC_WR_TIMEOUT].
+ * @host_wr_timeout: Reads or sets enable for
+ *   NPS_CORE_INT[HOST_WR_TIMEOUT].
+ * @host_wr_err: Reads or sets enable for
+ *   NPS_CORE_INT[HOST_WR_ERR]
+ */
+union nps_core_int_ena_w1s {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz4 : 55;
+                u64 host_nps_wr_err : 1;
+                u64 npco_dma_malform : 1;
+                u64 exec_wr_timeout : 1;
+                u64 host_wr_timeout : 1;
+                u64 host_wr_err : 1;
+                u64 raz3 : 1;
+                u64 raz2 : 1;
+                u64 raz1 : 1;
+                u64 raz0 : 1;
+#else
+                u64 raz0 : 1;
+                u64 raz1 : 1;
+                u64 raz2 : 1;
+                u64 raz3 : 1;
+                u64 host_wr_err : 1;
+                u64 host_wr_timeout : 1;
+                u64 exec_wr_timeout : 1;
+                u64 npco_dma_malform : 1;
+                u64 host_nps_wr_err : 1;
+                u64 raz4 : 55;
+#endif
+        } s;
+};
+
+/**
+ * struct nps_core_gbl_vfcfg - Global VF Configuration Register.
+ * @ilk_disable: When set, this bit indicates that the ILK interface has
+ *    been disabled.
+ * @obaf: BMO allocation control
+ *    0 = allocate per queue
+ *    1 = allocate per VF
+ * @ibaf: BMI allocation control
+ *    0 = allocate per queue
+ *    1 = allocate per VF
+ * @zaf: ZIP allocation control
+ *    0 = allocate per queue
+ *    1 = allocate per VF
+ * @aeaf: AE allocation control
+ *    0 = allocate per queue
+ *    1 = allocate per VF
+ * @seaf: SE allocation control
+ *    0 = allocation per queue
+ *    1 = allocate per VF
+ * @cfg: VF/PF mode.
+ */
+union nps_core_gbl_vfcfg {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64  raz :55;
+                u64  ilk_disable :1;
+                u64  obaf :1;
+                u64  ibaf :1;
+                u64  zaf :1;
+                u64  aeaf :1;
+                u64  seaf :1;
+                u64  cfg :3;
+#else
+                u64  cfg :3;
+                u64  seaf :1;
+                u64  aeaf :1;
+                u64  zaf :1;
+                u64  ibaf :1;
+                u64  obaf :1;
+                u64  ilk_disable :1;
+                u64  raz :55;
+#endif
+        } s;
+};
+
+/**
+ * struct nps_core_int_active - NPS Core Interrupt Active Register
+ * @resend: Resend MSI-X interrupt if needs to handle interrupts
+ *    Sofware can set this bit and then exit the ISR.
+ * @ocla: Set when any OCLA(0)_INT and corresponding OCLA(0_INT_ENA_W1C
+ *    bit are set
+ * @mbox: Set when any NPS_PKT_MBOX_INT_LO/HI and corresponding
+ *    NPS_PKT_MBOX_INT_LO_ENA_W1C/HI_ENA_W1C bits are set
+ * @emu: bit i is set in [EMU] when any EMU(i)_INT bit is set
+ * @bmo: Set when any BMO_INT bit is set
+ * @bmi: Set when any BMI_INT bit is set or when any non-RO
+ *    BMI_INT and corresponding BMI_INT_ENA_W1C bits are both set
+ * @aqm: Set when any AQM_INT bit is set
+ * @zqm: Set when any ZQM_INT bit is set
+ * @efl: Set when any EFL_INT RO bit is set or when any non-RO EFL_INT
+ *    and corresponding EFL_INT_ENA_W1C bits are both set
+ * @ilk: Set when any ILK_INT bit is set
+ * @lbc: Set when any LBC_INT RO bit is set or when any non-RO LBC_INT
+ *    and corresponding LBC_INT_ENA_W1C bits are bot set
+ * @pem: Set when any PEM(0)_INT RO bit is set or when any non-RO
+ *    PEM(0)_INT and corresponding PEM(0)_INT_ENA_W1C bit are both set
+ * @ucd: Set when any UCD_INT bit is set
+ * @zctl: Set when any ZIP_INT RO bit is set or when any non-RO ZIP_INT
+ *    and corresponding ZIP_INT_ENA_W1C bits are both set
+ * @lbm: Set when any LBM_INT bit is set
+ * @nps_pkt: Set when any NPS_PKT_INT bit is set
+ * @nps_core: Set when any NPS_CORE_INT RO bit is set or when non-RO
+ *    NPS_CORE_INT and corresponding NSP_CORE_INT_ENA_W1C bits are both set
+ */
+union nps_core_int_active {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 resend : 1;
+                u64 raz : 43;
+                u64 ocla : 1;
+                u64 mbox : 1;
+                u64 emu : 4;
+                u64 bmo : 1;
+                u64 bmi : 1;
+                u64 aqm : 1;
+                u64 zqm : 1;
+                u64 efl : 1;
+                u64 ilk : 1;
+                u64 lbc : 1;
+                u64 pem : 1;
+                u64 pom : 1;
+                u64 ucd : 1;
+                u64 zctl : 1;
+                u64 lbm : 1;
+                u64 nps_pkt : 1;
+                u64 nps_core : 1;
+#else
+                u64 nps_core : 1;
+                u64 nps_pkt : 1;
+                u64 lbm : 1;
+                u64 zctl: 1;
+                u64 ucd : 1;
+                u64 pom : 1;
+                u64 pem : 1;
+                u64 lbc : 1;
+                u64 ilk : 1;
+                u64 efl : 1;
+                u64 zqm : 1;
+                u64 aqm : 1;
+                u64 bmi : 1;
+                u64 bmo : 1;
+                u64 emu : 4;
+                u64 mbox : 1;
+                u64 ocla : 1;
+                u64 raz : 43;
+                u64 resend : 1;
+#endif
+        } s;
+};
+
+/**
+ * struct efl_core_int - EFL Interrupt Registers
+ * @epci_decode_err: EPCI decoded a transacation that was unknown
+ *    This error should only occurred when there is a micrcode/SE error
+ *    and should be considered fatal
+ * @ae_err: An AE uncorrectable error occurred.
+ *    See EFL_CORE(0..3)_AE_ERR_INT
+ * @se_err: An SE uncorrectable error occurred.
+ *    See EFL_CORE(0..3)_SE_ERR_INT
+ * @dbe: Double-bit error occurred in EFL
+ * @sbe: Single-bit error occurred in EFL
+ * @d_left: Asserted when new POM-Header-BMI-data is
+ *    being sent to an Exec, and that Exec has Not read all BMI
+ *    data associated with the previous POM header
+ * @len_ovr: Asserted when an Exec-Read is issued that is more than
+ *    14 greater in length that the BMI data left to be read
+ */
+union efl_core_int {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz : 57;
+                u64 epci_decode_err : 1;
+                u64 ae_err : 1;
+                u64 se_err : 1;
+                u64 dbe : 1;
+                u64 sbe : 1;
+                u64 d_left : 1;
+                u64 len_ovr : 1;
+#else
+                u64 len_ovr : 1;
+                u64 d_left : 1;
+                u64 sbe : 1;
+                u64 dbe : 1;
+                u64 se_err : 1;
+                u64 ae_err : 1;
+                u64 epci_decode_err  : 1;
+                u64 raz : 57;
+#endif
+        } s;
+};
+
+/**
+ * struct efl_core_int_ena_w1s - EFL core interrupt enable set register
+ * @epci_decode_err: Reads or sets enable for
+ *   EFL_CORE(0..3)_INT[EPCI_DECODE_ERR].
+ * @d_left: Reads or sets enable for
+ *   EFL_CORE(0..3)_INT[D_LEFT].
+ * @len_ovr: Reads or sets enable for
+ *   EFL_CORE(0..3)_INT[LEN_OVR].
+ */
+union efl_core_int_ena_w1s {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz_7_63 : 57;
+                u64 epci_decode_err : 1;
+                u64 raz_2_5 : 4;
+                u64 d_left : 1;
+                u64 len_ovr : 1;
+#else
+                u64 len_ovr : 1;
+                u64 d_left : 1;
+                u64 raz_2_5 : 4;
+                u64 epci_decode_err : 1;
+                u64 raz_7_63 : 57;
+#endif
+        } s;
+};
+
+/**
+ * struct efl_rnm_ctl_status - RNM Control and Status Register
+ * @ent_sel: Select input to RNM FIFO
+ * @exp_ent: Exported entropy enable for random number generator
+ * @rng_rst: Reset to RNG. Setting this bit to 1 cancels the generation
+ *    of the current random number.
+ * @rnm_rst: Reset the RNM. Setting this bit to 1 clears all sorted numbers
+ *    in the random number memory.
+ * @rng_en: Enabled the output of the RNG.
+ * @ent_en: Entropy enable for random number generator.
+ */
+union efl_rnm_ctl_status {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz_9_63 : 55;
+                u64 ent_sel : 4;
+                u64 exp_ent : 1;
+                u64 rng_rst : 1;
+                u64 rnm_rst : 1;
+                u64 rng_en : 1;
+                u64 ent_en : 1;
+#else
+                u64 ent_en : 1;
+                u64 rng_en : 1;
+                u64 rnm_rst : 1;
+                u64 rng_rst : 1;
+                u64 exp_ent : 1;
+                u64 ent_sel : 4;
+                u64 raz_9_63 : 55;
+#endif
+        } s;
+};
+
+/**
+ * struct bmi_ctl - BMI control register
+ * @ilk_hdrq_thrsh: Maximum number of header queue locations
+ *   that ILK packets may consume. When the threshold is
+ *   exceeded ILK_XOFF is sent to the BMI_X2P_ARB.
+ * @nps_hdrq_thrsh: Maximum number of header queue locations
+ *   that NPS packets may consume. When the threshold is
+ *   exceeded NPS_XOFF is sent to the BMI_X2P_ARB.
+ * @totl_hdrq_thrsh: Maximum number of header queue locations
+ *   that the sum of ILK and NPS packets may consume.
+ * @ilk_free_thrsh: Maximum number of buffers that ILK packet
+ *   flows may consume before ILK_XOFF is sent to the BMI_X2P_ARB.
+ * @nps_free_thrsh: Maximum number of buffers that NPS packet
+ *   flows may consume before NPS XOFF is sent to the BMI_X2p_ARB.
+ * @totl_free_thrsh: Maximum number of buffers that bot ILK and NPS
+ *   packet flows may consume before both NPS_XOFF and ILK_XOFF
+ *   are asserted to the BMI_X2P_ARB.
+ * @max_pkt_len: Maximum packet length, integral number of 256B
+ *   buffers.
+ */
+union bmi_ctl {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz_56_63 : 8;
+                u64 ilk_hdrq_thrsh : 8;
+                u64 nps_hdrq_thrsh : 8;
+                u64 totl_hdrq_thrsh : 8;
+                u64 ilk_free_thrsh : 8;
+                u64 nps_free_thrsh : 8;
+                u64 totl_free_thrsh : 8;
+                u64 max_pkt_len : 8;
+#else
+                u64 max_pkt_len : 8;
+                u64 totl_free_thrsh : 8;
+                u64 nps_free_thrsh : 8;
+                u64 ilk_free_thrsh : 8;
+                u64 totl_hdrq_thrsh : 8;
+                u64 nps_hdrq_thrsh : 8;
+                u64 ilk_hdrq_thrsh : 8;
+                u64 raz_56_63 : 8;
+#endif
+        } s;
+};
+
+/**
+ * struct bmi_int_ena_w1s - BMI interrupt enable set register
+ * @ilk_req_oflw: Reads or sets enable for
+ *   BMI_INT[ILK_REQ_OFLW].
+ * @nps_req_oflw: Reads or sets enable for
+ *   BMI_INT[NPS_REQ_OFLW].
+ * @fpf_undrrn: Reads or sets enable for
+ *   BMI_INT[FPF_UNDRRN].
+ * @eop_err_ilk: Reads or sets enable for
+ *   BMI_INT[EOP_ERR_ILK].
+ * @eop_err_nps: Reads or sets enable for
+ *   BMI_INT[EOP_ERR_NPS].
+ * @sop_err_ilk: Reads or sets enable for
+ *   BMI_INT[SOP_ERR_ILK].
+ * @sop_err_nps: Reads or sets enable for
+ *   BMI_INT[SOP_ERR_NPS].
+ * @pkt_rcv_err_ilk: Reads or sets enable for
+ *   BMI_INT[PKT_RCV_ERR_ILK].
+ * @pkt_rcv_err_nps: Reads or sets enable for
+ *   BMI_INT[PKT_RCV_ERR_NPS].
+ * @max_len_err_ilk: Reads or sets enable for
+ *   BMI_INT[MAX_LEN_ERR_ILK].
+ * @max_len_err_nps: Reads or sets enable for
+ *   BMI_INT[MAX_LEN_ERR_NPS].
+ */
+union bmi_int_ena_w1s {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz_13_63   : 51;
+                u64 ilk_req_oflw : 1;
+                u64 nps_req_oflw : 1;
+                u64 raz_10 : 1;
+                u64 raz_9 : 1;
+                u64 fpf_undrrn  : 1;
+                u64 eop_err_ilk : 1;
+                u64 eop_err_nps : 1;
+                u64 sop_err_ilk : 1;
+                u64 sop_err_nps : 1;
+                u64 pkt_rcv_err_ilk : 1;
+                u64 pkt_rcv_err_nps : 1;
+                u64 max_len_err_ilk : 1;
+                u64 max_len_err_nps : 1;
+#else
+                u64 max_len_err_nps : 1;
+                u64 max_len_err_ilk : 1;
+                u64 pkt_rcv_err_nps : 1;
+                u64 pkt_rcv_err_ilk : 1;
+                u64 sop_err_nps : 1;
+                u64 sop_err_ilk : 1;
+                u64 eop_err_nps : 1;
+                u64 eop_err_ilk : 1;
+                u64 fpf_undrrn  : 1;
+                u64 raz_9 : 1;
+                u64 raz_10 : 1;
+                u64 nps_req_oflw : 1;
+                u64 ilk_req_oflw : 1;
+                u64 raz_13_63 : 51;
+#endif
+        } s;
+};
+
+/**
+ * struct bmo_ctl2 - BMO Control2 Register
+ * @arb_sel: Determines P2X Arbitration
+ * @ilk_buf_thrsh: Maximum number of buffers that the
+ *    ILK packet flows may consume before ILK XOFF is
+ *    asserted to the POM.
+ * @nps_slc_buf_thrsh: Maximum number of buffers that the
+ *    NPS_SLC packet flow may consume before NPS_SLC XOFF is
+ *    asserted to the POM.
+ * @nps_uns_buf_thrsh: Maximum number of buffers that the
+ *    NPS_UNS packet flow may consume before NPS_UNS XOFF is
+ *    asserted to the POM.
+ * @totl_buf_thrsh: Maximum number of buffers that ILK, NPS_UNS and
+ *    NPS_SLC packet flows may consume before NPS_UNS XOFF, NSP_SLC and
+ *    ILK_XOFF are all asserted POM.
+ */
+union bmo_ctl2 {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 arb_sel : 1;
+                u64 raz_32_62 : 31;
+                u64 ilk_buf_thrsh : 8;
+                u64 nps_slc_buf_thrsh : 8;
+                u64 nps_uns_buf_thrsh : 8;
+                u64 totl_buf_thrsh : 8;
+#else
+                u64 totl_buf_thrsh : 8;
+                u64 nps_uns_buf_thrsh : 8;
+                u64 nps_slc_buf_thrsh : 8;
+                u64 ilk_buf_thrsh : 8;
+                u64 raz_32_62 : 31;
+                u64 arb_sel : 1;
+#endif
+        } s;
+};
+
+/**
+ * struct pom_int_ena_w1s - POM interrupt enable set register
+ * @illegal_intf: Reads or sets enable for POM_INT[ILLEGAL_INTF].
+ * @illegal_dport: Reads or sets enable for POM_INT[ILLEGAL_DPORT].
+ */
+union pom_int_ena_w1s {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz2 : 60;
+                u64 illegal_intf : 1;
+                u64 illegal_dport : 1;
+                u64 raz1 : 1;
+                u64 raz0 : 1;
+#else
+                u64 raz0 : 1;
+                u64 raz1 : 1;
+                u64 illegal_dport : 1;
+                u64 illegal_intf : 1;
+                u64 raz2 : 60;
+#endif
+        } s;
+};
+
+/**
+ * struct lbc_inval_ctl - LBC invalidation control register
+ * @wait_timer: Wait timer for wait state. [WAIT_TIMER] must
+ *   always be written with its reset value.
+ * @cam_inval_start: Software should write [CAM_INVAL_START]=1
+ *   to initiate an LBC cache invalidation. After this, software
+ *   should read LBC_INVAL_STATUS until LBC_INVAL_STATUS[DONE] is set.
+ *   LBC hardware clears [CAVM_INVAL_START] before software can
+ *   observed LBC_INVAL_STATUS[DONE] to be set
+ */
+union lbc_inval_ctl {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz2 : 48;
+                u64 wait_timer : 8;
+                u64 raz1 : 6;
+                u64 cam_inval_start : 1;
+                u64 raz0 : 1;
+#else
+                u64 raz0 : 1;
+                u64 cam_inval_start : 1;
+                u64 raz1 : 6;
+                u64 wait_timer : 8;
+                u64 raz2 : 48;
+#endif
+        } s;
+};
+
+/**
+ * struct lbc_int_ena_w1s - LBC interrupt enable set register
+ * @cam_hard_err: Reads or sets enable for LBC_INT[CAM_HARD_ERR].
+ * @cam_inval_abort: Reads or sets enable for LBC_INT[CAM_INVAL_ABORT].
+ * @over_fetch_err: Reads or sets enable for LBC_INT[OVER_FETCH_ERR].
+ * @cache_line_to_err: Reads or sets enable for
+ *   LBC_INT[CACHE_LINE_TO_ERR].
+ * @cam_soft_err: Reads or sets enable for
+ *   LBC_INT[CAM_SOFT_ERR].
+ * @dma_rd_err: Reads or sets enable for
+ *   LBC_INT[DMA_RD_ERR].
+ */
+union lbc_int_ena_w1s {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz_10_63 : 54;
+                u64 cam_hard_err : 1;
+                u64 cam_inval_abort : 1;
+                u64 over_fetch_err : 1;
+                u64 cache_line_to_err : 1;
+                u64 raz_2_5 : 4;
+                u64 cam_soft_err : 1;
+                u64 dma_rd_err : 1;
+#else
+                u64 dma_rd_err : 1;
+                u64 cam_soft_err : 1;
+                u64 raz_2_5 : 4;
+                u64 cache_line_to_err : 1;
+                u64 over_fetch_err : 1;
+                u64 cam_inval_abort : 1;
+                u64 cam_hard_err : 1;
+                u64 raz_10_63 : 54;
+#endif
+        } s;
+};
+
+/**
+ * struct lbc_int - LBC interrupt summary register
+ * @cam_hard_err: indicates a fatal hardware error.
+ *   It requires system reset.
+ *   When [CAM_HARD_ERR] is set, LBC stops logging any new information in
+ *   LBC_POM_MISS_INFO_LOG,
+ *   LBC_POM_MISS_ADDR_LOG,
+ *   LBC_EFL_MISS_INFO_LOG, and
+ *   LBC_EFL_MISS_ADDR_LOG.
+ *   Software should sample them.
+ * @cam_inval_abort: indicates a fatal hardware error.
+ *   System reset is required.
+ * @over_fetch_err: indicates a fatal hardware error
+ *   System reset is required
+ * @cache_line_to_err: is a debug feature.
+ *   This timeout interrupt bit tells the software that
+ *   a cacheline in LBC has non-zero usage and the context
+ *   has not been used for greater than the
+ *   LBC_TO_CNT[TO_CNT] time interval.
+ * @sbe: Memory SBE error. This is recoverable via ECC.
+ *   See LBC_ECC_INT for more details.
+ * @dbe: Memory DBE error. This is a fatal and requires a
+ *   system reset.
+ * @pref_dat_len_mismatch_err: Summary bit for context length
+ *   mismatch errors.
+ * @rd_dat_len_mismatch_err: Summary bit for SE read data length
+ *   greater than data prefect length errors.
+ * @cam_soft_err: is recoverable. Software must complete a
+ *   LBC_INVAL_CTL[CAM_INVAL_START] invalidation sequence and
+ *   then clear [CAM_SOFT_ERR].
+ * @dma_rd_err: A context prefect read of host memory returned with
+ *   a read error.
+ */
+union lbc_int {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz_10_63 : 54;
+                u64 cam_hard_err : 1;
+                u64 cam_inval_abort : 1;
+                u64 over_fetch_err : 1;
+                u64 cache_line_to_err : 1;
+                u64 sbe : 1;
+                u64 dbe : 1;
+                u64 pref_dat_len_mismatch_err : 1;
+                u64 rd_dat_len_mismatch_err : 1;
+                u64 cam_soft_err : 1;
+                u64 dma_rd_err : 1;
+#else
+                u64 dma_rd_err : 1;
+                u64 cam_soft_err : 1;
+                u64 rd_dat_len_mismatch_err : 1;
+                u64 pref_dat_len_mismatch_err : 1;
+                u64 dbe : 1;
+                u64 sbe : 1;
+                u64 cache_line_to_err : 1;
+                u64 over_fetch_err : 1;
+                u64 cam_inval_abort : 1;
+                u64 cam_hard_err : 1;
+                u64 raz_10_63 : 54;
+#endif
+        } s;
+};
+
+/**
+ * struct lbc_inval_status: LBC Invalidation status register
+ * @cam_clean_entry_complete_cnt: The number of entries that are
+ *   cleaned up successfully.
+ * @cam_clean_entry_cnt: The number of entries that have the CAM
+ *   inval command issued.
+ * @cam_inval_state: cam invalidation FSM state
+ * @cam_inval_abort: cam invalidation abort
+ * @cam_rst_rdy: lbc_cam reset ready
+ * @done: LBC clears [DONE] when
+ *   LBC_INVAL_CTL[CAM_INVAL_START] is written with a one,
+ *   and sets [DONE] when it completes the invalidation
+ *   sequence.
+ */
+union lbc_inval_status {
+        u64 value;
+        struct {
+#if (defined(__BIG_ENDIAN_BITFIELD))
+                u64 raz3 : 23;
+                u64 cam_clean_entry_complete_cnt : 9;
+                u64 raz2 : 7;
+                u64 cam_clean_entry_cnt : 9;
+                u64 raz1 : 5;
+                u64 cam_inval_state : 3;
+                u64 raz0 : 5;
+                u64 cam_inval_abort : 1;
+                u64 cam_rst_rdy : 1;
+                u64 done : 1;
+#else
+                u64 done : 1;
+                u64 cam_rst_rdy : 1;
+                u64 cam_inval_abort : 1;
+                u64 raz0 : 5;
+                u64 cam_inval_state : 3;
+                u64 raz1 : 5;
+                u64 cam_clean_entry_cnt : 9;
+                u64 raz2 : 7;
+                u64 cam_clean_entry_complete_cnt : 9;
+                u64 raz3 : 23;
+#endif
+        } s;
+};
+
+#endif /* __NITROX_CSR_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h
new file mode 100644
index 000000000000..57858b04f165
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h
@@ -0,0 +1,179 @@
+#ifndef __NITROX_DEV_H
+#define __NITROX_DEV_H
+
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+
+#define VERSION_LEN 32
+
+struct nitrox_cmdq {
+        /* command queue lock */
+        spinlock_t cmdq_lock;
+        /* response list lock */
+        spinlock_t response_lock;
+        /* backlog list lock */
+        spinlock_t backlog_lock;
+
+        /* request submitted to chip, in progress */
+        struct list_head response_head;
+        /* hw queue full, hold in backlog list */
+        struct list_head backlog_head;
+
+        /* doorbell address */
+        u8 __iomem *dbell_csr_addr;
+        /* base address of the queue */
+        u8 *head;
+
+        struct nitrox_device *ndev;
+        /* flush pending backlog commands */
+        struct work_struct backlog_qflush;
+
+        /* requests posted waiting for completion */
+        atomic_t pending_count;
+        /* requests in backlog queues */
+        atomic_t backlog_count;
+
+        /* command size 32B/64B */
+        u8 instr_size;
+        u8 qno;
+        u32 qsize;
+
+        /* unaligned addresses */
+        u8 *head_unaligned;
+        dma_addr_t dma_unaligned;
+        /* dma address of the base */
+        dma_addr_t dma;
+};
+
+struct nitrox_hw {
+        /* firmware version */
+        char fw_name[VERSION_LEN];
+
+        u16 vendor_id;
+        u16 device_id;
+        u8 revision_id;
+
+        /* CNN55XX cores */
+        u8 se_cores;
+        u8 ae_cores;
+        u8 zip_cores;
+};
+
+#define MAX_MSIX_VECTOR_NAME    20
+/**
+ * vectors for queues (64 AE, 64 SE and 64 ZIP) and
+ * error condition/mailbox.
+ */
+#define MAX_MSIX_VECTORS        192
+
+struct nitrox_msix {
+        struct msix_entry *entries;
+        char **names;
+        DECLARE_BITMAP(irqs, MAX_MSIX_VECTORS);
+        u32 nr_entries;
+};
+
+struct bh_data {
+        /* slc port completion count address */
+        u8 __iomem *completion_cnt_csr_addr;
+
+        struct nitrox_cmdq *cmdq;
+        struct tasklet_struct resp_handler;
+};
+
+struct nitrox_bh {
+        struct bh_data *slc;
+};
+
+/* NITROX-5 driver state */
+#define NITROX_UCODE_LOADED     0
+#define NITROX_READY            1
+
+/* command queue size */
+#define DEFAULT_CMD_QLEN 2048
+/* command timeout in milliseconds */
+#define CMD_TIMEOUT 2000
+
+#define DEV(ndev) ((struct device *)(&(ndev)->pdev->dev))
+#define PF_MODE 0
+
+#define NITROX_CSR_ADDR(ndev, offset) \
+        ((ndev)->bar_addr + (offset))
+
+/**
+ * struct nitrox_device - NITROX Device Information.
+ * @list: pointer to linked list of devices
+ * @bar_addr: iomap address
+ * @pdev: PCI device information
+ * @status: NITROX status
+ * @timeout: Request timeout in jiffies
+ * @refcnt: Device usage count
+ * @idx: device index (0..N)
+ * @node: NUMA node id attached
+ * @qlen: Command queue length
+ * @nr_queues: Number of command queues
+ * @ctx_pool: DMA pool for crypto context
+ * @pkt_cmdqs: SE Command queues
+ * @msix: MSI-X information
+ * @bh: post processing work
+ * @hw: hardware information
+ * @debugfs_dir: debugfs directory
+ */
+struct nitrox_device {
+        struct list_head list;
+
+        u8 __iomem *bar_addr;
+        struct pci_dev *pdev;
+
+        unsigned long status;
+        unsigned long timeout;
+        refcount_t refcnt;
+
+        u8 idx;
+        int node;
+        u16 qlen;
+        u16 nr_queues;
+
+        struct dma_pool *ctx_pool;
+        struct nitrox_cmdq *pkt_cmdqs;
+
+        struct nitrox_msix msix;
+        struct nitrox_bh bh;
+
+        struct nitrox_hw hw;
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+        struct dentry *debugfs_dir;
+#endif
+};
+
+/**
+ * nitrox_read_csr - Read from device register
+ * @ndev: NITROX device
+ * @offset: offset of the register to read
+ *
+ * Returns: value read
+ */
+static inline u64 nitrox_read_csr(struct nitrox_device *ndev, u64 offset)
+{
+        return readq(ndev->bar_addr + offset);
+}
+
+/**
+ * nitrox_write_csr - Write to device register
+ * @ndev: NITROX device
+ * @offset: offset of the register to write
+ * @value: value to write
+ */
+static inline void nitrox_write_csr(struct nitrox_device *ndev, u64 offset,
+                                    u64 value)
+{
+        writeq(value, (ndev->bar_addr + offset));
+}
+
+static inline int nitrox_ready(struct nitrox_device *ndev)
+{
+        return test_bit(NITROX_READY, &ndev->status);
+}
+
+#endif /* __NITROX_DEV_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c
new file mode 100644
index 000000000000..f0655f82fa7d
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c
@@ -0,0 +1,401 @@
+#include <linux/delay.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_csr.h"
+
+/**
+ * emu_enable_cores - Enable EMU cluster cores.
+ * @ndev: N5 device
+ */
+static void emu_enable_cores(struct nitrox_device *ndev)
+{
+        union emu_se_enable emu_se;
+        union emu_ae_enable emu_ae;
+        int i;
+
+        /* AE cores 20 per cluster */
+        emu_ae.value = 0;
+        emu_ae.s.enable = 0xfffff;
+
+        /* SE cores 16 per cluster */
+        emu_se.value = 0;
+        emu_se.s.enable = 0xffff;
+
+        /* enable per cluster cores */
+        for (i = 0; i < NR_CLUSTERS; i++) {
+                nitrox_write_csr(ndev, EMU_AE_ENABLEX(i), emu_ae.value);
+                nitrox_write_csr(ndev, EMU_SE_ENABLEX(i), emu_se.value);
+        }
+}
+
+/**
+ * nitrox_config_emu_unit - configure EMU unit.
+ * @ndev: N5 device
+ */
+void nitrox_config_emu_unit(struct nitrox_device *ndev)
+{
+        union emu_wd_int_ena_w1s emu_wd_int;
+        union emu_ge_int_ena_w1s emu_ge_int;
+        u64 offset;
+        int i;
+
+        /* enable cores */
+        emu_enable_cores(ndev);
+
+        /* enable general error and watch dog interrupts */
+        emu_ge_int.value = 0;
+        emu_ge_int.s.se_ge = 0xffff;
+        emu_ge_int.s.ae_ge = 0xfffff;
+        emu_wd_int.value = 0;
+        emu_wd_int.s.se_wd = 1;
+
+        for (i = 0; i < NR_CLUSTERS; i++) {
+                offset = EMU_WD_INT_ENA_W1SX(i);
+                nitrox_write_csr(ndev, offset, emu_wd_int.value);
+                offset = EMU_GE_INT_ENA_W1SX(i);
+                nitrox_write_csr(ndev, offset, emu_ge_int.value);
+        }
+}
+
+static void reset_pkt_input_ring(struct nitrox_device *ndev, int ring)
+{
+        union nps_pkt_in_instr_ctl pkt_in_ctl;
+        union nps_pkt_in_instr_baoff_dbell pkt_in_dbell;
+        union nps_pkt_in_done_cnts pkt_in_cnts;
+        u64 offset;
+
+        offset = NPS_PKT_IN_INSTR_CTLX(ring);
+        /* disable the ring */
+        pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
+        pkt_in_ctl.s.enb = 0;
+        nitrox_write_csr(ndev, offset, pkt_in_ctl.value);
+        usleep_range(100, 150);
+
+        /* wait to clear [ENB] */
+        do {
+                pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
+        } while (pkt_in_ctl.s.enb);
+
+        /* clear off door bell counts */
+        offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(ring);
+        pkt_in_dbell.value = 0;
+        pkt_in_dbell.s.dbell = 0xffffffff;
+        nitrox_write_csr(ndev, offset, pkt_in_dbell.value);
+
+        /* clear done counts */
+        offset = NPS_PKT_IN_DONE_CNTSX(ring);
+        pkt_in_cnts.value = nitrox_read_csr(ndev, offset);
+        nitrox_write_csr(ndev, offset, pkt_in_cnts.value);
+        usleep_range(50, 100);
+}
+
+void enable_pkt_input_ring(struct nitrox_device *ndev, int ring)
+{
+        union nps_pkt_in_instr_ctl pkt_in_ctl;
+        u64 offset;
+
+        /* 64-byte instruction size */
+        offset = NPS_PKT_IN_INSTR_CTLX(ring);
+        pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
+        pkt_in_ctl.s.is64b = 1;
+        pkt_in_ctl.s.enb = 1;
+        nitrox_write_csr(ndev, offset, pkt_in_ctl.value);
+
+        /* wait for set [ENB] */
+        do {
+                pkt_in_ctl.value = nitrox_read_csr(ndev, offset);
+        } while (!pkt_in_ctl.s.enb);
+}
+
+/**
+ * nitrox_config_pkt_input_rings - configure Packet Input Rings
+ * @ndev: N5 device
+ */
+void nitrox_config_pkt_input_rings(struct nitrox_device *ndev)
+{
+        int i;
+
+        for (i = 0; i < ndev->nr_queues; i++) {
+                struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i];
+                union nps_pkt_in_instr_rsize pkt_in_rsize;
+                u64 offset;
+
+                reset_pkt_input_ring(ndev, i);
+
+                /* configure ring base address 16-byte aligned,
+                 * size and interrupt threshold.
+                 */
+                offset = NPS_PKT_IN_INSTR_BADDRX(i);
+                nitrox_write_csr(ndev, NPS_PKT_IN_INSTR_BADDRX(i), cmdq->dma);
+
+                /* configure ring size */
+                offset = NPS_PKT_IN_INSTR_RSIZEX(i);
+                pkt_in_rsize.value = 0;
+                pkt_in_rsize.s.rsize = ndev->qlen;
+                nitrox_write_csr(ndev, offset, pkt_in_rsize.value);
+
+                /* set high threshold for pkt input ring interrupts */
+                offset = NPS_PKT_IN_INT_LEVELSX(i);
+                nitrox_write_csr(ndev, offset, 0xffffffff);
+
+                enable_pkt_input_ring(ndev, i);
+        }
+}
+
+static void reset_pkt_solicit_port(struct nitrox_device *ndev, int port)
+{
+        union nps_pkt_slc_ctl pkt_slc_ctl;
+        union nps_pkt_slc_cnts pkt_slc_cnts;
+        u64 offset;
+
+        /* disable slc port */
+        offset = NPS_PKT_SLC_CTLX(port);
+        pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
+        pkt_slc_ctl.s.enb = 0;
+        nitrox_write_csr(ndev, offset, pkt_slc_ctl.value);
+        usleep_range(100, 150);
+
+        /* wait to clear [ENB] */
+        do {
+                pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
+        } while (pkt_slc_ctl.s.enb);
+
+        /* clear slc counters */
+        offset = NPS_PKT_SLC_CNTSX(port);
+        pkt_slc_cnts.value = nitrox_read_csr(ndev, offset);
+        nitrox_write_csr(ndev, offset, pkt_slc_cnts.value);
+        usleep_range(50, 100);
+}
+
+void enable_pkt_solicit_port(struct nitrox_device *ndev, int port)
+{
+        union nps_pkt_slc_ctl pkt_slc_ctl;
+        u64 offset;
+
+        offset = NPS_PKT_SLC_CTLX(port);
+        pkt_slc_ctl.value = 0;
+        pkt_slc_ctl.s.enb = 1;
+
+        /*
+         * 8 trailing 0x00 bytes will be added
+         * to the end of the outgoing packet.
+         */
+        pkt_slc_ctl.s.z = 1;
+        /* enable response header */
+        pkt_slc_ctl.s.rh = 1;
+        nitrox_write_csr(ndev, offset, pkt_slc_ctl.value);
+
+        /* wait to set [ENB] */
+        do {
+                pkt_slc_ctl.value = nitrox_read_csr(ndev, offset);
+        } while (!pkt_slc_ctl.s.enb);
+}
+
+static void config_single_pkt_solicit_port(struct nitrox_device *ndev,
+                                           int port)
+{
+        union nps_pkt_slc_int_levels pkt_slc_int;
+        u64 offset;
+
+        reset_pkt_solicit_port(ndev, port);
+
+        offset = NPS_PKT_SLC_INT_LEVELSX(port);
+        pkt_slc_int.value = 0;
+        /* time interrupt threshold */
+        pkt_slc_int.s.timet = 0x3fffff;
+        nitrox_write_csr(ndev, offset, pkt_slc_int.value);
+
+        enable_pkt_solicit_port(ndev, port);
+}
+
+void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev)
+{
+        int i;
+
+        for (i = 0; i < ndev->nr_queues; i++)
+                config_single_pkt_solicit_port(ndev, i);
+}
+
+/**
+ * enable_nps_interrupts - enable NPS interrutps
+ * @ndev: N5 device.
+ *
+ * This includes NPS core, packet in and slc interrupts.
+ */
+static void enable_nps_interrupts(struct nitrox_device *ndev)
+{
+        union nps_core_int_ena_w1s core_int;
+
+        /* NPS core interrutps */
+        core_int.value = 0;
+        core_int.s.host_wr_err = 1;
+        core_int.s.host_wr_timeout = 1;
+        core_int.s.exec_wr_timeout = 1;
+        core_int.s.npco_dma_malform = 1;
+        core_int.s.host_nps_wr_err = 1;
+        nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value);
+
+        /* NPS packet in ring interrupts */
+        nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL));
+        nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL));
+        nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL));
+        /* NPS packet slc port interrupts */
+        nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL));
+        nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL));
+        nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL));
+}
+
+void nitrox_config_nps_unit(struct nitrox_device *ndev)
+{
+        union nps_core_gbl_vfcfg core_gbl_vfcfg;
+
+        /* endian control information */
+        nitrox_write_csr(ndev, NPS_CORE_CONTROL, 1ULL);
+
+        /* disable ILK interface */
+        core_gbl_vfcfg.value = 0;
+        core_gbl_vfcfg.s.ilk_disable = 1;
+        core_gbl_vfcfg.s.cfg = PF_MODE;
+        nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value);
+        /* config input and solicit ports */
+        nitrox_config_pkt_input_rings(ndev);
+        nitrox_config_pkt_solicit_ports(ndev);
+
+        /* enable interrupts */
+        enable_nps_interrupts(ndev);
+}
+
+void nitrox_config_pom_unit(struct nitrox_device *ndev)
+{
+        union pom_int_ena_w1s pom_int;
+        int i;
+
+        /* enable pom interrupts */
+        pom_int.value = 0;
+        pom_int.s.illegal_dport = 1;
+        nitrox_write_csr(ndev, POM_INT_ENA_W1S, pom_int.value);
+
+        /* enable perf counters */
+        for (i = 0; i < ndev->hw.se_cores; i++)
+                nitrox_write_csr(ndev, POM_PERF_CTL, BIT_ULL(i));
+}
+
+/**
+ * nitrox_config_rand_unit - enable N5 random number unit
+ * @ndev: N5 device
+ */
+void nitrox_config_rand_unit(struct nitrox_device *ndev)
+{
+        union efl_rnm_ctl_status efl_rnm_ctl;
+        u64 offset;
+
+        offset = EFL_RNM_CTL_STATUS;
+        efl_rnm_ctl.value = nitrox_read_csr(ndev, offset);
+        efl_rnm_ctl.s.ent_en = 1;
+        efl_rnm_ctl.s.rng_en = 1;
+        nitrox_write_csr(ndev, offset, efl_rnm_ctl.value);
+}
+
+void nitrox_config_efl_unit(struct nitrox_device *ndev)
+{
+        int i;
+
+        for (i = 0; i < NR_CLUSTERS; i++) {
+                union efl_core_int_ena_w1s efl_core_int;
+                u64 offset;
+
+                /* EFL core interrupts */
+                offset = EFL_CORE_INT_ENA_W1SX(i);
+                efl_core_int.value = 0;
+                efl_core_int.s.len_ovr = 1;
+                efl_core_int.s.d_left = 1;
+                efl_core_int.s.epci_decode_err = 1;
+                nitrox_write_csr(ndev, offset, efl_core_int.value);
+
+                offset = EFL_CORE_VF_ERR_INT0_ENA_W1SX(i);
+                nitrox_write_csr(ndev, offset, (~0ULL));
+                offset = EFL_CORE_VF_ERR_INT1_ENA_W1SX(i);
+                nitrox_write_csr(ndev, offset, (~0ULL));
+        }
+}
+
+void nitrox_config_bmi_unit(struct nitrox_device *ndev)
+{
+        union bmi_ctl bmi_ctl;
+        union bmi_int_ena_w1s bmi_int_ena;
+        u64 offset;
+
+        /* no threshold limits for PCIe */
+        offset = BMI_CTL;
+        bmi_ctl.value = nitrox_read_csr(ndev, offset);
+        bmi_ctl.s.max_pkt_len = 0xff;
+        bmi_ctl.s.nps_free_thrsh = 0xff;
+        bmi_ctl.s.nps_hdrq_thrsh = 0x7a;
+        nitrox_write_csr(ndev, offset, bmi_ctl.value);
+
+        /* enable interrupts */
+        offset = BMI_INT_ENA_W1S;
+        bmi_int_ena.value = 0;
+        bmi_int_ena.s.max_len_err_nps = 1;
+        bmi_int_ena.s.pkt_rcv_err_nps = 1;
+        bmi_int_ena.s.fpf_undrrn = 1;
+        nitrox_write_csr(ndev, offset, bmi_int_ena.value);
+}
+
+void nitrox_config_bmo_unit(struct nitrox_device *ndev)
+{
+        union bmo_ctl2 bmo_ctl2;
+        u64 offset;
+
+        /* no threshold limits for PCIe */
+        offset = BMO_CTL2;
+        bmo_ctl2.value = nitrox_read_csr(ndev, offset);
+        bmo_ctl2.s.nps_slc_buf_thrsh = 0xff;
+        nitrox_write_csr(ndev, offset, bmo_ctl2.value);
+}
+
+void invalidate_lbc(struct nitrox_device *ndev)
+{
+        union lbc_inval_ctl lbc_ctl;
+        union lbc_inval_status lbc_stat;
+        u64 offset;
+
+        /* invalidate LBC */
+        offset = LBC_INVAL_CTL;
+        lbc_ctl.value = nitrox_read_csr(ndev, offset);
+        lbc_ctl.s.cam_inval_start = 1;
+        nitrox_write_csr(ndev, offset, lbc_ctl.value);
+
+        offset = LBC_INVAL_STATUS;
+
+        do {
+                lbc_stat.value = nitrox_read_csr(ndev, offset);
+        } while (!lbc_stat.s.done);
+}
+
+void nitrox_config_lbc_unit(struct nitrox_device *ndev)
+{
+        union lbc_int_ena_w1s lbc_int_ena;
+        u64 offset;
+
+        invalidate_lbc(ndev);
+
+        /* enable interrupts */
+        offset = LBC_INT_ENA_W1S;
+        lbc_int_ena.value = 0;
+        lbc_int_ena.s.dma_rd_err = 1;
+        lbc_int_ena.s.over_fetch_err = 1;
+        lbc_int_ena.s.cam_inval_abort = 1;
+        lbc_int_ena.s.cam_hard_err = 1;
+        nitrox_write_csr(ndev, offset, lbc_int_ena.value);
+
+        offset = LBC_PLM_VF1_64_INT_ENA_W1S;
+        nitrox_write_csr(ndev, offset, (~0ULL));
+        offset = LBC_PLM_VF65_128_INT_ENA_W1S;
+        nitrox_write_csr(ndev, offset, (~0ULL));
+
+        offset = LBC_ELM_VF1_64_INT_ENA_W1S;
+        nitrox_write_csr(ndev, offset, (~0ULL));
+        offset = LBC_ELM_VF65_128_INT_ENA_W1S;
+        nitrox_write_csr(ndev, offset, (~0ULL));
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c
new file mode 100644
index 000000000000..71f934871a89
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c
@@ -0,0 +1,467 @@
+#include <linux/pci.h>
+#include <linux/printk.h>
+#include <linux/slab.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_csr.h"
+#include "nitrox_common.h"
+
+#define NR_RING_VECTORS 3
+#define NPS_CORE_INT_ACTIVE_ENTRY 192
+
+/**
+ * nps_pkt_slc_isr - IRQ handler for NPS solicit port
+ * @irq: irq number
+ * @data: argument
+ */
+static irqreturn_t nps_pkt_slc_isr(int irq, void *data)
+{
+        struct bh_data *slc = data;
+        union nps_pkt_slc_cnts pkt_slc_cnts;
+
+        pkt_slc_cnts.value = readq(slc->completion_cnt_csr_addr);
+        /* New packet on SLC output port */
+        if (pkt_slc_cnts.s.slc_int)
+                tasklet_hi_schedule(&slc->resp_handler);
+
+        return IRQ_HANDLED;
+}
+
+static void clear_nps_core_err_intr(struct nitrox_device *ndev)
+{
+        u64 value;
+
+        /* Write 1 to clear */
+        value = nitrox_read_csr(ndev, NPS_CORE_INT);
+        nitrox_write_csr(ndev, NPS_CORE_INT, value);
+
+        dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT  0x%016llx\n", value);
+}
+
+static void clear_nps_pkt_err_intr(struct nitrox_device *ndev)
+{
+        union nps_pkt_int pkt_int;
+        unsigned long value, offset;
+        int i;
+
+        pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT);
+        dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT  0x%016llx\n",
+                            pkt_int.value);
+
+        if (pkt_int.s.slc_err) {
+                offset = NPS_PKT_SLC_ERR_TYPE;
+                value = nitrox_read_csr(ndev, offset);
+                nitrox_write_csr(ndev, offset, value);
+                dev_err_ratelimited(DEV(ndev),
+                                    "NPS_PKT_SLC_ERR_TYPE  0x%016lx\n", value);
+
+                offset = NPS_PKT_SLC_RERR_LO;
+                value = nitrox_read_csr(ndev, offset);
+                nitrox_write_csr(ndev, offset, value);
+                /* enable the solicit ports */
+                for_each_set_bit(i, &value, BITS_PER_LONG)
+                        enable_pkt_solicit_port(ndev, i);
+
+                dev_err_ratelimited(DEV(ndev),
+                                    "NPS_PKT_SLC_RERR_LO  0x%016lx\n", value);
+
+                offset = NPS_PKT_SLC_RERR_HI;
+                value = nitrox_read_csr(ndev, offset);
+                nitrox_write_csr(ndev, offset, value);
+                dev_err_ratelimited(DEV(ndev),
+                                    "NPS_PKT_SLC_RERR_HI  0x%016lx\n", value);
+        }
+
+        if (pkt_int.s.in_err) {
+                offset = NPS_PKT_IN_ERR_TYPE;
+                value = nitrox_read_csr(ndev, offset);
+                nitrox_write_csr(ndev, offset, value);
+                dev_err_ratelimited(DEV(ndev),
+                                    "NPS_PKT_IN_ERR_TYPE  0x%016lx\n", value);
+                offset = NPS_PKT_IN_RERR_LO;
+                value = nitrox_read_csr(ndev, offset);
+                nitrox_write_csr(ndev, offset, value);
+                /* enable the input ring */
+                for_each_set_bit(i, &value, BITS_PER_LONG)
+                        enable_pkt_input_ring(ndev, i);
+
+                dev_err_ratelimited(DEV(ndev),
+                                    "NPS_PKT_IN_RERR_LO  0x%016lx\n", value);
+
+                offset = NPS_PKT_IN_RERR_HI;
+                value = nitrox_read_csr(ndev, offset);
+                nitrox_write_csr(ndev, offset, value);
+                dev_err_ratelimited(DEV(ndev),
+                                    "NPS_PKT_IN_RERR_HI  0x%016lx\n", value);
+        }
+}
+
+static void clear_pom_err_intr(struct nitrox_device *ndev)
+{
+        u64 value;
+
+        value = nitrox_read_csr(ndev, POM_INT);
+        nitrox_write_csr(ndev, POM_INT, value);
+        dev_err_ratelimited(DEV(ndev), "POM_INT  0x%016llx\n", value);
+}
+
+static void clear_pem_err_intr(struct nitrox_device *ndev)
+{
+        u64 value;
+
+        value = nitrox_read_csr(ndev, PEM0_INT);
+        nitrox_write_csr(ndev, PEM0_INT, value);
+        dev_err_ratelimited(DEV(ndev), "PEM(0)_INT  0x%016llx\n", value);
+}
+
+static void clear_lbc_err_intr(struct nitrox_device *ndev)
+{
+        union lbc_int lbc_int;
+        u64 value, offset;
+        int i;
+
+        lbc_int.value = nitrox_read_csr(ndev, LBC_INT);
+        dev_err_ratelimited(DEV(ndev), "LBC_INT  0x%016llx\n", lbc_int.value);
+
+        if (lbc_int.s.dma_rd_err) {
+                for (i = 0; i < NR_CLUSTERS; i++) {
+                        offset = EFL_CORE_VF_ERR_INT0X(i);
+                        value = nitrox_read_csr(ndev, offset);
+                        nitrox_write_csr(ndev, offset, value);
+                        offset = EFL_CORE_VF_ERR_INT1X(i);
+                        value = nitrox_read_csr(ndev, offset);
+                        nitrox_write_csr(ndev, offset, value);
+                }
+        }
+
+        if (lbc_int.s.cam_soft_err) {
+                dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n");
+                invalidate_lbc(ndev);
+        }
+
+        if (lbc_int.s.pref_dat_len_mismatch_err) {
+                offset = LBC_PLM_VF1_64_INT;
+                value = nitrox_read_csr(ndev, offset);
+                nitrox_write_csr(ndev, offset, value);
+                offset = LBC_PLM_VF65_128_INT;
+                value = nitrox_read_csr(ndev, offset);
+                nitrox_write_csr(ndev, offset, value);
+        }
+
+        if (lbc_int.s.rd_dat_len_mismatch_err) {
+                offset = LBC_ELM_VF1_64_INT;
+                value = nitrox_read_csr(ndev, offset);
+                nitrox_write_csr(ndev, offset, value);
+                offset = LBC_ELM_VF65_128_INT;
+                value = nitrox_read_csr(ndev, offset);
+                nitrox_write_csr(ndev, offset, value);
+        }
+        nitrox_write_csr(ndev, LBC_INT, lbc_int.value);
+}
+
+static void clear_efl_err_intr(struct nitrox_device *ndev)
+{
+        int i;
+
+        for (i = 0; i < NR_CLUSTERS; i++) {
+                union efl_core_int core_int;
+                u64 value, offset;
+
+                offset = EFL_CORE_INTX(i);
+                core_int.value = nitrox_read_csr(ndev, offset);
+                nitrox_write_csr(ndev, offset, core_int.value);
+                dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT  0x%016llx\n",
+                                    i, core_int.value);
+                if (core_int.s.se_err) {
+                        offset = EFL_CORE_SE_ERR_INTX(i);
+                        value = nitrox_read_csr(ndev, offset);
+                        nitrox_write_csr(ndev, offset, value);
+                }
+        }
+}
+
+static void clear_bmi_err_intr(struct nitrox_device *ndev)
+{
+        u64 value;
+
+        value = nitrox_read_csr(ndev, BMI_INT);
+        nitrox_write_csr(ndev, BMI_INT, value);
+        dev_err_ratelimited(DEV(ndev), "BMI_INT  0x%016llx\n", value);
+}
+
+/**
+ * clear_nps_core_int_active - clear NPS_CORE_INT_ACTIVE interrupts
+ * @ndev: NITROX device
+ */
+static void clear_nps_core_int_active(struct nitrox_device *ndev)
+{
+        union nps_core_int_active core_int_active;
+
+        core_int_active.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE);
+
+        if (core_int_active.s.nps_core)
+                clear_nps_core_err_intr(ndev);
+
+        if (core_int_active.s.nps_pkt)
+                clear_nps_pkt_err_intr(ndev);
+
+        if (core_int_active.s.pom)
+                clear_pom_err_intr(ndev);
+
+        if (core_int_active.s.pem)
+                clear_pem_err_intr(ndev);
+
+        if (core_int_active.s.lbc)
+                clear_lbc_err_intr(ndev);
+
+        if (core_int_active.s.efl)
+                clear_efl_err_intr(ndev);
+
+        if (core_int_active.s.bmi)
+                clear_bmi_err_intr(ndev);
+
+        /* If more work callback the ISR, set resend */
+        core_int_active.s.resend = 1;
+        nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int_active.value);
+}
+
+static irqreturn_t nps_core_int_isr(int irq, void *data)
+{
+        struct nitrox_device *ndev = data;
+
+        clear_nps_core_int_active(ndev);
+
+        return IRQ_HANDLED;
+}
+
+static int nitrox_enable_msix(struct nitrox_device *ndev)
+{
+        struct msix_entry *entries;
+        char **names;
+        int i, nr_entries, ret;
+
+        /*
+         * PF MSI-X vectors
+         *
+         * Entry 0: NPS PKT ring 0
+         * Entry 1: AQMQ ring 0
+         * Entry 2: ZQM ring 0
+         * Entry 3: NPS PKT ring 1
+         * Entry 4: AQMQ ring 1
+         * Entry 5: ZQM ring 1
+         * ....
+         * Entry 192: NPS_CORE_INT_ACTIVE
+         */
+        nr_entries = (ndev->nr_queues * NR_RING_VECTORS) + 1;
+        entries = kzalloc_node(nr_entries * sizeof(struct msix_entry),
+                               GFP_KERNEL, ndev->node);
+        if (!entries)
+                return -ENOMEM;
+
+        names = kcalloc(nr_entries, sizeof(char *), GFP_KERNEL);
+        if (!names) {
+                kfree(entries);
+                return -ENOMEM;
+        }
+
+        /* fill entires */
+        for (i = 0; i < (nr_entries - 1); i++)
+                entries[i].entry = i;
+
+        entries[i].entry = NPS_CORE_INT_ACTIVE_ENTRY;
+
+        for (i = 0; i < nr_entries; i++) {
+                *(names + i) = kzalloc(MAX_MSIX_VECTOR_NAME, GFP_KERNEL);
+                if (!(*(names + i))) {
+                        ret = -ENOMEM;
+                        goto msix_fail;
+                }
+        }
+        ndev->msix.entries = entries;
+        ndev->msix.names = names;
+        ndev->msix.nr_entries = nr_entries;
+
+        ret = pci_enable_msix_exact(ndev->pdev, ndev->msix.entries,
+                                    ndev->msix.nr_entries);
+        if (ret) {
+                dev_err(&ndev->pdev->dev, "Failed to enable MSI-X IRQ(s) %d\n",
+                        ret);
+                goto msix_fail;
+        }
+        return 0;
+
+msix_fail:
+        for (i = 0; i < nr_entries; i++)
+                kfree(*(names + i));
+
+        kfree(entries);
+        kfree(names);
+        return ret;
+}
+
+static void nitrox_cleanup_pkt_slc_bh(struct nitrox_device *ndev)
+{
+        int i;
+
+        if (!ndev->bh.slc)
+                return;
+
+        for (i = 0; i < ndev->nr_queues; i++) {
+                struct bh_data *bh = &ndev->bh.slc[i];
+
+                tasklet_disable(&bh->resp_handler);
+                tasklet_kill(&bh->resp_handler);
+        }
+        kfree(ndev->bh.slc);
+        ndev->bh.slc = NULL;
+}
+
+static int nitrox_setup_pkt_slc_bh(struct nitrox_device *ndev)
+{
+        u32 size;
+        int i;
+
+        size = ndev->nr_queues * sizeof(struct bh_data);
+        ndev->bh.slc = kzalloc(size, GFP_KERNEL);
+        if (!ndev->bh.slc)
+                return -ENOMEM;
+
+        for (i = 0; i < ndev->nr_queues; i++) {
+                struct bh_data *bh = &ndev->bh.slc[i];
+                u64 offset;
+
+                offset = NPS_PKT_SLC_CNTSX(i);
+                /* pre calculate completion count address */
+                bh->completion_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset);
+                bh->cmdq = &ndev->pkt_cmdqs[i];
+
+                tasklet_init(&bh->resp_handler, pkt_slc_resp_handler,
+                             (unsigned long)bh);
+        }
+
+        return 0;
+}
+
+static int nitrox_request_irqs(struct nitrox_device *ndev)
+{
+        struct pci_dev *pdev = ndev->pdev;
+        struct msix_entry *msix_ent = ndev->msix.entries;
+        int nr_ring_vectors, i = 0, ring, cpu, ret;
+        char *name;
+
+        /*
+         * PF MSI-X vectors
+         *
+         * Entry 0: NPS PKT ring 0
+         * Entry 1: AQMQ ring 0
+         * Entry 2: ZQM ring 0
+         * Entry 3: NPS PKT ring 1
+         * ....
+         * Entry 192: NPS_CORE_INT_ACTIVE
+         */
+        nr_ring_vectors = ndev->nr_queues * NR_RING_VECTORS;
+
+        /* request irq for pkt ring/ports only */
+        while (i < nr_ring_vectors) {
+                name = *(ndev->msix.names + i);
+                ring = (i / NR_RING_VECTORS);
+                snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-slc-ring%d",
+                         ndev->idx, ring);
+
+                ret = request_irq(msix_ent[i].vector, nps_pkt_slc_isr, 0,
+                                  name, &ndev->bh.slc[ring]);
+                if (ret) {
+                        dev_err(&pdev->dev, "failed to get irq %d for %s\n",
+                                msix_ent[i].vector, name);
+                        return ret;
+                }
+                cpu = ring % num_online_cpus();
+                irq_set_affinity_hint(msix_ent[i].vector, get_cpu_mask(cpu));
+
+                set_bit(i, ndev->msix.irqs);
+                i += NR_RING_VECTORS;
+        }
+
+        /* Request IRQ for NPS_CORE_INT_ACTIVE */
+        name = *(ndev->msix.names + i);
+        snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-nps-core-int", ndev->idx);
+        ret = request_irq(msix_ent[i].vector, nps_core_int_isr, 0, name, ndev);
+        if (ret) {
+                dev_err(&pdev->dev, "failed to get irq %d for %s\n",
+                        msix_ent[i].vector, name);
+                return ret;
+        }
+        set_bit(i, ndev->msix.irqs);
+
+        return 0;
+}
+
+static void nitrox_disable_msix(struct nitrox_device *ndev)
+{
+        struct msix_entry *msix_ent = ndev->msix.entries;
+        char **names = ndev->msix.names;
+        int i = 0, ring, nr_ring_vectors;
+
+        nr_ring_vectors = ndev->msix.nr_entries - 1;
+
+        /* clear pkt ring irqs */
+        while (i < nr_ring_vectors) {
+                if (test_and_clear_bit(i, ndev->msix.irqs)) {
+                        ring = (i / NR_RING_VECTORS);
+                        irq_set_affinity_hint(msix_ent[i].vector, NULL);
+                        free_irq(msix_ent[i].vector, &ndev->bh.slc[ring]);
+                }
+                i += NR_RING_VECTORS;
+        }
+        irq_set_affinity_hint(msix_ent[i].vector, NULL);
+        free_irq(msix_ent[i].vector, ndev);
+        clear_bit(i, ndev->msix.irqs);
+
+        kfree(ndev->msix.entries);
+        for (i = 0; i < ndev->msix.nr_entries; i++)
+                kfree(*(names + i));
+
+        kfree(names);
+        pci_disable_msix(ndev->pdev);
+}
+
+/**
+ * nitrox_pf_cleanup_isr: Cleanup PF MSI-X and IRQ
+ * @ndev: NITROX device
+ */
+void nitrox_pf_cleanup_isr(struct nitrox_device *ndev)
+{
+        nitrox_disable_msix(ndev);
+        nitrox_cleanup_pkt_slc_bh(ndev);
+}
+
+/**
+ * nitrox_init_isr - Initialize PF MSI-X vectors and IRQ
+ * @ndev: NITROX device
+ *
+ * Return: 0 on success, a negative value on failure.
+ */
+int nitrox_pf_init_isr(struct nitrox_device *ndev)
+{
+        int err;
+
+        err = nitrox_setup_pkt_slc_bh(ndev);
+        if (err)
+                return err;
+
+        err = nitrox_enable_msix(ndev);
+        if (err)
+                goto msix_fail;
+
+        err = nitrox_request_irqs(ndev);
+        if (err)
+                goto irq_fail;
+
+        return 0;
+
+irq_fail:
+        nitrox_disable_msix(ndev);
+msix_fail:
+        nitrox_cleanup_pkt_slc_bh(ndev);
+        return err;
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c
new file mode 100644
index 000000000000..b4a391adb9b6
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c
@@ -0,0 +1,210 @@
+#include <linux/cpumask.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/delay.h>
+#include <linux/gfp.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci_regs.h>
+#include <linux/vmalloc.h>
+#include <linux/pci.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_common.h"
+#include "nitrox_req.h"
+#include "nitrox_csr.h"
+
+#define CRYPTO_CTX_SIZE 256
+
+/* command queue alignments */
+#define PKT_IN_ALIGN    16
+
+static int cmdq_common_init(struct nitrox_cmdq *cmdq)
+{
+        struct nitrox_device *ndev = cmdq->ndev;
+        u32 qsize;
+
+        qsize = (ndev->qlen) * cmdq->instr_size;
+        cmdq->head_unaligned = dma_zalloc_coherent(DEV(ndev),
+                                                   (qsize + PKT_IN_ALIGN),
+                                                   &cmdq->dma_unaligned,
+                                                   GFP_KERNEL);
+        if (!cmdq->head_unaligned)
+                return -ENOMEM;
+
+        cmdq->head = PTR_ALIGN(cmdq->head_unaligned, PKT_IN_ALIGN);
+        cmdq->dma = PTR_ALIGN(cmdq->dma_unaligned, PKT_IN_ALIGN);
+        cmdq->qsize = (qsize + PKT_IN_ALIGN);
+
+        spin_lock_init(&cmdq->response_lock);
+        spin_lock_init(&cmdq->cmdq_lock);
+        spin_lock_init(&cmdq->backlog_lock);
+
+        INIT_LIST_HEAD(&cmdq->response_head);
+        INIT_LIST_HEAD(&cmdq->backlog_head);
+        INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work);
+
+        atomic_set(&cmdq->pending_count, 0);
+        atomic_set(&cmdq->backlog_count, 0);
+        return 0;
+}
+
+static void cmdq_common_cleanup(struct nitrox_cmdq *cmdq)
+{
+        struct nitrox_device *ndev = cmdq->ndev;
+
+        cancel_work_sync(&cmdq->backlog_qflush);
+
+        dma_free_coherent(DEV(ndev), cmdq->qsize,
+                          cmdq->head_unaligned, cmdq->dma_unaligned);
+
+        atomic_set(&cmdq->pending_count, 0);
+        atomic_set(&cmdq->backlog_count, 0);
+
+        cmdq->dbell_csr_addr = NULL;
+        cmdq->head = NULL;
+        cmdq->dma = 0;
+        cmdq->qsize = 0;
+        cmdq->instr_size = 0;
+}
+
+static void nitrox_cleanup_pkt_cmdqs(struct nitrox_device *ndev)
+{
+        int i;
+
+        for (i = 0; i < ndev->nr_queues; i++) {
+                struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i];
+
+                cmdq_common_cleanup(cmdq);
+        }
+        kfree(ndev->pkt_cmdqs);
+        ndev->pkt_cmdqs = NULL;
+}
+
+static int nitrox_init_pkt_cmdqs(struct nitrox_device *ndev)
+{
+        int i, err, size;
+
+        size = ndev->nr_queues * sizeof(struct nitrox_cmdq);
+        ndev->pkt_cmdqs = kzalloc(size, GFP_KERNEL);
+        if (!ndev->pkt_cmdqs)
+                return -ENOMEM;
+
+        for (i = 0; i < ndev->nr_queues; i++) {
+                struct nitrox_cmdq *cmdq;
+                u64 offset;
+
+                cmdq = &ndev->pkt_cmdqs[i];
+                cmdq->ndev = ndev;
+                cmdq->qno = i;
+                cmdq->instr_size = sizeof(struct nps_pkt_instr);
+
+                offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i);
+                /* SE ring doorbell address for this queue */
+                cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset);
+
+                err = cmdq_common_init(cmdq);
+                if (err)
+                        goto pkt_cmdq_fail;
+        }
+        return 0;
+
+pkt_cmdq_fail:
+        nitrox_cleanup_pkt_cmdqs(ndev);
+        return err;
+}
+
+static int create_crypto_dma_pool(struct nitrox_device *ndev)
+{
+        size_t size;
+
+        /* Crypto context pool, 16 byte aligned */
+        size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr);
+        ndev->ctx_pool = dma_pool_create("crypto-context",
+                                         DEV(ndev), size, 16, 0);
+        if (!ndev->ctx_pool)
+                return -ENOMEM;
+
+        return 0;
+}
+
+static void destroy_crypto_dma_pool(struct nitrox_device *ndev)
+{
+        if (!ndev->ctx_pool)
+                return;
+
+        dma_pool_destroy(ndev->ctx_pool);
+        ndev->ctx_pool = NULL;
+}
+
+/*
+ * crypto_alloc_context - Allocate crypto context from pool
+ * @ndev: NITROX Device
+ */
+void *crypto_alloc_context(struct nitrox_device *ndev)
+{
+        struct ctx_hdr *ctx;
+        void *vaddr;
+        dma_addr_t dma;
+
+        vaddr = dma_pool_alloc(ndev->ctx_pool, (GFP_ATOMIC | __GFP_ZERO), &dma);
+        if (!vaddr)
+                return NULL;
+
+        /* fill meta data */
+        ctx = vaddr;
+        ctx->pool = ndev->ctx_pool;
+        ctx->dma = dma;
+        ctx->ctx_dma = dma + sizeof(struct ctx_hdr);
+
+        return ((u8 *)vaddr + sizeof(struct ctx_hdr));
+}
+
+/**
+ * crypto_free_context - Free crypto context to pool
+ * @ctx: context to free
+ */
+void crypto_free_context(void *ctx)
+{
+        struct ctx_hdr *ctxp;
+
+        if (!ctx)
+                return;
+
+        ctxp = (struct ctx_hdr *)((u8 *)ctx - sizeof(struct ctx_hdr));
+        dma_pool_free(ctxp->pool, ctxp, ctxp->dma);
+}
+
+/**
+ * nitrox_common_sw_init - allocate software resources.
+ * @ndev: NITROX device
+ *
+ * Allocates crypto context pools and command queues etc.
+ *
+ * Return: 0 on success, or a negative error code on error.
+ */
+int nitrox_common_sw_init(struct nitrox_device *ndev)
+{
+        int err = 0;
+
+        /* per device crypto context pool */
+        err = create_crypto_dma_pool(ndev);
+        if (err)
+                return err;
+
+        err = nitrox_init_pkt_cmdqs(ndev);
+        if (err)
+                destroy_crypto_dma_pool(ndev);
+
+        return err;
+}
+
+/**
+ * nitrox_common_sw_cleanup - free software resources.
+ * @ndev: NITROX device
+ */
+void nitrox_common_sw_cleanup(struct nitrox_device *ndev)
+{
+        nitrox_cleanup_pkt_cmdqs(ndev);
+        destroy_crypto_dma_pool(ndev);
+}
diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c
new file mode 100644
index 000000000000..ae44a464cd2d
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_main.c
@@ -0,0 +1,640 @@
+#include <linux/aer.h>
+#include <linux/delay.h>
+#include <linux/debugfs.h>
+#include <linux/firmware.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_common.h"
+#include "nitrox_csr.h"
+
+#define CNN55XX_DEV_ID  0x12
+#define MAX_PF_QUEUES   64
+#define UCODE_HLEN 48
+#define SE_GROUP 0
+
+#define DRIVER_VERSION "1.0"
+/* SE microcode */
+#define SE_FW   "cnn55xx_se.fw"
+
+static const char nitrox_driver_name[] = "CNN55XX";
+
+static LIST_HEAD(ndevlist);
+static DEFINE_MUTEX(devlist_lock);
+static unsigned int num_devices;
+
+/**
+ * nitrox_pci_tbl - PCI Device ID Table
+ */
+static const struct pci_device_id nitrox_pci_tbl[] = {
+        {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
+        /* required last entry */
+        {0, }
+};
+MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
+
+static unsigned int qlen = DEFAULT_CMD_QLEN;
+module_param(qlen, uint, 0644);
+MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
+
+/**
+ * struct ucode - Firmware Header
+ * @id: microcode ID
+ * @version: firmware version
+ * @code_size: code section size
+ * @raz: alignment
+ * @code: code section
+ */
+struct ucode {
+        u8 id;
+        char version[VERSION_LEN - 1];
+        __be32 code_size;
+        u8 raz[12];
+        u64 code[0];
+};
+
+/**
+ * write_to_ucd_unit - Write Firmware to NITROX UCD unit
+ */
+static void write_to_ucd_unit(struct nitrox_device *ndev,
+                              struct ucode *ucode)
+{
+        u32 code_size = be32_to_cpu(ucode->code_size) * 2;
+        u64 offset, data;
+        int i = 0;
+
+        /*
+         * UCD structure
+         *
+         *  -------------
+         *  |    BLK 7  |
+         *  -------------
+         *  |    BLK 6  |
+         *  -------------
+         *  |    ...    |
+         *  -------------
+         *  |    BLK 0  |
+         *  -------------
+         *  Total of 8 blocks, each size 32KB
+         */
+
+        /* set the block number */
+        offset = UCD_UCODE_LOAD_BLOCK_NUM;
+        nitrox_write_csr(ndev, offset, 0);
+
+        code_size = roundup(code_size, 8);
+        while (code_size) {
+                data = ucode->code[i];
+                /* write 8 bytes at a time */
+                offset = UCD_UCODE_LOAD_IDX_DATAX(i);
+                nitrox_write_csr(ndev, offset, data);
+                code_size -= 8;
+                i++;
+        }
+
+        /* put all SE cores in group 0 */
+        offset = POM_GRP_EXECMASKX(SE_GROUP);
+        nitrox_write_csr(ndev, offset, (~0ULL));
+
+        for (i = 0; i < ndev->hw.se_cores; i++) {
+                /*
+                 * write block number and firware length
+                 * bit:<2:0> block number
+                 * bit:3 is set SE uses 32KB microcode
+                 * bit:3 is clear SE uses 64KB microcode
+                 */
+                offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
+                nitrox_write_csr(ndev, offset, 0x8);
+        }
+        usleep_range(300, 400);
+}
+
+static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name)
+{
+        const struct firmware *fw;
+        struct ucode *ucode;
+        int ret;
+
+        dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
+
+        ret = request_firmware(&fw, fw_name, DEV(ndev));
+        if (ret < 0) {
+                dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
+                return ret;
+        }
+
+        ucode = (struct ucode *)fw->data;
+        /* copy the firmware version */
+        memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2));
+        ndev->hw.fw_name[VERSION_LEN - 1] = '\0';
+
+        write_to_ucd_unit(ndev, ucode);
+        release_firmware(fw);
+
+        set_bit(NITROX_UCODE_LOADED, &ndev->status);
+        /* barrier to sync with other cpus */
+        smp_mb__after_atomic();
+        return 0;
+}
+
+/**
+ * nitrox_add_to_devlist - add NITROX device to global device list
+ * @ndev: NITROX device
+ */
+static int nitrox_add_to_devlist(struct nitrox_device *ndev)
+{
+        struct nitrox_device *dev;
+        int ret = 0;
+
+        INIT_LIST_HEAD(&ndev->list);
+        refcount_set(&ndev->refcnt, 1);
+
+        mutex_lock(&devlist_lock);
+        list_for_each_entry(dev, &ndevlist, list) {
+                if (dev == ndev) {
+                        ret = -EEXIST;
+                        goto unlock;
+                }
+        }
+        ndev->idx = num_devices++;
+        list_add_tail(&ndev->list, &ndevlist);
+unlock:
+        mutex_unlock(&devlist_lock);
+        return ret;
+}
+
+/**
+ * nitrox_remove_from_devlist - remove NITROX device from
+ *   global device list
+ * @ndev: NITROX device
+ */
+static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
+{
+        mutex_lock(&devlist_lock);
+        list_del(&ndev->list);
+        num_devices--;
+        mutex_unlock(&devlist_lock);
+}
+
+struct nitrox_device *nitrox_get_first_device(void)
+{
+        struct nitrox_device *ndev = NULL;
+
+        mutex_lock(&devlist_lock);
+        list_for_each_entry(ndev, &ndevlist, list) {
+                if (nitrox_ready(ndev))
+                        break;
+        }
+        mutex_unlock(&devlist_lock);
+        if (!ndev)
+                return NULL;
+
+        refcount_inc(&ndev->refcnt);
+        /* barrier to sync with other cpus */
+        smp_mb__after_atomic();
+        return ndev;
+}
+
+void nitrox_put_device(struct nitrox_device *ndev)
+{
+        if (!ndev)
+                return;
+
+        refcount_dec(&ndev->refcnt);
+        /* barrier to sync with other cpus */
+        smp_mb__after_atomic();
+}
+
+static int nitrox_reset_device(struct pci_dev *pdev)
+{
+        int pos = 0;
+
+        pos = pci_save_state(pdev);
+        if (pos) {
+                dev_err(&pdev->dev, "Failed to save pci state\n");
+                return -ENOMEM;
+        }
+
+        pos = pci_pcie_cap(pdev);
+        if (!pos)
+                return -ENOTTY;
+
+        if (!pci_wait_for_pending_transaction(pdev))
+                dev_err(&pdev->dev, "waiting for pending transaction\n");
+
+        pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
+        msleep(100);
+        pci_restore_state(pdev);
+
+        return 0;
+}
+
+static int nitrox_pf_sw_init(struct nitrox_device *ndev)
+{
+        int err;
+
+        err = nitrox_common_sw_init(ndev);
+        if (err)
+                return err;
+
+        err = nitrox_pf_init_isr(ndev);
+        if (err)
+                nitrox_common_sw_cleanup(ndev);
+
+        return err;
+}
+
+static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
+{
+        nitrox_pf_cleanup_isr(ndev);
+        nitrox_common_sw_cleanup(ndev);
+}
+
+/**
+ * nitrox_bist_check - Check NITORX BIST registers status
+ * @ndev: NITROX device
+ */
+static int nitrox_bist_check(struct nitrox_device *ndev)
+{
+        u64 value = 0;
+        int i;
+
+        for (i = 0; i < NR_CLUSTERS; i++) {
+                value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
+                value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
+        }
+        value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
+        value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
+        value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
+        value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
+        value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
+        value += nitrox_read_csr(ndev, POM_BIST_REG);
+        value += nitrox_read_csr(ndev, BMI_BIST_REG);
+        value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
+        value += nitrox_read_csr(ndev, BMO_BIST_REG);
+        value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
+        value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
+        if (value)
+                return -EIO;
+        return 0;
+}
+
+static void nitrox_get_hwinfo(struct nitrox_device *ndev)
+{
+        union emu_fuse_map emu_fuse;
+        u64 offset;
+        int i;
+
+        for (i = 0; i < NR_CLUSTERS; i++) {
+                u8 dead_cores;
+
+                offset = EMU_FUSE_MAPX(i);
+                emu_fuse.value = nitrox_read_csr(ndev, offset);
+                if (emu_fuse.s.valid) {
+                        dead_cores = hweight32(emu_fuse.s.ae_fuse);
+                        ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores;
+                        dead_cores = hweight16(emu_fuse.s.se_fuse);
+                        ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores;
+                }
+        }
+}
+
+static int nitrox_pf_hw_init(struct nitrox_device *ndev)
+{
+        int err;
+
+        err = nitrox_bist_check(ndev);
+        if (err) {
+                dev_err(&ndev->pdev->dev, "BIST check failed\n");
+                return err;
+        }
+        /* get cores information */
+        nitrox_get_hwinfo(ndev);
+
+        nitrox_config_nps_unit(ndev);
+        nitrox_config_pom_unit(ndev);
+        nitrox_config_efl_unit(ndev);
+        /* configure IO units */
+        nitrox_config_bmi_unit(ndev);
+        nitrox_config_bmo_unit(ndev);
+        /* configure Local Buffer Cache */
+        nitrox_config_lbc_unit(ndev);
+        nitrox_config_rand_unit(ndev);
+
+        /* load firmware on SE cores */
+        err = nitrox_load_fw(ndev, SE_FW);
+        if (err)
+                return err;
+
+        nitrox_config_emu_unit(ndev);
+
+        return 0;
+}
+
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+static int registers_show(struct seq_file *s, void *v)
+{
+        struct nitrox_device *ndev = s->private;
+        u64 offset;
+
+        /* NPS DMA stats */
+        offset = NPS_STATS_PKT_DMA_RD_CNT;
+        seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT  0x%016llx\n",
+                   nitrox_read_csr(ndev, offset));
+        offset = NPS_STATS_PKT_DMA_WR_CNT;
+        seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT  0x%016llx\n",
+                   nitrox_read_csr(ndev, offset));
+
+        /* BMI/BMO stats */
+        offset = BMI_NPS_PKT_CNT;
+        seq_printf(s, "BMI_NPS_PKT_CNT  0x%016llx\n",
+                   nitrox_read_csr(ndev, offset));
+        offset = BMO_NPS_SLC_PKT_CNT;
+        seq_printf(s, "BMO_NPS_PKT_CNT  0x%016llx\n",
+                   nitrox_read_csr(ndev, offset));
+
+        return 0;
+}
+
+static int registers_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, registers_show, inode->i_private);
+}
+
+static const struct file_operations register_fops = {
+        .owner = THIS_MODULE,
+        .open = registers_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+
+static int firmware_show(struct seq_file *s, void *v)
+{
+        struct nitrox_device *ndev = s->private;
+
+        seq_printf(s, "Version: %s\n", ndev->hw.fw_name);
+        return 0;
+}
+
+static int firmware_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, firmware_show, inode->i_private);
+}
+
+static const struct file_operations firmware_fops = {
+        .owner = THIS_MODULE,
+        .open = firmware_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+
+static int nitrox_show(struct seq_file *s, void *v)
+{
+        struct nitrox_device *ndev = s->private;
+
+        seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx);
+        seq_printf(s, "  Revision ID: 0x%0x\n", ndev->hw.revision_id);
+        seq_printf(s, "  Cores [AE: %u  SE: %u]\n",
+                   ndev->hw.ae_cores, ndev->hw.se_cores);
+        seq_printf(s, "  Number of Queues: %u\n", ndev->nr_queues);
+        seq_printf(s, "  Queue length: %u\n", ndev->qlen);
+        seq_printf(s, "  Node: %u\n", ndev->node);
+
+        return 0;
+}
+
+static int nitrox_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, nitrox_show, inode->i_private);
+}
+
+static const struct file_operations nitrox_fops = {
+        .owner = THIS_MODULE,
+        .open = nitrox_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+
+static void nitrox_debugfs_exit(struct nitrox_device *ndev)
+{
+        debugfs_remove_recursive(ndev->debugfs_dir);
+        ndev->debugfs_dir = NULL;
+}
+
+static int nitrox_debugfs_init(struct nitrox_device *ndev)
+{
+        struct dentry *dir, *f;
+
+        dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
+        if (!dir)
+                return -ENOMEM;
+
+        ndev->debugfs_dir = dir;
+        f = debugfs_create_file("counters", 0400, dir, ndev, &register_fops);
+        if (!f)
+                goto err;
+        f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops);
+        if (!f)
+                goto err;
+        f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops);
+        if (!f)
+                goto err;
+
+        return 0;
+
+err:
+        nitrox_debugfs_exit(ndev);
+        return -ENODEV;
+}
+#else
+static int nitrox_debugfs_init(struct nitrox_device *ndev)
+{
+        return 0;
+}
+
+static void nitrox_debugfs_exit(struct nitrox_device *ndev)
+{
+}
+#endif
+
+/**
+ * nitrox_probe - NITROX Initialization function.
+ * @pdev: PCI device information struct
+ * @id: entry in nitrox_pci_tbl
+ *
+ * Return: 0, if the driver is bound to the device, or
+ *         a negative error if there is failure.
+ */
+static int nitrox_probe(struct pci_dev *pdev,
+                        const struct pci_device_id *id)
+{
+        struct nitrox_device *ndev;
+        int err;
+
+        dev_info_once(&pdev->dev, "%s driver version %s\n",
+                      nitrox_driver_name, DRIVER_VERSION);
+
+        err = pci_enable_device_mem(pdev);
+        if (err)
+                return err;
+
+        /* do FLR */
+        err = nitrox_reset_device(pdev);
+        if (err) {
+                dev_err(&pdev->dev, "FLR failed\n");
+                pci_disable_device(pdev);
+                return err;
+        }
+
+        if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
+                dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
+        } else {
+                err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+                if (err) {
+                        dev_err(&pdev->dev, "DMA configuration failed\n");
+                        pci_disable_device(pdev);
+                        return err;
+                }
+        }
+
+        err = pci_request_mem_regions(pdev, nitrox_driver_name);
+        if (err) {
+                pci_disable_device(pdev);
+                return err;
+        }
+        pci_set_master(pdev);
+
+        ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
+        if (!ndev)
+                goto ndev_fail;
+
+        pci_set_drvdata(pdev, ndev);
+        ndev->pdev = pdev;
+
+        /* add to device list */
+        nitrox_add_to_devlist(ndev);
+
+        ndev->hw.vendor_id = pdev->vendor;
+        ndev->hw.device_id = pdev->device;
+        ndev->hw.revision_id = pdev->revision;
+        /* command timeout in jiffies */
+        ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
+        ndev->node = dev_to_node(&pdev->dev);
+        if (ndev->node == NUMA_NO_NODE)
+                ndev->node = 0;
+
+        ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
+                                 pci_resource_len(pdev, 0));
+        if (!ndev->bar_addr) {
+                err = -EIO;
+                goto ioremap_err;
+        }
+        /* allocate command queus based on cpus, max queues are 64 */
+        ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
+        ndev->qlen = qlen;
+
+        err = nitrox_pf_sw_init(ndev);
+        if (err)
+                goto ioremap_err;
+
+        err = nitrox_pf_hw_init(ndev);
+        if (err)
+                goto pf_hw_fail;
+
+        err = nitrox_debugfs_init(ndev);
+        if (err)
+                goto pf_hw_fail;
+
+        set_bit(NITROX_READY, &ndev->status);
+        /* barrier to sync with other cpus */
+        smp_mb__after_atomic();
+
+        err = nitrox_crypto_register();
+        if (err)
+                goto crypto_fail;
+
+        return 0;
+
+crypto_fail:
+        nitrox_debugfs_exit(ndev);
+        clear_bit(NITROX_READY, &ndev->status);
+        /* barrier to sync with other cpus */
+        smp_mb__after_atomic();
+pf_hw_fail:
+        nitrox_pf_sw_cleanup(ndev);
+ioremap_err:
+        nitrox_remove_from_devlist(ndev);
+        kfree(ndev);
+        pci_set_drvdata(pdev, NULL);
+ndev_fail:
+        pci_release_mem_regions(pdev);
+        pci_disable_device(pdev);
+        return err;
+}
+
+/**
+ * nitrox_remove - Unbind the driver from the device.
+ * @pdev: PCI device information struct
+ */
+static void nitrox_remove(struct pci_dev *pdev)
+{
+        struct nitrox_device *ndev = pci_get_drvdata(pdev);
+
+        if (!ndev)
+                return;
+
+        if (!refcount_dec_and_test(&ndev->refcnt)) {
+                dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
+                        refcount_read(&ndev->refcnt));
+                return;
+        }
+
+        dev_info(DEV(ndev), "Removing Device %x:%x\n",
+                 ndev->hw.vendor_id, ndev->hw.device_id);
+
+        clear_bit(NITROX_READY, &ndev->status);
+        /* barrier to sync with other cpus */
+        smp_mb__after_atomic();
+
+        nitrox_remove_from_devlist(ndev);
+        nitrox_crypto_unregister();
+        nitrox_debugfs_exit(ndev);
+        nitrox_pf_sw_cleanup(ndev);
+
+        iounmap(ndev->bar_addr);
+        kfree(ndev);
+
+        pci_set_drvdata(pdev, NULL);
+        pci_release_mem_regions(pdev);
+        pci_disable_device(pdev);
+}
+
+static void nitrox_shutdown(struct pci_dev *pdev)
+{
+        pci_set_drvdata(pdev, NULL);
+        pci_release_mem_regions(pdev);
+        pci_disable_device(pdev);
+}
+
+static struct pci_driver nitrox_driver = {
+        .name = nitrox_driver_name,
+        .id_table = nitrox_pci_tbl,
+        .probe = nitrox_probe,
+        .remove = nitrox_remove,
+        .shutdown = nitrox_shutdown,
+};
+
+module_pci_driver(nitrox_driver);
+
+MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
+MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_FIRMWARE(SE_FW);
diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h
new file mode 100644
index 000000000000..74f4c20dc87d
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_req.h
@@ -0,0 +1,445 @@
+#ifndef __NITROX_REQ_H
+#define __NITROX_REQ_H
+
+#include <linux/dma-mapping.h>
+#include <crypto/aes.h>
+
+#include "nitrox_dev.h"
+
+/**
+ * struct gphdr - General purpose Header
+ * @param0: first parameter.
+ * @param1: second parameter.
+ * @param2: third parameter.
+ * @param3: fourth parameter.
+ *
+ * Params tell the iv and enc/dec data offsets.
+ */
+struct gphdr {
+        __be16 param0;
+        __be16 param1;
+        __be16 param2;
+        __be16 param3;
+};
+
+/**
+ * struct se_req_ctrl - SE request information.
+ * @arg: Minor number of the opcode
+ * @ctxc: Context control.
+ * @unca: Uncertainity enabled.
+ * @info: Additional information for SE cores.
+ * @ctxl: Context length in bytes.
+ * @uddl: User defined data length
+ */
+union se_req_ctrl {
+        u64 value;
+        struct {
+                u64 raz : 22;
+                u64 arg : 8;
+                u64 ctxc : 2;
+                u64 unca : 1;
+                u64 info : 3;
+                u64 unc : 8;
+                u64 ctxl : 12;
+                u64 uddl : 8;
+        } s;
+};
+
+struct nitrox_sglist {
+        u16 len;
+        u16 raz0;
+        u32 raz1;
+        dma_addr_t dma;
+};
+
+#define MAX_IV_LEN 16
+
+/**
+ * struct se_crypto_request - SE crypto request structure.
+ * @opcode: Request opcode (enc/dec)
+ * @flags: flags from crypto subsystem
+ * @ctx_handle: Crypto context handle.
+ * @gph: GP Header
+ * @ctrl: Request Information.
+ * @in: Input sglist
+ * @out: Output sglist
+ */
+struct se_crypto_request {
+        u8 opcode;
+        gfp_t gfp;
+        u32 flags;
+        u64 ctx_handle;
+
+        struct gphdr gph;
+        union se_req_ctrl ctrl;
+
+        u8 iv[MAX_IV_LEN];
+        u16 ivsize;
+
+        struct scatterlist *src;
+        struct scatterlist *dst;
+};
+
+/* Crypto opcodes */
+#define FLEXI_CRYPTO_ENCRYPT_HMAC       0x33
+#define ENCRYPT 0
+#define DECRYPT 1
+
+/* IV from context */
+#define IV_FROM_CTX     0
+/* IV from Input data */
+#define IV_FROM_DPTR    1
+
+/**
+ * cipher opcodes for firmware
+ */
+enum flexi_cipher {
+        CIPHER_NULL = 0,
+        CIPHER_3DES_CBC,
+        CIPHER_3DES_ECB,
+        CIPHER_AES_CBC,
+        CIPHER_AES_ECB,
+        CIPHER_AES_CFB,
+        CIPHER_AES_CTR,
+        CIPHER_AES_GCM,
+        CIPHER_AES_XTS,
+        CIPHER_AES_CCM,
+        CIPHER_AES_CBC_CTS,
+        CIPHER_AES_ECB_CTS,
+        CIPHER_INVALID
+};
+
+/**
+ * struct crypto_keys - Crypto keys
+ * @key: Encryption key or KEY1 for AES-XTS
+ * @iv: Encryption IV or Tweak for AES-XTS
+ */
+struct crypto_keys {
+        union {
+                u8 key[AES_MAX_KEY_SIZE];
+                u8 key1[AES_MAX_KEY_SIZE];
+        } u;
+        u8 iv[AES_BLOCK_SIZE];
+};
+
+/**
+ * struct auth_keys - Authentication keys
+ * @ipad: IPAD or KEY2 for AES-XTS
+ * @opad: OPAD or AUTH KEY if auth_input_type = 1
+ */
+struct auth_keys {
+        union {
+                u8 ipad[64];
+                u8 key2[64];
+        } u;
+        u8 opad[64];
+};
+
+/**
+ * struct flexi_crypto_context - Crypto context
+ * @cipher_type: Encryption cipher type
+ * @aes_keylen: AES key length
+ * @iv_source: Encryption IV source
+ * @hash_type: Authentication type
+ * @auth_input_type: Authentication input type
+ *   1 - Authentication IV and KEY, microcode calculates OPAD/IPAD
+ *   0 - Authentication OPAD/IPAD
+ * @mac_len: mac length
+ * @crypto: Crypto keys
+ * @auth: Authentication keys
+ */
+struct flexi_crypto_context {
+        union {
+                __be64 flags;
+                struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+                        u64 cipher_type : 4;
+                        u64 reserved_59 : 1;
+                        u64 aes_keylen : 2;
+                        u64 iv_source : 1;
+                        u64 hash_type : 4;
+                        u64 reserved_49_51 : 3;
+                        u64 auth_input_type: 1;
+                        u64 mac_len : 8;
+                        u64 reserved_0_39 : 40;
+#else
+                        u64 reserved_0_39 : 40;
+                        u64 mac_len : 8;
+                        u64 auth_input_type: 1;
+                        u64 reserved_49_51 : 3;
+                        u64 hash_type : 4;
+                        u64 iv_source : 1;
+                        u64 aes_keylen : 2;
+                        u64 reserved_59 : 1;
+                        u64 cipher_type : 4;
+#endif
+                } w0;
+        };
+
+        struct crypto_keys crypto;
+        struct auth_keys auth;
+};
+
+struct nitrox_crypto_ctx {
+        struct nitrox_device *ndev;
+        union {
+                u64 ctx_handle;
+                struct flexi_crypto_context *fctx;
+        } u;
+};
+
+struct nitrox_kcrypt_request {
+        struct se_crypto_request creq;
+        struct nitrox_crypto_ctx *nctx;
+        struct skcipher_request *skreq;
+};
+
+/**
+ * struct pkt_instr_hdr - Packet Instruction Header
+ * @g: Gather used
+ *   When [G] is set and [GSZ] != 0, the instruction is
+ *   indirect gather instruction.
+ *   When [G] is set and [GSZ] = 0, the instruction is
+ *   direct gather instruction.
+ * @gsz: Number of pointers in the indirect gather list
+ * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr
+ *   and adds them to the packet after the pkt_instr_hdr but before any UDD
+ * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ]
+ *   when [IHI] is set.
+ * @fsz: The number of front data bytes directly included in the
+ *   PCIe instruction.
+ * @tlen: The length of the input packet in bytes, include:
+ *   - 16B pkt_hdr
+ *   - Inline context bytes if any,
+ *   - UDD if any,
+ *   - packet payload bytes
+ */
+union pkt_instr_hdr {
+        u64 value;
+        struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+                u64 raz_48_63 : 16;
+                u64 g : 1;
+                u64 gsz : 7;
+                u64 ihi : 1;
+                u64 ssz : 7;
+                u64 raz_30_31 : 2;
+                u64 fsz : 6;
+                u64 raz_16_23 : 8;
+                u64 tlen : 16;
+#else
+                u64 tlen : 16;
+                u64 raz_16_23 : 8;
+                u64 fsz : 6;
+                u64 raz_30_31 : 2;
+                u64 ssz : 7;
+                u64 ihi : 1;
+                u64 gsz : 7;
+                u64 g : 1;
+                u64 raz_48_63 : 16;
+#endif
+        } s;
+};
+
+/**
+ * struct pkt_hdr - Packet Input Header
+ * @opcode: Request opcode (Major)
+ * @arg: Request opcode (Minor)
+ * @ctxc: Context control.
+ * @unca: When set [UNC] is the uncertainty count for an input packet.
+ *        The hardware uses uncertainty counts to predict
+ *        output buffer use and avoid deadlock.
+ * @info: Not used by input hardware. Available for use
+ *        during SE processing.
+ * @destport: The expected destination port/ring/channel for the packet.
+ * @unc: Uncertainty count for an input packet.
+ * @grp: SE group that will process the input packet.
+ * @ctxl: Context Length in 64-bit words.
+ * @uddl: User-defined data (UDD) length in bytes.
+ * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases.
+ */
+union pkt_hdr {
+        u64 value[2];
+        struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+                u64 opcode : 8;
+                u64 arg : 8;
+                u64 ctxc : 2;
+                u64 unca : 1;
+                u64 raz_44 : 1;
+                u64 info : 3;
+                u64 destport : 9;
+                u64 unc : 8;
+                u64 raz_19_23 : 5;
+                u64 grp : 3;
+                u64 raz_15 : 1;
+                u64 ctxl : 7;
+                u64 uddl : 8;
+#else
+                u64 uddl : 8;
+                u64 ctxl : 7;
+                u64 raz_15 : 1;
+                u64 grp : 3;
+                u64 raz_19_23 : 5;
+                u64 unc : 8;
+                u64 destport : 9;
+                u64 info : 3;
+                u64 raz_44 : 1;
+                u64 unca : 1;
+                u64 ctxc : 2;
+                u64 arg : 8;
+                u64 opcode : 8;
+#endif
+                __be64 ctxp;
+        } s;
+};
+
+/**
+ * struct slc_store_info - Solicited Paceket Output Store Information.
+ * @ssz: The number of scatterlist pointers for the solicited output port
+ *       packet.
+ * @rptr: The result pointer for the solicited output port packet.
+ *        If [SSZ]=0, [RPTR] must point directly to a buffer on the remote
+ *        host that is large enough to hold the entire output packet.
+ *        If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4
+ *        sglist components at [RPTR] on the remote host.
+ */
+union slc_store_info {
+        u64 value[2];
+        struct {
+#if defined(__BIG_ENDIAN_BITFIELD)
+                u64 raz_39_63 : 25;
+                u64 ssz : 7;
+                u64 raz_0_31 : 32;
+#else
+                u64 raz_0_31 : 32;
+                u64 ssz : 7;
+                u64 raz_39_63 : 25;
+#endif
+                __be64 rptr;
+        } s;
+};
+
+/**
+ * struct nps_pkt_instr - NPS Packet Instruction of SE cores.
+ * @dptr0 : Input pointer points to buffer in remote host.
+ * @ih: Packet Instruction Header (8 bytes)
+ * @irh: Packet Input Header (16 bytes)
+ * @slc: Solicited Packet Output Store Information (16 bytes)
+ * @fdata: Front data
+ *
+ * 64-Byte Instruction Format
+ */
+struct nps_pkt_instr {
+        __be64 dptr0;
+        union pkt_instr_hdr ih;
+        union pkt_hdr irh;
+        union slc_store_info slc;
+        u64 fdata[2];
+};
+
+/**
+ * struct ctx_hdr - Book keeping data about the crypto context
+ * @pool: Pool used to allocate crypto context
+ * @dma: Base DMA address of the cypto context
+ * @ctx_dma: Actual usable crypto context for NITROX
+ */
+struct ctx_hdr {
+        struct dma_pool *pool;
+        dma_addr_t dma;
+        dma_addr_t ctx_dma;
+};
+
+/*
+ * struct sglist_component - SG list component format
+ * @len0: The number of bytes at [PTR0] on the remote host.
+ * @len1: The number of bytes at [PTR1] on the remote host.
+ * @len2: The number of bytes at [PTR2] on the remote host.
+ * @len3: The number of bytes at [PTR3] on the remote host.
+ * @dma0: First pointer point to buffer in remote host.
+ * @dma1: Second pointer point to buffer in remote host.
+ * @dma2: Third pointer point to buffer in remote host.
+ * @dma3: Fourth pointer point to buffer in remote host.
+ */
+struct nitrox_sgcomp {
+        __be16 len[4];
+        __be64 dma[4];
+};
+
+/*
+ * strutct nitrox_sgtable - SG list information
+ * @map_cnt: Number of buffers mapped
+ * @nr_comp: Number of sglist components
+ * @total_bytes: Total bytes in sglist.
+ * @len: Total sglist components length.
+ * @dma: DMA address of sglist component.
+ * @dir: DMA direction.
+ * @buf: crypto request buffer.
+ * @sglist: SG list of input/output buffers.
+ * @sgcomp: sglist component for NITROX.
+ */
+struct nitrox_sgtable {
+        u8 map_bufs_cnt;
+        u8 nr_sgcomp;
+        u16 total_bytes;
+        u32 len;
+        dma_addr_t dma;
+        enum dma_data_direction dir;
+
+        struct scatterlist *buf;
+        struct nitrox_sglist *sglist;
+        struct nitrox_sgcomp *sgcomp;
+};
+
+/* Response Header Length */
+#define ORH_HLEN        8
+/* Completion bytes Length */
+#define COMP_HLEN       8
+
+struct resp_hdr {
+        u64 orh;
+        dma_addr_t orh_dma;
+        u64 completion;
+        dma_addr_t completion_dma;
+};
+
+typedef void (*completion_t)(struct skcipher_request *skreq, int err);
+
+/**
+ * struct nitrox_softreq - Represents the NIROX Request.
+ * @response: response list entry
+ * @backlog: Backlog list entry
+ * @ndev: Device used to submit the request
+ * @cmdq: Command queue for submission
+ * @resp: Response headers
+ * @instr: 64B instruction
+ * @in: SG table for input
+ * @out SG table for output
+ * @tstamp: Request submitted time in jiffies
+ * @callback: callback after request completion/timeout
+ * @cb_arg: callback argument
+ */
+struct nitrox_softreq {
+        struct list_head response;
+        struct list_head backlog;
+
+        u32 flags;
+        gfp_t gfp;
+        atomic_t status;
+        bool inplace;
+
+        struct nitrox_device *ndev;
+        struct nitrox_cmdq *cmdq;
+
+        struct nps_pkt_instr instr;
+        struct resp_hdr resp;
+        struct nitrox_sgtable in;
+        struct nitrox_sgtable out;
+
+        unsigned long tstamp;
+
+        completion_t callback;
+        struct skcipher_request *skreq;
+};
+
+#endif /* __NITROX_REQ_H */
diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
new file mode 100644
index 000000000000..4bb4377c5ac0
--- /dev/null
+++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
@@ -0,0 +1,735 @@
+#include <linux/gfp.h>
+#include <linux/workqueue.h>
+#include <crypto/internal/skcipher.h>
+
+#include "nitrox_dev.h"
+#include "nitrox_req.h"
+#include "nitrox_csr.h"
+#include "nitrox_req.h"
+
+/* SLC_STORE_INFO */
+#define MIN_UDD_LEN 16
+/* PKT_IN_HDR + SLC_STORE_INFO */
+#define FDATA_SIZE 32
+/* Base destination port for the solicited requests */
+#define SOLICIT_BASE_DPORT 256
+#define PENDING_SIG     0xFFFFFFFFFFFFFFFFUL
+
+#define REQ_NOT_POSTED 1
+#define REQ_BACKLOG    2
+#define REQ_POSTED     3
+
+/**
+ * Response codes from SE microcode
+ * 0x00 - Success
+ *   Completion with no error
+ * 0x43 - ERR_GC_DATA_LEN_INVALID
+ *   Invalid Data length if Encryption Data length is
+ *   less than 16 bytes for AES-XTS and AES-CTS.
+ * 0x45 - ERR_GC_CTX_LEN_INVALID
+ *   Invalid context length: CTXL != 23 words.
+ * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID
+ *   DOCSIS support is enabled with other than
+ *   AES/DES-CBC mode encryption.
+ * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID
+ *   Authentication offset is other than 0 with
+ *   Encryption IV source = 0.
+ *   Authentication offset is other than 8 (DES)/16 (AES)
+ *   with Encryption IV source = 1
+ * 0x51 - ERR_GC_CRC32_INVALID_SELECTION
+ *   CRC32 is enabled for other than DOCSIS encryption.
+ * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID
+ *   Invalid flag options in AES-CCM IV.
+ */
+
+/**
+ * dma_free_sglist - unmap and free the sg lists.
+ * @ndev: N5 device
+ * @sgtbl: SG table
+ */
+static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
+{
+        struct nitrox_device *ndev = sr->ndev;
+        struct device *dev = DEV(ndev);
+        struct nitrox_sglist *sglist;
+
+        /* unmap in sgbuf */
+        sglist = sr->in.sglist;
+        if (!sglist)
+                goto out_unmap;
+
+        /* unmap iv */
+        dma_unmap_single(dev, sglist->dma, sglist->len, DMA_BIDIRECTIONAL);
+        /* unmpa src sglist */
+        dma_unmap_sg(dev, sr->in.buf, (sr->in.map_bufs_cnt - 1), sr->in.dir);
+        /* unamp gather component */
+        dma_unmap_single(dev, sr->in.dma, sr->in.len, DMA_TO_DEVICE);
+        kfree(sr->in.sglist);
+        kfree(sr->in.sgcomp);
+        sr->in.sglist = NULL;
+        sr->in.buf = NULL;
+        sr->in.map_bufs_cnt = 0;
+
+out_unmap:
+        /* unmap out sgbuf */
+        sglist = sr->out.sglist;
+        if (!sglist)
+                return;
+
+        /* unmap orh */
+        dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
+
+        /* unmap dst sglist */
+        if (!sr->inplace) {
+                dma_unmap_sg(dev, sr->out.buf, (sr->out.map_bufs_cnt - 3),
+                             sr->out.dir);
+        }
+        /* unmap completion */
+        dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
+
+        /* unmap scatter component */
+        dma_unmap_single(dev, sr->out.dma, sr->out.len, DMA_TO_DEVICE);
+        kfree(sr->out.sglist);
+        kfree(sr->out.sgcomp);
+        sr->out.sglist = NULL;
+        sr->out.buf = NULL;
+        sr->out.map_bufs_cnt = 0;
+}
+
+static void softreq_destroy(struct nitrox_softreq *sr)
+{
+        softreq_unmap_sgbufs(sr);
+        kfree(sr);
+}
+
+/**
+ * create_sg_component - create SG componets for N5 device.
+ * @sr: Request structure
+ * @sgtbl: SG table
+ * @nr_comp: total number of components required
+ *
+ * Component structure
+ *
+ *   63     48 47     32 31    16 15      0
+ *   --------------------------------------
+ *   |   LEN0  |  LEN1  |  LEN2  |  LEN3  |
+ *   |-------------------------------------
+ *   |               PTR0                 |
+ *   --------------------------------------
+ *   |               PTR1                 |
+ *   --------------------------------------
+ *   |               PTR2                 |
+ *   --------------------------------------
+ *   |               PTR3                 |
+ *   --------------------------------------
+ *
+ *   Returns 0 if success or a negative errno code on error.
+ */
+static int create_sg_component(struct nitrox_softreq *sr,
+                               struct nitrox_sgtable *sgtbl, int map_nents)
+{
+        struct nitrox_device *ndev = sr->ndev;
+        struct nitrox_sgcomp *sgcomp;
+        struct nitrox_sglist *sglist;
+        dma_addr_t dma;
+        size_t sz_comp;
+        int i, j, nr_sgcomp;
+
+        nr_sgcomp = roundup(map_nents, 4) / 4;
+
+        /* each component holds 4 dma pointers */
+        sz_comp = nr_sgcomp * sizeof(*sgcomp);
+        sgcomp = kzalloc(sz_comp, sr->gfp);
+        if (!sgcomp)
+                return -ENOMEM;
+
+        sgtbl->sgcomp = sgcomp;
+        sgtbl->nr_sgcomp = nr_sgcomp;
+
+        sglist = sgtbl->sglist;
+        /* populate device sg component */
+        for (i = 0; i < nr_sgcomp; i++) {
+                for (j = 0; j < 4; j++) {
+                        sgcomp->len[j] = cpu_to_be16(sglist->len);
+                        sgcomp->dma[j] = cpu_to_be64(sglist->dma);
+                        sglist++;
+                }
+                sgcomp++;
+        }
+        /* map the device sg component */
+        dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
+        if (dma_mapping_error(DEV(ndev), dma)) {
+                kfree(sgtbl->sgcomp);
+                sgtbl->sgcomp = NULL;
+                return -ENOMEM;
+        }
+
+        sgtbl->dma = dma;
+        sgtbl->len = sz_comp;
+
+        return 0;
+}
+
+/**
+ * dma_map_inbufs - DMA map input sglist and creates sglist component
+ *                  for N5 device.
+ * @sr: Request structure
+ * @req: Crypto request structre
+ *
+ * Returns 0 if successful or a negative errno code on error.
+ */
+static int dma_map_inbufs(struct nitrox_softreq *sr,
+                          struct se_crypto_request *req)
+{
+        struct device *dev = DEV(sr->ndev);
+        struct scatterlist *sg = req->src;
+        struct nitrox_sglist *glist;
+        int i, nents, ret = 0;
+        dma_addr_t dma;
+        size_t sz;
+
+        nents = sg_nents(req->src);
+
+        /* creater gather list IV and src entries */
+        sz = roundup((1 + nents), 4) * sizeof(*glist);
+        glist = kzalloc(sz, sr->gfp);
+        if (!glist)
+                return -ENOMEM;
+
+        sr->in.sglist = glist;
+        /* map IV */
+        dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL);
+        if (dma_mapping_error(dev, dma)) {
+                ret = -EINVAL;
+                goto iv_map_err;
+        }
+
+        sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+        /* map src entries */
+        nents = dma_map_sg(dev, req->src, nents, sr->in.dir);
+        if (!nents) {
+                ret = -EINVAL;
+                goto src_map_err;
+        }
+        sr->in.buf = req->src;
+
+        /* store the mappings */
+        glist->len = req->ivsize;
+        glist->dma = dma;
+        glist++;
+        sr->in.total_bytes += req->ivsize;
+
+        for_each_sg(req->src, sg, nents, i) {
+                glist->len = sg_dma_len(sg);
+                glist->dma = sg_dma_address(sg);
+                sr->in.total_bytes += glist->len;
+                glist++;
+        }
+        /* roundup map count to align with entires in sg component */
+        sr->in.map_bufs_cnt = (1 + nents);
+
+        /* create NITROX gather component */
+        ret = create_sg_component(sr, &sr->in, sr->in.map_bufs_cnt);
+        if (ret)
+                goto incomp_err;
+
+        return 0;
+
+incomp_err:
+        dma_unmap_sg(dev, req->src, nents, sr->in.dir);
+        sr->in.map_bufs_cnt = 0;
+src_map_err:
+        dma_unmap_single(dev, dma, req->ivsize, DMA_BIDIRECTIONAL);
+iv_map_err:
+        kfree(sr->in.sglist);
+        sr->in.sglist = NULL;
+        return ret;
+}
+
+static int dma_map_outbufs(struct nitrox_softreq *sr,
+                           struct se_crypto_request *req)
+{
+        struct device *dev = DEV(sr->ndev);
+        struct nitrox_sglist *glist = sr->in.sglist;
+        struct nitrox_sglist *slist;
+        struct scatterlist *sg;
+        int i, nents, map_bufs_cnt, ret = 0;
+        size_t sz;
+
+        nents = sg_nents(req->dst);
+
+        /* create scatter list ORH, IV, dst entries and Completion header */
+        sz = roundup((3 + nents), 4) * sizeof(*slist);
+        slist = kzalloc(sz, sr->gfp);
+        if (!slist)
+                return -ENOMEM;
+
+        sr->out.sglist = slist;
+        sr->out.dir = DMA_BIDIRECTIONAL;
+        /* map ORH */
+        sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN,
+                                          sr->out.dir);
+        if (dma_mapping_error(dev, sr->resp.orh_dma)) {
+                ret = -EINVAL;
+                goto orh_map_err;
+        }
+
+        /* map completion */
+        sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion,
+                                                 COMP_HLEN, sr->out.dir);
+        if (dma_mapping_error(dev, sr->resp.completion_dma)) {
+                ret = -EINVAL;
+                goto compl_map_err;
+        }
+
+        sr->inplace = (req->src == req->dst) ? true : false;
+        /* out place */
+        if (!sr->inplace) {
+                nents = dma_map_sg(dev, req->dst, nents, sr->out.dir);
+                if (!nents) {
+                        ret = -EINVAL;
+                        goto dst_map_err;
+                }
+        }
+        sr->out.buf = req->dst;
+
+        /* store the mappings */
+        /* orh */
+        slist->len = ORH_HLEN;
+        slist->dma = sr->resp.orh_dma;
+        slist++;
+
+        /* copy the glist mappings */
+        if (sr->inplace) {
+                nents = sr->in.map_bufs_cnt - 1;
+                map_bufs_cnt = sr->in.map_bufs_cnt;
+                while (map_bufs_cnt--) {
+                        slist->len = glist->len;
+                        slist->dma = glist->dma;
+                        slist++;
+                        glist++;
+                }
+        } else {
+                /* copy iv mapping */
+                slist->len = glist->len;
+                slist->dma = glist->dma;
+                slist++;
+                /* copy remaining maps */
+                for_each_sg(req->dst, sg, nents, i) {
+                        slist->len = sg_dma_len(sg);
+                        slist->dma = sg_dma_address(sg);
+                        slist++;
+                }
+        }
+
+        /* completion */
+        slist->len = COMP_HLEN;
+        slist->dma = sr->resp.completion_dma;
+
+        sr->out.map_bufs_cnt = (3 + nents);
+
+        ret = create_sg_component(sr, &sr->out, sr->out.map_bufs_cnt);
+        if (ret)
+                goto outcomp_map_err;
+
+        return 0;
+
+outcomp_map_err:
+        if (!sr->inplace)
+                dma_unmap_sg(dev, req->dst, nents, sr->out.dir);
+        sr->out.map_bufs_cnt = 0;
+        sr->out.buf = NULL;
+dst_map_err:
+        dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
+        sr->resp.completion_dma = 0;
+compl_map_err:
+        dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
+        sr->resp.orh_dma = 0;
+orh_map_err:
+        kfree(sr->out.sglist);
+        sr->out.sglist = NULL;
+        return ret;
+}
+
+static inline int softreq_map_iobuf(struct nitrox_softreq *sr,
+                                    struct se_crypto_request *creq)
+{
+        int ret;
+
+        ret = dma_map_inbufs(sr, creq);
+        if (ret)
+                return ret;
+
+        ret = dma_map_outbufs(sr, creq);
+        if (ret)
+                softreq_unmap_sgbufs(sr);
+
+        return ret;
+}
+
+static inline void backlog_list_add(struct nitrox_softreq *sr,
+                                    struct nitrox_cmdq *cmdq)
+{
+        INIT_LIST_HEAD(&sr->backlog);
+
+        spin_lock_bh(&cmdq->backlog_lock);
+        list_add_tail(&sr->backlog, &cmdq->backlog_head);
+        atomic_inc(&cmdq->backlog_count);
+        atomic_set(&sr->status, REQ_BACKLOG);
+        spin_unlock_bh(&cmdq->backlog_lock);
+}
+
+static inline void response_list_add(struct nitrox_softreq *sr,
+                                     struct nitrox_cmdq *cmdq)
+{
+        INIT_LIST_HEAD(&sr->response);
+
+        spin_lock_bh(&cmdq->response_lock);
+        list_add_tail(&sr->response, &cmdq->response_head);
+        spin_unlock_bh(&cmdq->response_lock);
+}
+
+static inline void response_list_del(struct nitrox_softreq *sr,
+                                     struct nitrox_cmdq *cmdq)
+{
+        spin_lock_bh(&cmdq->response_lock);
+        list_del(&sr->response);
+        spin_unlock_bh(&cmdq->response_lock);
+}
+
+static struct nitrox_softreq *
+get_first_response_entry(struct nitrox_cmdq *cmdq)
+{
+        return list_first_entry_or_null(&cmdq->response_head,
+                                        struct nitrox_softreq, response);
+}
+
+static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen)
+{
+        if (atomic_inc_return(&cmdq->pending_count) > qlen) {
+                atomic_dec(&cmdq->pending_count);
+                /* sync with other cpus */
+                smp_mb__after_atomic();
+                return true;
+        }
+        return false;
+}
+
+/**
+ * post_se_instr - Post SE instruction to Packet Input ring
+ * @sr: Request structure
+ *
+ * Returns 0 if successful or a negative error code,
+ * if no space in ring.
+ */
+static void post_se_instr(struct nitrox_softreq *sr,
+                          struct nitrox_cmdq *cmdq)
+{
+        struct nitrox_device *ndev = sr->ndev;
+        union nps_pkt_in_instr_baoff_dbell pkt_in_baoff_dbell;
+        u64 offset;
+        u8 *ent;
+
+        spin_lock_bh(&cmdq->cmdq_lock);
+
+        /* get the next write offset */
+        offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(cmdq->qno);
+        pkt_in_baoff_dbell.value = nitrox_read_csr(ndev, offset);
+        /* copy the instruction */
+        ent = cmdq->head + pkt_in_baoff_dbell.s.aoff;
+        memcpy(ent, &sr->instr, cmdq->instr_size);
+        /* flush the command queue updates */
+        dma_wmb();
+
+        sr->tstamp = jiffies;
+        atomic_set(&sr->status, REQ_POSTED);
+        response_list_add(sr, cmdq);
+
+        /* Ring doorbell with count 1 */
+        writeq(1, cmdq->dbell_csr_addr);
+        /* orders the doorbell rings */
+        mmiowb();
+
+        spin_unlock_bh(&cmdq->cmdq_lock);
+}
+
+static int post_backlog_cmds(struct nitrox_cmdq *cmdq)
+{
+        struct nitrox_device *ndev = cmdq->ndev;
+        struct nitrox_softreq *sr, *tmp;
+        int ret = 0;
+
+        spin_lock_bh(&cmdq->backlog_lock);
+
+        list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) {
+                struct skcipher_request *skreq;
+
+                /* submit until space available */
+                if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
+                        ret = -EBUSY;
+                        break;
+                }
+                /* delete from backlog list */
+                list_del(&sr->backlog);
+                atomic_dec(&cmdq->backlog_count);
+                /* sync with other cpus */
+                smp_mb__after_atomic();
+
+                skreq = sr->skreq;
+                /* post the command */
+                post_se_instr(sr, cmdq);
+
+                /* backlog requests are posted, wakeup with -EINPROGRESS */
+                skcipher_request_complete(skreq, -EINPROGRESS);
+        }
+        spin_unlock_bh(&cmdq->backlog_lock);
+
+        return ret;
+}
+
+static int nitrox_enqueue_request(struct nitrox_softreq *sr)
+{
+        struct nitrox_cmdq *cmdq = sr->cmdq;
+        struct nitrox_device *ndev = sr->ndev;
+        int ret = -EBUSY;
+
+        if (unlikely(cmdq_full(cmdq, ndev->qlen))) {
+                if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+                        return -EAGAIN;
+
+                backlog_list_add(sr, cmdq);
+        } else {
+                ret = post_backlog_cmds(cmdq);
+                if (ret) {
+                        backlog_list_add(sr, cmdq);
+                        return ret;
+                }
+                post_se_instr(sr, cmdq);
+                ret = -EINPROGRESS;
+        }
+        return ret;
+}
+
+/**
+ * nitrox_se_request - Send request to SE core
+ * @ndev: NITROX device
+ * @req: Crypto request
+ *
+ * Returns 0 on success, or a negative error code.
+ */
+int nitrox_process_se_request(struct nitrox_device *ndev,
+                              struct se_crypto_request *req,
+                              completion_t callback,
+                              struct skcipher_request *skreq)
+{
+        struct nitrox_softreq *sr;
+        dma_addr_t ctx_handle = 0;
+        int qno, ret = 0;
+
+        if (!nitrox_ready(ndev))
+                return -ENODEV;
+
+        sr = kzalloc(sizeof(*sr), req->gfp);
+        if (!sr)
+                return -ENOMEM;
+
+        sr->ndev = ndev;
+        sr->flags = req->flags;
+        sr->gfp = req->gfp;
+        sr->callback = callback;
+        sr->skreq = skreq;
+
+        atomic_set(&sr->status, REQ_NOT_POSTED);
+
+        WRITE_ONCE(sr->resp.orh, PENDING_SIG);
+        WRITE_ONCE(sr->resp.completion, PENDING_SIG);
+
+        ret = softreq_map_iobuf(sr, req);
+        if (ret) {
+                kfree(sr);
+                return ret;
+        }
+
+        /* get the context handle */
+        if (req->ctx_handle) {
+                struct ctx_hdr *hdr;
+                u8 *ctx_ptr;
+
+                ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle;
+                hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr));
+                ctx_handle = hdr->ctx_dma;
+        }
+
+        /* select the queue */
+        qno = smp_processor_id() % ndev->nr_queues;
+
+        sr->cmdq = &ndev->pkt_cmdqs[qno];
+
+        /*
+         * 64-Byte Instruction Format
+         *
+         *  ----------------------
+         *  |      DPTR0         | 8 bytes
+         *  ----------------------
+         *  |  PKT_IN_INSTR_HDR  | 8 bytes
+         *  ----------------------
+         *  |    PKT_IN_HDR      | 16 bytes
+         *  ----------------------
+         *  |    SLC_INFO        | 16 bytes
+         *  ----------------------
+         *  |   Front data       | 16 bytes
+         *  ----------------------
+         */
+
+        /* fill the packet instruction */
+        /* word 0 */
+        sr->instr.dptr0 = cpu_to_be64(sr->in.dma);
+
+        /* word 1 */
+        sr->instr.ih.value = 0;
+        sr->instr.ih.s.g = 1;
+        sr->instr.ih.s.gsz = sr->in.map_bufs_cnt;
+        sr->instr.ih.s.ssz = sr->out.map_bufs_cnt;
+        sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
+        sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
+        sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value);
+
+        /* word 2 */
+        sr->instr.irh.value[0] = 0;
+        sr->instr.irh.s.uddl = MIN_UDD_LEN;
+        /* context length in 64-bit words */
+        sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8);
+        /* offset from solicit base port 256 */
+        sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno;
+        sr->instr.irh.s.ctxc = req->ctrl.s.ctxc;
+        sr->instr.irh.s.arg = req->ctrl.s.arg;
+        sr->instr.irh.s.opcode = req->opcode;
+        sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]);
+
+        /* word 3 */
+        sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle);
+
+        /* word 4 */
+        sr->instr.slc.value[0] = 0;
+        sr->instr.slc.s.ssz = sr->out.map_bufs_cnt;
+        sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]);
+
+        /* word 5 */
+        sr->instr.slc.s.rptr = cpu_to_be64(sr->out.dma);
+
+        /*
+         * No conversion for front data,
+         * It goes into payload
+         * put GP Header in front data
+         */
+        sr->instr.fdata[0] = *((u64 *)&req->gph);
+        sr->instr.fdata[1] = 0;
+        /* flush the soft_req changes before posting the cmd */
+        wmb();
+
+        ret = nitrox_enqueue_request(sr);
+        if (ret == -EAGAIN)
+                goto send_fail;
+
+        return ret;
+
+send_fail:
+        softreq_destroy(sr);
+        return ret;
+}
+
+static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout)
+{
+        return time_after_eq(jiffies, (tstamp + timeout));
+}
+
+void backlog_qflush_work(struct work_struct *work)
+{
+        struct nitrox_cmdq *cmdq;
+
+        cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush);
+        post_backlog_cmds(cmdq);
+}
+
+/**
+ * process_request_list - process completed requests
+ * @ndev: N5 device
+ * @qno: queue to operate
+ *
+ * Returns the number of responses processed.
+ */
+static void process_response_list(struct nitrox_cmdq *cmdq)
+{
+        struct nitrox_device *ndev = cmdq->ndev;
+        struct nitrox_softreq *sr;
+        struct skcipher_request *skreq;
+        completion_t callback;
+        int req_completed = 0, err = 0, budget;
+
+        /* check all pending requests */
+        budget = atomic_read(&cmdq->pending_count);
+
+        while (req_completed < budget) {
+                sr = get_first_response_entry(cmdq);
+                if (!sr)
+                        break;
+
+                if (atomic_read(&sr->status) != REQ_POSTED)
+                        break;
+
+                /* check orh and completion bytes updates */
+                if (READ_ONCE(sr->resp.orh) == READ_ONCE(sr->resp.completion)) {
+                        /* request not completed, check for timeout */
+                        if (!cmd_timeout(sr->tstamp, ndev->timeout))
+                                break;
+                        dev_err_ratelimited(DEV(ndev),
+                                            "Request timeout, orh 0x%016llx\n",
+                                            READ_ONCE(sr->resp.orh));
+                }
+                atomic_dec(&cmdq->pending_count);
+                /* sync with other cpus */
+                smp_mb__after_atomic();
+                /* remove from response list */
+                response_list_del(sr, cmdq);
+
+                callback = sr->callback;
+                skreq = sr->skreq;
+
+                /* ORH error code */
+                err = READ_ONCE(sr->resp.orh) & 0xff;
+                softreq_destroy(sr);
+
+                if (callback)
+                        callback(skreq, err);
+
+                req_completed++;
+        }
+}
+
+/**
+ * pkt_slc_resp_handler - post processing of SE responses
+ */
+void pkt_slc_resp_handler(unsigned long data)
+{
+        struct bh_data *bh = (void *)(uintptr_t)(data);
+        struct nitrox_cmdq *cmdq = bh->cmdq;
+        union nps_pkt_slc_cnts pkt_slc_cnts;
+
+        /* read completion count */
+        pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr);
+        /* resend the interrupt if more work to do */
+        pkt_slc_cnts.s.resend = 1;
+
+        process_response_list(cmdq);
+
+        /*
+         * clear the interrupt with resend bit enabled,
+         * MSI-X interrupt generates if Completion count > Threshold
+         */
+        writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr);
+        /* order the writes */
+        mmiowb();
+
+        if (atomic_read(&cmdq->backlog_count))
+                schedule_work(&cmdq->backlog_qflush);
+}
diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile
index 60919a3ec53b..59493fd3a751 100644
--- a/drivers/crypto/ccp/Makefile
+++ b/drivers/crypto/ccp/Makefile
@@ -4,7 +4,8 @@ ccp-objs := ccp-dev.o \
             ccp-dev-v3.o \
             ccp-dev-v5.o \
             ccp-platform.o \
-            ccp-dmaengine.o
+            ccp-dmaengine.o \
+            ccp-debugfs.o
 ccp-$(CONFIG_PCI) += ccp-pci.o
 
 obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o
diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c
index 6b46eea94932..ce97b3868f4a 100644
--- a/drivers/crypto/ccp/ccp-crypto-sha.c
+++ b/drivers/crypto/ccp/ccp-crypto-sha.c
@@ -18,6 +18,7 @@
 #include <linux/crypto.h>
 #include <crypto/algapi.h>
 #include <crypto/hash.h>
+#include <crypto/hmac.h>
 #include <crypto/internal/hash.h>
 #include <crypto/sha.h>
 #include <crypto/scatterwalk.h>
@@ -308,8 +309,8 @@ static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
         }
 
         for (i = 0; i < block_size; i++) {
-                ctx->u.sha.ipad[i] = ctx->u.sha.key[i] ^ 0x36;
-                ctx->u.sha.opad[i] = ctx->u.sha.key[i] ^ 0x5c;
+                ctx->u.sha.ipad[i] = ctx->u.sha.key[i] ^ HMAC_IPAD_VALUE;
+                ctx->u.sha.opad[i] = ctx->u.sha.key[i] ^ HMAC_OPAD_VALUE;
         }
 
         sg_init_one(&ctx->u.sha.opad_sg, ctx->u.sha.opad, block_size);
diff --git a/drivers/crypto/ccp/ccp-debugfs.c b/drivers/crypto/ccp/ccp-debugfs.c
new file mode 100644
index 000000000000..3cd6c83754e0
--- /dev/null
+++ b/drivers/crypto/ccp/ccp-debugfs.c
@@ -0,0 +1,344 @@
+/*
+ * AMD Cryptographic Coprocessor (CCP) driver
+ *
+ * Copyright (C) 2017 Advanced Micro Devices, Inc.
+ *
+ * Author: Gary R Hook <gary.hook@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/debugfs.h>
+#include <linux/ccp.h>
+
+#include "ccp-dev.h"
+
+/* DebugFS helpers */
+#define OBUFP           (obuf + oboff)
+#define OBUFLEN         512
+#define OBUFSPC         (OBUFLEN - oboff)
+#define OSCNPRINTF(fmt, ...) \
+                scnprintf(OBUFP, OBUFSPC, fmt, ## __VA_ARGS__)
+
+#define BUFLEN  63
+
+#define RI_VERSION_NUM  0x0000003F
+#define RI_AES_PRESENT  0x00000040
+#define RI_3DES_PRESENT 0x00000080
+#define RI_SHA_PRESENT  0x00000100
+#define RI_RSA_PRESENT  0x00000200
+#define RI_ECC_PRESENT  0x00000400
+#define RI_ZDE_PRESENT  0x00000800
+#define RI_ZCE_PRESENT  0x00001000
+#define RI_TRNG_PRESENT 0x00002000
+#define RI_ELFC_PRESENT 0x00004000
+#define RI_ELFC_SHIFT   14
+#define RI_NUM_VQM      0x00078000
+#define RI_NVQM_SHIFT   15
+#define RI_NVQM(r)      (((r) * RI_NUM_VQM) >> RI_NVQM_SHIFT)
+#define RI_LSB_ENTRIES  0x0FF80000
+#define RI_NLSB_SHIFT   19
+#define RI_NLSB(r)      (((r) * RI_LSB_ENTRIES) >> RI_NLSB_SHIFT)
+
+static ssize_t ccp5_debugfs_info_read(struct file *filp, char __user *ubuf,
+                                      size_t count, loff_t *offp)
+{
+        struct ccp_device *ccp = filp->private_data;
+        unsigned int oboff = 0;
+        unsigned int regval;
+        ssize_t ret;
+        char *obuf;
+
+        if (!ccp)
+                return 0;
+
+        obuf = kmalloc(OBUFLEN, GFP_KERNEL);
+        if (!obuf)
+                return -ENOMEM;
+
+        oboff += OSCNPRINTF("Device name: %s\n", ccp->name);
+        oboff += OSCNPRINTF("   RNG name: %s\n", ccp->rngname);
+        oboff += OSCNPRINTF("   # Queues: %d\n", ccp->cmd_q_count);
+        oboff += OSCNPRINTF("     # Cmds: %d\n", ccp->cmd_count);
+
+        regval = ioread32(ccp->io_regs + CMD5_PSP_CCP_VERSION);
+        oboff += OSCNPRINTF("    Version: %d\n", regval & RI_VERSION_NUM);
+        oboff += OSCNPRINTF("    Engines:");
+        if (regval & RI_AES_PRESENT)
+                oboff += OSCNPRINTF(" AES");
+        if (regval & RI_3DES_PRESENT)
+                oboff += OSCNPRINTF(" 3DES");
+        if (regval & RI_SHA_PRESENT)
+                oboff += OSCNPRINTF(" SHA");
+        if (regval & RI_RSA_PRESENT)
+                oboff += OSCNPRINTF(" RSA");
+        if (regval & RI_ECC_PRESENT)
+                oboff += OSCNPRINTF(" ECC");
+        if (regval & RI_ZDE_PRESENT)
+                oboff += OSCNPRINTF(" ZDE");
+        if (regval & RI_ZCE_PRESENT)
+                oboff += OSCNPRINTF(" ZCE");
+        if (regval & RI_TRNG_PRESENT)
+                oboff += OSCNPRINTF(" TRNG");
+        oboff += OSCNPRINTF("\n");
+        oboff += OSCNPRINTF("     Queues: %d\n",
+                   (regval & RI_NUM_VQM) >> RI_NVQM_SHIFT);
+        oboff += OSCNPRINTF("LSB Entries: %d\n",
+                   (regval & RI_LSB_ENTRIES) >> RI_NLSB_SHIFT);
+
+        ret = simple_read_from_buffer(ubuf, count, offp, obuf, oboff);
+        kfree(obuf);
+
+        return ret;
+}
+
+/* Return a formatted buffer containing the current
+ * statistics across all queues for a CCP.
+ */
+static ssize_t ccp5_debugfs_stats_read(struct file *filp, char __user *ubuf,
+                                       size_t count, loff_t *offp)
+{
+        struct ccp_device *ccp = filp->private_data;
+        unsigned long total_xts_aes_ops = 0;
+        unsigned long total_3des_ops = 0;
+        unsigned long total_aes_ops = 0;
+        unsigned long total_sha_ops = 0;
+        unsigned long total_rsa_ops = 0;
+        unsigned long total_ecc_ops = 0;
+        unsigned long total_pt_ops = 0;
+        unsigned long total_ops = 0;
+        unsigned int oboff = 0;
+        ssize_t ret = 0;
+        unsigned int i;
+        char *obuf;
+
+        for (i = 0; i < ccp->cmd_q_count; i++) {
+                struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
+
+                total_ops += cmd_q->total_ops;
+                total_aes_ops += cmd_q->total_aes_ops;
+                total_xts_aes_ops += cmd_q->total_xts_aes_ops;
+                total_3des_ops += cmd_q->total_3des_ops;
+                total_sha_ops += cmd_q->total_sha_ops;
+                total_rsa_ops += cmd_q->total_rsa_ops;
+                total_pt_ops += cmd_q->total_pt_ops;
+                total_ecc_ops += cmd_q->total_ecc_ops;
+        }
+
+        obuf = kmalloc(OBUFLEN, GFP_KERNEL);
+        if (!obuf)
+                return -ENOMEM;
+
+        oboff += OSCNPRINTF("Total Interrupts Handled: %ld\n",
+                            ccp->total_interrupts);
+        oboff += OSCNPRINTF("        Total Operations: %ld\n",
+                            total_ops);
+        oboff += OSCNPRINTF("                     AES: %ld\n",
+                            total_aes_ops);
+        oboff += OSCNPRINTF("                 XTS AES: %ld\n",
+                            total_xts_aes_ops);
+        oboff += OSCNPRINTF("                     SHA: %ld\n",
+                            total_3des_ops);
+        oboff += OSCNPRINTF("                     SHA: %ld\n",
+                            total_sha_ops);
+        oboff += OSCNPRINTF("                     RSA: %ld\n",
+                            total_rsa_ops);
+        oboff += OSCNPRINTF("               Pass-Thru: %ld\n",
+                            total_pt_ops);
+        oboff += OSCNPRINTF("                     ECC: %ld\n",
+                            total_ecc_ops);
+
+        ret = simple_read_from_buffer(ubuf, count, offp, obuf, oboff);
+        kfree(obuf);
+
+        return ret;
+}
+
+/* Reset the counters in a queue
+ */
+static void ccp5_debugfs_reset_queue_stats(struct ccp_cmd_queue *cmd_q)
+{
+        cmd_q->total_ops = 0L;
+        cmd_q->total_aes_ops = 0L;
+        cmd_q->total_xts_aes_ops = 0L;
+        cmd_q->total_3des_ops = 0L;
+        cmd_q->total_sha_ops = 0L;
+        cmd_q->total_rsa_ops = 0L;
+        cmd_q->total_pt_ops = 0L;
+        cmd_q->total_ecc_ops = 0L;
+}
+
+/* A value was written to the stats variable, which
+ * should be used to reset the queue counters across
+ * that device.
+ */
+static ssize_t ccp5_debugfs_stats_write(struct file *filp,
+                                        const char __user *ubuf,
+                                        size_t count, loff_t *offp)
+{
+        struct ccp_device *ccp = filp->private_data;
+        int i;
+
+        for (i = 0; i < ccp->cmd_q_count; i++)
+                ccp5_debugfs_reset_queue_stats(&ccp->cmd_q[i]);
+        ccp->total_interrupts = 0L;
+
+        return count;
+}
+
+/* Return a formatted buffer containing the current information
+ * for that queue
+ */
+static ssize_t ccp5_debugfs_queue_read(struct file *filp, char __user *ubuf,
+                                       size_t count, loff_t *offp)
+{
+        struct ccp_cmd_queue *cmd_q = filp->private_data;
+        unsigned int oboff = 0;
+        unsigned int regval;
+        ssize_t ret;
+        char *obuf;
+
+        if (!cmd_q)
+                return 0;
+
+        obuf = kmalloc(OBUFLEN, GFP_KERNEL);
+        if (!obuf)
+                return -ENOMEM;
+
+        oboff += OSCNPRINTF("  Total Queue Operations: %ld\n",
+                            cmd_q->total_ops);
+        oboff += OSCNPRINTF("                     AES: %ld\n",
+                            cmd_q->total_aes_ops);
+        oboff += OSCNPRINTF("                 XTS AES: %ld\n",
+                            cmd_q->total_xts_aes_ops);
+        oboff += OSCNPRINTF("                     SHA: %ld\n",
+                            cmd_q->total_3des_ops);
+        oboff += OSCNPRINTF("                     SHA: %ld\n",
+                            cmd_q->total_sha_ops);
+        oboff += OSCNPRINTF("                     RSA: %ld\n",
+                            cmd_q->total_rsa_ops);
+        oboff += OSCNPRINTF("               Pass-Thru: %ld\n",
+                            cmd_q->total_pt_ops);
+        oboff += OSCNPRINTF("                     ECC: %ld\n",
+                            cmd_q->total_ecc_ops);
+
+        regval = ioread32(cmd_q->reg_int_enable);
+        oboff += OSCNPRINTF("      Enabled Interrupts:");
+        if (regval & INT_EMPTY_QUEUE)
+                oboff += OSCNPRINTF(" EMPTY");
+        if (regval & INT_QUEUE_STOPPED)
+                oboff += OSCNPRINTF(" STOPPED");
+        if (regval & INT_ERROR)
+                oboff += OSCNPRINTF(" ERROR");
+        if (regval & INT_COMPLETION)
+                oboff += OSCNPRINTF(" COMPLETION");
+        oboff += OSCNPRINTF("\n");
+
+        ret = simple_read_from_buffer(ubuf, count, offp, obuf, oboff);
+        kfree(obuf);
+
+        return ret;
+}
+
+/* A value was written to the stats variable for a
+ * queue. Reset the queue counters to this value.
+ */
+static ssize_t ccp5_debugfs_queue_write(struct file *filp,
+                                        const char __user *ubuf,
+                                        size_t count, loff_t *offp)
+{
+        struct ccp_cmd_queue *cmd_q = filp->private_data;
+
+        ccp5_debugfs_reset_queue_stats(cmd_q);
+
+        return count;
+}
+
+static const struct file_operations ccp_debugfs_info_ops = {
+        .owner = THIS_MODULE,
+        .open = simple_open,
+        .read = ccp5_debugfs_info_read,
+        .write = NULL,
+};
+
+static const struct file_operations ccp_debugfs_queue_ops = {
+        .owner = THIS_MODULE,
+        .open = simple_open,
+        .read = ccp5_debugfs_queue_read,
+        .write = ccp5_debugfs_queue_write,
+};
+
+static const struct file_operations ccp_debugfs_stats_ops = {
+        .owner = THIS_MODULE,
+        .open = simple_open,
+        .read = ccp5_debugfs_stats_read,
+        .write = ccp5_debugfs_stats_write,
+};
+
+static struct dentry *ccp_debugfs_dir;
+static DEFINE_RWLOCK(ccp_debugfs_lock);
+
+#define MAX_NAME_LEN    20
+
+void ccp5_debugfs_setup(struct ccp_device *ccp)
+{
+        struct ccp_cmd_queue *cmd_q;
+        char name[MAX_NAME_LEN + 1];
+        struct dentry *debugfs_info;
+        struct dentry *debugfs_stats;
+        struct dentry *debugfs_q_instance;
+        struct dentry *debugfs_q_stats;
+        unsigned long flags;
+        int i;
+
+        if (!debugfs_initialized())
+                return;
+
+        write_lock_irqsave(&ccp_debugfs_lock, flags);
+        if (!ccp_debugfs_dir)
+                ccp_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
+        write_unlock_irqrestore(&ccp_debugfs_lock, flags);
+        if (!ccp_debugfs_dir)
+                return;
+
+        ccp->debugfs_instance = debugfs_create_dir(ccp->name, ccp_debugfs_dir);
+        if (!ccp->debugfs_instance)
+                return;
+
+        debugfs_info = debugfs_create_file("info", 0400,
+                                           ccp->debugfs_instance, ccp,
+                                           &ccp_debugfs_info_ops);
+        if (!debugfs_info)
+                return;
+
+        debugfs_stats = debugfs_create_file("stats", 0600,
+                                            ccp->debugfs_instance, ccp,
+                                            &ccp_debugfs_stats_ops);
+        if (!debugfs_stats)
+                return;
+
+        for (i = 0; i < ccp->cmd_q_count; i++) {
+                cmd_q = &ccp->cmd_q[i];
+
+                snprintf(name, MAX_NAME_LEN - 1, "q%d", cmd_q->id);
+
+                debugfs_q_instance =
+                        debugfs_create_dir(name, ccp->debugfs_instance);
+                if (!debugfs_q_instance)
+                        return;
+
+                debugfs_q_stats =
+                        debugfs_create_file("stats", 0600,
+                                            debugfs_q_instance, cmd_q,
+                                            &ccp_debugfs_queue_ops);
+                if (!debugfs_q_stats)
+                        return;
+        }
+}
+
+void ccp5_debugfs_destroy(void)
+{
+        debugfs_remove_recursive(ccp_debugfs_dir);
+}
diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c
index ccbe32d5dd1c..b10d2d2075cb 100644
--- a/drivers/crypto/ccp/ccp-dev-v5.c
+++ b/drivers/crypto/ccp/ccp-dev-v5.c
@@ -14,6 +14,7 @@
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/kthread.h>
+#include <linux/debugfs.h>
 #include <linux/dma-mapping.h>
 #include <linux/interrupt.h>
 #include <linux/compiler.h>
@@ -231,6 +232,8 @@ static int ccp5_do_cmd(struct ccp5_desc *desc,
         int     i;
         int ret = 0;
 
+        cmd_q->total_ops++;
+
         if (CCP5_CMD_SOC(desc)) {
                 CCP5_CMD_IOC(desc) = 1;
                 CCP5_CMD_SOC(desc) = 0;
@@ -282,6 +285,8 @@ static int ccp5_perform_aes(struct ccp_op *op)
         union ccp_function function;
         u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
 
+        op->cmd_q->total_aes_ops++;
+
         /* Zero out all the fields of the command desc */
         memset(&desc, 0, Q_DESC_SIZE);
 
@@ -325,6 +330,8 @@ static int ccp5_perform_xts_aes(struct ccp_op *op)
         union ccp_function function;
         u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
 
+        op->cmd_q->total_xts_aes_ops++;
+
         /* Zero out all the fields of the command desc */
         memset(&desc, 0, Q_DESC_SIZE);
 
@@ -364,6 +371,8 @@ static int ccp5_perform_sha(struct ccp_op *op)
         struct ccp5_desc desc;
         union ccp_function function;
 
+        op->cmd_q->total_sha_ops++;
+
         /* Zero out all the fields of the command desc */
         memset(&desc, 0, Q_DESC_SIZE);
 
@@ -404,6 +413,8 @@ static int ccp5_perform_des3(struct ccp_op *op)
         union ccp_function function;
         u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
 
+        op->cmd_q->total_3des_ops++;
+
         /* Zero out all the fields of the command desc */
         memset(&desc, 0, sizeof(struct ccp5_desc));
 
@@ -444,6 +455,8 @@ static int ccp5_perform_rsa(struct ccp_op *op)
         struct ccp5_desc desc;
         union ccp_function function;
 
+        op->cmd_q->total_rsa_ops++;
+
         /* Zero out all the fields of the command desc */
         memset(&desc, 0, Q_DESC_SIZE);
 
@@ -487,6 +500,8 @@ static int ccp5_perform_passthru(struct ccp_op *op)
         struct ccp_dma_info *daddr = &op->dst.u.dma;
 
 
+        op->cmd_q->total_pt_ops++;
+
         memset(&desc, 0, Q_DESC_SIZE);
 
         CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
@@ -543,6 +558,8 @@ static int ccp5_perform_ecc(struct ccp_op *op)
         struct ccp5_desc desc;
         union ccp_function function;
 
+        op->cmd_q->total_ecc_ops++;
+
         /* Zero out all the fields of the command desc */
         memset(&desc, 0, Q_DESC_SIZE);
 
@@ -592,7 +609,6 @@ static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
         return queues ? 0 : -EINVAL;
 }
 
-
 static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
                                         int lsb_cnt, int n_lsbs,
                                         unsigned long *lsb_pub)
@@ -757,6 +773,7 @@ static irqreturn_t ccp5_irq_handler(int irq, void *data)
         struct ccp_device *ccp = dev_get_drvdata(dev);
 
         ccp5_disable_queue_interrupts(ccp);
+        ccp->total_interrupts++;
         if (ccp->use_tasklet)
                 tasklet_schedule(&ccp->irq_tasklet);
         else
@@ -956,6 +973,9 @@ static int ccp5_init(struct ccp_device *ccp)
         if (ret)
                 goto e_hwrng;
 
+        /* Set up debugfs entries */
+        ccp5_debugfs_setup(ccp);
+
         return 0;
 
 e_hwrng:
@@ -992,6 +1012,12 @@ static void ccp5_destroy(struct ccp_device *ccp)
         /* Remove this device from the list of available units first */
         ccp_del_device(ccp);
 
+        /* We're in the process of tearing down the entire driver;
+         * when all the devices are gone clean up debugfs
+         */
+        if (ccp_present())
+                ccp5_debugfs_destroy();
+
         /* Disable and clear interrupts */
         ccp5_disable_queue_interrupts(ccp);
         for (i = 0; i < ccp->cmd_q_count; i++) {
diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c
index 92d1c6959f08..2506b5025700 100644
--- a/drivers/crypto/ccp/ccp-dev.c
+++ b/drivers/crypto/ccp/ccp-dev.c
@@ -31,8 +31,9 @@
 #include "ccp-dev.h"
 
 MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>");
+MODULE_AUTHOR("Gary R Hook <gary.hook@amd.com>");
 MODULE_LICENSE("GPL");
-MODULE_VERSION("1.0.0");
+MODULE_VERSION("1.1.0");
 MODULE_DESCRIPTION("AMD Cryptographic Coprocessor driver");
 
 struct ccp_tasklet_data {
diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h
index 0cb09d0feeaf..a70154ac7405 100644
--- a/drivers/crypto/ccp/ccp-dev.h
+++ b/drivers/crypto/ccp/ccp-dev.h
@@ -70,6 +70,7 @@
 #define LSB_PUBLIC_MASK_HI_OFFSET       0x1C
 #define LSB_PRIVATE_MASK_LO_OFFSET      0x20
 #define LSB_PRIVATE_MASK_HI_OFFSET      0x24
+#define CMD5_PSP_CCP_VERSION            0x100
 
 #define CMD5_Q_CONTROL_BASE             0x0000
 #define CMD5_Q_TAIL_LO_BASE             0x0004
@@ -322,6 +323,16 @@ struct ccp_cmd_queue {
         /* Interrupt wait queue */
         wait_queue_head_t int_queue;
         unsigned int int_rcvd;
+
+        /* Per-queue Statistics */
+        unsigned long total_ops;
+        unsigned long total_aes_ops;
+        unsigned long total_xts_aes_ops;
+        unsigned long total_3des_ops;
+        unsigned long total_sha_ops;
+        unsigned long total_rsa_ops;
+        unsigned long total_pt_ops;
+        unsigned long total_ecc_ops;
 } ____cacheline_aligned;
 
 struct ccp_device {
@@ -419,6 +430,12 @@ struct ccp_device {
 
         /* DMA caching attribute support */
         unsigned int axcache;
+
+        /* Device Statistics */
+        unsigned long total_interrupts;
+
+        /* DebugFS info */
+        struct dentry *debugfs_instance;
 };
 
 enum ccp_memtype {
@@ -632,6 +649,9 @@ void ccp_unregister_rng(struct ccp_device *ccp);
 int ccp_dmaengine_register(struct ccp_device *ccp);
 void ccp_dmaengine_unregister(struct ccp_device *ccp);
 
+void ccp5_debugfs_setup(struct ccp_device *ccp);
+void ccp5_debugfs_destroy(void);
+
 /* Structure for computation functions that are device-specific */
 struct ccp_actions {
         int (*aes)(struct ccp_op *);
diff --git a/drivers/crypto/ccp/ccp-platform.c b/drivers/crypto/ccp/ccp-platform.c
index 351f28d8c336..e26969e601ad 100644
--- a/drivers/crypto/ccp/ccp-platform.c
+++ b/drivers/crypto/ccp/ccp-platform.c
@@ -44,7 +44,7 @@ static struct ccp_vdata *ccp_get_of_version(struct platform_device *pdev)
         if (match && match->data)
                 return (struct ccp_vdata *)match->data;
 #endif
-        return 0;
+        return NULL;
 }
 
 static struct ccp_vdata *ccp_get_acpi_version(struct platform_device *pdev)
@@ -56,7 +56,7 @@ static struct ccp_vdata *ccp_get_acpi_version(struct platform_device *pdev)
         if (match && match->driver_data)
                 return (struct ccp_vdata *)match->driver_data;
 #endif
-        return 0;
+        return NULL;
 }
 
 static int ccp_get_irq(struct ccp_device *ccp)
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index f00e0d8bd039..aa4e5b88483d 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -55,6 +55,8 @@
 #include <crypto/hash.h>
 #include <crypto/sha.h>
 #include <crypto/authenc.h>
+#include <crypto/ctr.h>
+#include <crypto/gf128mul.h>
 #include <crypto/internal/aead.h>
 #include <crypto/null.h>
 #include <crypto/internal/skcipher.h>
@@ -126,13 +128,13 @@ static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
         fw6_pld = (struct cpl_fw6_pld *)input;
         if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
             (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
-                cmp = memcmp(&fw6_pld->data[2], (fw6_pld + 1), authsize);
+                cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize);
         } else {
 
                 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
                                 authsize, req->assoclen +
                                 req->cryptlen - authsize);
-                cmp = memcmp(temp, (fw6_pld + 1), authsize);
+                cmp = crypto_memneq(temp, (fw6_pld + 1), authsize);
         }
         if (cmp)
                 *err = -EBADMSG;
@@ -151,12 +153,12 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
         struct chcr_context *ctx = crypto_tfm_ctx(tfm);
         struct uld_ctx *u_ctx = ULD_CTX(ctx);
         struct chcr_req_ctx ctx_req;
-        struct cpl_fw6_pld *fw6_pld;
         unsigned int digestsize, updated_digestsize;
+        struct adapter *adap = padap(ctx->dev);
 
         switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
         case CRYPTO_ALG_TYPE_AEAD:
-                ctx_req.req.aead_req = (struct aead_request *)req;
+                ctx_req.req.aead_req = aead_request_cast(req);
                 ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req);
                 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst,
                              ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE);
@@ -164,32 +166,23 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
                         kfree_skb(ctx_req.ctx.reqctx->skb);
                         ctx_req.ctx.reqctx->skb = NULL;
                 }
+                free_new_sg(ctx_req.ctx.reqctx->newdstsg);
+                ctx_req.ctx.reqctx->newdstsg = NULL;
                 if (ctx_req.ctx.reqctx->verify == VERIFY_SW) {
                         chcr_verify_tag(ctx_req.req.aead_req, input,
                                         &err);
                         ctx_req.ctx.reqctx->verify = VERIFY_HW;
                 }
+                ctx_req.req.aead_req->base.complete(req, err);
                 break;
 
         case CRYPTO_ALG_TYPE_ABLKCIPHER:
-                ctx_req.req.ablk_req = (struct ablkcipher_request *)req;
-                ctx_req.ctx.ablk_ctx =
-                        ablkcipher_request_ctx(ctx_req.req.ablk_req);
-                if (!err) {
-                        fw6_pld = (struct cpl_fw6_pld *)input;
-                        memcpy(ctx_req.req.ablk_req->info, &fw6_pld->data[2],
-                               AES_BLOCK_SIZE);
-                }
-                dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.ablk_req->dst,
-                             ctx_req.ctx.ablk_ctx->dst_nents, DMA_FROM_DEVICE);
-                if (ctx_req.ctx.ablk_ctx->skb) {
-                        kfree_skb(ctx_req.ctx.ablk_ctx->skb);
-                        ctx_req.ctx.ablk_ctx->skb = NULL;
-                }
+                 err = chcr_handle_cipher_resp(ablkcipher_request_cast(req),
+                                               input, err);
                 break;
 
         case CRYPTO_ALG_TYPE_AHASH:
-                ctx_req.req.ahash_req = (struct ahash_request *)req;
+                ctx_req.req.ahash_req = ahash_request_cast(req);
                 ctx_req.ctx.ahash_ctx =
                         ahash_request_ctx(ctx_req.req.ahash_req);
                 digestsize =
@@ -214,8 +207,10 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
                                sizeof(struct cpl_fw6_pld),
                                updated_digestsize);
                 }
+                ctx_req.req.ahash_req->base.complete(req, err);
                 break;
         }
+        atomic_inc(&adap->chcr_stats.complete);
         return err;
 }
 
@@ -392,7 +387,7 @@ static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
                            struct phys_sge_parm *sg_param)
 {
         struct phys_sge_pairs *to;
-        int out_buf_size = sg_param->obsize;
+        unsigned int len = 0, left_size = sg_param->obsize;
         unsigned int nents = sg_param->nents, i, j = 0;
 
         phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
@@ -409,20 +404,15 @@ static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
         phys_cpl->rss_hdr_int.hash_val = 0;
         to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl +
                                        sizeof(struct cpl_rx_phys_dsgl));
-
-        for (i = 0; nents; to++) {
-                for (j = 0; j < 8 && nents; j++, nents--) {
-                        out_buf_size -= sg_dma_len(sg);
-                        to->len[j] = htons(sg_dma_len(sg));
+        for (i = 0; nents && left_size; to++) {
+                for (j = 0; j < 8 && nents && left_size; j++, nents--) {
+                        len = min(left_size, sg_dma_len(sg));
+                        to->len[j] = htons(len);
                         to->addr[j] = cpu_to_be64(sg_dma_address(sg));
+                        left_size -= len;
                         sg = sg_next(sg);
                 }
         }
-        if (out_buf_size) {
-                j--;
-                to--;
-                to->len[j] = htons(ntohs(to->len[j]) + (out_buf_size));
-        }
 }
 
 static inline int map_writesg_phys_cpl(struct device *dev,
@@ -431,7 +421,7 @@ static inline int map_writesg_phys_cpl(struct device *dev,
                                         struct phys_sge_parm *sg_param)
 {
         if (!sg || !sg_param->nents)
-                return 0;
+                return -EINVAL;
 
         sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE);
         if (sg_param->nents == 0) {
@@ -498,6 +488,24 @@ write_sg_to_skb(struct sk_buff *skb, unsigned int *frags,
         }
 }
 
+static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
+{
+        struct adapter *adap = netdev2adap(dev);
+        struct sge_uld_txq_info *txq_info =
+                adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
+        struct sge_uld_txq *txq;
+        int ret = 0;
+
+        local_bh_disable();
+        txq = &txq_info->uldtxq[idx];
+        spin_lock(&txq->sendq.lock);
+        if (txq->full)
+                ret = -1;
+        spin_unlock(&txq->sendq.lock);
+        local_bh_enable();
+        return ret;
+}
+
 static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
                                struct _key_ctx *key_ctx)
 {
@@ -512,13 +520,67 @@ static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
         }
         return 0;
 }
+static int chcr_sg_ent_in_wr(struct scatterlist *src,
+                             struct scatterlist *dst,
+                             unsigned int minsg,
+                             unsigned int space,
+                             short int *sent,
+                             short int *dent)
+{
+        int srclen = 0, dstlen = 0;
+        int srcsg = minsg, dstsg = 0;
+
+        *sent = 0;
+        *dent = 0;
+        while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) &&
+               space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
+                srclen += src->length;
+                srcsg++;
+                while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
+                       space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
+                        if (srclen <= dstlen)
+                                break;
+                        dstlen += dst->length;
+                        dst = sg_next(dst);
+                        dstsg++;
+                }
+                src = sg_next(src);
+        }
+        *sent = srcsg - minsg;
+        *dent = dstsg;
+        return min(srclen, dstlen);
+}
+
+static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
+                                u32 flags,
+                                struct scatterlist *src,
+                                struct scatterlist *dst,
+                                unsigned int nbytes,
+                                u8 *iv,
+                                unsigned short op_type)
+{
+        int err;
 
+        SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
+        skcipher_request_set_tfm(subreq, cipher);
+        skcipher_request_set_callback(subreq, flags, NULL, NULL);
+        skcipher_request_set_crypt(subreq, src, dst,
+                                   nbytes, iv);
+
+        err = op_type ? crypto_skcipher_decrypt(subreq) :
+                crypto_skcipher_encrypt(subreq);
+        skcipher_request_zero(subreq);
+
+        return err;
+
+}
 static inline void create_wreq(struct chcr_context *ctx,
                                struct chcr_wr *chcr_req,
                                void *req, struct sk_buff *skb,
                                int kctx_len, int hash_sz,
                                int is_iv,
-                               unsigned int sc_len)
+                               unsigned int sc_len,
+                               unsigned int lcb)
 {
         struct uld_ctx *u_ctx = ULD_CTX(ctx);
         int iv_loc = IV_DSGL;
@@ -543,7 +605,8 @@ static inline void create_wreq(struct chcr_context *ctx,
         chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
         chcr_req->wreq.rx_chid_to_rx_q_id =
                 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
-                                is_iv ? iv_loc : IV_NOP, ctx->tx_qidx);
+                                is_iv ? iv_loc : IV_NOP, !!lcb,
+                                ctx->tx_qidx);
 
         chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id,
                                                        qid);
@@ -563,69 +626,62 @@ static inline void create_wreq(struct chcr_context *ctx,
  *      @qid: ingress qid where response of this WR should be received.
  *      @op_type:       encryption or decryption
  */
-static struct sk_buff
-*create_cipher_wr(struct ablkcipher_request *req,
-                  unsigned short qid,
-                  unsigned short op_type)
+static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req);
         struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
         struct uld_ctx *u_ctx = ULD_CTX(ctx);
         struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
         struct sk_buff *skb = NULL;
         struct chcr_wr *chcr_req;
         struct cpl_rx_phys_dsgl *phys_cpl;
-        struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
+        struct chcr_blkcipher_req_ctx *reqctx =
+                ablkcipher_request_ctx(wrparam->req);
         struct phys_sge_parm sg_param;
         unsigned int frags = 0, transhdr_len, phys_dsgl;
-        unsigned int ivsize = crypto_ablkcipher_ivsize(tfm), kctx_len;
-        gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
-                        GFP_ATOMIC;
-
-        if (!req->info)
-                return ERR_PTR(-EINVAL);
-        reqctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes);
-        if (reqctx->dst_nents <= 0) {
-                pr_err("AES:Invalid Destination sg lists\n");
-                return ERR_PTR(-EINVAL);
-        }
-        if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
-            (req->nbytes <= 0) || (req->nbytes % AES_BLOCK_SIZE)) {
-                pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
-                       ablkctx->enckey_len, req->nbytes, ivsize);
-                return ERR_PTR(-EINVAL);
-        }
+        int error;
+        unsigned int ivsize = AES_BLOCK_SIZE, kctx_len;
+        gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+                        GFP_KERNEL : GFP_ATOMIC;
+        struct adapter *adap = padap(ctx->dev);
 
         phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents);
 
         kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
         transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
         skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
-        if (!skb)
-                return ERR_PTR(-ENOMEM);
+        if (!skb) {
+                error = -ENOMEM;
+                goto err;
+        }
         skb_reserve(skb, sizeof(struct sge_opaque_hdr));
         chcr_req = (struct chcr_wr *)__skb_put(skb, transhdr_len);
         memset(chcr_req, 0, transhdr_len);
         chcr_req->sec_cpl.op_ivinsrtofst =
                 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1);
 
-        chcr_req->sec_cpl.pldlen = htonl(ivsize + req->nbytes);
+        chcr_req->sec_cpl.pldlen = htonl(ivsize + wrparam->bytes);
         chcr_req->sec_cpl.aadstart_cipherstop_hi =
                         FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0);
 
         chcr_req->sec_cpl.cipherstop_lo_authinsert =
                         FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
-        chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, 0,
+        chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
                                                          ablkctx->ciph_mode,
                                                          0, 0, ivsize >> 1);
         chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
                                                           0, 1, phys_dsgl);
 
         chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
-        if (op_type == CHCR_DECRYPT_OP) {
+        if ((reqctx->op == CHCR_DECRYPT_OP) &&
+            (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
+               CRYPTO_ALG_SUB_TYPE_CTR)) &&
+            (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
+               CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
                 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
         } else {
-                if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
+                if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
+                    (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
                         memcpy(chcr_req->key_ctx.key, ablkctx->key,
                                ablkctx->enckey_len);
                 } else {
@@ -640,45 +696,80 @@ static struct sk_buff
         }
         phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
         sg_param.nents = reqctx->dst_nents;
-        sg_param.obsize = req->nbytes;
-        sg_param.qid = qid;
-        sg_param.align = 1;
-        if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, req->dst,
-                                 &sg_param))
+        sg_param.obsize =  wrparam->bytes;
+        sg_param.qid = wrparam->qid;
+        error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
+                                       reqctx->dst, &sg_param);
+        if (error)
                 goto map_fail1;
 
         skb_set_transport_header(skb, transhdr_len);
-        memcpy(reqctx->iv, req->info, ivsize);
         write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
-        write_sg_to_skb(skb, &frags, req->src, req->nbytes);
-        create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1,
-                        sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl);
+        write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes);
+        atomic_inc(&adap->chcr_stats.cipher_rqst);
+        create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1,
+                        sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl,
+                        ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
         reqctx->skb = skb;
         skb_get(skb);
         return skb;
 map_fail1:
         kfree_skb(skb);
-        return ERR_PTR(-ENOMEM);
+err:
+        return ERR_PTR(error);
 }
 
-static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+static inline int chcr_keyctx_ck_size(unsigned int keylen)
+{
+        int ck_size = 0;
+
+        if (keylen == AES_KEYSIZE_128)
+                ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
+        else if (keylen == AES_KEYSIZE_192)
+                ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
+        else if (keylen == AES_KEYSIZE_256)
+                ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
+        else
+                ck_size = 0;
+
+        return ck_size;
+}
+static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher,
+                                       const u8 *key,
+                                       unsigned int keylen)
+{
+        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+        struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
+        struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+        int err = 0;
+
+        crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
+        crypto_skcipher_set_flags(ablkctx->sw_cipher, cipher->base.crt_flags &
+                                  CRYPTO_TFM_REQ_MASK);
+        err = crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
+        tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
+        tfm->crt_flags |=
+                crypto_skcipher_get_flags(ablkctx->sw_cipher) &
+                CRYPTO_TFM_RES_MASK;
+        return err;
+}
+
+static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher,
+                               const u8 *key,
                                unsigned int keylen)
 {
-        struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+        struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
         struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
         unsigned int ck_size, context_size;
         u16 alignment = 0;
+        int err;
 
-        if (keylen == AES_KEYSIZE_128) {
-                ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
-        } else if (keylen == AES_KEYSIZE_192) {
-                alignment = 8;
-                ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
-        } else if (keylen == AES_KEYSIZE_256) {
-                ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
-        } else {
+        err = chcr_cipher_fallback_setkey(cipher, key, keylen);
+        if (err)
                 goto badkey_err;
-        }
+
+        ck_size = chcr_keyctx_ck_size(keylen);
+        alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
         memcpy(ablkctx->key, key, keylen);
         ablkctx->enckey_len = keylen;
         get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
@@ -690,35 +781,398 @@ static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
         ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
         return 0;
 badkey_err:
-        crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+        crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
         ablkctx->enckey_len = 0;
-        return -EINVAL;
+
+        return err;
 }
 
-static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
+static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher,
+                                   const u8 *key,
+                                   unsigned int keylen)
 {
-        struct adapter *adap = netdev2adap(dev);
-        struct sge_uld_txq_info *txq_info =
-                adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
-        struct sge_uld_txq *txq;
+        struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
+        struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+        unsigned int ck_size, context_size;
+        u16 alignment = 0;
+        int err;
+
+        err = chcr_cipher_fallback_setkey(cipher, key, keylen);
+        if (err)
+                goto badkey_err;
+        ck_size = chcr_keyctx_ck_size(keylen);
+        alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
+        memcpy(ablkctx->key, key, keylen);
+        ablkctx->enckey_len = keylen;
+        context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
+                        keylen + alignment) >> 4;
+
+        ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
+                                                0, 0, context_size);
+        ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
+
+        return 0;
+badkey_err:
+        crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+        ablkctx->enckey_len = 0;
+
+        return err;
+}
+
+static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher,
+                                   const u8 *key,
+                                   unsigned int keylen)
+{
+        struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
+        struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+        unsigned int ck_size, context_size;
+        u16 alignment = 0;
+        int err;
+
+        if (keylen < CTR_RFC3686_NONCE_SIZE)
+                return -EINVAL;
+        memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
+               CTR_RFC3686_NONCE_SIZE);
+
+        keylen -= CTR_RFC3686_NONCE_SIZE;
+        err = chcr_cipher_fallback_setkey(cipher, key, keylen);
+        if (err)
+                goto badkey_err;
+
+        ck_size = chcr_keyctx_ck_size(keylen);
+        alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
+        memcpy(ablkctx->key, key, keylen);
+        ablkctx->enckey_len = keylen;
+        context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
+                        keylen + alignment) >> 4;
+
+        ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
+                                                0, 0, context_size);
+        ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
+
+        return 0;
+badkey_err:
+        crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+        ablkctx->enckey_len = 0;
+
+        return err;
+}
+static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
+{
+        unsigned int size = AES_BLOCK_SIZE;
+        __be32 *b = (__be32 *)(dstiv + size);
+        u32 c, prev;
+
+        memcpy(dstiv, srciv, AES_BLOCK_SIZE);
+        for (; size >= 4; size -= 4) {
+                prev = be32_to_cpu(*--b);
+                c = prev + add;
+                *b = cpu_to_be32(c);
+                if (prev < c)
+                        break;
+                add = 1;
+        }
+
+}
+
+static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
+{
+        __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
+        u64 c;
+        u32 temp = be32_to_cpu(*--b);
+
+        temp = ~temp;
+        c = (u64)temp +  1; // No of block can processed withou overflow
+        if ((bytes / AES_BLOCK_SIZE) > c)
+                bytes = c * AES_BLOCK_SIZE;
+        return bytes;
+}
+
+static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv)
+{
+        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+        struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+        struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+        struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
+        struct crypto_cipher *cipher;
+        int ret, i;
+        u8 *key;
+        unsigned int keylen;
+
+        cipher = crypto_alloc_cipher("aes-generic", 0, 0);
+        memcpy(iv, req->info, AES_BLOCK_SIZE);
+
+        if (IS_ERR(cipher)) {
+                ret = -ENOMEM;
+                goto out;
+        }
+        keylen = ablkctx->enckey_len / 2;
+        key = ablkctx->key + keylen;
+        ret = crypto_cipher_setkey(cipher, key, keylen);
+        if (ret)
+                goto out1;
+
+        crypto_cipher_encrypt_one(cipher, iv, iv);
+        for (i = 0; i < (reqctx->processed / AES_BLOCK_SIZE); i++)
+                gf128mul_x_ble((le128 *)iv, (le128 *)iv);
+
+        crypto_cipher_decrypt_one(cipher, iv, iv);
+out1:
+        crypto_free_cipher(cipher);
+out:
+        return ret;
+}
+
+static int chcr_update_cipher_iv(struct ablkcipher_request *req,
+                                   struct cpl_fw6_pld *fw6_pld, u8 *iv)
+{
+        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+        struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
+        int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
         int ret = 0;
 
-        local_bh_disable();
-        txq = &txq_info->uldtxq[idx];
-        spin_lock(&txq->sendq.lock);
-        if (txq->full)
-                ret = -1;
-        spin_unlock(&txq->sendq.lock);
-        local_bh_enable();
+        if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
+                ctr_add_iv(iv, req->info, (reqctx->processed /
+                           AES_BLOCK_SIZE));
+        else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
+                *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
+                        CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
+                                                AES_BLOCK_SIZE) + 1);
+        else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
+                ret = chcr_update_tweak(req, iv);
+        else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
+                if (reqctx->op)
+                        sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
+                                           16,
+                                           reqctx->processed - AES_BLOCK_SIZE);
+                else
+                        memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
+        }
+
         return ret;
+
 }
 
-static int chcr_aes_encrypt(struct ablkcipher_request *req)
+/* We need separate function for final iv because in rfc3686  Initial counter
+ * starts from 1 and buffer size of iv is 8 byte only which remains constant
+ * for subsequent update requests
+ */
+
+static int chcr_final_cipher_iv(struct ablkcipher_request *req,
+                                   struct cpl_fw6_pld *fw6_pld, u8 *iv)
+{
+        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+        struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
+        int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
+        int ret = 0;
+
+        if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
+                ctr_add_iv(iv, req->info, (reqctx->processed /
+                           AES_BLOCK_SIZE));
+        else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
+                ret = chcr_update_tweak(req, iv);
+        else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
+                if (reqctx->op)
+                        sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
+                                           16,
+                                           reqctx->processed - AES_BLOCK_SIZE);
+                else
+                        memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
+
+        }
+        return ret;
+
+}
+
+
+static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
+                                   unsigned char *input, int err)
 {
         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
         struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
         struct uld_ctx *u_ctx = ULD_CTX(ctx);
+        struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
         struct sk_buff *skb;
+        struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
+        struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
+        struct  cipher_wr_param wrparam;
+        int bytes;
+
+        dma_unmap_sg(&u_ctx->lldi.pdev->dev, reqctx->dst, reqctx->dst_nents,
+                     DMA_FROM_DEVICE);
+
+        if (reqctx->skb) {
+                kfree_skb(reqctx->skb);
+                reqctx->skb = NULL;
+        }
+        if (err)
+                goto complete;
+
+        if (req->nbytes == reqctx->processed) {
+                err = chcr_final_cipher_iv(req, fw6_pld, req->info);
+                goto complete;
+        }
+
+        if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
+                                            ctx->tx_qidx))) {
+                if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+                        err = -EBUSY;
+                        goto complete;
+                }
+
+        }
+        wrparam.srcsg = scatterwalk_ffwd(reqctx->srcffwd, req->src,
+                                       reqctx->processed);
+        reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, reqctx->dstsg,
+                                         reqctx->processed);
+        if (!wrparam.srcsg || !reqctx->dst) {
+                pr_err("Input sg list length less that nbytes\n");
+                err = -EINVAL;
+                goto complete;
+        }
+        bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1,
+                                 SPACE_LEFT(ablkctx->enckey_len),
+                                 &wrparam.snent, &reqctx->dst_nents);
+        if ((bytes + reqctx->processed) >= req->nbytes)
+                bytes  = req->nbytes - reqctx->processed;
+        else
+                bytes = ROUND_16(bytes);
+        err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
+        if (err)
+                goto complete;
+
+        if (unlikely(bytes == 0)) {
+                err = chcr_cipher_fallback(ablkctx->sw_cipher,
+                                     req->base.flags,
+                                     wrparam.srcsg,
+                                     reqctx->dst,
+                                     req->nbytes - reqctx->processed,
+                                     reqctx->iv,
+                                     reqctx->op);
+                goto complete;
+        }
+
+        if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
+            CRYPTO_ALG_SUB_TYPE_CTR)
+                bytes = adjust_ctr_overflow(reqctx->iv, bytes);
+        reqctx->processed += bytes;
+        wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
+        wrparam.req = req;
+        wrparam.bytes = bytes;
+        skb = create_cipher_wr(&wrparam);
+        if (IS_ERR(skb)) {
+                pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
+                err = PTR_ERR(skb);
+                goto complete;
+        }
+        skb->dev = u_ctx->lldi.ports[0];
+        set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
+        chcr_send_wr(skb);
+        return 0;
+complete:
+        free_new_sg(reqctx->newdstsg);
+        reqctx->newdstsg = NULL;
+        req->base.complete(&req->base, err);
+        return err;
+}
+
+static int process_cipher(struct ablkcipher_request *req,
+                                  unsigned short qid,
+                                  struct sk_buff **skb,
+                                  unsigned short op_type)
+{
+        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+        unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
+        struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
+        struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+        struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+        struct  cipher_wr_param wrparam;
+        int bytes, nents, err = -EINVAL;
+
+        reqctx->newdstsg = NULL;
+        reqctx->processed = 0;
+        if (!req->info)
+                goto error;
+        if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
+            (req->nbytes == 0) ||
+            (req->nbytes % crypto_ablkcipher_blocksize(tfm))) {
+                pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
+                       ablkctx->enckey_len, req->nbytes, ivsize);
+                goto error;
+        }
+        wrparam.srcsg = req->src;
+        if (is_newsg(req->dst, &nents)) {
+                reqctx->newdstsg = alloc_new_sg(req->dst, nents);
+                if (IS_ERR(reqctx->newdstsg))
+                        return PTR_ERR(reqctx->newdstsg);
+                reqctx->dstsg = reqctx->newdstsg;
+        } else {
+                reqctx->dstsg = req->dst;
+        }
+        bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG,
+                                 SPACE_LEFT(ablkctx->enckey_len),
+                                 &wrparam.snent,
+                                 &reqctx->dst_nents);
+        if ((bytes + reqctx->processed) >= req->nbytes)
+                bytes  = req->nbytes - reqctx->processed;
+        else
+                bytes = ROUND_16(bytes);
+        if (unlikely(bytes > req->nbytes))
+                bytes = req->nbytes;
+        if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
+                                  CRYPTO_ALG_SUB_TYPE_CTR) {
+                bytes = adjust_ctr_overflow(req->info, bytes);
+        }
+        if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
+            CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
+                memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
+                memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info,
+                                CTR_RFC3686_IV_SIZE);
+
+                /* initialize counter portion of counter block */
+                *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
+                        CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
+
+        } else {
+
+                memcpy(reqctx->iv, req->info, ivsize);
+        }
+        if (unlikely(bytes == 0)) {
+                err = chcr_cipher_fallback(ablkctx->sw_cipher,
+                                           req->base.flags,
+                                           req->src,
+                                           req->dst,
+                                           req->nbytes,
+                                           req->info,
+                                           op_type);
+                goto error;
+        }
+        reqctx->processed = bytes;
+        reqctx->dst = reqctx->dstsg;
+        reqctx->op = op_type;
+        wrparam.qid = qid;
+        wrparam.req = req;
+        wrparam.bytes = bytes;
+        *skb = create_cipher_wr(&wrparam);
+        if (IS_ERR(*skb)) {
+                err = PTR_ERR(*skb);
+                goto error;
+        }
+
+        return 0;
+error:
+        free_new_sg(reqctx->newdstsg);
+        reqctx->newdstsg = NULL;
+        return err;
+}
+
+static int chcr_aes_encrypt(struct ablkcipher_request *req)
+{
+        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+        struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+        struct sk_buff *skb = NULL;
+        int err;
+        struct uld_ctx *u_ctx = ULD_CTX(ctx);
 
         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
                                             ctx->tx_qidx))) {
@@ -726,12 +1180,10 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req)
                         return -EBUSY;
         }
 
-        skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx],
+        err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb,
                                CHCR_ENCRYPT_OP);
-        if (IS_ERR(skb)) {
-                pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
-                return  PTR_ERR(skb);
-        }
+        if (err || !skb)
+                return  err;
         skb->dev = u_ctx->lldi.ports[0];
         set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
         chcr_send_wr(skb);
@@ -743,7 +1195,8 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req)
         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
         struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
         struct uld_ctx *u_ctx = ULD_CTX(ctx);
-        struct sk_buff *skb;
+        struct sk_buff *skb = NULL;
+        int err;
 
         if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
                                             ctx->tx_qidx))) {
@@ -751,12 +1204,10 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req)
                         return -EBUSY;
         }
 
-        skb = create_cipher_wr(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx],
+         err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb,
                                CHCR_DECRYPT_OP);
-        if (IS_ERR(skb)) {
-                pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
-                return PTR_ERR(skb);
-        }
+        if (err || !skb)
+                return err;
         skb->dev = u_ctx->lldi.ports[0];
         set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
         chcr_send_wr(skb);
@@ -765,7 +1216,7 @@ static int chcr_aes_decrypt(struct ablkcipher_request *req)
 
 static int chcr_device_init(struct chcr_context *ctx)
 {
-        struct uld_ctx *u_ctx;
+        struct uld_ctx *u_ctx = NULL;
         struct adapter *adap;
         unsigned int id;
         int txq_perchan, txq_idx, ntxq;
@@ -773,12 +1224,12 @@ static int chcr_device_init(struct chcr_context *ctx)
 
         id = smp_processor_id();
         if (!ctx->dev) {
-                err = assign_chcr_device(&ctx->dev);
-                if (err) {
+                u_ctx = assign_chcr_device();
+                if (!u_ctx) {
                         pr_err("chcr device assignment fails\n");
                         goto out;
                 }
-                u_ctx = ULD_CTX(ctx);
+                ctx->dev = u_ctx->dev;
                 adap = padap(ctx->dev);
                 ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq,
                                     adap->vres.ncrypto_fc);
@@ -801,10 +1252,48 @@ out:
 
 static int chcr_cra_init(struct crypto_tfm *tfm)
 {
+        struct crypto_alg *alg = tfm->__crt_alg;
+        struct chcr_context *ctx = crypto_tfm_ctx(tfm);
+        struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+
+        ablkctx->sw_cipher = crypto_alloc_skcipher(alg->cra_name, 0,
+                                CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+        if (IS_ERR(ablkctx->sw_cipher)) {
+                pr_err("failed to allocate fallback for %s\n", alg->cra_name);
+                return PTR_ERR(ablkctx->sw_cipher);
+        }
+        tfm->crt_ablkcipher.reqsize =  sizeof(struct chcr_blkcipher_req_ctx);
+        return chcr_device_init(crypto_tfm_ctx(tfm));
+}
+
+static int chcr_rfc3686_init(struct crypto_tfm *tfm)
+{
+        struct crypto_alg *alg = tfm->__crt_alg;
+        struct chcr_context *ctx = crypto_tfm_ctx(tfm);
+        struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+
+        /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
+         * cannot be used as fallback in chcr_handle_cipher_response
+         */
+        ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0,
+                                CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+        if (IS_ERR(ablkctx->sw_cipher)) {
+                pr_err("failed to allocate fallback for %s\n", alg->cra_name);
+                return PTR_ERR(ablkctx->sw_cipher);
+        }
         tfm->crt_ablkcipher.reqsize =  sizeof(struct chcr_blkcipher_req_ctx);
         return chcr_device_init(crypto_tfm_ctx(tfm));
 }
 
+
+static void chcr_cra_exit(struct crypto_tfm *tfm)
+{
+        struct chcr_context *ctx = crypto_tfm_ctx(tfm);
+        struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
+
+        crypto_free_skcipher(ablkctx->sw_cipher);
+}
+
 static int get_alg_config(struct algo_param *params,
                           unsigned int auth_size)
 {
@@ -865,6 +1354,7 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req,
         u8 hash_size_in_response = 0;
         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
                 GFP_ATOMIC;
+        struct adapter *adap = padap(ctx->dev);
 
         iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
         kctx_len = param->alg_prm.result_size + iopad_alignment;
@@ -921,9 +1411,9 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req,
                                     param->bfr_len);
         if (param->sg_len != 0)
                 write_sg_to_skb(skb, &frags, req->src, param->sg_len);
-
-        create_wreq(ctx, chcr_req, req, skb, kctx_len, hash_size_in_response, 0,
-                        DUMMY_BYTES);
+        atomic_inc(&adap->chcr_stats.digest_rqst);
+        create_wreq(ctx, chcr_req, &req->base, skb, kctx_len,
+                    hash_size_in_response, 0, DUMMY_BYTES, 0);
         req_ctx->skb = skb;
         skb_get(skb);
         return skb;
@@ -1226,21 +1716,17 @@ out:
         return err;
 }
 
-static int chcr_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
                                unsigned int key_len)
 {
-        struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
+        struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
         struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
         unsigned short context_size = 0;
+        int err;
 
-        if ((key_len != (AES_KEYSIZE_128 << 1)) &&
-            (key_len != (AES_KEYSIZE_256 << 1))) {
-                crypto_tfm_set_flags((struct crypto_tfm *)tfm,
-                                     CRYPTO_TFM_RES_BAD_KEY_LEN);
-                ablkctx->enckey_len = 0;
-                return -EINVAL;
-
-        }
+        err = chcr_cipher_fallback_setkey(cipher, key, key_len);
+        if (err)
+                goto badkey_err;
 
         memcpy(ablkctx->key, key, key_len);
         ablkctx->enckey_len = key_len;
@@ -1254,6 +1740,11 @@ static int chcr_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
                                  0, context_size);
         ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
         return 0;
+badkey_err:
+        crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+        ablkctx->enckey_len = 0;
+
+        return err;
 }
 
 static int chcr_sha_init(struct ahash_request *areq)
@@ -1330,6 +1821,63 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
         }
 }
 
+static int is_newsg(struct scatterlist *sgl, unsigned int *newents)
+{
+        int nents = 0;
+        int ret = 0;
+
+        while (sgl) {
+                if (sgl->length > CHCR_SG_SIZE)
+                        ret = 1;
+                nents += DIV_ROUND_UP(sgl->length, CHCR_SG_SIZE);
+                sgl = sg_next(sgl);
+        }
+        *newents = nents;
+        return ret;
+}
+
+static inline void free_new_sg(struct scatterlist *sgl)
+{
+        kfree(sgl);
+}
+
+static struct scatterlist *alloc_new_sg(struct scatterlist *sgl,
+                                       unsigned int nents)
+{
+        struct scatterlist *newsg, *sg;
+        int i, len, processed = 0;
+        struct page *spage;
+        int offset;
+
+        newsg = kmalloc_array(nents, sizeof(struct scatterlist), GFP_KERNEL);
+        if (!newsg)
+                return ERR_PTR(-ENOMEM);
+        sg = newsg;
+        sg_init_table(sg, nents);
+        offset = sgl->offset;
+        spage = sg_page(sgl);
+        for (i = 0; i < nents; i++) {
+                len = min_t(u32, sgl->length - processed, CHCR_SG_SIZE);
+                sg_set_page(sg, spage, len, offset);
+                processed += len;
+                offset += len;
+                if (offset >= PAGE_SIZE) {
+                        offset = offset % PAGE_SIZE;
+                        spage++;
+                }
+                if (processed == sgl->length) {
+                        processed = 0;
+                        sgl = sg_next(sgl);
+                        if (!sgl)
+                                break;
+                        spage = sg_page(sgl);
+                        offset = sgl->offset;
+                }
+                sg = sg_next(sg);
+        }
+        return newsg;
+}
+
 static int chcr_copy_assoc(struct aead_request *req,
                                 struct chcr_aead_ctx *ctx)
 {
@@ -1392,16 +1940,20 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
         struct scatterlist *src;
         unsigned int frags = 0, transhdr_len;
         unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0;
-        unsigned int   kctx_len = 0;
+        unsigned int   kctx_len = 0, nents;
         unsigned short stop_offset = 0;
         unsigned int  assoclen = req->assoclen;
         unsigned int  authsize = crypto_aead_authsize(tfm);
-        int err = -EINVAL, src_nent;
+        int error = -EINVAL, src_nent;
         int null = 0;
         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
                 GFP_ATOMIC;
+        struct adapter *adap = padap(ctx->dev);
 
-        if (aeadctx->enckey_len == 0 || (req->cryptlen == 0))
+        reqctx->newdstsg = NULL;
+        dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
+                                                   authsize);
+        if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0))
                 goto err;
 
         if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
@@ -1410,14 +1962,24 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
         if (src_nent < 0)
                 goto err;
         src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
-        reqctx->dst = src;
 
         if (req->src != req->dst) {
-                err = chcr_copy_assoc(req, aeadctx);
-                if (err)
-                        return ERR_PTR(err);
-                reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
-                                               req->assoclen);
+                error = chcr_copy_assoc(req, aeadctx);
+                if (error)
+                        return ERR_PTR(error);
+        }
+        if (dst_size && is_newsg(req->dst, &nents)) {
+                reqctx->newdstsg = alloc_new_sg(req->dst, nents);
+                if (IS_ERR(reqctx->newdstsg))
+                        return ERR_CAST(reqctx->newdstsg);
+                reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
+                                               reqctx->newdstsg, req->assoclen);
+        } else {
+                if (req->src == req->dst)
+                        reqctx->dst = src;
+                else
+                        reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
+                                                       req->dst, req->assoclen);
         }
         if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
                 null = 1;
@@ -1427,6 +1989,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
                                              (op_type ? -authsize : authsize));
         if (reqctx->dst_nents < 0) {
                 pr_err("AUTHENC:Invalid Destination sg entries\n");
+                error = -EINVAL;
                 goto err;
         }
         dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
@@ -1437,11 +2000,16 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
                         T6_MAX_AAD_SIZE,
                         transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8),
                                 op_type)) {
+                atomic_inc(&adap->chcr_stats.fallback);
+                free_new_sg(reqctx->newdstsg);
+                reqctx->newdstsg = NULL;
                 return ERR_PTR(chcr_aead_fallback(req, op_type));
         }
         skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
-        if (!skb)
+        if (!skb) {
+                error = -ENOMEM;
                 goto err;
+        }
 
         /* LLD is going to write the sge hdr. */
         skb_reserve(skb, sizeof(struct sge_opaque_hdr));
@@ -1493,9 +2061,9 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
         sg_param.nents = reqctx->dst_nents;
         sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
         sg_param.qid = qid;
-        sg_param.align = 0;
-        if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
-                                  &sg_param))
+        error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
+                                        reqctx->dst, &sg_param);
+        if (error)
                 goto dstmap_fail;
 
         skb_set_transport_header(skb, transhdr_len);
@@ -1507,8 +2075,9 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
         }
         write_buffer_to_skb(skb, &frags, req->iv, ivsize);
         write_sg_to_skb(skb, &frags, src, req->cryptlen);
-        create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1,
-                   sizeof(struct cpl_rx_phys_dsgl) + dst_size);
+        atomic_inc(&adap->chcr_stats.cipher_rqst);
+        create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1,
+                   sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
         reqctx->skb = skb;
         skb_get(skb);
 
@@ -1517,7 +2086,9 @@ dstmap_fail:
         /* ivmap_fail: */
         kfree_skb(skb);
 err:
-        return ERR_PTR(-EINVAL);
+        free_new_sg(reqctx->newdstsg);
+        reqctx->newdstsg = NULL;
+        return ERR_PTR(error);
 }
 
 static int set_msg_len(u8 *block, unsigned int msglen, int csize)
@@ -1724,14 +2295,17 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
         struct phys_sge_parm sg_param;
         struct scatterlist *src;
         unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE;
-        unsigned int dst_size = 0, kctx_len;
+        unsigned int dst_size = 0, kctx_len, nents;
         unsigned int sub_type;
         unsigned int authsize = crypto_aead_authsize(tfm);
-        int err = -EINVAL, src_nent;
+        int error = -EINVAL, src_nent;
         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
                 GFP_ATOMIC;
+        struct adapter *adap = padap(ctx->dev);
 
-
+        dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
+                                                   authsize);
+        reqctx->newdstsg = NULL;
         if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
                 goto err;
         src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
@@ -1740,26 +2314,35 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
 
         sub_type = get_aead_subtype(tfm);
         src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
-        reqctx->dst = src;
-
         if (req->src != req->dst) {
-                err = chcr_copy_assoc(req, aeadctx);
-                if (err) {
+                error = chcr_copy_assoc(req, aeadctx);
+                if (error) {
                         pr_err("AAD copy to destination buffer fails\n");
-                        return ERR_PTR(err);
+                        return ERR_PTR(error);
                 }
-                reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
-                                               req->assoclen);
+        }
+        if (dst_size && is_newsg(req->dst, &nents)) {
+                reqctx->newdstsg = alloc_new_sg(req->dst, nents);
+                if (IS_ERR(reqctx->newdstsg))
+                        return ERR_CAST(reqctx->newdstsg);
+                reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
+                                               reqctx->newdstsg, req->assoclen);
+        } else {
+                if (req->src == req->dst)
+                        reqctx->dst = src;
+                else
+                        reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
+                                                       req->dst, req->assoclen);
         }
         reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
                                              (op_type ? -authsize : authsize));
         if (reqctx->dst_nents < 0) {
                 pr_err("CCM:Invalid Destination sg entries\n");
+                error = -EINVAL;
                 goto err;
         }
-
-
-        if (aead_ccm_validate_input(op_type, req, aeadctx, sub_type))
+        error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type);
+        if (error)
                 goto err;
 
         dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
@@ -1769,13 +2352,18 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
                             T6_MAX_AAD_SIZE - 18,
                             transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8),
                             op_type)) {
+                atomic_inc(&adap->chcr_stats.fallback);
+                free_new_sg(reqctx->newdstsg);
+                reqctx->newdstsg = NULL;
                 return ERR_PTR(chcr_aead_fallback(req, op_type));
         }
 
         skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)),  flags);
 
-        if (!skb)
+        if (!skb) {
+                error = -ENOMEM;
                 goto err;
+        }
 
         skb_reserve(skb, sizeof(struct sge_opaque_hdr));
 
@@ -1790,29 +2378,32 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
                                         16), aeadctx->key, aeadctx->enckey_len);
 
         phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
-        if (ccm_format_packet(req, aeadctx, sub_type, op_type))
+        error = ccm_format_packet(req, aeadctx, sub_type, op_type);
+        if (error)
                 goto dstmap_fail;
 
         sg_param.nents = reqctx->dst_nents;
         sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
         sg_param.qid = qid;
-        sg_param.align = 0;
-        if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
-                                  &sg_param))
+        error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
+                                 reqctx->dst, &sg_param);
+        if (error)
                 goto dstmap_fail;
 
         skb_set_transport_header(skb, transhdr_len);
         frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx);
-        create_wreq(ctx, chcr_req, req, skb, kctx_len, 0, 1,
-                    sizeof(struct cpl_rx_phys_dsgl) + dst_size);
+        atomic_inc(&adap->chcr_stats.aead_rqst);
+        create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1,
+                    sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
         reqctx->skb = skb;
         skb_get(skb);
         return skb;
 dstmap_fail:
         kfree_skb(skb);
-        skb = NULL;
 err:
-        return ERR_PTR(-EINVAL);
+        free_new_sg(reqctx->newdstsg);
+        reqctx->newdstsg = NULL;
+        return ERR_PTR(error);
 }
 
 static struct sk_buff *create_gcm_wr(struct aead_request *req,
@@ -1832,45 +2423,53 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
         struct scatterlist *src;
         unsigned int frags = 0, transhdr_len;
         unsigned int ivsize = AES_BLOCK_SIZE;
-        unsigned int dst_size = 0, kctx_len;
+        unsigned int dst_size = 0, kctx_len, nents, assoclen = req->assoclen;
         unsigned char tag_offset = 0;
-        unsigned int crypt_len = 0;
         unsigned int authsize = crypto_aead_authsize(tfm);
-        int err = -EINVAL, src_nent;
+        int error = -EINVAL, src_nent;
         gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
                 GFP_ATOMIC;
+        struct adapter *adap = padap(ctx->dev);
 
+        reqctx->newdstsg = NULL;
+        dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
+                                                    authsize);
         /* validate key size */
         if (aeadctx->enckey_len == 0)
                 goto err;
 
         if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
                 goto err;
-        src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
+        src_nent = sg_nents_for_len(req->src, assoclen + req->cryptlen);
         if (src_nent < 0)
                 goto err;
 
-        src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
-        reqctx->dst = src;
+        src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen);
         if (req->src != req->dst) {
-                err = chcr_copy_assoc(req, aeadctx);
-                if (err)
-                        return  ERR_PTR(err);
-                reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
-                                               req->assoclen);
+                error = chcr_copy_assoc(req, aeadctx);
+                if (error)
+                        return  ERR_PTR(error);
+        }
+
+        if (dst_size && is_newsg(req->dst, &nents)) {
+                reqctx->newdstsg = alloc_new_sg(req->dst, nents);
+                if (IS_ERR(reqctx->newdstsg))
+                        return ERR_CAST(reqctx->newdstsg);
+                reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
+                                               reqctx->newdstsg, assoclen);
+        } else {
+                if (req->src == req->dst)
+                        reqctx->dst = src;
+                else
+                        reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
+                                                       req->dst, assoclen);
         }
 
-        if (!req->cryptlen)
-                /* null-payload is not supported in the hardware.
-                 * software is sending block size
-                 */
-                crypt_len = AES_BLOCK_SIZE;
-        else
-                crypt_len = req->cryptlen;
         reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
                                              (op_type ? -authsize : authsize));
         if (reqctx->dst_nents < 0) {
                 pr_err("GCM:Invalid Destination sg entries\n");
+                error = -EINVAL;
                 goto err;
         }
 
@@ -1883,11 +2482,16 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
                             T6_MAX_AAD_SIZE,
                             transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8),
                             op_type)) {
+                atomic_inc(&adap->chcr_stats.fallback);
+                free_new_sg(reqctx->newdstsg);
+                reqctx->newdstsg = NULL;
                 return ERR_PTR(chcr_aead_fallback(req, op_type));
         }
         skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
-        if (!skb)
+        if (!skb) {
+                error = -ENOMEM;
                 goto err;
+        }
 
         /* NIC driver is going to write the sge hdr. */
         skb_reserve(skb, sizeof(struct sge_opaque_hdr));
@@ -1896,19 +2500,19 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
         memset(chcr_req, 0, transhdr_len);
 
         if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
-                req->assoclen -= 8;
+                assoclen = req->assoclen - 8;
 
         tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
         chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
                                         ctx->dev->rx_channel_id, 2, (ivsize ?
-                                        (req->assoclen + 1) : 0));
+                                        (assoclen + 1) : 0));
         chcr_req->sec_cpl.pldlen =
-                htonl(req->assoclen + ivsize + req->cryptlen);
+                htonl(assoclen + ivsize + req->cryptlen);
         chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
-                                        req->assoclen ? 1 : 0, req->assoclen,
-                                        req->assoclen + ivsize + 1, 0);
+                                        assoclen ? 1 : 0, assoclen,
+                                        assoclen + ivsize + 1, 0);
                 chcr_req->sec_cpl.cipherstop_lo_authinsert =
-                        FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + ivsize + 1,
+                        FILL_SEC_CPL_AUTHINSERT(0, assoclen + ivsize + 1,
                                                 tag_offset, tag_offset);
                 chcr_req->sec_cpl.seqno_numivs =
                         FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
@@ -1938,19 +2542,19 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
         sg_param.nents = reqctx->dst_nents;
         sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
         sg_param.qid = qid;
-        sg_param.align = 0;
-        if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
-                                  &sg_param))
+        error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
+                                          reqctx->dst, &sg_param);
+        if (error)
                 goto dstmap_fail;
 
         skb_set_transport_header(skb, transhdr_len);
-
-        write_sg_to_skb(skb, &frags, req->src, req->assoclen);
-
+        write_sg_to_skb(skb, &frags, req->src, assoclen);
         write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
         write_sg_to_skb(skb, &frags, src, req->cryptlen);
-        create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1,
-                        sizeof(struct cpl_rx_phys_dsgl) + dst_size);
+        atomic_inc(&adap->chcr_stats.aead_rqst);
+        create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1,
+                        sizeof(struct cpl_rx_phys_dsgl) + dst_size,
+                        reqctx->verify);
         reqctx->skb = skb;
         skb_get(skb);
         return skb;
@@ -1958,9 +2562,10 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
 dstmap_fail:
         /* ivmap_fail: */
         kfree_skb(skb);
-        skb = NULL;
 err:
-        return skb;
+        free_new_sg(reqctx->newdstsg);
+        reqctx->newdstsg = NULL;
+        return ERR_PTR(error);
 }
 
 
@@ -1972,7 +2577,8 @@ static int chcr_aead_cra_init(struct crypto_aead *tfm)
         struct aead_alg *alg = crypto_aead_alg(tfm);
 
         aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
-                                               CRYPTO_ALG_NEED_FALLBACK);
+                                               CRYPTO_ALG_NEED_FALLBACK |
+                                               CRYPTO_ALG_ASYNC);
         if  (IS_ERR(aeadctx->sw_cipher))
                 return PTR_ERR(aeadctx->sw_cipher);
         crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
@@ -2206,7 +2812,8 @@ static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
                                     unsigned int keylen)
 {
         struct chcr_context *ctx = crypto_aead_ctx(aead);
-         struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+        struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
+        int error;
 
         if (keylen < 3) {
                 crypto_tfm_set_flags((struct crypto_tfm *)aead,
@@ -2214,6 +2821,15 @@ static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
                 aeadctx->enckey_len = 0;
                 return  -EINVAL;
         }
+        crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
+        crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
+                              CRYPTO_TFM_REQ_MASK);
+        error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
+        crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
+        crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
+                              CRYPTO_TFM_RES_MASK);
+        if (error)
+                return error;
         keylen -= 3;
         memcpy(aeadctx->salt, key + keylen, 3);
         return chcr_ccm_common_setkey(aead, key, keylen);
@@ -2552,22 +3168,14 @@ static int chcr_aead_op(struct aead_request *req,
 static struct chcr_alg_template driver_algs[] = {
         /* AES-CBC */
         {
-                .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+                .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
                 .is_registered = 0,
                 .alg.crypto = {
                         .cra_name               = "cbc(aes)",
                         .cra_driver_name        = "cbc-aes-chcr",
-                        .cra_priority           = CHCR_CRA_PRIORITY,
-                        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
-                                CRYPTO_ALG_ASYNC,
                         .cra_blocksize          = AES_BLOCK_SIZE,
-                        .cra_ctxsize            = sizeof(struct chcr_context)
-                                + sizeof(struct ablk_ctx),
-                        .cra_alignmask          = 0,
-                        .cra_type               = &crypto_ablkcipher_type,
-                        .cra_module             = THIS_MODULE,
                         .cra_init               = chcr_cra_init,
-                        .cra_exit               = NULL,
+                        .cra_exit               = chcr_cra_exit,
                         .cra_u.ablkcipher       = {
                                 .min_keysize    = AES_MIN_KEY_SIZE,
                                 .max_keysize    = AES_MAX_KEY_SIZE,
@@ -2579,24 +3187,15 @@ static struct chcr_alg_template driver_algs[] = {
                 }
         },
         {
-                .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+                .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
                 .is_registered = 0,
                 .alg.crypto =   {
                         .cra_name               = "xts(aes)",
                         .cra_driver_name        = "xts-aes-chcr",
-                        .cra_priority           = CHCR_CRA_PRIORITY,
-                        .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
-                                CRYPTO_ALG_ASYNC,
                         .cra_blocksize          = AES_BLOCK_SIZE,
-                        .cra_ctxsize            = sizeof(struct chcr_context) +
-                                sizeof(struct ablk_ctx),
-                        .cra_alignmask          = 0,
-                        .cra_type               = &crypto_ablkcipher_type,
-                        .cra_module             = THIS_MODULE,
                         .cra_init               = chcr_cra_init,
                         .cra_exit               = NULL,
-                        .cra_u = {
-                                .ablkcipher = {
+                        .cra_u .ablkcipher = {
                                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
                                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
                                         .ivsize         = AES_BLOCK_SIZE,
@@ -2605,6 +3204,47 @@ static struct chcr_alg_template driver_algs[] = {
                                         .decrypt        = chcr_aes_decrypt,
                                 }
                         }
+        },
+        {
+                .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
+                .is_registered = 0,
+                .alg.crypto = {
+                        .cra_name               = "ctr(aes)",
+                        .cra_driver_name        = "ctr-aes-chcr",
+                        .cra_blocksize          = 1,
+                        .cra_init               = chcr_cra_init,
+                        .cra_exit               = chcr_cra_exit,
+                        .cra_u.ablkcipher       = {
+                                .min_keysize    = AES_MIN_KEY_SIZE,
+                                .max_keysize    = AES_MAX_KEY_SIZE,
+                                .ivsize         = AES_BLOCK_SIZE,
+                                .setkey         = chcr_aes_ctr_setkey,
+                                .encrypt        = chcr_aes_encrypt,
+                                .decrypt        = chcr_aes_decrypt,
+                        }
+                }
+        },
+        {
+                .type = CRYPTO_ALG_TYPE_ABLKCIPHER |
+                        CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
+                .is_registered = 0,
+                .alg.crypto = {
+                        .cra_name               = "rfc3686(ctr(aes))",
+                        .cra_driver_name        = "rfc3686-ctr-aes-chcr",
+                        .cra_blocksize          = 1,
+                        .cra_init               = chcr_rfc3686_init,
+                        .cra_exit               = chcr_cra_exit,
+                        .cra_u.ablkcipher       = {
+                                .min_keysize    = AES_MIN_KEY_SIZE +
+                                        CTR_RFC3686_NONCE_SIZE,
+                                .max_keysize    = AES_MAX_KEY_SIZE +
+                                        CTR_RFC3686_NONCE_SIZE,
+                                .ivsize         = CTR_RFC3686_IV_SIZE,
+                                .setkey         = chcr_aes_rfc3686_setkey,
+                                .encrypt        = chcr_aes_encrypt,
+                                .decrypt        = chcr_aes_decrypt,
+                                .geniv          = "seqiv",
+                        }
                 }
         },
         /* SHA */
@@ -2986,6 +3626,18 @@ static int chcr_register_alg(void)
                         continue;
                 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
                 case CRYPTO_ALG_TYPE_ABLKCIPHER:
+                        driver_algs[i].alg.crypto.cra_priority =
+                                CHCR_CRA_PRIORITY;
+                        driver_algs[i].alg.crypto.cra_module = THIS_MODULE;
+                        driver_algs[i].alg.crypto.cra_flags =
+                                CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC |
+                                CRYPTO_ALG_NEED_FALLBACK;
+                        driver_algs[i].alg.crypto.cra_ctxsize =
+                                sizeof(struct chcr_context) +
+                                sizeof(struct ablk_ctx);
+                        driver_algs[i].alg.crypto.cra_alignmask = 0;
+                        driver_algs[i].alg.crypto.cra_type =
+                                &crypto_ablkcipher_type;
                         err = crypto_register_alg(&driver_algs[i].alg.crypto);
                         name = driver_algs[i].alg.crypto.cra_driver_name;
                         break;
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
index 751d06a58101..583008de51a3 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -185,11 +185,11 @@
                         FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \
                         FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len)))
 
-#define FILL_WR_RX_Q_ID(cid, qid, wr_iv, fid) \
+#define FILL_WR_RX_Q_ID(cid, qid, wr_iv, lcb, fid) \
                 htonl( \
                         FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \
                         FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \
-                        FW_CRYPTO_LOOKASIDE_WR_LCB_V(0) | \
+                        FW_CRYPTO_LOOKASIDE_WR_LCB_V((lcb)) | \
                         FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv)) | \
                         FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(fid))
 
@@ -219,9 +219,26 @@
 #define MAX_NK 8
 #define CRYPTO_MAX_IMM_TX_PKT_LEN 256
 #define MAX_WR_SIZE                     512
+#define ROUND_16(bytes)         ((bytes) & 0xFFFFFFF0)
+#define MAX_DSGL_ENT                    32
+#define MAX_DIGEST_SKB_SGE      (MAX_SKB_FRAGS - 2)
+#define MIN_CIPHER_SG                   1 /* IV */
 #define MIN_AUTH_SG                     2 /*IV + AAD*/
 #define MIN_GCM_SG                      2 /* IV + AAD*/
+#define MIN_DIGEST_SG                   1 /*Partial Buffer*/
 #define MIN_CCM_SG                      3 /*IV+AAD+B0*/
+#define SPACE_LEFT(len) \
+        ((MAX_WR_SIZE - WR_MIN_LEN - (len)))
+
+unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40,
+                                48, 64, 72, 88,
+                                96, 112, 120, 136,
+                                144, 160, 168, 184,
+                                192};
+unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80,
+                                112, 112, 128, 128, 144, 144, 160, 160,
+                                192, 192, 208, 208, 224, 224, 240, 240,
+                                272, 272, 288, 288, 304, 304, 320, 320};
 
 struct algo_param {
         unsigned int auth_mode;
@@ -239,6 +256,14 @@ struct hash_wr_param {
         u64 scmd1;
 };
 
+struct cipher_wr_param {
+        struct ablkcipher_request *req;
+        struct scatterlist *srcsg;
+        char *iv;
+        int bytes;
+        short int snent;
+        unsigned short qid;
+};
 enum {
         AES_KEYLENGTH_128BIT = 128,
         AES_KEYLENGTH_192BIT = 192,
@@ -293,7 +318,6 @@ struct phys_sge_parm {
         unsigned int nents;
         unsigned int obsize;
         unsigned short qid;
-        unsigned char align;
 };
 
 struct crypto_result {
diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c
index c28e018e0773..b6dd9cbe815f 100644
--- a/drivers/crypto/chelsio/chcr_core.c
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -29,6 +29,7 @@
 static LIST_HEAD(uld_ctx_list);
 static DEFINE_MUTEX(dev_mutex);
 static atomic_t dev_count;
+static struct uld_ctx *ctx_rr;
 
 typedef int (*chcr_handler_func)(struct chcr_dev *dev, unsigned char *input);
 static int cpl_fw6_pld_handler(struct chcr_dev *dev, unsigned char *input);
@@ -49,25 +50,28 @@ static struct cxgb4_uld_info chcr_uld_info = {
         .rx_handler = chcr_uld_rx_handler,
 };
 
-int assign_chcr_device(struct chcr_dev **dev)
+struct uld_ctx *assign_chcr_device(void)
 {
-        struct uld_ctx *u_ctx;
-        int ret = -ENXIO;
+        struct uld_ctx *u_ctx = NULL;
 
         /*
-         * Which device to use if multiple devices are available TODO
-         * May be select the device based on round robin. One session
-         * must go to the same device to maintain the ordering.
+         * When multiple devices are present in system select
+         * device in round-robin fashion for crypto operations
+         * Although One session must use the same device to
+         * maintain request-response ordering.
          */
-        mutex_lock(&dev_mutex); /* TODO ? */
-        list_for_each_entry(u_ctx, &uld_ctx_list, entry)
-                if (u_ctx->dev) {
-                        *dev = u_ctx->dev;
-                        ret = 0;
-                        break;
+        mutex_lock(&dev_mutex);
+        if (!list_empty(&uld_ctx_list)) {
+                u_ctx = ctx_rr;
+                if (list_is_last(&ctx_rr->entry, &uld_ctx_list))
+                        ctx_rr = list_first_entry(&uld_ctx_list,
+                                                  struct uld_ctx,
+                                                  entry);
+                else
+                        ctx_rr = list_next_entry(ctx_rr, entry);
         }
         mutex_unlock(&dev_mutex);
-        return ret;
+        return u_ctx;
 }
 
 static int chcr_dev_add(struct uld_ctx *u_ctx)
@@ -82,11 +86,27 @@ static int chcr_dev_add(struct uld_ctx *u_ctx)
         u_ctx->dev = dev;
         dev->u_ctx = u_ctx;
         atomic_inc(&dev_count);
+        mutex_lock(&dev_mutex);
+        list_add_tail(&u_ctx->entry, &uld_ctx_list);
+        if (!ctx_rr)
+                ctx_rr = u_ctx;
+        mutex_unlock(&dev_mutex);
         return 0;
 }
 
 static int chcr_dev_remove(struct uld_ctx *u_ctx)
 {
+        if (ctx_rr == u_ctx) {
+                if (list_is_last(&ctx_rr->entry, &uld_ctx_list))
+                        ctx_rr = list_first_entry(&uld_ctx_list,
+                                                  struct uld_ctx,
+                                                  entry);
+                else
+                        ctx_rr = list_next_entry(ctx_rr, entry);
+        }
+        list_del(&u_ctx->entry);
+        if (list_empty(&uld_ctx_list))
+                ctx_rr = NULL;
         kfree(u_ctx->dev);
         u_ctx->dev = NULL;
         atomic_dec(&dev_count);
@@ -100,6 +120,7 @@ static int cpl_fw6_pld_handler(struct chcr_dev *dev,
         struct cpl_fw6_pld *fw6_pld;
         u32 ack_err_status = 0;
         int error_status = 0;
+        struct adapter *adap = padap(dev);
 
         fw6_pld = (struct cpl_fw6_pld *)input;
         req = (struct crypto_async_request *)(uintptr_t)be64_to_cpu(
@@ -111,11 +132,11 @@ static int cpl_fw6_pld_handler(struct chcr_dev *dev,
                 if (CHK_MAC_ERR_BIT(ack_err_status) ||
                     CHK_PAD_ERR_BIT(ack_err_status))
                         error_status = -EBADMSG;
+                atomic_inc(&adap->chcr_stats.error);
         }
         /* call completion callback with failure status */
         if (req) {
                 error_status = chcr_handle_resp(req, input, error_status);
-                req->complete(req, error_status);
         } else {
                 pr_err("Incorrect request address from the firmware\n");
                 return -EFAULT;
@@ -138,10 +159,11 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld)
                 u_ctx = ERR_PTR(-ENOMEM);
                 goto out;
         }
+        if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) {
+                u_ctx = ERR_PTR(-ENOMEM);
+                goto out;
+        }
         u_ctx->lldi = *lld;
-        mutex_lock(&dev_mutex);
-        list_add_tail(&u_ctx->entry, &uld_ctx_list);
-        mutex_unlock(&dev_mutex);
 out:
         return u_ctx;
 }
diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h
index cd0c35a18d92..c9a19b2a1e9f 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -53,6 +53,9 @@
 #define MAC_ERROR_BIT           0
 #define CHK_MAC_ERR_BIT(x)      (((x) >> MAC_ERROR_BIT) & 1)
 #define MAX_SALT                4
+#define WR_MIN_LEN (sizeof(struct chcr_wr) + \
+                    sizeof(struct cpl_rx_phys_dsgl) + \
+                    sizeof(struct ulptx_sgl))
 
 #define padap(dev) pci_get_drvdata(dev->u_ctx->lldi.pdev)
 
@@ -86,7 +89,7 @@ struct uld_ctx {
         struct chcr_dev *dev;
 };
 
-int assign_chcr_device(struct chcr_dev **dev);
+struct uld_ctx * assign_chcr_device(void);
 int chcr_send_wr(struct sk_buff *skb);
 int start_crypto(void);
 int stop_crypto(void);
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index 5b2fabb14229..a4f95b014b46 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -139,6 +139,9 @@
 #define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309    0x06000000
 #define CRYPTO_ALG_SUB_TYPE_AEAD_NULL       0x07000000
 #define CRYPTO_ALG_SUB_TYPE_CTR             0x08000000
+#define CRYPTO_ALG_SUB_TYPE_CTR_RFC3686     0x09000000
+#define CRYPTO_ALG_SUB_TYPE_XTS             0x0a000000
+#define CRYPTO_ALG_SUB_TYPE_CBC             0x0b000000
 #define CRYPTO_ALG_TYPE_HMAC (CRYPTO_ALG_TYPE_AHASH |\
                               CRYPTO_ALG_SUB_TYPE_HASH_HMAC)
 
@@ -146,19 +149,23 @@
 
 #define CHCR_HASH_MAX_BLOCK_SIZE_64  64
 #define CHCR_HASH_MAX_BLOCK_SIZE_128 128
+#define CHCR_SG_SIZE 2048
 
 /* Aligned to 128 bit boundary */
 
 struct ablk_ctx {
+        struct crypto_skcipher *sw_cipher;
         __be32 key_ctx_hdr;
         unsigned int enckey_len;
-        u8 key[CHCR_AES_MAX_KEY_LEN];
         unsigned char ciph_mode;
+        u8 key[CHCR_AES_MAX_KEY_LEN];
+        u8 nonce[4];
         u8 rrkey[AES_MAX_KEY_SIZE];
 };
 struct chcr_aead_reqctx {
         struct  sk_buff *skb;
         struct scatterlist *dst;
+        struct scatterlist *newdstsg;
         struct scatterlist srcffwd[2];
         struct scatterlist dstffwd[2];
         short int dst_nents;
@@ -233,7 +240,14 @@ struct chcr_ahash_req_ctx {
 
 struct chcr_blkcipher_req_ctx {
         struct sk_buff *skb;
-        unsigned int dst_nents;
+        struct scatterlist srcffwd[2];
+        struct scatterlist dstffwd[2];
+        struct scatterlist *dstsg;
+        struct scatterlist *dst;
+        struct scatterlist *newdstsg;
+        unsigned int processed;
+        unsigned int op;
+        short int dst_nents;
         u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
 };
 
@@ -275,5 +289,10 @@ static int chcr_aead_op(struct aead_request *req_base,
                           int size,
                           create_wr_t create_wr_fn);
 static inline int get_aead_subtype(struct crypto_aead *aead);
-
+static int is_newsg(struct scatterlist *sgl, unsigned int *newents);
+static struct scatterlist *alloc_new_sg(struct scatterlist *sgl,
+                                        unsigned int nents);
+static inline void free_new_sg(struct scatterlist *sgl);
+static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
+                                   unsigned char *input, int err);
 #endif /* __CHCR_CRYPTO_H__ */
diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c
index 9b07f3d88feb..0c6a917a9ab8 100644
--- a/drivers/crypto/img-hash.c
+++ b/drivers/crypto/img-hash.c
@@ -1088,9 +1088,17 @@ static int img_hash_suspend(struct device *dev)
 static int img_hash_resume(struct device *dev)
 {
         struct img_hash_dev *hdev = dev_get_drvdata(dev);
+        int ret;
 
-        clk_prepare_enable(hdev->hash_clk);
-        clk_prepare_enable(hdev->sys_clk);
+        ret = clk_prepare_enable(hdev->hash_clk);
+        if (ret)
+                return ret;
+
+        ret = clk_prepare_enable(hdev->sys_clk);
+        if (ret) {
+                clk_disable_unprepare(hdev->hash_clk);
+                return ret;
+        }
 
         return 0;
 }
diff --git a/drivers/crypto/inside-secure/Makefile b/drivers/crypto/inside-secure/Makefile
new file mode 100644
index 000000000000..302f07dde98c
--- /dev/null
+++ b/drivers/crypto/inside-secure/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += crypto_safexcel.o
+crypto_safexcel-objs := safexcel.o safexcel_ring.o safexcel_cipher.o safexcel_hash.o
diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c
new file mode 100644
index 000000000000..e7f87ac12685
--- /dev/null
+++ b/drivers/crypto/inside-secure/safexcel.c
@@ -0,0 +1,926 @@
+/*
+ * Copyright (C) 2017 Marvell
+ *
+ * Antoine Tenart <antoine.tenart@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/firmware.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/workqueue.h>
+
+#include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
+
+#include "safexcel.h"
+
+static u32 max_rings = EIP197_MAX_RINGS;
+module_param(max_rings, uint, 0644);
+MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
+
+static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
+{
+        u32 val, htable_offset;
+        int i;
+
+        /* Enable the record cache memory access */
+        val = readl(priv->base + EIP197_CS_RAM_CTRL);
+        val &= ~EIP197_TRC_ENABLE_MASK;
+        val |= EIP197_TRC_ENABLE_0;
+        writel(val, priv->base + EIP197_CS_RAM_CTRL);
+
+        /* Clear all ECC errors */
+        writel(0, priv->base + EIP197_TRC_ECCCTRL);
+
+        /*
+         * Make sure the cache memory is accessible by taking record cache into
+         * reset.
+         */
+        val = readl(priv->base + EIP197_TRC_PARAMS);
+        val |= EIP197_TRC_PARAMS_SW_RESET;
+        val &= ~EIP197_TRC_PARAMS_DATA_ACCESS;
+        writel(val, priv->base + EIP197_TRC_PARAMS);
+
+        /* Clear all records */
+        for (i = 0; i < EIP197_CS_RC_MAX; i++) {
+                u32 val, offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
+
+                writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
+                       EIP197_CS_RC_PREV(EIP197_RC_NULL),
+                       priv->base + offset);
+
+                val = EIP197_CS_RC_NEXT(i+1) | EIP197_CS_RC_PREV(i-1);
+                if (i == 0)
+                        val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
+                else if (i == EIP197_CS_RC_MAX - 1)
+                        val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
+                writel(val, priv->base + offset + sizeof(u32));
+        }
+
+        /* Clear the hash table entries */
+        htable_offset = EIP197_CS_RC_MAX * EIP197_CS_RC_SIZE;
+        for (i = 0; i < 64; i++)
+                writel(GENMASK(29, 0),
+                       priv->base + EIP197_CLASSIFICATION_RAMS + htable_offset + i * sizeof(u32));
+
+        /* Disable the record cache memory access */
+        val = readl(priv->base + EIP197_CS_RAM_CTRL);
+        val &= ~EIP197_TRC_ENABLE_MASK;
+        writel(val, priv->base + EIP197_CS_RAM_CTRL);
+
+        /* Write head and tail pointers of the record free chain */
+        val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
+              EIP197_TRC_FREECHAIN_TAIL_PTR(EIP197_CS_RC_MAX - 1);
+        writel(val, priv->base + EIP197_TRC_FREECHAIN);
+
+        /* Configure the record cache #1 */
+        val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(EIP197_CS_TRC_REC_WC) |
+              EIP197_TRC_PARAMS2_HTABLE_PTR(EIP197_CS_RC_MAX);
+        writel(val, priv->base + EIP197_TRC_PARAMS2);
+
+        /* Configure the record cache #2 */
+        val = EIP197_TRC_PARAMS_RC_SZ_LARGE(EIP197_CS_TRC_LG_REC_WC) |
+              EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
+              EIP197_TRC_PARAMS_HTABLE_SZ(2);
+        writel(val, priv->base + EIP197_TRC_PARAMS);
+}
+
+static void eip197_write_firmware(struct safexcel_crypto_priv *priv,
+                                  const struct firmware *fw, u32 ctrl,
+                                  u32 prog_en)
+{
+        const u32 *data = (const u32 *)fw->data;
+        u32 val;
+        int i;
+
+        /* Reset the engine to make its program memory accessible */
+        writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
+               EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
+               EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
+               priv->base + ctrl);
+
+        /* Enable access to the program memory */
+        writel(prog_en, priv->base + EIP197_PE_ICE_RAM_CTRL);
+
+        /* Write the firmware */
+        for (i = 0; i < fw->size / sizeof(u32); i++)
+                writel(be32_to_cpu(data[i]),
+                       priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32));
+
+        /* Disable access to the program memory */
+        writel(0, priv->base + EIP197_PE_ICE_RAM_CTRL);
+
+        /* Release engine from reset */
+        val = readl(priv->base + ctrl);
+        val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET;
+        writel(val, priv->base + ctrl);
+}
+
+static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
+{
+        const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
+        const struct firmware *fw[FW_NB];
+        int i, j, ret = 0;
+        u32 val;
+
+        for (i = 0; i < FW_NB; i++) {
+                ret = request_firmware(&fw[i], fw_name[i], priv->dev);
+                if (ret) {
+                        dev_err(priv->dev,
+                                "Failed to request firmware %s (%d)\n",
+                                fw_name[i], ret);
+                        goto release_fw;
+                }
+         }
+
+        /* Clear the scratchpad memory */
+        val = readl(priv->base + EIP197_PE_ICE_SCRATCH_CTRL);
+        val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
+               EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
+               EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
+               EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
+        writel(val, priv->base + EIP197_PE_ICE_SCRATCH_CTRL);
+
+        memset(priv->base + EIP197_PE_ICE_SCRATCH_RAM, 0,
+               EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32));
+
+        eip197_write_firmware(priv, fw[FW_IFPP], EIP197_PE_ICE_FPP_CTRL,
+                              EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN);
+
+        eip197_write_firmware(priv, fw[FW_IPUE], EIP197_PE_ICE_PUE_CTRL,
+                              EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN);
+
+release_fw:
+        for (j = 0; j < i; j++)
+                release_firmware(fw[j]);
+
+        return ret;
+}
+
+static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
+{
+        u32 hdw, cd_size_rnd, val;
+        int i;
+
+        hdw = readl(priv->base + EIP197_HIA_OPTIONS);
+        hdw &= GENMASK(27, 25);
+        hdw >>= 25;
+
+        cd_size_rnd = (priv->config.cd_size + (BIT(hdw) - 1)) >> hdw;
+
+        for (i = 0; i < priv->config.rings; i++) {
+                /* ring base address */
+                writel(lower_32_bits(priv->ring[i].cdr.base_dma),
+                       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
+                writel(upper_32_bits(priv->ring[i].cdr.base_dma),
+                       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
+
+                writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) |
+                       priv->config.cd_size,
+                       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DESC_SIZE);
+                writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) |
+                       (EIP197_FETCH_COUNT * priv->config.cd_offset),
+                       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG);
+
+                /* Configure DMA tx control */
+                val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
+                val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
+                writel(val,
+                       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DMA_CFG);
+
+                /* clear any pending interrupt */
+                writel(GENMASK(5, 0),
+                       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_STAT);
+        }
+
+        return 0;
+}
+
+static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
+{
+        u32 hdw, rd_size_rnd, val;
+        int i;
+
+        hdw = readl(priv->base + EIP197_HIA_OPTIONS);
+        hdw &= GENMASK(27, 25);
+        hdw >>= 25;
+
+        rd_size_rnd = (priv->config.rd_size + (BIT(hdw) - 1)) >> hdw;
+
+        for (i = 0; i < priv->config.rings; i++) {
+                /* ring base address */
+                writel(lower_32_bits(priv->ring[i].rdr.base_dma),
+                       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
+                writel(upper_32_bits(priv->ring[i].rdr.base_dma),
+                       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
+
+                writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) |
+                       priv->config.rd_size,
+                       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DESC_SIZE);
+
+                writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) |
+                       (EIP197_FETCH_COUNT * priv->config.rd_offset),
+                       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG);
+
+                /* Configure DMA tx control */
+                val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
+                val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
+                val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUG;
+                writel(val,
+                       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DMA_CFG);
+
+                /* clear any pending interrupt */
+                writel(GENMASK(7, 0),
+                       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_STAT);
+
+                /* enable ring interrupt */
+                val = readl(priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
+                val |= EIP197_RDR_IRQ(i);
+                writel(val, priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
+        }
+
+        return 0;
+}
+
+static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
+{
+        u32 version, val;
+        int i, ret;
+
+        /* Determine endianess and configure byte swap */
+        version = readl(priv->base + EIP197_HIA_VERSION);
+        val = readl(priv->base + EIP197_HIA_MST_CTRL);
+
+        if ((version & 0xffff) == EIP197_HIA_VERSION_BE)
+                val |= EIP197_MST_CTRL_BYTE_SWAP;
+        else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE)
+                val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24);
+
+        writel(val, priv->base + EIP197_HIA_MST_CTRL);
+
+
+        /* Configure wr/rd cache values */
+        writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
+               EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
+               priv->base + EIP197_MST_CTRL);
+
+        /* Interrupts reset */
+
+        /* Disable all global interrupts */
+        writel(0, priv->base + EIP197_HIA_AIC_G_ENABLE_CTRL);
+
+        /* Clear any pending interrupt */
+        writel(GENMASK(31, 0), priv->base + EIP197_HIA_AIC_G_ACK);
+
+        /* Data Fetch Engine configuration */
+
+        /* Reset all DFE threads */
+        writel(EIP197_DxE_THR_CTRL_RESET_PE,
+               priv->base + EIP197_HIA_DFE_THR_CTRL);
+
+        /* Reset HIA input interface arbiter */
+        writel(EIP197_HIA_RA_PE_CTRL_RESET,
+               priv->base + EIP197_HIA_RA_PE_CTRL);
+
+        /* DMA transfer size to use */
+        val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
+        val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
+        val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
+        val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
+        val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
+        writel(val, priv->base + EIP197_HIA_DFE_CFG);
+
+        /* Leave the DFE threads reset state */
+        writel(0, priv->base + EIP197_HIA_DFE_THR_CTRL);
+
+        /* Configure the procesing engine thresholds */
+        writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(9),
+              priv->base + EIP197_PE_IN_DBUF_THRES);
+        writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(7),
+              priv->base + EIP197_PE_IN_TBUF_THRES);
+
+        /* enable HIA input interface arbiter and rings */
+        writel(EIP197_HIA_RA_PE_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
+               priv->base + EIP197_HIA_RA_PE_CTRL);
+
+        /* Data Store Engine configuration */
+
+        /* Reset all DSE threads */
+        writel(EIP197_DxE_THR_CTRL_RESET_PE,
+               priv->base + EIP197_HIA_DSE_THR_CTRL);
+
+        /* Wait for all DSE threads to complete */
+        while ((readl(priv->base + EIP197_HIA_DSE_THR_STAT) &
+                GENMASK(15, 12)) != GENMASK(15, 12))
+                ;
+
+        /* DMA transfer size to use */
+        val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
+        val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
+        val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
+        val |= EIP197_HIA_DSE_CFG_ALLWAYS_BUFFERABLE;
+        val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
+        writel(val, priv->base + EIP197_HIA_DSE_CFG);
+
+        /* Leave the DSE threads reset state */
+        writel(0, priv->base + EIP197_HIA_DSE_THR_CTRL);
+
+        /* Configure the procesing engine thresholds */
+        writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | EIP197_PE_OUT_DBUF_THRES_MAX(8),
+               priv->base + EIP197_PE_OUT_DBUF_THRES);
+
+        /* Processing Engine configuration */
+
+        /* H/W capabilities selection */
+        val = EIP197_FUNCTION_RSVD;
+        val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY;
+        val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC;
+        val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1;
+        val |= EIP197_ALG_SHA2;
+        writel(val, priv->base + EIP197_PE_EIP96_FUNCTION_EN);
+
+        /* Command Descriptor Rings prepare */
+        for (i = 0; i < priv->config.rings; i++) {
+                /* Clear interrupts for this ring */
+                writel(GENMASK(31, 0),
+                       priv->base + EIP197_HIA_AIC_R_ENABLE_CLR(i));
+
+                /* Disable external triggering */
+                writel(0, priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG);
+
+                /* Clear the pending prepared counter */
+                writel(EIP197_xDR_PREP_CLR_COUNT,
+                       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_COUNT);
+
+                /* Clear the pending processed counter */
+                writel(EIP197_xDR_PROC_CLR_COUNT,
+                       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_COUNT);
+
+                writel(0,
+                       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_PNTR);
+                writel(0,
+                       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_PNTR);
+
+                writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2,
+                       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_SIZE);
+        }
+
+        /* Result Descriptor Ring prepare */
+        for (i = 0; i < priv->config.rings; i++) {
+                /* Disable external triggering*/
+                writel(0, priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG);
+
+                /* Clear the pending prepared counter */
+                writel(EIP197_xDR_PREP_CLR_COUNT,
+                       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_COUNT);
+
+                /* Clear the pending processed counter */
+                writel(EIP197_xDR_PROC_CLR_COUNT,
+                       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_COUNT);
+
+                writel(0,
+                       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_PNTR);
+                writel(0,
+                       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_PNTR);
+
+                /* Ring size */
+                writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2,
+                       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_SIZE);
+        }
+
+        /* Enable command descriptor rings */
+        writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
+               priv->base + EIP197_HIA_DFE_THR_CTRL);
+
+        /* Enable result descriptor rings */
+        writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
+               priv->base + EIP197_HIA_DSE_THR_CTRL);
+
+        /* Clear any HIA interrupt */
+        writel(GENMASK(30, 20), priv->base + EIP197_HIA_AIC_G_ACK);
+
+        eip197_trc_cache_init(priv);
+
+        ret = eip197_load_firmwares(priv);
+        if (ret)
+                return ret;
+
+        safexcel_hw_setup_cdesc_rings(priv);
+        safexcel_hw_setup_rdesc_rings(priv);
+
+        return 0;
+}
+
+void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
+{
+        struct crypto_async_request *req, *backlog;
+        struct safexcel_context *ctx;
+        struct safexcel_request *request;
+        int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
+
+        priv->ring[ring].need_dequeue = false;
+
+        do {
+                spin_lock_bh(&priv->ring[ring].queue_lock);
+                backlog = crypto_get_backlog(&priv->ring[ring].queue);
+                req = crypto_dequeue_request(&priv->ring[ring].queue);
+                spin_unlock_bh(&priv->ring[ring].queue_lock);
+
+                if (!req)
+                        goto finalize;
+
+                request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req));
+                if (!request) {
+                        spin_lock_bh(&priv->ring[ring].queue_lock);
+                        crypto_enqueue_request(&priv->ring[ring].queue, req);
+                        spin_unlock_bh(&priv->ring[ring].queue_lock);
+
+                        priv->ring[ring].need_dequeue = true;
+                        goto finalize;
+                }
+
+                ctx = crypto_tfm_ctx(req->tfm);
+                ret = ctx->send(req, ring, request, &commands, &results);
+                if (ret) {
+                        kfree(request);
+                        req->complete(req, ret);
+                        priv->ring[ring].need_dequeue = true;
+                        goto finalize;
+                }
+
+                if (backlog)
+                        backlog->complete(backlog, -EINPROGRESS);
+
+                spin_lock_bh(&priv->ring[ring].egress_lock);
+                list_add_tail(&request->list, &priv->ring[ring].list);
+                spin_unlock_bh(&priv->ring[ring].egress_lock);
+
+                cdesc += commands;
+                rdesc += results;
+        } while (nreq++ < EIP197_MAX_BATCH_SZ);
+
+finalize:
+        if (nreq == EIP197_MAX_BATCH_SZ)
+                priv->ring[ring].need_dequeue = true;
+        else if (!nreq)
+                return;
+
+        spin_lock_bh(&priv->ring[ring].lock);
+
+        /* Configure when we want an interrupt */
+        writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
+               EIP197_HIA_RDR_THRESH_PROC_PKT(nreq),
+               priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_THRESH);
+
+        /* let the RDR know we have pending descriptors */
+        writel((rdesc * priv->config.rd_offset) << 2,
+               priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PREP_COUNT);
+
+        /* let the CDR know we have pending descriptors */
+        writel((cdesc * priv->config.cd_offset) << 2,
+               priv->base + EIP197_HIA_CDR(ring) + EIP197_HIA_xDR_PREP_COUNT);
+
+        spin_unlock_bh(&priv->ring[ring].lock);
+}
+
+void safexcel_free_context(struct safexcel_crypto_priv *priv,
+                           struct crypto_async_request *req,
+                           int result_sz)
+{
+        struct safexcel_context *ctx = crypto_tfm_ctx(req->tfm);
+
+        if (ctx->result_dma)
+                dma_unmap_single(priv->dev, ctx->result_dma, result_sz,
+                                 DMA_FROM_DEVICE);
+
+        if (ctx->cache) {
+                dma_unmap_single(priv->dev, ctx->cache_dma, ctx->cache_sz,
+                                 DMA_TO_DEVICE);
+                kfree(ctx->cache);
+                ctx->cache = NULL;
+                ctx->cache_sz = 0;
+        }
+}
+
+void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
+{
+        struct safexcel_command_desc *cdesc;
+
+        /* Acknowledge the command descriptors */
+        do {
+                cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
+                if (IS_ERR(cdesc)) {
+                        dev_err(priv->dev,
+                                "Could not retrieve the command descriptor\n");
+                        return;
+                }
+        } while (!cdesc->last_seg);
+}
+
+void safexcel_inv_complete(struct crypto_async_request *req, int error)
+{
+        struct safexcel_inv_result *result = req->data;
+
+        if (error == -EINPROGRESS)
+                return;
+
+        result->error = error;
+        complete(&result->completion);
+}
+
+int safexcel_invalidate_cache(struct crypto_async_request *async,
+                              struct safexcel_context *ctx,
+                              struct safexcel_crypto_priv *priv,
+                              dma_addr_t ctxr_dma, int ring,
+                              struct safexcel_request *request)
+{
+        struct safexcel_command_desc *cdesc;
+        struct safexcel_result_desc *rdesc;
+        int ret = 0;
+
+        spin_lock_bh(&priv->ring[ring].egress_lock);
+
+        /* Prepare command descriptor */
+        cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma);
+        if (IS_ERR(cdesc)) {
+                ret = PTR_ERR(cdesc);
+                goto unlock;
+        }
+
+        cdesc->control_data.type = EIP197_TYPE_EXTENDED;
+        cdesc->control_data.options = 0;
+        cdesc->control_data.refresh = 0;
+        cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;
+
+        /* Prepare result descriptor */
+        rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
+
+        if (IS_ERR(rdesc)) {
+                ret = PTR_ERR(rdesc);
+                goto cdesc_rollback;
+        }
+
+        request->req = async;
+        goto unlock;
+
+cdesc_rollback:
+        safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
+
+unlock:
+        spin_unlock_bh(&priv->ring[ring].egress_lock);
+        return ret;
+}
+
+static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
+                                                     int ring)
+{
+        struct safexcel_request *sreq;
+        struct safexcel_context *ctx;
+        int ret, i, nreq, ndesc = 0;
+        bool should_complete;
+
+        nreq = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT);
+        nreq >>= 24;
+        nreq &= GENMASK(6, 0);
+        if (!nreq)
+                return;
+
+        for (i = 0; i < nreq; i++) {
+                spin_lock_bh(&priv->ring[ring].egress_lock);
+                sreq = list_first_entry(&priv->ring[ring].list,
+                                        struct safexcel_request, list);
+                list_del(&sreq->list);
+                spin_unlock_bh(&priv->ring[ring].egress_lock);
+
+                ctx = crypto_tfm_ctx(sreq->req->tfm);
+                ndesc = ctx->handle_result(priv, ring, sreq->req,
+                                           &should_complete, &ret);
+                if (ndesc < 0) {
+                        dev_err(priv->dev, "failed to handle result (%d)", ndesc);
+                        return;
+                }
+
+                writel(EIP197_xDR_PROC_xD_PKT(1) |
+                       EIP197_xDR_PROC_xD_COUNT(ndesc * priv->config.rd_offset),
+                       priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT);
+
+                if (should_complete) {
+                        local_bh_disable();
+                        sreq->req->complete(sreq->req, ret);
+                        local_bh_enable();
+                }
+
+                kfree(sreq);
+        }
+}
+
+static void safexcel_handle_result_work(struct work_struct *work)
+{
+        struct safexcel_work_data *data =
+                        container_of(work, struct safexcel_work_data, work);
+        struct safexcel_crypto_priv *priv = data->priv;
+
+        safexcel_handle_result_descriptor(priv, data->ring);
+
+        if (priv->ring[data->ring].need_dequeue)
+                safexcel_dequeue(data->priv, data->ring);
+}
+
+struct safexcel_ring_irq_data {
+        struct safexcel_crypto_priv *priv;
+        int ring;
+};
+
+static irqreturn_t safexcel_irq_ring(int irq, void *data)
+{
+        struct safexcel_ring_irq_data *irq_data = data;
+        struct safexcel_crypto_priv *priv = irq_data->priv;
+        int ring = irq_data->ring;
+        u32 status, stat;
+
+        status = readl(priv->base + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
+        if (!status)
+                return IRQ_NONE;
+
+        /* RDR interrupts */
+        if (status & EIP197_RDR_IRQ(ring)) {
+                stat = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT);
+
+                if (unlikely(stat & EIP197_xDR_ERR)) {
+                        /*
+                         * Fatal error, the RDR is unusable and must be
+                         * reinitialized. This should not happen under
+                         * normal circumstances.
+                         */
+                        dev_err(priv->dev, "RDR: fatal error.");
+                } else if (likely(stat & EIP197_xDR_THRESH)) {
+                        queue_work(priv->ring[ring].workqueue, &priv->ring[ring].work_data.work);
+                }
+
+                /* ACK the interrupts */
+                writel(stat & 0xff,
+                       priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT);
+        }
+
+        /* ACK the interrupts */
+        writel(status, priv->base + EIP197_HIA_AIC_R_ACK(ring));
+
+        return IRQ_HANDLED;
+}
+
+static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name,
+                                     irq_handler_t handler,
+                                     struct safexcel_ring_irq_data *ring_irq_priv)
+{
+        int ret, irq = platform_get_irq_byname(pdev, name);
+
+        if (irq < 0) {
+                dev_err(&pdev->dev, "unable to get IRQ '%s'\n", name);
+                return irq;
+        }
+
+        ret = devm_request_irq(&pdev->dev, irq, handler, 0,
+                               dev_name(&pdev->dev), ring_irq_priv);
+        if (ret) {
+                dev_err(&pdev->dev, "unable to request IRQ %d\n", irq);
+                return ret;
+        }
+
+        return irq;
+}
+
+static struct safexcel_alg_template *safexcel_algs[] = {
+        &safexcel_alg_ecb_aes,
+        &safexcel_alg_cbc_aes,
+        &safexcel_alg_sha1,
+        &safexcel_alg_sha224,
+        &safexcel_alg_sha256,
+        &safexcel_alg_hmac_sha1,
+};
+
+static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
+{
+        int i, j, ret = 0;
+
+        for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
+                safexcel_algs[i]->priv = priv;
+
+                if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
+                        ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
+                else
+                        ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
+
+                if (ret)
+                        goto fail;
+        }
+
+        return 0;
+
+fail:
+        for (j = 0; j < i; j++) {
+                if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
+                        crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
+                else
+                        crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
+        }
+
+        return ret;
+}
+
+static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
+{
+        int i;
+
+        for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
+                if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
+                        crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
+                else
+                        crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
+        }
+}
+
+static void safexcel_configure(struct safexcel_crypto_priv *priv)
+{
+        u32 val, mask;
+
+        val = readl(priv->base + EIP197_HIA_OPTIONS);
+        val = (val & GENMASK(27, 25)) >> 25;
+        mask = BIT(val) - 1;
+
+        val = readl(priv->base + EIP197_HIA_OPTIONS);
+        priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings);
+
+        priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32));
+        priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
+
+        priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32));
+        priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
+}
+
+static int safexcel_probe(struct platform_device *pdev)
+{
+        struct device *dev = &pdev->dev;
+        struct resource *res;
+        struct safexcel_crypto_priv *priv;
+        u64 dma_mask;
+        int i, ret;
+
+        priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+        if (!priv)
+                return -ENOMEM;
+
+        priv->dev = dev;
+
+        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        priv->base = devm_ioremap_resource(dev, res);
+        if (IS_ERR(priv->base)) {
+                dev_err(dev, "failed to get resource\n");
+                return PTR_ERR(priv->base);
+        }
+
+        priv->clk = of_clk_get(dev->of_node, 0);
+        if (!IS_ERR(priv->clk)) {
+                ret = clk_prepare_enable(priv->clk);
+                if (ret) {
+                        dev_err(dev, "unable to enable clk (%d)\n", ret);
+                        return ret;
+                }
+        } else {
+                /* The clock isn't mandatory */
+                if (PTR_ERR(priv->clk) == -EPROBE_DEFER)
+                        return -EPROBE_DEFER;
+        }
+
+        if (of_property_read_u64(dev->of_node, "dma-mask", &dma_mask))
+                dma_mask = DMA_BIT_MASK(64);
+        ret = dma_set_mask_and_coherent(dev, dma_mask);
+        if (ret)
+                goto err_clk;
+
+        priv->context_pool = dmam_pool_create("safexcel-context", dev,
+                                              sizeof(struct safexcel_context_record),
+                                              1, 0);
+        if (!priv->context_pool) {
+                ret = -ENOMEM;
+                goto err_clk;
+        }
+
+        safexcel_configure(priv);
+
+        for (i = 0; i < priv->config.rings; i++) {
+                char irq_name[6] = {0}; /* "ringX\0" */
+                char wq_name[9] = {0}; /* "wq_ringX\0" */
+                int irq;
+                struct safexcel_ring_irq_data *ring_irq;
+
+                ret = safexcel_init_ring_descriptors(priv,
+                                                     &priv->ring[i].cdr,
+                                                     &priv->ring[i].rdr);
+                if (ret)
+                        goto err_clk;
+
+                ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
+                if (!ring_irq) {
+                        ret = -ENOMEM;
+                        goto err_clk;
+                }
+
+                ring_irq->priv = priv;
+                ring_irq->ring = i;
+
+                snprintf(irq_name, 6, "ring%d", i);
+                irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring,
+                                                ring_irq);
+
+                if (irq < 0)
+                        goto err_clk;
+
+                priv->ring[i].work_data.priv = priv;
+                priv->ring[i].work_data.ring = i;
+                INIT_WORK(&priv->ring[i].work_data.work, safexcel_handle_result_work);
+
+                snprintf(wq_name, 9, "wq_ring%d", i);
+                priv->ring[i].workqueue = create_singlethread_workqueue(wq_name);
+                if (!priv->ring[i].workqueue) {
+                        ret = -ENOMEM;
+                        goto err_clk;
+                }
+
+                crypto_init_queue(&priv->ring[i].queue,
+                                  EIP197_DEFAULT_RING_SIZE);
+
+                INIT_LIST_HEAD(&priv->ring[i].list);
+                spin_lock_init(&priv->ring[i].lock);
+                spin_lock_init(&priv->ring[i].egress_lock);
+                spin_lock_init(&priv->ring[i].queue_lock);
+        }
+
+        platform_set_drvdata(pdev, priv);
+        atomic_set(&priv->ring_used, 0);
+
+        ret = safexcel_hw_init(priv);
+        if (ret) {
+                dev_err(dev, "EIP h/w init failed (%d)\n", ret);
+                goto err_clk;
+        }
+
+        ret = safexcel_register_algorithms(priv);
+        if (ret) {
+                dev_err(dev, "Failed to register algorithms (%d)\n", ret);
+                goto err_clk;
+        }
+
+        return 0;
+
+err_clk:
+        clk_disable_unprepare(priv->clk);
+        return ret;
+}
+
+
+static int safexcel_remove(struct platform_device *pdev)
+{
+        struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
+        int i;
+
+        safexcel_unregister_algorithms(priv);
+        clk_disable_unprepare(priv->clk);
+
+        for (i = 0; i < priv->config.rings; i++)
+                destroy_workqueue(priv->ring[i].workqueue);
+
+        return 0;
+}
+
+static const struct of_device_id safexcel_of_match_table[] = {
+        { .compatible = "inside-secure,safexcel-eip197" },
+        {},
+};
+
+
+static struct platform_driver  crypto_safexcel = {
+        .probe          = safexcel_probe,
+        .remove         = safexcel_remove,
+        .driver         = {
+                .name   = "crypto-safexcel",
+                .of_match_table = safexcel_of_match_table,
+        },
+};
+module_platform_driver(crypto_safexcel);
+
+MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
+MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
+MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
+MODULE_DESCRIPTION("Support for SafeXcel cryptographic engine EIP197");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h
new file mode 100644
index 000000000000..304c5838c11a
--- /dev/null
+++ b/drivers/crypto/inside-secure/safexcel.h
@@ -0,0 +1,574 @@
+/*
+ * Copyright (C) 2017 Marvell
+ *
+ * Antoine Tenart <antoine.tenart@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#ifndef __SAFEXCEL_H__
+#define __SAFEXCEL_H__
+
+#include <crypto/algapi.h>
+#include <crypto/internal/hash.h>
+#include <crypto/skcipher.h>
+
+#define EIP197_HIA_VERSION_LE                   0xca35
+#define EIP197_HIA_VERSION_BE                   0x35ca
+
+/* Static configuration */
+#define EIP197_DEFAULT_RING_SIZE                64
+#define EIP197_MAX_TOKENS                       5
+#define EIP197_MAX_RINGS                        4
+#define EIP197_FETCH_COUNT                      1
+#define EIP197_MAX_BATCH_SZ                     EIP197_DEFAULT_RING_SIZE
+
+#define EIP197_GFP_FLAGS(base)  ((base).flags & CRYPTO_TFM_REQ_MAY_SLEEP ? \
+                                 GFP_KERNEL : GFP_ATOMIC)
+
+/* CDR/RDR register offsets */
+#define EIP197_HIA_xDR_OFF(r)                   (0x80000 + (r) * 0x1000)
+#define EIP197_HIA_CDR(r)                       (EIP197_HIA_xDR_OFF(r))
+#define EIP197_HIA_RDR(r)                       (EIP197_HIA_xDR_OFF(r) + 0x800)
+#define EIP197_HIA_xDR_RING_BASE_ADDR_LO        0x0
+#define EIP197_HIA_xDR_RING_BASE_ADDR_HI        0x4
+#define EIP197_HIA_xDR_RING_SIZE                0x18
+#define EIP197_HIA_xDR_DESC_SIZE                0x1c
+#define EIP197_HIA_xDR_CFG                      0x20
+#define EIP197_HIA_xDR_DMA_CFG                  0x24
+#define EIP197_HIA_xDR_THRESH                   0x28
+#define EIP197_HIA_xDR_PREP_COUNT               0x2c
+#define EIP197_HIA_xDR_PROC_COUNT               0x30
+#define EIP197_HIA_xDR_PREP_PNTR                0x34
+#define EIP197_HIA_xDR_PROC_PNTR                0x38
+#define EIP197_HIA_xDR_STAT                     0x3c
+
+/* register offsets */
+#define EIP197_HIA_DFE_CFG                      0x8c000
+#define EIP197_HIA_DFE_THR_CTRL                 0x8c040
+#define EIP197_HIA_DFE_THR_STAT                 0x8c044
+#define EIP197_HIA_DSE_CFG                      0x8d000
+#define EIP197_HIA_DSE_THR_CTRL                 0x8d040
+#define EIP197_HIA_DSE_THR_STAT                 0x8d044
+#define EIP197_HIA_RA_PE_CTRL                   0x90010
+#define EIP197_HIA_RA_PE_STAT                   0x90014
+#define EIP197_HIA_AIC_R_OFF(r)                 ((r) * 0x1000)
+#define EIP197_HIA_AIC_R_ENABLE_CTRL(r)         (0x9e808 - EIP197_HIA_AIC_R_OFF(r))
+#define EIP197_HIA_AIC_R_ENABLED_STAT(r)        (0x9e810 - EIP197_HIA_AIC_R_OFF(r))
+#define EIP197_HIA_AIC_R_ACK(r)                 (0x9e810 - EIP197_HIA_AIC_R_OFF(r))
+#define EIP197_HIA_AIC_R_ENABLE_CLR(r)          (0x9e814 - EIP197_HIA_AIC_R_OFF(r))
+#define EIP197_HIA_AIC_G_ENABLE_CTRL            0x9f808
+#define EIP197_HIA_AIC_G_ENABLED_STAT           0x9f810
+#define EIP197_HIA_AIC_G_ACK                    0x9f810
+#define EIP197_HIA_MST_CTRL                     0x9fff4
+#define EIP197_HIA_OPTIONS                      0x9fff8
+#define EIP197_HIA_VERSION                      0x9fffc
+#define EIP197_PE_IN_DBUF_THRES                 0xa0000
+#define EIP197_PE_IN_TBUF_THRES                 0xa0100
+#define EIP197_PE_ICE_SCRATCH_RAM               0xa0800
+#define EIP197_PE_ICE_PUE_CTRL                  0xa0c80
+#define EIP197_PE_ICE_SCRATCH_CTRL              0xa0d04
+#define EIP197_PE_ICE_FPP_CTRL                  0xa0d80
+#define EIP197_PE_ICE_RAM_CTRL                  0xa0ff0
+#define EIP197_PE_EIP96_FUNCTION_EN             0xa1004
+#define EIP197_PE_EIP96_CONTEXT_CTRL            0xa1008
+#define EIP197_PE_EIP96_CONTEXT_STAT            0xa100c
+#define EIP197_PE_OUT_DBUF_THRES                0xa1c00
+#define EIP197_PE_OUT_TBUF_THRES                0xa1d00
+#define EIP197_CLASSIFICATION_RAMS              0xe0000
+#define EIP197_TRC_CTRL                         0xf0800
+#define EIP197_TRC_LASTRES                      0xf0804
+#define EIP197_TRC_REGINDEX                     0xf0808
+#define EIP197_TRC_PARAMS                       0xf0820
+#define EIP197_TRC_FREECHAIN                    0xf0824
+#define EIP197_TRC_PARAMS2                      0xf0828
+#define EIP197_TRC_ECCCTRL                      0xf0830
+#define EIP197_TRC_ECCSTAT                      0xf0834
+#define EIP197_TRC_ECCADMINSTAT                 0xf0838
+#define EIP197_TRC_ECCDATASTAT                  0xf083c
+#define EIP197_TRC_ECCDATA                      0xf0840
+#define EIP197_CS_RAM_CTRL                      0xf7ff0
+#define EIP197_MST_CTRL                         0xffff4
+
+/* EIP197_HIA_xDR_DESC_SIZE */
+#define EIP197_xDR_DESC_MODE_64BIT              BIT(31)
+
+/* EIP197_HIA_xDR_DMA_CFG */
+#define EIP197_HIA_xDR_WR_RES_BUF               BIT(22)
+#define EIP197_HIA_xDR_WR_CTRL_BUG              BIT(23)
+#define EIP197_HIA_xDR_WR_OWN_BUF               BIT(24)
+#define EIP197_HIA_xDR_CFG_WR_CACHE(n)          (((n) & 0x7) << 25)
+#define EIP197_HIA_xDR_CFG_RD_CACHE(n)          (((n) & 0x7) << 29)
+
+/* EIP197_HIA_CDR_THRESH */
+#define EIP197_HIA_CDR_THRESH_PROC_PKT(n)       (n)
+#define EIP197_HIA_CDR_THRESH_PROC_MODE         BIT(22)
+#define EIP197_HIA_CDR_THRESH_PKT_MODE          BIT(23)
+#define EIP197_HIA_CDR_THRESH_TIMEOUT(n)        ((n) << 24) /* x256 clk cycles */
+
+/* EIP197_HIA_RDR_THRESH */
+#define EIP197_HIA_RDR_THRESH_PROC_PKT(n)       (n)
+#define EIP197_HIA_RDR_THRESH_PKT_MODE          BIT(23)
+#define EIP197_HIA_RDR_THRESH_TIMEOUT(n)        ((n) << 24) /* x256 clk cycles */
+
+/* EIP197_HIA_xDR_PREP_COUNT */
+#define EIP197_xDR_PREP_CLR_COUNT               BIT(31)
+
+/* EIP197_HIA_xDR_PROC_COUNT */
+#define EIP197_xDR_PROC_xD_COUNT(n)             ((n) << 2)
+#define EIP197_xDR_PROC_xD_PKT(n)               ((n) << 24)
+#define EIP197_xDR_PROC_CLR_COUNT               BIT(31)
+
+/* EIP197_HIA_xDR_STAT */
+#define EIP197_xDR_DMA_ERR                      BIT(0)
+#define EIP197_xDR_PREP_CMD_THRES               BIT(1)
+#define EIP197_xDR_ERR                          BIT(2)
+#define EIP197_xDR_THRESH                       BIT(4)
+#define EIP197_xDR_TIMEOUT                      BIT(5)
+
+#define EIP197_HIA_RA_PE_CTRL_RESET             BIT(31)
+#define EIP197_HIA_RA_PE_CTRL_EN                BIT(30)
+
+/* EIP197_HIA_AIC_R_ENABLE_CTRL */
+#define EIP197_CDR_IRQ(n)                       BIT((n) * 2)
+#define EIP197_RDR_IRQ(n)                       BIT((n) * 2 + 1)
+
+/* EIP197_HIA_DFE/DSE_CFG */
+#define EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(n)     ((n) << 0)
+#define EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(n)   (((n) & 0x7) << 4)
+#define EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(n)     ((n) << 8)
+#define EIP197_HIA_DSE_CFG_ALLWAYS_BUFFERABLE   GENMASK(15, 14)
+#define EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(n)     ((n) << 16)
+#define EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(n)   (((n) & 0x7) << 20)
+#define EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(n)     ((n) << 24)
+#define EIP197_HIA_DFE_CFG_DIS_DEBUG            (BIT(31) | BIT(29))
+#define EIP197_HIA_DSE_CFG_EN_SINGLE_WR         BIT(29)
+#define EIP197_HIA_DSE_CFG_DIS_DEBUG            BIT(31)
+
+/* EIP197_HIA_DFE/DSE_THR_CTRL */
+#define EIP197_DxE_THR_CTRL_EN                  BIT(30)
+#define EIP197_DxE_THR_CTRL_RESET_PE            BIT(31)
+
+/* EIP197_HIA_AIC_G_ENABLED_STAT */
+#define EIP197_G_IRQ_DFE(n)                     BIT((n) << 1)
+#define EIP197_G_IRQ_DSE(n)                     BIT(((n) << 1) + 1)
+#define EIP197_G_IRQ_RING                       BIT(16)
+#define EIP197_G_IRQ_PE(n)                      BIT((n) + 20)
+
+/* EIP197_HIA_MST_CTRL */
+#define RD_CACHE_3BITS                          0x5
+#define WR_CACHE_3BITS                          0x3
+#define RD_CACHE_4BITS                          (RD_CACHE_3BITS << 1 | BIT(0))
+#define WR_CACHE_4BITS                          (WR_CACHE_3BITS << 1 | BIT(0))
+#define EIP197_MST_CTRL_RD_CACHE(n)             (((n) & 0xf) << 0)
+#define EIP197_MST_CTRL_WD_CACHE(n)             (((n) & 0xf) << 4)
+#define EIP197_MST_CTRL_BYTE_SWAP               BIT(24)
+#define EIP197_MST_CTRL_NO_BYTE_SWAP            BIT(25)
+
+/* EIP197_PE_IN_DBUF/TBUF_THRES */
+#define EIP197_PE_IN_xBUF_THRES_MIN(n)          ((n) << 8)
+#define EIP197_PE_IN_xBUF_THRES_MAX(n)          ((n) << 12)
+
+/* EIP197_PE_OUT_DBUF_THRES */
+#define EIP197_PE_OUT_DBUF_THRES_MIN(n)         ((n) << 0)
+#define EIP197_PE_OUT_DBUF_THRES_MAX(n)         ((n) << 4)
+
+/* EIP197_PE_ICE_SCRATCH_CTRL */
+#define EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER         BIT(2)
+#define EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN             BIT(3)
+#define EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS        BIT(24)
+#define EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS       BIT(25)
+
+/* EIP197_PE_ICE_SCRATCH_RAM */
+#define EIP197_NUM_OF_SCRATCH_BLOCKS            32
+
+/* EIP197_PE_ICE_PUE/FPP_CTRL */
+#define EIP197_PE_ICE_x_CTRL_SW_RESET                   BIT(0)
+#define EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR           BIT(14)
+#define EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR               BIT(15)
+
+/* EIP197_PE_ICE_RAM_CTRL */
+#define EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN      BIT(0)
+#define EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN      BIT(1)
+
+/* EIP197_PE_EIP96_FUNCTION_EN */
+#define EIP197_FUNCTION_RSVD                    (BIT(6) | BIT(15) | BIT(20) | BIT(23))
+#define EIP197_PROTOCOL_HASH_ONLY               BIT(0)
+#define EIP197_PROTOCOL_ENCRYPT_ONLY            BIT(1)
+#define EIP197_PROTOCOL_HASH_ENCRYPT            BIT(2)
+#define EIP197_PROTOCOL_HASH_DECRYPT            BIT(3)
+#define EIP197_PROTOCOL_ENCRYPT_HASH            BIT(4)
+#define EIP197_PROTOCOL_DECRYPT_HASH            BIT(5)
+#define EIP197_ALG_ARC4                         BIT(7)
+#define EIP197_ALG_AES_ECB                      BIT(8)
+#define EIP197_ALG_AES_CBC                      BIT(9)
+#define EIP197_ALG_AES_CTR_ICM                  BIT(10)
+#define EIP197_ALG_AES_OFB                      BIT(11)
+#define EIP197_ALG_AES_CFB                      BIT(12)
+#define EIP197_ALG_DES_ECB                      BIT(13)
+#define EIP197_ALG_DES_CBC                      BIT(14)
+#define EIP197_ALG_DES_OFB                      BIT(16)
+#define EIP197_ALG_DES_CFB                      BIT(17)
+#define EIP197_ALG_3DES_ECB                     BIT(18)
+#define EIP197_ALG_3DES_CBC                     BIT(19)
+#define EIP197_ALG_3DES_OFB                     BIT(21)
+#define EIP197_ALG_3DES_CFB                     BIT(22)
+#define EIP197_ALG_MD5                          BIT(24)
+#define EIP197_ALG_HMAC_MD5                     BIT(25)
+#define EIP197_ALG_SHA1                         BIT(26)
+#define EIP197_ALG_HMAC_SHA1                    BIT(27)
+#define EIP197_ALG_SHA2                         BIT(28)
+#define EIP197_ALG_HMAC_SHA2                    BIT(29)
+#define EIP197_ALG_AES_XCBC_MAC                 BIT(30)
+#define EIP197_ALG_GCM_HASH                     BIT(31)
+
+/* EIP197_PE_EIP96_CONTEXT_CTRL */
+#define EIP197_CONTEXT_SIZE(n)                  (n)
+#define EIP197_ADDRESS_MODE                     BIT(8)
+#define EIP197_CONTROL_MODE                     BIT(9)
+
+/* Context Control */
+struct safexcel_context_record {
+        u32 control0;
+        u32 control1;
+
+        __le32 data[12];
+} __packed;
+
+/* control0 */
+#define CONTEXT_CONTROL_TYPE_NULL_OUT           0x0
+#define CONTEXT_CONTROL_TYPE_NULL_IN            0x1
+#define CONTEXT_CONTROL_TYPE_HASH_OUT           0x2
+#define CONTEXT_CONTROL_TYPE_HASH_IN            0x3
+#define CONTEXT_CONTROL_TYPE_CRYPTO_OUT         0x4
+#define CONTEXT_CONTROL_TYPE_CRYPTO_IN          0x5
+#define CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT   0x6
+#define CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN    0x7
+#define CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT   0x14
+#define CONTEXT_CONTROL_TYPE_HASH_DECRYPT_OUT   0x15
+#define CONTEXT_CONTROL_RESTART_HASH            BIT(4)
+#define CONTEXT_CONTROL_NO_FINISH_HASH          BIT(5)
+#define CONTEXT_CONTROL_SIZE(n)                 ((n) << 8)
+#define CONTEXT_CONTROL_KEY_EN                  BIT(16)
+#define CONTEXT_CONTROL_CRYPTO_ALG_AES128       (0x5 << 17)
+#define CONTEXT_CONTROL_CRYPTO_ALG_AES192       (0x6 << 17)
+#define CONTEXT_CONTROL_CRYPTO_ALG_AES256       (0x7 << 17)
+#define CONTEXT_CONTROL_DIGEST_PRECOMPUTED      (0x1 << 21)
+#define CONTEXT_CONTROL_DIGEST_HMAC             (0x3 << 21)
+#define CONTEXT_CONTROL_CRYPTO_ALG_SHA1         (0x2 << 23)
+#define CONTEXT_CONTROL_CRYPTO_ALG_SHA224       (0x4 << 23)
+#define CONTEXT_CONTROL_CRYPTO_ALG_SHA256       (0x3 << 23)
+#define CONTEXT_CONTROL_INV_FR                  (0x5 << 24)
+#define CONTEXT_CONTROL_INV_TR                  (0x6 << 24)
+
+/* control1 */
+#define CONTEXT_CONTROL_CRYPTO_MODE_ECB         (0 << 0)
+#define CONTEXT_CONTROL_CRYPTO_MODE_CBC         (1 << 0)
+#define CONTEXT_CONTROL_IV0                     BIT(5)
+#define CONTEXT_CONTROL_IV1                     BIT(6)
+#define CONTEXT_CONTROL_IV2                     BIT(7)
+#define CONTEXT_CONTROL_IV3                     BIT(8)
+#define CONTEXT_CONTROL_DIGEST_CNT              BIT(9)
+#define CONTEXT_CONTROL_COUNTER_MODE            BIT(10)
+#define CONTEXT_CONTROL_HASH_STORE              BIT(19)
+
+/* EIP197_CS_RAM_CTRL */
+#define EIP197_TRC_ENABLE_0                     BIT(4)
+#define EIP197_TRC_ENABLE_1                     BIT(5)
+#define EIP197_TRC_ENABLE_2                     BIT(6)
+#define EIP197_TRC_ENABLE_MASK                  GENMASK(6, 4)
+
+/* EIP197_TRC_PARAMS */
+#define EIP197_TRC_PARAMS_SW_RESET              BIT(0)
+#define EIP197_TRC_PARAMS_DATA_ACCESS           BIT(2)
+#define EIP197_TRC_PARAMS_HTABLE_SZ(x)          ((x) << 4)
+#define EIP197_TRC_PARAMS_BLK_TIMER_SPEED(x)    ((x) << 10)
+#define EIP197_TRC_PARAMS_RC_SZ_LARGE(n)        ((n) << 18)
+
+/* EIP197_TRC_FREECHAIN */
+#define EIP197_TRC_FREECHAIN_HEAD_PTR(p)        (p)
+#define EIP197_TRC_FREECHAIN_TAIL_PTR(p)        ((p) << 16)
+
+/* EIP197_TRC_PARAMS2 */
+#define EIP197_TRC_PARAMS2_HTABLE_PTR(p)        (p)
+#define EIP197_TRC_PARAMS2_RC_SZ_SMALL(n)       ((n) << 18)
+
+/* Cache helpers */
+#define EIP197_CS_RC_MAX                        52
+#define EIP197_CS_RC_SIZE                       (4 * sizeof(u32))
+#define EIP197_CS_RC_NEXT(x)                    (x)
+#define EIP197_CS_RC_PREV(x)                    ((x) << 10)
+#define EIP197_RC_NULL                          0x3ff
+#define EIP197_CS_TRC_REC_WC                    59
+#define EIP197_CS_TRC_LG_REC_WC                 73
+
+/* Result data */
+struct result_data_desc {
+        u32 packet_length:17;
+        u32 error_code:15;
+
+        u8 bypass_length:4;
+        u8 e15:1;
+        u16 rsvd0;
+        u8 hash_bytes:1;
+        u8 hash_length:6;
+        u8 generic_bytes:1;
+        u8 checksum:1;
+        u8 next_header:1;
+        u8 length:1;
+
+        u16 application_id;
+        u16 rsvd1;
+
+        u32 rsvd2;
+} __packed;
+
+
+/* Basic Result Descriptor format */
+struct safexcel_result_desc {
+        u32 particle_size:17;
+        u8 rsvd0:3;
+        u8 descriptor_overflow:1;
+        u8 buffer_overflow:1;
+        u8 last_seg:1;
+        u8 first_seg:1;
+        u16 result_size:8;
+
+        u32 rsvd1;
+
+        u32 data_lo;
+        u32 data_hi;
+
+        struct result_data_desc result_data;
+} __packed;
+
+struct safexcel_token {
+        u32 packet_length:17;
+        u8 stat:2;
+        u16 instructions:9;
+        u8 opcode:4;
+} __packed;
+
+#define EIP197_TOKEN_STAT_LAST_HASH             BIT(0)
+#define EIP197_TOKEN_STAT_LAST_PACKET           BIT(1)
+#define EIP197_TOKEN_OPCODE_DIRECTION           0x0
+#define EIP197_TOKEN_OPCODE_INSERT              0x2
+#define EIP197_TOKEN_OPCODE_NOOP                EIP197_TOKEN_OPCODE_INSERT
+#define EIP197_TOKEN_OPCODE_BYPASS              GENMASK(3, 0)
+
+static inline void eip197_noop_token(struct safexcel_token *token)
+{
+        token->opcode = EIP197_TOKEN_OPCODE_NOOP;
+        token->packet_length = BIT(2);
+}
+
+/* Instructions */
+#define EIP197_TOKEN_INS_INSERT_HASH_DIGEST     0x1c
+#define EIP197_TOKEN_INS_TYPE_OUTPUT            BIT(5)
+#define EIP197_TOKEN_INS_TYPE_HASH              BIT(6)
+#define EIP197_TOKEN_INS_TYPE_CRYTO             BIT(7)
+#define EIP197_TOKEN_INS_LAST                   BIT(8)
+
+/* Processing Engine Control Data  */
+struct safexcel_control_data_desc {
+        u32 packet_length:17;
+        u16 options:13;
+        u8 type:2;
+
+        u16 application_id;
+        u16 rsvd;
+
+        u8 refresh:2;
+        u32 context_lo:30;
+        u32 context_hi;
+
+        u32 control0;
+        u32 control1;
+
+        u32 token[EIP197_MAX_TOKENS];
+} __packed;
+
+#define EIP197_OPTION_MAGIC_VALUE       BIT(0)
+#define EIP197_OPTION_64BIT_CTX         BIT(1)
+#define EIP197_OPTION_CTX_CTRL_IN_CMD   BIT(8)
+#define EIP197_OPTION_4_TOKEN_IV_CMD    GENMASK(11, 9)
+
+#define EIP197_TYPE_EXTENDED            0x3
+
+/* Basic Command Descriptor format */
+struct safexcel_command_desc {
+        u32 particle_size:17;
+        u8 rsvd0:5;
+        u8 last_seg:1;
+        u8 first_seg:1;
+        u16 additional_cdata_size:8;
+
+        u32 rsvd1;
+
+        u32 data_lo;
+        u32 data_hi;
+
+        struct safexcel_control_data_desc control_data;
+} __packed;
+
+/*
+ * Internal structures & functions
+ */
+
+enum eip197_fw {
+        FW_IFPP = 0,
+        FW_IPUE,
+        FW_NB
+};
+
+struct safexcel_ring {
+        void *base;
+        void *base_end;
+        dma_addr_t base_dma;
+
+        /* write and read pointers */
+        void *write;
+        void *read;
+
+        /* number of elements used in the ring */
+        unsigned nr;
+        unsigned offset;
+};
+
+enum safexcel_alg_type {
+        SAFEXCEL_ALG_TYPE_SKCIPHER,
+        SAFEXCEL_ALG_TYPE_AHASH,
+};
+
+struct safexcel_request {
+        struct list_head list;
+        struct crypto_async_request *req;
+};
+
+struct safexcel_config {
+        u32 rings;
+
+        u32 cd_size;
+        u32 cd_offset;
+
+        u32 rd_size;
+        u32 rd_offset;
+};
+
+struct safexcel_work_data {
+        struct work_struct work;
+        struct safexcel_crypto_priv *priv;
+        int ring;
+};
+
+struct safexcel_crypto_priv {
+        void __iomem *base;
+        struct device *dev;
+        struct clk *clk;
+        struct safexcel_config config;
+
+        /* context DMA pool */
+        struct dma_pool *context_pool;
+
+        atomic_t ring_used;
+
+        struct {
+                spinlock_t lock;
+                spinlock_t egress_lock;
+
+                struct list_head list;
+                struct workqueue_struct *workqueue;
+                struct safexcel_work_data work_data;
+
+                /* command/result rings */
+                struct safexcel_ring cdr;
+                struct safexcel_ring rdr;
+
+                /* queue */
+                struct crypto_queue queue;
+                spinlock_t queue_lock;
+                bool need_dequeue;
+        } ring[EIP197_MAX_RINGS];
+};
+
+struct safexcel_context {
+        int (*send)(struct crypto_async_request *req, int ring,
+                    struct safexcel_request *request, int *commands,
+                    int *results);
+        int (*handle_result)(struct safexcel_crypto_priv *priv, int ring,
+                             struct crypto_async_request *req, bool *complete,
+                             int *ret);
+        struct safexcel_context_record *ctxr;
+        dma_addr_t ctxr_dma;
+
+        int ring;
+        bool needs_inv;
+        bool exit_inv;
+
+        /* Used for ahash requests */
+        dma_addr_t result_dma;
+        void *cache;
+        dma_addr_t cache_dma;
+        unsigned int cache_sz;
+};
+
+/*
+ * Template structure to describe the algorithms in order to register them.
+ * It also has the purpose to contain our private structure and is actually
+ * the only way I know in this framework to avoid having global pointers...
+ */
+struct safexcel_alg_template {
+        struct safexcel_crypto_priv *priv;
+        enum safexcel_alg_type type;
+        union {
+                struct skcipher_alg skcipher;
+                struct ahash_alg ahash;
+        } alg;
+};
+
+struct safexcel_inv_result {
+        struct completion completion;
+        int error;
+};
+
+void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring);
+void safexcel_complete(struct safexcel_crypto_priv *priv, int ring);
+void safexcel_free_context(struct safexcel_crypto_priv *priv,
+                                  struct crypto_async_request *req,
+                                  int result_sz);
+int safexcel_invalidate_cache(struct crypto_async_request *async,
+                              struct safexcel_context *ctx,
+                              struct safexcel_crypto_priv *priv,
+                              dma_addr_t ctxr_dma, int ring,
+                              struct safexcel_request *request);
+int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
+                                   struct safexcel_ring *cdr,
+                                   struct safexcel_ring *rdr);
+int safexcel_select_ring(struct safexcel_crypto_priv *priv);
+void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv,
+                              struct safexcel_ring *ring);
+void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,
+                                 struct safexcel_ring *ring);
+struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv,
+                                                 int ring_id,
+                                                 bool first, bool last,
+                                                 dma_addr_t data, u32 len,
+                                                 u32 full_data_len,
+                                                 dma_addr_t context);
+struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv,
+                                                 int ring_id,
+                                                bool first, bool last,
+                                                dma_addr_t data, u32 len);
+void safexcel_inv_complete(struct crypto_async_request *req, int error);
+
+/* available algorithms */
+extern struct safexcel_alg_template safexcel_alg_ecb_aes;
+extern struct safexcel_alg_template safexcel_alg_cbc_aes;
+extern struct safexcel_alg_template safexcel_alg_sha1;
+extern struct safexcel_alg_template safexcel_alg_sha224;
+extern struct safexcel_alg_template safexcel_alg_sha256;
+extern struct safexcel_alg_template safexcel_alg_hmac_sha1;
+
+#endif
diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c
new file mode 100644
index 000000000000..d2207ac5ba19
--- /dev/null
+++ b/drivers/crypto/inside-secure/safexcel_cipher.c
@@ -0,0 +1,561 @@
+/*
+ * Copyright (C) 2017 Marvell
+ *
+ * Antoine Tenart <antoine.tenart@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+
+#include <crypto/aes.h>
+#include <crypto/skcipher.h>
+
+#include "safexcel.h"
+
+enum safexcel_cipher_direction {
+        SAFEXCEL_ENCRYPT,
+        SAFEXCEL_DECRYPT,
+};
+
+struct safexcel_cipher_ctx {
+        struct safexcel_context base;
+        struct safexcel_crypto_priv *priv;
+
+        enum safexcel_cipher_direction direction;
+        u32 mode;
+
+        __le32 key[8];
+        unsigned int key_len;
+};
+
+static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx,
+                                  struct crypto_async_request *async,
+                                  struct safexcel_command_desc *cdesc,
+                                  u32 length)
+{
+        struct skcipher_request *req = skcipher_request_cast(async);
+        struct safexcel_token *token;
+        unsigned offset = 0;
+
+        if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
+                offset = AES_BLOCK_SIZE / sizeof(u32);
+                memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_SIZE);
+
+                cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
+        }
+
+        token = (struct safexcel_token *)(cdesc->control_data.token + offset);
+
+        token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
+        token[0].packet_length = length;
+        token[0].stat = EIP197_TOKEN_STAT_LAST_PACKET;
+        token[0].instructions = EIP197_TOKEN_INS_LAST |
+                                EIP197_TOKEN_INS_TYPE_CRYTO |
+                                EIP197_TOKEN_INS_TYPE_OUTPUT;
+}
+
+static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key,
+                               unsigned int len)
+{
+        struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
+        struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct crypto_aes_ctx aes;
+        int ret, i;
+
+        ret = crypto_aes_expand_key(&aes, key, len);
+        if (ret) {
+                crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return ret;
+        }
+
+        for (i = 0; i < len / sizeof(u32); i++) {
+                if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
+                        ctx->base.needs_inv = true;
+                        break;
+                }
+        }
+
+        for (i = 0; i < len / sizeof(u32); i++)
+                ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
+
+        ctx->key_len = len;
+
+        memzero_explicit(&aes, sizeof(aes));
+        return 0;
+}
+
+static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
+                                    struct safexcel_command_desc *cdesc)
+{
+        struct safexcel_crypto_priv *priv = ctx->priv;
+        int ctrl_size;
+
+        if (ctx->direction == SAFEXCEL_ENCRYPT)
+                cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT;
+        else
+                cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_IN;
+
+        cdesc->control_data.control0 |= CONTEXT_CONTROL_KEY_EN;
+        cdesc->control_data.control1 |= ctx->mode;
+
+        switch (ctx->key_len) {
+        case AES_KEYSIZE_128:
+                cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES128;
+                ctrl_size = 4;
+                break;
+        case AES_KEYSIZE_192:
+                cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES192;
+                ctrl_size = 6;
+                break;
+        case AES_KEYSIZE_256:
+                cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES256;
+                ctrl_size = 8;
+                break;
+        default:
+                dev_err(priv->dev, "aes keysize not supported: %u\n",
+                        ctx->key_len);
+                return -EINVAL;
+        }
+        cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(ctrl_size);
+
+        return 0;
+}
+
+static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
+                                  struct crypto_async_request *async,
+                                  bool *should_complete, int *ret)
+{
+        struct skcipher_request *req = skcipher_request_cast(async);
+        struct safexcel_result_desc *rdesc;
+        int ndesc = 0;
+
+        *ret = 0;
+
+        spin_lock_bh(&priv->ring[ring].egress_lock);
+        do {
+                rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
+                if (IS_ERR(rdesc)) {
+                        dev_err(priv->dev,
+                                "cipher: result: could not retrieve the result descriptor\n");
+                        *ret = PTR_ERR(rdesc);
+                        break;
+                }
+
+                if (rdesc->result_data.error_code) {
+                        dev_err(priv->dev,
+                                "cipher: result: result descriptor error (%d)\n",
+                                rdesc->result_data.error_code);
+                        *ret = -EIO;
+                }
+
+                ndesc++;
+        } while (!rdesc->last_seg);
+
+        safexcel_complete(priv, ring);
+        spin_unlock_bh(&priv->ring[ring].egress_lock);
+
+        if (req->src == req->dst) {
+                dma_unmap_sg(priv->dev, req->src,
+                             sg_nents_for_len(req->src, req->cryptlen),
+                             DMA_BIDIRECTIONAL);
+        } else {
+                dma_unmap_sg(priv->dev, req->src,
+                             sg_nents_for_len(req->src, req->cryptlen),
+                             DMA_TO_DEVICE);
+                dma_unmap_sg(priv->dev, req->dst,
+                             sg_nents_for_len(req->dst, req->cryptlen),
+                             DMA_FROM_DEVICE);
+        }
+
+        *should_complete = true;
+
+        return ndesc;
+}
+
+static int safexcel_aes_send(struct crypto_async_request *async,
+                             int ring, struct safexcel_request *request,
+                             int *commands, int *results)
+{
+        struct skcipher_request *req = skcipher_request_cast(async);
+        struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+        struct safexcel_crypto_priv *priv = ctx->priv;
+        struct safexcel_command_desc *cdesc;
+        struct safexcel_result_desc *rdesc;
+        struct scatterlist *sg;
+        int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = req->cryptlen;
+        int i, ret = 0;
+
+        if (req->src == req->dst) {
+                nr_src = dma_map_sg(priv->dev, req->src,
+                                    sg_nents_for_len(req->src, req->cryptlen),
+                                    DMA_BIDIRECTIONAL);
+                nr_dst = nr_src;
+                if (!nr_src)
+                        return -EINVAL;
+        } else {
+                nr_src = dma_map_sg(priv->dev, req->src,
+                                    sg_nents_for_len(req->src, req->cryptlen),
+                                    DMA_TO_DEVICE);
+                if (!nr_src)
+                        return -EINVAL;
+
+                nr_dst = dma_map_sg(priv->dev, req->dst,
+                                    sg_nents_for_len(req->dst, req->cryptlen),
+                                    DMA_FROM_DEVICE);
+                if (!nr_dst) {
+                        dma_unmap_sg(priv->dev, req->src,
+                                     sg_nents_for_len(req->src, req->cryptlen),
+                                     DMA_TO_DEVICE);
+                        return -EINVAL;
+                }
+        }
+
+        memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);
+
+        spin_lock_bh(&priv->ring[ring].egress_lock);
+
+        /* command descriptors */
+        for_each_sg(req->src, sg, nr_src, i) {
+                int len = sg_dma_len(sg);
+
+                /* Do not overflow the request */
+                if (queued - len < 0)
+                        len = queued;
+
+                cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, !(queued - len),
+                                           sg_dma_address(sg), len, req->cryptlen,
+                                           ctx->base.ctxr_dma);
+                if (IS_ERR(cdesc)) {
+                        /* No space left in the command descriptor ring */
+                        ret = PTR_ERR(cdesc);
+                        goto cdesc_rollback;
+                }
+                n_cdesc++;
+
+                if (n_cdesc == 1) {
+                        safexcel_context_control(ctx, cdesc);
+                        safexcel_cipher_token(ctx, async, cdesc, req->cryptlen);
+                }
+
+                queued -= len;
+                if (!queued)
+                        break;
+        }
+
+        /* result descriptors */
+        for_each_sg(req->dst, sg, nr_dst, i) {
+                bool first = !i, last = (i == nr_dst - 1);
+                u32 len = sg_dma_len(sg);
+
+                rdesc = safexcel_add_rdesc(priv, ring, first, last,
+                                           sg_dma_address(sg), len);
+                if (IS_ERR(rdesc)) {
+                        /* No space left in the result descriptor ring */
+                        ret = PTR_ERR(rdesc);
+                        goto rdesc_rollback;
+                }
+                n_rdesc++;
+        }
+
+        spin_unlock_bh(&priv->ring[ring].egress_lock);
+
+        request->req = &req->base;
+        ctx->base.handle_result = safexcel_handle_result;
+
+        *commands = n_cdesc;
+        *results = n_rdesc;
+        return 0;
+
+rdesc_rollback:
+        for (i = 0; i < n_rdesc; i++)
+                safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr);
+cdesc_rollback:
+        for (i = 0; i < n_cdesc; i++)
+                safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
+
+        spin_unlock_bh(&priv->ring[ring].egress_lock);
+
+        if (req->src == req->dst) {
+                dma_unmap_sg(priv->dev, req->src,
+                             sg_nents_for_len(req->src, req->cryptlen),
+                             DMA_BIDIRECTIONAL);
+        } else {
+                dma_unmap_sg(priv->dev, req->src,
+                             sg_nents_for_len(req->src, req->cryptlen),
+                             DMA_TO_DEVICE);
+                dma_unmap_sg(priv->dev, req->dst,
+                             sg_nents_for_len(req->dst, req->cryptlen),
+                             DMA_FROM_DEVICE);
+        }
+
+        return ret;
+}
+
+static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
+                                      int ring,
+                                      struct crypto_async_request *async,
+                                      bool *should_complete, int *ret)
+{
+        struct skcipher_request *req = skcipher_request_cast(async);
+        struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+        struct safexcel_result_desc *rdesc;
+        int ndesc = 0, enq_ret;
+
+        *ret = 0;
+
+        spin_lock_bh(&priv->ring[ring].egress_lock);
+        do {
+                rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
+                if (IS_ERR(rdesc)) {
+                        dev_err(priv->dev,
+                                "cipher: invalidate: could not retrieve the result descriptor\n");
+                        *ret = PTR_ERR(rdesc);
+                        break;
+                }
+
+                if (rdesc->result_data.error_code) {
+                        dev_err(priv->dev, "cipher: invalidate: result descriptor error (%d)\n",
+                                rdesc->result_data.error_code);
+                        *ret = -EIO;
+                }
+
+                ndesc++;
+        } while (!rdesc->last_seg);
+
+        safexcel_complete(priv, ring);
+        spin_unlock_bh(&priv->ring[ring].egress_lock);
+
+        if (ctx->base.exit_inv) {
+                dma_pool_free(priv->context_pool, ctx->base.ctxr,
+                              ctx->base.ctxr_dma);
+
+                *should_complete = true;
+
+                return ndesc;
+        }
+
+        ring = safexcel_select_ring(priv);
+        ctx->base.ring = ring;
+        ctx->base.needs_inv = false;
+        ctx->base.send = safexcel_aes_send;
+
+        spin_lock_bh(&priv->ring[ring].queue_lock);
+        enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
+        spin_unlock_bh(&priv->ring[ring].queue_lock);
+
+        if (enq_ret != -EINPROGRESS)
+                *ret = enq_ret;
+
+        if (!priv->ring[ring].need_dequeue)
+                safexcel_dequeue(priv, ring);
+
+        *should_complete = false;
+
+        return ndesc;
+}
+
+static int safexcel_cipher_send_inv(struct crypto_async_request *async,
+                                    int ring, struct safexcel_request *request,
+                                    int *commands, int *results)
+{
+        struct skcipher_request *req = skcipher_request_cast(async);
+        struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+        struct safexcel_crypto_priv *priv = ctx->priv;
+        int ret;
+
+        ctx->base.handle_result = safexcel_handle_inv_result;
+
+        ret = safexcel_invalidate_cache(async, &ctx->base, priv,
+                                        ctx->base.ctxr_dma, ring, request);
+        if (unlikely(ret))
+                return ret;
+
+        *commands = 1;
+        *results = 1;
+
+        return 0;
+}
+
+static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm)
+{
+        struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct safexcel_crypto_priv *priv = ctx->priv;
+        struct skcipher_request req;
+        struct safexcel_inv_result result = { 0 };
+        int ring = ctx->base.ring;
+
+        memset(&req, 0, sizeof(struct skcipher_request));
+
+        /* create invalidation request */
+        init_completion(&result.completion);
+        skcipher_request_set_callback(&req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+                                        safexcel_inv_complete, &result);
+
+        skcipher_request_set_tfm(&req, __crypto_skcipher_cast(tfm));
+        ctx = crypto_tfm_ctx(req.base.tfm);
+        ctx->base.exit_inv = true;
+        ctx->base.send = safexcel_cipher_send_inv;
+
+        spin_lock_bh(&priv->ring[ring].queue_lock);
+        crypto_enqueue_request(&priv->ring[ring].queue, &req.base);
+        spin_unlock_bh(&priv->ring[ring].queue_lock);
+
+        if (!priv->ring[ring].need_dequeue)
+                safexcel_dequeue(priv, ring);
+
+        wait_for_completion_interruptible(&result.completion);
+
+        if (result.error) {
+                dev_warn(priv->dev,
+                        "cipher: sync: invalidate: completion error %d\n",
+                         result.error);
+                return result.error;
+        }
+
+        return 0;
+}
+
+static int safexcel_aes(struct skcipher_request *req,
+                        enum safexcel_cipher_direction dir, u32 mode)
+{
+        struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+        struct safexcel_crypto_priv *priv = ctx->priv;
+        int ret, ring;
+
+        ctx->direction = dir;
+        ctx->mode = mode;
+
+        if (ctx->base.ctxr) {
+                if (ctx->base.needs_inv)
+                        ctx->base.send = safexcel_cipher_send_inv;
+        } else {
+                ctx->base.ring = safexcel_select_ring(priv);
+                ctx->base.send = safexcel_aes_send;
+
+                ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
+                                                 EIP197_GFP_FLAGS(req->base),
+                                                 &ctx->base.ctxr_dma);
+                if (!ctx->base.ctxr)
+                        return -ENOMEM;
+        }
+
+        ring = ctx->base.ring;
+
+        spin_lock_bh(&priv->ring[ring].queue_lock);
+        ret = crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
+        spin_unlock_bh(&priv->ring[ring].queue_lock);
+
+        if (!priv->ring[ring].need_dequeue)
+                safexcel_dequeue(priv, ring);
+
+        return ret;
+}
+
+static int safexcel_ecb_aes_encrypt(struct skcipher_request *req)
+{
+        return safexcel_aes(req, SAFEXCEL_ENCRYPT,
+                            CONTEXT_CONTROL_CRYPTO_MODE_ECB);
+}
+
+static int safexcel_ecb_aes_decrypt(struct skcipher_request *req)
+{
+        return safexcel_aes(req, SAFEXCEL_DECRYPT,
+                            CONTEXT_CONTROL_CRYPTO_MODE_ECB);
+}
+
+static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
+{
+        struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct safexcel_alg_template *tmpl =
+                container_of(tfm->__crt_alg, struct safexcel_alg_template,
+                             alg.skcipher.base);
+
+        ctx->priv = tmpl->priv;
+
+        return 0;
+}
+
+static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
+{
+        struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct safexcel_crypto_priv *priv = ctx->priv;
+        int ret;
+
+        memzero_explicit(ctx->key, 8 * sizeof(u32));
+
+        /* context not allocated, skip invalidation */
+        if (!ctx->base.ctxr)
+                return;
+
+        memzero_explicit(ctx->base.ctxr->data, 8 * sizeof(u32));
+
+        ret = safexcel_cipher_exit_inv(tfm);
+        if (ret)
+                dev_warn(priv->dev, "cipher: invalidation error %d\n", ret);
+}
+
+struct safexcel_alg_template safexcel_alg_ecb_aes = {
+        .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
+        .alg.skcipher = {
+                .setkey = safexcel_aes_setkey,
+                .encrypt = safexcel_ecb_aes_encrypt,
+                .decrypt = safexcel_ecb_aes_decrypt,
+                .min_keysize = AES_MIN_KEY_SIZE,
+                .max_keysize = AES_MAX_KEY_SIZE,
+                .base = {
+                        .cra_name = "ecb(aes)",
+                        .cra_driver_name = "safexcel-ecb-aes",
+                        .cra_priority = 300,
+                        .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
+                                     CRYPTO_ALG_KERN_DRIVER_ONLY,
+                        .cra_blocksize = AES_BLOCK_SIZE,
+                        .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+                        .cra_alignmask = 0,
+                        .cra_init = safexcel_skcipher_cra_init,
+                        .cra_exit = safexcel_skcipher_cra_exit,
+                        .cra_module = THIS_MODULE,
+                },
+        },
+};
+
+static int safexcel_cbc_aes_encrypt(struct skcipher_request *req)
+{
+        return safexcel_aes(req, SAFEXCEL_ENCRYPT,
+                            CONTEXT_CONTROL_CRYPTO_MODE_CBC);
+}
+
+static int safexcel_cbc_aes_decrypt(struct skcipher_request *req)
+{
+        return safexcel_aes(req, SAFEXCEL_DECRYPT,
+                            CONTEXT_CONTROL_CRYPTO_MODE_CBC);
+}
+
+struct safexcel_alg_template safexcel_alg_cbc_aes = {
+        .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
+        .alg.skcipher = {
+                .setkey = safexcel_aes_setkey,
+                .encrypt = safexcel_cbc_aes_encrypt,
+                .decrypt = safexcel_cbc_aes_decrypt,
+                .min_keysize = AES_MIN_KEY_SIZE,
+                .max_keysize = AES_MAX_KEY_SIZE,
+                .ivsize = AES_BLOCK_SIZE,
+                .base = {
+                        .cra_name = "cbc(aes)",
+                        .cra_driver_name = "safexcel-cbc-aes",
+                        .cra_priority = 300,
+                        .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
+                                     CRYPTO_ALG_KERN_DRIVER_ONLY,
+                        .cra_blocksize = AES_BLOCK_SIZE,
+                        .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
+                        .cra_alignmask = 0,
+                        .cra_init = safexcel_skcipher_cra_init,
+                        .cra_exit = safexcel_skcipher_cra_exit,
+                        .cra_module = THIS_MODULE,
+                },
+        },
+};
diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c
new file mode 100644
index 000000000000..8527a5899a2f
--- /dev/null
+++ b/drivers/crypto/inside-secure/safexcel_hash.c
@@ -0,0 +1,1052 @@
+/*
+ * Copyright (C) 2017 Marvell
+ *
+ * Antoine Tenart <antoine.tenart@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <crypto/hmac.h>
+#include <crypto/sha.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+
+
+#include "safexcel.h"
+
+struct safexcel_ahash_ctx {
+        struct safexcel_context base;
+        struct safexcel_crypto_priv *priv;
+
+        u32 alg;
+        u32 digest;
+
+        u32 ipad[SHA1_DIGEST_SIZE / sizeof(u32)];
+        u32 opad[SHA1_DIGEST_SIZE / sizeof(u32)];
+};
+
+struct safexcel_ahash_req {
+        bool last_req;
+        bool finish;
+        bool hmac;
+
+        u8 state_sz;    /* expected sate size, only set once */
+        u32 state[SHA256_DIGEST_SIZE / sizeof(u32)];
+
+        u64 len;
+        u64 processed;
+
+        u8 cache[SHA256_BLOCK_SIZE] __aligned(sizeof(u32));
+        u8 cache_next[SHA256_BLOCK_SIZE] __aligned(sizeof(u32));
+};
+
+struct safexcel_ahash_export_state {
+        u64 len;
+        u64 processed;
+
+        u32 state[SHA256_DIGEST_SIZE / sizeof(u32)];
+        u8 cache[SHA256_BLOCK_SIZE];
+};
+
+static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
+                                u32 input_length, u32 result_length)
+{
+        struct safexcel_token *token =
+                (struct safexcel_token *)cdesc->control_data.token;
+
+        token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
+        token[0].packet_length = input_length;
+        token[0].stat = EIP197_TOKEN_STAT_LAST_HASH;
+        token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH;
+
+        token[1].opcode = EIP197_TOKEN_OPCODE_INSERT;
+        token[1].packet_length = result_length;
+        token[1].stat = EIP197_TOKEN_STAT_LAST_HASH |
+                        EIP197_TOKEN_STAT_LAST_PACKET;
+        token[1].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
+                                EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
+}
+
+static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
+                                     struct safexcel_ahash_req *req,
+                                     struct safexcel_command_desc *cdesc,
+                                     unsigned int digestsize,
+                                     unsigned int blocksize)
+{
+        int i;
+
+        cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_OUT;
+        cdesc->control_data.control0 |= ctx->alg;
+        cdesc->control_data.control0 |= ctx->digest;
+
+        if (ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) {
+                if (req->processed) {
+                        if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
+                                cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(6);
+                        else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224 ||
+                                 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
+                                cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(9);
+
+                        cdesc->control_data.control1 |= CONTEXT_CONTROL_DIGEST_CNT;
+                } else {
+                        cdesc->control_data.control0 |= CONTEXT_CONTROL_RESTART_HASH;
+                }
+
+                if (!req->finish)
+                        cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH;
+
+                /*
+                 * Copy the input digest if needed, and setup the context
+                 * fields. Do this now as we need it to setup the first command
+                 * descriptor.
+                 */
+                if (req->processed) {
+                        for (i = 0; i < digestsize / sizeof(u32); i++)
+                                ctx->base.ctxr->data[i] = cpu_to_le32(req->state[i]);
+
+                        if (req->finish)
+                                ctx->base.ctxr->data[i] = cpu_to_le32(req->processed / blocksize);
+                }
+        } else if (ctx->digest == CONTEXT_CONTROL_DIGEST_HMAC) {
+                cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(10);
+
+                memcpy(ctx->base.ctxr->data, ctx->ipad, digestsize);
+                memcpy(ctx->base.ctxr->data + digestsize / sizeof(u32),
+                       ctx->opad, digestsize);
+        }
+}
+
+static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
+                                  struct crypto_async_request *async,
+                                  bool *should_complete, int *ret)
+{
+        struct safexcel_result_desc *rdesc;
+        struct ahash_request *areq = ahash_request_cast(async);
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
+        struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);
+        int cache_len, result_sz = sreq->state_sz;
+
+        *ret = 0;
+
+        spin_lock_bh(&priv->ring[ring].egress_lock);
+        rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
+        if (IS_ERR(rdesc)) {
+                dev_err(priv->dev,
+                        "hash: result: could not retrieve the result descriptor\n");
+                *ret = PTR_ERR(rdesc);
+        } else if (rdesc->result_data.error_code) {
+                dev_err(priv->dev,
+                        "hash: result: result descriptor error (%d)\n",
+                        rdesc->result_data.error_code);
+                *ret = -EINVAL;
+        }
+
+        safexcel_complete(priv, ring);
+        spin_unlock_bh(&priv->ring[ring].egress_lock);
+
+        if (sreq->finish)
+                result_sz = crypto_ahash_digestsize(ahash);
+        memcpy(sreq->state, areq->result, result_sz);
+
+        dma_unmap_sg(priv->dev, areq->src,
+                     sg_nents_for_len(areq->src, areq->nbytes), DMA_TO_DEVICE);
+
+        safexcel_free_context(priv, async, sreq->state_sz);
+
+        cache_len = sreq->len - sreq->processed;
+        if (cache_len)
+                memcpy(sreq->cache, sreq->cache_next, cache_len);
+
+        *should_complete = true;
+
+        return 1;
+}
+
+static int safexcel_ahash_send(struct crypto_async_request *async, int ring,
+                               struct safexcel_request *request, int *commands,
+                               int *results)
+{
+        struct ahash_request *areq = ahash_request_cast(async);
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+        struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+        struct safexcel_crypto_priv *priv = ctx->priv;
+        struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
+        struct safexcel_result_desc *rdesc;
+        struct scatterlist *sg;
+        int i, nents, queued, len, cache_len, extra, n_cdesc = 0, ret = 0;
+
+        queued = len = req->len - req->processed;
+        if (queued < crypto_ahash_blocksize(ahash))
+                cache_len = queued;
+        else
+                cache_len = queued - areq->nbytes;
+
+        /*
+         * If this is not the last request and the queued data does not fit
+         * into full blocks, cache it for the next send() call.
+         */
+        extra = queued & (crypto_ahash_blocksize(ahash) - 1);
+        if (!req->last_req && extra) {
+                sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
+                                   req->cache_next, extra, areq->nbytes - extra);
+
+                queued -= extra;
+                len -= extra;
+        }
+
+        spin_lock_bh(&priv->ring[ring].egress_lock);
+
+        /* Add a command descriptor for the cached data, if any */
+        if (cache_len) {
+                ctx->base.cache = kzalloc(cache_len, EIP197_GFP_FLAGS(*async));
+                if (!ctx->base.cache) {
+                        ret = -ENOMEM;
+                        goto unlock;
+                }
+                memcpy(ctx->base.cache, req->cache, cache_len);
+                ctx->base.cache_dma = dma_map_single(priv->dev, ctx->base.cache,
+                                                     cache_len, DMA_TO_DEVICE);
+                if (dma_mapping_error(priv->dev, ctx->base.cache_dma)) {
+                        ret = -EINVAL;
+                        goto free_cache;
+                }
+
+                ctx->base.cache_sz = cache_len;
+                first_cdesc = safexcel_add_cdesc(priv, ring, 1,
+                                                 (cache_len == len),
+                                                 ctx->base.cache_dma,
+                                                 cache_len, len,
+                                                 ctx->base.ctxr_dma);
+                if (IS_ERR(first_cdesc)) {
+                        ret = PTR_ERR(first_cdesc);
+                        goto unmap_cache;
+                }
+                n_cdesc++;
+
+                queued -= cache_len;
+                if (!queued)
+                        goto send_command;
+        }
+
+        /* Now handle the current ahash request buffer(s) */
+        nents = dma_map_sg(priv->dev, areq->src,
+                       sg_nents_for_len(areq->src, areq->nbytes),
+                       DMA_TO_DEVICE);
+        if (!nents) {
+                ret = -ENOMEM;
+                goto cdesc_rollback;
+        }
+
+        for_each_sg(areq->src, sg, nents, i) {
+                int sglen = sg_dma_len(sg);
+
+                /* Do not overflow the request */
+                if (queued - sglen < 0)
+                        sglen = queued;
+
+                cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
+                                           !(queued - sglen), sg_dma_address(sg),
+                                           sglen, len, ctx->base.ctxr_dma);
+                if (IS_ERR(cdesc)) {
+                        ret = PTR_ERR(cdesc);
+                        goto cdesc_rollback;
+                }
+                n_cdesc++;
+
+                if (n_cdesc == 1)
+                        first_cdesc = cdesc;
+
+                queued -= sglen;
+                if (!queued)
+                        break;
+        }
+
+send_command:
+        /* Setup the context options */
+        safexcel_context_control(ctx, req, first_cdesc, req->state_sz,
+                                 crypto_ahash_blocksize(ahash));
+
+        /* Add the token */
+        safexcel_hash_token(first_cdesc, len, req->state_sz);
+
+        ctx->base.result_dma = dma_map_single(priv->dev, areq->result,
+                                              req->state_sz, DMA_FROM_DEVICE);
+        if (dma_mapping_error(priv->dev, ctx->base.result_dma)) {
+                ret = -EINVAL;
+                goto cdesc_rollback;
+        }
+
+        /* Add a result descriptor */
+        rdesc = safexcel_add_rdesc(priv, ring, 1, 1, ctx->base.result_dma,
+                                   req->state_sz);
+        if (IS_ERR(rdesc)) {
+                ret = PTR_ERR(rdesc);
+                goto cdesc_rollback;
+        }
+
+        spin_unlock_bh(&priv->ring[ring].egress_lock);
+
+        req->processed += len;
+        request->req = &areq->base;
+        ctx->base.handle_result = safexcel_handle_result;
+
+        *commands = n_cdesc;
+        *results = 1;
+        return 0;
+
+cdesc_rollback:
+        for (i = 0; i < n_cdesc; i++)
+                safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
+unmap_cache:
+        if (ctx->base.cache_dma) {
+                dma_unmap_single(priv->dev, ctx->base.cache_dma,
+                                 ctx->base.cache_sz, DMA_TO_DEVICE);
+                ctx->base.cache_sz = 0;
+        }
+free_cache:
+        if (ctx->base.cache) {
+                kfree(ctx->base.cache);
+                ctx->base.cache = NULL;
+        }
+
+unlock:
+        spin_unlock_bh(&priv->ring[ring].egress_lock);
+        return ret;
+}
+
+static inline bool safexcel_ahash_needs_inv_get(struct ahash_request *areq)
+{
+        struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
+        unsigned int state_w_sz = req->state_sz / sizeof(u32);
+        int i;
+
+        for (i = 0; i < state_w_sz; i++)
+                if (ctx->base.ctxr->data[i] != cpu_to_le32(req->state[i]))
+                        return true;
+
+        if (ctx->base.ctxr->data[state_w_sz] !=
+            cpu_to_le32(req->processed / crypto_ahash_blocksize(ahash)))
+                return true;
+
+        return false;
+}
+
+static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
+                                      int ring,
+                                      struct crypto_async_request *async,
+                                      bool *should_complete, int *ret)
+{
+        struct safexcel_result_desc *rdesc;
+        struct ahash_request *areq = ahash_request_cast(async);
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
+        struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
+        int enq_ret;
+
+        *ret = 0;
+
+        spin_lock_bh(&priv->ring[ring].egress_lock);
+        rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
+        if (IS_ERR(rdesc)) {
+                dev_err(priv->dev,
+                        "hash: invalidate: could not retrieve the result descriptor\n");
+                *ret = PTR_ERR(rdesc);
+        } else if (rdesc->result_data.error_code) {
+                dev_err(priv->dev,
+                        "hash: invalidate: result descriptor error (%d)\n",
+                        rdesc->result_data.error_code);
+                *ret = -EINVAL;
+        }
+
+        safexcel_complete(priv, ring);
+        spin_unlock_bh(&priv->ring[ring].egress_lock);
+
+        if (ctx->base.exit_inv) {
+                dma_pool_free(priv->context_pool, ctx->base.ctxr,
+                              ctx->base.ctxr_dma);
+
+                *should_complete = true;
+                return 1;
+        }
+
+        ring = safexcel_select_ring(priv);
+        ctx->base.ring = ring;
+        ctx->base.needs_inv = false;
+        ctx->base.send = safexcel_ahash_send;
+
+        spin_lock_bh(&priv->ring[ring].queue_lock);
+        enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
+        spin_unlock_bh(&priv->ring[ring].queue_lock);
+
+        if (enq_ret != -EINPROGRESS)
+                *ret = enq_ret;
+
+        if (!priv->ring[ring].need_dequeue)
+                safexcel_dequeue(priv, ring);
+
+        *should_complete = false;
+
+        return 1;
+}
+
+static int safexcel_ahash_send_inv(struct crypto_async_request *async,
+                                   int ring, struct safexcel_request *request,
+                                   int *commands, int *results)
+{
+        struct ahash_request *areq = ahash_request_cast(async);
+        struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+        int ret;
+
+        ctx->base.handle_result = safexcel_handle_inv_result;
+        ret = safexcel_invalidate_cache(async, &ctx->base, ctx->priv,
+                                        ctx->base.ctxr_dma, ring, request);
+        if (unlikely(ret))
+                return ret;
+
+        *commands = 1;
+        *results = 1;
+
+        return 0;
+}
+
+static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
+{
+        struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct safexcel_crypto_priv *priv = ctx->priv;
+        struct ahash_request req;
+        struct safexcel_inv_result result = { 0 };
+        int ring = ctx->base.ring;
+
+        memset(&req, 0, sizeof(struct ahash_request));
+
+        /* create invalidation request */
+        init_completion(&result.completion);
+        ahash_request_set_callback(&req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+                                   safexcel_inv_complete, &result);
+
+        ahash_request_set_tfm(&req, __crypto_ahash_cast(tfm));
+        ctx = crypto_tfm_ctx(req.base.tfm);
+        ctx->base.exit_inv = true;
+        ctx->base.send = safexcel_ahash_send_inv;
+
+        spin_lock_bh(&priv->ring[ring].queue_lock);
+        crypto_enqueue_request(&priv->ring[ring].queue, &req.base);
+        spin_unlock_bh(&priv->ring[ring].queue_lock);
+
+        if (!priv->ring[ring].need_dequeue)
+                safexcel_dequeue(priv, ring);
+
+        wait_for_completion_interruptible(&result.completion);
+
+        if (result.error) {
+                dev_warn(priv->dev, "hash: completion error (%d)\n",
+                         result.error);
+                return result.error;
+        }
+
+        return 0;
+}
+
+static int safexcel_ahash_cache(struct ahash_request *areq)
+{
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
+        int queued, cache_len;
+
+        cache_len = req->len - areq->nbytes - req->processed;
+        queued = req->len - req->processed;
+
+        /*
+         * In case there isn't enough bytes to proceed (less than a
+         * block size), cache the data until we have enough.
+         */
+        if (cache_len + areq->nbytes <= crypto_ahash_blocksize(ahash)) {
+                sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
+                                   req->cache + cache_len,
+                                   areq->nbytes, 0);
+                return areq->nbytes;
+        }
+
+        /* We could'nt cache all the data */
+        return -E2BIG;
+}
+
+static int safexcel_ahash_enqueue(struct ahash_request *areq)
+{
+        struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+        struct safexcel_crypto_priv *priv = ctx->priv;
+        int ret, ring;
+
+        ctx->base.send = safexcel_ahash_send;
+
+        if (req->processed && ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)
+                ctx->base.needs_inv = safexcel_ahash_needs_inv_get(areq);
+
+        if (ctx->base.ctxr) {
+                if (ctx->base.needs_inv)
+                        ctx->base.send = safexcel_ahash_send_inv;
+        } else {
+                ctx->base.ring = safexcel_select_ring(priv);
+                ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
+                                                 EIP197_GFP_FLAGS(areq->base),
+                                                 &ctx->base.ctxr_dma);
+                if (!ctx->base.ctxr)
+                        return -ENOMEM;
+        }
+
+        ring = ctx->base.ring;
+
+        spin_lock_bh(&priv->ring[ring].queue_lock);
+        ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
+        spin_unlock_bh(&priv->ring[ring].queue_lock);
+
+        if (!priv->ring[ring].need_dequeue)
+                safexcel_dequeue(priv, ring);
+
+        return ret;
+}
+
+static int safexcel_ahash_update(struct ahash_request *areq)
+{
+        struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
+
+        /* If the request is 0 length, do nothing */
+        if (!areq->nbytes)
+                return 0;
+
+        req->len += areq->nbytes;
+
+        safexcel_ahash_cache(areq);
+
+        /*
+         * We're not doing partial updates when performing an hmac request.
+         * Everything will be handled by the final() call.
+         */
+        if (ctx->digest == CONTEXT_CONTROL_DIGEST_HMAC)
+                return 0;
+
+        if (req->hmac)
+                return safexcel_ahash_enqueue(areq);
+
+        if (!req->last_req &&
+            req->len - req->processed > crypto_ahash_blocksize(ahash))
+                return safexcel_ahash_enqueue(areq);
+
+        return 0;
+}
+
+static int safexcel_ahash_final(struct ahash_request *areq)
+{
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+        struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+
+        req->last_req = true;
+        req->finish = true;
+
+        /* If we have an overall 0 length request */
+        if (!(req->len + areq->nbytes)) {
+                if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
+                        memcpy(areq->result, sha1_zero_message_hash,
+                               SHA1_DIGEST_SIZE);
+                else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
+                        memcpy(areq->result, sha224_zero_message_hash,
+                               SHA224_DIGEST_SIZE);
+                else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
+                        memcpy(areq->result, sha256_zero_message_hash,
+                               SHA256_DIGEST_SIZE);
+
+                return 0;
+        }
+
+        return safexcel_ahash_enqueue(areq);
+}
+
+static int safexcel_ahash_finup(struct ahash_request *areq)
+{
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+
+        req->last_req = true;
+        req->finish = true;
+
+        safexcel_ahash_update(areq);
+        return safexcel_ahash_final(areq);
+}
+
+static int safexcel_ahash_export(struct ahash_request *areq, void *out)
+{
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+        struct safexcel_ahash_export_state *export = out;
+
+        export->len = req->len;
+        export->processed = req->processed;
+
+        memcpy(export->state, req->state, req->state_sz);
+        memset(export->cache, 0, crypto_ahash_blocksize(ahash));
+        memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash));
+
+        return 0;
+}
+
+static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
+{
+        struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+        const struct safexcel_ahash_export_state *export = in;
+        int ret;
+
+        ret = crypto_ahash_init(areq);
+        if (ret)
+                return ret;
+
+        req->len = export->len;
+        req->processed = export->processed;
+
+        memcpy(req->cache, export->cache, crypto_ahash_blocksize(ahash));
+        memcpy(req->state, export->state, req->state_sz);
+
+        return 0;
+}
+
+static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
+{
+        struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct safexcel_alg_template *tmpl =
+                container_of(__crypto_ahash_alg(tfm->__crt_alg),
+                             struct safexcel_alg_template, alg.ahash);
+
+        ctx->priv = tmpl->priv;
+
+        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+                                 sizeof(struct safexcel_ahash_req));
+        return 0;
+}
+
+static int safexcel_sha1_init(struct ahash_request *areq)
+{
+        struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+
+        memset(req, 0, sizeof(*req));
+
+        req->state[0] = SHA1_H0;
+        req->state[1] = SHA1_H1;
+        req->state[2] = SHA1_H2;
+        req->state[3] = SHA1_H3;
+        req->state[4] = SHA1_H4;
+
+        ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
+        ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+        req->state_sz = SHA1_DIGEST_SIZE;
+
+        return 0;
+}
+
+static int safexcel_sha1_digest(struct ahash_request *areq)
+{
+        int ret = safexcel_sha1_init(areq);
+
+        if (ret)
+                return ret;
+
+        return safexcel_ahash_finup(areq);
+}
+
+static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
+{
+        struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct safexcel_crypto_priv *priv = ctx->priv;
+        int ret;
+
+        /* context not allocated, skip invalidation */
+        if (!ctx->base.ctxr)
+                return;
+
+        ret = safexcel_ahash_exit_inv(tfm);
+        if (ret)
+                dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
+}
+
+struct safexcel_alg_template safexcel_alg_sha1 = {
+        .type = SAFEXCEL_ALG_TYPE_AHASH,
+        .alg.ahash = {
+                .init = safexcel_sha1_init,
+                .update = safexcel_ahash_update,
+                .final = safexcel_ahash_final,
+                .finup = safexcel_ahash_finup,
+                .digest = safexcel_sha1_digest,
+                .export = safexcel_ahash_export,
+                .import = safexcel_ahash_import,
+                .halg = {
+                        .digestsize = SHA1_DIGEST_SIZE,
+                        .statesize = sizeof(struct safexcel_ahash_export_state),
+                        .base = {
+                                .cra_name = "sha1",
+                                .cra_driver_name = "safexcel-sha1",
+                                .cra_priority = 300,
+                                .cra_flags = CRYPTO_ALG_ASYNC |
+                                             CRYPTO_ALG_KERN_DRIVER_ONLY,
+                                .cra_blocksize = SHA1_BLOCK_SIZE,
+                                .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
+                                .cra_init = safexcel_ahash_cra_init,
+                                .cra_exit = safexcel_ahash_cra_exit,
+                                .cra_module = THIS_MODULE,
+                        },
+                },
+        },
+};
+
+static int safexcel_hmac_sha1_init(struct ahash_request *areq)
+{
+        struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+
+        safexcel_sha1_init(areq);
+        ctx->digest = CONTEXT_CONTROL_DIGEST_HMAC;
+        return 0;
+}
+
+static int safexcel_hmac_sha1_digest(struct ahash_request *areq)
+{
+        int ret = safexcel_hmac_sha1_init(areq);
+
+        if (ret)
+                return ret;
+
+        return safexcel_ahash_finup(areq);
+}
+
+struct safexcel_ahash_result {
+        struct completion completion;
+        int error;
+};
+
+static void safexcel_ahash_complete(struct crypto_async_request *req, int error)
+{
+        struct safexcel_ahash_result *result = req->data;
+
+        if (error == -EINPROGRESS)
+                return;
+
+        result->error = error;
+        complete(&result->completion);
+}
+
+static int safexcel_hmac_init_pad(struct ahash_request *areq,
+                                  unsigned int blocksize, const u8 *key,
+                                  unsigned int keylen, u8 *ipad, u8 *opad)
+{
+        struct safexcel_ahash_result result;
+        struct scatterlist sg;
+        int ret, i;
+        u8 *keydup;
+
+        if (keylen <= blocksize) {
+                memcpy(ipad, key, keylen);
+        } else {
+                keydup = kmemdup(key, keylen, GFP_KERNEL);
+                if (!keydup)
+                        return -ENOMEM;
+
+                ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
+                                           safexcel_ahash_complete, &result);
+                sg_init_one(&sg, keydup, keylen);
+                ahash_request_set_crypt(areq, &sg, ipad, keylen);
+                init_completion(&result.completion);
+
+                ret = crypto_ahash_digest(areq);
+                if (ret == -EINPROGRESS) {
+                        wait_for_completion_interruptible(&result.completion);
+                        ret = result.error;
+                }
+
+                /* Avoid leaking */
+                memzero_explicit(keydup, keylen);
+                kfree(keydup);
+
+                if (ret)
+                        return ret;
+
+                keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
+        }
+
+        memset(ipad + keylen, 0, blocksize - keylen);
+        memcpy(opad, ipad, blocksize);
+
+        for (i = 0; i < blocksize; i++) {
+                ipad[i] ^= HMAC_IPAD_VALUE;
+                opad[i] ^= HMAC_OPAD_VALUE;
+        }
+
+        return 0;
+}
+
+static int safexcel_hmac_init_iv(struct ahash_request *areq,
+                                 unsigned int blocksize, u8 *pad, void *state)
+{
+        struct safexcel_ahash_result result;
+        struct safexcel_ahash_req *req;
+        struct scatterlist sg;
+        int ret;
+
+        ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
+                                   safexcel_ahash_complete, &result);
+        sg_init_one(&sg, pad, blocksize);
+        ahash_request_set_crypt(areq, &sg, pad, blocksize);
+        init_completion(&result.completion);
+
+        ret = crypto_ahash_init(areq);
+        if (ret)
+                return ret;
+
+        req = ahash_request_ctx(areq);
+        req->hmac = true;
+        req->last_req = true;
+
+        ret = crypto_ahash_update(areq);
+        if (ret && ret != -EINPROGRESS)
+                return ret;
+
+        wait_for_completion_interruptible(&result.completion);
+        if (result.error)
+                return result.error;
+
+        return crypto_ahash_export(areq, state);
+}
+
+static int safexcel_hmac_setkey(const char *alg, const u8 *key,
+                                unsigned int keylen, void *istate, void *ostate)
+{
+        struct ahash_request *areq;
+        struct crypto_ahash *tfm;
+        unsigned int blocksize;
+        u8 *ipad, *opad;
+        int ret;
+
+        tfm = crypto_alloc_ahash(alg, CRYPTO_ALG_TYPE_AHASH,
+                                 CRYPTO_ALG_TYPE_AHASH_MASK);
+        if (IS_ERR(tfm))
+                return PTR_ERR(tfm);
+
+        areq = ahash_request_alloc(tfm, GFP_KERNEL);
+        if (!areq) {
+                ret = -ENOMEM;
+                goto free_ahash;
+        }
+
+        crypto_ahash_clear_flags(tfm, ~0);
+        blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+        ipad = kzalloc(2 * blocksize, GFP_KERNEL);
+        if (!ipad) {
+                ret = -ENOMEM;
+                goto free_request;
+        }
+
+        opad = ipad + blocksize;
+
+        ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);
+        if (ret)
+                goto free_ipad;
+
+        ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);
+        if (ret)
+                goto free_ipad;
+
+        ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);
+
+free_ipad:
+        kfree(ipad);
+free_request:
+        ahash_request_free(areq);
+free_ahash:
+        crypto_free_ahash(tfm);
+
+        return ret;
+}
+
+static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
+                                     unsigned int keylen)
+{
+        struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
+        struct safexcel_ahash_export_state istate, ostate;
+        int ret, i;
+
+        ret = safexcel_hmac_setkey("safexcel-sha1", key, keylen, &istate, &ostate);
+        if (ret)
+                return ret;
+
+        memcpy(ctx->ipad, &istate.state, SHA1_DIGEST_SIZE);
+        memcpy(ctx->opad, &ostate.state, SHA1_DIGEST_SIZE);
+
+        for (i = 0; i < ARRAY_SIZE(istate.state); i++) {
+                if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) ||
+                    ctx->opad[i] != le32_to_cpu(ostate.state[i])) {
+                        ctx->base.needs_inv = true;
+                        break;
+                }
+        }
+
+        return 0;
+}
+
+struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
+        .type = SAFEXCEL_ALG_TYPE_AHASH,
+        .alg.ahash = {
+                .init = safexcel_hmac_sha1_init,
+                .update = safexcel_ahash_update,
+                .final = safexcel_ahash_final,
+                .finup = safexcel_ahash_finup,
+                .digest = safexcel_hmac_sha1_digest,
+                .setkey = safexcel_hmac_sha1_setkey,
+                .export = safexcel_ahash_export,
+                .import = safexcel_ahash_import,
+                .halg = {
+                        .digestsize = SHA1_DIGEST_SIZE,
+                        .statesize = sizeof(struct safexcel_ahash_export_state),
+                        .base = {
+                                .cra_name = "hmac(sha1)",
+                                .cra_driver_name = "safexcel-hmac-sha1",
+                                .cra_priority = 300,
+                                .cra_flags = CRYPTO_ALG_ASYNC |
+                                             CRYPTO_ALG_KERN_DRIVER_ONLY,
+                                .cra_blocksize = SHA1_BLOCK_SIZE,
+                                .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
+                                .cra_init = safexcel_ahash_cra_init,
+                                .cra_exit = safexcel_ahash_cra_exit,
+                                .cra_module = THIS_MODULE,
+                        },
+                },
+        },
+};
+
+static int safexcel_sha256_init(struct ahash_request *areq)
+{
+        struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+
+        memset(req, 0, sizeof(*req));
+
+        req->state[0] = SHA256_H0;
+        req->state[1] = SHA256_H1;
+        req->state[2] = SHA256_H2;
+        req->state[3] = SHA256_H3;
+        req->state[4] = SHA256_H4;
+        req->state[5] = SHA256_H5;
+        req->state[6] = SHA256_H6;
+        req->state[7] = SHA256_H7;
+
+        ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
+        ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+        req->state_sz = SHA256_DIGEST_SIZE;
+
+        return 0;
+}
+
+static int safexcel_sha256_digest(struct ahash_request *areq)
+{
+        int ret = safexcel_sha256_init(areq);
+
+        if (ret)
+                return ret;
+
+        return safexcel_ahash_finup(areq);
+}
+
+struct safexcel_alg_template safexcel_alg_sha256 = {
+        .type = SAFEXCEL_ALG_TYPE_AHASH,
+        .alg.ahash = {
+                .init = safexcel_sha256_init,
+                .update = safexcel_ahash_update,
+                .final = safexcel_ahash_final,
+                .finup = safexcel_ahash_finup,
+                .digest = safexcel_sha256_digest,
+                .export = safexcel_ahash_export,
+                .import = safexcel_ahash_import,
+                .halg = {
+                        .digestsize = SHA256_DIGEST_SIZE,
+                        .statesize = sizeof(struct safexcel_ahash_export_state),
+                        .base = {
+                                .cra_name = "sha256",
+                                .cra_driver_name = "safexcel-sha256",
+                                .cra_priority = 300,
+                                .cra_flags = CRYPTO_ALG_ASYNC |
+                                             CRYPTO_ALG_KERN_DRIVER_ONLY,
+                                .cra_blocksize = SHA256_BLOCK_SIZE,
+                                .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
+                                .cra_init = safexcel_ahash_cra_init,
+                                .cra_exit = safexcel_ahash_cra_exit,
+                                .cra_module = THIS_MODULE,
+                        },
+                },
+        },
+};
+
+static int safexcel_sha224_init(struct ahash_request *areq)
+{
+        struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+        struct safexcel_ahash_req *req = ahash_request_ctx(areq);
+
+        memset(req, 0, sizeof(*req));
+
+        req->state[0] = SHA224_H0;
+        req->state[1] = SHA224_H1;
+        req->state[2] = SHA224_H2;
+        req->state[3] = SHA224_H3;
+        req->state[4] = SHA224_H4;
+        req->state[5] = SHA224_H5;
+        req->state[6] = SHA224_H6;
+        req->state[7] = SHA224_H7;
+
+        ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
+        ctx->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
+        req->state_sz = SHA256_DIGEST_SIZE;
+
+        return 0;
+}
+
+static int safexcel_sha224_digest(struct ahash_request *areq)
+{
+        int ret = safexcel_sha224_init(areq);
+
+        if (ret)
+                return ret;
+
+        return safexcel_ahash_finup(areq);
+}
+
+struct safexcel_alg_template safexcel_alg_sha224 = {
+        .type = SAFEXCEL_ALG_TYPE_AHASH,
+        .alg.ahash = {
+                .init = safexcel_sha224_init,
+                .update = safexcel_ahash_update,
+                .final = safexcel_ahash_final,
+                .finup = safexcel_ahash_finup,
+                .digest = safexcel_sha224_digest,
+                .export = safexcel_ahash_export,
+                .import = safexcel_ahash_import,
+                .halg = {
+                        .digestsize = SHA224_DIGEST_SIZE,
+                        .statesize = sizeof(struct safexcel_ahash_export_state),
+                        .base = {
+                                .cra_name = "sha224",
+                                .cra_driver_name = "safexcel-sha224",
+                                .cra_priority = 300,
+                                .cra_flags = CRYPTO_ALG_ASYNC |
+                                             CRYPTO_ALG_KERN_DRIVER_ONLY,
+                                .cra_blocksize = SHA224_BLOCK_SIZE,
+                                .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
+                                .cra_init = safexcel_ahash_cra_init,
+                                .cra_exit = safexcel_ahash_cra_exit,
+                                .cra_module = THIS_MODULE,
+                        },
+                },
+        },
+};
diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c
new file mode 100644
index 000000000000..c9d2a8716b5b
--- /dev/null
+++ b/drivers/crypto/inside-secure/safexcel_ring.c
@@ -0,0 +1,157 @@
+/*
+ * Copyright (C) 2017 Marvell
+ *
+ * Antoine Tenart <antoine.tenart@free-electrons.com>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2. This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/spinlock.h>
+
+#include "safexcel.h"
+
+int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
+                                   struct safexcel_ring *cdr,
+                                   struct safexcel_ring *rdr)
+{
+        cdr->offset = sizeof(u32) * priv->config.cd_offset;
+        cdr->base = dmam_alloc_coherent(priv->dev,
+                                        cdr->offset * EIP197_DEFAULT_RING_SIZE,
+                                        &cdr->base_dma, GFP_KERNEL);
+        if (!cdr->base)
+                return -ENOMEM;
+        cdr->write = cdr->base;
+        cdr->base_end = cdr->base + cdr->offset * EIP197_DEFAULT_RING_SIZE;
+        cdr->read = cdr->base;
+
+        rdr->offset = sizeof(u32) * priv->config.rd_offset;
+        rdr->base = dmam_alloc_coherent(priv->dev,
+                                        rdr->offset * EIP197_DEFAULT_RING_SIZE,
+                                        &rdr->base_dma, GFP_KERNEL);
+        if (!rdr->base)
+                return -ENOMEM;
+        rdr->write = rdr->base;
+        rdr->base_end = rdr->base + rdr->offset * EIP197_DEFAULT_RING_SIZE;
+        rdr->read = rdr->base;
+
+        return 0;
+}
+
+inline int safexcel_select_ring(struct safexcel_crypto_priv *priv)
+{
+        return (atomic_inc_return(&priv->ring_used) % priv->config.rings);
+}
+
+static void *safexcel_ring_next_wptr(struct safexcel_crypto_priv *priv,
+                                     struct safexcel_ring *ring)
+{
+        void *ptr = ring->write;
+
+        if (ring->nr == EIP197_DEFAULT_RING_SIZE - 1)
+                return ERR_PTR(-ENOMEM);
+
+        ring->write += ring->offset;
+        if (ring->write == ring->base_end)
+                ring->write = ring->base;
+
+        ring->nr++;
+        return ptr;
+}
+
+void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv,
+                              struct safexcel_ring *ring)
+{
+        void *ptr = ring->read;
+
+        if (!ring->nr)
+                return ERR_PTR(-ENOENT);
+
+        ring->read += ring->offset;
+        if (ring->read == ring->base_end)
+                ring->read = ring->base;
+
+        ring->nr--;
+        return ptr;
+}
+
+void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,
+                                 struct safexcel_ring *ring)
+{
+        if (!ring->nr)
+                return;
+
+        if (ring->write == ring->base)
+                ring->write = ring->base_end - ring->offset;
+        else
+                ring->write -= ring->offset;
+
+        ring->nr--;
+}
+
+struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv,
+                                                 int ring_id,
+                                                 bool first, bool last,
+                                                 dma_addr_t data, u32 data_len,
+                                                 u32 full_data_len,
+                                                 dma_addr_t context) {
+        struct safexcel_command_desc *cdesc;
+        int i;
+
+        cdesc = safexcel_ring_next_wptr(priv, &priv->ring[ring_id].cdr);
+        if (IS_ERR(cdesc))
+                return cdesc;
+
+        memset(cdesc, 0, sizeof(struct safexcel_command_desc));
+
+        cdesc->first_seg = first;
+        cdesc->last_seg = last;
+        cdesc->particle_size = data_len;
+        cdesc->data_lo = lower_32_bits(data);
+        cdesc->data_hi = upper_32_bits(data);
+
+        if (first && context) {
+                struct safexcel_token *token =
+                        (struct safexcel_token *)cdesc->control_data.token;
+
+                cdesc->control_data.packet_length = full_data_len;
+                cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE |
+                                              EIP197_OPTION_64BIT_CTX |
+                                              EIP197_OPTION_CTX_CTRL_IN_CMD;
+                cdesc->control_data.context_lo =
+                        (lower_32_bits(context) & GENMASK(31, 2)) >> 2;
+                cdesc->control_data.context_hi = upper_32_bits(context);
+
+                /* TODO: large xform HMAC with SHA-384/512 uses refresh = 3 */
+                cdesc->control_data.refresh = 2;
+
+                for (i = 0; i < EIP197_MAX_TOKENS; i++)
+                        eip197_noop_token(&token[i]);
+        }
+
+        return cdesc;
+}
+
+struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv,
+                                                int ring_id,
+                                                bool first, bool last,
+                                                dma_addr_t data, u32 len)
+{
+        struct safexcel_result_desc *rdesc;
+
+        rdesc = safexcel_ring_next_wptr(priv, &priv->ring[ring_id].rdr);
+        if (IS_ERR(rdesc))
+                return rdesc;
+
+        memset(rdesc, 0, sizeof(struct safexcel_result_desc));
+
+        rdesc->first_seg = first;
+        rdesc->last_seg = last;
+        rdesc->particle_size = len;
+        rdesc->data_lo = lower_32_bits(data);
+        rdesc->data_hi = upper_32_bits(data);
+
+        return rdesc;
+}
diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c
index 771dd26c7076..427cbe012729 100644
--- a/drivers/crypto/ixp4xx_crypto.c
+++ b/drivers/crypto/ixp4xx_crypto.c
@@ -23,6 +23,7 @@
 #include <crypto/ctr.h>
 #include <crypto/des.h>
 #include <crypto/aes.h>
+#include <crypto/hmac.h>
 #include <crypto/sha.h>
 #include <crypto/algapi.h>
 #include <crypto/internal/aead.h>
@@ -90,8 +91,6 @@
 #define CTL_FLAG_PERFORM_AEAD   0x0008
 #define CTL_FLAG_MASK           0x000f
 
-#define HMAC_IPAD_VALUE   0x36
-#define HMAC_OPAD_VALUE   0x5C
 #define HMAC_PAD_BLOCKLEN SHA1_BLOCK_SIZE
 
 #define MD5_DIGEST_SIZE   16
diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c
index 77c0fb936f47..e61b08566093 100644
--- a/drivers/crypto/marvell/hash.c
+++ b/drivers/crypto/marvell/hash.c
@@ -12,6 +12,7 @@
  * by the Free Software Foundation.
  */
 
+#include <crypto/hmac.h>
 #include <crypto/md5.h>
 #include <crypto/sha.h>
 
@@ -1164,8 +1165,8 @@ static int mv_cesa_ahmac_pad_init(struct ahash_request *req,
         memcpy(opad, ipad, blocksize);
 
         for (i = 0; i < blocksize; i++) {
-                ipad[i] ^= 0x36;
-                opad[i] ^= 0x5c;
+                ipad[i] ^= HMAC_IPAD_VALUE;
+                opad[i] ^= HMAC_OPAD_VALUE;
         }
 
         return 0;
diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c
index b6ecc288b540..000b6500a22d 100644
--- a/drivers/crypto/mediatek/mtk-platform.c
+++ b/drivers/crypto/mediatek/mtk-platform.c
@@ -504,19 +504,14 @@ static int mtk_crypto_probe(struct platform_device *pdev)
                 }
         }
 
-        cryp->clk_ethif = devm_clk_get(&pdev->dev, "ethif");
         cryp->clk_cryp = devm_clk_get(&pdev->dev, "cryp");
-        if (IS_ERR(cryp->clk_ethif) || IS_ERR(cryp->clk_cryp))
+        if (IS_ERR(cryp->clk_cryp))
                 return -EPROBE_DEFER;
 
         cryp->dev = &pdev->dev;
         pm_runtime_enable(cryp->dev);
         pm_runtime_get_sync(cryp->dev);
 
-        err = clk_prepare_enable(cryp->clk_ethif);
-        if (err)
-                goto err_clk_ethif;
-
         err = clk_prepare_enable(cryp->clk_cryp);
         if (err)
                 goto err_clk_cryp;
@@ -559,8 +554,6 @@ err_engine:
 err_resource:
         clk_disable_unprepare(cryp->clk_cryp);
 err_clk_cryp:
-        clk_disable_unprepare(cryp->clk_ethif);
-err_clk_ethif:
         pm_runtime_put_sync(cryp->dev);
         pm_runtime_disable(cryp->dev);
 
@@ -576,7 +569,6 @@ static int mtk_crypto_remove(struct platform_device *pdev)
         mtk_desc_dma_free(cryp);
 
         clk_disable_unprepare(cryp->clk_cryp);
-        clk_disable_unprepare(cryp->clk_ethif);
 
         pm_runtime_put_sync(cryp->dev);
         pm_runtime_disable(cryp->dev);
@@ -596,7 +588,6 @@ static struct platform_driver mtk_crypto_driver = {
         .remove = mtk_crypto_remove,
         .driver = {
                    .name = "mtk-crypto",
-                   .owner = THIS_MODULE,
                    .of_match_table = of_crypto_id,
         },
 };
diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h
index 303c152dc931..f0831f1742ab 100644
--- a/drivers/crypto/mediatek/mtk-platform.h
+++ b/drivers/crypto/mediatek/mtk-platform.h
@@ -200,7 +200,6 @@ struct mtk_sha_rec {
  * struct mtk_cryp - Cryptographic device
  * @base:       pointer to mapped register I/O base
  * @dev:        pointer to device
- * @clk_ethif:  pointer to ethif clock
  * @clk_cryp:   pointer to crypto clock
  * @irq:        global system and rings IRQ
  * @ring:       pointer to descriptor rings
@@ -215,7 +214,6 @@ struct mtk_sha_rec {
 struct mtk_cryp {
         void __iomem *base;
         struct device *dev;
-        struct clk *clk_ethif;
         struct clk *clk_cryp;
         int irq[MTK_IRQ_NUM];
 
diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c
index 2226f12d1c7a..5f4f845adbb8 100644
--- a/drivers/crypto/mediatek/mtk-sha.c
+++ b/drivers/crypto/mediatek/mtk-sha.c
@@ -12,6 +12,7 @@
  * Some ideas are from atmel-sha.c and omap-sham.c drivers.
  */
 
+#include <crypto/hmac.h>
 #include <crypto/sha.h>
 #include "mtk-platform.h"
 
@@ -825,8 +826,8 @@ static int mtk_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
         memcpy(bctx->opad, bctx->ipad, bs);
 
         for (i = 0; i < bs; i++) {
-                bctx->ipad[i] ^= 0x36;
-                bctx->opad[i] ^= 0x5c;
+                bctx->ipad[i] ^= HMAC_IPAD_VALUE;
+                bctx->opad[i] ^= HMAC_OPAD_VALUE;
         }
 
         return 0;
diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c
index 451fa18c1c7b..bf25f415eea6 100644
--- a/drivers/crypto/mv_cesa.c
+++ b/drivers/crypto/mv_cesa.c
@@ -18,6 +18,7 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/clk.h>
+#include <crypto/hmac.h>
 #include <crypto/internal/hash.h>
 #include <crypto/sha.h>
 #include <linux/of.h>
@@ -822,8 +823,8 @@ static int mv_hash_setkey(struct crypto_ahash *tfm, const u8 * key,
                 memcpy(opad, ipad, bs);
 
                 for (i = 0; i < bs; i++) {
-                        ipad[i] ^= 0x36;
-                        opad[i] ^= 0x5c;
+                        ipad[i] ^= HMAC_IPAD_VALUE;
+                        opad[i] ^= HMAC_OPAD_VALUE;
                 }
 
                 rc = crypto_shash_init(shash) ? :
diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c
index 4ecb77aa60e1..269451375b63 100644
--- a/drivers/crypto/n2_core.c
+++ b/drivers/crypto/n2_core.c
@@ -2169,7 +2169,7 @@ static int n2_mau_remove(struct platform_device *dev)
         return 0;
 }
 
-static struct of_device_id n2_crypto_match[] = {
+static const struct of_device_id n2_crypto_match[] = {
         {
                 .name = "n2cp",
                 .compatible = "SUNW,n2-cwq",
@@ -2196,7 +2196,7 @@ static struct platform_driver n2_crypto_driver = {
         .remove         =       n2_crypto_remove,
 };
 
-static struct of_device_id n2_mau_match[] = {
+static const struct of_device_id n2_mau_match[] = {
         {
                 .name = "ncp",
                 .compatible = "SUNW,n2-mau",
diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c
new file mode 100644
index 000000000000..7d4f8a4be6d8
--- /dev/null
+++ b/drivers/crypto/omap-aes-gcm.c
@@ -0,0 +1,408 @@
+/*
+ * Cryptographic API.
+ *
+ * Support for OMAP AES GCM HW acceleration.
+ *
+ * Copyright (c) 2016 Texas Instruments Incorporated
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ *
+ */
+
+#include <linux/errno.h>
+#include <linux/scatterlist.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
+#include <linux/omap-dma.h>
+#include <linux/interrupt.h>
+#include <crypto/aes.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/skcipher.h>
+#include <crypto/internal/aead.h>
+
+#include "omap-crypto.h"
+#include "omap-aes.h"
+
+static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
+                                     struct aead_request *req);
+
+static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
+{
+        struct aead_request *req = dd->aead_req;
+
+        dd->flags &= ~FLAGS_BUSY;
+        dd->in_sg = NULL;
+        dd->out_sg = NULL;
+
+        req->base.complete(&req->base, ret);
+}
+
+static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
+{
+        u8 *tag;
+        int alen, clen, i, ret = 0, nsg;
+        struct omap_aes_reqctx *rctx;
+
+        alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
+        clen = ALIGN(dd->total, AES_BLOCK_SIZE);
+        rctx = aead_request_ctx(dd->aead_req);
+
+        nsg = !!(dd->assoc_len && dd->total);
+
+        dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
+                               DMA_FROM_DEVICE);
+        dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
+        dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
+        omap_aes_crypt_dma_stop(dd);
+
+        omap_crypto_cleanup(dd->out_sg, dd->orig_out,
+                            dd->aead_req->assoclen, dd->total,
+                            FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
+
+        if (dd->flags & FLAGS_ENCRYPT)
+                scatterwalk_map_and_copy(rctx->auth_tag,
+                                         dd->aead_req->dst,
+                                         dd->total + dd->aead_req->assoclen,
+                                         dd->authsize, 1);
+
+        omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
+                            FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
+
+        omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
+                            FLAGS_IN_DATA_ST_SHIFT, dd->flags);
+
+        if (!(dd->flags & FLAGS_ENCRYPT)) {
+                tag = (u8 *)rctx->auth_tag;
+                for (i = 0; i < dd->authsize; i++) {
+                        if (tag[i]) {
+                                dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n");
+                                ret = -EBADMSG;
+                        }
+                }
+        }
+
+        omap_aes_gcm_finish_req(dd, ret);
+        omap_aes_gcm_handle_queue(dd, NULL);
+}
+
+static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
+                                     struct aead_request *req)
+{
+        int alen, clen, cryptlen, assoclen, ret;
+        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        unsigned int authlen = crypto_aead_authsize(aead);
+        struct scatterlist *tmp, sg_arr[2];
+        int nsg;
+        u16 flags;
+
+        assoclen = req->assoclen;
+        cryptlen = req->cryptlen;
+
+        if (dd->flags & FLAGS_RFC4106_GCM)
+                assoclen -= 8;
+
+        if (!(dd->flags & FLAGS_ENCRYPT))
+                cryptlen -= authlen;
+
+        alen = ALIGN(assoclen, AES_BLOCK_SIZE);
+        clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
+
+        nsg = !!(assoclen && cryptlen);
+
+        omap_aes_clear_copy_flags(dd);
+
+        sg_init_table(dd->in_sgl, nsg + 1);
+        if (assoclen) {
+                tmp = req->src;
+                ret = omap_crypto_align_sg(&tmp, assoclen,
+                                           AES_BLOCK_SIZE, dd->in_sgl,
+                                           OMAP_CRYPTO_COPY_DATA |
+                                           OMAP_CRYPTO_ZERO_BUF |
+                                           OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
+                                           FLAGS_ASSOC_DATA_ST_SHIFT,
+                                           &dd->flags);
+        }
+
+        if (cryptlen) {
+                tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
+
+                ret = omap_crypto_align_sg(&tmp, cryptlen,
+                                           AES_BLOCK_SIZE, &dd->in_sgl[nsg],
+                                           OMAP_CRYPTO_COPY_DATA |
+                                           OMAP_CRYPTO_ZERO_BUF |
+                                           OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
+                                           FLAGS_IN_DATA_ST_SHIFT,
+                                           &dd->flags);
+        }
+
+        dd->in_sg = dd->in_sgl;
+        dd->total = cryptlen;
+        dd->assoc_len = assoclen;
+        dd->authsize = authlen;
+
+        dd->out_sg = req->dst;
+        dd->orig_out = req->dst;
+
+        dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, assoclen);
+
+        flags = 0;
+        if (req->src == req->dst || dd->out_sg == sg_arr)
+                flags |= OMAP_CRYPTO_FORCE_COPY;
+
+        ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
+                                   AES_BLOCK_SIZE, &dd->out_sgl,
+                                   flags,
+                                   FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
+        if (ret)
+                return ret;
+
+        dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
+        dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
+
+        return 0;
+}
+
+static void omap_aes_gcm_complete(struct crypto_async_request *req, int err)
+{
+        struct omap_aes_gcm_result *res = req->data;
+
+        if (err == -EINPROGRESS)
+                return;
+
+        res->err = err;
+        complete(&res->completion);
+}
+
+static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
+{
+        struct scatterlist iv_sg, tag_sg;
+        struct skcipher_request *sk_req;
+        struct omap_aes_gcm_result result;
+        struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+        int ret = 0;
+
+        sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL);
+        if (!sk_req) {
+                pr_err("skcipher: Failed to allocate request\n");
+                return -1;
+        }
+
+        init_completion(&result.completion);
+
+        sg_init_one(&iv_sg, iv, AES_BLOCK_SIZE);
+        sg_init_one(&tag_sg, tag, AES_BLOCK_SIZE);
+        skcipher_request_set_callback(sk_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+                                      omap_aes_gcm_complete, &result);
+        ret = crypto_skcipher_setkey(ctx->ctr, (u8 *)ctx->key, ctx->keylen);
+        skcipher_request_set_crypt(sk_req, &iv_sg, &tag_sg, AES_BLOCK_SIZE,
+                                   NULL);
+        ret = crypto_skcipher_encrypt(sk_req);
+        switch (ret) {
+        case 0:
+                break;
+        case -EINPROGRESS:
+        case -EBUSY:
+                ret = wait_for_completion_interruptible(&result.completion);
+                if (!ret) {
+                        ret = result.err;
+                        if (!ret) {
+                                reinit_completion(&result.completion);
+                                break;
+                        }
+                }
+                /* fall through */
+        default:
+                pr_err("Encryption of IV failed for GCM mode");
+                break;
+        }
+
+        skcipher_request_free(sk_req);
+        return ret;
+}
+
+void omap_aes_gcm_dma_out_callback(void *data)
+{
+        struct omap_aes_dev *dd = data;
+        struct omap_aes_reqctx *rctx;
+        int i, val;
+        u32 *auth_tag, tag[4];
+
+        if (!(dd->flags & FLAGS_ENCRYPT))
+                scatterwalk_map_and_copy(tag, dd->aead_req->src,
+                                         dd->total + dd->aead_req->assoclen,
+                                         dd->authsize, 0);
+
+        rctx = aead_request_ctx(dd->aead_req);
+        auth_tag = (u32 *)rctx->auth_tag;
+        for (i = 0; i < 4; i++) {
+                val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
+                auth_tag[i] = val ^ auth_tag[i];
+                if (!(dd->flags & FLAGS_ENCRYPT))
+                        auth_tag[i] = auth_tag[i] ^ tag[i];
+        }
+
+        omap_aes_gcm_done_task(dd);
+}
+
+static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
+                                     struct aead_request *req)
+{
+        struct omap_aes_ctx *ctx;
+        struct aead_request *backlog;
+        struct omap_aes_reqctx *rctx;
+        unsigned long flags;
+        int err, ret = 0;
+
+        spin_lock_irqsave(&dd->lock, flags);
+        if (req)
+                ret = aead_enqueue_request(&dd->aead_queue, req);
+        if (dd->flags & FLAGS_BUSY) {
+                spin_unlock_irqrestore(&dd->lock, flags);
+                return ret;
+        }
+
+        backlog = aead_get_backlog(&dd->aead_queue);
+        req = aead_dequeue_request(&dd->aead_queue);
+        if (req)
+                dd->flags |= FLAGS_BUSY;
+        spin_unlock_irqrestore(&dd->lock, flags);
+
+        if (!req)
+                return ret;
+
+        if (backlog)
+                backlog->base.complete(&backlog->base, -EINPROGRESS);
+
+        ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+        rctx = aead_request_ctx(req);
+
+        dd->ctx = ctx;
+        rctx->dd = dd;
+        dd->aead_req = req;
+
+        rctx->mode &= FLAGS_MODE_MASK;
+        dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
+
+        err = omap_aes_gcm_copy_buffers(dd, req);
+        if (err)
+                return err;
+
+        err = omap_aes_write_ctrl(dd);
+        if (!err)
+                err = omap_aes_crypt_dma_start(dd);
+
+        if (err) {
+                omap_aes_gcm_finish_req(dd, err);
+                omap_aes_gcm_handle_queue(dd, NULL);
+        }
+
+        return ret;
+}
+
+static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
+{
+        struct omap_aes_reqctx *rctx = aead_request_ctx(req);
+        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        unsigned int authlen = crypto_aead_authsize(aead);
+        struct omap_aes_dev *dd;
+        __be32 counter = cpu_to_be32(1);
+        int err, assoclen;
+
+        memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
+        memcpy(rctx->iv + 12, &counter, 4);
+
+        err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
+        if (err)
+                return err;
+
+        if (mode & FLAGS_RFC4106_GCM)
+                assoclen = req->assoclen - 8;
+        else
+                assoclen = req->assoclen;
+        if (assoclen + req->cryptlen == 0) {
+                scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
+                                         1);
+                return 0;
+        }
+
+        dd = omap_aes_find_dev(rctx);
+        if (!dd)
+                return -ENODEV;
+        rctx->mode = mode;
+
+        return omap_aes_gcm_handle_queue(dd, req);
+}
+
+int omap_aes_gcm_encrypt(struct aead_request *req)
+{
+        struct omap_aes_reqctx *rctx = aead_request_ctx(req);
+
+        memcpy(rctx->iv, req->iv, 12);
+        return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
+}
+
+int omap_aes_gcm_decrypt(struct aead_request *req)
+{
+        struct omap_aes_reqctx *rctx = aead_request_ctx(req);
+
+        memcpy(rctx->iv, req->iv, 12);
+        return omap_aes_gcm_crypt(req, FLAGS_GCM);
+}
+
+int omap_aes_4106gcm_encrypt(struct aead_request *req)
+{
+        struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+        struct omap_aes_reqctx *rctx = aead_request_ctx(req);
+
+        memcpy(rctx->iv, ctx->nonce, 4);
+        memcpy(rctx->iv + 4, req->iv, 8);
+        return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
+                                  FLAGS_RFC4106_GCM);
+}
+
+int omap_aes_4106gcm_decrypt(struct aead_request *req)
+{
+        struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+        struct omap_aes_reqctx *rctx = aead_request_ctx(req);
+
+        memcpy(rctx->iv, ctx->nonce, 4);
+        memcpy(rctx->iv + 4, req->iv, 8);
+        return omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
+}
+
+int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
+                        unsigned int keylen)
+{
+        struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
+
+        if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
+            keylen != AES_KEYSIZE_256)
+                return -EINVAL;
+
+        memcpy(ctx->key, key, keylen);
+        ctx->keylen = keylen;
+
+        return 0;
+}
+
+int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
+                            unsigned int keylen)
+{
+        struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
+
+        if (keylen < 4)
+                return -EINVAL;
+
+        keylen -= 4;
+        if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
+            keylen != AES_KEYSIZE_256)
+                return -EINVAL;
+
+        memcpy(ctx->key, key, keylen);
+        memcpy(ctx->nonce, key + keylen, 4);
+        ctx->keylen = keylen;
+
+        return 0;
+}
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index fe32dd95ae4f..5120a17731d0 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -37,155 +37,10 @@
 #include <crypto/aes.h>
 #include <crypto/engine.h>
 #include <crypto/internal/skcipher.h>
+#include <crypto/internal/aead.h>
 
-#define DST_MAXBURST                    4
-#define DMA_MIN                         (DST_MAXBURST * sizeof(u32))
-
-#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset)
-
-/* OMAP TRM gives bitfields as start:end, where start is the higher bit
-   number. For example 7:0 */
-#define FLD_MASK(start, end)    (((1 << ((start) - (end) + 1)) - 1) << (end))
-#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
-
-#define AES_REG_KEY(dd, x)              ((dd)->pdata->key_ofs - \
-                                                ((x ^ 0x01) * 0x04))
-#define AES_REG_IV(dd, x)               ((dd)->pdata->iv_ofs + ((x) * 0x04))
-
-#define AES_REG_CTRL(dd)                ((dd)->pdata->ctrl_ofs)
-#define AES_REG_CTRL_CTR_WIDTH_MASK     GENMASK(8, 7)
-#define AES_REG_CTRL_CTR_WIDTH_32       0
-#define AES_REG_CTRL_CTR_WIDTH_64       BIT(7)
-#define AES_REG_CTRL_CTR_WIDTH_96       BIT(8)
-#define AES_REG_CTRL_CTR_WIDTH_128      GENMASK(8, 7)
-#define AES_REG_CTRL_CTR                BIT(6)
-#define AES_REG_CTRL_CBC                BIT(5)
-#define AES_REG_CTRL_KEY_SIZE           GENMASK(4, 3)
-#define AES_REG_CTRL_DIRECTION          BIT(2)
-#define AES_REG_CTRL_INPUT_READY        BIT(1)
-#define AES_REG_CTRL_OUTPUT_READY       BIT(0)
-#define AES_REG_CTRL_MASK               GENMASK(24, 2)
-
-#define AES_REG_DATA_N(dd, x)           ((dd)->pdata->data_ofs + ((x) * 0x04))
-
-#define AES_REG_REV(dd)                 ((dd)->pdata->rev_ofs)
-
-#define AES_REG_MASK(dd)                ((dd)->pdata->mask_ofs)
-#define AES_REG_MASK_SIDLE              BIT(6)
-#define AES_REG_MASK_START              BIT(5)
-#define AES_REG_MASK_DMA_OUT_EN         BIT(3)
-#define AES_REG_MASK_DMA_IN_EN          BIT(2)
-#define AES_REG_MASK_SOFTRESET          BIT(1)
-#define AES_REG_AUTOIDLE                BIT(0)
-
-#define AES_REG_LENGTH_N(x)             (0x54 + ((x) * 0x04))
-
-#define AES_REG_IRQ_STATUS(dd)         ((dd)->pdata->irq_status_ofs)
-#define AES_REG_IRQ_ENABLE(dd)         ((dd)->pdata->irq_enable_ofs)
-#define AES_REG_IRQ_DATA_IN            BIT(1)
-#define AES_REG_IRQ_DATA_OUT           BIT(2)
-#define DEFAULT_TIMEOUT         (5*HZ)
-
-#define DEFAULT_AUTOSUSPEND_DELAY       1000
-
-#define FLAGS_MODE_MASK         0x000f
-#define FLAGS_ENCRYPT           BIT(0)
-#define FLAGS_CBC               BIT(1)
-#define FLAGS_GIV               BIT(2)
-#define FLAGS_CTR               BIT(3)
-
-#define FLAGS_INIT              BIT(4)
-#define FLAGS_FAST              BIT(5)
-#define FLAGS_BUSY              BIT(6)
-
-#define AES_BLOCK_WORDS         (AES_BLOCK_SIZE >> 2)
-
-struct omap_aes_ctx {
-        struct omap_aes_dev *dd;
-
-        int             keylen;
-        u32             key[AES_KEYSIZE_256 / sizeof(u32)];
-        unsigned long   flags;
-        struct crypto_skcipher  *fallback;
-};
-
-struct omap_aes_reqctx {
-        unsigned long mode;
-};
-
-#define OMAP_AES_QUEUE_LENGTH   1
-#define OMAP_AES_CACHE_SIZE     0
-
-struct omap_aes_algs_info {
-        struct crypto_alg       *algs_list;
-        unsigned int            size;
-        unsigned int            registered;
-};
-
-struct omap_aes_pdata {
-        struct omap_aes_algs_info       *algs_info;
-        unsigned int    algs_info_size;
-
-        void            (*trigger)(struct omap_aes_dev *dd, int length);
-
-        u32             key_ofs;
-        u32             iv_ofs;
-        u32             ctrl_ofs;
-        u32             data_ofs;
-        u32             rev_ofs;
-        u32             mask_ofs;
-        u32             irq_enable_ofs;
-        u32             irq_status_ofs;
-
-        u32             dma_enable_in;
-        u32             dma_enable_out;
-        u32             dma_start;
-
-        u32             major_mask;
-        u32             major_shift;
-        u32             minor_mask;
-        u32             minor_shift;
-};
-
-struct omap_aes_dev {
-        struct list_head        list;
-        unsigned long           phys_base;
-        void __iomem            *io_base;
-        struct omap_aes_ctx     *ctx;
-        struct device           *dev;
-        unsigned long           flags;
-        int                     err;
-
-        struct tasklet_struct   done_task;
-
-        struct ablkcipher_request       *req;
-        struct crypto_engine            *engine;
-
-        /*
-         * total is used by PIO mode for book keeping so introduce
-         * variable total_save as need it to calc page_order
-         */
-        size_t                          total;
-        size_t                          total_save;
-
-        struct scatterlist              *in_sg;
-        struct scatterlist              *out_sg;
-
-        /* Buffers for copying for unaligned cases */
-        struct scatterlist              in_sgl;
-        struct scatterlist              out_sgl;
-        struct scatterlist              *orig_out;
-        int                             sgs_copied;
-
-        struct scatter_walk             in_walk;
-        struct scatter_walk             out_walk;
-        struct dma_chan         *dma_lch_in;
-        struct dma_chan         *dma_lch_out;
-        int                     in_sg_len;
-        int                     out_sg_len;
-        int                     pio_only;
-        const struct omap_aes_pdata     *pdata;
-};
+#include "omap-crypto.h"
+#include "omap-aes.h"
 
 /* keep registered devices data here */
 static LIST_HEAD(dev_list);
@@ -201,7 +56,7 @@ static DEFINE_SPINLOCK(list_lock);
         _read_ret;                                              \
 })
 #else
-static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
+inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
 {
         return __raw_readl(dd->io_base + offset);
 }
@@ -215,7 +70,7 @@ static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
                 __raw_writel(value, dd->io_base + offset);              \
         } while (0)
 #else
-static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
+inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
                                   u32 value)
 {
         __raw_writel(value, dd->io_base + offset);
@@ -258,8 +113,16 @@ static int omap_aes_hw_init(struct omap_aes_dev *dd)
         return 0;
 }
 
-static int omap_aes_write_ctrl(struct omap_aes_dev *dd)
+void omap_aes_clear_copy_flags(struct omap_aes_dev *dd)
+{
+        dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_IN_DATA_ST_SHIFT);
+        dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_OUT_DATA_ST_SHIFT);
+        dd->flags &= ~(OMAP_CRYPTO_COPY_MASK << FLAGS_ASSOC_DATA_ST_SHIFT);
+}
+
+int omap_aes_write_ctrl(struct omap_aes_dev *dd)
 {
+        struct omap_aes_reqctx *rctx;
         unsigned int key32;
         int i, err;
         u32 val;
@@ -270,7 +133,11 @@ static int omap_aes_write_ctrl(struct omap_aes_dev *dd)
 
         key32 = dd->ctx->keylen / sizeof(u32);
 
-        /* it seems a key should always be set even if it has not changed */
+        /* RESET the key as previous HASH keys should not get affected*/
+        if (dd->flags & FLAGS_GCM)
+                for (i = 0; i < 0x40; i = i + 4)
+                        omap_aes_write(dd, i, 0x0);
+
         for (i = 0; i < key32; i++) {
                 omap_aes_write(dd, AES_REG_KEY(dd, i),
                         __le32_to_cpu(dd->ctx->key[i]));
@@ -279,12 +146,21 @@ static int omap_aes_write_ctrl(struct omap_aes_dev *dd)
         if ((dd->flags & (FLAGS_CBC | FLAGS_CTR)) && dd->req->info)
                 omap_aes_write_n(dd, AES_REG_IV(dd, 0), dd->req->info, 4);
 
+        if ((dd->flags & (FLAGS_GCM)) && dd->aead_req->iv) {
+                rctx = aead_request_ctx(dd->aead_req);
+                omap_aes_write_n(dd, AES_REG_IV(dd, 0), (u32 *)rctx->iv, 4);
+        }
+
         val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
         if (dd->flags & FLAGS_CBC)
                 val |= AES_REG_CTRL_CBC;
-        if (dd->flags & FLAGS_CTR)
+
+        if (dd->flags & (FLAGS_CTR | FLAGS_GCM))
                 val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_128;
 
+        if (dd->flags & FLAGS_GCM)
+                val |= AES_REG_CTRL_GCM;
+
         if (dd->flags & FLAGS_ENCRYPT)
                 val |= AES_REG_CTRL_DIRECTION;
 
@@ -315,6 +191,8 @@ static void omap_aes_dma_trigger_omap4(struct omap_aes_dev *dd, int length)
 {
         omap_aes_write(dd, AES_REG_LENGTH_N(0), length);
         omap_aes_write(dd, AES_REG_LENGTH_N(1), 0);
+        if (dd->flags & FLAGS_GCM)
+                omap_aes_write(dd, AES_REG_A_LEN, dd->assoc_len);
 
         omap_aes_dma_trigger_omap2(dd, length);
 }
@@ -329,14 +207,14 @@ static void omap_aes_dma_stop(struct omap_aes_dev *dd)
         omap_aes_write_mask(dd, AES_REG_MASK(dd), 0, mask);
 }
 
-static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx)
+struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx)
 {
         struct omap_aes_dev *dd;
 
         spin_lock_bh(&list_lock);
         dd = list_first_entry(&dev_list, struct omap_aes_dev, list);
         list_move_tail(&dd->list, &dev_list);
-        ctx->dd = dd;
+        rctx->dd = dd;
         spin_unlock_bh(&list_lock);
 
         return dd;
@@ -387,26 +265,11 @@ static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
         dma_release_channel(dd->dma_lch_in);
 }
 
-static void sg_copy_buf(void *buf, struct scatterlist *sg,
-                              unsigned int start, unsigned int nbytes, int out)
+static int omap_aes_crypt_dma(struct omap_aes_dev *dd,
+                              struct scatterlist *in_sg,
+                              struct scatterlist *out_sg,
+                              int in_sg_len, int out_sg_len)
 {
-        struct scatter_walk walk;
-
-        if (!nbytes)
-                return;
-
-        scatterwalk_start(&walk, sg);
-        scatterwalk_advance(&walk, start);
-        scatterwalk_copychunks(buf, &walk, nbytes, out);
-        scatterwalk_done(&walk, out, 0);
-}
-
-static int omap_aes_crypt_dma(struct crypto_tfm *tfm,
-                struct scatterlist *in_sg, struct scatterlist *out_sg,
-                int in_sg_len, int out_sg_len)
-{
-        struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-        struct omap_aes_dev *dd = ctx->dd;
         struct dma_async_tx_descriptor *tx_in, *tx_out;
         struct dma_slave_config cfg;
         int ret;
@@ -467,7 +330,10 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm,
                 return -EINVAL;
         }
 
-        tx_out->callback = omap_aes_dma_out_callback;
+        if (dd->flags & FLAGS_GCM)
+                tx_out->callback = omap_aes_gcm_dma_out_callback;
+        else
+                tx_out->callback = omap_aes_dma_out_callback;
         tx_out->callback_param = dd;
 
         dmaengine_submit(tx_in);
@@ -482,10 +348,8 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm,
         return 0;
 }
 
-static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
+int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
 {
-        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
-                                        crypto_ablkcipher_reqtfm(dd->req));
         int err;
 
         pr_debug("total: %d\n", dd->total);
@@ -506,7 +370,7 @@ static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
                 }
         }
 
-        err = omap_aes_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len,
+        err = omap_aes_crypt_dma(dd, dd->in_sg, dd->out_sg, dd->in_sg_len,
                                  dd->out_sg_len);
         if (err && !dd->pio_only) {
                 dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
@@ -529,7 +393,7 @@ static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
         pm_runtime_put_autosuspend(dd->dev);
 }
 
-static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
+int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
 {
         pr_debug("total: %d\n", dd->total);
 
@@ -539,62 +403,6 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
         return 0;
 }
 
-static int omap_aes_check_aligned(struct scatterlist *sg, int total)
-{
-        int len = 0;
-
-        if (!IS_ALIGNED(total, AES_BLOCK_SIZE))
-                return -EINVAL;
-
-        while (sg) {
-                if (!IS_ALIGNED(sg->offset, 4))
-                        return -1;
-                if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
-                        return -1;
-
-                len += sg->length;
-                sg = sg_next(sg);
-        }
-
-        if (len != total)
-                return -1;
-
-        return 0;
-}
-
-static int omap_aes_copy_sgs(struct omap_aes_dev *dd)
-{
-        void *buf_in, *buf_out;
-        int pages, total;
-
-        total = ALIGN(dd->total, AES_BLOCK_SIZE);
-        pages = get_order(total);
-
-        buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
-        buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
-
-        if (!buf_in || !buf_out) {
-                pr_err("Couldn't allocated pages for unaligned cases.\n");
-                return -1;
-        }
-
-        dd->orig_out = dd->out_sg;
-
-        sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0);
-
-        sg_init_table(&dd->in_sgl, 1);
-        sg_set_buf(&dd->in_sgl, buf_in, total);
-        dd->in_sg = &dd->in_sgl;
-        dd->in_sg_len = 1;
-
-        sg_init_table(&dd->out_sgl, 1);
-        sg_set_buf(&dd->out_sgl, buf_out, total);
-        dd->out_sg = &dd->out_sgl;
-        dd->out_sg_len = 1;
-
-        return 0;
-}
-
 static int omap_aes_handle_queue(struct omap_aes_dev *dd,
                                  struct ablkcipher_request *req)
 {
@@ -609,8 +417,10 @@ static int omap_aes_prepare_req(struct crypto_engine *engine,
 {
         struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
                         crypto_ablkcipher_reqtfm(req));
-        struct omap_aes_dev *dd = ctx->dd;
-        struct omap_aes_reqctx *rctx;
+        struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
+        struct omap_aes_dev *dd = rctx->dd;
+        int ret;
+        u16 flags;
 
         if (!dd)
                 return -ENODEV;
@@ -621,6 +431,23 @@ static int omap_aes_prepare_req(struct crypto_engine *engine,
         dd->total_save = req->nbytes;
         dd->in_sg = req->src;
         dd->out_sg = req->dst;
+        dd->orig_out = req->dst;
+
+        flags = OMAP_CRYPTO_COPY_DATA;
+        if (req->src == req->dst)
+                flags |= OMAP_CRYPTO_FORCE_COPY;
+
+        ret = omap_crypto_align_sg(&dd->in_sg, dd->total, AES_BLOCK_SIZE,
+                                   dd->in_sgl, flags,
+                                   FLAGS_IN_DATA_ST_SHIFT, &dd->flags);
+        if (ret)
+                return ret;
+
+        ret = omap_crypto_align_sg(&dd->out_sg, dd->total, AES_BLOCK_SIZE,
+                                   &dd->out_sgl, 0,
+                                   FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
+        if (ret)
+                return ret;
 
         dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total);
         if (dd->in_sg_len < 0)
@@ -630,22 +457,11 @@ static int omap_aes_prepare_req(struct crypto_engine *engine,
         if (dd->out_sg_len < 0)
                 return dd->out_sg_len;
 
-        if (omap_aes_check_aligned(dd->in_sg, dd->total) ||
-            omap_aes_check_aligned(dd->out_sg, dd->total)) {
-                if (omap_aes_copy_sgs(dd))
-                        pr_err("Failed to copy SGs for unaligned cases\n");
-                dd->sgs_copied = 1;
-        } else {
-                dd->sgs_copied = 0;
-        }
-
-        rctx = ablkcipher_request_ctx(req);
-        ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
         rctx->mode &= FLAGS_MODE_MASK;
         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 
         dd->ctx = ctx;
-        ctx->dd = dd;
+        rctx->dd = dd;
 
         return omap_aes_write_ctrl(dd);
 }
@@ -653,9 +469,8 @@ static int omap_aes_prepare_req(struct crypto_engine *engine,
 static int omap_aes_crypt_req(struct crypto_engine *engine,
                               struct ablkcipher_request *req)
 {
-        struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
-                        crypto_ablkcipher_reqtfm(req));
-        struct omap_aes_dev *dd = ctx->dd;
+        struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
+        struct omap_aes_dev *dd = rctx->dd;
 
         if (!dd)
                 return -ENODEV;
@@ -666,8 +481,6 @@ static int omap_aes_crypt_req(struct crypto_engine *engine,
 static void omap_aes_done_task(unsigned long data)
 {
         struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
-        void *buf_in, *buf_out;
-        int pages, len;
 
         pr_debug("enter done_task\n");
 
@@ -680,17 +493,11 @@ static void omap_aes_done_task(unsigned long data)
                 omap_aes_crypt_dma_stop(dd);
         }
 
-        if (dd->sgs_copied) {
-                buf_in = sg_virt(&dd->in_sgl);
-                buf_out = sg_virt(&dd->out_sgl);
+        omap_crypto_cleanup(dd->in_sgl, NULL, 0, dd->total_save,
+                            FLAGS_IN_DATA_ST_SHIFT, dd->flags);
 
-                sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1);
-
-                len = ALIGN(dd->total_save, AES_BLOCK_SIZE);
-                pages = get_order(len);
-                free_pages((unsigned long)buf_in, pages);
-                free_pages((unsigned long)buf_out, pages);
-        }
+        omap_crypto_cleanup(&dd->out_sgl, dd->orig_out, 0, dd->total_save,
+                            FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
 
         omap_aes_finish_req(dd, 0);
 
@@ -726,7 +533,7 @@ static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
                 skcipher_request_zero(subreq);
                 return ret;
         }
-        dd = omap_aes_find_dev(ctx);
+        dd = omap_aes_find_dev(rctx);
         if (!dd)
                 return -ENODEV;
 
@@ -811,6 +618,36 @@ static int omap_aes_cra_init(struct crypto_tfm *tfm)
         return 0;
 }
 
+static int omap_aes_gcm_cra_init(struct crypto_aead *tfm)
+{
+        struct omap_aes_dev *dd = NULL;
+        struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
+        int err;
+
+        /* Find AES device, currently picks the first device */
+        spin_lock_bh(&list_lock);
+        list_for_each_entry(dd, &dev_list, list) {
+                break;
+        }
+        spin_unlock_bh(&list_lock);
+
+        err = pm_runtime_get_sync(dd->dev);
+        if (err < 0) {
+                dev_err(dd->dev, "%s: failed to get_sync(%d)\n",
+                        __func__, err);
+                return err;
+        }
+
+        tfm->reqsize = sizeof(struct omap_aes_reqctx);
+        ctx->ctr = crypto_alloc_skcipher("ecb(aes)", 0, 0);
+        if (IS_ERR(ctx->ctr)) {
+                pr_warn("could not load aes driver for encrypting IV\n");
+                return PTR_ERR(ctx->ctr);
+        }
+
+        return 0;
+}
+
 static void omap_aes_cra_exit(struct crypto_tfm *tfm)
 {
         struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
@@ -821,6 +658,16 @@ static void omap_aes_cra_exit(struct crypto_tfm *tfm)
         ctx->fallback = NULL;
 }
 
+static void omap_aes_gcm_cra_exit(struct crypto_aead *tfm)
+{
+        struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
+
+        omap_aes_cra_exit(crypto_aead_tfm(tfm));
+
+        if (ctx->ctr)
+                crypto_free_skcipher(ctx->ctr);
+}
+
 /* ********************** ALGS ************************************ */
 
 static struct crypto_alg algs_ecb_cbc[] = {
@@ -905,6 +752,54 @@ static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc[] = {
         },
 };
 
+static struct aead_alg algs_aead_gcm[] = {
+{
+        .base = {
+                .cra_name               = "gcm(aes)",
+                .cra_driver_name        = "gcm-aes-omap",
+                .cra_priority           = 300,
+                .cra_flags              = CRYPTO_ALG_ASYNC |
+                                          CRYPTO_ALG_KERN_DRIVER_ONLY,
+                .cra_blocksize          = 1,
+                .cra_ctxsize            = sizeof(struct omap_aes_ctx),
+                .cra_alignmask          = 0xf,
+                .cra_module             = THIS_MODULE,
+        },
+        .init           = omap_aes_gcm_cra_init,
+        .exit           = omap_aes_gcm_cra_exit,
+        .ivsize         = 12,
+        .maxauthsize    = AES_BLOCK_SIZE,
+        .setkey         = omap_aes_gcm_setkey,
+        .encrypt        = omap_aes_gcm_encrypt,
+        .decrypt        = omap_aes_gcm_decrypt,
+},
+{
+        .base = {
+                .cra_name               = "rfc4106(gcm(aes))",
+                .cra_driver_name        = "rfc4106-gcm-aes-omap",
+                .cra_priority           = 300,
+                .cra_flags              = CRYPTO_ALG_ASYNC |
+                                          CRYPTO_ALG_KERN_DRIVER_ONLY,
+                .cra_blocksize          = 1,
+                .cra_ctxsize            = sizeof(struct omap_aes_ctx),
+                .cra_alignmask          = 0xf,
+                .cra_module             = THIS_MODULE,
+        },
+        .init           = omap_aes_gcm_cra_init,
+        .exit           = omap_aes_gcm_cra_exit,
+        .maxauthsize    = AES_BLOCK_SIZE,
+        .ivsize         = 8,
+        .setkey         = omap_aes_4106gcm_setkey,
+        .encrypt        = omap_aes_4106gcm_encrypt,
+        .decrypt        = omap_aes_4106gcm_decrypt,
+},
+};
+
+static struct omap_aes_aead_algs omap_aes_aead_info = {
+        .algs_list      =       algs_aead_gcm,
+        .size           =       ARRAY_SIZE(algs_aead_gcm),
+};
+
 static const struct omap_aes_pdata omap_aes_pdata_omap2 = {
         .algs_info      = omap_aes_algs_info_ecb_cbc,
         .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc),
@@ -958,6 +853,7 @@ static const struct omap_aes_pdata omap_aes_pdata_omap3 = {
 static const struct omap_aes_pdata omap_aes_pdata_omap4 = {
         .algs_info      = omap_aes_algs_info_ecb_cbc_ctr,
         .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr),
+        .aead_algs_info = &omap_aes_aead_info,
         .trigger        = omap_aes_dma_trigger_omap4,
         .key_ofs        = 0x3c,
         .iv_ofs         = 0x40,
@@ -1140,6 +1036,7 @@ static int omap_aes_probe(struct platform_device *pdev)
         struct device *dev = &pdev->dev;
         struct omap_aes_dev *dd;
         struct crypto_alg *algp;
+        struct aead_alg *aalg;
         struct resource res;
         int err = -ENOMEM, i, j, irq = -1;
         u32 reg;
@@ -1152,6 +1049,8 @@ static int omap_aes_probe(struct platform_device *pdev)
         dd->dev = dev;
         platform_set_drvdata(pdev, dd);
 
+        aead_init_queue(&dd->aead_queue, OMAP_AES_QUEUE_LENGTH);
+
         err = (dev->of_node) ? omap_aes_get_res_of(dd, dev, &res) :
                                omap_aes_get_res_pdev(dd, pdev, &res);
         if (err)
@@ -1207,6 +1106,7 @@ static int omap_aes_probe(struct platform_device *pdev)
                 }
         }
 
+        spin_lock_init(&dd->lock);
 
         INIT_LIST_HEAD(&dd->list);
         spin_lock(&list_lock);
@@ -1243,7 +1143,29 @@ static int omap_aes_probe(struct platform_device *pdev)
                 }
         }
 
+        if (dd->pdata->aead_algs_info &&
+            !dd->pdata->aead_algs_info->registered) {
+                for (i = 0; i < dd->pdata->aead_algs_info->size; i++) {
+                        aalg = &dd->pdata->aead_algs_info->algs_list[i];
+                        algp = &aalg->base;
+
+                        pr_debug("reg alg: %s\n", algp->cra_name);
+                        INIT_LIST_HEAD(&algp->cra_list);
+
+                        err = crypto_register_aead(aalg);
+                        if (err)
+                                goto err_aead_algs;
+
+                        dd->pdata->aead_algs_info->registered++;
+                }
+        }
+
         return 0;
+err_aead_algs:
+        for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) {
+                aalg = &dd->pdata->aead_algs_info->algs_list[i];
+                crypto_unregister_aead(aalg);
+        }
 err_algs:
         for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
                 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
@@ -1268,6 +1190,7 @@ err_data:
 static int omap_aes_remove(struct platform_device *pdev)
 {
         struct omap_aes_dev *dd = platform_get_drvdata(pdev);
+        struct aead_alg *aalg;
         int i, j;
 
         if (!dd)
@@ -1282,7 +1205,13 @@ static int omap_aes_remove(struct platform_device *pdev)
                         crypto_unregister_alg(
                                         &dd->pdata->algs_info[i].algs_list[j]);
 
+        for (i = dd->pdata->aead_algs_info->size - 1; i >= 0; i--) {
+                aalg = &dd->pdata->aead_algs_info->algs_list[i];
+                crypto_unregister_aead(aalg);
+        }
+
         crypto_engine_exit(dd->engine);
+
         tasklet_kill(&dd->done_task);
         omap_aes_dma_cleanup(dd);
         pm_runtime_disable(dd->dev);
diff --git a/drivers/crypto/omap-aes.h b/drivers/crypto/omap-aes.h
new file mode 100644
index 000000000000..8906342e2b9a
--- /dev/null
+++ b/drivers/crypto/omap-aes.h
@@ -0,0 +1,214 @@
+/*
+ * Cryptographic API.
+ *
+ * Support for OMAP AES HW ACCELERATOR defines
+ *
+ * Copyright (c) 2015 Texas Instruments Incorporated
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ *
+ */
+#ifndef __OMAP_AES_H__
+#define __OMAP_AES_H__
+
+#define DST_MAXBURST                    4
+#define DMA_MIN                         (DST_MAXBURST * sizeof(u32))
+
+#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset)
+
+/*
+ * OMAP TRM gives bitfields as start:end, where start is the higher bit
+ * number. For example 7:0
+ */
+#define FLD_MASK(start, end)    (((1 << ((start) - (end) + 1)) - 1) << (end))
+#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
+
+#define AES_REG_KEY(dd, x)              ((dd)->pdata->key_ofs - \
+                                                (((x) ^ 0x01) * 0x04))
+#define AES_REG_IV(dd, x)               ((dd)->pdata->iv_ofs + ((x) * 0x04))
+
+#define AES_REG_CTRL(dd)                ((dd)->pdata->ctrl_ofs)
+#define AES_REG_CTRL_CONTEXT_READY      BIT(31)
+#define AES_REG_CTRL_CTR_WIDTH_MASK     GENMASK(8, 7)
+#define AES_REG_CTRL_CTR_WIDTH_32       0
+#define AES_REG_CTRL_CTR_WIDTH_64       BIT(7)
+#define AES_REG_CTRL_CTR_WIDTH_96       BIT(8)
+#define AES_REG_CTRL_CTR_WIDTH_128      GENMASK(8, 7)
+#define AES_REG_CTRL_GCM                GENMASK(17, 16)
+#define AES_REG_CTRL_CTR                BIT(6)
+#define AES_REG_CTRL_CBC                BIT(5)
+#define AES_REG_CTRL_KEY_SIZE           GENMASK(4, 3)
+#define AES_REG_CTRL_DIRECTION          BIT(2)
+#define AES_REG_CTRL_INPUT_READY        BIT(1)
+#define AES_REG_CTRL_OUTPUT_READY       BIT(0)
+#define AES_REG_CTRL_MASK               GENMASK(24, 2)
+
+#define AES_REG_C_LEN_0                 0x54
+#define AES_REG_C_LEN_1                 0x58
+#define AES_REG_A_LEN                   0x5C
+
+#define AES_REG_DATA_N(dd, x)           ((dd)->pdata->data_ofs + ((x) * 0x04))
+#define AES_REG_TAG_N(dd, x)            (0x70 + ((x) * 0x04))
+
+#define AES_REG_REV(dd)                 ((dd)->pdata->rev_ofs)
+
+#define AES_REG_MASK(dd)                ((dd)->pdata->mask_ofs)
+#define AES_REG_MASK_SIDLE              BIT(6)
+#define AES_REG_MASK_START              BIT(5)
+#define AES_REG_MASK_DMA_OUT_EN         BIT(3)
+#define AES_REG_MASK_DMA_IN_EN          BIT(2)
+#define AES_REG_MASK_SOFTRESET          BIT(1)
+#define AES_REG_AUTOIDLE                BIT(0)
+
+#define AES_REG_LENGTH_N(x)             (0x54 + ((x) * 0x04))
+
+#define AES_REG_IRQ_STATUS(dd)         ((dd)->pdata->irq_status_ofs)
+#define AES_REG_IRQ_ENABLE(dd)         ((dd)->pdata->irq_enable_ofs)
+#define AES_REG_IRQ_DATA_IN            BIT(1)
+#define AES_REG_IRQ_DATA_OUT           BIT(2)
+#define DEFAULT_TIMEOUT         (5 * HZ)
+
+#define DEFAULT_AUTOSUSPEND_DELAY       1000
+
+#define FLAGS_MODE_MASK         0x001f
+#define FLAGS_ENCRYPT           BIT(0)
+#define FLAGS_CBC               BIT(1)
+#define FLAGS_CTR               BIT(2)
+#define FLAGS_GCM               BIT(3)
+#define FLAGS_RFC4106_GCM       BIT(4)
+
+#define FLAGS_INIT              BIT(5)
+#define FLAGS_FAST              BIT(6)
+#define FLAGS_BUSY              BIT(7)
+
+#define FLAGS_IN_DATA_ST_SHIFT  8
+#define FLAGS_OUT_DATA_ST_SHIFT 10
+#define FLAGS_ASSOC_DATA_ST_SHIFT       12
+
+#define AES_BLOCK_WORDS         (AES_BLOCK_SIZE >> 2)
+
+struct omap_aes_gcm_result {
+        struct completion completion;
+        int err;
+};
+
+struct omap_aes_ctx {
+        int             keylen;
+        u32             key[AES_KEYSIZE_256 / sizeof(u32)];
+        u8              nonce[4];
+        struct crypto_skcipher  *fallback;
+        struct crypto_skcipher  *ctr;
+};
+
+struct omap_aes_reqctx {
+        struct omap_aes_dev *dd;
+        unsigned long mode;
+        u8 iv[AES_BLOCK_SIZE];
+        u32 auth_tag[AES_BLOCK_SIZE / sizeof(u32)];
+};
+
+#define OMAP_AES_QUEUE_LENGTH   1
+#define OMAP_AES_CACHE_SIZE     0
+
+struct omap_aes_algs_info {
+        struct crypto_alg       *algs_list;
+        unsigned int            size;
+        unsigned int            registered;
+};
+
+struct omap_aes_aead_algs {
+        struct aead_alg *algs_list;
+        unsigned int    size;
+        unsigned int    registered;
+};
+
+struct omap_aes_pdata {
+        struct omap_aes_algs_info       *algs_info;
+        unsigned int    algs_info_size;
+        struct omap_aes_aead_algs       *aead_algs_info;
+
+        void            (*trigger)(struct omap_aes_dev *dd, int length);
+
+        u32             key_ofs;
+        u32             iv_ofs;
+        u32             ctrl_ofs;
+        u32             data_ofs;
+        u32             rev_ofs;
+        u32             mask_ofs;
+        u32             irq_enable_ofs;
+        u32             irq_status_ofs;
+
+        u32             dma_enable_in;
+        u32             dma_enable_out;
+        u32             dma_start;
+
+        u32             major_mask;
+        u32             major_shift;
+        u32             minor_mask;
+        u32             minor_shift;
+};
+
+struct omap_aes_dev {
+        struct list_head        list;
+        unsigned long           phys_base;
+        void __iomem            *io_base;
+        struct omap_aes_ctx     *ctx;
+        struct device           *dev;
+        unsigned long           flags;
+        int                     err;
+
+        struct tasklet_struct   done_task;
+        struct aead_queue       aead_queue;
+        spinlock_t              lock;
+
+        struct ablkcipher_request       *req;
+        struct aead_request             *aead_req;
+        struct crypto_engine            *engine;
+
+        /*
+         * total is used by PIO mode for book keeping so introduce
+         * variable total_save as need it to calc page_order
+         */
+        size_t                          total;
+        size_t                          total_save;
+        size_t                          assoc_len;
+        size_t                          authsize;
+
+        struct scatterlist              *in_sg;
+        struct scatterlist              *out_sg;
+
+        /* Buffers for copying for unaligned cases */
+        struct scatterlist              in_sgl[2];
+        struct scatterlist              out_sgl;
+        struct scatterlist              *orig_out;
+
+        struct scatter_walk             in_walk;
+        struct scatter_walk             out_walk;
+        struct dma_chan         *dma_lch_in;
+        struct dma_chan         *dma_lch_out;
+        int                     in_sg_len;
+        int                     out_sg_len;
+        int                     pio_only;
+        const struct omap_aes_pdata     *pdata;
+};
+
+u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset);
+void omap_aes_write(struct omap_aes_dev *dd, u32 offset, u32 value);
+struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_reqctx *rctx);
+int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
+                        unsigned int keylen);
+int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
+                            unsigned int keylen);
+int omap_aes_gcm_encrypt(struct aead_request *req);
+int omap_aes_gcm_decrypt(struct aead_request *req);
+int omap_aes_4106gcm_encrypt(struct aead_request *req);
+int omap_aes_4106gcm_decrypt(struct aead_request *req);
+int omap_aes_write_ctrl(struct omap_aes_dev *dd);
+int omap_aes_crypt_dma_start(struct omap_aes_dev *dd);
+int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd);
+void omap_aes_gcm_dma_out_callback(void *data);
+void omap_aes_clear_copy_flags(struct omap_aes_dev *dd);
+
+#endif
diff --git a/drivers/crypto/omap-crypto.c b/drivers/crypto/omap-crypto.c
new file mode 100644
index 000000000000..23e37779317e
--- /dev/null
+++ b/drivers/crypto/omap-crypto.c
@@ -0,0 +1,184 @@
+/*
+ * OMAP Crypto driver common support routines.
+ *
+ * Copyright (c) 2017 Texas Instruments Incorporated
+ *   Tero Kristo <t-kristo@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/scatterlist.h>
+#include <crypto/scatterwalk.h>
+
+#include "omap-crypto.h"
+
+static int omap_crypto_copy_sg_lists(int total, int bs,
+                                     struct scatterlist **sg,
+                                     struct scatterlist *new_sg, u16 flags)
+{
+        int n = sg_nents(*sg);
+        struct scatterlist *tmp;
+
+        if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY)) {
+                new_sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL);
+                if (!new_sg)
+                        return -ENOMEM;
+
+                sg_init_table(new_sg, n);
+        }
+
+        tmp = new_sg;
+
+        while (*sg && total) {
+                int len = (*sg)->length;
+
+                if (total < len)
+                        len = total;
+
+                if (len > 0) {
+                        total -= len;
+                        sg_set_page(tmp, sg_page(*sg), len, (*sg)->offset);
+                        if (total <= 0)
+                                sg_mark_end(tmp);
+                        tmp = sg_next(tmp);
+                }
+
+                *sg = sg_next(*sg);
+        }
+
+        *sg = new_sg;
+
+        return 0;
+}
+
+static int omap_crypto_copy_sgs(int total, int bs, struct scatterlist **sg,
+                                struct scatterlist *new_sg, u16 flags)
+{
+        void *buf;
+        int pages;
+        int new_len;
+
+        new_len = ALIGN(total, bs);
+        pages = get_order(new_len);
+
+        buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
+        if (!buf) {
+                pr_err("%s: Couldn't allocate pages for unaligned cases.\n",
+                       __func__);
+                return -ENOMEM;
+        }
+
+        if (flags & OMAP_CRYPTO_COPY_DATA) {
+                scatterwalk_map_and_copy(buf, *sg, 0, total, 0);
+                if (flags & OMAP_CRYPTO_ZERO_BUF)
+                        memset(buf + total, 0, new_len - total);
+        }
+
+        if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY))
+                sg_init_table(new_sg, 1);
+
+        sg_set_buf(new_sg, buf, new_len);
+
+        *sg = new_sg;
+
+        return 0;
+}
+
+static int omap_crypto_check_sg(struct scatterlist *sg, int total, int bs,
+                                u16 flags)
+{
+        int len = 0;
+        int num_sg = 0;
+
+        if (!IS_ALIGNED(total, bs))
+                return OMAP_CRYPTO_NOT_ALIGNED;
+
+        while (sg) {
+                num_sg++;
+
+                if (!IS_ALIGNED(sg->offset, 4))
+                        return OMAP_CRYPTO_NOT_ALIGNED;
+                if (!IS_ALIGNED(sg->length, bs))
+                        return OMAP_CRYPTO_NOT_ALIGNED;
+
+                len += sg->length;
+                sg = sg_next(sg);
+
+                if (len >= total)
+                        break;
+        }
+
+        if ((flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY) && num_sg > 1)
+                return OMAP_CRYPTO_NOT_ALIGNED;
+
+        if (len != total)
+                return OMAP_CRYPTO_BAD_DATA_LENGTH;
+
+        return 0;
+}
+
+int omap_crypto_align_sg(struct scatterlist **sg, int total, int bs,
+                         struct scatterlist *new_sg, u16 flags,
+                         u8 flags_shift, unsigned long *dd_flags)
+{
+        int ret;
+
+        *dd_flags &= ~(OMAP_CRYPTO_COPY_MASK << flags_shift);
+
+        if (flags & OMAP_CRYPTO_FORCE_COPY)
+                ret = OMAP_CRYPTO_NOT_ALIGNED;
+        else
+                ret = omap_crypto_check_sg(*sg, total, bs, flags);
+
+        if (ret == OMAP_CRYPTO_NOT_ALIGNED) {
+                ret = omap_crypto_copy_sgs(total, bs, sg, new_sg, flags);
+                if (ret)
+                        return ret;
+                *dd_flags |= OMAP_CRYPTO_DATA_COPIED << flags_shift;
+        } else if (ret == OMAP_CRYPTO_BAD_DATA_LENGTH) {
+                ret = omap_crypto_copy_sg_lists(total, bs, sg, new_sg, flags);
+                if (ret)
+                        return ret;
+                if (!(flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY))
+                        *dd_flags |= OMAP_CRYPTO_SG_COPIED << flags_shift;
+        } else if (flags & OMAP_CRYPTO_FORCE_SINGLE_ENTRY) {
+                sg_set_buf(new_sg, sg_virt(*sg), (*sg)->length);
+        }
+
+        return 0;
+}
+EXPORT_SYMBOL_GPL(omap_crypto_align_sg);
+
+void omap_crypto_cleanup(struct scatterlist *sg, struct scatterlist *orig,
+                         int offset, int len, u8 flags_shift,
+                         unsigned long flags)
+{
+        void *buf;
+        int pages;
+
+        flags >>= flags_shift;
+        flags &= OMAP_CRYPTO_COPY_MASK;
+
+        if (!flags)
+                return;
+
+        buf = sg_virt(sg);
+        pages = get_order(len);
+
+        if (orig && (flags & OMAP_CRYPTO_COPY_MASK))
+                scatterwalk_map_and_copy(buf, orig, offset, len, 1);
+
+        if (flags & OMAP_CRYPTO_DATA_COPIED)
+                free_pages((unsigned long)buf, pages);
+        else if (flags & OMAP_CRYPTO_SG_COPIED)
+                kfree(sg);
+}
+EXPORT_SYMBOL_GPL(omap_crypto_cleanup);
+
+MODULE_DESCRIPTION("OMAP crypto support library.");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Tero Kristo <t-kristo@ti.com>");
diff --git a/drivers/crypto/omap-crypto.h b/drivers/crypto/omap-crypto.h
new file mode 100644
index 000000000000..36a230eb87af
--- /dev/null
+++ b/drivers/crypto/omap-crypto.h
@@ -0,0 +1,37 @@
+/*
+ * OMAP Crypto driver common support routines.
+ *
+ * Copyright (c) 2017 Texas Instruments Incorporated
+ *   Tero Kristo <t-kristo@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+
+#ifndef __CRYPTO_OMAP_CRYPTO_H
+#define __CRYPTO_OMAP_CRYPTO_H
+
+enum {
+        OMAP_CRYPTO_NOT_ALIGNED = 1,
+        OMAP_CRYPTO_BAD_DATA_LENGTH,
+};
+
+#define OMAP_CRYPTO_DATA_COPIED         BIT(0)
+#define OMAP_CRYPTO_SG_COPIED           BIT(1)
+
+#define OMAP_CRYPTO_COPY_MASK           0x3
+
+#define OMAP_CRYPTO_COPY_DATA           BIT(0)
+#define OMAP_CRYPTO_FORCE_COPY          BIT(1)
+#define OMAP_CRYPTO_ZERO_BUF            BIT(2)
+#define OMAP_CRYPTO_FORCE_SINGLE_ENTRY  BIT(3)
+
+int omap_crypto_align_sg(struct scatterlist **sg, int total, int bs,
+                         struct scatterlist *new_sg, u16 flags,
+                         u8 flags_shift, unsigned long *dd_flags);
+void omap_crypto_cleanup(struct scatterlist *sg, struct scatterlist *orig,
+                         int offset, int len, u8 flags_shift,
+                         unsigned long flags);
+
+#endif
diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c
index a6f65532fd16..0bcab00e0ff5 100644
--- a/drivers/crypto/omap-des.c
+++ b/drivers/crypto/omap-des.c
@@ -41,6 +41,8 @@
 #include <crypto/algapi.h>
 #include <crypto/engine.h>
 
+#include "omap-crypto.h"
+
 #define DST_MAXBURST                    2
 
 #define DES_BLOCK_WORDS         (DES_BLOCK_SIZE >> 2)
@@ -78,6 +80,11 @@
 #define FLAGS_INIT              BIT(4)
 #define FLAGS_BUSY              BIT(6)
 
+#define DEFAULT_AUTOSUSPEND_DELAY       1000
+
+#define FLAGS_IN_DATA_ST_SHIFT  8
+#define FLAGS_OUT_DATA_ST_SHIFT 10
+
 struct omap_des_ctx {
         struct omap_des_dev *dd;
 
@@ -151,7 +158,6 @@ struct omap_des_dev {
         struct scatterlist              in_sgl;
         struct scatterlist              out_sgl;
         struct scatterlist              *orig_out;
-        int                             sgs_copied;
 
         struct scatter_walk             in_walk;
         struct scatter_walk             out_walk;
@@ -370,20 +376,6 @@ static void omap_des_dma_cleanup(struct omap_des_dev *dd)
         dma_release_channel(dd->dma_lch_in);
 }
 
-static void sg_copy_buf(void *buf, struct scatterlist *sg,
-                              unsigned int start, unsigned int nbytes, int out)
-{
-        struct scatter_walk walk;
-
-        if (!nbytes)
-                return;
-
-        scatterwalk_start(&walk, sg);
-        scatterwalk_advance(&walk, start);
-        scatterwalk_copychunks(buf, &walk, nbytes, out);
-        scatterwalk_done(&walk, out, 0);
-}
-
 static int omap_des_crypt_dma(struct crypto_tfm *tfm,
                 struct scatterlist *in_sg, struct scatterlist *out_sg,
                 int in_sg_len, int out_sg_len)
@@ -506,8 +498,10 @@ static void omap_des_finish_req(struct omap_des_dev *dd, int err)
 
         pr_debug("err: %d\n", err);
 
-        pm_runtime_put(dd->dev);
         crypto_finalize_cipher_request(dd->engine, req, err);
+
+        pm_runtime_mark_last_busy(dd->dev);
+        pm_runtime_put_autosuspend(dd->dev);
 }
 
 static int omap_des_crypt_dma_stop(struct omap_des_dev *dd)
@@ -522,55 +516,6 @@ static int omap_des_crypt_dma_stop(struct omap_des_dev *dd)
         return 0;
 }
 
-static int omap_des_copy_needed(struct scatterlist *sg)
-{
-        while (sg) {
-                if (!IS_ALIGNED(sg->offset, 4))
-                        return -1;
-                if (!IS_ALIGNED(sg->length, DES_BLOCK_SIZE))
-                        return -1;
-                sg = sg_next(sg);
-        }
-        return 0;
-}
-
-static int omap_des_copy_sgs(struct omap_des_dev *dd)
-{
-        void *buf_in, *buf_out;
-        int pages;
-
-        pages = dd->total >> PAGE_SHIFT;
-
-        if (dd->total & (PAGE_SIZE-1))
-                pages++;
-
-        BUG_ON(!pages);
-
-        buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
-        buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
-
-        if (!buf_in || !buf_out) {
-                pr_err("Couldn't allocated pages for unaligned cases.\n");
-                return -1;
-        }
-
-        dd->orig_out = dd->out_sg;
-
-        sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0);
-
-        sg_init_table(&dd->in_sgl, 1);
-        sg_set_buf(&dd->in_sgl, buf_in, dd->total);
-        dd->in_sg = &dd->in_sgl;
-        dd->in_sg_len = 1;
-
-        sg_init_table(&dd->out_sgl, 1);
-        sg_set_buf(&dd->out_sgl, buf_out, dd->total);
-        dd->out_sg = &dd->out_sgl;
-        dd->out_sg_len = 1;
-
-        return 0;
-}
-
 static int omap_des_handle_queue(struct omap_des_dev *dd,
                                  struct ablkcipher_request *req)
 {
@@ -587,6 +532,8 @@ static int omap_des_prepare_req(struct crypto_engine *engine,
                         crypto_ablkcipher_reqtfm(req));
         struct omap_des_dev *dd = omap_des_find_dev(ctx);
         struct omap_des_reqctx *rctx;
+        int ret;
+        u16 flags;
 
         if (!dd)
                 return -ENODEV;
@@ -597,6 +544,23 @@ static int omap_des_prepare_req(struct crypto_engine *engine,
         dd->total_save = req->nbytes;
         dd->in_sg = req->src;
         dd->out_sg = req->dst;
+        dd->orig_out = req->dst;
+
+        flags = OMAP_CRYPTO_COPY_DATA;
+        if (req->src == req->dst)
+                flags |= OMAP_CRYPTO_FORCE_COPY;
+
+        ret = omap_crypto_align_sg(&dd->in_sg, dd->total, DES_BLOCK_SIZE,
+                                   &dd->in_sgl, flags,
+                                   FLAGS_IN_DATA_ST_SHIFT, &dd->flags);
+        if (ret)
+                return ret;
+
+        ret = omap_crypto_align_sg(&dd->out_sg, dd->total, DES_BLOCK_SIZE,
+                                   &dd->out_sgl, 0,
+                                   FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
+        if (ret)
+                return ret;
 
         dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total);
         if (dd->in_sg_len < 0)
@@ -606,15 +570,6 @@ static int omap_des_prepare_req(struct crypto_engine *engine,
         if (dd->out_sg_len < 0)
                 return dd->out_sg_len;
 
-        if (omap_des_copy_needed(dd->in_sg) ||
-            omap_des_copy_needed(dd->out_sg)) {
-                if (omap_des_copy_sgs(dd))
-                        pr_err("Failed to copy SGs for unaligned cases\n");
-                dd->sgs_copied = 1;
-        } else {
-                dd->sgs_copied = 0;
-        }
-
         rctx = ablkcipher_request_ctx(req);
         ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
         rctx->mode &= FLAGS_MODE_MASK;
@@ -642,8 +597,6 @@ static int omap_des_crypt_req(struct crypto_engine *engine,
 static void omap_des_done_task(unsigned long data)
 {
         struct omap_des_dev *dd = (struct omap_des_dev *)data;
-        void *buf_in, *buf_out;
-        int pages;
 
         pr_debug("enter done_task\n");
 
@@ -656,16 +609,11 @@ static void omap_des_done_task(unsigned long data)
                 omap_des_crypt_dma_stop(dd);
         }
 
-        if (dd->sgs_copied) {
-                buf_in = sg_virt(&dd->in_sgl);
-                buf_out = sg_virt(&dd->out_sgl);
+        omap_crypto_cleanup(&dd->in_sgl, NULL, 0, dd->total_save,
+                            FLAGS_IN_DATA_ST_SHIFT, dd->flags);
 
-                sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1);
-
-                pages = get_order(dd->total_save);
-                free_pages((unsigned long)buf_in, pages);
-                free_pages((unsigned long)buf_out, pages);
-        }
+        omap_crypto_cleanup(&dd->out_sgl, dd->orig_out, 0, dd->total_save,
+                            FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
 
         omap_des_finish_req(dd, 0);
 
@@ -699,16 +647,28 @@ static int omap_des_crypt(struct ablkcipher_request *req, unsigned long mode)
 
 /* ********************** ALG API ************************************ */
 
-static int omap_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+static int omap_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
                            unsigned int keylen)
 {
-        struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+        struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(cipher);
+        struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
 
         if (keylen != DES_KEY_SIZE && keylen != (3*DES_KEY_SIZE))
                 return -EINVAL;
 
         pr_debug("enter, keylen: %d\n", keylen);
 
+        /* Do we need to test against weak key? */
+        if (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY) {
+                u32 tmp[DES_EXPKEY_WORDS];
+                int ret = des_ekey(tmp, key);
+
+                if (!ret) {
+                        tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
+                        return -EINVAL;
+                }
+        }
+
         memcpy(ctx->key, key, keylen);
         ctx->keylen = keylen;
 
@@ -1032,8 +992,10 @@ static int omap_des_probe(struct platform_device *pdev)
         }
         dd->phys_base = res->start;
 
+        pm_runtime_use_autosuspend(dev);
+        pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
+
         pm_runtime_enable(dev);
-        pm_runtime_irq_safe(dev);
         err = pm_runtime_get_sync(dev);
         if (err < 0) {
                 pm_runtime_put_noidle(dev);
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index d0b16e5e4ee5..9ad9d399daf1 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -41,6 +41,7 @@
 #include <crypto/algapi.h>
 #include <crypto/sha.h>
 #include <crypto/hash.h>
+#include <crypto/hmac.h>
 #include <crypto/internal/hash.h>
 
 #define MD5_DIGEST_SIZE                 16
@@ -225,7 +226,7 @@ struct omap_sham_dev {
         struct dma_chan         *dma_lch;
         struct tasklet_struct   done_task;
         u8                      polling_mode;
-        u8                      xmit_buf[BUFLEN];
+        u8                      xmit_buf[BUFLEN] OMAP_ALIGNED;
 
         unsigned long           flags;
         struct crypto_queue     queue;
@@ -750,7 +751,10 @@ static int omap_sham_align_sgs(struct scatterlist *sg,
         if (final)
                 new_len = DIV_ROUND_UP(new_len, bs) * bs;
         else
-                new_len = new_len / bs * bs;
+                new_len = (new_len - 1) / bs * bs;
+
+        if (nbytes != new_len)
+                list_ok = false;
 
         while (nbytes > 0 && sg_tmp) {
                 n++;
@@ -846,6 +850,8 @@ static int omap_sham_prepare_request(struct ahash_request *req, bool update)
                         xmit_len = DIV_ROUND_UP(xmit_len, bs) * bs;
                 else
                         xmit_len = xmit_len / bs * bs;
+        } else if (!final) {
+                xmit_len -= bs;
         }
 
         hash_later = rctx->total - xmit_len;
@@ -873,14 +879,21 @@ static int omap_sham_prepare_request(struct ahash_request *req, bool update)
         }
 
         if (hash_later) {
-                if (req->nbytes) {
-                        scatterwalk_map_and_copy(rctx->buffer, req->src,
-                                                 req->nbytes - hash_later,
-                                                 hash_later, 0);
-                } else {
+                int offset = 0;
+
+                if (hash_later > req->nbytes) {
                         memcpy(rctx->buffer, rctx->buffer + xmit_len,
-                               hash_later);
+                               hash_later - req->nbytes);
+                        offset = hash_later - req->nbytes;
                 }
+
+                if (req->nbytes) {
+                        scatterwalk_map_and_copy(rctx->buffer + offset,
+                                                 req->src,
+                                                 offset + req->nbytes -
+                                                 hash_later, hash_later, 0);
+                }
+
                 rctx->bufcnt = hash_later;
         } else {
                 rctx->bufcnt = 0;
@@ -1130,7 +1143,7 @@ retry:
         ctx = ahash_request_ctx(req);
 
         err = omap_sham_prepare_request(req, ctx->op == OP_UPDATE);
-        if (err)
+        if (err || !ctx->total)
                 goto err1;
 
         dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
@@ -1189,11 +1202,10 @@ static int omap_sham_update(struct ahash_request *req)
         if (!req->nbytes)
                 return 0;
 
-        if (ctx->total + req->nbytes < ctx->buflen) {
+        if (ctx->bufcnt + req->nbytes <= ctx->buflen) {
                 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,
                                          0, req->nbytes, 0);
                 ctx->bufcnt += req->nbytes;
-                ctx->total += req->nbytes;
                 return 0;
         }
 
@@ -1326,8 +1338,8 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
                 memcpy(bctx->opad, bctx->ipad, bs);
 
                 for (i = 0; i < bs; i++) {
-                        bctx->ipad[i] ^= 0x36;
-                        bctx->opad[i] ^= 0x5c;
+                        bctx->ipad[i] ^= HMAC_IPAD_VALUE;
+                        bctx->opad[i] ^= HMAC_OPAD_VALUE;
                 }
         }
 
diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c
index 2839fccdd84b..d3e25c37dc33 100644
--- a/drivers/crypto/qat/qat_common/adf_aer.c
+++ b/drivers/crypto/qat/qat_common/adf_aer.c
@@ -109,20 +109,7 @@ EXPORT_SYMBOL_GPL(adf_reset_sbr);
 
 void adf_reset_flr(struct adf_accel_dev *accel_dev)
 {
-        struct pci_dev *pdev = accel_to_pci_dev(accel_dev);
-        u16 control = 0;
-        int pos = 0;
-
-        dev_info(&GET_DEV(accel_dev), "Function level reset\n");
-        pos = pci_pcie_cap(pdev);
-        if (!pos) {
-                dev_err(&GET_DEV(accel_dev), "Restart device failed\n");
-                return;
-        }
-        pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &control);
-        control |= PCI_EXP_DEVCTL_BCR_FLR;
-        pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, control);
-        msleep(100);
+        pcie_flr(accel_to_pci_dev(accel_dev));
 }
 EXPORT_SYMBOL_GPL(adf_reset_flr);
 
diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c
index 20f35df8a01f..5b5efcc52cb5 100644
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_algs.c
@@ -51,6 +51,7 @@
 #include <crypto/aes.h>
 #include <crypto/sha.h>
 #include <crypto/hash.h>
+#include <crypto/hmac.h>
 #include <crypto/algapi.h>
 #include <crypto/authenc.h>
 #include <linux/dma-mapping.h>
@@ -178,8 +179,8 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
         for (i = 0; i < block_size; i++) {
                 char *ipad_ptr = ipad + i;
                 char *opad_ptr = opad + i;
-                *ipad_ptr ^= 0x36;
-                *opad_ptr ^= 0x5C;
+                *ipad_ptr ^= HMAC_IPAD_VALUE;
+                *opad_ptr ^= HMAC_OPAD_VALUE;
         }
 
         if (crypto_shash_init(shash))
diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c
index 2aab80bc241f..6f5dd68449c6 100644
--- a/drivers/crypto/qat/qat_common/qat_asym_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c
@@ -521,11 +521,11 @@ static int qat_dh_set_secret(struct crypto_kpp *tfm, const void *buf,
         return 0;
 }
 
-static int qat_dh_max_size(struct crypto_kpp *tfm)
+static unsigned int qat_dh_max_size(struct crypto_kpp *tfm)
 {
         struct qat_dh_ctx *ctx = kpp_tfm_ctx(tfm);
 
-        return ctx->p ? ctx->p_size : -EINVAL;
+        return ctx->p_size;
 }
 
 static int qat_dh_init_tfm(struct crypto_kpp *tfm)
@@ -1256,11 +1256,11 @@ static int qat_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
         return qat_rsa_setkey(tfm, key, keylen, true);
 }
 
-static int qat_rsa_max_size(struct crypto_akcipher *tfm)
+static unsigned int qat_rsa_max_size(struct crypto_akcipher *tfm)
 {
         struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
 
-        return (ctx->n) ? ctx->key_sz : -EINVAL;
+        return ctx->key_sz;
 }
 
 static int qat_rsa_init_tfm(struct crypto_akcipher *tfm)
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
index 90efd10d57a1..5cf64746731a 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
@@ -16,13 +16,13 @@
  */
 #include "sun4i-ss.h"
 
-static int sun4i_ss_opti_poll(struct ablkcipher_request *areq)
+static int sun4i_ss_opti_poll(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
         struct sun4i_ss_ctx *ss = op->ss;
-        unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
-        struct sun4i_cipher_req_ctx *ctx = ablkcipher_request_ctx(areq);
+        unsigned int ivsize = crypto_skcipher_ivsize(tfm);
+        struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq);
         u32 mode = ctx->mode;
         /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */
         u32 rx_cnt = SS_RX_DEFAULT;
@@ -31,17 +31,17 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq)
         u32 v;
         int err = 0;
         unsigned int i;
-        unsigned int ileft = areq->nbytes;
-        unsigned int oleft = areq->nbytes;
+        unsigned int ileft = areq->cryptlen;
+        unsigned int oleft = areq->cryptlen;
         unsigned int todo;
         struct sg_mapping_iter mi, mo;
         unsigned int oi, oo; /* offset for in and out */
         unsigned long flags;
 
-        if (areq->nbytes == 0)
+        if (!areq->cryptlen)
                 return 0;
 
-        if (!areq->info) {
+        if (!areq->iv) {
                 dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n");
                 return -EINVAL;
         }
@@ -56,9 +56,9 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq)
         for (i = 0; i < op->keylen; i += 4)
                 writel(*(op->key + i / 4), ss->base + SS_KEY0 + i);
 
-        if (areq->info) {
+        if (areq->iv) {
                 for (i = 0; i < 4 && i < ivsize / 4; i++) {
-                        v = *(u32 *)(areq->info + i * 4);
+                        v = *(u32 *)(areq->iv + i * 4);
                         writel(v, ss->base + SS_IV0 + i * 4);
                 }
         }
@@ -76,13 +76,13 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq)
                 goto release_ss;
         }
 
-        ileft = areq->nbytes / 4;
-        oleft = areq->nbytes / 4;
+        ileft = areq->cryptlen / 4;
+        oleft = areq->cryptlen / 4;
         oi = 0;
         oo = 0;
         do {
                 todo = min3(rx_cnt, ileft, (mi.length - oi) / 4);
-                if (todo > 0) {
+                if (todo) {
                         ileft -= todo;
                         writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo);
                         oi += todo * 4;
@@ -97,7 +97,7 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq)
                 tx_cnt = SS_TXFIFO_SPACES(spaces);
 
                 todo = min3(tx_cnt, oleft, (mo.length - oo) / 4);
-                if (todo > 0) {
+                if (todo) {
                         oleft -= todo;
                         readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo);
                         oo += todo * 4;
@@ -106,12 +106,12 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq)
                         sg_miter_next(&mo);
                         oo = 0;
                 }
-        } while (oleft > 0);
+        } while (oleft);
 
-        if (areq->info) {
+        if (areq->iv) {
                 for (i = 0; i < 4 && i < ivsize / 4; i++) {
                         v = readl(ss->base + SS_IV0 + i * 4);
-                        *(u32 *)(areq->info + i * 4) = v;
+                        *(u32 *)(areq->iv + i * 4) = v;
                 }
         }
 
@@ -124,16 +124,16 @@ release_ss:
 }
 
 /* Generic function that support SG with size not multiple of 4 */
-static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq)
+static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
         struct sun4i_ss_ctx *ss = op->ss;
         int no_chunk = 1;
         struct scatterlist *in_sg = areq->src;
         struct scatterlist *out_sg = areq->dst;
-        unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
-        struct sun4i_cipher_req_ctx *ctx = ablkcipher_request_ctx(areq);
+        unsigned int ivsize = crypto_skcipher_ivsize(tfm);
+        struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq);
         u32 mode = ctx->mode;
         /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */
         u32 rx_cnt = SS_RX_DEFAULT;
@@ -142,8 +142,8 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq)
         u32 spaces;
         int err = 0;
         unsigned int i;
-        unsigned int ileft = areq->nbytes;
-        unsigned int oleft = areq->nbytes;
+        unsigned int ileft = areq->cryptlen;
+        unsigned int oleft = areq->cryptlen;
         unsigned int todo;
         struct sg_mapping_iter mi, mo;
         unsigned int oi, oo;    /* offset for in and out */
@@ -154,10 +154,10 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq)
         unsigned int obl = 0;   /* length of data in bufo */
         unsigned long flags;
 
-        if (areq->nbytes == 0)
+        if (!areq->cryptlen)
                 return 0;
 
-        if (!areq->info) {
+        if (!areq->iv) {
                 dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n");
                 return -EINVAL;
         }
@@ -172,12 +172,12 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq)
          * we can use the SS optimized function
          */
         while (in_sg && no_chunk == 1) {
-                if ((in_sg->length % 4) != 0)
+                if (in_sg->length % 4)
                         no_chunk = 0;
                 in_sg = sg_next(in_sg);
         }
         while (out_sg && no_chunk == 1) {
-                if ((out_sg->length % 4) != 0)
+                if (out_sg->length % 4)
                         no_chunk = 0;
                 out_sg = sg_next(out_sg);
         }
@@ -190,9 +190,9 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq)
         for (i = 0; i < op->keylen; i += 4)
                 writel(*(op->key + i / 4), ss->base + SS_KEY0 + i);
 
-        if (areq->info) {
+        if (areq->iv) {
                 for (i = 0; i < 4 && i < ivsize / 4; i++) {
-                        v = *(u32 *)(areq->info + i * 4);
+                        v = *(u32 *)(areq->iv + i * 4);
                         writel(v, ss->base + SS_IV0 + i * 4);
                 }
         }
@@ -209,19 +209,19 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq)
                 err = -EINVAL;
                 goto release_ss;
         }
-        ileft = areq->nbytes;
-        oleft = areq->nbytes;
+        ileft = areq->cryptlen;
+        oleft = areq->cryptlen;
         oi = 0;
         oo = 0;
 
-        while (oleft > 0) {
-                if (ileft > 0) {
+        while (oleft) {
+                if (ileft) {
                         /*
                          * todo is the number of consecutive 4byte word that we
                          * can read from current SG
                          */
                         todo = min3(rx_cnt, ileft / 4, (mi.length - oi) / 4);
-                        if (todo > 0 && ob == 0) {
+                        if (todo && !ob) {
                                 writesl(ss->base + SS_RXFIFO, mi.addr + oi,
                                         todo);
                                 ileft -= todo * 4;
@@ -240,7 +240,7 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq)
                                 ileft -= todo;
                                 oi += todo;
                                 ob += todo;
-                                if (ob % 4 == 0) {
+                                if (!(ob % 4)) {
                                         writesl(ss->base + SS_RXFIFO, buf,
                                                 ob / 4);
                                         ob = 0;
@@ -257,14 +257,14 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq)
                 tx_cnt = SS_TXFIFO_SPACES(spaces);
                 dev_dbg(ss->dev, "%x %u/%u %u/%u cnt=%u %u/%u %u/%u cnt=%u %u\n",
                         mode,
-                        oi, mi.length, ileft, areq->nbytes, rx_cnt,
-                        oo, mo.length, oleft, areq->nbytes, tx_cnt, ob);
+                        oi, mi.length, ileft, areq->cryptlen, rx_cnt,
+                        oo, mo.length, oleft, areq->cryptlen, tx_cnt, ob);
 
-                if (tx_cnt == 0)
+                if (!tx_cnt)
                         continue;
                 /* todo in 4bytes word */
                 todo = min3(tx_cnt, oleft / 4, (mo.length - oo) / 4);
-                if (todo > 0) {
+                if (todo) {
                         readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo);
                         oleft -= todo * 4;
                         oo += todo * 4;
@@ -300,10 +300,10 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq)
                         /* bufo must be fully used here */
                 }
         }
-        if (areq->info) {
+        if (areq->iv) {
                 for (i = 0; i < 4 && i < ivsize / 4; i++) {
                         v = readl(ss->base + SS_IV0 + i * 4);
-                        *(u32 *)(areq->info + i * 4) = v;
+                        *(u32 *)(areq->iv + i * 4) = v;
                 }
         }
 
@@ -317,22 +317,22 @@ release_ss:
 }
 
 /* CBC AES */
-int sun4i_ss_cbc_aes_encrypt(struct ablkcipher_request *areq)
+int sun4i_ss_cbc_aes_encrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |
                 op->keymode;
         return sun4i_ss_cipher_poll(areq);
 }
 
-int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request *areq)
+int sun4i_ss_cbc_aes_decrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_DECRYPTION |
                 op->keymode;
@@ -340,22 +340,22 @@ int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request *areq)
 }
 
 /* ECB AES */
-int sun4i_ss_ecb_aes_encrypt(struct ablkcipher_request *areq)
+int sun4i_ss_ecb_aes_encrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |
                 op->keymode;
         return sun4i_ss_cipher_poll(areq);
 }
 
-int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request *areq)
+int sun4i_ss_ecb_aes_decrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_DECRYPTION |
                 op->keymode;
@@ -363,22 +363,22 @@ int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request *areq)
 }
 
 /* CBC DES */
-int sun4i_ss_cbc_des_encrypt(struct ablkcipher_request *areq)
+int sun4i_ss_cbc_des_encrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |
                 op->keymode;
         return sun4i_ss_cipher_poll(areq);
 }
 
-int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request *areq)
+int sun4i_ss_cbc_des_decrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_DECRYPTION |
                 op->keymode;
@@ -386,22 +386,22 @@ int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request *areq)
 }
 
 /* ECB DES */
-int sun4i_ss_ecb_des_encrypt(struct ablkcipher_request *areq)
+int sun4i_ss_ecb_des_encrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |
                 op->keymode;
         return sun4i_ss_cipher_poll(areq);
 }
 
-int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request *areq)
+int sun4i_ss_ecb_des_decrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_DECRYPTION |
                 op->keymode;
@@ -409,22 +409,22 @@ int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request *areq)
 }
 
 /* CBC 3DES */
-int sun4i_ss_cbc_des3_encrypt(struct ablkcipher_request *areq)
+int sun4i_ss_cbc_des3_encrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION |
                 op->keymode;
         return sun4i_ss_cipher_poll(areq);
 }
 
-int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request *areq)
+int sun4i_ss_cbc_des3_decrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_DECRYPTION |
                 op->keymode;
@@ -432,22 +432,22 @@ int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request *areq)
 }
 
 /* ECB 3DES */
-int sun4i_ss_ecb_des3_encrypt(struct ablkcipher_request *areq)
+int sun4i_ss_ecb_des3_encrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION |
                 op->keymode;
         return sun4i_ss_cipher_poll(areq);
 }
 
-int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request *areq)
+int sun4i_ss_ecb_des3_decrypt(struct skcipher_request *areq)
 {
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq);
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
-        struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq);
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 
         rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_DECRYPTION |
                 op->keymode;
@@ -457,24 +457,25 @@ int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request *areq)
 int sun4i_ss_cipher_init(struct crypto_tfm *tfm)
 {
         struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm);
-        struct crypto_alg *alg = tfm->__crt_alg;
         struct sun4i_ss_alg_template *algt;
 
         memset(op, 0, sizeof(struct sun4i_tfm_ctx));
 
-        algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto);
+        algt = container_of(tfm->__crt_alg, struct sun4i_ss_alg_template,
+                            alg.crypto.base);
         op->ss = algt->ss;
 
-        tfm->crt_ablkcipher.reqsize = sizeof(struct sun4i_cipher_req_ctx);
+        crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
+                                    sizeof(struct sun4i_cipher_req_ctx));
 
         return 0;
 }
 
 /* check and set the AES key, prepare the mode to be used */
-int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+int sun4i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
                         unsigned int keylen)
 {
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
         struct sun4i_ss_ctx *ss = op->ss;
 
         switch (keylen) {
@@ -489,7 +490,7 @@ int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
                 break;
         default:
                 dev_err(ss->dev, "ERROR: Invalid keylen %u\n", keylen);
-                crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
                 return -EINVAL;
         }
         op->keylen = keylen;
@@ -498,10 +499,10 @@ int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 }
 
 /* check and set the DES key, prepare the mode to be used */
-int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
                         unsigned int keylen)
 {
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
         struct sun4i_ss_ctx *ss = op->ss;
         u32 flags;
         u32 tmp[DES_EXPKEY_WORDS];
@@ -509,15 +510,15 @@ int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 
         if (unlikely(keylen != DES_KEY_SIZE)) {
                 dev_err(ss->dev, "Invalid keylen %u\n", keylen);
-                crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
                 return -EINVAL;
         }
 
-        flags = crypto_ablkcipher_get_flags(tfm);
+        flags = crypto_skcipher_get_flags(tfm);
 
         ret = des_ekey(tmp, key);
-        if (unlikely(ret == 0) && (flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
-                crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY);
+        if (unlikely(!ret) && (flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
+                crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY);
                 dev_dbg(ss->dev, "Weak key %u\n", keylen);
                 return -EINVAL;
         }
@@ -528,15 +529,15 @@ int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
 }
 
 /* check and set the 3DES key, prepare the mode to be used */
-int sun4i_ss_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
                          unsigned int keylen)
 {
-        struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
         struct sun4i_ss_ctx *ss = op->ss;
 
         if (unlikely(keylen != 3 * DES_KEY_SIZE)) {
                 dev_err(ss->dev, "Invalid keylen %u\n", keylen);
-                crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
                 return -EINVAL;
         }
         op->keylen = keylen;
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c
index 3ac6c6c4ad18..02ad8256e900 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-core.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c
@@ -83,134 +83,133 @@ static struct sun4i_ss_alg_template ss_algs[] = {
                 }
         }
 },
-{       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+{       .type = CRYPTO_ALG_TYPE_SKCIPHER,
         .alg.crypto = {
-                .cra_name = "cbc(aes)",
-                .cra_driver_name = "cbc-aes-sun4i-ss",
-                .cra_priority = 300,
-                .cra_blocksize = AES_BLOCK_SIZE,
-                .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER,
-                .cra_ctxsize = sizeof(struct sun4i_tfm_ctx),
-                .cra_module = THIS_MODULE,
-                .cra_alignmask = 3,
-                .cra_type = &crypto_ablkcipher_type,
-                .cra_init = sun4i_ss_cipher_init,
-                .cra_ablkcipher = {
-                        .min_keysize    = AES_MIN_KEY_SIZE,
-                        .max_keysize    = AES_MAX_KEY_SIZE,
-                        .ivsize         = AES_BLOCK_SIZE,
-                        .setkey         = sun4i_ss_aes_setkey,
-                        .encrypt        = sun4i_ss_cbc_aes_encrypt,
-                        .decrypt        = sun4i_ss_cbc_aes_decrypt,
+                .setkey         = sun4i_ss_aes_setkey,
+                .encrypt        = sun4i_ss_cbc_aes_encrypt,
+                .decrypt        = sun4i_ss_cbc_aes_decrypt,
+                .min_keysize    = AES_MIN_KEY_SIZE,
+                .max_keysize    = AES_MAX_KEY_SIZE,
+                .ivsize         = AES_BLOCK_SIZE,
+                .base = {
+                        .cra_name = "cbc(aes)",
+                        .cra_driver_name = "cbc-aes-sun4i-ss",
+                        .cra_priority = 300,
+                        .cra_blocksize = AES_BLOCK_SIZE,
+                        .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+                                     CRYPTO_ALG_KERN_DRIVER_ONLY,
+                        .cra_ctxsize = sizeof(struct sun4i_tfm_ctx),
+                        .cra_module = THIS_MODULE,
+                        .cra_alignmask = 3,
+                        .cra_init = sun4i_ss_cipher_init,
                 }
         }
 },
-{       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+{       .type = CRYPTO_ALG_TYPE_SKCIPHER,
         .alg.crypto = {
-                .cra_name = "ecb(aes)",
-                .cra_driver_name = "ecb-aes-sun4i-ss",
-                .cra_priority = 300,
-                .cra_blocksize = AES_BLOCK_SIZE,
-                .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER,
-                .cra_ctxsize = sizeof(struct sun4i_tfm_ctx),
-                .cra_module = THIS_MODULE,
-                .cra_alignmask = 3,
-                .cra_type = &crypto_ablkcipher_type,
-                .cra_init = sun4i_ss_cipher_init,
-                .cra_ablkcipher = {
-                        .min_keysize    = AES_MIN_KEY_SIZE,
-                        .max_keysize    = AES_MAX_KEY_SIZE,
-                        .ivsize         = AES_BLOCK_SIZE,
-                        .setkey         = sun4i_ss_aes_setkey,
-                        .encrypt        = sun4i_ss_ecb_aes_encrypt,
-                        .decrypt        = sun4i_ss_ecb_aes_decrypt,
+                .setkey         = sun4i_ss_aes_setkey,
+                .encrypt        = sun4i_ss_ecb_aes_encrypt,
+                .decrypt        = sun4i_ss_ecb_aes_decrypt,
+                .min_keysize    = AES_MIN_KEY_SIZE,
+                .max_keysize    = AES_MAX_KEY_SIZE,
+                .ivsize         = AES_BLOCK_SIZE,
+                .base = {
+                        .cra_name = "ecb(aes)",
+                        .cra_driver_name = "ecb-aes-sun4i-ss",
+                        .cra_priority = 300,
+                        .cra_blocksize = AES_BLOCK_SIZE,
+                        .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+                                     CRYPTO_ALG_KERN_DRIVER_ONLY,
+                        .cra_ctxsize = sizeof(struct sun4i_tfm_ctx),
+                        .cra_module = THIS_MODULE,
+                        .cra_alignmask = 3,
+                        .cra_init = sun4i_ss_cipher_init,
                 }
         }
 },
-{       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+{       .type = CRYPTO_ALG_TYPE_SKCIPHER,
         .alg.crypto = {
-                .cra_name = "cbc(des)",
-                .cra_driver_name = "cbc-des-sun4i-ss",
-                .cra_priority = 300,
-                .cra_blocksize = DES_BLOCK_SIZE,
-                .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER,
-                .cra_ctxsize = sizeof(struct sun4i_req_ctx),
-                .cra_module = THIS_MODULE,
-                .cra_alignmask = 3,
-                .cra_type = &crypto_ablkcipher_type,
-                .cra_init = sun4i_ss_cipher_init,
-                .cra_u.ablkcipher = {
-                        .min_keysize    = DES_KEY_SIZE,
-                        .max_keysize    = DES_KEY_SIZE,
-                        .ivsize         = DES_BLOCK_SIZE,
-                        .setkey         = sun4i_ss_des_setkey,
-                        .encrypt        = sun4i_ss_cbc_des_encrypt,
-                        .decrypt        = sun4i_ss_cbc_des_decrypt,
+                .setkey         = sun4i_ss_des_setkey,
+                .encrypt        = sun4i_ss_cbc_des_encrypt,
+                .decrypt        = sun4i_ss_cbc_des_decrypt,
+                .min_keysize    = DES_KEY_SIZE,
+                .max_keysize    = DES_KEY_SIZE,
+                .ivsize         = DES_BLOCK_SIZE,
+                .base = {
+                        .cra_name = "cbc(des)",
+                        .cra_driver_name = "cbc-des-sun4i-ss",
+                        .cra_priority = 300,
+                        .cra_blocksize = DES_BLOCK_SIZE,
+                        .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+                                     CRYPTO_ALG_KERN_DRIVER_ONLY,
+                        .cra_ctxsize = sizeof(struct sun4i_req_ctx),
+                        .cra_module = THIS_MODULE,
+                        .cra_alignmask = 3,
+                        .cra_init = sun4i_ss_cipher_init,
                 }
         }
 },
-{       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+{       .type = CRYPTO_ALG_TYPE_SKCIPHER,
         .alg.crypto = {
-                .cra_name = "ecb(des)",
-                .cra_driver_name = "ecb-des-sun4i-ss",
-                .cra_priority = 300,
-                .cra_blocksize = DES_BLOCK_SIZE,
-                .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER,
-                .cra_ctxsize = sizeof(struct sun4i_req_ctx),
-                .cra_module = THIS_MODULE,
-                .cra_alignmask = 3,
-                .cra_type = &crypto_ablkcipher_type,
-                .cra_init = sun4i_ss_cipher_init,
-                .cra_u.ablkcipher = {
-                        .min_keysize    = DES_KEY_SIZE,
-                        .max_keysize    = DES_KEY_SIZE,
-                        .setkey         = sun4i_ss_des_setkey,
-                        .encrypt        = sun4i_ss_ecb_des_encrypt,
-                        .decrypt        = sun4i_ss_ecb_des_decrypt,
+                .setkey         = sun4i_ss_des_setkey,
+                .encrypt        = sun4i_ss_ecb_des_encrypt,
+                .decrypt        = sun4i_ss_ecb_des_decrypt,
+                .min_keysize    = DES_KEY_SIZE,
+                .max_keysize    = DES_KEY_SIZE,
+                .base = {
+                        .cra_name = "ecb(des)",
+                        .cra_driver_name = "ecb-des-sun4i-ss",
+                        .cra_priority = 300,
+                        .cra_blocksize = DES_BLOCK_SIZE,
+                        .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+                                     CRYPTO_ALG_KERN_DRIVER_ONLY,
+                        .cra_ctxsize = sizeof(struct sun4i_req_ctx),
+                        .cra_module = THIS_MODULE,
+                        .cra_alignmask = 3,
+                        .cra_init = sun4i_ss_cipher_init,
                 }
         }
 },
-{       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+{       .type = CRYPTO_ALG_TYPE_SKCIPHER,
         .alg.crypto = {
-                .cra_name = "cbc(des3_ede)",
-                .cra_driver_name = "cbc-des3-sun4i-ss",
-                .cra_priority = 300,
-                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
-                .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER,
-                .cra_ctxsize = sizeof(struct sun4i_req_ctx),
-                .cra_module = THIS_MODULE,
-                .cra_alignmask = 3,
-                .cra_type = &crypto_ablkcipher_type,
-                .cra_init = sun4i_ss_cipher_init,
-                .cra_u.ablkcipher = {
-                        .min_keysize    = DES3_EDE_KEY_SIZE,
-                        .max_keysize    = DES3_EDE_KEY_SIZE,
-                        .ivsize         = DES3_EDE_BLOCK_SIZE,
-                        .setkey         = sun4i_ss_des3_setkey,
-                        .encrypt        = sun4i_ss_cbc_des3_encrypt,
-                        .decrypt        = sun4i_ss_cbc_des3_decrypt,
+                .setkey         = sun4i_ss_des3_setkey,
+                .encrypt        = sun4i_ss_cbc_des3_encrypt,
+                .decrypt        = sun4i_ss_cbc_des3_decrypt,
+                .min_keysize    = DES3_EDE_KEY_SIZE,
+                .max_keysize    = DES3_EDE_KEY_SIZE,
+                .ivsize         = DES3_EDE_BLOCK_SIZE,
+                .base = {
+                        .cra_name = "cbc(des3_ede)",
+                        .cra_driver_name = "cbc-des3-sun4i-ss",
+                        .cra_priority = 300,
+                        .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
+                                     CRYPTO_ALG_KERN_DRIVER_ONLY,
+                        .cra_ctxsize = sizeof(struct sun4i_req_ctx),
+                        .cra_module = THIS_MODULE,
+                        .cra_alignmask = 3,
+                        .cra_init = sun4i_ss_cipher_init,
                 }
         }
 },
-{       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+{       .type = CRYPTO_ALG_TYPE_SKCIPHER,
         .alg.crypto = {
-                .cra_name = "ecb(des3_ede)",
-                .cra_driver_name = "ecb-des3-sun4i-ss",
-                .cra_priority = 300,
-                .cra_blocksize = DES3_EDE_BLOCK_SIZE,
-                .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER,
-                .cra_ctxsize = sizeof(struct sun4i_req_ctx),
-                .cra_module = THIS_MODULE,
-                .cra_alignmask = 3,
-                .cra_type = &crypto_ablkcipher_type,
-                .cra_init = sun4i_ss_cipher_init,
-                .cra_u.ablkcipher = {
-                        .min_keysize    = DES3_EDE_KEY_SIZE,
-                        .max_keysize    = DES3_EDE_KEY_SIZE,
-                        .ivsize         = DES3_EDE_BLOCK_SIZE,
-                        .setkey         = sun4i_ss_des3_setkey,
-                        .encrypt        = sun4i_ss_ecb_des3_encrypt,
-                        .decrypt        = sun4i_ss_ecb_des3_decrypt,
+                .setkey         = sun4i_ss_des3_setkey,
+                .encrypt        = sun4i_ss_ecb_des3_encrypt,
+                .decrypt        = sun4i_ss_ecb_des3_decrypt,
+                .min_keysize    = DES3_EDE_KEY_SIZE,
+                .max_keysize    = DES3_EDE_KEY_SIZE,
+                .ivsize         = DES3_EDE_BLOCK_SIZE,
+                .base = {
+                        .cra_name = "ecb(des3_ede)",
+                        .cra_driver_name = "ecb-des3-sun4i-ss",
+                        .cra_priority = 300,
+                        .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+                        .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER,
+                        .cra_ctxsize = sizeof(struct sun4i_req_ctx),
+                        .cra_module = THIS_MODULE,
+                        .cra_alignmask = 3,
+                        .cra_init = sun4i_ss_cipher_init,
                 }
         }
 },
@@ -266,12 +265,12 @@ static int sun4i_ss_probe(struct platform_device *pdev)
 
         /* Enable both clocks */
         err = clk_prepare_enable(ss->busclk);
-        if (err != 0) {
+        if (err) {
                 dev_err(&pdev->dev, "Cannot prepare_enable busclk\n");
                 return err;
         }
         err = clk_prepare_enable(ss->ssclk);
-        if (err != 0) {
+        if (err) {
                 dev_err(&pdev->dev, "Cannot prepare_enable ssclk\n");
                 goto error_ssclk;
         }
@@ -281,7 +280,7 @@ static int sun4i_ss_probe(struct platform_device *pdev)
          * Try to set the clock to the maximum allowed
          */
         err = clk_set_rate(ss->ssclk, cr_mod);
-        if (err != 0) {
+        if (err) {
                 dev_err(&pdev->dev, "Cannot set clock rate to ssclk\n");
                 goto error_clk;
         }
@@ -340,17 +339,17 @@ static int sun4i_ss_probe(struct platform_device *pdev)
         for (i = 0; i < ARRAY_SIZE(ss_algs); i++) {
                 ss_algs[i].ss = ss;
                 switch (ss_algs[i].type) {
-                case CRYPTO_ALG_TYPE_ABLKCIPHER:
-                        err = crypto_register_alg(&ss_algs[i].alg.crypto);
-                        if (err != 0) {
+                case CRYPTO_ALG_TYPE_SKCIPHER:
+                        err = crypto_register_skcipher(&ss_algs[i].alg.crypto);
+                        if (err) {
                                 dev_err(ss->dev, "Fail to register %s\n",
-                                        ss_algs[i].alg.crypto.cra_name);
+                                        ss_algs[i].alg.crypto.base.cra_name);
                                 goto error_alg;
                         }
                         break;
                 case CRYPTO_ALG_TYPE_AHASH:
                         err = crypto_register_ahash(&ss_algs[i].alg.hash);
-                        if (err != 0) {
+                        if (err) {
                                 dev_err(ss->dev, "Fail to register %s\n",
                                         ss_algs[i].alg.hash.halg.base.cra_name);
                                 goto error_alg;
@@ -364,8 +363,8 @@ error_alg:
         i--;
         for (; i >= 0; i--) {
                 switch (ss_algs[i].type) {
-                case CRYPTO_ALG_TYPE_ABLKCIPHER:
-                        crypto_unregister_alg(&ss_algs[i].alg.crypto);
+                case CRYPTO_ALG_TYPE_SKCIPHER:
+                        crypto_unregister_skcipher(&ss_algs[i].alg.crypto);
                         break;
                 case CRYPTO_ALG_TYPE_AHASH:
                         crypto_unregister_ahash(&ss_algs[i].alg.hash);
@@ -388,8 +387,8 @@ static int sun4i_ss_remove(struct platform_device *pdev)
 
         for (i = 0; i < ARRAY_SIZE(ss_algs); i++) {
                 switch (ss_algs[i].type) {
-                case CRYPTO_ALG_TYPE_ABLKCIPHER:
-                        crypto_unregister_alg(&ss_algs[i].alg.crypto);
+                case CRYPTO_ALG_TYPE_SKCIPHER:
+                        crypto_unregister_skcipher(&ss_algs[i].alg.crypto);
                         break;
                 case CRYPTO_ALG_TYPE_AHASH:
                         crypto_unregister_ahash(&ss_algs[i].alg.hash);
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c
index 0de2f62d51ff..a4b5ff2b72f8 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-hash.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-hash.c
@@ -60,7 +60,7 @@ int sun4i_hash_export_md5(struct ahash_request *areq, void *out)
 
         memcpy(octx->block, op->buf, op->len);
 
-        if (op->byte_count > 0) {
+        if (op->byte_count) {
                 for (i = 0; i < 4; i++)
                         octx->hash[i] = op->hash[i];
         } else {
@@ -102,7 +102,7 @@ int sun4i_hash_export_sha1(struct ahash_request *areq, void *out)
 
         memcpy(octx->buffer, op->buf, op->len);
 
-        if (op->byte_count > 0) {
+        if (op->byte_count) {
                 for (i = 0; i < 5; i++)
                         octx->state[i] = op->hash[i];
         } else {
@@ -167,44 +167,34 @@ int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in)
  */
 static int sun4i_hash(struct ahash_request *areq)
 {
-        u32 v, ivmode = 0;
-        unsigned int i = 0;
         /*
          * i is the total bytes read from SGs, to be compared to areq->nbytes
          * i is important because we cannot rely on SG length since the sum of
          * SG->length could be greater than areq->nbytes
+         *
+         * end is the position when we need to stop writing to the device,
+         * to be compared to i
+         *
+         * in_i: advancement in the current SG
          */
-
+        unsigned int i = 0, end, fill, min_fill, nwait, nbw = 0, j = 0, todo;
+        unsigned int in_i = 0;
+        u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32] = {0}, wb = 0, v, ivmode = 0;
         struct sun4i_req_ctx *op = ahash_request_ctx(areq);
         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
         struct sun4i_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm);
         struct sun4i_ss_ctx *ss = tfmctx->ss;
-        unsigned int in_i = 0; /* advancement in the current SG */
-        unsigned int end;
-        /*
-         * end is the position when we need to stop writing to the device,
-         * to be compared to i
-         */
+        struct scatterlist *in_sg = areq->src;
+        struct sg_mapping_iter mi;
         int in_r, err = 0;
-        unsigned int todo;
-        u32 spaces, rx_cnt = SS_RX_DEFAULT;
         size_t copied = 0;
-        struct sg_mapping_iter mi;
-        unsigned int j = 0;
-        int zeros;
-        unsigned int index, padlen;
-        __be64 bits;
-        u32 bf[32];
-        u32 wb = 0;
-        unsigned int nwait, nbw = 0;
-        struct scatterlist *in_sg = areq->src;
 
         dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x",
                 __func__, crypto_tfm_alg_name(areq->base.tfm),
                 op->byte_count, areq->nbytes, op->mode,
                 op->len, op->hash[0]);
 
-        if (unlikely(areq->nbytes == 0) && (op->flags & SS_HASH_FINAL) == 0)
+        if (unlikely(!areq->nbytes) && !(op->flags & SS_HASH_FINAL))
                 return 0;
 
         /* protect against overflow */
@@ -213,7 +203,7 @@ static int sun4i_hash(struct ahash_request *areq)
                 return -EINVAL;
         }
 
-        if (op->len + areq->nbytes < 64 && (op->flags & SS_HASH_FINAL) == 0) {
+        if (op->len + areq->nbytes < 64 && !(op->flags & SS_HASH_FINAL)) {
                 /* linearize data to op->buf */
                 copied = sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
                                             op->buf + op->len, areq->nbytes, 0);
@@ -227,7 +217,7 @@ static int sun4i_hash(struct ahash_request *areq)
          * if some data have been processed before,
          * we need to restore the partial hash state
          */
-        if (op->byte_count > 0) {
+        if (op->byte_count) {
                 ivmode = SS_IV_ARBITRARY;
                 for (i = 0; i < 5; i++)
                         writel(op->hash[i], ss->base + SS_IV0 + i * 4);
@@ -235,11 +225,11 @@ static int sun4i_hash(struct ahash_request *areq)
         /* Enable the device */
         writel(op->mode | SS_ENABLED | ivmode, ss->base + SS_CTL);
 
-        if ((op->flags & SS_HASH_UPDATE) == 0)
+        if (!(op->flags & SS_HASH_UPDATE))
                 goto hash_final;
 
         /* start of handling data */
-        if ((op->flags & SS_HASH_FINAL) == 0) {
+        if (!(op->flags & SS_HASH_FINAL)) {
                 end = ((areq->nbytes + op->len) / 64) * 64 - op->len;
 
                 if (end > areq->nbytes || areq->nbytes - end > 63) {
@@ -253,14 +243,14 @@ static int sun4i_hash(struct ahash_request *areq)
                 end = ((areq->nbytes + op->len) / 4) * 4 - op->len;
         }
 
-        /* TODO if SGlen % 4 and op->len == 0 then DMA */
+        /* TODO if SGlen % 4 and !op->len then DMA */
         i = 1;
         while (in_sg && i == 1) {
-                if ((in_sg->length % 4) != 0)
+                if (in_sg->length % 4)
                         i = 0;
                 in_sg = sg_next(in_sg);
         }
-        if (i == 1 && op->len == 0)
+        if (i == 1 && !op->len && areq->nbytes)
                 dev_dbg(ss->dev, "We can DMA\n");
 
         i = 0;
@@ -275,7 +265,7 @@ static int sun4i_hash(struct ahash_request *areq)
                  * - the buffer is already used
                  * - the SG does not have enough byte remaining ( < 4)
                  */
-                if (op->len > 0 || (mi.length - in_i) < 4) {
+                if (op->len || (mi.length - in_i) < 4) {
                         /*
                          * if we have entered here we have two reason to stop
                          * - the buffer is full
@@ -294,7 +284,7 @@ static int sun4i_hash(struct ahash_request *areq)
                                         in_i = 0;
                                 }
                         }
-                        if (op->len > 3 && (op->len % 4) == 0) {
+                        if (op->len > 3 && !(op->len % 4)) {
                                 /* write buf to the device */
                                 writesl(ss->base + SS_RXFIFO, op->buf,
                                         op->len / 4);
@@ -313,7 +303,7 @@ static int sun4i_hash(struct ahash_request *areq)
                         i += todo * 4;
                         in_i += todo * 4;
                         rx_cnt -= todo;
-                        if (rx_cnt == 0) {
+                        if (!rx_cnt) {
                                 spaces = readl(ss->base + SS_FCSR);
                                 rx_cnt = SS_RXFIFO_SPACES(spaces);
                         }
@@ -351,7 +341,7 @@ static int sun4i_hash(struct ahash_request *areq)
          * Now if we have the flag final go to finalize part
          * If not, store the partial hash
          */
-        if ((op->flags & SS_HASH_FINAL) > 0)
+        if (op->flags & SS_HASH_FINAL)
                 goto hash_final;
 
         writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL);
@@ -359,7 +349,7 @@ static int sun4i_hash(struct ahash_request *areq)
         do {
                 v = readl(ss->base + SS_CTL);
                 i++;
-        } while (i < SS_TIMEOUT && (v & SS_DATA_END) > 0);
+        } while (i < SS_TIMEOUT && (v & SS_DATA_END));
         if (unlikely(i >= SS_TIMEOUT)) {
                 dev_err_ratelimited(ss->dev,
                                     "ERROR: hash end timeout %d>%d ctl=%x len=%u\n",
@@ -368,6 +358,15 @@ static int sun4i_hash(struct ahash_request *areq)
                 goto release_ss;
         }
 
+        /*
+         * The datasheet isn't very clear about when to retrieve the digest. The
+         * bit SS_DATA_END is cleared when the engine has processed the data and
+         * when the digest is computed *but* it doesn't mean the digest is
+         * available in the digest registers. Hence the delay to be sure we can
+         * read it.
+         */
+        ndelay(1);
+
         for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++)
                 op->hash[i] = readl(ss->base + SS_MD0 + i * 4);
 
@@ -388,56 +387,50 @@ static int sun4i_hash(struct ahash_request *areq)
 hash_final:
 
         /* write the remaining words of the wait buffer */
-        if (op->len > 0) {
+        if (op->len) {
                 nwait = op->len / 4;
-                if (nwait > 0) {
+                if (nwait) {
                         writesl(ss->base + SS_RXFIFO, op->buf, nwait);
                         op->byte_count += 4 * nwait;
                 }
+
                 nbw = op->len - 4 * nwait;
-                wb = *(u32 *)(op->buf + nwait * 4);
-                wb &= (0xFFFFFFFF >> (4 - nbw) * 8);
+                if (nbw) {
+                        wb = *(u32 *)(op->buf + nwait * 4);
+                        wb &= GENMASK((nbw * 8) - 1, 0);
+
+                        op->byte_count += nbw;
+                }
         }
 
         /* write the remaining bytes of the nbw buffer */
-        if (nbw > 0) {
-                wb |= ((1 << 7) << (nbw * 8));
-                bf[j++] = wb;
-        } else {
-                bf[j++] = 1 << 7;
-        }
+        wb |= ((1 << 7) << (nbw * 8));
+        bf[j++] = wb;
 
         /*
          * number of space to pad to obtain 64o minus 8(size) minus 4 (final 1)
          * I take the operations from other MD5/SHA1 implementations
          */
 
-        /* we have already send 4 more byte of which nbw data */
-        if (op->mode == SS_OP_MD5) {
-                index = (op->byte_count + 4) & 0x3f;
-                op->byte_count += nbw;
-                if (index > 56)
-                        zeros = (120 - index) / 4;
-                else
-                        zeros = (56 - index) / 4;
-        } else {
-                op->byte_count += nbw;
-                index = op->byte_count & 0x3f;
-                padlen = (index < 56) ? (56 - index) : ((64 + 56) - index);
-                zeros = (padlen - 1) / 4;
-        }
+        /* last block size */
+        fill = 64 - (op->byte_count % 64);
+        min_fill = 2 * sizeof(u32) + (nbw ? 0 : sizeof(u32));
+
+        /* if we can't fill all data, jump to the next 64 block */
+        if (fill < min_fill)
+                fill += 64;
 
-        memset(bf + j, 0, 4 * zeros);
-        j += zeros;
+        j += (fill - min_fill) / sizeof(u32);
 
         /* write the length of data */
         if (op->mode == SS_OP_SHA1) {
-                bits = cpu_to_be64(op->byte_count << 3);
-                bf[j++] = bits & 0xffffffff;
-                bf[j++] = (bits >> 32) & 0xffffffff;
+                __be64 bits = cpu_to_be64(op->byte_count << 3);
+                bf[j++] = lower_32_bits(bits);
+                bf[j++] = upper_32_bits(bits);
         } else {
-                bf[j++] = (op->byte_count << 3) & 0xffffffff;
-                bf[j++] = (op->byte_count >> 29) & 0xffffffff;
+                __le64 bits = op->byte_count << 3;
+                bf[j++] = lower_32_bits(bits);
+                bf[j++] = upper_32_bits(bits);
         }
         writesl(ss->base + SS_RXFIFO, bf, j);
 
@@ -453,7 +446,7 @@ hash_final:
         do {
                 v = readl(ss->base + SS_CTL);
                 i++;
-        } while (i < SS_TIMEOUT && (v & SS_DATA_END) > 0);
+        } while (i < SS_TIMEOUT && (v & SS_DATA_END));
         if (unlikely(i >= SS_TIMEOUT)) {
                 dev_err_ratelimited(ss->dev,
                                     "ERROR: hash end timeout %d>%d ctl=%x len=%u\n",
@@ -462,6 +455,15 @@ hash_final:
                 goto release_ss;
         }
 
+        /*
+         * The datasheet isn't very clear about when to retrieve the digest. The
+         * bit SS_DATA_END is cleared when the engine has processed the data and
+         * when the digest is computed *but* it doesn't mean the digest is
+         * available in the digest registers. Hence the delay to be sure we can
+         * read it.
+         */
+        ndelay(1);
+
         /* Get the hash from the device */
         if (op->mode == SS_OP_SHA1) {
                 for (i = 0; i < 5; i++) {
@@ -513,7 +515,7 @@ int sun4i_hash_digest(struct ahash_request *areq)
         struct sun4i_req_ctx *op = ahash_request_ctx(areq);
 
         err = sun4i_hash_init(areq);
-        if (err != 0)
+        if (err)
                 return err;
 
         op->flags = SS_HASH_UPDATE | SS_HASH_FINAL;
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss.h b/drivers/crypto/sunxi-ss/sun4i-ss.h
index f04c0f8cf026..a0e1efc1cb2a 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss.h
+++ b/drivers/crypto/sunxi-ss/sun4i-ss.h
@@ -24,9 +24,11 @@
 #include <linux/interrupt.h>
 #include <linux/delay.h>
 #include <crypto/md5.h>
+#include <crypto/skcipher.h>
 #include <crypto/sha.h>
 #include <crypto/hash.h>
 #include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
 #include <crypto/aes.h>
 #include <crypto/des.h>
 #include <crypto/internal/rng.h>
@@ -140,7 +142,7 @@ struct sun4i_ss_alg_template {
         u32 type;
         u32 mode;
         union {
-                struct crypto_alg crypto;
+                struct skcipher_alg crypto;
                 struct ahash_alg hash;
         } alg;
         struct sun4i_ss_ctx *ss;
@@ -177,25 +179,25 @@ int sun4i_hash_import_md5(struct ahash_request *areq, const void *in);
 int sun4i_hash_export_sha1(struct ahash_request *areq, void *out);
 int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in);
 
-int sun4i_ss_cbc_aes_encrypt(struct ablkcipher_request *areq);
-int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request *areq);
-int sun4i_ss_ecb_aes_encrypt(struct ablkcipher_request *areq);
-int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request *areq);
+int sun4i_ss_cbc_aes_encrypt(struct skcipher_request *areq);
+int sun4i_ss_cbc_aes_decrypt(struct skcipher_request *areq);
+int sun4i_ss_ecb_aes_encrypt(struct skcipher_request *areq);
+int sun4i_ss_ecb_aes_decrypt(struct skcipher_request *areq);
 
-int sun4i_ss_cbc_des_encrypt(struct ablkcipher_request *areq);
-int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request *areq);
-int sun4i_ss_ecb_des_encrypt(struct ablkcipher_request *areq);
-int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request *areq);
+int sun4i_ss_cbc_des_encrypt(struct skcipher_request *areq);
+int sun4i_ss_cbc_des_decrypt(struct skcipher_request *areq);
+int sun4i_ss_ecb_des_encrypt(struct skcipher_request *areq);
+int sun4i_ss_ecb_des_decrypt(struct skcipher_request *areq);
 
-int sun4i_ss_cbc_des3_encrypt(struct ablkcipher_request *areq);
-int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request *areq);
-int sun4i_ss_ecb_des3_encrypt(struct ablkcipher_request *areq);
-int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request *areq);
+int sun4i_ss_cbc_des3_encrypt(struct skcipher_request *areq);
+int sun4i_ss_cbc_des3_decrypt(struct skcipher_request *areq);
+int sun4i_ss_ecb_des3_encrypt(struct skcipher_request *areq);
+int sun4i_ss_ecb_des3_decrypt(struct skcipher_request *areq);
 
 int sun4i_ss_cipher_init(struct crypto_tfm *tfm);
-int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+int sun4i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
                         unsigned int keylen);
-int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
                         unsigned int keylen);
-int sun4i_ss_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
                          unsigned int keylen);
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index 0bba6a19d36a..79791c690858 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -816,7 +816,7 @@ static void talitos_unregister_rng(struct device *dev)
  * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
  */
 #define TALITOS_CRA_PRIORITY_AEAD_HSNA  (TALITOS_CRA_PRIORITY - 1)
-#define TALITOS_MAX_KEY_SIZE            96
+#define TALITOS_MAX_KEY_SIZE            (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 
 struct talitos_ctx {
@@ -1495,6 +1495,11 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
 {
         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
 
+        if (keylen > TALITOS_MAX_KEY_SIZE) {
+                crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+
         memcpy(&ctx->key, key, keylen);
         ctx->keylen = keylen;
 
diff --git a/drivers/crypto/vmx/aes.c b/drivers/crypto/vmx/aes.c
index 022c7ab7351a..96072b9b55c4 100644
--- a/drivers/crypto/vmx/aes.c
+++ b/drivers/crypto/vmx/aes.c
@@ -37,15 +37,10 @@ struct p8_aes_ctx {
 
 static int p8_aes_init(struct crypto_tfm *tfm)
 {
-        const char *alg;
+        const char *alg = crypto_tfm_alg_name(tfm);
         struct crypto_cipher *fallback;
         struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 
-        if (!(alg = crypto_tfm_alg_name(tfm))) {
-                printk(KERN_ERR "Failed to get algorithm name.\n");
-                return -ENOENT;
-        }
-
         fallback = crypto_alloc_cipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
         if (IS_ERR(fallback)) {
                 printk(KERN_ERR
diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c
index 72a26eb4e954..7394d35d5936 100644
--- a/drivers/crypto/vmx/aes_cbc.c
+++ b/drivers/crypto/vmx/aes_cbc.c
@@ -39,15 +39,10 @@ struct p8_aes_cbc_ctx {
 
 static int p8_aes_cbc_init(struct crypto_tfm *tfm)
 {
-        const char *alg;
+        const char *alg = crypto_tfm_alg_name(tfm);
         struct crypto_skcipher *fallback;
         struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
 
-        if (!(alg = crypto_tfm_alg_name(tfm))) {
-                printk(KERN_ERR "Failed to get algorithm name.\n");
-                return -ENOENT;
-        }
-
         fallback = crypto_alloc_skcipher(alg, 0,
                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
 
diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
index 7cf6d31c1123..9c26d9e8dbea 100644
--- a/drivers/crypto/vmx/aes_ctr.c
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -36,15 +36,10 @@ struct p8_aes_ctr_ctx {
 
 static int p8_aes_ctr_init(struct crypto_tfm *tfm)
 {
-        const char *alg;
+        const char *alg = crypto_tfm_alg_name(tfm);
         struct crypto_blkcipher *fallback;
         struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
 
-        if (!(alg = crypto_tfm_alg_name(tfm))) {
-                printk(KERN_ERR "Failed to get algorithm name.\n");
-                return -ENOENT;
-        }
-
         fallback =
             crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
         if (IS_ERR(fallback)) {
diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c
index 6adc9290557a..8cd6e62e4c90 100644
--- a/drivers/crypto/vmx/aes_xts.c
+++ b/drivers/crypto/vmx/aes_xts.c
@@ -41,15 +41,10 @@ struct p8_aes_xts_ctx {
 
 static int p8_aes_xts_init(struct crypto_tfm *tfm)
 {
-        const char *alg;
+        const char *alg = crypto_tfm_alg_name(tfm);
         struct crypto_skcipher *fallback;
         struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 
-        if (!(alg = crypto_tfm_alg_name(tfm))) {
-                printk(KERN_ERR "Failed to get algorithm name.\n");
-                return -ENOENT;
-        }
-
         fallback = crypto_alloc_skcipher(alg, 0,
                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
         if (IS_ERR(fallback)) {
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index e88c1808e46f..b32eb8c7c1e3 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -868,6 +868,7 @@ struct adapter {
 
         /* TC u32 offload */
         struct cxgb4_tc_u32_table *tc_u32;
+        struct chcr_stats_debug chcr_stats;
 };
 
 /* Support for "sched-class" command to allow a TX Scheduling Class to be
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index 1fa34b009891..77a59d7db7f5 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -3069,6 +3069,40 @@ static const struct file_operations meminfo_fops = {
         .llseek  = seq_lseek,
         .release = single_release,
 };
+
+static int chcr_show(struct seq_file *seq, void *v)
+{
+        struct adapter *adap = seq->private;
+
+        seq_puts(seq, "Chelsio Crypto Accelerator Stats \n");
+        seq_printf(seq, "Cipher Ops: %10u \n",
+                   atomic_read(&adap->chcr_stats.cipher_rqst));
+        seq_printf(seq, "Digest Ops: %10u \n",
+                   atomic_read(&adap->chcr_stats.digest_rqst));
+        seq_printf(seq, "Aead Ops: %10u \n",
+                   atomic_read(&adap->chcr_stats.aead_rqst));
+        seq_printf(seq, "Completion: %10u \n",
+                   atomic_read(&adap->chcr_stats.complete));
+        seq_printf(seq, "Error: %10u \n",
+                   atomic_read(&adap->chcr_stats.error));
+        seq_printf(seq, "Fallback: %10u \n",
+                   atomic_read(&adap->chcr_stats.fallback));
+        return 0;
+}
+
+
+static int chcr_stats_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, chcr_show, inode->i_private);
+}
+
+static const struct file_operations chcr_stats_debugfs_fops = {
+        .owner   = THIS_MODULE,
+        .open    = chcr_stats_open,
+        .read    = seq_read,
+        .llseek  = seq_lseek,
+        .release = single_release,
+};
 /* Add an array of Debug FS files.
  */
 void add_debugfs_files(struct adapter *adap,
@@ -3143,6 +3177,7 @@ int t4_setup_debugfs(struct adapter *adap)
                 { "tids", &tid_info_debugfs_fops, S_IRUSR, 0},
                 { "blocked_fl", &blocked_fl_fops, S_IRUSR | S_IWUSR, 0 },
                 { "meminfo", &meminfo_fops, S_IRUSR, 0 },
+                { "crypto", &chcr_stats_debugfs_fops, S_IRUSR, 0 },
         };
 
         /* Debug FS nodes common to all T5 and later adapters.
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
index d0868c2320da..ec53fe9dec68 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
@@ -642,6 +642,7 @@ static void uld_init(struct adapter *adap, struct cxgb4_lld_info *lld)
         lld->sge_ingpadboundary = adap->sge.fl_align;
         lld->sge_egrstatuspagesize = adap->sge.stat_len;
         lld->sge_pktshift = adap->sge.pktshift;
+        lld->ulp_crypto = adap->params.crypto;
         lld->enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
         lld->max_ordird_qp = adap->params.max_ordird_qp;
         lld->max_ird_adapter = adap->params.max_ird_adapter;
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
index 6e74040af49a..8f1c874cfe21 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
@@ -275,6 +275,15 @@ struct cxgb4_virt_res {                      /* virtualized HW resources */
         unsigned int ncrypto_fc;
 };
 
+struct chcr_stats_debug {
+        atomic_t cipher_rqst;
+        atomic_t digest_rqst;
+        atomic_t aead_rqst;
+        atomic_t complete;
+        atomic_t error;
+        atomic_t fallback;
+};
+
 #define OCQ_WIN_OFFSET(pdev, vres) \
         (pci_resource_len((pdev), 2) - roundup_pow_of_two((vres)->ocq.size))
 
@@ -322,6 +331,7 @@ struct cxgb4_lld_info {
         unsigned int iscsi_tagmask;          /* iscsi ddp tag mask */
         unsigned int iscsi_pgsz_order;       /* iscsi ddp page size orders */
         unsigned int iscsi_llimit;           /* chip's iscsi region llimit */
+        unsigned int ulp_crypto;             /* crypto lookaside support */
         void **iscsi_ppm;                    /* iscsi page pod manager */
         int nodeid;                          /* device numa node id */
         bool fr_nsmr_tpte_wr_support;        /* FW supports FR_NSMR_TPTE_WR */
diff --git a/include/crypto/akcipher.h b/include/crypto/akcipher.h
index c37cc59e9bf2..b5e11de4d497 100644
--- a/include/crypto/akcipher.h
+++ b/include/crypto/akcipher.h
@@ -98,7 +98,7 @@ struct akcipher_alg {
                            unsigned int keylen);
         int (*set_priv_key)(struct crypto_akcipher *tfm, const void *key,
                             unsigned int keylen);
-        int (*max_size)(struct crypto_akcipher *tfm);
+        unsigned int (*max_size)(struct crypto_akcipher *tfm);
         int (*init)(struct crypto_akcipher *tfm);
         void (*exit)(struct crypto_akcipher *tfm);
 
@@ -257,13 +257,14 @@ static inline void akcipher_request_set_crypt(struct akcipher_request *req,
 /**
  * crypto_akcipher_maxsize() - Get len for output buffer
  *
- * Function returns the dest buffer size required for a given key
+ * Function returns the dest buffer size required for a given key.
+ * Function assumes that the key is already set in the transformation. If this
+ * function is called without a setkey or with a failed setkey, you will end up
+ * in a NULL dereference.
  *
  * @tfm:        AKCIPHER tfm handle allocated with crypto_alloc_akcipher()
- *
- * Return: minimum len for output buffer or error code in key hasn't been set
  */
-static inline int crypto_akcipher_maxsize(struct crypto_akcipher *tfm)
+static inline unsigned int crypto_akcipher_maxsize(struct crypto_akcipher *tfm)
 {
         struct akcipher_alg *alg = crypto_akcipher_alg(tfm);
 
diff --git a/include/crypto/dh.h b/include/crypto/dh.h
index 6b424ad3482e..f638998fb6d0 100644
--- a/include/crypto/dh.h
+++ b/include/crypto/dh.h
@@ -73,9 +73,9 @@ int crypto_dh_encode_key(char *buf, unsigned int len, const struct dh *params);
 /**
  * crypto_dh_decode_key() - decode a private key
  * @buf:        Buffer holding a packet key that should be decoded
- * @len:        Lenth of the packet private key buffer
+ * @len:        Length of the packet private key buffer
  * @params:     Buffer allocated by the caller that is filled with the
- *              unpacket DH private key.
+ *              unpacked DH private key.
  *
  * The unpacking obtains the private key by pointing @p to the correct location
  * in @buf. Thus, both pointers refer to the same memory.
diff --git a/include/crypto/ecdh.h b/include/crypto/ecdh.h
index 03a64f62ba7a..1aff2a8a3a68 100644
--- a/include/crypto/ecdh.h
+++ b/include/crypto/ecdh.h
@@ -74,9 +74,9 @@ int crypto_ecdh_encode_key(char *buf, unsigned int len, const struct ecdh *p);
 /**
  * crypto_ecdh_decode_key() - decode a private key
  * @buf:        Buffer holding a packet key that should be decoded
- * @len:        Lenth of the packet private key buffer
+ * @len:        Length of the packet private key buffer
  * @p:          Buffer allocated by the caller that is filled with the
- *              unpacket ECDH private key.
+ *              unpacked ECDH private key.
  *
  * The unpacking obtains the private key by pointing @p to the correct location
  * in @buf. Thus, both pointers refer to the same memory.
diff --git a/include/crypto/engine.h b/include/crypto/engine.h
index 1bf600fc99f7..dd04c1699b51 100644
--- a/include/crypto/engine.h
+++ b/include/crypto/engine.h
@@ -58,6 +58,7 @@ struct crypto_engine {
         struct list_head        list;
         spinlock_t              queue_lock;
         struct crypto_queue     queue;
+        struct device           *dev;
 
         bool                    rt;
 
diff --git a/include/crypto/hmac.h b/include/crypto/hmac.h
new file mode 100644
index 000000000000..ef09f7938204
--- /dev/null
+++ b/include/crypto/hmac.h
@@ -0,0 +1,7 @@
+#ifndef _CRYPTO_HMAC_H
+#define _CRYPTO_HMAC_H
+
+#define HMAC_IPAD_VALUE 0x36
+#define HMAC_OPAD_VALUE 0x5c
+
+#endif /* _CRYPTO_HMAC_H */
diff --git a/include/crypto/kpp.h b/include/crypto/kpp.h
index ce8e1f79374b..2133d17b7156 100644
--- a/include/crypto/kpp.h
+++ b/include/crypto/kpp.h
@@ -53,7 +53,7 @@ struct crypto_kpp {
  *
  * @set_secret:         Function invokes the protocol specific function to
  *                      store the secret private key along with parameters.
- *                      The implementation knows how to decode thie buffer
+ *                      The implementation knows how to decode the buffer
  * @generate_public_key: Function generate the public key to be sent to the
  *                      counterpart. In case of error, where output is not big
  *                      enough req->dst_len will be updated to the size
@@ -79,7 +79,7 @@ struct kpp_alg {
         int (*generate_public_key)(struct kpp_request *req);
         int (*compute_shared_secret)(struct kpp_request *req);
 
-        int (*max_size)(struct crypto_kpp *tfm);
+        unsigned int (*max_size)(struct crypto_kpp *tfm);
 
         int (*init)(struct crypto_kpp *tfm);
         void (*exit)(struct crypto_kpp *tfm);
@@ -102,7 +102,7 @@ struct kpp_alg {
  * @mask: specifies the mask for the algorithm
  *
  * Allocate a handle for kpp algorithm. The returned struct crypto_kpp
- * is requeried for any following API invocation
+ * is required for any following API invocation
  *
  * Return: allocated handle in case of success; IS_ERR() is true in case of
  *         an error, PTR_ERR() returns the error code.
@@ -323,13 +323,14 @@ static inline int crypto_kpp_compute_shared_secret(struct kpp_request *req)
 /**
  * crypto_kpp_maxsize() - Get len for output buffer
  *
- * Function returns the output buffer size required
+ * Function returns the output buffer size required for a given key.
+ * Function assumes that the key is already set in the transformation. If this
+ * function is called without a setkey or with a failed setkey, you will end up
+ * in a NULL dereference.
  *
  * @tfm:        KPP tfm handle allocated with crypto_alloc_kpp()
- *
- * Return: minimum len for output buffer or error code if key hasn't been set
  */
-static inline int crypto_kpp_maxsize(struct crypto_kpp *tfm)
+static inline unsigned int crypto_kpp_maxsize(struct crypto_kpp *tfm)
 {
         struct kpp_alg *alg = crypto_kpp_alg(tfm);
 
diff --git a/include/linux/timeriomem-rng.h b/include/linux/timeriomem-rng.h
index 46eb27ddbfab..3e00122bcf88 100644
--- a/include/linux/timeriomem-rng.h
+++ b/include/linux/timeriomem-rng.h
@@ -13,4 +13,7 @@ struct timeriomem_rng_data {
 
         /* measures in usecs */
         unsigned int            period;
+
+        /* bits of entropy per 1024 bits read */
+        unsigned int            quality;
 };