Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Pull crypto updates from Herbert Xu:
 "Here is the crypto update for 5.3:

  API:
   - Test shash interface directly in testmgr
   - cra_driver_name is now mandatory

  Algorithms:
   - Replace arc4 crypto_cipher with library helper
   - Implement 5 way interleave for ECB, CBC and CTR on arm64
   - Add xxhash
   - Add continuous self-test on noise source to drbg
   - Update jitter RNG

  Drivers:
   - Add support for SHA204A random number generator
   - Add support for 7211 in iproc-rng200
   - Fix fuzz test failures in inside-secure
   - Fix fuzz test failures in talitos
   - Fix fuzz test failures in qat"

* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (143 commits)
  crypto: stm32/hash - remove interruptible condition for dma
  crypto: stm32/hash - Fix hmac issue more than 256 bytes
  crypto: stm32/crc32 - rename driver file
  crypto: amcc - remove memset after dma_alloc_coherent
  crypto: ccp - Switch to SPDX license identifiers
  crypto: ccp - Validate the the error value used to index error messages
  crypto: doc - Fix formatting of new crypto engine content
  crypto: doc - Add parameter documentation
  crypto: arm64/aes-ce - implement 5 way interleave for ECB, CBC and CTR
  crypto: arm64/aes-ce - add 5 way interleave routines
  crypto: talitos - drop icv_ool
  crypto: talitos - fix hash on SEC1.
  crypto: talitos - move struct talitos_edesc into talitos.h
  lib/scatterlist: Fix mapping iterator when sg->offset is greater than PAGE_SIZE
  crypto/NX: Set receive window credits to max number of CRBs in RxFIFO
  crypto: asymmetric_keys - select CRYPTO_HASH where needed
  crypto: serpent - mark __serpent_setkey_sbox noinline
  crypto: testmgr - dynamically allocate crypto_shash
  crypto: testmgr - dynamically allocate testvec_config
  crypto: talitos - eliminate unneeded 'done' functions at build time
  ...

168 files changed, 4527 insertions, 3798 deletions

diff --git a/Documentation/crypto/api-samples.rst b/Documentation/crypto/api-samples.rst
index f14afaaf2f32..e923f17bc2bd 100644
--- a/Documentation/crypto/api-samples.rst
+++ b/Documentation/crypto/api-samples.rst
@@ -4,111 +4,89 @@ Code Examples
 Code Example For Symmetric Key Cipher Operation
 -----------------------------------------------
 
-::
-
-
-    /* tie all data structures together */
-    struct skcipher_def {
-        struct scatterlist sg;
-        struct crypto_skcipher *tfm;
-        struct skcipher_request *req;
-        struct crypto_wait wait;
-    };
-
-    /* Perform cipher operation */
-    static unsigned int test_skcipher_encdec(struct skcipher_def *sk,
-                         int enc)
-    {
-        int rc;
-
-        if (enc)
-            rc = crypto_wait_req(crypto_skcipher_encrypt(sk->req), &sk->wait);
-        else
-            rc = crypto_wait_req(crypto_skcipher_decrypt(sk->req), &sk->wait);
-
-        if (rc)
-                pr_info("skcipher encrypt returned with result %d\n", rc);
+This code encrypts some data with AES-256-XTS.  For sake of example,
+all inputs are random bytes, the encryption is done in-place, and it's
+assumed the code is running in a context where it can sleep.
 
-        return rc;
-    }
+::
 
-    /* Initialize and trigger cipher operation */
     static int test_skcipher(void)
     {
-        struct skcipher_def sk;
-        struct crypto_skcipher *skcipher = NULL;
-        struct skcipher_request *req = NULL;
-        char *scratchpad = NULL;
-        char *ivdata = NULL;
-        unsigned char key[32];
-        int ret = -EFAULT;
-
-        skcipher = crypto_alloc_skcipher("cbc-aes-aesni", 0, 0);
-        if (IS_ERR(skcipher)) {
-            pr_info("could not allocate skcipher handle\n");
-            return PTR_ERR(skcipher);
-        }
-
-        req = skcipher_request_alloc(skcipher, GFP_KERNEL);
-        if (!req) {
-            pr_info("could not allocate skcipher request\n");
-            ret = -ENOMEM;
-            goto out;
-        }
-
-        skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
-                          crypto_req_done,
-                          &sk.wait);
-
-        /* AES 256 with random key */
-        get_random_bytes(&key, 32);
-        if (crypto_skcipher_setkey(skcipher, key, 32)) {
-            pr_info("key could not be set\n");
-            ret = -EAGAIN;
-            goto out;
-        }
-
-        /* IV will be random */
-        ivdata = kmalloc(16, GFP_KERNEL);
-        if (!ivdata) {
-            pr_info("could not allocate ivdata\n");
-            goto out;
-        }
-        get_random_bytes(ivdata, 16);
-
-        /* Input data will be random */
-        scratchpad = kmalloc(16, GFP_KERNEL);
-        if (!scratchpad) {
-            pr_info("could not allocate scratchpad\n");
-            goto out;
-        }
-        get_random_bytes(scratchpad, 16);
-
-        sk.tfm = skcipher;
-        sk.req = req;
-
-        /* We encrypt one block */
-        sg_init_one(&sk.sg, scratchpad, 16);
-        skcipher_request_set_crypt(req, &sk.sg, &sk.sg, 16, ivdata);
-        crypto_init_wait(&sk.wait);
-
-        /* encrypt data */
-        ret = test_skcipher_encdec(&sk, 1);
-        if (ret)
-            goto out;
-
-        pr_info("Encryption triggered successfully\n");
-
+            struct crypto_skcipher *tfm = NULL;
+            struct skcipher_request *req = NULL;
+            u8 *data = NULL;
+            const size_t datasize = 512; /* data size in bytes */
+            struct scatterlist sg;
+            DECLARE_CRYPTO_WAIT(wait);
+            u8 iv[16];  /* AES-256-XTS takes a 16-byte IV */
+            u8 key[64]; /* AES-256-XTS takes a 64-byte key */
+            int err;
+
+            /*
+             * Allocate a tfm (a transformation object) and set the key.
+             *
+             * In real-world use, a tfm and key are typically used for many
+             * encryption/decryption operations.  But in this example, we'll just do a
+             * single encryption operation with it (which is not very efficient).
+             */
+
+            tfm = crypto_alloc_skcipher("xts(aes)", 0, 0);
+            if (IS_ERR(tfm)) {
+                    pr_err("Error allocating xts(aes) handle: %ld\n", PTR_ERR(tfm));
+                    return PTR_ERR(tfm);
+            }
+
+            get_random_bytes(key, sizeof(key));
+            err = crypto_skcipher_setkey(tfm, key, sizeof(key));
+            if (err) {
+                    pr_err("Error setting key: %d\n", err);
+                    goto out;
+            }
+
+            /* Allocate a request object */
+            req = skcipher_request_alloc(tfm, GFP_KERNEL);
+            if (!req) {
+                    err = -ENOMEM;
+                    goto out;
+            }
+
+            /* Prepare the input data */
+            data = kmalloc(datasize, GFP_KERNEL);
+            if (!data) {
+                    err = -ENOMEM;
+                    goto out;
+            }
+            get_random_bytes(data, datasize);
+
+            /* Initialize the IV */
+            get_random_bytes(iv, sizeof(iv));
+
+            /*
+             * Encrypt the data in-place.
+             *
+             * For simplicity, in this example we wait for the request to complete
+             * before proceeding, even if the underlying implementation is asynchronous.
+             *
+             * To decrypt instead of encrypt, just change crypto_skcipher_encrypt() to
+             * crypto_skcipher_decrypt().
+             */
+            sg_init_one(&sg, data, datasize);
+            skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
+                                               CRYPTO_TFM_REQ_MAY_SLEEP,
+                                          crypto_req_done, &wait);
+            skcipher_request_set_crypt(req, &sg, &sg, datasize, iv);
+            err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
+            if (err) {
+                    pr_err("Error encrypting data: %d\n", err);
+                    goto out;
+            }
+
+            pr_debug("Encryption was successful\n");
     out:
-        if (skcipher)
-            crypto_free_skcipher(skcipher);
-        if (req)
+            crypto_free_skcipher(tfm);
             skcipher_request_free(req);
-        if (ivdata)
-            kfree(ivdata);
-        if (scratchpad)
-            kfree(scratchpad);
-        return ret;
+            kfree(data);
+            return err;
     }
 
 
diff --git a/Documentation/crypto/api-skcipher.rst b/Documentation/crypto/api-skcipher.rst
index 4eec4a93f7e3..20ba08dddf2e 100644
--- a/Documentation/crypto/api-skcipher.rst
+++ b/Documentation/crypto/api-skcipher.rst
@@ -5,7 +5,7 @@ Block Cipher Algorithm Definitions
    :doc: Block Cipher Algorithm Definitions
 
 .. kernel-doc:: include/linux/crypto.h
-   :functions: crypto_alg ablkcipher_alg blkcipher_alg cipher_alg
+   :functions: crypto_alg ablkcipher_alg blkcipher_alg cipher_alg compress_alg
 
 Symmetric Key Cipher API
 ------------------------
diff --git a/Documentation/crypto/architecture.rst b/Documentation/crypto/architecture.rst
index ee8ff0762d7f..3eae1ae7f798 100644
--- a/Documentation/crypto/architecture.rst
+++ b/Documentation/crypto/architecture.rst
@@ -208,9 +208,7 @@ the aforementioned cipher types:
 -  CRYPTO_ALG_TYPE_KPP Key-agreement Protocol Primitive (KPP) such as
    an ECDH or DH implementation
 
--  CRYPTO_ALG_TYPE_DIGEST Raw message digest
-
--  CRYPTO_ALG_TYPE_HASH Alias for CRYPTO_ALG_TYPE_DIGEST
+-  CRYPTO_ALG_TYPE_HASH Raw message digest
 
 -  CRYPTO_ALG_TYPE_SHASH Synchronous multi-block hash
 
diff --git a/Documentation/crypto/crypto_engine.rst b/Documentation/crypto/crypto_engine.rst
index 1d56221dfe35..236c674d6897 100644
--- a/Documentation/crypto/crypto_engine.rst
+++ b/Documentation/crypto/crypto_engine.rst
@@ -1,50 +1,85 @@
-=============
-CRYPTO ENGINE
+.. SPDX-License-Identifier: GPL-2.0
+Crypto Engine
 =============
 
 Overview
 --------
-The crypto engine API (CE), is a crypto queue manager.
+The crypto engine (CE) API is a crypto queue manager.
 
 Requirement
 -----------
-You have to put at start of your tfm_ctx the struct crypto_engine_ctx::
+You must put, at the start of your transform context your_tfm_ctx, the structure
+crypto_engine:
+
+::
 
-  struct your_tfm_ctx {
-        struct crypto_engine_ctx enginectx;
-        ...
-  };
+        struct your_tfm_ctx {
+                struct crypto_engine engine;
+                ...
+        };
 
-Why: Since CE manage only crypto_async_request, it cannot know the underlying
-request_type and so have access only on the TFM.
-So using container_of for accessing __ctx is impossible.
-Furthermore, the crypto engine cannot know the "struct your_tfm_ctx",
-so it must assume that crypto_engine_ctx is at start of it.
+The crypto engine only manages asynchronous requests in the form of
+crypto_async_request. It cannot know the underlying request type and thus only
+has access to the transform structure. It is not possible to access the context
+using container_of. In addition, the engine knows nothing about your
+structure "``struct your_tfm_ctx``". The engine assumes (requires) the placement
+of the known member ``struct crypto_engine`` at the beginning.
 
 Order of operations
 -------------------
-You have to obtain a struct crypto_engine via crypto_engine_alloc_init().
-And start it via crypto_engine_start().
-
-Before transferring any request, you have to fill the enginectx.
-- prepare_request: (taking a function pointer) If you need to do some processing before doing the request
-- unprepare_request: (taking a function pointer) Undoing what's done in prepare_request
-- do_one_request: (taking a function pointer) Do encryption for current request
-
-Note: that those three functions get the crypto_async_request associated with the received request.
-So your need to get the original request via container_of(areq, struct yourrequesttype_request, base);
-
-When your driver receive a crypto_request, you have to transfer it to
-the cryptoengine via one of:
-- crypto_transfer_ablkcipher_request_to_engine()
-- crypto_transfer_aead_request_to_engine()
-- crypto_transfer_akcipher_request_to_engine()
-- crypto_transfer_hash_request_to_engine()
-- crypto_transfer_skcipher_request_to_engine()
-
-At the end of the request process, a call to one of the following function is needed:
-- crypto_finalize_ablkcipher_request
-- crypto_finalize_aead_request
-- crypto_finalize_akcipher_request
-- crypto_finalize_hash_request
-- crypto_finalize_skcipher_request
+You are required to obtain a struct crypto_engine via ``crypto_engine_alloc_init()``.
+Start it via ``crypto_engine_start()``. When finished with your work, shut down the
+engine using ``crypto_engine_stop()`` and destroy the engine with
+``crypto_engine_exit()``.
+
+Before transferring any request, you have to fill the context enginectx by
+providing functions for the following:
+
+* ``prepare_crypt_hardware``: Called once before any prepare functions are
+  called.
+
+* ``unprepare_crypt_hardware``: Called once after all unprepare functions have
+  been called.
+
+* ``prepare_cipher_request``/``prepare_hash_request``: Called before each
+  corresponding request is performed. If some processing or other preparatory
+  work is required, do it here.
+
+* ``unprepare_cipher_request``/``unprepare_hash_request``: Called after each
+  request is handled. Clean up / undo what was done in the prepare function.
+
+* ``cipher_one_request``/``hash_one_request``: Handle the current request by
+  performing the operation.
+
+Note that these functions access the crypto_async_request structure
+associated with the received request. You are able to retrieve the original
+request by using:
+
+::
+
+        container_of(areq, struct yourrequesttype_request, base);
+
+When your driver receives a crypto_request, you must to transfer it to
+the crypto engine via one of:
+
+* crypto_transfer_ablkcipher_request_to_engine()
+
+* crypto_transfer_aead_request_to_engine()
+
+* crypto_transfer_akcipher_request_to_engine()
+
+* crypto_transfer_hash_request_to_engine()
+
+* crypto_transfer_skcipher_request_to_engine()
+
+At the end of the request process, a call to one of the following functions is needed:
+
+* crypto_finalize_ablkcipher_request()
+
+* crypto_finalize_aead_request()
+
+* crypto_finalize_akcipher_request()
+
+* crypto_finalize_hash_request()
+
+* crypto_finalize_skcipher_request()
diff --git a/Documentation/devicetree/bindings/crypto/atmel-crypto.txt b/Documentation/devicetree/bindings/crypto/atmel-crypto.txt
index 6b458bb2440d..f2aab3dc2b52 100644
--- a/Documentation/devicetree/bindings/crypto/atmel-crypto.txt
+++ b/Documentation/devicetree/bindings/crypto/atmel-crypto.txt
@@ -66,16 +66,3 @@ sha@f8034000 {
         dmas = <&dma1 2 17>;
         dma-names = "tx";
 };
-
-* Eliptic Curve Cryptography (I2C)
-
-Required properties:
-- compatible : must be "atmel,atecc508a".
-- reg: I2C bus address of the device.
-- clock-frequency: must be present in the i2c controller node.
-
-Example:
-atecc508a@c0 {
-        compatible = "atmel,atecc508a";
-        reg = <0xC0>;
-};
diff --git a/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt b/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt
index 0014da9145af..c223e54452da 100644
--- a/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt
+++ b/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt
@@ -2,6 +2,7 @@ HWRNG support for the iproc-rng200 driver
 
 Required properties:
 - compatible : Must be one of:
+               "brcm,bcm7211-rng200"
                "brcm,bcm7278-rng200"
                "brcm,iproc-rng200"
 - reg : base address and size of control register block
diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml
index 747fd3f689dc..2e742d399e87 100644
--- a/Documentation/devicetree/bindings/trivial-devices.yaml
+++ b/Documentation/devicetree/bindings/trivial-devices.yaml
@@ -52,6 +52,10 @@ properties:
           - at,24c08
             # i2c trusted platform module (TPM)
           - atmel,at97sc3204t
+            # i2c h/w symmetric crypto module
+          - atmel,atsha204a
+            # i2c h/w elliptic curve crypto module
+          - atmel,atecc508a
             # CM32181: Ambient Light Sensor
           - capella,cm32181
             # CM3232: Ambient Light Sensor
diff --git a/MAINTAINERS b/MAINTAINERS
index a2b97d38a704..4a9e8e5b2432 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -4257,6 +4257,7 @@ F:	crypto/
 F:      drivers/crypto/
 F:      include/crypto/
 F:      include/linux/crypto*
+F:      lib/crypto/
 
 CRYPTOGRAPHIC RANDOM NUMBER GENERATOR
 M:      Neil Horman <nhorman@tuxdriver.com>
diff --git a/arch/arm/boot/dts/imx7ulp.dtsi b/arch/arm/boot/dts/imx7ulp.dtsi
index d6b711011cba..e20483714be5 100644
--- a/arch/arm/boot/dts/imx7ulp.dtsi
+++ b/arch/arm/boot/dts/imx7ulp.dtsi
@@ -100,6 +100,29 @@
                 reg = <0x40000000 0x800000>;
                 ranges;
 
+                crypto: crypto@40240000 {
+                        compatible = "fsl,sec-v4.0";
+                        #address-cells = <1>;
+                        #size-cells = <1>;
+                        reg = <0x40240000 0x10000>;
+                        ranges = <0 0x40240000 0x10000>;
+                        clocks = <&pcc2 IMX7ULP_CLK_CAAM>,
+                                 <&scg1 IMX7ULP_CLK_NIC1_BUS_DIV>;
+                        clock-names = "aclk", "ipg";
+
+                        sec_jr0: jr0@1000 {
+                                compatible = "fsl,sec-v4.0-job-ring";
+                                reg = <0x1000 0x1000>;
+                                interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>;
+                        };
+
+                        sec_jr1: jr1@2000 {
+                                compatible = "fsl,sec-v4.0-job-ring";
+                                reg = <0x2000 0x1000>;
+                                interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>;
+                        };
+                };
+
                 lpuart4: serial@402d0000 {
                         compatible = "fsl,imx7ulp-lpuart";
                         reg = <0x402d0000 0x1000>;
diff --git a/arch/arm/crypto/chacha-neon-glue.c b/arch/arm/crypto/chacha-neon-glue.c
index 48a89537b828..a8e9b534c8da 100644
--- a/arch/arm/crypto/chacha-neon-glue.c
+++ b/arch/arm/crypto/chacha-neon-glue.c
@@ -63,7 +63,7 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
 }
 
 static int chacha_neon_stream_xor(struct skcipher_request *req,
-                                  struct chacha_ctx *ctx, u8 *iv)
+                                  const struct chacha_ctx *ctx, const u8 *iv)
 {
         struct skcipher_walk walk;
         u32 state[16];
diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c
index 232eeab1ec37..8775aa42bbbe 100644
--- a/arch/arm/crypto/sha512-glue.c
+++ b/arch/arm/crypto/sha512-glue.c
@@ -34,7 +34,7 @@ int sha512_arm_update(struct shash_desc *desc, const u8 *data,
                 (sha512_block_fn *)sha512_block_data_order);
 }
 
-int sha512_arm_final(struct shash_desc *desc, u8 *out)
+static int sha512_arm_final(struct shash_desc *desc, u8 *out)
 {
         sha512_base_do_finalize(desc,
                 (sha512_block_fn *)sha512_block_data_order);
diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S
index 3ebfaec97e27..00bd2885feaa 100644
--- a/arch/arm64/crypto/aes-ce.S
+++ b/arch/arm64/crypto/aes-ce.S
@@ -15,6 +15,8 @@
         .arch           armv8-a+crypto
 
         xtsmask         .req    v16
+        cbciv           .req    v16
+        vctr            .req    v16
 
         .macro          xts_reload_mask, tmp
         .endm
@@ -49,7 +51,7 @@
         load_round_keys \rounds, \temp
         .endm
 
-        .macro          do_enc_Nx, de, mc, k, i0, i1, i2, i3
+        .macro          do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4
         aes\de          \i0\().16b, \k\().16b
         aes\mc          \i0\().16b, \i0\().16b
         .ifnb           \i1
@@ -60,27 +62,34 @@
         aes\mc          \i2\().16b, \i2\().16b
         aes\de          \i3\().16b, \k\().16b
         aes\mc          \i3\().16b, \i3\().16b
+        .ifnb           \i4
+        aes\de          \i4\().16b, \k\().16b
+        aes\mc          \i4\().16b, \i4\().16b
+        .endif
         .endif
         .endif
         .endm
 
-        /* up to 4 interleaved encryption rounds with the same round key */
-        .macro          round_Nx, enc, k, i0, i1, i2, i3
+        /* up to 5 interleaved encryption rounds with the same round key */
+        .macro          round_Nx, enc, k, i0, i1, i2, i3, i4
         .ifc            \enc, e
-        do_enc_Nx       e, mc, \k, \i0, \i1, \i2, \i3
+        do_enc_Nx       e, mc, \k, \i0, \i1, \i2, \i3, \i4
         .else
-        do_enc_Nx       d, imc, \k, \i0, \i1, \i2, \i3
+        do_enc_Nx       d, imc, \k, \i0, \i1, \i2, \i3, \i4
         .endif
         .endm
 
-        /* up to 4 interleaved final rounds */
-        .macro          fin_round_Nx, de, k, k2, i0, i1, i2, i3
+        /* up to 5 interleaved final rounds */
+        .macro          fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4
         aes\de          \i0\().16b, \k\().16b
         .ifnb           \i1
         aes\de          \i1\().16b, \k\().16b
         .ifnb           \i3
         aes\de          \i2\().16b, \k\().16b
         aes\de          \i3\().16b, \k\().16b
+        .ifnb           \i4
+        aes\de          \i4\().16b, \k\().16b
+        .endif
         .endif
         .endif
         eor             \i0\().16b, \i0\().16b, \k2\().16b
@@ -89,47 +98,52 @@
         .ifnb           \i3
         eor             \i2\().16b, \i2\().16b, \k2\().16b
         eor             \i3\().16b, \i3\().16b, \k2\().16b
+        .ifnb           \i4
+        eor             \i4\().16b, \i4\().16b, \k2\().16b
+        .endif
         .endif
         .endif
         .endm
 
-        /* up to 4 interleaved blocks */
-        .macro          do_block_Nx, enc, rounds, i0, i1, i2, i3
+        /* up to 5 interleaved blocks */
+        .macro          do_block_Nx, enc, rounds, i0, i1, i2, i3, i4
         cmp             \rounds, #12
         blo             2222f           /* 128 bits */
         beq             1111f           /* 192 bits */
-        round_Nx        \enc, v17, \i0, \i1, \i2, \i3
-        round_Nx        \enc, v18, \i0, \i1, \i2, \i3
-1111:   round_Nx        \enc, v19, \i0, \i1, \i2, \i3
-        round_Nx        \enc, v20, \i0, \i1, \i2, \i3
+        round_Nx        \enc, v17, \i0, \i1, \i2, \i3, \i4
+        round_Nx        \enc, v18, \i0, \i1, \i2, \i3, \i4
+1111:   round_Nx        \enc, v19, \i0, \i1, \i2, \i3, \i4
+        round_Nx        \enc, v20, \i0, \i1, \i2, \i3, \i4
 2222:   .irp            key, v21, v22, v23, v24, v25, v26, v27, v28, v29
-        round_Nx        \enc, \key, \i0, \i1, \i2, \i3
+        round_Nx        \enc, \key, \i0, \i1, \i2, \i3, \i4
         .endr
-        fin_round_Nx    \enc, v30, v31, \i0, \i1, \i2, \i3
+        fin_round_Nx    \enc, v30, v31, \i0, \i1, \i2, \i3, \i4
         .endm
 
         .macro          encrypt_block, in, rounds, t0, t1, t2
         do_block_Nx     e, \rounds, \in
         .endm
 
-        .macro          encrypt_block2x, i0, i1, rounds, t0, t1, t2
-        do_block_Nx     e, \rounds, \i0, \i1
-        .endm
-
         .macro          encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
         do_block_Nx     e, \rounds, \i0, \i1, \i2, \i3
         .endm
 
-        .macro          decrypt_block, in, rounds, t0, t1, t2
-        do_block_Nx     d, \rounds, \in
+        .macro          encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
+        do_block_Nx     e, \rounds, \i0, \i1, \i2, \i3, \i4
         .endm
 
-        .macro          decrypt_block2x, i0, i1, rounds, t0, t1, t2
-        do_block_Nx     d, \rounds, \i0, \i1
+        .macro          decrypt_block, in, rounds, t0, t1, t2
+        do_block_Nx     d, \rounds, \in
         .endm
 
         .macro          decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
         do_block_Nx     d, \rounds, \i0, \i1, \i2, \i3
         .endm
 
+        .macro          decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2
+        do_block_Nx     d, \rounds, \i0, \i1, \i2, \i3, \i4
+        .endm
+
+#define MAX_STRIDE      5
+
 #include "aes-modes.S"
diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S
index 2883def14be5..324039b72094 100644
--- a/arch/arm64/crypto/aes-modes.S
+++ b/arch/arm64/crypto/aes-modes.S
@@ -10,6 +10,18 @@
         .text
         .align          4
 
+#ifndef MAX_STRIDE
+#define MAX_STRIDE      4
+#endif
+
+#if MAX_STRIDE == 4
+#define ST4(x...) x
+#define ST5(x...)
+#else
+#define ST4(x...)
+#define ST5(x...) x
+#endif
+
 aes_encrypt_block4x:
         encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7
         ret
@@ -20,6 +32,18 @@ aes_decrypt_block4x:
         ret
 ENDPROC(aes_decrypt_block4x)
 
+#if MAX_STRIDE == 5
+aes_encrypt_block5x:
+        encrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7
+        ret
+ENDPROC(aes_encrypt_block5x)
+
+aes_decrypt_block5x:
+        decrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7
+        ret
+ENDPROC(aes_decrypt_block5x)
+#endif
+
         /*
          * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
          *                 int blocks)
@@ -34,14 +58,17 @@ AES_ENTRY(aes_ecb_encrypt)
         enc_prepare     w3, x2, x5
 
 .LecbencloopNx:
-        subs            w4, w4, #4
+        subs            w4, w4, #MAX_STRIDE
         bmi             .Lecbenc1x
         ld1             {v0.16b-v3.16b}, [x1], #64      /* get 4 pt blocks */
-        bl              aes_encrypt_block4x
+ST4(    bl              aes_encrypt_block4x             )
+ST5(    ld1             {v4.16b}, [x1], #16             )
+ST5(    bl              aes_encrypt_block5x             )
         st1             {v0.16b-v3.16b}, [x0], #64
+ST5(    st1             {v4.16b}, [x0], #16             )
         b               .LecbencloopNx
 .Lecbenc1x:
-        adds            w4, w4, #4
+        adds            w4, w4, #MAX_STRIDE
         beq             .Lecbencout
 .Lecbencloop:
         ld1             {v0.16b}, [x1], #16             /* get next pt block */
@@ -62,14 +89,17 @@ AES_ENTRY(aes_ecb_decrypt)
         dec_prepare     w3, x2, x5
 
 .LecbdecloopNx:
-        subs            w4, w4, #4
+        subs            w4, w4, #MAX_STRIDE
         bmi             .Lecbdec1x
         ld1             {v0.16b-v3.16b}, [x1], #64      /* get 4 ct blocks */
-        bl              aes_decrypt_block4x
+ST4(    bl              aes_decrypt_block4x             )
+ST5(    ld1             {v4.16b}, [x1], #16             )
+ST5(    bl              aes_decrypt_block5x             )
         st1             {v0.16b-v3.16b}, [x0], #64
+ST5(    st1             {v4.16b}, [x0], #16             )
         b               .LecbdecloopNx
 .Lecbdec1x:
-        adds            w4, w4, #4
+        adds            w4, w4, #MAX_STRIDE
         beq             .Lecbdecout
 .Lecbdecloop:
         ld1             {v0.16b}, [x1], #16             /* get next ct block */
@@ -129,39 +159,56 @@ AES_ENTRY(aes_cbc_decrypt)
         stp             x29, x30, [sp, #-16]!
         mov             x29, sp
 
-        ld1             {v7.16b}, [x5]                  /* get iv */
+        ld1             {cbciv.16b}, [x5]               /* get iv */
         dec_prepare     w3, x2, x6
 
 .LcbcdecloopNx:
-        subs            w4, w4, #4
+        subs            w4, w4, #MAX_STRIDE
         bmi             .Lcbcdec1x
         ld1             {v0.16b-v3.16b}, [x1], #64      /* get 4 ct blocks */
+#if MAX_STRIDE == 5
+        ld1             {v4.16b}, [x1], #16             /* get 1 ct block */
+        mov             v5.16b, v0.16b
+        mov             v6.16b, v1.16b
+        mov             v7.16b, v2.16b
+        bl              aes_decrypt_block5x
+        sub             x1, x1, #32
+        eor             v0.16b, v0.16b, cbciv.16b
+        eor             v1.16b, v1.16b, v5.16b
+        ld1             {v5.16b}, [x1], #16             /* reload 1 ct block */
+        ld1             {cbciv.16b}, [x1], #16          /* reload 1 ct block */
+        eor             v2.16b, v2.16b, v6.16b
+        eor             v3.16b, v3.16b, v7.16b
+        eor             v4.16b, v4.16b, v5.16b
+#else
         mov             v4.16b, v0.16b
         mov             v5.16b, v1.16b
         mov             v6.16b, v2.16b
         bl              aes_decrypt_block4x
         sub             x1, x1, #16
-        eor             v0.16b, v0.16b, v7.16b
+        eor             v0.16b, v0.16b, cbciv.16b
         eor             v1.16b, v1.16b, v4.16b
-        ld1             {v7.16b}, [x1], #16             /* reload 1 ct block */
+        ld1             {cbciv.16b}, [x1], #16          /* reload 1 ct block */
         eor             v2.16b, v2.16b, v5.16b
         eor             v3.16b, v3.16b, v6.16b
+#endif
         st1             {v0.16b-v3.16b}, [x0], #64
+ST5(    st1             {v4.16b}, [x0], #16             )
         b               .LcbcdecloopNx
 .Lcbcdec1x:
-        adds            w4, w4, #4
+        adds            w4, w4, #MAX_STRIDE
         beq             .Lcbcdecout
 .Lcbcdecloop:
         ld1             {v1.16b}, [x1], #16             /* get next ct block */
         mov             v0.16b, v1.16b                  /* ...and copy to v0 */
         decrypt_block   v0, w3, x2, x6, w7
-        eor             v0.16b, v0.16b, v7.16b          /* xor with iv => pt */
-        mov             v7.16b, v1.16b                  /* ct is next iv */
+        eor             v0.16b, v0.16b, cbciv.16b       /* xor with iv => pt */
+        mov             cbciv.16b, v1.16b               /* ct is next iv */
         st1             {v0.16b}, [x0], #16
         subs            w4, w4, #1
         bne             .Lcbcdecloop
 .Lcbcdecout:
-        st1             {v7.16b}, [x5]                  /* return iv */
+        st1             {cbciv.16b}, [x5]               /* return iv */
         ldp             x29, x30, [sp], #16
         ret
 AES_ENDPROC(aes_cbc_decrypt)
@@ -255,51 +302,60 @@ AES_ENTRY(aes_ctr_encrypt)
         mov             x29, sp
 
         enc_prepare     w3, x2, x6
-        ld1             {v4.16b}, [x5]
+        ld1             {vctr.16b}, [x5]
 
-        umov            x6, v4.d[1]             /* keep swabbed ctr in reg */
+        umov            x6, vctr.d[1]           /* keep swabbed ctr in reg */
         rev             x6, x6
         cmn             w6, w4                  /* 32 bit overflow? */
         bcs             .Lctrloop
 .LctrloopNx:
-        subs            w4, w4, #4
+        subs            w4, w4, #MAX_STRIDE
         bmi             .Lctr1x
         add             w7, w6, #1
-        mov             v0.16b, v4.16b
+        mov             v0.16b, vctr.16b
         add             w8, w6, #2
-        mov             v1.16b, v4.16b
+        mov             v1.16b, vctr.16b
+        add             w9, w6, #3
+        mov             v2.16b, vctr.16b
         add             w9, w6, #3
-        mov             v2.16b, v4.16b
         rev             w7, w7
-        mov             v3.16b, v4.16b
+        mov             v3.16b, vctr.16b
         rev             w8, w8
+ST5(    mov             v4.16b, vctr.16b                )
         mov             v1.s[3], w7
         rev             w9, w9
+ST5(    add             w10, w6, #4                     )
         mov             v2.s[3], w8
+ST5(    rev             w10, w10                        )
         mov             v3.s[3], w9
+ST5(    mov             v4.s[3], w10                    )
         ld1             {v5.16b-v7.16b}, [x1], #48      /* get 3 input blocks */
-        bl              aes_encrypt_block4x
+ST4(    bl              aes_encrypt_block4x             )
+ST5(    bl              aes_encrypt_block5x             )
         eor             v0.16b, v5.16b, v0.16b
-        ld1             {v5.16b}, [x1], #16             /* get 1 input block  */
+ST4(    ld1             {v5.16b}, [x1], #16             )
         eor             v1.16b, v6.16b, v1.16b
+ST5(    ld1             {v5.16b-v6.16b}, [x1], #32      )
         eor             v2.16b, v7.16b, v2.16b
         eor             v3.16b, v5.16b, v3.16b
+ST5(    eor             v4.16b, v6.16b, v4.16b          )
         st1             {v0.16b-v3.16b}, [x0], #64
-        add             x6, x6, #4
+ST5(    st1             {v4.16b}, [x0], #16             )
+        add             x6, x6, #MAX_STRIDE
         rev             x7, x6
-        ins             v4.d[1], x7
+        ins             vctr.d[1], x7
         cbz             w4, .Lctrout
         b               .LctrloopNx
 .Lctr1x:
-        adds            w4, w4, #4
+        adds            w4, w4, #MAX_STRIDE
         beq             .Lctrout
 .Lctrloop:
-        mov             v0.16b, v4.16b
+        mov             v0.16b, vctr.16b
         encrypt_block   v0, w3, x2, x8, w7
 
         adds            x6, x6, #1              /* increment BE ctr */
         rev             x7, x6
-        ins             v4.d[1], x7
+        ins             vctr.d[1], x7
         bcs             .Lctrcarry              /* overflow? */
 
 .Lctrcarrydone:
@@ -311,7 +367,7 @@ AES_ENTRY(aes_ctr_encrypt)
         bne             .Lctrloop
 
 .Lctrout:
-        st1             {v4.16b}, [x5]          /* return next CTR value */
+        st1             {vctr.16b}, [x5]        /* return next CTR value */
         ldp             x29, x30, [sp], #16
         ret
 
@@ -320,11 +376,11 @@ AES_ENTRY(aes_ctr_encrypt)
         b               .Lctrout
 
 .Lctrcarry:
-        umov            x7, v4.d[0]             /* load upper word of ctr  */
+        umov            x7, vctr.d[0]           /* load upper word of ctr  */
         rev             x7, x7                  /* ... to handle the carry */
         add             x7, x7, #1
         rev             x7, x7
-        ins             v4.d[0], x7
+        ins             vctr.d[0], x7
         b               .Lctrcarrydone
 AES_ENDPROC(aes_ctr_encrypt)
 
diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S
index d261331747f2..2bebccc73869 100644
--- a/arch/arm64/crypto/aes-neon.S
+++ b/arch/arm64/crypto/aes-neon.S
@@ -12,6 +12,8 @@
 #define AES_ENDPROC(func)       ENDPROC(neon_ ## func)
 
         xtsmask         .req    v7
+        cbciv           .req    v7
+        vctr            .req    v4
 
         .macro          xts_reload_mask, tmp
         xts_load_mask   \tmp
@@ -114,26 +116,9 @@
 
         /*
          * Interleaved versions: functionally equivalent to the
-         * ones above, but applied to 2 or 4 AES states in parallel.
+         * ones above, but applied to AES states in parallel.
          */
 
-        .macro          sub_bytes_2x, in0, in1
-        sub             v8.16b, \in0\().16b, v15.16b
-        tbl             \in0\().16b, {v16.16b-v19.16b}, \in0\().16b
-        sub             v9.16b, \in1\().16b, v15.16b
-        tbl             \in1\().16b, {v16.16b-v19.16b}, \in1\().16b
-        sub             v10.16b, v8.16b, v15.16b
-        tbx             \in0\().16b, {v20.16b-v23.16b}, v8.16b
-        sub             v11.16b, v9.16b, v15.16b
-        tbx             \in1\().16b, {v20.16b-v23.16b}, v9.16b
-        sub             v8.16b, v10.16b, v15.16b
-        tbx             \in0\().16b, {v24.16b-v27.16b}, v10.16b
-        sub             v9.16b, v11.16b, v15.16b
-        tbx             \in1\().16b, {v24.16b-v27.16b}, v11.16b
-        tbx             \in0\().16b, {v28.16b-v31.16b}, v8.16b
-        tbx             \in1\().16b, {v28.16b-v31.16b}, v9.16b
-        .endm
-
         .macro          sub_bytes_4x, in0, in1, in2, in3
         sub             v8.16b, \in0\().16b, v15.16b
         tbl             \in0\().16b, {v16.16b-v19.16b}, \in0\().16b
@@ -212,25 +197,6 @@
         eor             \in1\().16b, \in1\().16b, v11.16b
         .endm
 
-        .macro          do_block_2x, enc, in0, in1, rounds, rk, rkp, i
-        ld1             {v15.4s}, [\rk]
-        add             \rkp, \rk, #16
-        mov             \i, \rounds
-1111:   eor             \in0\().16b, \in0\().16b, v15.16b       /* ^round key */
-        eor             \in1\().16b, \in1\().16b, v15.16b       /* ^round key */
-        movi            v15.16b, #0x40
-        tbl             \in0\().16b, {\in0\().16b}, v13.16b     /* ShiftRows */
-        tbl             \in1\().16b, {\in1\().16b}, v13.16b     /* ShiftRows */
-        sub_bytes_2x    \in0, \in1
-        subs            \i, \i, #1
-        ld1             {v15.4s}, [\rkp], #16
-        beq             2222f
-        mix_columns_2x  \in0, \in1, \enc
-        b               1111b
-2222:   eor             \in0\().16b, \in0\().16b, v15.16b       /* ^round key */
-        eor             \in1\().16b, \in1\().16b, v15.16b       /* ^round key */
-        .endm
-
         .macro          do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i
         ld1             {v15.4s}, [\rk]
         add             \rkp, \rk, #16
@@ -257,14 +223,6 @@
         eor             \in3\().16b, \in3\().16b, v15.16b       /* ^round key */
         .endm
 
-        .macro          encrypt_block2x, in0, in1, rounds, rk, rkp, i
-        do_block_2x     1, \in0, \in1, \rounds, \rk, \rkp, \i
-        .endm
-
-        .macro          decrypt_block2x, in0, in1, rounds, rk, rkp, i
-        do_block_2x     0, \in0, \in1, \rounds, \rk, \rkp, \i
-        .endm
-
         .macro          encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
         do_block_4x     1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
         .endm
diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c
index 82029cda2e77..1495d2b18518 100644
--- a/arch/arm64/crypto/chacha-neon-glue.c
+++ b/arch/arm64/crypto/chacha-neon-glue.c
@@ -60,7 +60,7 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
 }
 
 static int chacha_neon_stream_xor(struct skcipher_request *req,
-                                  struct chacha_ctx *ctx, u8 *iv)
+                                  const struct chacha_ctx *ctx, const u8 *iv)
 {
         struct skcipher_walk walk;
         u32 state[16];
diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c
index ecb0f67e5998..bdc1b6d7aff7 100644
--- a/arch/arm64/crypto/sha1-ce-glue.c
+++ b/arch/arm64/crypto/sha1-ce-glue.c
@@ -52,7 +52,7 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data,
                          unsigned int len, u8 *out)
 {
         struct sha1_ce_state *sctx = shash_desc_ctx(desc);
-        bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE);
+        bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE) && len;
 
         if (!crypto_simd_usable())
                 return crypto_sha1_finup(desc, data, len, out);
diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c
index 955c3c2d3f5a..604a01a4ede6 100644
--- a/arch/arm64/crypto/sha2-ce-glue.c
+++ b/arch/arm64/crypto/sha2-ce-glue.c
@@ -57,7 +57,7 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data,
                            unsigned int len, u8 *out)
 {
         struct sha256_ce_state *sctx = shash_desc_ctx(desc);
-        bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE);
+        bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE) && len;
 
         if (!crypto_simd_usable()) {
                 if (len)
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index e9b866e87d48..73c0ccb009a0 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -371,20 +371,6 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
         }
 }
 
-static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
-
-        aesni_enc(ctx, dst, src);
-}
-
-static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
-
-        aesni_dec(ctx, dst, src);
-}
-
 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
                                  unsigned int len)
 {
@@ -920,7 +906,7 @@ static int helper_rfc4106_decrypt(struct aead_request *req)
 }
 #endif
 
-static struct crypto_alg aesni_algs[] = { {
+static struct crypto_alg aesni_cipher_alg = {
         .cra_name               = "aes",
         .cra_driver_name        = "aes-aesni",
         .cra_priority           = 300,
@@ -937,24 +923,7 @@ static struct crypto_alg aesni_algs[] = { {
                         .cia_decrypt            = aes_decrypt
                 }
         }
-}, {
-        .cra_name               = "__aes",
-        .cra_driver_name        = "__aes-aesni",
-        .cra_priority           = 300,
-        .cra_flags              = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
-        .cra_blocksize          = AES_BLOCK_SIZE,
-        .cra_ctxsize            = CRYPTO_AES_CTX_SIZE,
-        .cra_module             = THIS_MODULE,
-        .cra_u  = {
-                .cipher = {
-                        .cia_min_keysize        = AES_MIN_KEY_SIZE,
-                        .cia_max_keysize        = AES_MAX_KEY_SIZE,
-                        .cia_setkey             = aes_set_key,
-                        .cia_encrypt            = __aes_encrypt,
-                        .cia_decrypt            = __aes_decrypt
-                }
-        }
-} };
+};
 
 static struct skcipher_alg aesni_skciphers[] = {
         {
@@ -1150,7 +1119,7 @@ static int __init aesni_init(void)
 #endif
 #endif
 
-        err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
+        err = crypto_register_alg(&aesni_cipher_alg);
         if (err)
                 return err;
 
@@ -1158,7 +1127,7 @@ static int __init aesni_init(void)
                                              ARRAY_SIZE(aesni_skciphers),
                                              aesni_simd_skciphers);
         if (err)
-                goto unregister_algs;
+                goto unregister_cipher;
 
         err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads),
                                          aesni_simd_aeads);
@@ -1170,8 +1139,8 @@ static int __init aesni_init(void)
 unregister_skciphers:
         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
                                   aesni_simd_skciphers);
-unregister_algs:
-        crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
+unregister_cipher:
+        crypto_unregister_alg(&aesni_cipher_alg);
         return err;
 }
 
@@ -1181,7 +1150,7 @@ static void __exit aesni_exit(void)
                               aesni_simd_aeads);
         simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers),
                                   aesni_simd_skciphers);
-        crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
+        crypto_unregister_alg(&aesni_cipher_alg);
 }
 
 late_initcall(aesni_init);
diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c
index 1ce0019c059c..388f95a4ec24 100644
--- a/arch/x86/crypto/chacha_glue.c
+++ b/arch/x86/crypto/chacha_glue.c
@@ -124,7 +124,7 @@ static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src,
 }
 
 static int chacha_simd_stream_xor(struct skcipher_walk *walk,
-                                  struct chacha_ctx *ctx, u8 *iv)
+                                  const struct chacha_ctx *ctx, const u8 *iv)
 {
         u32 *state, state_buf[16 + 2] __aligned(8);
         int next_yield = 4096; /* bytes until next FPU yield */
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 3d056e7da65f..e801450bcb1c 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -61,7 +61,6 @@ config CRYPTO_BLKCIPHER2
         tristate
         select CRYPTO_ALGAPI2
         select CRYPTO_RNG2
-        select CRYPTO_WORKQUEUE
 
 config CRYPTO_HASH
         tristate
@@ -137,10 +136,11 @@ config CRYPTO_USER
           Userspace configuration for cryptographic instantiations such as
           cbc(aes).
 
+if CRYPTO_MANAGER2
+
 config CRYPTO_MANAGER_DISABLE_TESTS
         bool "Disable run-time self tests"
         default y
-        depends on CRYPTO_MANAGER2
         help
           Disable run-time self tests that normally take place at
           algorithm registration.
@@ -155,14 +155,10 @@ config CRYPTO_MANAGER_EXTRA_TESTS
           This is intended for developer use only, as these tests take much
           longer to run than the normal self tests.
 
+endif   # if CRYPTO_MANAGER2
+
 config CRYPTO_GF128MUL
-        tristate "GF(2^128) multiplication functions"
-        help
-          Efficient table driven implementation of multiplications in the
-          field GF(2^128).  This is needed by some cypher modes. This
-          option will be selected automatically if you select such a
-          cipher mode.  Only select this option by hand if you expect to load
-          an external module that requires these functions.
+        tristate
 
 config CRYPTO_NULL
         tristate "Null algorithms"
@@ -186,15 +182,11 @@ config CRYPTO_PCRYPT
           This converts an arbitrary crypto algorithm into a parallel
           algorithm that executes in kernel threads.
 
-config CRYPTO_WORKQUEUE
-       tristate
-
 config CRYPTO_CRYPTD
         tristate "Software async crypto daemon"
         select CRYPTO_BLKCIPHER
         select CRYPTO_HASH
         select CRYPTO_MANAGER
-        select CRYPTO_WORKQUEUE
         help
           This is a generic software asynchronous crypto daemon that
           converts an arbitrary synchronous software crypto algorithm
@@ -279,6 +271,7 @@ config CRYPTO_CCM
         select CRYPTO_CTR
         select CRYPTO_HASH
         select CRYPTO_AEAD
+        select CRYPTO_MANAGER
         help
           Support for Counter with CBC MAC. Required for IPsec.
 
@@ -288,6 +281,7 @@ config CRYPTO_GCM
         select CRYPTO_AEAD
         select CRYPTO_GHASH
         select CRYPTO_NULL
+        select CRYPTO_MANAGER
         help
           Support for Galois/Counter Mode (GCM) and Galois Message
           Authentication Code (GMAC). Required for IPSec.
@@ -297,6 +291,7 @@ config CRYPTO_CHACHA20POLY1305
         select CRYPTO_CHACHA20
         select CRYPTO_POLY1305
         select CRYPTO_AEAD
+        select CRYPTO_MANAGER
         help
           ChaCha20-Poly1305 AEAD support, RFC7539.
 
@@ -411,6 +406,7 @@ config CRYPTO_SEQIV
         select CRYPTO_BLKCIPHER
         select CRYPTO_NULL
         select CRYPTO_RNG_DEFAULT
+        select CRYPTO_MANAGER
         help
           This IV generator generates an IV based on a sequence number by
           xoring it with a salt.  This algorithm is mainly useful for CTR
@@ -420,7 +416,7 @@ config CRYPTO_ECHAINIV
         select CRYPTO_AEAD
         select CRYPTO_NULL
         select CRYPTO_RNG_DEFAULT
-        default m
+        select CRYPTO_MANAGER
         help
           This IV generator generates an IV based on the encryption of
           a sequence number xored with a salt.  This is the default
@@ -456,6 +452,7 @@ config CRYPTO_CTR
 config CRYPTO_CTS
         tristate "CTS support"
         select CRYPTO_BLKCIPHER
+        select CRYPTO_MANAGER
         help
           CTS: Cipher Text Stealing
           This is the Cipher Text Stealing mode as described by
@@ -521,6 +518,7 @@ config CRYPTO_XTS
 config CRYPTO_KEYWRAP
         tristate "Key wrapping support"
         select CRYPTO_BLKCIPHER
+        select CRYPTO_MANAGER
         help
           Support for key wrapping (NIST SP800-38F / RFC3394) without
           padding.
@@ -551,6 +549,7 @@ config CRYPTO_ADIANTUM
         select CRYPTO_CHACHA20
         select CRYPTO_POLY1305
         select CRYPTO_NHPOLY1305
+        select CRYPTO_MANAGER
         help
           Adiantum is a tweakable, length-preserving encryption mode
           designed for fast and secure disk encryption, especially on
@@ -684,6 +683,14 @@ config CRYPTO_CRC32_MIPS
           instructions, when available.
 
 
+config CRYPTO_XXHASH
+        tristate "xxHash hash algorithm"
+        select CRYPTO_HASH
+        select XXHASH
+        help
+          xxHash non-cryptographic hash algorithm. Extremely fast, working at
+          speeds close to RAM limits.
+
 config CRYPTO_CRCT10DIF
         tristate "CRCT10DIF algorithm"
         select CRYPTO_HASH
@@ -1230,9 +1237,13 @@ config CRYPTO_ANUBIS
           <https://www.cosic.esat.kuleuven.be/nessie/reports/>
           <http://www.larc.usp.br/~pbarreto/AnubisPage.html>
 
+config CRYPTO_LIB_ARC4
+        tristate
+
 config CRYPTO_ARC4
         tristate "ARC4 cipher algorithm"
         select CRYPTO_BLKCIPHER
+        select CRYPTO_LIB_ARC4
         help
           ARC4 cipher algorithm.
 
diff --git a/crypto/Makefile b/crypto/Makefile
index 266a4cdbb9e2..9479e1a45d8c 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -6,8 +6,6 @@
 obj-$(CONFIG_CRYPTO) += crypto.o
 crypto-y := api.o cipher.o compress.o memneq.o
 
-obj-$(CONFIG_CRYPTO_WORKQUEUE) += crypto_wq.o
-
 obj-$(CONFIG_CRYPTO_ENGINE) += crypto_engine.o
 obj-$(CONFIG_CRYPTO_FIPS) += fips.o
 
@@ -131,6 +129,7 @@ obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
 obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o
 obj-$(CONFIG_CRYPTO_LZ4) += lz4.o
 obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o
+obj-$(CONFIG_CRYPTO_XXHASH) += xxhash_generic.o
 obj-$(CONFIG_CRYPTO_842) += 842.o
 obj-$(CONFIG_CRYPTO_RNG2) += rng.o
 obj-$(CONFIG_CRYPTO_ANSI_CPRNG) += ansi_cprng.o
diff --git a/crypto/aead.c b/crypto/aead.c
index c3c158ba9883..fbf0ec93bc8e 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -84,6 +84,42 @@ int crypto_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
 }
 EXPORT_SYMBOL_GPL(crypto_aead_setauthsize);
 
+int crypto_aead_encrypt(struct aead_request *req)
+{
+        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        struct crypto_alg *alg = aead->base.__crt_alg;
+        unsigned int cryptlen = req->cryptlen;
+        int ret;
+
+        crypto_stats_get(alg);
+        if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
+                ret = -ENOKEY;
+        else
+                ret = crypto_aead_alg(aead)->encrypt(req);
+        crypto_stats_aead_encrypt(cryptlen, alg, ret);
+        return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_aead_encrypt);
+
+int crypto_aead_decrypt(struct aead_request *req)
+{
+        struct crypto_aead *aead = crypto_aead_reqtfm(req);
+        struct crypto_alg *alg = aead->base.__crt_alg;
+        unsigned int cryptlen = req->cryptlen;
+        int ret;
+
+        crypto_stats_get(alg);
+        if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
+                ret = -ENOKEY;
+        else if (req->cryptlen < crypto_aead_authsize(aead))
+                ret = -EINVAL;
+        else
+                ret = crypto_aead_alg(aead)->decrypt(req);
+        crypto_stats_aead_decrypt(cryptlen, alg, ret);
+        return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_aead_decrypt);
+
 static void crypto_aead_exit_tfm(struct crypto_tfm *tfm)
 {
         struct crypto_aead *aead = __crypto_aead_cast(tfm);
diff --git a/crypto/algapi.c b/crypto/algapi.c
index 313a7682cef1..de30ddc952d8 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -21,23 +21,6 @@
 
 static LIST_HEAD(crypto_template_list);
 
-static inline int crypto_set_driver_name(struct crypto_alg *alg)
-{
-        static const char suffix[] = "-generic";
-        char *driver_name = alg->cra_driver_name;
-        int len;
-
-        if (*driver_name)
-                return 0;
-
-        len = strlcpy(driver_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
-        if (len + sizeof(suffix) > CRYPTO_MAX_ALG_NAME)
-                return -ENAMETOOLONG;
-
-        memcpy(driver_name + len, suffix, sizeof(suffix));
-        return 0;
-}
-
 static inline void crypto_check_module_sig(struct module *mod)
 {
         if (fips_enabled && mod && !module_sig_ok(mod))
@@ -49,6 +32,9 @@ static int crypto_check_alg(struct crypto_alg *alg)
 {
         crypto_check_module_sig(alg->cra_module);
 
+        if (!alg->cra_name[0] || !alg->cra_driver_name[0])
+                return -EINVAL;
+
         if (alg->cra_alignmask & (alg->cra_alignmask + 1))
                 return -EINVAL;
 
@@ -74,7 +60,7 @@ static int crypto_check_alg(struct crypto_alg *alg)
 
         refcount_set(&alg->cra_refcnt, 1);
 
-        return crypto_set_driver_name(alg);
+        return 0;
 }
 
 static void crypto_free_instance(struct crypto_instance *inst)
@@ -947,19 +933,6 @@ struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue)
 }
 EXPORT_SYMBOL_GPL(crypto_dequeue_request);
 
-int crypto_tfm_in_queue(struct crypto_queue *queue, struct crypto_tfm *tfm)
-{
-        struct crypto_async_request *req;
-
-        list_for_each_entry(req, &queue->list, list) {
-                if (req->tfm == tfm)
-                        return 1;
-        }
-
-        return 0;
-}
-EXPORT_SYMBOL_GPL(crypto_tfm_in_queue);
-
 static inline void crypto_inc_byte(u8 *a, unsigned int size)
 {
         u8 *b = (a + size);
diff --git a/crypto/anubis.c b/crypto/anubis.c
index 673927de0eb9..f9ce78fde6ee 100644
--- a/crypto/anubis.c
+++ b/crypto/anubis.c
@@ -673,6 +673,7 @@ static void anubis_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 
 static struct crypto_alg anubis_alg = {
         .cra_name               =       "anubis",
+        .cra_driver_name        =       "anubis-generic",
         .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
         .cra_blocksize          =       ANUBIS_BLOCK_SIZE,
         .cra_ctxsize            =       sizeof (struct anubis_ctx),
diff --git a/crypto/arc4.c b/crypto/arc4.c
index a2120e06bf84..aa79571dbd49 100644
--- a/crypto/arc4.c
+++ b/crypto/arc4.c
@@ -13,84 +13,15 @@
 #include <linux/init.h>
 #include <linux/module.h>
 
-struct arc4_ctx {
-        u32 S[256];
-        u32 x, y;
-};
-
-static int arc4_set_key(struct crypto_tfm *tfm, const u8 *in_key,
-                        unsigned int key_len)
-{
-        struct arc4_ctx *ctx = crypto_tfm_ctx(tfm);
-        int i, j = 0, k = 0;
-
-        ctx->x = 1;
-        ctx->y = 0;
-
-        for (i = 0; i < 256; i++)
-                ctx->S[i] = i;
-
-        for (i = 0; i < 256; i++) {
-                u32 a = ctx->S[i];
-                j = (j + in_key[k] + a) & 0xff;
-                ctx->S[i] = ctx->S[j];
-                ctx->S[j] = a;
-                if (++k >= key_len)
-                        k = 0;
-        }
-
-        return 0;
-}
-
-static int arc4_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
-                                 unsigned int key_len)
+static int crypto_arc4_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
+                              unsigned int key_len)
 {
-        return arc4_set_key(&tfm->base, in_key, key_len);
-}
-
-static void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in,
-                       unsigned int len)
-{
-        u32 *const S = ctx->S;
-        u32 x, y, a, b;
-        u32 ty, ta, tb;
-
-        if (len == 0)
-                return;
-
-        x = ctx->x;
-        y = ctx->y;
-
-        a = S[x];
-        y = (y + a) & 0xff;
-        b = S[y];
-
-        do {
-                S[y] = a;
-                a = (a + b) & 0xff;
-                S[x] = b;
-                x = (x + 1) & 0xff;
-                ta = S[x];
-                ty = (y + ta) & 0xff;
-                tb = S[ty];
-                *out++ = *in++ ^ S[a];
-                if (--len == 0)
-                        break;
-                y = ty;
-                a = ta;
-                b = tb;
-        } while (true);
-
-        ctx->x = x;
-        ctx->y = y;
-}
+        struct arc4_ctx *ctx = crypto_skcipher_ctx(tfm);
 
-static void arc4_crypt_one(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-        arc4_crypt(crypto_tfm_ctx(tfm), out, in, 1);
+        return arc4_setkey(ctx, in_key, key_len);
 }
 
-static int ecb_arc4_crypt(struct skcipher_request *req)
+static int crypto_arc4_crypt(struct skcipher_request *req)
 {
         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
         struct arc4_ctx *ctx = crypto_skcipher_ctx(tfm);
@@ -108,54 +39,32 @@ static int ecb_arc4_crypt(struct skcipher_request *req)
         return err;
 }
 
-static struct crypto_alg arc4_cipher = {
-        .cra_name               =       "arc4",
-        .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
-        .cra_blocksize          =       ARC4_BLOCK_SIZE,
-        .cra_ctxsize            =       sizeof(struct arc4_ctx),
-        .cra_module             =       THIS_MODULE,
-        .cra_u                  =       {
-                .cipher = {
-                        .cia_min_keysize        =       ARC4_MIN_KEY_SIZE,
-                        .cia_max_keysize        =       ARC4_MAX_KEY_SIZE,
-                        .cia_setkey             =       arc4_set_key,
-                        .cia_encrypt            =       arc4_crypt_one,
-                        .cia_decrypt            =       arc4_crypt_one,
-                },
-        },
-};
-
-static struct skcipher_alg arc4_skcipher = {
+static struct skcipher_alg arc4_alg = {
+        /*
+         * For legacy reasons, this is named "ecb(arc4)", not "arc4".
+         * Nevertheless it's actually a stream cipher, not a block cipher.
+         */
         .base.cra_name          =       "ecb(arc4)",
+        .base.cra_driver_name   =       "ecb(arc4)-generic",
         .base.cra_priority      =       100,
         .base.cra_blocksize     =       ARC4_BLOCK_SIZE,
         .base.cra_ctxsize       =       sizeof(struct arc4_ctx),
         .base.cra_module        =       THIS_MODULE,
         .min_keysize            =       ARC4_MIN_KEY_SIZE,
         .max_keysize            =       ARC4_MAX_KEY_SIZE,
-        .setkey                 =       arc4_set_key_skcipher,
-        .encrypt                =       ecb_arc4_crypt,
-        .decrypt                =       ecb_arc4_crypt,
+        .setkey                 =       crypto_arc4_setkey,
+        .encrypt                =       crypto_arc4_crypt,
+        .decrypt                =       crypto_arc4_crypt,
 };
 
 static int __init arc4_init(void)
 {
-        int err;
-
-        err = crypto_register_alg(&arc4_cipher);
-        if (err)
-                return err;
-
-        err = crypto_register_skcipher(&arc4_skcipher);
-        if (err)
-                crypto_unregister_alg(&arc4_cipher);
-        return err;
+        return crypto_register_skcipher(&arc4_alg);
 }
 
 static void __exit arc4_exit(void)
 {
-        crypto_unregister_alg(&arc4_cipher);
-        crypto_unregister_skcipher(&arc4_skcipher);
+        crypto_unregister_skcipher(&arc4_alg);
 }
 
 subsys_initcall(arc4_init);
@@ -164,4 +73,4 @@ module_exit(arc4_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("ARC4 Cipher Algorithm");
 MODULE_AUTHOR("Jon Oberheide <jon@oberheide.org>");
-MODULE_ALIAS_CRYPTO("arc4");
+MODULE_ALIAS_CRYPTO("ecb(arc4)");
diff --git a/crypto/asymmetric_keys/Kconfig b/crypto/asymmetric_keys/Kconfig
index be70ca6c85d3..1f1f004dc757 100644
--- a/crypto/asymmetric_keys/Kconfig
+++ b/crypto/asymmetric_keys/Kconfig
@@ -15,6 +15,7 @@ config ASYMMETRIC_PUBLIC_KEY_SUBTYPE
         select MPILIB
         select CRYPTO_HASH_INFO
         select CRYPTO_AKCIPHER
+        select CRYPTO_HASH
         help
           This option provides support for asymmetric public key type handling.
           If signature generation and/or verification are to be used,
@@ -65,6 +66,7 @@ config TPM_KEY_PARSER
 config PKCS7_MESSAGE_PARSER
         tristate "PKCS#7 message parser"
         depends on X509_CERTIFICATE_PARSER
+        select CRYPTO_HASH
         select ASN1
         select OID_REGISTRY
         help
@@ -87,6 +89,7 @@ config SIGNED_PE_FILE_VERIFICATION
         bool "Support for PE file signature verification"
         depends on PKCS7_MESSAGE_PARSER=y
         depends on SYSTEM_DATA_VERIFICATION
+        select CRYPTO_HASH
         select ASN1
         select OID_REGISTRY
         help
diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c
index 2db7eac4bf3b..74e824e537e6 100644
--- a/crypto/chacha20poly1305.c
+++ b/crypto/chacha20poly1305.c
@@ -61,6 +61,8 @@ struct chachapoly_req_ctx {
         unsigned int cryptlen;
         /* Actual AD, excluding IV */
         unsigned int assoclen;
+        /* request flags, with MAY_SLEEP cleared if needed */
+        u32 flags;
         union {
                 struct poly_req poly;
                 struct chacha_req chacha;
@@ -70,8 +72,12 @@ struct chachapoly_req_ctx {
 static inline void async_done_continue(struct aead_request *req, int err,
                                        int (*cont)(struct aead_request *))
 {
-        if (!err)
+        if (!err) {
+                struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
+
+                rctx->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
                 err = cont(req);
+        }
 
         if (err != -EINPROGRESS && err != -EBUSY)
                 aead_request_complete(req, err);
@@ -129,16 +135,12 @@ static int chacha_decrypt(struct aead_request *req)
 
         chacha_iv(creq->iv, req, 1);
 
-        sg_init_table(rctx->src, 2);
         src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
         dst = src;
-
-        if (req->src != req->dst) {
-                sg_init_table(rctx->dst, 2);
+        if (req->src != req->dst)
                 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
-        }
 
-        skcipher_request_set_callback(&creq->req, aead_request_flags(req),
+        skcipher_request_set_callback(&creq->req, rctx->flags,
                                       chacha_decrypt_done, req);
         skcipher_request_set_tfm(&creq->req, ctx->chacha);
         skcipher_request_set_crypt(&creq->req, src, dst,
@@ -172,17 +174,13 @@ static int poly_tail(struct aead_request *req)
         struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
         struct poly_req *preq = &rctx->u.poly;
-        __le64 len;
         int err;
 
-        sg_init_table(preq->src, 1);
-        len = cpu_to_le64(rctx->assoclen);
-        memcpy(&preq->tail.assoclen, &len, sizeof(len));
-        len = cpu_to_le64(rctx->cryptlen);
-        memcpy(&preq->tail.cryptlen, &len, sizeof(len));
-        sg_set_buf(preq->src, &preq->tail, sizeof(preq->tail));
+        preq->tail.assoclen = cpu_to_le64(rctx->assoclen);
+        preq->tail.cryptlen = cpu_to_le64(rctx->cryptlen);
+        sg_init_one(preq->src, &preq->tail, sizeof(preq->tail));
 
-        ahash_request_set_callback(&preq->req, aead_request_flags(req),
+        ahash_request_set_callback(&preq->req, rctx->flags,
                                    poly_tail_done, req);
         ahash_request_set_tfm(&preq->req, ctx->poly);
         ahash_request_set_crypt(&preq->req, preq->src,
@@ -205,15 +203,14 @@ static int poly_cipherpad(struct aead_request *req)
         struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
         struct poly_req *preq = &rctx->u.poly;
-        unsigned int padlen, bs = POLY1305_BLOCK_SIZE;
+        unsigned int padlen;
         int err;
 
-        padlen = (bs - (rctx->cryptlen % bs)) % bs;
+        padlen = -rctx->cryptlen % POLY1305_BLOCK_SIZE;
         memset(preq->pad, 0, sizeof(preq->pad));
-        sg_init_table(preq->src, 1);
-        sg_set_buf(preq->src, &preq->pad, padlen);
+        sg_init_one(preq->src, preq->pad, padlen);
 
-        ahash_request_set_callback(&preq->req, aead_request_flags(req),
+        ahash_request_set_callback(&preq->req, rctx->flags,
                                    poly_cipherpad_done, req);
         ahash_request_set_tfm(&preq->req, ctx->poly);
         ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
@@ -241,10 +238,9 @@ static int poly_cipher(struct aead_request *req)
         if (rctx->cryptlen == req->cryptlen) /* encrypting */
                 crypt = req->dst;
 
-        sg_init_table(rctx->src, 2);
         crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen);
 
-        ahash_request_set_callback(&preq->req, aead_request_flags(req),
+        ahash_request_set_callback(&preq->req, rctx->flags,
                                    poly_cipher_done, req);
         ahash_request_set_tfm(&preq->req, ctx->poly);
         ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen);
@@ -266,15 +262,14 @@ static int poly_adpad(struct aead_request *req)
         struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
         struct poly_req *preq = &rctx->u.poly;
-        unsigned int padlen, bs = POLY1305_BLOCK_SIZE;
+        unsigned int padlen;
         int err;
 
-        padlen = (bs - (rctx->assoclen % bs)) % bs;
+        padlen = -rctx->assoclen % POLY1305_BLOCK_SIZE;
         memset(preq->pad, 0, sizeof(preq->pad));
-        sg_init_table(preq->src, 1);
-        sg_set_buf(preq->src, preq->pad, padlen);
+        sg_init_one(preq->src, preq->pad, padlen);
 
-        ahash_request_set_callback(&preq->req, aead_request_flags(req),
+        ahash_request_set_callback(&preq->req, rctx->flags,
                                    poly_adpad_done, req);
         ahash_request_set_tfm(&preq->req, ctx->poly);
         ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
@@ -298,7 +293,7 @@ static int poly_ad(struct aead_request *req)
         struct poly_req *preq = &rctx->u.poly;
         int err;
 
-        ahash_request_set_callback(&preq->req, aead_request_flags(req),
+        ahash_request_set_callback(&preq->req, rctx->flags,
                                    poly_ad_done, req);
         ahash_request_set_tfm(&preq->req, ctx->poly);
         ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen);
@@ -322,10 +317,9 @@ static int poly_setkey(struct aead_request *req)
         struct poly_req *preq = &rctx->u.poly;
         int err;
 
-        sg_init_table(preq->src, 1);
-        sg_set_buf(preq->src, rctx->key, sizeof(rctx->key));
+        sg_init_one(preq->src, rctx->key, sizeof(rctx->key));
 
-        ahash_request_set_callback(&preq->req, aead_request_flags(req),
+        ahash_request_set_callback(&preq->req, rctx->flags,
                                    poly_setkey_done, req);
         ahash_request_set_tfm(&preq->req, ctx->poly);
         ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key));
@@ -349,7 +343,7 @@ static int poly_init(struct aead_request *req)
         struct poly_req *preq = &rctx->u.poly;
         int err;
 
-        ahash_request_set_callback(&preq->req, aead_request_flags(req),
+        ahash_request_set_callback(&preq->req, rctx->flags,
                                    poly_init_done, req);
         ahash_request_set_tfm(&preq->req, ctx->poly);
 
@@ -381,13 +375,12 @@ static int poly_genkey(struct aead_request *req)
                 rctx->assoclen -= 8;
         }
 
-        sg_init_table(creq->src, 1);
         memset(rctx->key, 0, sizeof(rctx->key));
-        sg_set_buf(creq->src, rctx->key, sizeof(rctx->key));
+        sg_init_one(creq->src, rctx->key, sizeof(rctx->key));
 
         chacha_iv(creq->iv, req, 0);
 
-        skcipher_request_set_callback(&creq->req, aead_request_flags(req),
+        skcipher_request_set_callback(&creq->req, rctx->flags,
                                       poly_genkey_done, req);
         skcipher_request_set_tfm(&creq->req, ctx->chacha);
         skcipher_request_set_crypt(&creq->req, creq->src, creq->src,
@@ -418,16 +411,12 @@ static int chacha_encrypt(struct aead_request *req)
 
         chacha_iv(creq->iv, req, 1);
 
-        sg_init_table(rctx->src, 2);
         src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
         dst = src;
-
-        if (req->src != req->dst) {
-                sg_init_table(rctx->dst, 2);
+        if (req->src != req->dst)
                 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
-        }
 
-        skcipher_request_set_callback(&creq->req, aead_request_flags(req),
+        skcipher_request_set_callback(&creq->req, rctx->flags,
                                       chacha_encrypt_done, req);
         skcipher_request_set_tfm(&creq->req, ctx->chacha);
         skcipher_request_set_crypt(&creq->req, src, dst,
@@ -445,6 +434,7 @@ static int chachapoly_encrypt(struct aead_request *req)
         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 
         rctx->cryptlen = req->cryptlen;
+        rctx->flags = aead_request_flags(req);
 
         /* encrypt call chain:
          * - chacha_encrypt/done()
@@ -466,6 +456,7 @@ static int chachapoly_decrypt(struct aead_request *req)
         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 
         rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE;
+        rctx->flags = aead_request_flags(req);
 
         /* decrypt call chain:
          * - poly_genkey/done()
diff --git a/crypto/chacha_generic.c b/crypto/chacha_generic.c
index 04404c479e68..085d8d219987 100644
--- a/crypto/chacha_generic.c
+++ b/crypto/chacha_generic.c
@@ -32,7 +32,7 @@ static void chacha_docrypt(u32 *state, u8 *dst, const u8 *src,
 }
 
 static int chacha_stream_xor(struct skcipher_request *req,
-                             struct chacha_ctx *ctx, u8 *iv)
+                             const struct chacha_ctx *ctx, const u8 *iv)
 {
         struct skcipher_walk walk;
         u32 state[16];
@@ -56,7 +56,7 @@ static int chacha_stream_xor(struct skcipher_request *req,
         return err;
 }
 
-void crypto_chacha_init(u32 *state, struct chacha_ctx *ctx, u8 *iv)
+void crypto_chacha_init(u32 *state, const struct chacha_ctx *ctx, const u8 *iv)
 {
         state[0]  = 0x61707865; /* "expa" */
         state[1]  = 0x3320646e; /* "nd 3" */
diff --git a/crypto/cryptd.c b/crypto/cryptd.c
index 5f76c6e222c6..3748f9b4516d 100644
--- a/crypto/cryptd.c
+++ b/crypto/cryptd.c
@@ -16,7 +16,6 @@
 #include <crypto/internal/aead.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/cryptd.h>
-#include <crypto/crypto_wq.h>
 #include <linux/atomic.h>
 #include <linux/err.h>
 #include <linux/init.h>
@@ -26,11 +25,14 @@
 #include <linux/scatterlist.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
+#include <linux/workqueue.h>
 
 static unsigned int cryptd_max_cpu_qlen = 1000;
 module_param(cryptd_max_cpu_qlen, uint, 0);
 MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth");
 
+static struct workqueue_struct *cryptd_wq;
+
 struct cryptd_cpu_queue {
         struct crypto_queue queue;
         struct work_struct work;
@@ -136,7 +138,7 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue,
         if (err == -ENOSPC)
                 goto out_put_cpu;
 
-        queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
+        queue_work_on(cpu, cryptd_wq, &cpu_queue->work);
 
         if (!atomic_read(refcnt))
                 goto out_put_cpu;
@@ -179,7 +181,7 @@ static void cryptd_queue_worker(struct work_struct *work)
         req->complete(req, 0);
 
         if (cpu_queue->queue.qlen)
-                queue_work(kcrypto_wq, &cpu_queue->work);
+                queue_work(cryptd_wq, &cpu_queue->work);
 }
 
 static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
@@ -919,7 +921,7 @@ static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
         switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
         case CRYPTO_ALG_TYPE_BLKCIPHER:
                 return cryptd_create_skcipher(tmpl, tb, &queue);
-        case CRYPTO_ALG_TYPE_DIGEST:
+        case CRYPTO_ALG_TYPE_HASH:
                 return cryptd_create_hash(tmpl, tb, &queue);
         case CRYPTO_ALG_TYPE_AEAD:
                 return cryptd_create_aead(tmpl, tb, &queue);
@@ -1119,19 +1121,31 @@ static int __init cryptd_init(void)
 {
         int err;
 
+        cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE,
+                                    1);
+        if (!cryptd_wq)
+                return -ENOMEM;
+
         err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen);
         if (err)
-                return err;
+                goto err_destroy_wq;
 
         err = crypto_register_template(&cryptd_tmpl);
         if (err)
-                cryptd_fini_queue(&queue);
+                goto err_fini_queue;
 
+        return 0;
+
+err_fini_queue:
+        cryptd_fini_queue(&queue);
+err_destroy_wq:
+        destroy_workqueue(cryptd_wq);
         return err;
 }
 
 static void __exit cryptd_exit(void)
 {
+        destroy_workqueue(cryptd_wq);
         cryptd_fini_queue(&queue);
         crypto_unregister_template(&cryptd_tmpl);
 }
diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c
index 0d341ddecd54..5b84b0f7cc17 100644
--- a/crypto/crypto_null.c
+++ b/crypto/crypto_null.c
@@ -100,6 +100,7 @@ static struct shash_alg digest_null = {
         .final                  =       null_final,
         .base                   =       {
                 .cra_name               =       "digest_null",
+                .cra_driver_name        =       "digest_null-generic",
                 .cra_blocksize          =       NULL_BLOCK_SIZE,
                 .cra_module             =       THIS_MODULE,
         }
@@ -122,6 +123,7 @@ static struct skcipher_alg skcipher_null = {
 
 static struct crypto_alg null_algs[] = { {
         .cra_name               =       "cipher_null",
+        .cra_driver_name        =       "cipher_null-generic",
         .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
         .cra_blocksize          =       NULL_BLOCK_SIZE,
         .cra_ctxsize            =       0,
@@ -134,6 +136,7 @@ static struct crypto_alg null_algs[] = { {
         .cia_decrypt            =       null_crypt } }
 }, {
         .cra_name               =       "compress_null",
+        .cra_driver_name        =       "compress_null-generic",
         .cra_flags              =       CRYPTO_ALG_TYPE_COMPRESS,
         .cra_blocksize          =       NULL_BLOCK_SIZE,
         .cra_ctxsize            =       0,
diff --git a/crypto/crypto_wq.c b/crypto/crypto_wq.c
deleted file mode 100644
index 80501928e0bb..000000000000
--- a/crypto/crypto_wq.c
+++ /dev/null
@@ -1,35 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Workqueue for crypto subsystem
- *
- * Copyright (c) 2009 Intel Corp.
- *   Author: Huang Ying <ying.huang@intel.com>
- */
-
-#include <linux/workqueue.h>
-#include <linux/module.h>
-#include <crypto/algapi.h>
-#include <crypto/crypto_wq.h>
-
-struct workqueue_struct *kcrypto_wq;
-EXPORT_SYMBOL_GPL(kcrypto_wq);
-
-static int __init crypto_wq_init(void)
-{
-        kcrypto_wq = alloc_workqueue("crypto",
-                                     WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1);
-        if (unlikely(!kcrypto_wq))
-                return -ENOMEM;
-        return 0;
-}
-
-static void __exit crypto_wq_exit(void)
-{
-        destroy_workqueue(kcrypto_wq);
-}
-
-subsys_initcall(crypto_wq_init);
-module_exit(crypto_wq_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Workqueue for crypto subsystem");
diff --git a/crypto/deflate.c b/crypto/deflate.c
index 65be51456398..4c0e6c9d942a 100644
--- a/crypto/deflate.c
+++ b/crypto/deflate.c
@@ -275,6 +275,7 @@ static int deflate_sdecompress(struct crypto_scomp *tfm, const u8 *src,
 
 static struct crypto_alg alg = {
         .cra_name               = "deflate",
+        .cra_driver_name        = "deflate-generic",
         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
         .cra_ctxsize            = sizeof(struct deflate_ctx),
         .cra_module             = THIS_MODULE,
diff --git a/crypto/drbg.c b/crypto/drbg.c
index 2a5b16bb000c..b6929eb5f565 100644
--- a/crypto/drbg.c
+++ b/crypto/drbg.c
@@ -220,6 +220,57 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags)
 }
 
 /*
+ * FIPS 140-2 continuous self test for the noise source
+ * The test is performed on the noise source input data. Thus, the function
+ * implicitly knows the size of the buffer to be equal to the security
+ * strength.
+ *
+ * Note, this function disregards the nonce trailing the entropy data during
+ * initial seeding.
+ *
+ * drbg->drbg_mutex must have been taken.
+ *
+ * @drbg DRBG handle
+ * @entropy buffer of seed data to be checked
+ *
+ * return:
+ *      0 on success
+ *      -EAGAIN on when the CTRNG is not yet primed
+ *      < 0 on error
+ */
+static int drbg_fips_continuous_test(struct drbg_state *drbg,
+                                     const unsigned char *entropy)
+{
+        unsigned short entropylen = drbg_sec_strength(drbg->core->flags);
+        int ret = 0;
+
+        if (!IS_ENABLED(CONFIG_CRYPTO_FIPS))
+                return 0;
+
+        /* skip test if we test the overall system */
+        if (list_empty(&drbg->test_data.list))
+                return 0;
+        /* only perform test in FIPS mode */
+        if (!fips_enabled)
+                return 0;
+
+        if (!drbg->fips_primed) {
+                /* Priming of FIPS test */
+                memcpy(drbg->prev, entropy, entropylen);
+                drbg->fips_primed = true;
+                /* priming: another round is needed */
+                return -EAGAIN;
+        }
+        ret = memcmp(drbg->prev, entropy, entropylen);
+        if (!ret)
+                panic("DRBG continuous self test failed\n");
+        memcpy(drbg->prev, entropy, entropylen);
+
+        /* the test shall pass when the two values are not equal */
+        return 0;
+}
+
+/*
  * Convert an integer into a byte representation of this integer.
  * The byte representation is big-endian
  *
@@ -998,6 +1049,22 @@ static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed,
         return ret;
 }
 
+static inline int drbg_get_random_bytes(struct drbg_state *drbg,
+                                        unsigned char *entropy,
+                                        unsigned int entropylen)
+{
+        int ret;
+
+        do {
+                get_random_bytes(entropy, entropylen);
+                ret = drbg_fips_continuous_test(drbg, entropy);
+                if (ret && ret != -EAGAIN)
+                        return ret;
+        } while (ret);
+
+        return 0;
+}
+
 static void drbg_async_seed(struct work_struct *work)
 {
         struct drbg_string data;
@@ -1006,16 +1073,20 @@ static void drbg_async_seed(struct work_struct *work)
                                                seed_work);
         unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
         unsigned char entropy[32];
+        int ret;
 
         BUG_ON(!entropylen);
         BUG_ON(entropylen > sizeof(entropy));
-        get_random_bytes(entropy, entropylen);
 
         drbg_string_fill(&data, entropy, entropylen);
         list_add_tail(&data.list, &seedlist);
 
         mutex_lock(&drbg->drbg_mutex);
 
+        ret = drbg_get_random_bytes(drbg, entropy, entropylen);
+        if (ret)
+                goto unlock;
+
         /* If nonblocking pool is initialized, deactivate Jitter RNG */
         crypto_free_rng(drbg->jent);
         drbg->jent = NULL;
@@ -1030,6 +1101,7 @@ static void drbg_async_seed(struct work_struct *work)
         if (drbg->seeded)
                 drbg->reseed_threshold = drbg_max_requests(drbg);
 
+unlock:
         mutex_unlock(&drbg->drbg_mutex);
 
         memzero_explicit(entropy, entropylen);
@@ -1081,7 +1153,9 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
                 BUG_ON((entropylen * 2) > sizeof(entropy));
 
                 /* Get seed from in-kernel /dev/urandom */
-                get_random_bytes(entropy, entropylen);
+                ret = drbg_get_random_bytes(drbg, entropy, entropylen);
+                if (ret)
+                        goto out;
 
                 if (!drbg->jent) {
                         drbg_string_fill(&data1, entropy, entropylen);
@@ -1094,7 +1168,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
                                                    entropylen);
                         if (ret) {
                                 pr_devel("DRBG: jent failed with %d\n", ret);
-                                return ret;
+                                goto out;
                         }
 
                         drbg_string_fill(&data1, entropy, entropylen * 2);
@@ -1121,6 +1195,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
 
         ret = __drbg_seed(drbg, &seedlist, reseed);
 
+out:
         memzero_explicit(entropy, entropylen * 2);
 
         return ret;
@@ -1142,6 +1217,11 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg)
         drbg->reseed_ctr = 0;
         drbg->d_ops = NULL;
         drbg->core = NULL;
+        if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) {
+                kzfree(drbg->prev);
+                drbg->prev = NULL;
+                drbg->fips_primed = false;
+        }
 }
 
 /*
@@ -1211,6 +1291,14 @@ static inline int drbg_alloc_state(struct drbg_state *drbg)
                 drbg->scratchpad = PTR_ALIGN(drbg->scratchpadbuf, ret + 1);
         }
 
+        if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) {
+                drbg->prev = kzalloc(drbg_sec_strength(drbg->core->flags),
+                                     GFP_KERNEL);
+                if (!drbg->prev)
+                        goto fini;
+                drbg->fips_primed = false;
+        }
+
         return 0;
 
 fini:
diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c
index 4e8704405a3b..58f935315cf8 100644
--- a/crypto/fcrypt.c
+++ b/crypto/fcrypt.c
@@ -391,6 +391,7 @@ static int fcrypt_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key
 
 static struct crypto_alg fcrypt_alg = {
         .cra_name               =       "fcrypt",
+        .cra_driver_name        =       "fcrypt-generic",
         .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
         .cra_blocksize          =       8,
         .cra_ctxsize            =       sizeof(struct fcrypt_ctx),
diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c
index 6425b9cd718e..dad9e1f91a78 100644
--- a/crypto/ghash-generic.c
+++ b/crypto/ghash-generic.c
@@ -31,6 +31,7 @@ static int ghash_setkey(struct crypto_shash *tfm,
                         const u8 *key, unsigned int keylen)
 {
         struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
+        be128 k;
 
         if (keylen != GHASH_BLOCK_SIZE) {
                 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
@@ -39,7 +40,12 @@ static int ghash_setkey(struct crypto_shash *tfm,
 
         if (ctx->gf128)
                 gf128mul_free_4k(ctx->gf128);
-        ctx->gf128 = gf128mul_init_4k_lle((be128 *)key);
+
+        BUILD_BUG_ON(sizeof(k) != GHASH_BLOCK_SIZE);
+        memcpy(&k, key, GHASH_BLOCK_SIZE); /* avoid violating alignment rules */
+        ctx->gf128 = gf128mul_init_4k_lle(&k);
+        memzero_explicit(&k, GHASH_BLOCK_SIZE);
+
         if (!ctx->gf128)
                 return -ENOMEM;
 
diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c
index 787dccca3715..701b8d86ab49 100644
--- a/crypto/jitterentropy-kcapi.c
+++ b/crypto/jitterentropy-kcapi.c
@@ -56,11 +56,6 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector);
  * Helper function
  ***************************************************************************/
 
-__u64 jent_rol64(__u64 word, unsigned int shift)
-{
-        return rol64(word, shift);
-}
-
 void *jent_zalloc(unsigned int len)
 {
         return kzalloc(len, GFP_KERNEL);
diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c
index acf44b2d2d1d..77fa2120fe0c 100644
--- a/crypto/jitterentropy.c
+++ b/crypto/jitterentropy.c
@@ -2,7 +2,7 @@
  * Non-physical true random number generator based on timing jitter --
  * Jitter RNG standalone code.
  *
- * Copyright Stephan Mueller <smueller@chronox.de>, 2015
+ * Copyright Stephan Mueller <smueller@chronox.de>, 2015 - 2019
  *
  * Design
  * ======
@@ -47,7 +47,7 @@
 
 /*
  * This Jitterentropy RNG is based on the jitterentropy library
- * version 1.1.0 provided at http://www.chronox.de/jent.html
+ * version 2.1.2 provided at http://www.chronox.de/jent.html
  */
 
 #ifdef __OPTIMIZE__
@@ -71,10 +71,7 @@ struct rand_data {
 #define DATA_SIZE_BITS ((sizeof(__u64)) * 8)
         __u64 last_delta;       /* SENSITIVE stuck test */
         __s64 last_delta2;      /* SENSITIVE stuck test */
-        unsigned int stuck:1;   /* Time measurement stuck */
         unsigned int osr;       /* Oversample rate */
-        unsigned int stir:1;            /* Post-processing stirring */
-        unsigned int disable_unbias:1;  /* Deactivate Von-Neuman unbias */
 #define JENT_MEMORY_BLOCKS 64
 #define JENT_MEMORY_BLOCKSIZE 32
 #define JENT_MEMORY_ACCESSLOOPS 128
@@ -89,8 +86,6 @@ struct rand_data {
 };
 
 /* Flags that can be used to initialize the RNG */
-#define JENT_DISABLE_STIR (1<<0) /* Disable stirring the entropy pool */
-#define JENT_DISABLE_UNBIAS (1<<1) /* Disable the Von-Neuman Unbiaser */
 #define JENT_DISABLE_MEMORY_ACCESS (1<<2) /* Disable memory access for more
                                            * entropy, saves MEMORY_SIZE RAM for
                                            * entropy collector */
@@ -99,19 +94,16 @@ struct rand_data {
 #define JENT_ENOTIME            1 /* Timer service not available */
 #define JENT_ECOARSETIME        2 /* Timer too coarse for RNG */
 #define JENT_ENOMONOTONIC       3 /* Timer is not monotonic increasing */
-#define JENT_EMINVARIATION      4 /* Timer variations too small for RNG */
 #define JENT_EVARVAR            5 /* Timer does not produce variations of
                                    * variations (2nd derivation of time is
                                    * zero). */
-#define JENT_EMINVARVAR         6 /* Timer variations of variations is tooi
-                                   * small. */
+#define JENT_ESTUCK             8 /* Too many stuck results during init. */
 
 /***************************************************************************
  * Helper functions
  ***************************************************************************/
 
 void jent_get_nstime(__u64 *out);
-__u64 jent_rol64(__u64 word, unsigned int shift);
 void *jent_zalloc(unsigned int len);
 void jent_zfree(void *ptr);
 int jent_fips_enabled(void);
@@ -140,16 +132,16 @@ static __u64 jent_loop_shuffle(struct rand_data *ec,
 
         jent_get_nstime(&time);
         /*
-         * mix the current state of the random number into the shuffle
-         * calculation to balance that shuffle a bit more
+         * Mix the current state of the random number into the shuffle
+         * calculation to balance that shuffle a bit more.
          */
         if (ec)
                 time ^= ec->data;
         /*
-         * we fold the time value as much as possible to ensure that as many
-         * bits of the time stamp are included as possible
+         * We fold the time value as much as possible to ensure that as many
+         * bits of the time stamp are included as possible.
          */
-        for (i = 0; (DATA_SIZE_BITS / bits) > i; i++) {
+        for (i = 0; ((DATA_SIZE_BITS + bits - 1) / bits) > i; i++) {
                 shuffle ^= time & mask;
                 time = time >> bits;
         }
@@ -169,38 +161,28 @@ static __u64 jent_loop_shuffle(struct rand_data *ec,
  * CPU Jitter noise source -- this is the noise source based on the CPU
  *                            execution time jitter
  *
- * This function folds the time into one bit units by iterating
- * through the DATA_SIZE_BITS bit time value as follows: assume our time value
- * is 0xabcd
- * 1st loop, 1st shift generates 0xd000
- * 1st loop, 2nd shift generates 0x000d
- * 2nd loop, 1st shift generates 0xcd00
- * 2nd loop, 2nd shift generates 0x000c
- * 3rd loop, 1st shift generates 0xbcd0
- * 3rd loop, 2nd shift generates 0x000b
- * 4th loop, 1st shift generates 0xabcd
- * 4th loop, 2nd shift generates 0x000a
- * Now, the values at the end of the 2nd shifts are XORed together.
+ * This function injects the individual bits of the time value into the
+ * entropy pool using an LFSR.
  *
- * The code is deliberately inefficient and shall stay that way. This function
- * is the root cause why the code shall be compiled without optimization. This
- * function not only acts as folding operation, but this function's execution
- * is used to measure the CPU execution time jitter. Any change to the loop in
- * this function implies that careful retesting must be done.
+ * The code is deliberately inefficient with respect to the bit shifting
+ * and shall stay that way. This function is the root cause why the code
+ * shall be compiled without optimization. This function not only acts as
+ * folding operation, but this function's execution is used to measure
+ * the CPU execution time jitter. Any change to the loop in this function
+ * implies that careful retesting must be done.
  *
  * Input:
  * @ec entropy collector struct -- may be NULL
- * @time time stamp to be folded
+ * @time time stamp to be injected
  * @loop_cnt if a value not equal to 0 is set, use the given value as number of
  *           loops to perform the folding
  *
  * Output:
- * @folded result of folding operation
+ * updated ec->data
  *
  * @return Number of loops the folding operation is performed
  */
-static __u64 jent_fold_time(struct rand_data *ec, __u64 time,
-                            __u64 *folded, __u64 loop_cnt)
+static __u64 jent_lfsr_time(struct rand_data *ec, __u64 time, __u64 loop_cnt)
 {
         unsigned int i;
         __u64 j = 0;
@@ -217,15 +199,34 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time,
         if (loop_cnt)
                 fold_loop_cnt = loop_cnt;
         for (j = 0; j < fold_loop_cnt; j++) {
-                new = 0;
+                new = ec->data;
                 for (i = 1; (DATA_SIZE_BITS) >= i; i++) {
                         __u64 tmp = time << (DATA_SIZE_BITS - i);
 
                         tmp = tmp >> (DATA_SIZE_BITS - 1);
+
+                        /*
+                        * Fibonacci LSFR with polynomial of
+                        *  x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is
+                        *  primitive according to
+                        *   http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf
+                        * (the shift values are the polynomial values minus one
+                        * due to counting bits from 0 to 63). As the current
+                        * position is always the LSB, the polynomial only needs
+                        * to shift data in from the left without wrap.
+                        */
+                        tmp ^= ((new >> 63) & 1);
+                        tmp ^= ((new >> 60) & 1);
+                        tmp ^= ((new >> 55) & 1);
+                        tmp ^= ((new >> 30) & 1);
+                        tmp ^= ((new >> 27) & 1);
+                        tmp ^= ((new >> 22) & 1);
+                        new <<= 1;
                         new ^= tmp;
                 }
         }
-        *folded = new;
+        ec->data = new;
+
         return fold_loop_cnt;
 }
 
@@ -258,7 +259,6 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time,
  */
 static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt)
 {
-        unsigned char *tmpval = NULL;
         unsigned int wrap = 0;
         __u64 i = 0;
 #define MAX_ACC_LOOP_BIT 7
@@ -278,7 +278,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt)
                 acc_loop_cnt = loop_cnt;
 
         for (i = 0; i < (ec->memaccessloops + acc_loop_cnt); i++) {
-                tmpval = ec->mem + ec->memlocation;
+                unsigned char *tmpval = ec->mem + ec->memlocation;
                 /*
                  * memory access: just add 1 to one byte,
                  * wrap at 255 -- memory access implies read
@@ -316,7 +316,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt)
  *      0 jitter measurement not stuck (good bit)
  *      1 jitter measurement stuck (reject bit)
  */
-static void jent_stuck(struct rand_data *ec, __u64 current_delta)
+static int jent_stuck(struct rand_data *ec, __u64 current_delta)
 {
         __s64 delta2 = ec->last_delta - current_delta;
         __s64 delta3 = delta2 - ec->last_delta2;
@@ -325,14 +325,15 @@ static void jent_stuck(struct rand_data *ec, __u64 current_delta)
         ec->last_delta2 = delta2;
 
         if (!current_delta || !delta2 || !delta3)
-                ec->stuck = 1;
+                return 1;
+
+        return 0;
 }
 
 /**
  * This is the heart of the entropy generation: calculate time deltas and
- * use the CPU jitter in the time deltas. The jitter is folded into one
- * bit. You can call this function the "random bit generator" as it
- * produces one random bit per invocation.
+ * use the CPU jitter in the time deltas. The jitter is injected into the
+ * entropy pool.
  *
  * WARNING: ensure that ->prev_time is primed before using the output
  *          of this function! This can be done by calling this function
@@ -341,12 +342,11 @@ static void jent_stuck(struct rand_data *ec, __u64 current_delta)
  * Input:
  * @entropy_collector Reference to entropy collector
  *
- * @return One random bit
+ * @return result of stuck test
  */
-static __u64 jent_measure_jitter(struct rand_data *ec)
+static int jent_measure_jitter(struct rand_data *ec)
 {
         __u64 time = 0;
-        __u64 data = 0;
         __u64 current_delta = 0;
 
         /* Invoke one noise source before time measurement to add variations */
@@ -360,109 +360,11 @@ static __u64 jent_measure_jitter(struct rand_data *ec)
         current_delta = time - ec->prev_time;
         ec->prev_time = time;
 
-        /* Now call the next noise sources which also folds the data */
-        jent_fold_time(ec, current_delta, &data, 0);
-
-        /*
-         * Check whether we have a stuck measurement. The enforcement
-         * is performed after the stuck value has been mixed into the
-         * entropy pool.
-         */
-        jent_stuck(ec, current_delta);
-
-        return data;
-}
-
-/**
- * Von Neuman unbias as explained in RFC 4086 section 4.2. As shown in the
- * documentation of that RNG, the bits from jent_measure_jitter are considered
- * independent which implies that the Von Neuman unbias operation is applicable.
- * A proof of the Von-Neumann unbias operation to remove skews is given in the
- * document "A proposal for: Functionality classes for random number
- * generators", version 2.0 by Werner Schindler, section 5.4.1.
- *
- * Input:
- * @entropy_collector Reference to entropy collector
- *
- * @return One random bit
- */
-static __u64 jent_unbiased_bit(struct rand_data *entropy_collector)
-{
-        do {
-                __u64 a = jent_measure_jitter(entropy_collector);
-                __u64 b = jent_measure_jitter(entropy_collector);
-
-                if (a == b)
-                        continue;
-                if (1 == a)
-                        return 1;
-                else
-                        return 0;
-        } while (1);
-}
-
-/**
- * Shuffle the pool a bit by mixing some value with a bijective function (XOR)
- * into the pool.
- *
- * The function generates a mixer value that depends on the bits set and the
- * location of the set bits in the random number generated by the entropy
- * source. Therefore, based on the generated random number, this mixer value
- * can have 2**64 different values. That mixer value is initialized with the
- * first two SHA-1 constants. After obtaining the mixer value, it is XORed into
- * the random number.
- *
- * The mixer value is not assumed to contain any entropy. But due to the XOR
- * operation, it can also not destroy any entropy present in the entropy pool.
- *
- * Input:
- * @entropy_collector Reference to entropy collector
- */
-static void jent_stir_pool(struct rand_data *entropy_collector)
-{
-        /*
-         * to shut up GCC on 32 bit, we have to initialize the 64 variable
-         * with two 32 bit variables
-         */
-        union c {
-                __u64 u64;
-                __u32 u32[2];
-        };
-        /*
-         * This constant is derived from the first two 32 bit initialization
-         * vectors of SHA-1 as defined in FIPS 180-4 section 5.3.1
-         */
-        union c constant;
-        /*
-         * The start value of the mixer variable is derived from the third
-         * and fourth 32 bit initialization vector of SHA-1 as defined in
-         * FIPS 180-4 section 5.3.1
-         */
-        union c mixer;
-        unsigned int i = 0;
-
-        /*
-         * Store the SHA-1 constants in reverse order to make up the 64 bit
-         * value -- this applies to a little endian system, on a big endian
-         * system, it reverses as expected. But this really does not matter
-         * as we do not rely on the specific numbers. We just pick the SHA-1
-         * constants as they have a good mix of bit set and unset.
-         */
-        constant.u32[1] = 0x67452301;
-        constant.u32[0] = 0xefcdab89;
-        mixer.u32[1] = 0x98badcfe;
-        mixer.u32[0] = 0x10325476;
+        /* Now call the next noise sources which also injects the data */
+        jent_lfsr_time(ec, current_delta, 0);
 
-        for (i = 0; i < DATA_SIZE_BITS; i++) {
-                /*
-                 * get the i-th bit of the input random number and only XOR
-                 * the constant into the mixer value when that bit is set
-                 */
-                if ((entropy_collector->data >> i) & 1)
-                        mixer.u64 ^= constant.u64;
-                mixer.u64 = jent_rol64(mixer.u64, 1);
-        }
-        entropy_collector->data ^= mixer.u64;
+        /* Check whether we have a stuck measurement. */
+        return jent_stuck(ec, current_delta);
 }
 
 /**
@@ -480,48 +382,9 @@ static void jent_gen_entropy(struct rand_data *ec)
         jent_measure_jitter(ec);
 
         while (1) {
-                __u64 data = 0;
-
-                if (ec->disable_unbias == 1)
-                        data = jent_measure_jitter(ec);
-                else
-                        data = jent_unbiased_bit(ec);
-
-                /* enforcement of the jent_stuck test */
-                if (ec->stuck) {
-                        /*
-                         * We only mix in the bit considered not appropriate
-                         * without the LSFR. The reason is that if we apply
-                         * the LSFR and we do not rotate, the 2nd bit with LSFR
-                         * will cancel out the first LSFR application on the
-                         * bad bit.
-                         *
-                         * And we do not rotate as we apply the next bit to the
-                         * current bit location again.
-                         */
-                        ec->data ^= data;
-                        ec->stuck = 0;
+                /* If a stuck measurement is received, repeat measurement */
+                if (jent_measure_jitter(ec))
                         continue;
-                }
-
-                /*
-                 * Fibonacci LSFR with polynom of
-                 *  x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is
-                 *  primitive according to
-                 *   http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf
-                 * (the shift values are the polynom values minus one
-                 * due to counting bits from 0 to 63). As the current
-                 * position is always the LSB, the polynom only needs
-                 * to shift data in from the left without wrap.
-                 */
-                ec->data ^= data;
-                ec->data ^= ((ec->data >> 63) & 1);
-                ec->data ^= ((ec->data >> 60) & 1);
-                ec->data ^= ((ec->data >> 55) & 1);
-                ec->data ^= ((ec->data >> 30) & 1);
-                ec->data ^= ((ec->data >> 27) & 1);
-                ec->data ^= ((ec->data >> 22) & 1);
-                ec->data = jent_rol64(ec->data, 1);
 
                 /*
                  * We multiply the loop value with ->osr to obtain the
@@ -530,8 +393,6 @@ static void jent_gen_entropy(struct rand_data *ec)
                 if (++k >= (DATA_SIZE_BITS * ec->osr))
                         break;
         }
-        if (ec->stir)
-                jent_stir_pool(ec);
 }
 
 /**
@@ -639,12 +500,6 @@ struct rand_data *jent_entropy_collector_alloc(unsigned int osr,
                 osr = 1; /* minimum sampling rate is 1 */
         entropy_collector->osr = osr;
 
-        entropy_collector->stir = 1;
-        if (flags & JENT_DISABLE_STIR)
-                entropy_collector->stir = 0;
-        if (flags & JENT_DISABLE_UNBIAS)
-                entropy_collector->disable_unbias = 1;
-
         /* fill the data pad with non-zero values */
         jent_gen_entropy(entropy_collector);
 
@@ -656,7 +511,6 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector)
         jent_zfree(entropy_collector->mem);
         entropy_collector->mem = NULL;
         jent_zfree(entropy_collector);
-        entropy_collector = NULL;
 }
 
 int jent_entropy_init(void)
@@ -665,8 +519,9 @@ int jent_entropy_init(void)
         __u64 delta_sum = 0;
         __u64 old_delta = 0;
         int time_backwards = 0;
-        int count_var = 0;
         int count_mod = 0;
+        int count_stuck = 0;
+        struct rand_data ec = { 0 };
 
         /* We could perform statistical tests here, but the problem is
          * that we only have a few loop counts to do testing. These
@@ -695,12 +550,14 @@ int jent_entropy_init(void)
         for (i = 0; (TESTLOOPCOUNT + CLEARCACHE) > i; i++) {
                 __u64 time = 0;
                 __u64 time2 = 0;
-                __u64 folded = 0;
                 __u64 delta = 0;
                 unsigned int lowdelta = 0;
+                int stuck;
 
+                /* Invoke core entropy collection logic */
                 jent_get_nstime(&time);
-                jent_fold_time(NULL, time, &folded, 1<<MIN_FOLD_LOOP_BIT);
+                ec.prev_time = time;
+                jent_lfsr_time(&ec, time, 0);
                 jent_get_nstime(&time2);
 
                 /* test whether timer works */
@@ -715,6 +572,8 @@ int jent_entropy_init(void)
                 if (!delta)
                         return JENT_ECOARSETIME;
 
+                stuck = jent_stuck(&ec, delta);
+
                 /*
                  * up to here we did not modify any variable that will be
                  * evaluated later, but we already performed some work. Thus we
@@ -725,14 +584,14 @@ int jent_entropy_init(void)
                 if (CLEARCACHE > i)
                         continue;
 
+                if (stuck)
+                        count_stuck++;
+
                 /* test whether we have an increasing timer */
                 if (!(time2 > time))
                         time_backwards++;
 
-                /*
-                 * Avoid modulo of 64 bit integer to allow code to compile
-                 * on 32 bit architectures.
-                 */
+                /* use 32 bit value to ensure compilation on 32 bit arches */
                 lowdelta = time2 - time;
                 if (!(lowdelta % 100))
                         count_mod++;
@@ -743,14 +602,10 @@ int jent_entropy_init(void)
                  * only after the first loop is executed as we need to prime
                  * the old_data value
                  */
-                if (i) {
-                        if (delta != old_delta)
-                                count_var++;
-                        if (delta > old_delta)
-                                delta_sum += (delta - old_delta);
-                        else
-                                delta_sum += (old_delta - delta);
-                }
+                if (delta > old_delta)
+                        delta_sum += (delta - old_delta);
+                else
+                        delta_sum += (old_delta - delta);
                 old_delta = delta;
         }
 
@@ -763,25 +618,29 @@ int jent_entropy_init(void)
          */
         if (3 < time_backwards)
                 return JENT_ENOMONOTONIC;
-        /* Error if the time variances are always identical */
-        if (!delta_sum)
-                return JENT_EVARVAR;
 
         /*
          * Variations of deltas of time must on average be larger
          * than 1 to ensure the entropy estimation
          * implied with 1 is preserved
          */
-        if (delta_sum <= 1)
-                return JENT_EMINVARVAR;
+        if ((delta_sum) <= 1)
+                return JENT_EVARVAR;
 
         /*
          * Ensure that we have variations in the time stamp below 10 for at
-         * least 10% of all checks -- on some platforms, the counter
-         * increments in multiples of 100, but not always
+         * least 10% of all checks -- on some platforms, the counter increments
+         * in multiples of 100, but not always
          */
         if ((TESTLOOPCOUNT/10 * 9) < count_mod)
                 return JENT_ECOARSETIME;
 
+        /*
+         * If we have more than 90% stuck results, then this Jitter RNG is
+         * likely to not work well.
+         */
+        if ((TESTLOOPCOUNT/10 * 9) < count_stuck)
+                return JENT_ESTUCK;
+
         return 0;
 }
diff --git a/crypto/khazad.c b/crypto/khazad.c
index b50aa8a3ab4c..14ca7f1631c7 100644
--- a/crypto/khazad.c
+++ b/crypto/khazad.c
@@ -848,6 +848,7 @@ static void khazad_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 
 static struct crypto_alg khazad_alg = {
         .cra_name               =       "khazad",
+        .cra_driver_name        =       "khazad-generic",
         .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
         .cra_blocksize          =       KHAZAD_BLOCK_SIZE,
         .cra_ctxsize            =       sizeof (struct khazad_ctx),
diff --git a/crypto/lrw.c b/crypto/lrw.c
index 58009cf63a6e..be829f6afc8e 100644
--- a/crypto/lrw.c
+++ b/crypto/lrw.c
@@ -384,7 +384,7 @@ static int create(struct crypto_template *tmpl, struct rtattr **tb)
         inst->alg.base.cra_priority = alg->base.cra_priority;
         inst->alg.base.cra_blocksize = LRW_BLOCK_SIZE;
         inst->alg.base.cra_alignmask = alg->base.cra_alignmask |
-                                       (__alignof__(__be32) - 1);
+                                       (__alignof__(be128) - 1);
 
         inst->alg.ivsize = LRW_BLOCK_SIZE;
         inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) +
diff --git a/crypto/lz4.c b/crypto/lz4.c
index 54673cf88385..0606f8862e78 100644
--- a/crypto/lz4.c
+++ b/crypto/lz4.c
@@ -106,6 +106,7 @@ static int lz4_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
 
 static struct crypto_alg alg_lz4 = {
         .cra_name               = "lz4",
+        .cra_driver_name        = "lz4-generic",
         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
         .cra_ctxsize            = sizeof(struct lz4_ctx),
         .cra_module             = THIS_MODULE,
diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c
index daae7bfb385d..d7cc94aa2fcf 100644
--- a/crypto/lz4hc.c
+++ b/crypto/lz4hc.c
@@ -107,6 +107,7 @@ static int lz4hc_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
 
 static struct crypto_alg alg_lz4hc = {
         .cra_name               = "lz4hc",
+        .cra_driver_name        = "lz4hc-generic",
         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
         .cra_ctxsize            = sizeof(struct lz4hc_ctx),
         .cra_module             = THIS_MODULE,
diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c
index c4303e96f2b1..0631d975bfac 100644
--- a/crypto/lzo-rle.c
+++ b/crypto/lzo-rle.c
@@ -109,6 +109,7 @@ static int lzorle_sdecompress(struct crypto_scomp *tfm, const u8 *src,
 
 static struct crypto_alg alg = {
         .cra_name               = "lzo-rle",
+        .cra_driver_name        = "lzo-rle-generic",
         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
         .cra_ctxsize            = sizeof(struct lzorle_ctx),
         .cra_module             = THIS_MODULE,
diff --git a/crypto/lzo.c b/crypto/lzo.c
index 97051a2ca08e..ebda132dd22b 100644
--- a/crypto/lzo.c
+++ b/crypto/lzo.c
@@ -109,6 +109,7 @@ static int lzo_sdecompress(struct crypto_scomp *tfm, const u8 *src,
 
 static struct crypto_alg alg = {
         .cra_name               = "lzo",
+        .cra_driver_name        = "lzo-generic",
         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
         .cra_ctxsize            = sizeof(struct lzo_ctx),
         .cra_module             = THIS_MODULE,
diff --git a/crypto/md4.c b/crypto/md4.c
index 9a1a228a0c69..2e7f2f319f95 100644
--- a/crypto/md4.c
+++ b/crypto/md4.c
@@ -216,9 +216,10 @@ static struct shash_alg alg = {
         .final          =       md4_final,
         .descsize       =       sizeof(struct md4_ctx),
         .base           =       {
-                .cra_name       =       "md4",
-                .cra_blocksize  =       MD4_HMAC_BLOCK_SIZE,
-                .cra_module     =       THIS_MODULE,
+                .cra_name        =      "md4",
+                .cra_driver_name =      "md4-generic",
+                .cra_blocksize   =      MD4_HMAC_BLOCK_SIZE,
+                .cra_module      =      THIS_MODULE,
         }
 };
 
diff --git a/crypto/md5.c b/crypto/md5.c
index 221c2c0932f8..22dc60bc0437 100644
--- a/crypto/md5.c
+++ b/crypto/md5.c
@@ -228,9 +228,10 @@ static struct shash_alg alg = {
         .descsize       =       sizeof(struct md5_state),
         .statesize      =       sizeof(struct md5_state),
         .base           =       {
-                .cra_name       =       "md5",
-                .cra_blocksize  =       MD5_HMAC_BLOCK_SIZE,
-                .cra_module     =       THIS_MODULE,
+                .cra_name        =      "md5",
+                .cra_driver_name =      "md5-generic",
+                .cra_blocksize   =      MD5_HMAC_BLOCK_SIZE,
+                .cra_module      =      THIS_MODULE,
         }
 };
 
diff --git a/crypto/michael_mic.c b/crypto/michael_mic.c
index b3d83ff709d3..20e6220f46f6 100644
--- a/crypto/michael_mic.c
+++ b/crypto/michael_mic.c
@@ -156,6 +156,7 @@ static struct shash_alg alg = {
         .descsize               =       sizeof(struct michael_mic_desc_ctx),
         .base                   =       {
                 .cra_name               =       "michael_mic",
+                .cra_driver_name        =       "michael_mic-generic",
                 .cra_blocksize          =       8,
                 .cra_alignmask          =       3,
                 .cra_ctxsize            =       sizeof(struct michael_mic_ctx),
diff --git a/crypto/rmd128.c b/crypto/rmd128.c
index d6c031a9fd14..29308fb97e7e 100644
--- a/crypto/rmd128.c
+++ b/crypto/rmd128.c
@@ -298,6 +298,7 @@ static struct shash_alg alg = {
         .descsize       =       sizeof(struct rmd128_ctx),
         .base           =       {
                 .cra_name        =      "rmd128",
+                .cra_driver_name =      "rmd128-generic",
                 .cra_blocksize   =      RMD128_BLOCK_SIZE,
                 .cra_module      =      THIS_MODULE,
         }
diff --git a/crypto/rmd160.c b/crypto/rmd160.c
index f3add4d54a22..c5fe4034b153 100644
--- a/crypto/rmd160.c
+++ b/crypto/rmd160.c
@@ -342,6 +342,7 @@ static struct shash_alg alg = {
         .descsize       =       sizeof(struct rmd160_ctx),
         .base           =       {
                 .cra_name        =      "rmd160",
+                .cra_driver_name =      "rmd160-generic",
                 .cra_blocksize   =      RMD160_BLOCK_SIZE,
                 .cra_module      =      THIS_MODULE,
         }
diff --git a/crypto/rmd256.c b/crypto/rmd256.c
index 79ca3029848f..3c730e9de5fd 100644
--- a/crypto/rmd256.c
+++ b/crypto/rmd256.c
@@ -317,6 +317,7 @@ static struct shash_alg alg = {
         .descsize       =       sizeof(struct rmd256_ctx),
         .base           =       {
                 .cra_name        =      "rmd256",
+                .cra_driver_name =      "rmd256-generic",
                 .cra_blocksize   =      RMD256_BLOCK_SIZE,
                 .cra_module      =      THIS_MODULE,
         }
diff --git a/crypto/rmd320.c b/crypto/rmd320.c
index b2392ef7467b..c919ad6c4705 100644
--- a/crypto/rmd320.c
+++ b/crypto/rmd320.c
@@ -366,6 +366,7 @@ static struct shash_alg alg = {
         .descsize       =       sizeof(struct rmd320_ctx),
         .base           =       {
                 .cra_name        =      "rmd320",
+                .cra_driver_name =      "rmd320-generic",
                 .cra_blocksize   =      RMD320_BLOCK_SIZE,
                 .cra_module      =      THIS_MODULE,
         }
diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c
index 16f612b6dbca..56fa665a4f01 100644
--- a/crypto/serpent_generic.c
+++ b/crypto/serpent_generic.c
@@ -225,7 +225,13 @@
         x4 ^= x2;                                       \
         })
 
-static void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2, u32 r3, u32 r4, u32 *k)
+/*
+ * both gcc and clang have misoptimized this function in the past,
+ * producing horrible object code from spilling temporary variables
+ * on the stack. Forcing this part out of line avoids that.
+ */
+static noinline void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2,
+                                           u32 r3, u32 r4, u32 *k)
 {
         k += 100;
         S3(r3, r4, r0, r1, r2); store_and_load_keys(r1, r2, r4, r3, 28, 24);
@@ -637,6 +643,7 @@ static struct crypto_alg srp_algs[2] = { {
         .cia_decrypt            =       serpent_decrypt } }
 }, {
         .cra_name               =       "tnepres",
+        .cra_driver_name        =       "tnepres-generic",
         .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
         .cra_blocksize          =       SERPENT_BLOCK_SIZE,
         .cra_ctxsize            =       sizeof(struct serpent_ctx),
diff --git a/crypto/skcipher.c b/crypto/skcipher.c
index df735148000f..5d836fc3df3e 100644
--- a/crypto/skcipher.c
+++ b/crypto/skcipher.c
@@ -837,6 +837,40 @@ static int skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
         return 0;
 }
 
+int crypto_skcipher_encrypt(struct skcipher_request *req)
+{
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+        struct crypto_alg *alg = tfm->base.__crt_alg;
+        unsigned int cryptlen = req->cryptlen;
+        int ret;
+
+        crypto_stats_get(alg);
+        if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
+                ret = -ENOKEY;
+        else
+                ret = tfm->encrypt(req);
+        crypto_stats_skcipher_encrypt(cryptlen, ret, alg);
+        return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_skcipher_encrypt);
+
+int crypto_skcipher_decrypt(struct skcipher_request *req)
+{
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+        struct crypto_alg *alg = tfm->base.__crt_alg;
+        unsigned int cryptlen = req->cryptlen;
+        int ret;
+
+        crypto_stats_get(alg);
+        if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
+                ret = -ENOKEY;
+        else
+                ret = tfm->decrypt(req);
+        crypto_stats_skcipher_decrypt(cryptlen, ret, alg);
+        return ret;
+}
+EXPORT_SYMBOL_GPL(crypto_skcipher_decrypt);
+
 static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm)
 {
         struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm);
diff --git a/crypto/tea.c b/crypto/tea.c
index 37a18a9be2f4..02efc5d81690 100644
--- a/crypto/tea.c
+++ b/crypto/tea.c
@@ -216,6 +216,7 @@ static void xeta_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 
 static struct crypto_alg tea_algs[3] = { {
         .cra_name               =       "tea",
+        .cra_driver_name        =       "tea-generic",
         .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
         .cra_blocksize          =       TEA_BLOCK_SIZE,
         .cra_ctxsize            =       sizeof (struct tea_ctx),
@@ -229,6 +230,7 @@ static struct crypto_alg tea_algs[3] = { {
         .cia_decrypt            =       tea_decrypt } }
 }, {
         .cra_name               =       "xtea",
+        .cra_driver_name        =       "xtea-generic",
         .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
         .cra_blocksize          =       XTEA_BLOCK_SIZE,
         .cra_ctxsize            =       sizeof (struct xtea_ctx),
@@ -242,6 +244,7 @@ static struct crypto_alg tea_algs[3] = { {
         .cia_decrypt            =       xtea_decrypt } }
 }, {
         .cra_name               =       "xeta",
+        .cra_driver_name        =       "xeta-generic",
         .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
         .cra_blocksize          =       XTEA_BLOCK_SIZE,
         .cra_ctxsize            =       sizeof (struct xtea_ctx),
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 658a7eeebab2..d0b5b33806a6 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -1032,6 +1032,205 @@ static void crypto_reenable_simd_for_test(void)
 }
 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
 
+static int build_hash_sglist(struct test_sglist *tsgl,
+                             const struct hash_testvec *vec,
+                             const struct testvec_config *cfg,
+                             unsigned int alignmask,
+                             const struct test_sg_division *divs[XBUFSIZE])
+{
+        struct kvec kv;
+        struct iov_iter input;
+
+        kv.iov_base = (void *)vec->plaintext;
+        kv.iov_len = vec->psize;
+        iov_iter_kvec(&input, WRITE, &kv, 1, vec->psize);
+        return build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize,
+                                 &input, divs);
+}
+
+static int check_hash_result(const char *type,
+                             const u8 *result, unsigned int digestsize,
+                             const struct hash_testvec *vec,
+                             const char *vec_name,
+                             const char *driver,
+                             const struct testvec_config *cfg)
+{
+        if (memcmp(result, vec->digest, digestsize) != 0) {
+                pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
+                       type, driver, vec_name, cfg->name);
+                return -EINVAL;
+        }
+        if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) {
+                pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n",
+                       type, driver, vec_name, cfg->name);
+                return -EOVERFLOW;
+        }
+        return 0;
+}
+
+static inline int check_shash_op(const char *op, int err,
+                                 const char *driver, const char *vec_name,
+                                 const struct testvec_config *cfg)
+{
+        if (err)
+                pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
+                       driver, op, err, vec_name, cfg->name);
+        return err;
+}
+
+static inline const void *sg_data(struct scatterlist *sg)
+{
+        return page_address(sg_page(sg)) + sg->offset;
+}
+
+/* Test one hash test vector in one configuration, using the shash API */
+static int test_shash_vec_cfg(const char *driver,
+                              const struct hash_testvec *vec,
+                              const char *vec_name,
+                              const struct testvec_config *cfg,
+                              struct shash_desc *desc,
+                              struct test_sglist *tsgl,
+                              u8 *hashstate)
+{
+        struct crypto_shash *tfm = desc->tfm;
+        const unsigned int alignmask = crypto_shash_alignmask(tfm);
+        const unsigned int digestsize = crypto_shash_digestsize(tfm);
+        const unsigned int statesize = crypto_shash_statesize(tfm);
+        const struct test_sg_division *divs[XBUFSIZE];
+        unsigned int i;
+        u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];
+        int err;
+
+        /* Set the key, if specified */
+        if (vec->ksize) {
+                err = crypto_shash_setkey(tfm, vec->key, vec->ksize);
+                if (err) {
+                        if (err == vec->setkey_error)
+                                return 0;
+                        pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
+                               driver, vec_name, vec->setkey_error, err,
+                               crypto_shash_get_flags(tfm));
+                        return err;
+                }
+                if (vec->setkey_error) {
+                        pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
+                               driver, vec_name, vec->setkey_error);
+                        return -EINVAL;
+                }
+        }
+
+        /* Build the scatterlist for the source data */
+        err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs);
+        if (err) {
+                pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
+                       driver, vec_name, cfg->name);
+                return err;
+        }
+
+        /* Do the actual hashing */
+
+        testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
+        testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);
+
+        if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||
+            vec->digest_error) {
+                /* Just using digest() */
+                if (tsgl->nents != 1)
+                        return 0;
+                if (cfg->nosimd)
+                        crypto_disable_simd_for_test();
+                err = crypto_shash_digest(desc, sg_data(&tsgl->sgl[0]),
+                                          tsgl->sgl[0].length, result);
+                if (cfg->nosimd)
+                        crypto_reenable_simd_for_test();
+                if (err) {
+                        if (err == vec->digest_error)
+                                return 0;
+                        pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
+                               driver, vec_name, vec->digest_error, err,
+                               cfg->name);
+                        return err;
+                }
+                if (vec->digest_error) {
+                        pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
+                               driver, vec_name, vec->digest_error, cfg->name);
+                        return -EINVAL;
+                }
+                goto result_ready;
+        }
+
+        /* Using init(), zero or more update(), then final() or finup() */
+
+        if (cfg->nosimd)
+                crypto_disable_simd_for_test();
+        err = crypto_shash_init(desc);
+        if (cfg->nosimd)
+                crypto_reenable_simd_for_test();
+        err = check_shash_op("init", err, driver, vec_name, cfg);
+        if (err)
+                return err;
+
+        for (i = 0; i < tsgl->nents; i++) {
+                if (i + 1 == tsgl->nents &&
+                    cfg->finalization_type == FINALIZATION_TYPE_FINUP) {
+                        if (divs[i]->nosimd)
+                                crypto_disable_simd_for_test();
+                        err = crypto_shash_finup(desc, sg_data(&tsgl->sgl[i]),
+                                                 tsgl->sgl[i].length, result);
+                        if (divs[i]->nosimd)
+                                crypto_reenable_simd_for_test();
+                        err = check_shash_op("finup", err, driver, vec_name,
+                                             cfg);
+                        if (err)
+                                return err;
+                        goto result_ready;
+                }
+                if (divs[i]->nosimd)
+                        crypto_disable_simd_for_test();
+                err = crypto_shash_update(desc, sg_data(&tsgl->sgl[i]),
+                                          tsgl->sgl[i].length);
+                if (divs[i]->nosimd)
+                        crypto_reenable_simd_for_test();
+                err = check_shash_op("update", err, driver, vec_name, cfg);
+                if (err)
+                        return err;
+                if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {
+                        /* Test ->export() and ->import() */
+                        testmgr_poison(hashstate + statesize,
+                                       TESTMGR_POISON_LEN);
+                        err = crypto_shash_export(desc, hashstate);
+                        err = check_shash_op("export", err, driver, vec_name,
+                                             cfg);
+                        if (err)
+                                return err;
+                        if (!testmgr_is_poison(hashstate + statesize,
+                                               TESTMGR_POISON_LEN)) {
+                                pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
+                                       driver, vec_name, cfg->name);
+                                return -EOVERFLOW;
+                        }
+                        testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));
+                        err = crypto_shash_import(desc, hashstate);
+                        err = check_shash_op("import", err, driver, vec_name,
+                                             cfg);
+                        if (err)
+                                return err;
+                }
+        }
+
+        if (cfg->nosimd)
+                crypto_disable_simd_for_test();
+        err = crypto_shash_final(desc, result);
+        if (cfg->nosimd)
+                crypto_reenable_simd_for_test();
+        err = check_shash_op("final", err, driver, vec_name, cfg);
+        if (err)
+                return err;
+result_ready:
+        return check_hash_result("shash", result, digestsize, vec, vec_name,
+                                 driver, cfg);
+}
+
 static int do_ahash_op(int (*op)(struct ahash_request *req),
                        struct ahash_request *req,
                        struct crypto_wait *wait, bool nosimd)
@@ -1049,31 +1248,32 @@ static int do_ahash_op(int (*op)(struct ahash_request *req),
         return crypto_wait_req(err, wait);
 }
 
-static int check_nonfinal_hash_op(const char *op, int err,
-                                  u8 *result, unsigned int digestsize,
-                                  const char *driver, const char *vec_name,
-                                  const struct testvec_config *cfg)
+static int check_nonfinal_ahash_op(const char *op, int err,
+                                   u8 *result, unsigned int digestsize,
+                                   const char *driver, const char *vec_name,
+                                   const struct testvec_config *cfg)
 {
         if (err) {
-                pr_err("alg: hash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
+                pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",
                        driver, op, err, vec_name, cfg->name);
                 return err;
         }
         if (!testmgr_is_poison(result, digestsize)) {
-                pr_err("alg: hash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",
+                pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",
                        driver, op, vec_name, cfg->name);
                 return -EINVAL;
         }
         return 0;
 }
 
-static int test_hash_vec_cfg(const char *driver,
-                             const struct hash_testvec *vec,
-                             const char *vec_name,
-                             const struct testvec_config *cfg,
-                             struct ahash_request *req,
-                             struct test_sglist *tsgl,
-                             u8 *hashstate)
+/* Test one hash test vector in one configuration, using the ahash API */
+static int test_ahash_vec_cfg(const char *driver,
+                              const struct hash_testvec *vec,
+                              const char *vec_name,
+                              const struct testvec_config *cfg,
+                              struct ahash_request *req,
+                              struct test_sglist *tsgl,
+                              u8 *hashstate)
 {
         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
         const unsigned int alignmask = crypto_ahash_alignmask(tfm);
@@ -1082,8 +1282,6 @@ static int test_hash_vec_cfg(const char *driver,
         const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;
         const struct test_sg_division *divs[XBUFSIZE];
         DECLARE_CRYPTO_WAIT(wait);
-        struct kvec _input;
-        struct iov_iter input;
         unsigned int i;
         struct scatterlist *pending_sgl;
         unsigned int pending_len;
@@ -1096,26 +1294,22 @@ static int test_hash_vec_cfg(const char *driver,
                 if (err) {
                         if (err == vec->setkey_error)
                                 return 0;
-                        pr_err("alg: hash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
+                        pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",
                                driver, vec_name, vec->setkey_error, err,
                                crypto_ahash_get_flags(tfm));
                         return err;
                 }
                 if (vec->setkey_error) {
-                        pr_err("alg: hash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
+                        pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",
                                driver, vec_name, vec->setkey_error);
                         return -EINVAL;
                 }
         }
 
         /* Build the scatterlist for the source data */
-        _input.iov_base = (void *)vec->plaintext;
-        _input.iov_len = vec->psize;
-        iov_iter_kvec(&input, WRITE, &_input, 1, vec->psize);
-        err = build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize,
-                                &input, divs);
+        err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs);
         if (err) {
-                pr_err("alg: hash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
+                pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",
                        driver, vec_name, cfg->name);
                 return err;
         }
@@ -1135,13 +1329,13 @@ static int test_hash_vec_cfg(const char *driver,
                 if (err) {
                         if (err == vec->digest_error)
                                 return 0;
-                        pr_err("alg: hash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
+                        pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",
                                driver, vec_name, vec->digest_error, err,
                                cfg->name);
                         return err;
                 }
                 if (vec->digest_error) {
-                        pr_err("alg: hash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
+                        pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",
                                driver, vec_name, vec->digest_error, cfg->name);
                         return -EINVAL;
                 }
@@ -1153,8 +1347,8 @@ static int test_hash_vec_cfg(const char *driver,
         ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);
         ahash_request_set_crypt(req, NULL, result, 0);
         err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd);
-        err = check_nonfinal_hash_op("init", err, result, digestsize,
-                                     driver, vec_name, cfg);
+        err = check_nonfinal_ahash_op("init", err, result, digestsize,
+                                      driver, vec_name, cfg);
         if (err)
                 return err;
 
@@ -1170,9 +1364,9 @@ static int test_hash_vec_cfg(const char *driver,
                                                 pending_len);
                         err = do_ahash_op(crypto_ahash_update, req, &wait,
                                           divs[i]->nosimd);
-                        err = check_nonfinal_hash_op("update", err,
-                                                     result, digestsize,
-                                                     driver, vec_name, cfg);
+                        err = check_nonfinal_ahash_op("update", err,
+                                                      result, digestsize,
+                                                      driver, vec_name, cfg);
                         if (err)
                                 return err;
                         pending_sgl = NULL;
@@ -1183,23 +1377,23 @@ static int test_hash_vec_cfg(const char *driver,
                         testmgr_poison(hashstate + statesize,
                                        TESTMGR_POISON_LEN);
                         err = crypto_ahash_export(req, hashstate);
-                        err = check_nonfinal_hash_op("export", err,
-                                                     result, digestsize,
-                                                     driver, vec_name, cfg);
+                        err = check_nonfinal_ahash_op("export", err,
+                                                      result, digestsize,
+                                                      driver, vec_name, cfg);
                         if (err)
                                 return err;
                         if (!testmgr_is_poison(hashstate + statesize,
                                                TESTMGR_POISON_LEN)) {
-                                pr_err("alg: hash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
+                                pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",
                                        driver, vec_name, cfg->name);
                                 return -EOVERFLOW;
                         }
 
                         testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));
                         err = crypto_ahash_import(req, hashstate);
-                        err = check_nonfinal_hash_op("import", err,
-                                                     result, digestsize,
-                                                     driver, vec_name, cfg);
+                        err = check_nonfinal_ahash_op("import", err,
+                                                      result, digestsize,
+                                                      driver, vec_name, cfg);
                         if (err)
                                 return err;
                 }
@@ -1213,13 +1407,13 @@ static int test_hash_vec_cfg(const char *driver,
         if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) {
                 /* finish with update() and final() */
                 err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd);
-                err = check_nonfinal_hash_op("update", err, result, digestsize,
-                                             driver, vec_name, cfg);
+                err = check_nonfinal_ahash_op("update", err, result, digestsize,
+                                              driver, vec_name, cfg);
                 if (err)
                         return err;
                 err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd);
                 if (err) {
-                        pr_err("alg: hash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",
+                        pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",
                                driver, err, vec_name, cfg->name);
                         return err;
                 }
@@ -1227,31 +1421,49 @@ static int test_hash_vec_cfg(const char *driver,
                 /* finish with finup() */
                 err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd);
                 if (err) {
-                        pr_err("alg: hash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",
+                        pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",
                                driver, err, vec_name, cfg->name);
                         return err;
                 }
         }
 
 result_ready:
-        /* Check that the algorithm produced the correct digest */
-        if (memcmp(result, vec->digest, digestsize) != 0) {
-                pr_err("alg: hash: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",
-                       driver, vec_name, cfg->name);
-                return -EINVAL;
-        }
-        if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) {
-                pr_err("alg: hash: %s overran result buffer on test vector %s, cfg=\"%s\"\n",
-                       driver, vec_name, cfg->name);
-                return -EOVERFLOW;
+        return check_hash_result("ahash", result, digestsize, vec, vec_name,
+                                 driver, cfg);
+}
+
+static int test_hash_vec_cfg(const char *driver,
+                             const struct hash_testvec *vec,
+                             const char *vec_name,
+                             const struct testvec_config *cfg,
+                             struct ahash_request *req,
+                             struct shash_desc *desc,
+                             struct test_sglist *tsgl,
+                             u8 *hashstate)
+{
+        int err;
+
+        /*
+         * For algorithms implemented as "shash", most bugs will be detected by
+         * both the shash and ahash tests.  Test the shash API first so that the
+         * failures involve less indirection, so are easier to debug.
+         */
+
+        if (desc) {
+                err = test_shash_vec_cfg(driver, vec, vec_name, cfg, desc, tsgl,
+                                         hashstate);
+                if (err)
+                        return err;
         }
 
-        return 0;
+        return test_ahash_vec_cfg(driver, vec, vec_name, cfg, req, tsgl,
+                                  hashstate);
 }
 
 static int test_hash_vec(const char *driver, const struct hash_testvec *vec,
                          unsigned int vec_num, struct ahash_request *req,
-                         struct test_sglist *tsgl, u8 *hashstate)
+                         struct shash_desc *desc, struct test_sglist *tsgl,
+                         u8 *hashstate)
 {
         char vec_name[16];
         unsigned int i;
@@ -1262,7 +1474,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec,
         for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) {
                 err = test_hash_vec_cfg(driver, vec, vec_name,
                                         &default_hash_testvec_configs[i],
-                                        req, tsgl, hashstate);
+                                        req, desc, tsgl, hashstate);
                 if (err)
                         return err;
         }
@@ -1276,9 +1488,10 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec,
                         generate_random_testvec_config(&cfg, cfgname,
                                                        sizeof(cfgname));
                         err = test_hash_vec_cfg(driver, vec, vec_name, &cfg,
-                                                req, tsgl, hashstate);
+                                                req, desc, tsgl, hashstate);
                         if (err)
                                 return err;
+                        cond_resched();
                 }
         }
 #endif
@@ -1290,14 +1503,12 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec,
  * Generate a hash test vector from the given implementation.
  * Assumes the buffers in 'vec' were already allocated.
  */
-static void generate_random_hash_testvec(struct crypto_shash *tfm,
+static void generate_random_hash_testvec(struct shash_desc *desc,
                                          struct hash_testvec *vec,
                                          unsigned int maxkeysize,
                                          unsigned int maxdatasize,
                                          char *name, size_t max_namelen)
 {
-        SHASH_DESC_ON_STACK(desc, tfm);
-
         /* Data */
         vec->psize = generate_random_length(maxdatasize);
         generate_random_bytes((u8 *)vec->plaintext, vec->psize);
@@ -1314,7 +1525,7 @@ static void generate_random_hash_testvec(struct crypto_shash *tfm,
                         vec->ksize = 1 + (prandom_u32() % maxkeysize);
                 generate_random_bytes((u8 *)vec->key, vec->ksize);
 
-                vec->setkey_error = crypto_shash_setkey(tfm, vec->key,
+                vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key,
                                                         vec->ksize);
                 /* If the key couldn't be set, no need to continue to digest. */
                 if (vec->setkey_error)
@@ -1322,7 +1533,6 @@ static void generate_random_hash_testvec(struct crypto_shash *tfm,
         }
 
         /* Digest */
-        desc->tfm = tfm;
         vec->digest_error = crypto_shash_digest(desc, vec->plaintext,
                                                 vec->psize, (u8 *)vec->digest);
 done:
@@ -1338,6 +1548,7 @@ static int test_hash_vs_generic_impl(const char *driver,
                                      const char *generic_driver,
                                      unsigned int maxkeysize,
                                      struct ahash_request *req,
+                                     struct shash_desc *desc,
                                      struct test_sglist *tsgl,
                                      u8 *hashstate)
 {
@@ -1348,10 +1559,11 @@ static int test_hash_vs_generic_impl(const char *driver,
         const char *algname = crypto_hash_alg_common(tfm)->base.cra_name;
         char _generic_driver[CRYPTO_MAX_ALG_NAME];
         struct crypto_shash *generic_tfm = NULL;
+        struct shash_desc *generic_desc = NULL;
         unsigned int i;
         struct hash_testvec vec = { 0 };
         char vec_name[64];
-        struct testvec_config cfg;
+        struct testvec_config *cfg;
         char cfgname[TESTVEC_CONFIG_NAMELEN];
         int err;
 
@@ -1381,6 +1593,20 @@ static int test_hash_vs_generic_impl(const char *driver,
                 return err;
         }
 
+        cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
+        if (!cfg) {
+                err = -ENOMEM;
+                goto out;
+        }
+
+        generic_desc = kzalloc(sizeof(*desc) +
+                               crypto_shash_descsize(generic_tfm), GFP_KERNEL);
+        if (!generic_desc) {
+                err = -ENOMEM;
+                goto out;
+        }
+        generic_desc->tfm = generic_tfm;
+
         /* Check the algorithm properties for consistency. */
 
         if (digestsize != crypto_shash_digestsize(generic_tfm)) {
@@ -1412,23 +1638,25 @@ static int test_hash_vs_generic_impl(const char *driver,
         }
 
         for (i = 0; i < fuzz_iterations * 8; i++) {
-                generate_random_hash_testvec(generic_tfm, &vec,
+                generate_random_hash_testvec(generic_desc, &vec,
                                              maxkeysize, maxdatasize,
                                              vec_name, sizeof(vec_name));
-                generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname));
+                generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
 
-                err = test_hash_vec_cfg(driver, &vec, vec_name, &cfg,
-                                        req, tsgl, hashstate);
+                err = test_hash_vec_cfg(driver, &vec, vec_name, cfg,
+                                        req, desc, tsgl, hashstate);
                 if (err)
                         goto out;
                 cond_resched();
         }
         err = 0;
 out:
+        kfree(cfg);
         kfree(vec.key);
         kfree(vec.plaintext);
         kfree(vec.digest);
         crypto_free_shash(generic_tfm);
+        kzfree(generic_desc);
         return err;
 }
 #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
@@ -1436,6 +1664,7 @@ static int test_hash_vs_generic_impl(const char *driver,
                                      const char *generic_driver,
                                      unsigned int maxkeysize,
                                      struct ahash_request *req,
+                                     struct shash_desc *desc,
                                      struct test_sglist *tsgl,
                                      u8 *hashstate)
 {
@@ -1443,26 +1672,67 @@ static int test_hash_vs_generic_impl(const char *driver,
 }
 #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */
 
+static int alloc_shash(const char *driver, u32 type, u32 mask,
+                       struct crypto_shash **tfm_ret,
+                       struct shash_desc **desc_ret)
+{
+        struct crypto_shash *tfm;
+        struct shash_desc *desc;
+
+        tfm = crypto_alloc_shash(driver, type, mask);
+        if (IS_ERR(tfm)) {
+                if (PTR_ERR(tfm) == -ENOENT) {
+                        /*
+                         * This algorithm is only available through the ahash
+                         * API, not the shash API, so skip the shash tests.
+                         */
+                        return 0;
+                }
+                pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n",
+                       driver, PTR_ERR(tfm));
+                return PTR_ERR(tfm);
+        }
+
+        desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL);
+        if (!desc) {
+                crypto_free_shash(tfm);
+                return -ENOMEM;
+        }
+        desc->tfm = tfm;
+
+        *tfm_ret = tfm;
+        *desc_ret = desc;
+        return 0;
+}
+
 static int __alg_test_hash(const struct hash_testvec *vecs,
                            unsigned int num_vecs, const char *driver,
                            u32 type, u32 mask,
                            const char *generic_driver, unsigned int maxkeysize)
 {
-        struct crypto_ahash *tfm;
+        struct crypto_ahash *atfm = NULL;
         struct ahash_request *req = NULL;
+        struct crypto_shash *stfm = NULL;
+        struct shash_desc *desc = NULL;
         struct test_sglist *tsgl = NULL;
         u8 *hashstate = NULL;
+        unsigned int statesize;
         unsigned int i;
         int err;
 
-        tfm = crypto_alloc_ahash(driver, type, mask);
-        if (IS_ERR(tfm)) {
+        /*
+         * Always test the ahash API.  This works regardless of whether the
+         * algorithm is implemented as ahash or shash.
+         */
+
+        atfm = crypto_alloc_ahash(driver, type, mask);
+        if (IS_ERR(atfm)) {
                 pr_err("alg: hash: failed to allocate transform for %s: %ld\n",
-                       driver, PTR_ERR(tfm));
-                return PTR_ERR(tfm);
+                       driver, PTR_ERR(atfm));
+                return PTR_ERR(atfm);
         }
 
-        req = ahash_request_alloc(tfm, GFP_KERNEL);
+        req = ahash_request_alloc(atfm, GFP_KERNEL);
         if (!req) {
                 pr_err("alg: hash: failed to allocate request for %s\n",
                        driver);
@@ -1470,6 +1740,14 @@ static int __alg_test_hash(const struct hash_testvec *vecs,
                 goto out;
         }
 
+        /*
+         * If available also test the shash API, to cover corner cases that may
+         * be missed by testing the ahash API only.
+         */
+        err = alloc_shash(driver, type, mask, &stfm, &desc);
+        if (err)
+                goto out;
+
         tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL);
         if (!tsgl || init_test_sglist(tsgl) != 0) {
                 pr_err("alg: hash: failed to allocate test buffers for %s\n",
@@ -1480,8 +1758,10 @@ static int __alg_test_hash(const struct hash_testvec *vecs,
                 goto out;
         }
 
-        hashstate = kmalloc(crypto_ahash_statesize(tfm) + TESTMGR_POISON_LEN,
-                            GFP_KERNEL);
+        statesize = crypto_ahash_statesize(atfm);
+        if (stfm)
+                statesize = max(statesize, crypto_shash_statesize(stfm));
+        hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL);
         if (!hashstate) {
                 pr_err("alg: hash: failed to allocate hash state buffer for %s\n",
                        driver);
@@ -1490,20 +1770,24 @@ static int __alg_test_hash(const struct hash_testvec *vecs,
         }
 
         for (i = 0; i < num_vecs; i++) {
-                err = test_hash_vec(driver, &vecs[i], i, req, tsgl, hashstate);
+                err = test_hash_vec(driver, &vecs[i], i, req, desc, tsgl,
+                                    hashstate);
                 if (err)
                         goto out;
+                cond_resched();
         }
         err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req,
-                                        tsgl, hashstate);
+                                        desc, tsgl, hashstate);
 out:
         kfree(hashstate);
         if (tsgl) {
                 destroy_test_sglist(tsgl);
                 kfree(tsgl);
         }
+        kfree(desc);
+        crypto_free_shash(stfm);
         ahash_request_free(req);
-        crypto_free_ahash(tfm);
+        crypto_free_ahash(atfm);
         return err;
 }
 
@@ -1755,6 +2039,7 @@ static int test_aead_vec(const char *driver, int enc,
                                                 &cfg, req, tsgls);
                         if (err)
                                 return err;
+                        cond_resched();
                 }
         }
 #endif
@@ -1864,7 +2149,7 @@ static int test_aead_vs_generic_impl(const char *driver,
         unsigned int i;
         struct aead_testvec vec = { 0 };
         char vec_name[64];
-        struct testvec_config cfg;
+        struct testvec_config *cfg;
         char cfgname[TESTVEC_CONFIG_NAMELEN];
         int err;
 
@@ -1894,6 +2179,12 @@ static int test_aead_vs_generic_impl(const char *driver,
                 return err;
         }
 
+        cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
+        if (!cfg) {
+                err = -ENOMEM;
+                goto out;
+        }
+
         generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL);
         if (!generic_req) {
                 err = -ENOMEM;
@@ -1948,13 +2239,13 @@ static int test_aead_vs_generic_impl(const char *driver,
                 generate_random_aead_testvec(generic_req, &vec,
                                              maxkeysize, maxdatasize,
                                              vec_name, sizeof(vec_name));
-                generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname));
+                generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
 
-                err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, &cfg,
+                err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, cfg,
                                         req, tsgls);
                 if (err)
                         goto out;
-                err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, &cfg,
+                err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, cfg,
                                         req, tsgls);
                 if (err)
                         goto out;
@@ -1962,6 +2253,7 @@ static int test_aead_vs_generic_impl(const char *driver,
         }
         err = 0;
 out:
+        kfree(cfg);
         kfree(vec.key);
         kfree(vec.iv);
         kfree(vec.assoc);
@@ -1994,6 +2286,7 @@ static int test_aead(const char *driver, int enc,
                                     tsgls);
                 if (err)
                         return err;
+                cond_resched();
         }
         return 0;
 }
@@ -2336,6 +2629,7 @@ static int test_skcipher_vec(const char *driver, int enc,
                                                     &cfg, req, tsgls);
                         if (err)
                                 return err;
+                        cond_resched();
                 }
         }
 #endif
@@ -2409,7 +2703,7 @@ static int test_skcipher_vs_generic_impl(const char *driver,
         unsigned int i;
         struct cipher_testvec vec = { 0 };
         char vec_name[64];
-        struct testvec_config cfg;
+        struct testvec_config *cfg;
         char cfgname[TESTVEC_CONFIG_NAMELEN];
         int err;
 
@@ -2443,6 +2737,12 @@ static int test_skcipher_vs_generic_impl(const char *driver,
                 return err;
         }
 
+        cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
+        if (!cfg) {
+                err = -ENOMEM;
+                goto out;
+        }
+
         generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL);
         if (!generic_req) {
                 err = -ENOMEM;
@@ -2490,20 +2790,21 @@ static int test_skcipher_vs_generic_impl(const char *driver,
         for (i = 0; i < fuzz_iterations * 8; i++) {
                 generate_random_cipher_testvec(generic_req, &vec, maxdatasize,
                                                vec_name, sizeof(vec_name));
-                generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname));
+                generate_random_testvec_config(cfg, cfgname, sizeof(cfgname));
 
                 err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name,
-                                            &cfg, req, tsgls);
+                                            cfg, req, tsgls);
                 if (err)
                         goto out;
                 err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name,
-                                            &cfg, req, tsgls);
+                                            cfg, req, tsgls);
                 if (err)
                         goto out;
                 cond_resched();
         }
         err = 0;
 out:
+        kfree(cfg);
         kfree(vec.key);
         kfree(vec.iv);
         kfree(vec.ptext);
@@ -2535,6 +2836,7 @@ static int test_skcipher(const char *driver, int enc,
                                         tsgls);
                 if (err)
                         return err;
+                cond_resched();
         }
         return 0;
 }
@@ -4125,6 +4427,7 @@ static const struct alg_test_desc alg_test_descs[] = {
                 }
         }, {
                 .alg = "ecb(arc4)",
+                .generic_driver = "ecb(arc4)-generic",
                 .test = alg_test_skcipher,
                 .suite = {
                         .cipher = __VECS(arc4_tv_template)
@@ -4790,6 +5093,13 @@ static const struct alg_test_desc alg_test_descs[] = {
                 .test = alg_test_null,
                 .fips_allowed = 1,
         }, {
+                .alg = "xxhash64",
+                .test = alg_test_hash,
+                .fips_allowed = 1,
+                .suite = {
+                        .hash = __VECS(xxhash64_tv_template)
+                }
+        }, {
                 .alg = "zlib-deflate",
                 .test = alg_test_comp,
                 .fips_allowed = 1,
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 1fdae5993bc3..073bd2efafca 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -38,7 +38,7 @@ struct hash_testvec {
         const char *key;
         const char *plaintext;
         const char *digest;
-        unsigned short psize;
+        unsigned int psize;
         unsigned short ksize;
         int setkey_error;
         int digest_error;
@@ -69,7 +69,7 @@ struct cipher_testvec {
         const char *ctext;
         unsigned char wk; /* weak key flag */
         unsigned short klen;
-        unsigned short len;
+        unsigned int len;
         bool fips_skip;
         bool generates_iv;
         int setkey_error;
@@ -105,9 +105,9 @@ struct aead_testvec {
         unsigned char novrfy;
         unsigned char wk;
         unsigned char klen;
-        unsigned short plen;
-        unsigned short clen;
-        unsigned short alen;
+        unsigned int plen;
+        unsigned int clen;
+        unsigned int alen;
         int setkey_error;
         int setauthsize_error;
         int crypt_error;
@@ -33382,6 +33382,112 @@ static const struct hash_testvec crc32c_tv_template[] = {
         }
 };
 
+static const struct hash_testvec xxhash64_tv_template[] = {
+        {
+                .psize = 0,
+                .digest = "\x99\xe9\xd8\x51\x37\xdb\x46\xef",
+        },
+        {
+                .plaintext = "\x40",
+                .psize = 1,
+                .digest = "\x20\x5c\x91\xaa\x88\xeb\x59\xd0",
+        },
+        {
+                .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d"
+                             "\x88\xc7\x9a\x09\x1a\x9b",
+                .psize = 14,
+                .digest = "\xa8\xe8\x2b\xa9\x92\xa1\x37\x4a",
+        },
+        {
+                .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d"
+                             "\x88\xc7\x9a\x09\x1a\x9b\x42\xe0"
+                             "\xd4\x38\xa5\x2a\x26\xa5\x19\x4b"
+                             "\x57\x65\x7f\xad\xc3\x7d\xca\x40"
+                             "\x31\x65\x05\xbb\x31\xae\x51\x11"
+                             "\xa8\xc0\xb3\x28\x42\xeb\x3c\x46"
+                             "\xc8\xed\xed\x0f\x8d\x0b\xfa\x6e"
+                             "\xbc\xe3\x88\x53\xca\x8f\xc8\xd9"
+                             "\x41\x26\x7a\x3d\x21\xdb\x1a\x3c"
+                             "\x01\x1d\xc9\xe9\xb7\x3a\x78\x67"
+                             "\x57\x20\x94\xf1\x1e\xfd\xce\x39"
+                             "\x99\x57\x69\x39\xa5\xd0\x8d\xd9"
+                             "\x43\xfe\x1d\x66\x04\x3c\x27\x6a"
+                             "\xe1\x0d\xe7\xc9\xfa\xc9\x07\x56"
+                             "\xa5\xb3\xec\xd9\x1f\x42\x65\x66"
+                             "\xaa\xbf\x87\x9b\xc5\x41\x9c\x27"
+                             "\x3f\x2f\xa9\x55\x93\x01\x27\x33"
+                             "\x43\x99\x4d\x81\x85\xae\x82\x00"
+                             "\x6c\xd0\xd1\xa3\x57\x18\x06\xcc"
+                             "\xec\x72\xf7\x8e\x87\x2d\x1f\x5e"
+                             "\xd7\x5b\x1f\x36\x4c\xfa\xfd\x18"
+                             "\x89\x76\xd3\x5e\xb5\x5a\xc0\x01"
+                             "\xd2\xa1\x9a\x50\xe6\x08\xb4\x76"
+                             "\x56\x4f\x0e\xbc\x54\xfc\x67\xe6"
+                             "\xb9\xc0\x28\x4b\xb5\xc3\xff\x79"
+                             "\x52\xea\xa1\x90\xc3\xaf\x08\x70"
+                             "\x12\x02\x0c\xdb\x94\x00\x38\x95"
+                             "\xed\xfd\x08\xf7\xe8\x04",
+                .psize = 222,
+                .digest = "\x41\xfc\xd4\x29\xfe\xe7\x85\x17",
+        },
+        {
+                .psize = 0,
+                .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00",
+                .ksize = 8,
+                .digest = "\xef\x17\x9b\x92\xa2\xfd\x75\xac",
+        },
+
+        {
+                .plaintext = "\x40",
+                .psize = 1,
+                .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00",
+                .ksize = 8,
+                .digest = "\xd1\x70\x4f\x14\x02\xc4\x9e\x71",
+        },
+        {
+                .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d"
+                             "\x88\xc7\x9a\x09\x1a\x9b",
+                .psize = 14,
+                .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00",
+                .ksize = 8,
+                .digest = "\xa4\xcd\xfe\x8e\x37\xe2\x1c\x64"
+        },
+        {
+                .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d"
+                             "\x88\xc7\x9a\x09\x1a\x9b\x42\xe0"
+                             "\xd4\x38\xa5\x2a\x26\xa5\x19\x4b"
+                             "\x57\x65\x7f\xad\xc3\x7d\xca\x40"
+                             "\x31\x65\x05\xbb\x31\xae\x51\x11"
+                             "\xa8\xc0\xb3\x28\x42\xeb\x3c\x46"
+                             "\xc8\xed\xed\x0f\x8d\x0b\xfa\x6e"
+                             "\xbc\xe3\x88\x53\xca\x8f\xc8\xd9"
+                             "\x41\x26\x7a\x3d\x21\xdb\x1a\x3c"
+                             "\x01\x1d\xc9\xe9\xb7\x3a\x78\x67"
+                             "\x57\x20\x94\xf1\x1e\xfd\xce\x39"
+                             "\x99\x57\x69\x39\xa5\xd0\x8d\xd9"
+                             "\x43\xfe\x1d\x66\x04\x3c\x27\x6a"
+                             "\xe1\x0d\xe7\xc9\xfa\xc9\x07\x56"
+                             "\xa5\xb3\xec\xd9\x1f\x42\x65\x66"
+                             "\xaa\xbf\x87\x9b\xc5\x41\x9c\x27"
+                             "\x3f\x2f\xa9\x55\x93\x01\x27\x33"
+                             "\x43\x99\x4d\x81\x85\xae\x82\x00"
+                             "\x6c\xd0\xd1\xa3\x57\x18\x06\xcc"
+                             "\xec\x72\xf7\x8e\x87\x2d\x1f\x5e"
+                             "\xd7\x5b\x1f\x36\x4c\xfa\xfd\x18"
+                             "\x89\x76\xd3\x5e\xb5\x5a\xc0\x01"
+                             "\xd2\xa1\x9a\x50\xe6\x08\xb4\x76"
+                             "\x56\x4f\x0e\xbc\x54\xfc\x67\xe6"
+                             "\xb9\xc0\x28\x4b\xb5\xc3\xff\x79"
+                             "\x52\xea\xa1\x90\xc3\xaf\x08\x70"
+                             "\x12\x02\x0c\xdb\x94\x00\x38\x95"
+                             "\xed\xfd\x08\xf7\xe8\x04",
+                .psize = 222,
+                .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00",
+                .ksize = 8,
+                .digest = "\x58\xbc\x55\xf2\x42\x81\x5c\xf0"
+        },
+};
+
 static const struct comp_testvec lz4_comp_tv_template[] = {
         {
                 .inlen  = 255,
diff --git a/crypto/tgr192.c b/crypto/tgr192.c
index 702c2c89c7a1..052648e24909 100644
--- a/crypto/tgr192.c
+++ b/crypto/tgr192.c
@@ -630,9 +630,10 @@ static struct shash_alg tgr_algs[3] = { {
         .final          =       tgr192_final,
         .descsize       =       sizeof(struct tgr192_ctx),
         .base           =       {
-                .cra_name       =       "tgr192",
-                .cra_blocksize  =       TGR192_BLOCK_SIZE,
-                .cra_module     =       THIS_MODULE,
+                .cra_name        =      "tgr192",
+                .cra_driver_name =      "tgr192-generic",
+                .cra_blocksize   =      TGR192_BLOCK_SIZE,
+                .cra_module      =      THIS_MODULE,
         }
 }, {
         .digestsize     =       TGR160_DIGEST_SIZE,
@@ -641,9 +642,10 @@ static struct shash_alg tgr_algs[3] = { {
         .final          =       tgr160_final,
         .descsize       =       sizeof(struct tgr192_ctx),
         .base           =       {
-                .cra_name       =       "tgr160",
-                .cra_blocksize  =       TGR192_BLOCK_SIZE,
-                .cra_module     =       THIS_MODULE,
+                .cra_name        =      "tgr160",
+                .cra_driver_name =      "tgr160-generic",
+                .cra_blocksize   =      TGR192_BLOCK_SIZE,
+                .cra_module      =      THIS_MODULE,
         }
 }, {
         .digestsize     =       TGR128_DIGEST_SIZE,
@@ -652,9 +654,10 @@ static struct shash_alg tgr_algs[3] = { {
         .final          =       tgr128_final,
         .descsize       =       sizeof(struct tgr192_ctx),
         .base           =       {
-                .cra_name       =       "tgr128",
-                .cra_blocksize  =       TGR192_BLOCK_SIZE,
-                .cra_module     =       THIS_MODULE,
+                .cra_name        =      "tgr128",
+                .cra_driver_name =      "tgr128-generic",
+                .cra_blocksize   =      TGR192_BLOCK_SIZE,
+                .cra_module      =      THIS_MODULE,
         }
 } };
 
diff --git a/crypto/wp512.c b/crypto/wp512.c
index 1b8e502d999f..feadc13ccae0 100644
--- a/crypto/wp512.c
+++ b/crypto/wp512.c
@@ -1126,9 +1126,10 @@ static struct shash_alg wp_algs[3] = { {
         .final          =       wp512_final,
         .descsize       =       sizeof(struct wp512_ctx),
         .base           =       {
-                .cra_name       =       "wp512",
-                .cra_blocksize  =       WP512_BLOCK_SIZE,
-                .cra_module     =       THIS_MODULE,
+                .cra_name        =      "wp512",
+                .cra_driver_name =      "wp512-generic",
+                .cra_blocksize   =      WP512_BLOCK_SIZE,
+                .cra_module      =      THIS_MODULE,
         }
 }, {
         .digestsize     =       WP384_DIGEST_SIZE,
@@ -1137,9 +1138,10 @@ static struct shash_alg wp_algs[3] = { {
         .final          =       wp384_final,
         .descsize       =       sizeof(struct wp512_ctx),
         .base           =       {
-                .cra_name       =       "wp384",
-                .cra_blocksize  =       WP512_BLOCK_SIZE,
-                .cra_module     =       THIS_MODULE,
+                .cra_name        =      "wp384",
+                .cra_driver_name =      "wp384-generic",
+                .cra_blocksize   =      WP512_BLOCK_SIZE,
+                .cra_module      =      THIS_MODULE,
         }
 }, {
         .digestsize     =       WP256_DIGEST_SIZE,
@@ -1148,9 +1150,10 @@ static struct shash_alg wp_algs[3] = { {
         .final          =       wp256_final,
         .descsize       =       sizeof(struct wp512_ctx),
         .base           =       {
-                .cra_name       =       "wp256",
-                .cra_blocksize  =       WP512_BLOCK_SIZE,
-                .cra_module     =       THIS_MODULE,
+                .cra_name        =      "wp256",
+                .cra_driver_name =      "wp256-generic",
+                .cra_blocksize   =      WP512_BLOCK_SIZE,
+                .cra_module      =      THIS_MODULE,
         }
 } };
 
diff --git a/crypto/xxhash_generic.c b/crypto/xxhash_generic.c
new file mode 100644
index 000000000000..4aad2c0f40a9
--- /dev/null
+++ b/crypto/xxhash_generic.c
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <crypto/internal/hash.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/xxhash.h>
+#include <asm/unaligned.h>
+
+#define XXHASH64_BLOCK_SIZE     32
+#define XXHASH64_DIGEST_SIZE    8
+
+struct xxhash64_tfm_ctx {
+        u64 seed;
+};
+
+struct xxhash64_desc_ctx {
+        struct xxh64_state xxhstate;
+};
+
+static int xxhash64_setkey(struct crypto_shash *tfm, const u8 *key,
+                         unsigned int keylen)
+{
+        struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(tfm);
+
+        if (keylen != sizeof(tctx->seed)) {
+                crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+                return -EINVAL;
+        }
+        tctx->seed = get_unaligned_le64(key);
+        return 0;
+}
+
+static int xxhash64_init(struct shash_desc *desc)
+{
+        struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
+        struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc);
+
+        xxh64_reset(&dctx->xxhstate, tctx->seed);
+
+        return 0;
+}
+
+static int xxhash64_update(struct shash_desc *desc, const u8 *data,
+                         unsigned int length)
+{
+        struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc);
+
+        xxh64_update(&dctx->xxhstate, data, length);
+
+        return 0;
+}
+
+static int xxhash64_final(struct shash_desc *desc, u8 *out)
+{
+        struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc);
+
+        put_unaligned_le64(xxh64_digest(&dctx->xxhstate), out);
+
+        return 0;
+}
+
+static int xxhash64_digest(struct shash_desc *desc, const u8 *data,
+                         unsigned int length, u8 *out)
+{
+        struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
+
+        put_unaligned_le64(xxh64(data, length, tctx->seed), out);
+
+        return 0;
+}
+
+static struct shash_alg alg = {
+        .digestsize     = XXHASH64_DIGEST_SIZE,
+        .setkey         = xxhash64_setkey,
+        .init           = xxhash64_init,
+        .update         = xxhash64_update,
+        .final          = xxhash64_final,
+        .digest         = xxhash64_digest,
+        .descsize       = sizeof(struct xxhash64_desc_ctx),
+        .base           = {
+                .cra_name        = "xxhash64",
+                .cra_driver_name = "xxhash64-generic",
+                .cra_priority    = 100,
+                .cra_flags       = CRYPTO_ALG_OPTIONAL_KEY,
+                .cra_blocksize   = XXHASH64_BLOCK_SIZE,
+                .cra_ctxsize     = sizeof(struct xxhash64_tfm_ctx),
+                .cra_module      = THIS_MODULE,
+        }
+};
+
+static int __init xxhash_mod_init(void)
+{
+        return crypto_register_shash(&alg);
+}
+
+static void __exit xxhash_mod_fini(void)
+{
+        crypto_unregister_shash(&alg);
+}
+
+subsys_initcall(xxhash_mod_init);
+module_exit(xxhash_mod_fini);
+
+MODULE_AUTHOR("Nikolay Borisov <nborisov@suse.com>");
+MODULE_DESCRIPTION("xxhash calculations wrapper for lib/xxhash.c");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS_CRYPTO("xxhash64");
+MODULE_ALIAS_CRYPTO("xxhash64-generic");
diff --git a/crypto/zstd.c b/crypto/zstd.c
index f1e4c70c9d24..5a3ff258d8f7 100644
--- a/crypto/zstd.c
+++ b/crypto/zstd.c
@@ -206,6 +206,7 @@ static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src,
 
 static struct crypto_alg alg = {
         .cra_name               = "zstd",
+        .cra_driver_name        = "zstd-generic",
         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
         .cra_ctxsize            = sizeof(struct zstd_ctx),
         .cra_module             = THIS_MODULE,
diff --git a/drivers/char/hw_random/iproc-rng200.c b/drivers/char/hw_random/iproc-rng200.c
index 8b5a20b35293..92be1c0ab99f 100644
--- a/drivers/char/hw_random/iproc-rng200.c
+++ b/drivers/char/hw_random/iproc-rng200.c
@@ -220,6 +220,7 @@ static int iproc_rng200_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id iproc_rng200_of_match[] = {
+        { .compatible = "brcm,bcm7211-rng200", },
         { .compatible = "brcm,bcm7278-rng200", },
         { .compatible = "brcm,iproc-rng200", },
         {},
diff --git a/drivers/char/hw_random/meson-rng.c b/drivers/char/hw_random/meson-rng.c
index 2e23be802a62..76e693da5dde 100644
--- a/drivers/char/hw_random/meson-rng.c
+++ b/drivers/char/hw_random/meson-rng.c
@@ -1,58 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
  * Copyright (c) 2016 BayLibre, SAS.
  * Author: Neil Armstrong <narmstrong@baylibre.com>
  * Copyright (C) 2014 Amlogic, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see <http://www.gnu.org/licenses/>.
- * The full GNU General Public License is included in this distribution
- * in the file called COPYING.
- *
- * BSD LICENSE
- *
- * Copyright (c) 2016 BayLibre, SAS.
- * Author: Neil Armstrong <narmstrong@baylibre.com>
- * Copyright (C) 2014 Amlogic, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *   * Redistributions of source code must retain the above copyright
- *     notice, this list of conditions and the following disclaimer.
- *   * Redistributions in binary form must reproduce the above copyright
- *     notice, this list of conditions and the following disclaimer in
- *     the documentation and/or other materials provided with the
- *     distribution.
- *   * Neither the name of Intel Corporation nor the names of its
- *     contributors may be used to endorse or promote products derived
- *     from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <linux/err.h>
 #include <linux/module.h>
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 0af08081e305..603413f28fa3 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -520,10 +520,13 @@ config CRYPTO_DEV_ATMEL_SHA
           To compile this driver as a module, choose M here: the module
           will be called atmel-sha.
 
+config CRYPTO_DEV_ATMEL_I2C
+        tristate
+
 config CRYPTO_DEV_ATMEL_ECC
         tristate "Support for Microchip / Atmel ECC hw accelerator"
-        depends on ARCH_AT91 || COMPILE_TEST
         depends on I2C
+        select CRYPTO_DEV_ATMEL_I2C
         select CRYPTO_ECDH
         select CRC16
         help
@@ -534,6 +537,21 @@ config CRYPTO_DEV_ATMEL_ECC
           To compile this driver as a module, choose M here: the module
           will be called atmel-ecc.
 
+config CRYPTO_DEV_ATMEL_SHA204A
+        tristate "Support for Microchip / Atmel SHA accelerator and RNG"
+        depends on I2C
+        select CRYPTO_DEV_ATMEL_I2C
+        select HW_RANDOM
+        select CRC16
+        help
+          Microhip / Atmel SHA accelerator and RNG.
+          Select this if you want to use the Microchip / Atmel SHA204A
+          module as a random number generator. (Other functions of the
+          chip are currently not exposed by this driver)
+
+          To compile this driver as a module, choose M here: the module
+          will be called atmel-sha204a.
+
 config CRYPTO_DEV_CCP
         bool "Support for AMD Secure Processor"
         depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index a23a7197fcd7..afc4753b5d28 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -2,7 +2,9 @@
 obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o
 obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o
 obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o
+obj-$(CONFIG_CRYPTO_DEV_ATMEL_I2C) += atmel-i2c.o
 obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o
+obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA204A) += atmel-sha204a.o
 obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/
 obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/
 obj-$(CONFIG_CRYPTO_DEV_CCREE) += ccree/
diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c
index 49f3e0ce242c..cbfc607282f4 100644
--- a/drivers/crypto/amcc/crypto4xx_alg.c
+++ b/drivers/crypto/amcc/crypto4xx_alg.c
@@ -67,12 +67,16 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm,
 }
 
 static inline int crypto4xx_crypt(struct skcipher_request *req,
-                                  const unsigned int ivlen, bool decrypt)
+                                  const unsigned int ivlen, bool decrypt,
+                                  bool check_blocksize)
 {
         struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
         struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher);
         __le32 iv[AES_IV_SIZE];
 
+        if (check_blocksize && !IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE))
+                return -EINVAL;
+
         if (ivlen)
                 crypto4xx_memcpy_to_le32(iv, req->iv, ivlen);
 
@@ -81,24 +85,34 @@ static inline int crypto4xx_crypt(struct skcipher_request *req,
                 ctx->sa_len, 0, NULL);
 }
 
-int crypto4xx_encrypt_noiv(struct skcipher_request *req)
+int crypto4xx_encrypt_noiv_block(struct skcipher_request *req)
+{
+        return crypto4xx_crypt(req, 0, false, true);
+}
+
+int crypto4xx_encrypt_iv_stream(struct skcipher_request *req)
+{
+        return crypto4xx_crypt(req, AES_IV_SIZE, false, false);
+}
+
+int crypto4xx_decrypt_noiv_block(struct skcipher_request *req)
 {
-        return crypto4xx_crypt(req, 0, false);
+        return crypto4xx_crypt(req, 0, true, true);
 }
 
-int crypto4xx_encrypt_iv(struct skcipher_request *req)
+int crypto4xx_decrypt_iv_stream(struct skcipher_request *req)
 {
-        return crypto4xx_crypt(req, AES_IV_SIZE, false);
+        return crypto4xx_crypt(req, AES_IV_SIZE, true, false);
 }
 
-int crypto4xx_decrypt_noiv(struct skcipher_request *req)
+int crypto4xx_encrypt_iv_block(struct skcipher_request *req)
 {
-        return crypto4xx_crypt(req, 0, true);
+        return crypto4xx_crypt(req, AES_IV_SIZE, false, true);
 }
 
-int crypto4xx_decrypt_iv(struct skcipher_request *req)
+int crypto4xx_decrypt_iv_block(struct skcipher_request *req)
 {
-        return crypto4xx_crypt(req, AES_IV_SIZE, true);
+        return crypto4xx_crypt(req, AES_IV_SIZE, true, true);
 }
 
 /**
@@ -269,8 +283,8 @@ crypto4xx_ctr_crypt(struct skcipher_request *req, bool encrypt)
                 return ret;
         }
 
-        return encrypt ? crypto4xx_encrypt_iv(req)
-                       : crypto4xx_decrypt_iv(req);
+        return encrypt ? crypto4xx_encrypt_iv_stream(req)
+                       : crypto4xx_decrypt_iv_stream(req);
 }
 
 static int crypto4xx_sk_setup_fallback(struct crypto4xx_ctx *ctx,
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index 16d911aaa508..de5e9352e920 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -182,7 +182,6 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev)
                                   dev->pdr_pa);
                 return -ENOMEM;
         }
-        memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD);
         dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device,
                                    sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD,
                                    &dev->shadow_sa_pool_pa,
@@ -1210,8 +1209,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
                 .max_keysize = AES_MAX_KEY_SIZE,
                 .ivsize = AES_IV_SIZE,
                 .setkey = crypto4xx_setkey_aes_cbc,
-                .encrypt = crypto4xx_encrypt_iv,
-                .decrypt = crypto4xx_decrypt_iv,
+                .encrypt = crypto4xx_encrypt_iv_block,
+                .decrypt = crypto4xx_decrypt_iv_block,
                 .init = crypto4xx_sk_init,
                 .exit = crypto4xx_sk_exit,
         } },
@@ -1222,7 +1221,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
                         .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
                         .cra_flags = CRYPTO_ALG_ASYNC |
                                 CRYPTO_ALG_KERN_DRIVER_ONLY,
-                        .cra_blocksize = AES_BLOCK_SIZE,
+                        .cra_blocksize = 1,
                         .cra_ctxsize = sizeof(struct crypto4xx_ctx),
                         .cra_module = THIS_MODULE,
                 },
@@ -1230,8 +1229,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
                 .max_keysize = AES_MAX_KEY_SIZE,
                 .ivsize = AES_IV_SIZE,
                 .setkey = crypto4xx_setkey_aes_cfb,
-                .encrypt = crypto4xx_encrypt_iv,
-                .decrypt = crypto4xx_decrypt_iv,
+                .encrypt = crypto4xx_encrypt_iv_stream,
+                .decrypt = crypto4xx_decrypt_iv_stream,
                 .init = crypto4xx_sk_init,
                 .exit = crypto4xx_sk_exit,
         } },
@@ -1243,7 +1242,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
                         .cra_flags = CRYPTO_ALG_NEED_FALLBACK |
                                 CRYPTO_ALG_ASYNC |
                                 CRYPTO_ALG_KERN_DRIVER_ONLY,
-                        .cra_blocksize = AES_BLOCK_SIZE,
+                        .cra_blocksize = 1,
                         .cra_ctxsize = sizeof(struct crypto4xx_ctx),
                         .cra_module = THIS_MODULE,
                 },
@@ -1263,7 +1262,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
                         .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
                         .cra_flags = CRYPTO_ALG_ASYNC |
                                 CRYPTO_ALG_KERN_DRIVER_ONLY,
-                        .cra_blocksize = AES_BLOCK_SIZE,
+                        .cra_blocksize = 1,
                         .cra_ctxsize = sizeof(struct crypto4xx_ctx),
                         .cra_module = THIS_MODULE,
                 },
@@ -1290,8 +1289,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
                 .min_keysize = AES_MIN_KEY_SIZE,
                 .max_keysize = AES_MAX_KEY_SIZE,
                 .setkey = crypto4xx_setkey_aes_ecb,
-                .encrypt = crypto4xx_encrypt_noiv,
-                .decrypt = crypto4xx_decrypt_noiv,
+                .encrypt = crypto4xx_encrypt_noiv_block,
+                .decrypt = crypto4xx_decrypt_noiv_block,
                 .init = crypto4xx_sk_init,
                 .exit = crypto4xx_sk_exit,
         } },
@@ -1302,7 +1301,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
                         .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
                         .cra_flags = CRYPTO_ALG_ASYNC |
                                 CRYPTO_ALG_KERN_DRIVER_ONLY,
-                        .cra_blocksize = AES_BLOCK_SIZE,
+                        .cra_blocksize = 1,
                         .cra_ctxsize = sizeof(struct crypto4xx_ctx),
                         .cra_module = THIS_MODULE,
                 },
@@ -1310,8 +1309,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = {
                 .max_keysize = AES_MAX_KEY_SIZE,
                 .ivsize = AES_IV_SIZE,
                 .setkey = crypto4xx_setkey_aes_ofb,
-                .encrypt = crypto4xx_encrypt_iv,
-                .decrypt = crypto4xx_decrypt_iv,
+                .encrypt = crypto4xx_encrypt_iv_stream,
+                .decrypt = crypto4xx_decrypt_iv_stream,
                 .init = crypto4xx_sk_init,
                 .exit = crypto4xx_sk_exit,
         } },
diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h
index ca1c25c40c23..6b6841359190 100644
--- a/drivers/crypto/amcc/crypto4xx_core.h
+++ b/drivers/crypto/amcc/crypto4xx_core.h
@@ -173,10 +173,12 @@ int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher,
                              const u8 *key, unsigned int keylen);
 int crypto4xx_encrypt_ctr(struct skcipher_request *req);
 int crypto4xx_decrypt_ctr(struct skcipher_request *req);
-int crypto4xx_encrypt_iv(struct skcipher_request *req);
-int crypto4xx_decrypt_iv(struct skcipher_request *req);
-int crypto4xx_encrypt_noiv(struct skcipher_request *req);
-int crypto4xx_decrypt_noiv(struct skcipher_request *req);
+int crypto4xx_encrypt_iv_stream(struct skcipher_request *req);
+int crypto4xx_decrypt_iv_stream(struct skcipher_request *req);
+int crypto4xx_encrypt_iv_block(struct skcipher_request *req);
+int crypto4xx_decrypt_iv_block(struct skcipher_request *req);
+int crypto4xx_encrypt_noiv_block(struct skcipher_request *req);
+int crypto4xx_decrypt_noiv_block(struct skcipher_request *req);
 int crypto4xx_rfc3686_encrypt(struct skcipher_request *req);
 int crypto4xx_rfc3686_decrypt(struct skcipher_request *req);
 int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm);
diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c
index ba00e4563ca0..ff02cc05affb 100644
--- a/drivers/crypto/atmel-ecc.c
+++ b/drivers/crypto/atmel-ecc.c
@@ -6,8 +6,6 @@
  * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
  */
 
-#include <linux/bitrev.h>
-#include <linux/crc16.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/err.h>
@@ -23,42 +21,11 @@
 #include <crypto/internal/kpp.h>
 #include <crypto/ecdh.h>
 #include <crypto/kpp.h>
-#include "atmel-ecc.h"
-
-/* Used for binding tfm objects to i2c clients. */
-struct atmel_ecc_driver_data {
-        struct list_head i2c_client_list;
-        spinlock_t i2c_list_lock;
-} ____cacheline_aligned;
+#include "atmel-i2c.h"
 
 static struct atmel_ecc_driver_data driver_data;
 
 /**
- * atmel_ecc_i2c_client_priv - i2c_client private data
- * @client              : pointer to i2c client device
- * @i2c_client_list_node: part of i2c_client_list
- * @lock                : lock for sending i2c commands
- * @wake_token          : wake token array of zeros
- * @wake_token_sz       : size in bytes of the wake_token
- * @tfm_count           : number of active crypto transformations on i2c client
- *
- * Reads and writes from/to the i2c client are sequential. The first byte
- * transmitted to the device is treated as the byte size. Any attempt to send
- * more than this number of bytes will cause the device to not ACK those bytes.
- * After the host writes a single command byte to the input buffer, reads are
- * prohibited until after the device completes command execution. Use a mutex
- * when sending i2c commands.
- */
-struct atmel_ecc_i2c_client_priv {
-        struct i2c_client *client;
-        struct list_head i2c_client_list_node;
-        struct mutex lock;
-        u8 wake_token[WAKE_TOKEN_MAX_SIZE];
-        size_t wake_token_sz;
-        atomic_t tfm_count ____cacheline_aligned;
-};
-
-/**
  * atmel_ecdh_ctx - transformation context
  * @client     : pointer to i2c client device
  * @fallback   : used for unsupported curves or when user wants to use its own
@@ -80,188 +47,12 @@ struct atmel_ecdh_ctx {
         bool do_fallback;
 };
 
-/**
- * atmel_ecc_work_data - data structure representing the work
- * @ctx : transformation context.
- * @cbk : pointer to a callback function to be invoked upon completion of this
- *        request. This has the form:
- *        callback(struct atmel_ecc_work_data *work_data, void *areq, u8 status)
- *        where:
- *        @work_data: data structure representing the work
- *        @areq     : optional pointer to an argument passed with the original
- *                    request.
- *        @status   : status returned from the i2c client device or i2c error.
- * @areq: optional pointer to a user argument for use at callback time.
- * @work: describes the task to be executed.
- * @cmd : structure used for communicating with the device.
- */
-struct atmel_ecc_work_data {
-        struct atmel_ecdh_ctx *ctx;
-        void (*cbk)(struct atmel_ecc_work_data *work_data, void *areq,
-                    int status);
-        void *areq;
-        struct work_struct work;
-        struct atmel_ecc_cmd cmd;
-};
-
-static u16 atmel_ecc_crc16(u16 crc, const u8 *buffer, size_t len)
-{
-        return cpu_to_le16(bitrev16(crc16(crc, buffer, len)));
-}
-
-/**
- * atmel_ecc_checksum() - Generate 16-bit CRC as required by ATMEL ECC.
- * CRC16 verification of the count, opcode, param1, param2 and data bytes.
- * The checksum is saved in little-endian format in the least significant
- * two bytes of the command. CRC polynomial is 0x8005 and the initial register
- * value should be zero.
- *
- * @cmd : structure used for communicating with the device.
- */
-static void atmel_ecc_checksum(struct atmel_ecc_cmd *cmd)
-{
-        u8 *data = &cmd->count;
-        size_t len = cmd->count - CRC_SIZE;
-        u16 *crc16 = (u16 *)(data + len);
-
-        *crc16 = atmel_ecc_crc16(0, data, len);
-}
-
-static void atmel_ecc_init_read_cmd(struct atmel_ecc_cmd *cmd)
-{
-        cmd->word_addr = COMMAND;
-        cmd->opcode = OPCODE_READ;
-        /*
-         * Read the word from Configuration zone that contains the lock bytes
-         * (UserExtra, Selector, LockValue, LockConfig).
-         */
-        cmd->param1 = CONFIG_ZONE;
-        cmd->param2 = DEVICE_LOCK_ADDR;
-        cmd->count = READ_COUNT;
-
-        atmel_ecc_checksum(cmd);
-
-        cmd->msecs = MAX_EXEC_TIME_READ;
-        cmd->rxsize = READ_RSP_SIZE;
-}
-
-static void atmel_ecc_init_genkey_cmd(struct atmel_ecc_cmd *cmd, u16 keyid)
-{
-        cmd->word_addr = COMMAND;
-        cmd->count = GENKEY_COUNT;
-        cmd->opcode = OPCODE_GENKEY;
-        cmd->param1 = GENKEY_MODE_PRIVATE;
-        /* a random private key will be generated and stored in slot keyID */
-        cmd->param2 = cpu_to_le16(keyid);
-
-        atmel_ecc_checksum(cmd);
-
-        cmd->msecs = MAX_EXEC_TIME_GENKEY;
-        cmd->rxsize = GENKEY_RSP_SIZE;
-}
-
-static int atmel_ecc_init_ecdh_cmd(struct atmel_ecc_cmd *cmd,
-                                   struct scatterlist *pubkey)
-{
-        size_t copied;
-
-        cmd->word_addr = COMMAND;
-        cmd->count = ECDH_COUNT;
-        cmd->opcode = OPCODE_ECDH;
-        cmd->param1 = ECDH_PREFIX_MODE;
-        /* private key slot */
-        cmd->param2 = cpu_to_le16(DATA_SLOT_2);
-
-        /*
-         * The device only supports NIST P256 ECC keys. The public key size will
-         * always be the same. Use a macro for the key size to avoid unnecessary
-         * computations.
-         */
-        copied = sg_copy_to_buffer(pubkey,
-                                   sg_nents_for_len(pubkey,
-                                                    ATMEL_ECC_PUBKEY_SIZE),
-                                   cmd->data, ATMEL_ECC_PUBKEY_SIZE);
-        if (copied != ATMEL_ECC_PUBKEY_SIZE)
-                return -EINVAL;
-
-        atmel_ecc_checksum(cmd);
-
-        cmd->msecs = MAX_EXEC_TIME_ECDH;
-        cmd->rxsize = ECDH_RSP_SIZE;
-
-        return 0;
-}
-
-/*
- * After wake and after execution of a command, there will be error, status, or
- * result bytes in the device's output register that can be retrieved by the
- * system. When the length of that group is four bytes, the codes returned are
- * detailed in error_list.
- */
-static int atmel_ecc_status(struct device *dev, u8 *status)
-{
-        size_t err_list_len = ARRAY_SIZE(error_list);
-        int i;
-        u8 err_id = status[1];
-
-        if (*status != STATUS_SIZE)
-                return 0;
-
-        if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR)
-                return 0;
-
-        for (i = 0; i < err_list_len; i++)
-                if (error_list[i].value == err_id)
-                        break;
-
-        /* if err_id is not in the error_list then ignore it */
-        if (i != err_list_len) {
-                dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text);
-                return err_id;
-        }
-
-        return 0;
-}
-
-static int atmel_ecc_wakeup(struct i2c_client *client)
-{
-        struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
-        u8 status[STATUS_RSP_SIZE];
-        int ret;
-
-        /*
-         * The device ignores any levels or transitions on the SCL pin when the
-         * device is idle, asleep or during waking up. Don't check for error
-         * when waking up the device.
-         */
-        i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz);
-
-        /*
-         * Wait to wake the device. Typical execution times for ecdh and genkey
-         * are around tens of milliseconds. Delta is chosen to 50 microseconds.
-         */
-        usleep_range(TWHI_MIN, TWHI_MAX);
-
-        ret = i2c_master_recv(client, status, STATUS_SIZE);
-        if (ret < 0)
-                return ret;
-
-        return atmel_ecc_status(&client->dev, status);
-}
-
-static int atmel_ecc_sleep(struct i2c_client *client)
-{
-        u8 sleep = SLEEP_TOKEN;
-
-        return i2c_master_send(client, &sleep, 1);
-}
-
-static void atmel_ecdh_done(struct atmel_ecc_work_data *work_data, void *areq,
+static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq,
                             int status)
 {
         struct kpp_request *req = areq;
         struct atmel_ecdh_ctx *ctx = work_data->ctx;
-        struct atmel_ecc_cmd *cmd = &work_data->cmd;
+        struct atmel_i2c_cmd *cmd = &work_data->cmd;
         size_t copied, n_sz;
 
         if (status)
@@ -282,82 +73,6 @@ free_work_data:
         kpp_request_complete(req, status);
 }
 
-/*
- * atmel_ecc_send_receive() - send a command to the device and receive its
- *                            response.
- * @client: i2c client device
- * @cmd   : structure used to communicate with the device
- *
- * After the device receives a Wake token, a watchdog counter starts within the
- * device. After the watchdog timer expires, the device enters sleep mode
- * regardless of whether some I/O transmission or command execution is in
- * progress. If a command is attempted when insufficient time remains prior to
- * watchdog timer execution, the device will return the watchdog timeout error
- * code without attempting to execute the command. There is no way to reset the
- * counter other than to put the device into sleep or idle mode and then
- * wake it up again.
- */
-static int atmel_ecc_send_receive(struct i2c_client *client,
-                                  struct atmel_ecc_cmd *cmd)
-{
-        struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
-        int ret;
-
-        mutex_lock(&i2c_priv->lock);
-
-        ret = atmel_ecc_wakeup(client);
-        if (ret)
-                goto err;
-
-        /* send the command */
-        ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE);
-        if (ret < 0)
-                goto err;
-
-        /* delay the appropriate amount of time for command to execute */
-        msleep(cmd->msecs);
-
-        /* receive the response */
-        ret = i2c_master_recv(client, cmd->data, cmd->rxsize);
-        if (ret < 0)
-                goto err;
-
-        /* put the device into low-power mode */
-        ret = atmel_ecc_sleep(client);
-        if (ret < 0)
-                goto err;
-
-        mutex_unlock(&i2c_priv->lock);
-        return atmel_ecc_status(&client->dev, cmd->data);
-err:
-        mutex_unlock(&i2c_priv->lock);
-        return ret;
-}
-
-static void atmel_ecc_work_handler(struct work_struct *work)
-{
-        struct atmel_ecc_work_data *work_data =
-                        container_of(work, struct atmel_ecc_work_data, work);
-        struct atmel_ecc_cmd *cmd = &work_data->cmd;
-        struct i2c_client *client = work_data->ctx->client;
-        int status;
-
-        status = atmel_ecc_send_receive(client, cmd);
-        work_data->cbk(work_data, work_data->areq, status);
-}
-
-static void atmel_ecc_enqueue(struct atmel_ecc_work_data *work_data,
-                              void (*cbk)(struct atmel_ecc_work_data *work_data,
-                                          void *areq, int status),
-                              void *areq)
-{
-        work_data->cbk = (void *)cbk;
-        work_data->areq = areq;
-
-        INIT_WORK(&work_data->work, atmel_ecc_work_handler);
-        schedule_work(&work_data->work);
-}
-
 static unsigned int atmel_ecdh_supported_curve(unsigned int curve_id)
 {
         if (curve_id == ECC_CURVE_NIST_P256)
@@ -374,7 +89,7 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
                                  unsigned int len)
 {
         struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
-        struct atmel_ecc_cmd *cmd;
+        struct atmel_i2c_cmd *cmd;
         void *public_key;
         struct ecdh params;
         int ret = -ENOMEM;
@@ -412,9 +127,9 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
         ctx->do_fallback = false;
         ctx->curve_id = params.curve_id;
 
-        atmel_ecc_init_genkey_cmd(cmd, DATA_SLOT_2);
+        atmel_i2c_init_genkey_cmd(cmd, DATA_SLOT_2);
 
-        ret = atmel_ecc_send_receive(ctx->client, cmd);
+        ret = atmel_i2c_send_receive(ctx->client, cmd);
         if (ret)
                 goto free_public_key;
 
@@ -444,6 +159,9 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req)
                 return crypto_kpp_generate_public_key(req);
         }
 
+        if (!ctx->public_key)
+                return -EINVAL;
+
         /* might want less than we've got */
         nbytes = min_t(size_t, ATMEL_ECC_PUBKEY_SIZE, req->dst_len);
 
@@ -461,7 +179,7 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req)
 {
         struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
         struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm);
-        struct atmel_ecc_work_data *work_data;
+        struct atmel_i2c_work_data *work_data;
         gfp_t gfp;
         int ret;
 
@@ -482,12 +200,13 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req)
                 return -ENOMEM;
 
         work_data->ctx = ctx;
+        work_data->client = ctx->client;
 
-        ret = atmel_ecc_init_ecdh_cmd(&work_data->cmd, req->src);
+        ret = atmel_i2c_init_ecdh_cmd(&work_data->cmd, req->src);
         if (ret)
                 goto free_work_data;
 
-        atmel_ecc_enqueue(work_data, atmel_ecdh_done, req);
+        atmel_i2c_enqueue(work_data, atmel_ecdh_done, req);
 
         return -EINPROGRESS;
 
@@ -498,7 +217,7 @@ free_work_data:
 
 static struct i2c_client *atmel_ecc_i2c_client_alloc(void)
 {
-        struct atmel_ecc_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL;
+        struct atmel_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL;
         struct i2c_client *client = ERR_PTR(-ENODEV);
         int min_tfm_cnt = INT_MAX;
         int tfm_cnt;
@@ -533,7 +252,7 @@ static struct i2c_client *atmel_ecc_i2c_client_alloc(void)
 
 static void atmel_ecc_i2c_client_free(struct i2c_client *client)
 {
-        struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+        struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
 
         atomic_dec(&i2c_priv->tfm_count);
 }
@@ -604,96 +323,18 @@ static struct kpp_alg atmel_ecdh = {
         },
 };
 
-static inline size_t atmel_ecc_wake_token_sz(u32 bus_clk_rate)
-{
-        u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC);
-
-        /* return the size of the wake_token in bytes */
-        return DIV_ROUND_UP(no_of_bits, 8);
-}
-
-static int device_sanity_check(struct i2c_client *client)
-{
-        struct atmel_ecc_cmd *cmd;
-        int ret;
-
-        cmd = kmalloc(sizeof(*cmd), GFP_KERNEL);
-        if (!cmd)
-                return -ENOMEM;
-
-        atmel_ecc_init_read_cmd(cmd);
-
-        ret = atmel_ecc_send_receive(client, cmd);
-        if (ret)
-                goto free_cmd;
-
-        /*
-         * It is vital that the Configuration, Data and OTP zones be locked
-         * prior to release into the field of the system containing the device.
-         * Failure to lock these zones may permit modification of any secret
-         * keys and may lead to other security problems.
-         */
-        if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) {
-                dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n");
-                ret = -ENOTSUPP;
-        }
-
-        /* fall through */
-free_cmd:
-        kfree(cmd);
-        return ret;
-}
-
 static int atmel_ecc_probe(struct i2c_client *client,
                            const struct i2c_device_id *id)
 {
-        struct atmel_ecc_i2c_client_priv *i2c_priv;
-        struct device *dev = &client->dev;
+        struct atmel_i2c_client_priv *i2c_priv;
         int ret;
-        u32 bus_clk_rate;
-
-        if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
-                dev_err(dev, "I2C_FUNC_I2C not supported\n");
-                return -ENODEV;
-        }
 
-        ret = of_property_read_u32(client->adapter->dev.of_node,
-                                   "clock-frequency", &bus_clk_rate);
-        if (ret) {
-                dev_err(dev, "of: failed to read clock-frequency property\n");
-                return ret;
-        }
-
-        if (bus_clk_rate > 1000000L) {
-                dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n",
-                        bus_clk_rate);
-                return -EINVAL;
-        }
-
-        i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL);
-        if (!i2c_priv)
-                return -ENOMEM;
-
-        i2c_priv->client = client;
-        mutex_init(&i2c_priv->lock);
-
-        /*
-         * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate -
-         * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz
-         * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE.
-         */
-        i2c_priv->wake_token_sz = atmel_ecc_wake_token_sz(bus_clk_rate);
-
-        memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token));
-
-        atomic_set(&i2c_priv->tfm_count, 0);
-
-        i2c_set_clientdata(client, i2c_priv);
-
-        ret = device_sanity_check(client);
+        ret = atmel_i2c_probe(client, id);
         if (ret)
                 return ret;
 
+        i2c_priv = i2c_get_clientdata(client);
+
         spin_lock(&driver_data.i2c_list_lock);
         list_add_tail(&i2c_priv->i2c_client_list_node,
                       &driver_data.i2c_client_list);
@@ -705,10 +346,10 @@ static int atmel_ecc_probe(struct i2c_client *client,
                 list_del(&i2c_priv->i2c_client_list_node);
                 spin_unlock(&driver_data.i2c_list_lock);
 
-                dev_err(dev, "%s alg registration failed\n",
+                dev_err(&client->dev, "%s alg registration failed\n",
                         atmel_ecdh.base.cra_driver_name);
         } else {
-                dev_info(dev, "atmel ecc algorithms registered in /proc/crypto\n");
+                dev_info(&client->dev, "atmel ecc algorithms registered in /proc/crypto\n");
         }
 
         return ret;
@@ -716,7 +357,7 @@ static int atmel_ecc_probe(struct i2c_client *client,
 
 static int atmel_ecc_remove(struct i2c_client *client)
 {
-        struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+        struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
 
         /* Return EBUSY if i2c client already allocated. */
         if (atomic_read(&i2c_priv->tfm_count)) {
diff --git a/drivers/crypto/atmel-ecc.h b/drivers/crypto/atmel-ecc.h
deleted file mode 100644
index 643a3b947338..000000000000
--- a/drivers/crypto/atmel-ecc.h
+++ /dev/null
@@ -1,116 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2017, Microchip Technology Inc.
- * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
- */
-
-#ifndef __ATMEL_ECC_H__
-#define __ATMEL_ECC_H__
-
-#define ATMEL_ECC_PRIORITY              300
-
-#define COMMAND                         0x03 /* packet function */
-#define SLEEP_TOKEN                     0x01
-#define WAKE_TOKEN_MAX_SIZE             8
-
-/* Definitions of Data and Command sizes */
-#define WORD_ADDR_SIZE                  1
-#define COUNT_SIZE                      1
-#define CRC_SIZE                        2
-#define CMD_OVERHEAD_SIZE               (COUNT_SIZE + CRC_SIZE)
-
-/* size in bytes of the n prime */
-#define ATMEL_ECC_NIST_P256_N_SIZE      32
-#define ATMEL_ECC_PUBKEY_SIZE           (2 * ATMEL_ECC_NIST_P256_N_SIZE)
-
-#define STATUS_RSP_SIZE                 4
-#define ECDH_RSP_SIZE                   (32 + CMD_OVERHEAD_SIZE)
-#define GENKEY_RSP_SIZE                 (ATMEL_ECC_PUBKEY_SIZE + \
-                                         CMD_OVERHEAD_SIZE)
-#define READ_RSP_SIZE                   (4 + CMD_OVERHEAD_SIZE)
-#define MAX_RSP_SIZE                    GENKEY_RSP_SIZE
-
-/**
- * atmel_ecc_cmd - structure used for communicating with the device.
- * @word_addr: indicates the function of the packet sent to the device. This
- *             byte should have a value of COMMAND for normal operation.
- * @count    : number of bytes to be transferred to (or from) the device.
- * @opcode   : the command code.
- * @param1   : the first parameter; always present.
- * @param2   : the second parameter; always present.
- * @data     : optional remaining input data. Includes a 2-byte CRC.
- * @rxsize   : size of the data received from i2c client.
- * @msecs    : command execution time in milliseconds
- */
-struct atmel_ecc_cmd {
-        u8 word_addr;
-        u8 count;
-        u8 opcode;
-        u8 param1;
-        u16 param2;
-        u8 data[MAX_RSP_SIZE];
-        u8 msecs;
-        u16 rxsize;
-} __packed;
-
-/* Status/Error codes */
-#define STATUS_SIZE                     0x04
-#define STATUS_NOERR                    0x00
-#define STATUS_WAKE_SUCCESSFUL          0x11
-
-static const struct {
-        u8 value;
-        const char *error_text;
-} error_list[] = {
-        { 0x01, "CheckMac or Verify miscompare" },
-        { 0x03, "Parse Error" },
-        { 0x05, "ECC Fault" },
-        { 0x0F, "Execution Error" },
-        { 0xEE, "Watchdog about to expire" },
-        { 0xFF, "CRC or other communication error" },
-};
-
-/* Definitions for eeprom organization */
-#define CONFIG_ZONE                     0
-
-/* Definitions for Indexes common to all commands */
-#define RSP_DATA_IDX                    1 /* buffer index of data in response */
-#define DATA_SLOT_2                     2 /* used for ECDH private key */
-
-/* Definitions for the device lock state */
-#define DEVICE_LOCK_ADDR                0x15
-#define LOCK_VALUE_IDX                  (RSP_DATA_IDX + 2)
-#define LOCK_CONFIG_IDX                 (RSP_DATA_IDX + 3)
-
-/*
- * Wake High delay to data communication (microseconds). SDA should be stable
- * high for this entire duration.
- */
-#define TWHI_MIN                        1500
-#define TWHI_MAX                        1550
-
-/* Wake Low duration */
-#define TWLO_USEC                       60
-
-/* Command execution time (milliseconds) */
-#define MAX_EXEC_TIME_ECDH              58
-#define MAX_EXEC_TIME_GENKEY            115
-#define MAX_EXEC_TIME_READ              1
-
-/* Command opcode */
-#define OPCODE_ECDH                     0x43
-#define OPCODE_GENKEY                   0x40
-#define OPCODE_READ                     0x02
-
-/* Definitions for the READ Command */
-#define READ_COUNT                      7
-
-/* Definitions for the GenKey Command */
-#define GENKEY_COUNT                    7
-#define GENKEY_MODE_PRIVATE             0x04
-
-/* Definitions for the ECDH Command */
-#define ECDH_COUNT                      71
-#define ECDH_PREFIX_MODE                0x00
-
-#endif /* __ATMEL_ECC_H__ */
diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c
new file mode 100644
index 000000000000..dc876fab2882
--- /dev/null
+++ b/drivers/crypto/atmel-i2c.c
@@ -0,0 +1,364 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Microchip / Atmel ECC (I2C) driver.
+ *
+ * Copyright (c) 2017, Microchip Technology Inc.
+ * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
+ */
+
+#include <linux/bitrev.h>
+#include <linux/crc16.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+#include "atmel-i2c.h"
+
+/**
+ * atmel_i2c_checksum() - Generate 16-bit CRC as required by ATMEL ECC.
+ * CRC16 verification of the count, opcode, param1, param2 and data bytes.
+ * The checksum is saved in little-endian format in the least significant
+ * two bytes of the command. CRC polynomial is 0x8005 and the initial register
+ * value should be zero.
+ *
+ * @cmd : structure used for communicating with the device.
+ */
+static void atmel_i2c_checksum(struct atmel_i2c_cmd *cmd)
+{
+        u8 *data = &cmd->count;
+        size_t len = cmd->count - CRC_SIZE;
+        __le16 *__crc16 = (__le16 *)(data + len);
+
+        *__crc16 = cpu_to_le16(bitrev16(crc16(0, data, len)));
+}
+
+void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd)
+{
+        cmd->word_addr = COMMAND;
+        cmd->opcode = OPCODE_READ;
+        /*
+         * Read the word from Configuration zone that contains the lock bytes
+         * (UserExtra, Selector, LockValue, LockConfig).
+         */
+        cmd->param1 = CONFIG_ZONE;
+        cmd->param2 = cpu_to_le16(DEVICE_LOCK_ADDR);
+        cmd->count = READ_COUNT;
+
+        atmel_i2c_checksum(cmd);
+
+        cmd->msecs = MAX_EXEC_TIME_READ;
+        cmd->rxsize = READ_RSP_SIZE;
+}
+EXPORT_SYMBOL(atmel_i2c_init_read_cmd);
+
+void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd)
+{
+        cmd->word_addr = COMMAND;
+        cmd->opcode = OPCODE_RANDOM;
+        cmd->param1 = 0;
+        cmd->param2 = 0;
+        cmd->count = RANDOM_COUNT;
+
+        atmel_i2c_checksum(cmd);
+
+        cmd->msecs = MAX_EXEC_TIME_RANDOM;
+        cmd->rxsize = RANDOM_RSP_SIZE;
+}
+EXPORT_SYMBOL(atmel_i2c_init_random_cmd);
+
+void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid)
+{
+        cmd->word_addr = COMMAND;
+        cmd->count = GENKEY_COUNT;
+        cmd->opcode = OPCODE_GENKEY;
+        cmd->param1 = GENKEY_MODE_PRIVATE;
+        /* a random private key will be generated and stored in slot keyID */
+        cmd->param2 = cpu_to_le16(keyid);
+
+        atmel_i2c_checksum(cmd);
+
+        cmd->msecs = MAX_EXEC_TIME_GENKEY;
+        cmd->rxsize = GENKEY_RSP_SIZE;
+}
+EXPORT_SYMBOL(atmel_i2c_init_genkey_cmd);
+
+int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd,
+                            struct scatterlist *pubkey)
+{
+        size_t copied;
+
+        cmd->word_addr = COMMAND;
+        cmd->count = ECDH_COUNT;
+        cmd->opcode = OPCODE_ECDH;
+        cmd->param1 = ECDH_PREFIX_MODE;
+        /* private key slot */
+        cmd->param2 = cpu_to_le16(DATA_SLOT_2);
+
+        /*
+         * The device only supports NIST P256 ECC keys. The public key size will
+         * always be the same. Use a macro for the key size to avoid unnecessary
+         * computations.
+         */
+        copied = sg_copy_to_buffer(pubkey,
+                                   sg_nents_for_len(pubkey,
+                                                    ATMEL_ECC_PUBKEY_SIZE),
+                                   cmd->data, ATMEL_ECC_PUBKEY_SIZE);
+        if (copied != ATMEL_ECC_PUBKEY_SIZE)
+                return -EINVAL;
+
+        atmel_i2c_checksum(cmd);
+
+        cmd->msecs = MAX_EXEC_TIME_ECDH;
+        cmd->rxsize = ECDH_RSP_SIZE;
+
+        return 0;
+}
+EXPORT_SYMBOL(atmel_i2c_init_ecdh_cmd);
+
+/*
+ * After wake and after execution of a command, there will be error, status, or
+ * result bytes in the device's output register that can be retrieved by the
+ * system. When the length of that group is four bytes, the codes returned are
+ * detailed in error_list.
+ */
+static int atmel_i2c_status(struct device *dev, u8 *status)
+{
+        size_t err_list_len = ARRAY_SIZE(error_list);
+        int i;
+        u8 err_id = status[1];
+
+        if (*status != STATUS_SIZE)
+                return 0;
+
+        if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR)
+                return 0;
+
+        for (i = 0; i < err_list_len; i++)
+                if (error_list[i].value == err_id)
+                        break;
+
+        /* if err_id is not in the error_list then ignore it */
+        if (i != err_list_len) {
+                dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text);
+                return err_id;
+        }
+
+        return 0;
+}
+
+static int atmel_i2c_wakeup(struct i2c_client *client)
+{
+        struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+        u8 status[STATUS_RSP_SIZE];
+        int ret;
+
+        /*
+         * The device ignores any levels or transitions on the SCL pin when the
+         * device is idle, asleep or during waking up. Don't check for error
+         * when waking up the device.
+         */
+        i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz);
+
+        /*
+         * Wait to wake the device. Typical execution times for ecdh and genkey
+         * are around tens of milliseconds. Delta is chosen to 50 microseconds.
+         */
+        usleep_range(TWHI_MIN, TWHI_MAX);
+
+        ret = i2c_master_recv(client, status, STATUS_SIZE);
+        if (ret < 0)
+                return ret;
+
+        return atmel_i2c_status(&client->dev, status);
+}
+
+static int atmel_i2c_sleep(struct i2c_client *client)
+{
+        u8 sleep = SLEEP_TOKEN;
+
+        return i2c_master_send(client, &sleep, 1);
+}
+
+/*
+ * atmel_i2c_send_receive() - send a command to the device and receive its
+ *                            response.
+ * @client: i2c client device
+ * @cmd   : structure used to communicate with the device
+ *
+ * After the device receives a Wake token, a watchdog counter starts within the
+ * device. After the watchdog timer expires, the device enters sleep mode
+ * regardless of whether some I/O transmission or command execution is in
+ * progress. If a command is attempted when insufficient time remains prior to
+ * watchdog timer execution, the device will return the watchdog timeout error
+ * code without attempting to execute the command. There is no way to reset the
+ * counter other than to put the device into sleep or idle mode and then
+ * wake it up again.
+ */
+int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd)
+{
+        struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+        int ret;
+
+        mutex_lock(&i2c_priv->lock);
+
+        ret = atmel_i2c_wakeup(client);
+        if (ret)
+                goto err;
+
+        /* send the command */
+        ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE);
+        if (ret < 0)
+                goto err;
+
+        /* delay the appropriate amount of time for command to execute */
+        msleep(cmd->msecs);
+
+        /* receive the response */
+        ret = i2c_master_recv(client, cmd->data, cmd->rxsize);
+        if (ret < 0)
+                goto err;
+
+        /* put the device into low-power mode */
+        ret = atmel_i2c_sleep(client);
+        if (ret < 0)
+                goto err;
+
+        mutex_unlock(&i2c_priv->lock);
+        return atmel_i2c_status(&client->dev, cmd->data);
+err:
+        mutex_unlock(&i2c_priv->lock);
+        return ret;
+}
+EXPORT_SYMBOL(atmel_i2c_send_receive);
+
+static void atmel_i2c_work_handler(struct work_struct *work)
+{
+        struct atmel_i2c_work_data *work_data =
+                        container_of(work, struct atmel_i2c_work_data, work);
+        struct atmel_i2c_cmd *cmd = &work_data->cmd;
+        struct i2c_client *client = work_data->client;
+        int status;
+
+        status = atmel_i2c_send_receive(client, cmd);
+        work_data->cbk(work_data, work_data->areq, status);
+}
+
+void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data,
+                       void (*cbk)(struct atmel_i2c_work_data *work_data,
+                                   void *areq, int status),
+                       void *areq)
+{
+        work_data->cbk = (void *)cbk;
+        work_data->areq = areq;
+
+        INIT_WORK(&work_data->work, atmel_i2c_work_handler);
+        schedule_work(&work_data->work);
+}
+EXPORT_SYMBOL(atmel_i2c_enqueue);
+
+static inline size_t atmel_i2c_wake_token_sz(u32 bus_clk_rate)
+{
+        u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC);
+
+        /* return the size of the wake_token in bytes */
+        return DIV_ROUND_UP(no_of_bits, 8);
+}
+
+static int device_sanity_check(struct i2c_client *client)
+{
+        struct atmel_i2c_cmd *cmd;
+        int ret;
+
+        cmd = kmalloc(sizeof(*cmd), GFP_KERNEL);
+        if (!cmd)
+                return -ENOMEM;
+
+        atmel_i2c_init_read_cmd(cmd);
+
+        ret = atmel_i2c_send_receive(client, cmd);
+        if (ret)
+                goto free_cmd;
+
+        /*
+         * It is vital that the Configuration, Data and OTP zones be locked
+         * prior to release into the field of the system containing the device.
+         * Failure to lock these zones may permit modification of any secret
+         * keys and may lead to other security problems.
+         */
+        if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) {
+                dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n");
+                ret = -ENOTSUPP;
+        }
+
+        /* fall through */
+free_cmd:
+        kfree(cmd);
+        return ret;
+}
+
+int atmel_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id)
+{
+        struct atmel_i2c_client_priv *i2c_priv;
+        struct device *dev = &client->dev;
+        int ret;
+        u32 bus_clk_rate;
+
+        if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
+                dev_err(dev, "I2C_FUNC_I2C not supported\n");
+                return -ENODEV;
+        }
+
+        bus_clk_rate = i2c_acpi_find_bus_speed(&client->adapter->dev);
+        if (!bus_clk_rate) {
+                ret = device_property_read_u32(&client->adapter->dev,
+                                               "clock-frequency", &bus_clk_rate);
+                if (ret) {
+                        dev_err(dev, "failed to read clock-frequency property\n");
+                        return ret;
+                }
+        }
+
+        if (bus_clk_rate > 1000000L) {
+                dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n",
+                        bus_clk_rate);
+                return -EINVAL;
+        }
+
+        i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL);
+        if (!i2c_priv)
+                return -ENOMEM;
+
+        i2c_priv->client = client;
+        mutex_init(&i2c_priv->lock);
+
+        /*
+         * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate -
+         * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz
+         * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE.
+         */
+        i2c_priv->wake_token_sz = atmel_i2c_wake_token_sz(bus_clk_rate);
+
+        memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token));
+
+        atomic_set(&i2c_priv->tfm_count, 0);
+
+        i2c_set_clientdata(client, i2c_priv);
+
+        ret = device_sanity_check(client);
+        if (ret)
+                return ret;
+
+        return 0;
+}
+EXPORT_SYMBOL(atmel_i2c_probe);
+
+MODULE_AUTHOR("Tudor Ambarus <tudor.ambarus@microchip.com>");
+MODULE_DESCRIPTION("Microchip / Atmel ECC (I2C) driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/atmel-i2c.h b/drivers/crypto/atmel-i2c.h
new file mode 100644
index 000000000000..21860b99c3e3
--- /dev/null
+++ b/drivers/crypto/atmel-i2c.h
@@ -0,0 +1,197 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2017, Microchip Technology Inc.
+ * Author: Tudor Ambarus <tudor.ambarus@microchip.com>
+ */
+
+#ifndef __ATMEL_I2C_H__
+#define __ATMEL_I2C_H__
+
+#include <linux/hw_random.h>
+#include <linux/types.h>
+
+#define ATMEL_ECC_PRIORITY              300
+
+#define COMMAND                         0x03 /* packet function */
+#define SLEEP_TOKEN                     0x01
+#define WAKE_TOKEN_MAX_SIZE             8
+
+/* Definitions of Data and Command sizes */
+#define WORD_ADDR_SIZE                  1
+#define COUNT_SIZE                      1
+#define CRC_SIZE                        2
+#define CMD_OVERHEAD_SIZE               (COUNT_SIZE + CRC_SIZE)
+
+/* size in bytes of the n prime */
+#define ATMEL_ECC_NIST_P256_N_SIZE      32
+#define ATMEL_ECC_PUBKEY_SIZE           (2 * ATMEL_ECC_NIST_P256_N_SIZE)
+
+#define STATUS_RSP_SIZE                 4
+#define ECDH_RSP_SIZE                   (32 + CMD_OVERHEAD_SIZE)
+#define GENKEY_RSP_SIZE                 (ATMEL_ECC_PUBKEY_SIZE + \
+                                         CMD_OVERHEAD_SIZE)
+#define READ_RSP_SIZE                   (4 + CMD_OVERHEAD_SIZE)
+#define RANDOM_RSP_SIZE                 (32 + CMD_OVERHEAD_SIZE)
+#define MAX_RSP_SIZE                    GENKEY_RSP_SIZE
+
+/**
+ * atmel_i2c_cmd - structure used for communicating with the device.
+ * @word_addr: indicates the function of the packet sent to the device. This
+ *             byte should have a value of COMMAND for normal operation.
+ * @count    : number of bytes to be transferred to (or from) the device.
+ * @opcode   : the command code.
+ * @param1   : the first parameter; always present.
+ * @param2   : the second parameter; always present.
+ * @data     : optional remaining input data. Includes a 2-byte CRC.
+ * @rxsize   : size of the data received from i2c client.
+ * @msecs    : command execution time in milliseconds
+ */
+struct atmel_i2c_cmd {
+        u8 word_addr;
+        u8 count;
+        u8 opcode;
+        u8 param1;
+        __le16 param2;
+        u8 data[MAX_RSP_SIZE];
+        u8 msecs;
+        u16 rxsize;
+} __packed;
+
+/* Status/Error codes */
+#define STATUS_SIZE                     0x04
+#define STATUS_NOERR                    0x00
+#define STATUS_WAKE_SUCCESSFUL          0x11
+
+static const struct {
+        u8 value;
+        const char *error_text;
+} error_list[] = {
+        { 0x01, "CheckMac or Verify miscompare" },
+        { 0x03, "Parse Error" },
+        { 0x05, "ECC Fault" },
+        { 0x0F, "Execution Error" },
+        { 0xEE, "Watchdog about to expire" },
+        { 0xFF, "CRC or other communication error" },
+};
+
+/* Definitions for eeprom organization */
+#define CONFIG_ZONE                     0
+
+/* Definitions for Indexes common to all commands */
+#define RSP_DATA_IDX                    1 /* buffer index of data in response */
+#define DATA_SLOT_2                     2 /* used for ECDH private key */
+
+/* Definitions for the device lock state */
+#define DEVICE_LOCK_ADDR                0x15
+#define LOCK_VALUE_IDX                  (RSP_DATA_IDX + 2)
+#define LOCK_CONFIG_IDX                 (RSP_DATA_IDX + 3)
+
+/*
+ * Wake High delay to data communication (microseconds). SDA should be stable
+ * high for this entire duration.
+ */
+#define TWHI_MIN                        1500
+#define TWHI_MAX                        1550
+
+/* Wake Low duration */
+#define TWLO_USEC                       60
+
+/* Command execution time (milliseconds) */
+#define MAX_EXEC_TIME_ECDH              58
+#define MAX_EXEC_TIME_GENKEY            115
+#define MAX_EXEC_TIME_READ              1
+#define MAX_EXEC_TIME_RANDOM            50
+
+/* Command opcode */
+#define OPCODE_ECDH                     0x43
+#define OPCODE_GENKEY                   0x40
+#define OPCODE_READ                     0x02
+#define OPCODE_RANDOM                   0x1b
+
+/* Definitions for the READ Command */
+#define READ_COUNT                      7
+
+/* Definitions for the RANDOM Command */
+#define RANDOM_COUNT                    7
+
+/* Definitions for the GenKey Command */
+#define GENKEY_COUNT                    7
+#define GENKEY_MODE_PRIVATE             0x04
+
+/* Definitions for the ECDH Command */
+#define ECDH_COUNT                      71
+#define ECDH_PREFIX_MODE                0x00
+
+/* Used for binding tfm objects to i2c clients. */
+struct atmel_ecc_driver_data {
+        struct list_head i2c_client_list;
+        spinlock_t i2c_list_lock;
+} ____cacheline_aligned;
+
+/**
+ * atmel_i2c_client_priv - i2c_client private data
+ * @client              : pointer to i2c client device
+ * @i2c_client_list_node: part of i2c_client_list
+ * @lock                : lock for sending i2c commands
+ * @wake_token          : wake token array of zeros
+ * @wake_token_sz       : size in bytes of the wake_token
+ * @tfm_count           : number of active crypto transformations on i2c client
+ *
+ * Reads and writes from/to the i2c client are sequential. The first byte
+ * transmitted to the device is treated as the byte size. Any attempt to send
+ * more than this number of bytes will cause the device to not ACK those bytes.
+ * After the host writes a single command byte to the input buffer, reads are
+ * prohibited until after the device completes command execution. Use a mutex
+ * when sending i2c commands.
+ */
+struct atmel_i2c_client_priv {
+        struct i2c_client *client;
+        struct list_head i2c_client_list_node;
+        struct mutex lock;
+        u8 wake_token[WAKE_TOKEN_MAX_SIZE];
+        size_t wake_token_sz;
+        atomic_t tfm_count ____cacheline_aligned;
+        struct hwrng hwrng;
+};
+
+/**
+ * atmel_i2c_work_data - data structure representing the work
+ * @ctx : transformation context.
+ * @cbk : pointer to a callback function to be invoked upon completion of this
+ *        request. This has the form:
+ *        callback(struct atmel_i2c_work_data *work_data, void *areq, u8 status)
+ *        where:
+ *        @work_data: data structure representing the work
+ *        @areq     : optional pointer to an argument passed with the original
+ *                    request.
+ *        @status   : status returned from the i2c client device or i2c error.
+ * @areq: optional pointer to a user argument for use at callback time.
+ * @work: describes the task to be executed.
+ * @cmd : structure used for communicating with the device.
+ */
+struct atmel_i2c_work_data {
+        void *ctx;
+        struct i2c_client *client;
+        void (*cbk)(struct atmel_i2c_work_data *work_data, void *areq,
+                    int status);
+        void *areq;
+        struct work_struct work;
+        struct atmel_i2c_cmd cmd;
+};
+
+int atmel_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id);
+
+void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data,
+                       void (*cbk)(struct atmel_i2c_work_data *work_data,
+                                   void *areq, int status),
+                       void *areq);
+
+int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd);
+
+void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd);
+void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd);
+void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid);
+int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd,
+                            struct scatterlist *pubkey);
+
+#endif /* __ATMEL_I2C_H__ */
diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c
new file mode 100644
index 000000000000..ea0d2068ea4f
--- /dev/null
+++ b/drivers/crypto/atmel-sha204a.c
@@ -0,0 +1,171 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Microchip / Atmel SHA204A (I2C) driver.
+ *
+ * Copyright (c) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+#include "atmel-i2c.h"
+
+static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data,
+                                   void *areq, int status)
+{
+        struct atmel_i2c_client_priv *i2c_priv = work_data->ctx;
+        struct hwrng *rng = areq;
+
+        if (status)
+                dev_warn_ratelimited(&i2c_priv->client->dev,
+                                     "i2c transaction failed (%d)\n",
+                                     status);
+
+        rng->priv = (unsigned long)work_data;
+        atomic_dec(&i2c_priv->tfm_count);
+}
+
+static int atmel_sha204a_rng_read_nonblocking(struct hwrng *rng, void *data,
+                                              size_t max)
+{
+        struct atmel_i2c_client_priv *i2c_priv;
+        struct atmel_i2c_work_data *work_data;
+
+        i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng);
+
+        /* keep maximum 1 asynchronous read in flight at any time */
+        if (!atomic_add_unless(&i2c_priv->tfm_count, 1, 1))
+                return 0;
+
+        if (rng->priv) {
+                work_data = (struct atmel_i2c_work_data *)rng->priv;
+                max = min(sizeof(work_data->cmd.data), max);
+                memcpy(data, &work_data->cmd.data, max);
+                rng->priv = 0;
+        } else {
+                work_data = kmalloc(sizeof(*work_data), GFP_ATOMIC);
+                if (!work_data)
+                        return -ENOMEM;
+
+                work_data->ctx = i2c_priv;
+                work_data->client = i2c_priv->client;
+
+                max = 0;
+        }
+
+        atmel_i2c_init_random_cmd(&work_data->cmd);
+        atmel_i2c_enqueue(work_data, atmel_sha204a_rng_done, rng);
+
+        return max;
+}
+
+static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max,
+                                  bool wait)
+{
+        struct atmel_i2c_client_priv *i2c_priv;
+        struct atmel_i2c_cmd cmd;
+        int ret;
+
+        if (!wait)
+                return atmel_sha204a_rng_read_nonblocking(rng, data, max);
+
+        i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng);
+
+        atmel_i2c_init_random_cmd(&cmd);
+
+        ret = atmel_i2c_send_receive(i2c_priv->client, &cmd);
+        if (ret)
+                return ret;
+
+        max = min(sizeof(cmd.data), max);
+        memcpy(data, cmd.data, max);
+
+        return max;
+}
+
+static int atmel_sha204a_probe(struct i2c_client *client,
+                               const struct i2c_device_id *id)
+{
+        struct atmel_i2c_client_priv *i2c_priv;
+        int ret;
+
+        ret = atmel_i2c_probe(client, id);
+        if (ret)
+                return ret;
+
+        i2c_priv = i2c_get_clientdata(client);
+
+        memset(&i2c_priv->hwrng, 0, sizeof(i2c_priv->hwrng));
+
+        i2c_priv->hwrng.name = dev_name(&client->dev);
+        i2c_priv->hwrng.read = atmel_sha204a_rng_read;
+        i2c_priv->hwrng.quality = 1024;
+
+        ret = hwrng_register(&i2c_priv->hwrng);
+        if (ret)
+                dev_warn(&client->dev, "failed to register RNG (%d)\n", ret);
+
+        return ret;
+}
+
+static int atmel_sha204a_remove(struct i2c_client *client)
+{
+        struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
+
+        if (atomic_read(&i2c_priv->tfm_count)) {
+                dev_err(&client->dev, "Device is busy\n");
+                return -EBUSY;
+        }
+
+        if (i2c_priv->hwrng.priv)
+                kfree((void *)i2c_priv->hwrng.priv);
+        hwrng_unregister(&i2c_priv->hwrng);
+
+        return 0;
+}
+
+static const struct of_device_id atmel_sha204a_dt_ids[] = {
+        { .compatible = "atmel,atsha204a", },
+        { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, atmel_sha204a_dt_ids);
+
+static const struct i2c_device_id atmel_sha204a_id[] = {
+        { "atsha204a", 0 },
+        { /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(i2c, atmel_sha204a_id);
+
+static struct i2c_driver atmel_sha204a_driver = {
+        .probe                  = atmel_sha204a_probe,
+        .remove                 = atmel_sha204a_remove,
+        .id_table               = atmel_sha204a_id,
+
+        .driver.name            = "atmel-sha204a",
+        .driver.of_match_table  = of_match_ptr(atmel_sha204a_dt_ids),
+};
+
+static int __init atmel_sha204a_init(void)
+{
+        return i2c_add_driver(&atmel_sha204a_driver);
+}
+
+static void __exit atmel_sha204a_exit(void)
+{
+        flush_scheduled_work();
+        i2c_del_driver(&atmel_sha204a_driver);
+}
+
+module_init(atmel_sha204a_init);
+module_exit(atmel_sha204a_exit);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c
index 18410c9e7b29..869602fcfd96 100644
--- a/drivers/crypto/bcm/cipher.c
+++ b/drivers/crypto/bcm/cipher.c
@@ -85,7 +85,7 @@ MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos");
  * 0x70 - ring 2
  * 0x78 - ring 3
  */
-char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
+static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 };
 /*
  * Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN
  * is set dynamically after reading SPU type from device tree.
@@ -2083,7 +2083,7 @@ static int __ahash_init(struct ahash_request *req)
  * Return: true if incremental hashing is not supported
  *         false otherwise
  */
-bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
+static bool spu_no_incr_hash(struct iproc_ctx_s *ctx)
 {
         struct spu_hw *spu = &iproc_priv.spu;
 
@@ -4809,7 +4809,7 @@ static int spu_dt_read(struct platform_device *pdev)
         return 0;
 }
 
-int bcm_spu_probe(struct platform_device *pdev)
+static int bcm_spu_probe(struct platform_device *pdev)
 {
         struct device *dev = &pdev->dev;
         struct spu_hw *spu = &iproc_priv.spu;
@@ -4853,7 +4853,7 @@ failure:
         return err;
 }
 
-int bcm_spu_remove(struct platform_device *pdev)
+static int bcm_spu_remove(struct platform_device *pdev)
 {
         int i;
         struct device *dev = &pdev->dev;
diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c
index cb477259a2e2..2add51024575 100644
--- a/drivers/crypto/bcm/spu2.c
+++ b/drivers/crypto/bcm/spu2.c
@@ -38,21 +38,21 @@ enum spu2_proto_sel {
         SPU2_DTLS_AEAD = 10
 };
 
-char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256",
+static char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256",
         "DES", "3DES"
 };
 
-char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB", "XTS",
-        "CCM", "GCM"
+static char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB",
+        "XTS", "CCM", "GCM"
 };
 
-char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256",
+static char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256",
         "Reserved", "Reserved", "MD5", "SHA1", "SHA224", "SHA256", "SHA384",
         "SHA512", "SHA512/224", "SHA512/256", "SHA3-224", "SHA3-256",
         "SHA3-384", "SHA3-512"
 };
 
-char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC",
+static char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC",
         "Rabin", "CCM", "GCM", "Reserved"
 };
 
diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig
index 577c9844b322..3720ddabb507 100644
--- a/drivers/crypto/caam/Kconfig
+++ b/drivers/crypto/caam/Kconfig
@@ -2,6 +2,12 @@
 config CRYPTO_DEV_FSL_CAAM_COMMON
         tristate
 
+config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
+        tristate
+
+config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
+        tristate
+
 config CRYPTO_DEV_FSL_CAAM
         tristate "Freescale CAAM-Multicore platform driver backend"
         depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE
@@ -25,7 +31,7 @@ config CRYPTO_DEV_FSL_CAAM_DEBUG
           Selecting this will enable printing of various debug
           information in the CAAM driver.
 
-config CRYPTO_DEV_FSL_CAAM_JR
+menuconfig CRYPTO_DEV_FSL_CAAM_JR
         tristate "Freescale CAAM Job Ring driver backend"
         default y
         help
@@ -86,8 +92,9 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD
           threshold. Range is 1-65535.
 
 config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
-        tristate "Register algorithm implementations with the Crypto API"
+        bool "Register algorithm implementations with the Crypto API"
         default y
+        select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
         select CRYPTO_AEAD
         select CRYPTO_AUTHENC
         select CRYPTO_BLKCIPHER
@@ -97,13 +104,11 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
           scatterlist crypto API (such as the linux native IPSec
           stack) to the SEC4 via job ring.
 
-          To compile this as a module, choose M here: the module
-          will be called caamalg.
-
 config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
-        tristate "Queue Interface as Crypto API backend"
+        bool "Queue Interface as Crypto API backend"
         depends on FSL_DPAA && NET
         default y
+        select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
         select CRYPTO_AUTHENC
         select CRYPTO_BLKCIPHER
         help
@@ -114,33 +119,26 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
           assigned to the kernel should also be more than the number of
           job rings.
 
-          To compile this as a module, choose M here: the module
-          will be called caamalg_qi.
-
 config CRYPTO_DEV_FSL_CAAM_AHASH_API
-        tristate "Register hash algorithm implementations with Crypto API"
+        bool "Register hash algorithm implementations with Crypto API"
         default y
+        select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
         select CRYPTO_HASH
         help
           Selecting this will offload ahash for users of the
           scatterlist crypto API to the SEC4 via job ring.
 
-          To compile this as a module, choose M here: the module
-          will be called caamhash.
-
 config CRYPTO_DEV_FSL_CAAM_PKC_API
-        tristate "Register public key cryptography implementations with Crypto API"
+        bool "Register public key cryptography implementations with Crypto API"
         default y
         select CRYPTO_RSA
         help
           Selecting this will allow SEC Public key support for RSA.
           Supported cryptographic primitives: encryption, decryption,
           signature and verification.
-          To compile this as a module, choose M here: the module
-          will be called caam_pkc.
 
 config CRYPTO_DEV_FSL_CAAM_RNG_API
-        tristate "Register caam device for hwrng API"
+        bool "Register caam device for hwrng API"
         default y
         select CRYPTO_RNG
         select HW_RANDOM
@@ -148,9 +146,6 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API
           Selecting this will register the SEC4 hardware rng to
           the hw_random API for suppying the kernel entropy pool.
 
-          To compile this as a module, choose M here: the module
-          will be called caamrng.
-
 endif # CRYPTO_DEV_FSL_CAAM_JR
 
 endif # CRYPTO_DEV_FSL_CAAM
@@ -160,6 +155,8 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM
         depends on FSL_MC_DPIO
         depends on NETDEVICES
         select CRYPTO_DEV_FSL_CAAM_COMMON
+        select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
+        select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
         select CRYPTO_BLKCIPHER
         select CRYPTO_AUTHENC
         select CRYPTO_AEAD
@@ -171,12 +168,3 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM
 
           To compile this as a module, choose M here: the module
           will be called dpaa2_caam.
-
-config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC
-        def_tristate (CRYPTO_DEV_FSL_CAAM_CRYPTO_API || \
-                      CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI || \
-                      CRYPTO_DEV_FSL_DPAA2_CAAM)
-
-config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC
-        def_tristate (CRYPTO_DEV_FSL_CAAM_AHASH_API || \
-                      CRYPTO_DEV_FSL_DPAA2_CAAM)
diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile
index 7bbfd06a11ff..9ab4e81ea21e 100644
--- a/drivers/crypto/caam/Makefile
+++ b/drivers/crypto/caam/Makefile
@@ -11,20 +11,20 @@ ccflags-y += -DVERSION=\"\"
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += error.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o
-obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
-obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o
-obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
 obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC) += caamhash_desc.o
-obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
-obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o
 
-caam-objs := ctrl.o
-caam_jr-objs := jr.o key_gen.o
-caam_pkc-y := caampkc.o pkc_desc.o
+caam-y := ctrl.o
+caam_jr-y := jr.o key_gen.o
+caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
+caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o
+caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o
+caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o
+caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caampkc.o pkc_desc.o
+
+caam-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += qi.o
 ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),)
         ccflags-y += -DCONFIG_CAAM_QI
-        caam-objs += qi.o
 endif
 
 obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index c0ece44f303b..43f18253e5b6 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -77,13 +77,6 @@
 #define DESC_MAX_USED_BYTES             (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
 
-#ifdef DEBUG
-/* for print_hex_dumps with line references */
-#define debug(format, arg...) printk(format, arg)
-#else
-#define debug(format, arg...)
-#endif
-
 struct caam_alg_entry {
         int class1_alg_type;
         int class2_alg_type;
@@ -583,13 +576,11 @@ static int aead_setkey(struct crypto_aead *aead,
         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
                 goto badkey;
 
-#ifdef DEBUG
-        printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
+        dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
                keys.authkeylen + keys.enckeylen, keys.enckeylen,
                keys.authkeylen);
-        print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+        print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
         /*
          * If DKP is supported, use it in the shared descriptor to generate
@@ -623,11 +614,10 @@ static int aead_setkey(struct crypto_aead *aead,
         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
         dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
                                    keys.enckeylen, ctx->dir);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
-                       ctx->adata.keylen_pad + keys.enckeylen, 1);
-#endif
+
+        print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
+                             ctx->adata.keylen_pad + keys.enckeylen, 1);
 
 skip_split_key:
         ctx->cdata.keylen = keys.enckeylen;
@@ -678,10 +668,8 @@ static int gcm_setkey(struct crypto_aead *aead,
         struct caam_ctx *ctx = crypto_aead_ctx(aead);
         struct device *jrdev = ctx->jrdev;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+        print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
         memcpy(ctx->key, key, keylen);
         dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
@@ -699,10 +687,8 @@ static int rfc4106_setkey(struct crypto_aead *aead,
         if (keylen < 4)
                 return -EINVAL;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+        print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
         memcpy(ctx->key, key, keylen);
 
@@ -725,10 +711,8 @@ static int rfc4543_setkey(struct crypto_aead *aead,
         if (keylen < 4)
                 return -EINVAL;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+        print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
         memcpy(ctx->key, key, keylen);
 
@@ -757,10 +741,8 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
                                OP_ALG_AAI_CTR_MOD128);
         const bool is_rfc3686 = alg->caam.rfc3686;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+        print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
         /*
          * AES-CTR needs to load IV in CONTEXT1 reg
          * at an offset of 128bits (16bytes)
@@ -916,7 +898,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
         }
 
         if (iv_dma)
-                dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
+                dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL);
         if (sec4_sg_bytes)
                 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
                                  DMA_TO_DEVICE);
@@ -949,9 +931,7 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
         struct aead_request *req = context;
         struct aead_edesc *edesc;
 
-#ifdef DEBUG
-        dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+        dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 
         edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
 
@@ -971,9 +951,7 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
         struct aead_request *req = context;
         struct aead_edesc *edesc;
 
-#ifdef DEBUG
-        dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+        dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 
         edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
 
@@ -1001,33 +979,32 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
         int ivsize = crypto_skcipher_ivsize(skcipher);
 
-#ifdef DEBUG
-        dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+        dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 
         edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
 
         if (err)
                 caam_jr_strstatus(jrdev, err);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
-                       edesc->src_nents > 1 ? 100 : ivsize, 1);
-#endif
-        caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
-                     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
-                     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
-
         skcipher_unmap(jrdev, edesc, req);
 
         /*
          * The crypto API expects us to set the IV (req->iv) to the last
-         * ciphertext block. This is used e.g. by the CTS mode.
+         * ciphertext block (CBC mode) or last counter (CTR mode).
+         * This is used e.g. by the CTS mode.
          */
-        if (ivsize)
-                scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen -
-                                         ivsize, ivsize, 0);
+        if (ivsize) {
+                memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
+                       ivsize);
+
+                print_hex_dump_debug("dstiv  @"__stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
+                                     edesc->src_nents > 1 ? 100 : ivsize, 1);
+        }
+
+        caam_dump_sg("dst    @" __stringify(__LINE__)": ",
+                     DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
+                     edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
 
         kfree(edesc);
 
@@ -1039,26 +1016,35 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 {
         struct skcipher_request *req = context;
         struct skcipher_edesc *edesc;
-#ifdef DEBUG
         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
         int ivsize = crypto_skcipher_ivsize(skcipher);
 
-        dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+        dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 
         edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]);
         if (err)
                 caam_jr_strstatus(jrdev, err);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
-#endif
-        caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
+        skcipher_unmap(jrdev, edesc, req);
+
+        /*
+         * The crypto API expects us to set the IV (req->iv) to the last
+         * ciphertext block (CBC mode) or last counter (CTR mode).
+         * This is used e.g. by the CTS mode.
+         */
+        if (ivsize) {
+                memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes,
+                       ivsize);
+
+                print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
+                                     ivsize, 1);
+        }
+
+        caam_dump_sg("dst    @" __stringify(__LINE__)": ",
                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
 
-        skcipher_unmap(jrdev, edesc, req);
         kfree(edesc);
 
         skcipher_request_complete(req, err);
@@ -1106,6 +1092,7 @@ static void init_aead_job(struct aead_request *req,
         if (unlikely(req->src != req->dst)) {
                 if (!edesc->mapped_dst_nents) {
                         dst_dma = 0;
+                        out_options = 0;
                 } else if (edesc->mapped_dst_nents == 1) {
                         dst_dma = sg_dma_address(req->dst);
                         out_options = 0;
@@ -1249,6 +1236,7 @@ static void init_skcipher_job(struct skcipher_request *req,
 {
         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
+        struct device *jrdev = ctx->jrdev;
         int ivsize = crypto_skcipher_ivsize(skcipher);
         u32 *desc = edesc->hw_desc;
         u32 *sh_desc;
@@ -1256,13 +1244,12 @@ static void init_skcipher_job(struct skcipher_request *req,
         dma_addr_t src_dma, dst_dma, ptr;
         int len, sec4_sg_index = 0;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
-        pr_err("asked=%d, cryptlen%d\n",
+        print_hex_dump_debug("presciv@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
+        dev_dbg(jrdev, "asked=%d, cryptlen%d\n",
                (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen);
-#endif
-        caam_dump_sg(KERN_ERR, "src    @" __stringify(__LINE__)": ",
+
+        caam_dump_sg("src    @" __stringify(__LINE__)": ",
                      DUMP_PREFIX_ADDRESS, 16, 4, req->src,
                      edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
 
@@ -1285,7 +1272,7 @@ static void init_skcipher_job(struct skcipher_request *req,
         if (likely(req->src == req->dst)) {
                 dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry);
                 out_options = in_options;
-        } else if (edesc->mapped_dst_nents == 1) {
+        } else if (!ivsize && edesc->mapped_dst_nents == 1) {
                 dst_dma = sg_dma_address(req->dst);
         } else {
                 dst_dma = edesc->sec4_sg_dma + sec4_sg_index *
@@ -1293,7 +1280,7 @@ static void init_skcipher_job(struct skcipher_request *req,
                 out_options = LDST_SGF;
         }
 
-        append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options);
+        append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options);
 }
 
 /*
@@ -1309,37 +1296,36 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                        GFP_KERNEL : GFP_ATOMIC;
         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
+        int src_len, dst_len = 0;
         struct aead_edesc *edesc;
         int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
         unsigned int authsize = ctx->authsize;
 
         if (unlikely(req->dst != req->src)) {
-                src_nents = sg_nents_for_len(req->src, req->assoclen +
-                                             req->cryptlen);
+                src_len = req->assoclen + req->cryptlen;
+                dst_len = src_len + (encrypt ? authsize : (-authsize));
+
+                src_nents = sg_nents_for_len(req->src, src_len);
                 if (unlikely(src_nents < 0)) {
                         dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
-                                req->assoclen + req->cryptlen);
+                                src_len);
                         return ERR_PTR(src_nents);
                 }
 
-                dst_nents = sg_nents_for_len(req->dst, req->assoclen +
-                                             req->cryptlen +
-                                                (encrypt ? authsize :
-                                                           (-authsize)));
+                dst_nents = sg_nents_for_len(req->dst, dst_len);
                 if (unlikely(dst_nents < 0)) {
                         dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
-                                req->assoclen + req->cryptlen +
-                                (encrypt ? authsize : (-authsize)));
+                                dst_len);
                         return ERR_PTR(dst_nents);
                 }
         } else {
-                src_nents = sg_nents_for_len(req->src, req->assoclen +
-                                             req->cryptlen +
-                                             (encrypt ? authsize : 0));
+                src_len = req->assoclen + req->cryptlen +
+                          (encrypt ? authsize : 0);
+
+                src_nents = sg_nents_for_len(req->src, src_len);
                 if (unlikely(src_nents < 0)) {
                         dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
-                                req->assoclen + req->cryptlen +
-                                (encrypt ? authsize : 0));
+                                src_len);
                         return ERR_PTR(src_nents);
                 }
         }
@@ -1380,8 +1366,16 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                 }
         }
 
+        /*
+         * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+         * the end of the table by allocating more S/G entries.
+         */
         sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
-        sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
+        if (mapped_dst_nents > 1)
+                sec4_sg_len += pad_sg_nents(mapped_dst_nents);
+        else
+                sec4_sg_len = pad_sg_nents(sec4_sg_len);
+
         sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
 
         /* allocate space for base edesc and hw desc commands, link tables */
@@ -1403,12 +1397,12 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 
         sec4_sg_index = 0;
         if (mapped_src_nents > 1) {
-                sg_to_sec4_sg_last(req->src, mapped_src_nents,
+                sg_to_sec4_sg_last(req->src, src_len,
                                    edesc->sec4_sg + sec4_sg_index, 0);
                 sec4_sg_index += mapped_src_nents;
         }
         if (mapped_dst_nents > 1) {
-                sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
+                sg_to_sec4_sg_last(req->dst, dst_len,
                                    edesc->sec4_sg + sec4_sg_index, 0);
         }
 
@@ -1446,11 +1440,10 @@ static int gcm_encrypt(struct aead_request *req)
 
         /* Create and submit job descriptor */
         init_gcm_job(req, edesc, all_contig, true);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
-                       desc_bytes(edesc->hw_desc), 1);
-#endif
+
+        print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+                             desc_bytes(edesc->hw_desc), 1);
 
         desc = edesc->hw_desc;
         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
@@ -1556,11 +1549,10 @@ static int aead_encrypt(struct aead_request *req)
 
         /* Create and submit job descriptor */
         init_authenc_job(req, edesc, all_contig, true);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
-                       desc_bytes(edesc->hw_desc), 1);
-#endif
+
+        print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+                             desc_bytes(edesc->hw_desc), 1);
 
         desc = edesc->hw_desc;
         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
@@ -1591,11 +1583,10 @@ static int gcm_decrypt(struct aead_request *req)
 
         /* Create and submit job descriptor*/
         init_gcm_job(req, edesc, all_contig, false);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
-                       desc_bytes(edesc->hw_desc), 1);
-#endif
+
+        print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+                             desc_bytes(edesc->hw_desc), 1);
 
         desc = edesc->hw_desc;
         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
@@ -1627,7 +1618,7 @@ static int aead_decrypt(struct aead_request *req)
         u32 *desc;
         int ret = 0;
 
-        caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ",
+        caam_dump_sg("dec src@" __stringify(__LINE__)": ",
                      DUMP_PREFIX_ADDRESS, 16, 4, req->src,
                      req->assoclen + req->cryptlen, 1);
 
@@ -1639,11 +1630,10 @@ static int aead_decrypt(struct aead_request *req)
 
         /* Create and submit job descriptor*/
         init_authenc_job(req, edesc, all_contig, false);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
-                       desc_bytes(edesc->hw_desc), 1);
-#endif
+
+        print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+                             desc_bytes(edesc->hw_desc), 1);
 
         desc = edesc->hw_desc;
         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
@@ -1719,7 +1709,29 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
         else
                 sec4_sg_ents = mapped_src_nents + !!ivsize;
         dst_sg_idx = sec4_sg_ents;
-        sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
+
+        /*
+         * Input, output HW S/G tables: [IV, src][dst, IV]
+         * IV entries point to the same buffer
+         * If src == dst, S/G entries are reused (S/G tables overlap)
+         *
+         * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+         * the end of the table by allocating more S/G entries. Logic:
+         * if (output S/G)
+         *      pad output S/G, if needed
+         * else if (input S/G) ...
+         *      pad input S/G, if needed
+         */
+        if (ivsize || mapped_dst_nents > 1) {
+                if (req->src == req->dst)
+                        sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents);
+                else
+                        sec4_sg_ents += pad_sg_nents(mapped_dst_nents +
+                                                     !!ivsize);
+        } else {
+                sec4_sg_ents = pad_sg_nents(sec4_sg_ents);
+        }
+
         sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
 
         /*
@@ -1744,10 +1756,10 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
 
         /* Make sure IV is located in a DMAable area */
         if (ivsize) {
-                iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes;
+                iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes;
                 memcpy(iv, req->iv, ivsize);
 
-                iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_TO_DEVICE);
+                iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL);
                 if (dma_mapping_error(jrdev, iv_dma)) {
                         dev_err(jrdev, "unable to map IV\n");
                         caam_unmap(jrdev, req->src, req->dst, src_nents,
@@ -1759,13 +1771,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
                 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
         }
         if (dst_sg_idx)
-                sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg +
-                                   !!ivsize, 0);
+                sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg +
+                              !!ivsize, 0);
 
-        if (mapped_dst_nents > 1) {
-                sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
-                                   edesc->sec4_sg + dst_sg_idx, 0);
-        }
+        if (req->src != req->dst && (ivsize || mapped_dst_nents > 1))
+                sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg +
+                              dst_sg_idx, 0);
+
+        if (ivsize)
+                dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx +
+                                   mapped_dst_nents, iv_dma, ivsize, 0);
+
+        if (ivsize || mapped_dst_nents > 1)
+                sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx +
+                                    mapped_dst_nents);
 
         if (sec4_sg_bytes) {
                 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
@@ -1782,11 +1801,9 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
 
         edesc->iv_dma = iv_dma;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "skcipher sec4_sg@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
-                       sec4_sg_bytes, 1);
-#endif
+        print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
+                             sec4_sg_bytes, 1);
 
         return edesc;
 }
@@ -1807,11 +1824,11 @@ static int skcipher_encrypt(struct skcipher_request *req)
 
         /* Create and submit job descriptor*/
         init_skcipher_job(req, edesc, true);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
-                       desc_bytes(edesc->hw_desc), 1);
-#endif
+
+        print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+                             desc_bytes(edesc->hw_desc), 1);
+
         desc = edesc->hw_desc;
         ret = caam_jr_enqueue(jrdev, desc, skcipher_encrypt_done, req);
 
@@ -1830,7 +1847,6 @@ static int skcipher_decrypt(struct skcipher_request *req)
         struct skcipher_edesc *edesc;
         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
-        int ivsize = crypto_skcipher_ivsize(skcipher);
         struct device *jrdev = ctx->jrdev;
         u32 *desc;
         int ret = 0;
@@ -1840,22 +1856,13 @@ static int skcipher_decrypt(struct skcipher_request *req)
         if (IS_ERR(edesc))
                 return PTR_ERR(edesc);
 
-        /*
-         * The crypto API expects us to set the IV (req->iv) to the last
-         * ciphertext block.
-         */
-        if (ivsize)
-                scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen -
-                                         ivsize, ivsize, 0);
-
         /* Create and submit job descriptor*/
         init_skcipher_job(req, edesc, false);
         desc = edesc->hw_desc;
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
-                       desc_bytes(edesc->hw_desc), 1);
-#endif
+
+        print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+                             desc_bytes(edesc->hw_desc), 1);
 
         ret = caam_jr_enqueue(jrdev, desc, skcipher_decrypt_done, req);
         if (!ret) {
@@ -3444,7 +3451,7 @@ static void caam_aead_exit(struct crypto_aead *tfm)
         caam_exit_common(crypto_aead_ctx(tfm));
 }
 
-static void __exit caam_algapi_exit(void)
+void caam_algapi_exit(void)
 {
         int i;
 
@@ -3489,43 +3496,15 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
         alg->exit = caam_aead_exit;
 }
 
-static int __init caam_algapi_init(void)
+int caam_algapi_init(struct device *ctrldev)
 {
-        struct device_node *dev_node;
-        struct platform_device *pdev;
-        struct caam_drv_private *priv;
+        struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
         int i = 0, err = 0;
         u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst;
         u32 arc4_inst;
         unsigned int md_limit = SHA512_DIGEST_SIZE;
         bool registered = false, gcm_support;
 
-        dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
-        if (!dev_node) {
-                dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
-                if (!dev_node)
-                        return -ENODEV;
-        }
-
-        pdev = of_find_device_by_node(dev_node);
-        if (!pdev) {
-                of_node_put(dev_node);
-                return -ENODEV;
-        }
-
-        priv = dev_get_drvdata(&pdev->dev);
-        of_node_put(dev_node);
-
-        /*
-         * If priv is NULL, it's probably because the caam driver wasn't
-         * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
-         */
-        if (!priv) {
-                err = -ENODEV;
-                goto out_put_dev;
-        }
-
-
         /*
          * Register crypto algorithms the device supports.
          * First, detect presence and attributes of DES, AES, and MD blocks.
@@ -3668,14 +3647,5 @@ static int __init caam_algapi_init(void)
         if (registered)
                 pr_info("caam algorithms registered in /proc/crypto\n");
 
-out_put_dev:
-        put_device(&pdev->dev);
         return err;
 }
-
-module_init(caam_algapi_init);
-module_exit(caam_algapi_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("FSL CAAM support for crypto API");
-MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c
index 1e1a376edc2f..72531837571e 100644
--- a/drivers/crypto/caam/caamalg_desc.c
+++ b/drivers/crypto/caam/caamalg_desc.c
@@ -33,12 +33,11 @@ static inline void append_dec_op1(u32 *desc, u32 type)
         }
 
         jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
-        append_operation(desc, type | OP_ALG_AS_INITFINAL |
-                         OP_ALG_DECRYPT);
+        append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
         uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
         set_jump_tgt_here(desc, jump_cmd);
-        append_operation(desc, type | OP_ALG_AS_INITFINAL |
-                         OP_ALG_DECRYPT | OP_ALG_AAI_DK);
+        append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
+                         OP_ALG_AAI_DK);
         set_jump_tgt_here(desc, uncond_jump_cmd);
 }
 
@@ -115,11 +114,9 @@ void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
                          LDST_SRCDST_BYTE_CONTEXT);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "aead null enc shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
 
@@ -204,11 +201,9 @@ void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "aead null dec shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
 
@@ -358,10 +353,9 @@ void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
                          LDST_SRCDST_BYTE_CONTEXT);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
 
@@ -475,10 +469,9 @@ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
 
@@ -613,11 +606,9 @@ copy_iv:
         append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
                          LDST_SRCDST_BYTE_CONTEXT);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "aead givenc shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
 
@@ -742,10 +733,9 @@ void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
                          LDST_SRCDST_BYTE_CONTEXT);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
 
@@ -838,10 +828,9 @@ void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
 
@@ -933,11 +922,9 @@ void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
                          LDST_SRCDST_BYTE_CONTEXT);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "rfc4106 enc shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
 
@@ -1030,11 +1017,9 @@ void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "rfc4106 dec shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
 
@@ -1115,11 +1100,9 @@ void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
         append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
                          LDST_SRCDST_BYTE_CONTEXT);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "rfc4543 enc shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
 
@@ -1205,11 +1188,9 @@ void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
         append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
                              FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "rfc4543 dec shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
 
@@ -1410,17 +1391,21 @@ void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
                                       LDST_OFFSET_SHIFT));
 
         /* Load operation */
-        append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+        append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
                          OP_ALG_ENCRYPT);
 
         /* Perform operation */
         skcipher_append_src_dst(desc);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "skcipher enc shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        /* Store IV */
+        if (ivsize)
+                append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
+                                 LDST_CLASS_1_CCB | (ctx1_iv_off <<
+                                 LDST_OFFSET_SHIFT));
+
+        print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
 
@@ -1479,7 +1464,7 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
 
         /* Choose operation */
         if (ctx1_iv_off)
-                append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
+                append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
                                  OP_ALG_DECRYPT);
         else
                 append_dec_op1(desc, cdata->algtype);
@@ -1487,11 +1472,15 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
         /* Perform operation */
         skcipher_append_src_dst(desc);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "skcipher dec shdesc@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        /* Store IV */
+        if (ivsize)
+                append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
+                                 LDST_CLASS_1_CCB | (ctx1_iv_off <<
+                                 LDST_OFFSET_SHIFT));
+
+        print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
 
@@ -1538,11 +1527,13 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
         /* Perform operation */
         skcipher_append_src_dst(desc);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "xts skcipher enc shdesc@" __stringify(__LINE__) ": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        /* Store upper 8B of IV */
+        append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
+                         (0x20 << LDST_OFFSET_SHIFT));
+
+        print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
+                             ": ", DUMP_PREFIX_ADDRESS, 16, 4,
+                             desc, desc_bytes(desc), 1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
 
@@ -1588,11 +1579,13 @@ void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
         /* Perform operation */
         skcipher_append_src_dst(desc);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "xts skcipher dec shdesc@" __stringify(__LINE__) ": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        /* Store upper 8B of IV */
+        append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
+                         (0x20 << LDST_OFFSET_SHIFT));
+
+        print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
+                             ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
+                             desc_bytes(desc), 1);
 }
 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
 
diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h
index d5ca42ff961a..da4a4ee60c80 100644
--- a/drivers/crypto/caam/caamalg_desc.h
+++ b/drivers/crypto/caam/caamalg_desc.h
@@ -44,9 +44,9 @@
 
 #define DESC_SKCIPHER_BASE              (3 * CAAM_CMD_SZ)
 #define DESC_SKCIPHER_ENC_LEN           (DESC_SKCIPHER_BASE + \
-                                         20 * CAAM_CMD_SZ)
+                                         21 * CAAM_CMD_SZ)
 #define DESC_SKCIPHER_DEC_LEN           (DESC_SKCIPHER_BASE + \
-                                         15 * CAAM_CMD_SZ)
+                                         16 * CAAM_CMD_SZ)
 
 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
                                  unsigned int icvsize, int era);
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index d290d6b41825..32f0f8a72067 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -4,7 +4,7 @@
  * Based on caamalg.c
  *
  * Copyright 2013-2016 Freescale Semiconductor, Inc.
- * Copyright 2016-2018 NXP
+ * Copyright 2016-2019 NXP
  */
 
 #include "compat.h"
@@ -214,13 +214,11 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
                 goto badkey;
 
-#ifdef DEBUG
-        dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
+        dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
                 keys.authkeylen + keys.enckeylen, keys.enckeylen,
                 keys.authkeylen);
-        print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+        print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
         /*
          * If DKP is supported, use it in the shared descriptor to generate
@@ -237,7 +235,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
                 memcpy(ctx->key, keys.authkey, keys.authkeylen);
                 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
                        keys.enckeylen);
-                dma_sync_single_for_device(jrdev, ctx->key_dma,
+                dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
                                            ctx->adata.keylen_pad +
                                            keys.enckeylen, ctx->dir);
                 goto skip_split_key;
@@ -251,8 +249,9 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 
         /* postpend encryption key to auth split key */
         memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
-        dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
-                                   keys.enckeylen, ctx->dir);
+        dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
+                                   ctx->adata.keylen_pad + keys.enckeylen,
+                                   ctx->dir);
 #ifdef DEBUG
         print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
@@ -386,13 +385,12 @@ static int gcm_setkey(struct crypto_aead *aead,
         struct device *jrdev = ctx->jrdev;
         int ret;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+        print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
         memcpy(ctx->key, key, keylen);
-        dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
+        dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen,
+                                   ctx->dir);
         ctx->cdata.keylen = keylen;
 
         ret = gcm_set_sh_desc(aead);
@@ -485,10 +483,8 @@ static int rfc4106_setkey(struct crypto_aead *aead,
         if (keylen < 4)
                 return -EINVAL;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+        print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
         memcpy(ctx->key, key, keylen);
         /*
@@ -496,8 +492,8 @@ static int rfc4106_setkey(struct crypto_aead *aead,
          * in the nonce. Update the AES key length.
          */
         ctx->cdata.keylen = keylen - 4;
-        dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
-                                   ctx->dir);
+        dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
+                                   ctx->cdata.keylen, ctx->dir);
 
         ret = rfc4106_set_sh_desc(aead);
         if (ret)
@@ -589,10 +585,8 @@ static int rfc4543_setkey(struct crypto_aead *aead,
         if (keylen < 4)
                 return -EINVAL;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+        print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
         memcpy(ctx->key, key, keylen);
         /*
@@ -600,8 +594,8 @@ static int rfc4543_setkey(struct crypto_aead *aead,
          * in the nonce. Update the AES key length.
          */
         ctx->cdata.keylen = keylen - 4;
-        dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
-                                   ctx->dir);
+        dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
+                                   ctx->cdata.keylen, ctx->dir);
 
         ret = rfc4543_set_sh_desc(aead);
         if (ret)
@@ -644,10 +638,9 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
         const bool is_rfc3686 = alg->caam.rfc3686;
         int ret = 0;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
-#endif
+        print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+
         /*
          * AES-CTR needs to load IV in CONTEXT1 reg
          * at an offset of 128bits (16bytes)
@@ -838,7 +831,8 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,
 static void caam_unmap(struct device *dev, struct scatterlist *src,
                        struct scatterlist *dst, int src_nents,
                        int dst_nents, dma_addr_t iv_dma, int ivsize,
-                       dma_addr_t qm_sg_dma, int qm_sg_bytes)
+                       enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
+                       int qm_sg_bytes)
 {
         if (dst != src) {
                 if (src_nents)
@@ -850,7 +844,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
         }
 
         if (iv_dma)
-                dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
+                dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
         if (qm_sg_bytes)
                 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
 }
@@ -863,7 +857,8 @@ static void aead_unmap(struct device *dev,
         int ivsize = crypto_aead_ivsize(aead);
 
         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
-                   edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
+                   edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
+                   edesc->qm_sg_bytes);
         dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 }
 
@@ -874,7 +869,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
         int ivsize = crypto_skcipher_ivsize(skcipher);
 
         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
-                   edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
+                   edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
+                   edesc->qm_sg_bytes);
 }
 
 static void aead_done(struct caam_drv_req *drv_req, u32 status)
@@ -924,6 +920,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                        GFP_KERNEL : GFP_ATOMIC;
         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
+        int src_len, dst_len = 0;
         struct aead_edesc *edesc;
         dma_addr_t qm_sg_dma, iv_dma = 0;
         int ivsize = 0;
@@ -945,13 +942,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         }
 
         if (likely(req->src == req->dst)) {
-                src_nents = sg_nents_for_len(req->src, req->assoclen +
-                                             req->cryptlen +
-                                                (encrypt ? authsize : 0));
+                src_len = req->assoclen + req->cryptlen +
+                          (encrypt ? authsize : 0);
+
+                src_nents = sg_nents_for_len(req->src, src_len);
                 if (unlikely(src_nents < 0)) {
                         dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
-                                req->assoclen + req->cryptlen +
-                                (encrypt ? authsize : 0));
+                                src_len);
                         qi_cache_free(edesc);
                         return ERR_PTR(src_nents);
                 }
@@ -964,23 +961,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                         return ERR_PTR(-ENOMEM);
                 }
         } else {
-                src_nents = sg_nents_for_len(req->src, req->assoclen +
-                                             req->cryptlen);
+                src_len = req->assoclen + req->cryptlen;
+                dst_len = src_len + (encrypt ? authsize : (-authsize));
+
+                src_nents = sg_nents_for_len(req->src, src_len);
                 if (unlikely(src_nents < 0)) {
                         dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
-                                req->assoclen + req->cryptlen);
+                                src_len);
                         qi_cache_free(edesc);
                         return ERR_PTR(src_nents);
                 }
 
-                dst_nents = sg_nents_for_len(req->dst, req->assoclen +
-                                             req->cryptlen +
-                                             (encrypt ? authsize :
-                                                        (-authsize)));
+                dst_nents = sg_nents_for_len(req->dst, dst_len);
                 if (unlikely(dst_nents < 0)) {
                         dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
-                                req->assoclen + req->cryptlen +
-                                (encrypt ? authsize : (-authsize)));
+                                dst_len);
                         qi_cache_free(edesc);
                         return ERR_PTR(dst_nents);
                 }
@@ -1019,9 +1014,24 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         /*
          * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
          * Input is not contiguous.
+         * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+         * the end of the table by allocating more S/G entries. Logic:
+         * if (src != dst && output S/G)
+         *      pad output S/G, if needed
+         * else if (src == dst && S/G)
+         *      overlapping S/Gs; pad one of them
+         * else if (input S/G) ...
+         *      pad input S/G, if needed
          */
-        qm_sg_ents = 1 + !!ivsize + mapped_src_nents +
-                     (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
+        qm_sg_ents = 1 + !!ivsize + mapped_src_nents;
+        if (mapped_dst_nents > 1)
+                qm_sg_ents += pad_sg_nents(mapped_dst_nents);
+        else if ((req->src == req->dst) && (mapped_src_nents > 1))
+                qm_sg_ents = max(pad_sg_nents(qm_sg_ents),
+                                 1 + !!ivsize + pad_sg_nents(mapped_src_nents));
+        else
+                qm_sg_ents = pad_sg_nents(qm_sg_ents);
+
         sg_table = &edesc->sgt[0];
         qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
         if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
@@ -1029,7 +1039,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
                         qm_sg_ents, ivsize);
                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0);
+                           0, DMA_NONE, 0, 0);
                 qi_cache_free(edesc);
                 return ERR_PTR(-ENOMEM);
         }
@@ -1044,7 +1054,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                 if (dma_mapping_error(qidev, iv_dma)) {
                         dev_err(qidev, "unable to map IV\n");
                         caam_unmap(qidev, req->src, req->dst, src_nents,
-                                   dst_nents, 0, 0, 0, 0);
+                                   dst_nents, 0, 0, DMA_NONE, 0, 0);
                         qi_cache_free(edesc);
                         return ERR_PTR(-ENOMEM);
                 }
@@ -1063,7 +1073,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         if (dma_mapping_error(qidev, edesc->assoclen_dma)) {
                 dev_err(qidev, "unable to map assoclen\n");
                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, 0, 0);
+                           iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
                 qi_cache_free(edesc);
                 return ERR_PTR(-ENOMEM);
         }
@@ -1074,19 +1084,18 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
                 qm_sg_index++;
         }
-        sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0);
+        sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
         qm_sg_index += mapped_src_nents;
 
         if (mapped_dst_nents > 1)
-                sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
-                                 qm_sg_index, 0);
+                sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
 
         qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
         if (dma_mapping_error(qidev, qm_sg_dma)) {
                 dev_err(qidev, "unable to map S/G table\n");
                 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, 0, 0);
+                           iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
                 qi_cache_free(edesc);
                 return ERR_PTR(-ENOMEM);
         }
@@ -1109,7 +1118,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                         dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma +
                                              (1 + !!ivsize) * sizeof(*sg_table),
                                              out_len, 0);
-        } else if (mapped_dst_nents == 1) {
+        } else if (mapped_dst_nents <= 1) {
                 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len,
                                  0);
         } else {
@@ -1182,33 +1191,28 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
         struct device *qidev = caam_ctx->qidev;
         int ivsize = crypto_skcipher_ivsize(skcipher);
 
-#ifdef DEBUG
-        dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
-#endif
+        dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
 
         edesc = container_of(drv_req, typeof(*edesc), drv_req);
 
         if (status)
                 caam_jr_strstatus(qidev, status);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "dstiv  @" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
-                       edesc->src_nents > 1 ? 100 : ivsize, 1);
-        caam_dump_sg(KERN_ERR, "dst    @" __stringify(__LINE__)": ",
+        print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
+                             edesc->src_nents > 1 ? 100 : ivsize, 1);
+        caam_dump_sg("dst    @" __stringify(__LINE__)": ",
                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
-#endif
 
         skcipher_unmap(qidev, edesc, req);
 
         /*
          * The crypto API expects us to set the IV (req->iv) to the last
-         * ciphertext block. This is used e.g. by the CTS mode.
+         * ciphertext block (CBC mode) or last counter (CTR mode).
+         * This is used e.g. by the CTS mode.
          */
-        if (edesc->drv_req.drv_ctx->op_type == ENCRYPT)
-                scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen -
-                                         ivsize, ivsize, 0);
+        memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize);
 
         qi_cache_free(edesc);
         skcipher_request_complete(req, status);
@@ -1276,14 +1280,26 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
         qm_sg_ents = 1 + mapped_src_nents;
         dst_sg_idx = qm_sg_ents;
 
-        qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
+        /*
+         * Input, output HW S/G tables: [IV, src][dst, IV]
+         * IV entries point to the same buffer
+         * If src == dst, S/G entries are reused (S/G tables overlap)
+         *
+         * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+         * the end of the table by allocating more S/G entries.
+         */
+        if (req->src != req->dst)
+                qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
+        else
+                qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
+
         qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
         if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
                      ivsize > CAAM_QI_MEMCACHE_SIZE)) {
                 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
                         qm_sg_ents, ivsize);
                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0);
+                           0, DMA_NONE, 0, 0);
                 return ERR_PTR(-ENOMEM);
         }
 
@@ -1292,7 +1308,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
         if (unlikely(!edesc)) {
                 dev_err(qidev, "could not allocate extended descriptor\n");
                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0);
+                           0, DMA_NONE, 0, 0);
                 return ERR_PTR(-ENOMEM);
         }
 
@@ -1301,11 +1317,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
         iv = (u8 *)(sg_table + qm_sg_ents);
         memcpy(iv, req->iv, ivsize);
 
-        iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
+        iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);
         if (dma_mapping_error(qidev, iv_dma)) {
                 dev_err(qidev, "unable to map IV\n");
                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0);
+                           0, DMA_NONE, 0, 0);
                 qi_cache_free(edesc);
                 return ERR_PTR(-ENOMEM);
         }
@@ -1319,18 +1335,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
         edesc->drv_req.drv_ctx = drv_ctx;
 
         dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
-        sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0);
+        sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
 
-        if (mapped_dst_nents > 1)
-                sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
-                                 dst_sg_idx, 0);
+        if (req->src != req->dst)
+                sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
+
+        dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
+                         ivsize, 0);
 
         edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes,
                                           DMA_TO_DEVICE);
         if (dma_mapping_error(qidev, edesc->qm_sg_dma)) {
                 dev_err(qidev, "unable to map S/G table\n");
                 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, 0, 0);
+                           iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
                 qi_cache_free(edesc);
                 return ERR_PTR(-ENOMEM);
         }
@@ -1340,16 +1358,14 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
         dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,
                                   ivsize + req->cryptlen, 0);
 
-        if (req->src == req->dst) {
+        if (req->src == req->dst)
                 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +
-                                     sizeof(*sg_table), req->cryptlen, 0);
-        } else if (mapped_dst_nents > 1) {
+                                     sizeof(*sg_table), req->cryptlen + ivsize,
+                                     0);
+        else
                 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
-                                     sizeof(*sg_table), req->cryptlen, 0);
-        } else {
-                dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst),
-                                 req->cryptlen, 0);
-        }
+                                     sizeof(*sg_table), req->cryptlen + ivsize,
+                                     0);
 
         return edesc;
 }
@@ -1359,7 +1375,6 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
         struct skcipher_edesc *edesc;
         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
-        int ivsize = crypto_skcipher_ivsize(skcipher);
         int ret;
 
         if (unlikely(caam_congested))
@@ -1370,14 +1385,6 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
         if (IS_ERR(edesc))
                 return PTR_ERR(edesc);
 
-        /*
-         * The crypto API expects us to set the IV (req->iv) to the last
-         * ciphertext block.
-         */
-        if (!encrypt)
-                scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen -
-                                         ivsize, ivsize, 0);
-
         ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
         if (!ret) {
                 ret = -EINPROGRESS;
@@ -2382,6 +2389,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
                             bool uses_dkp)
 {
         struct caam_drv_private *priv;
+        struct device *dev;
 
         /*
          * distribute tfms across job rings to ensure in-order
@@ -2393,16 +2401,17 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
                 return PTR_ERR(ctx->jrdev);
         }
 
-        priv = dev_get_drvdata(ctx->jrdev->parent);
+        dev = ctx->jrdev->parent;
+        priv = dev_get_drvdata(dev);
         if (priv->era >= 6 && uses_dkp)
                 ctx->dir = DMA_BIDIRECTIONAL;
         else
                 ctx->dir = DMA_TO_DEVICE;
 
-        ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key),
+        ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key),
                                       ctx->dir);
-        if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) {
-                dev_err(ctx->jrdev, "unable to map key\n");
+        if (dma_mapping_error(dev, ctx->key_dma)) {
+                dev_err(dev, "unable to map key\n");
                 caam_jr_free(ctx->jrdev);
                 return -ENOMEM;
         }
@@ -2411,7 +2420,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
         ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
         ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
 
-        ctx->qidev = priv->qidev;
+        ctx->qidev = dev;
 
         spin_lock_init(&ctx->lock);
         ctx->drv_ctx[ENCRYPT] = NULL;
@@ -2445,7 +2454,8 @@ static void caam_exit_common(struct caam_ctx *ctx)
         caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]);
         caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);
 
-        dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir);
+        dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key),
+                         ctx->dir);
 
         caam_jr_free(ctx->jrdev);
 }
@@ -2460,7 +2470,7 @@ static void caam_aead_exit(struct crypto_aead *tfm)
         caam_exit_common(crypto_aead_ctx(tfm));
 }
 
-static void __exit caam_qi_algapi_exit(void)
+void caam_qi_algapi_exit(void)
 {
         int i;
 
@@ -2505,45 +2515,17 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
         alg->exit = caam_aead_exit;
 }
 
-static int __init caam_qi_algapi_init(void)
+int caam_qi_algapi_init(struct device *ctrldev)
 {
-        struct device_node *dev_node;
-        struct platform_device *pdev;
-        struct device *ctrldev;
-        struct caam_drv_private *priv;
+        struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
         int i = 0, err = 0;
         u32 aes_vid, aes_inst, des_inst, md_vid, md_inst;
         unsigned int md_limit = SHA512_DIGEST_SIZE;
         bool registered = false;
 
-        dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
-        if (!dev_node) {
-                dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
-                if (!dev_node)
-                        return -ENODEV;
-        }
-
-        pdev = of_find_device_by_node(dev_node);
-        of_node_put(dev_node);
-        if (!pdev)
-                return -ENODEV;
-
-        ctrldev = &pdev->dev;
-        priv = dev_get_drvdata(ctrldev);
-
-        /*
-         * If priv is NULL, it's probably because the caam driver wasn't
-         * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
-         */
-        if (!priv || !priv->qi_present) {
-                err = -ENODEV;
-                goto out_put_dev;
-        }
-
         if (caam_dpaa2) {
                 dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n");
-                err = -ENODEV;
-                goto out_put_dev;
+                return -ENODEV;
         }
 
         /*
@@ -2598,7 +2580,7 @@ static int __init caam_qi_algapi_init(void)
 
                 err = crypto_register_skcipher(&t_alg->skcipher);
                 if (err) {
-                        dev_warn(priv->qidev, "%s alg registration failed\n",
+                        dev_warn(ctrldev, "%s alg registration failed\n",
                                  t_alg->skcipher.base.cra_driver_name);
                         continue;
                 }
@@ -2654,16 +2636,7 @@ static int __init caam_qi_algapi_init(void)
         }
 
         if (registered)
-                dev_info(priv->qidev, "algorithms registered in /proc/crypto\n");
+                dev_info(ctrldev, "algorithms registered in /proc/crypto\n");
 
-out_put_dev:
-        put_device(ctrldev);
         return err;
 }
-
-module_init(caam_qi_algapi_init);
-module_exit(caam_qi_algapi_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend");
-MODULE_AUTHOR("Freescale Semiconductor");
diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c
index 2b2980a8a9b9..06bf32c32cbd 100644
--- a/drivers/crypto/caam/caamalg_qi2.c
+++ b/drivers/crypto/caam/caamalg_qi2.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
 /*
  * Copyright 2015-2016 Freescale Semiconductor Inc.
- * Copyright 2017-2018 NXP
+ * Copyright 2017-2019 NXP
  */
 
 #include "compat.h"
@@ -140,7 +140,8 @@ static struct caam_request *to_caam_req(struct crypto_async_request *areq)
 static void caam_unmap(struct device *dev, struct scatterlist *src,
                        struct scatterlist *dst, int src_nents,
                        int dst_nents, dma_addr_t iv_dma, int ivsize,
-                       dma_addr_t qm_sg_dma, int qm_sg_bytes)
+                       enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
+                       int qm_sg_bytes)
 {
         if (dst != src) {
                 if (src_nents)
@@ -152,7 +153,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src,
         }
 
         if (iv_dma)
-                dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
+                dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
 
         if (qm_sg_bytes)
                 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
@@ -371,6 +372,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                       GFP_KERNEL : GFP_ATOMIC;
         int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
+        int src_len, dst_len = 0;
         struct aead_edesc *edesc;
         dma_addr_t qm_sg_dma, iv_dma = 0;
         int ivsize = 0;
@@ -387,23 +389,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         }
 
         if (unlikely(req->dst != req->src)) {
-                src_nents = sg_nents_for_len(req->src, req->assoclen +
-                                             req->cryptlen);
+                src_len = req->assoclen + req->cryptlen;
+                dst_len = src_len + (encrypt ? authsize : (-authsize));
+
+                src_nents = sg_nents_for_len(req->src, src_len);
                 if (unlikely(src_nents < 0)) {
                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
-                                req->assoclen + req->cryptlen);
+                                src_len);
                         qi_cache_free(edesc);
                         return ERR_PTR(src_nents);
                 }
 
-                dst_nents = sg_nents_for_len(req->dst, req->assoclen +
-                                             req->cryptlen +
-                                             (encrypt ? authsize :
-                                                        (-authsize)));
+                dst_nents = sg_nents_for_len(req->dst, dst_len);
                 if (unlikely(dst_nents < 0)) {
                         dev_err(dev, "Insufficient bytes (%d) in dst S/G\n",
-                                req->assoclen + req->cryptlen +
-                                (encrypt ? authsize : (-authsize)));
+                                dst_len);
                         qi_cache_free(edesc);
                         return ERR_PTR(dst_nents);
                 }
@@ -434,13 +434,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                         mapped_dst_nents = 0;
                 }
         } else {
-                src_nents = sg_nents_for_len(req->src, req->assoclen +
-                                             req->cryptlen +
-                                                (encrypt ? authsize : 0));
+                src_len = req->assoclen + req->cryptlen +
+                          (encrypt ? authsize : 0);
+
+                src_nents = sg_nents_for_len(req->src, src_len);
                 if (unlikely(src_nents < 0)) {
                         dev_err(dev, "Insufficient bytes (%d) in src S/G\n",
-                                req->assoclen + req->cryptlen +
-                                (encrypt ? authsize : 0));
+                                src_len);
                         qi_cache_free(edesc);
                         return ERR_PTR(src_nents);
                 }
@@ -460,9 +460,25 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         /*
          * Create S/G table: req->assoclen, [IV,] req->src [, req->dst].
          * Input is not contiguous.
+         * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+         * the end of the table by allocating more S/G entries. Logic:
+         * if (src != dst && output S/G)
+         *      pad output S/G, if needed
+         * else if (src == dst && S/G)
+         *      overlapping S/Gs; pad one of them
+         * else if (input S/G) ...
+         *      pad input S/G, if needed
          */
-        qm_sg_nents = 1 + !!ivsize + mapped_src_nents +
-                      (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
+        qm_sg_nents = 1 + !!ivsize + mapped_src_nents;
+        if (mapped_dst_nents > 1)
+                qm_sg_nents += pad_sg_nents(mapped_dst_nents);
+        else if ((req->src == req->dst) && (mapped_src_nents > 1))
+                qm_sg_nents = max(pad_sg_nents(qm_sg_nents),
+                                  1 + !!ivsize +
+                                  pad_sg_nents(mapped_src_nents));
+        else
+                qm_sg_nents = pad_sg_nents(qm_sg_nents);
+
         sg_table = &edesc->sgt[0];
         qm_sg_bytes = qm_sg_nents * sizeof(*sg_table);
         if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
@@ -470,7 +486,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
                         qm_sg_nents, ivsize);
                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0);
+                           0, DMA_NONE, 0, 0);
                 qi_cache_free(edesc);
                 return ERR_PTR(-ENOMEM);
         }
@@ -485,7 +501,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                 if (dma_mapping_error(dev, iv_dma)) {
                         dev_err(dev, "unable to map IV\n");
                         caam_unmap(dev, req->src, req->dst, src_nents,
-                                   dst_nents, 0, 0, 0, 0);
+                                   dst_nents, 0, 0, DMA_NONE, 0, 0);
                         qi_cache_free(edesc);
                         return ERR_PTR(-ENOMEM);
                 }
@@ -509,7 +525,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
         if (dma_mapping_error(dev, edesc->assoclen_dma)) {
                 dev_err(dev, "unable to map assoclen\n");
                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, 0, 0);
+                           iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
                 qi_cache_free(edesc);
                 return ERR_PTR(-ENOMEM);
         }
@@ -520,19 +536,18 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
                 qm_sg_index++;
         }
-        sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0);
+        sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
         qm_sg_index += mapped_src_nents;
 
         if (mapped_dst_nents > 1)
-                sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
-                                 qm_sg_index, 0);
+                sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
 
         qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
         if (dma_mapping_error(dev, qm_sg_dma)) {
                 dev_err(dev, "unable to map S/G table\n");
                 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, 0, 0);
+                           iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
                 qi_cache_free(edesc);
                 return ERR_PTR(-ENOMEM);
         }
@@ -559,6 +574,14 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
                         dpaa2_fl_set_addr(out_fle, qm_sg_dma +
                                           (1 + !!ivsize) * sizeof(*sg_table));
                 }
+        } else if (!mapped_dst_nents) {
+                /*
+                 * crypto engine requires the output entry to be present when
+                 * "frame list" FD is used.
+                 * Since engine does not support FMT=2'b11 (unused entry type),
+                 * leaving out_fle zeroized is the best option.
+                 */
+                goto skip_out_fle;
         } else if (mapped_dst_nents == 1) {
                 dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
                 dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
@@ -570,6 +593,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 
         dpaa2_fl_set_len(out_fle, out_len);
 
+skip_out_fle:
         return edesc;
 }
 
@@ -1077,14 +1101,26 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
         qm_sg_ents = 1 + mapped_src_nents;
         dst_sg_idx = qm_sg_ents;
 
-        qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
+        /*
+         * Input, output HW S/G tables: [IV, src][dst, IV]
+         * IV entries point to the same buffer
+         * If src == dst, S/G entries are reused (S/G tables overlap)
+         *
+         * HW reads 4 S/G entries at a time; make sure the reads don't go beyond
+         * the end of the table by allocating more S/G entries.
+         */
+        if (req->src != req->dst)
+                qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
+        else
+                qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
+
         qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry);
         if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
                      ivsize > CAAM_QI_MEMCACHE_SIZE)) {
                 dev_err(dev, "No space for %d S/G entries and/or %dB IV\n",
                         qm_sg_ents, ivsize);
                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0);
+                           0, DMA_NONE, 0, 0);
                 return ERR_PTR(-ENOMEM);
         }
 
@@ -1093,7 +1129,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
         if (unlikely(!edesc)) {
                 dev_err(dev, "could not allocate extended descriptor\n");
                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0);
+                           0, DMA_NONE, 0, 0);
                 return ERR_PTR(-ENOMEM);
         }
 
@@ -1102,11 +1138,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
         iv = (u8 *)(sg_table + qm_sg_ents);
         memcpy(iv, req->iv, ivsize);
 
-        iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
+        iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL);
         if (dma_mapping_error(dev, iv_dma)) {
                 dev_err(dev, "unable to map IV\n");
                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0,
-                           0, 0, 0);
+                           0, DMA_NONE, 0, 0);
                 qi_cache_free(edesc);
                 return ERR_PTR(-ENOMEM);
         }
@@ -1117,18 +1153,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
         edesc->qm_sg_bytes = qm_sg_bytes;
 
         dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
-        sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0);
+        sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
 
-        if (mapped_dst_nents > 1)
-                sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table +
-                                 dst_sg_idx, 0);
+        if (req->src != req->dst)
+                sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
+
+        dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
+                         ivsize, 0);
 
         edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes,
                                           DMA_TO_DEVICE);
         if (dma_mapping_error(dev, edesc->qm_sg_dma)) {
                 dev_err(dev, "unable to map S/G table\n");
                 caam_unmap(dev, req->src, req->dst, src_nents, dst_nents,
-                           iv_dma, ivsize, 0, 0);
+                           iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
                 qi_cache_free(edesc);
                 return ERR_PTR(-ENOMEM);
         }
@@ -1136,23 +1174,19 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req)
         memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt));
         dpaa2_fl_set_final(in_fle, true);
         dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize);
-        dpaa2_fl_set_len(out_fle, req->cryptlen);
+        dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize);
 
         dpaa2_fl_set_format(in_fle, dpaa2_fl_sg);
         dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma);
 
-        if (req->src == req->dst) {
-                dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
+        dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
+
+        if (req->src == req->dst)
                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma +
                                   sizeof(*sg_table));
-        } else if (mapped_dst_nents > 1) {
-                dpaa2_fl_set_format(out_fle, dpaa2_fl_sg);
+        else
                 dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx *
                                   sizeof(*sg_table));
-        } else {
-                dpaa2_fl_set_format(out_fle, dpaa2_fl_single);
-                dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst));
-        }
 
         return edesc;
 }
@@ -1164,7 +1198,8 @@ static void aead_unmap(struct device *dev, struct aead_edesc *edesc,
         int ivsize = crypto_aead_ivsize(aead);
 
         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
-                   edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
+                   edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
+                   edesc->qm_sg_bytes);
         dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
 }
 
@@ -1175,7 +1210,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
         int ivsize = crypto_skcipher_ivsize(skcipher);
 
         caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
-                   edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes);
+                   edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
+                   edesc->qm_sg_bytes);
 }
 
 static void aead_encrypt_done(void *cbk_ctx, u32 status)
@@ -1324,7 +1360,7 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
                              edesc->src_nents > 1 ? 100 : ivsize, 1);
-        caam_dump_sg(KERN_DEBUG, "dst    @" __stringify(__LINE__)": ",
+        caam_dump_sg("dst    @" __stringify(__LINE__)": ",
                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
 
@@ -1332,10 +1368,10 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status)
 
         /*
          * The crypto API expects us to set the IV (req->iv) to the last
-         * ciphertext block. This is used e.g. by the CTS mode.
+         * ciphertext block (CBC mode) or last counter (CTR mode).
+         * This is used e.g. by the CTS mode.
          */
-        scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize,
-                                 ivsize, 0);
+        memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize);
 
         qi_cache_free(edesc);
         skcipher_request_complete(req, ecode);
@@ -1362,11 +1398,19 @@ static void skcipher_decrypt_done(void *cbk_ctx, u32 status)
         print_hex_dump_debug("dstiv  @" __stringify(__LINE__)": ",
                              DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
                              edesc->src_nents > 1 ? 100 : ivsize, 1);
-        caam_dump_sg(KERN_DEBUG, "dst    @" __stringify(__LINE__)": ",
+        caam_dump_sg("dst    @" __stringify(__LINE__)": ",
                      DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
                      edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
 
         skcipher_unmap(ctx->dev, edesc, req);
+
+        /*
+         * The crypto API expects us to set the IV (req->iv) to the last
+         * ciphertext block (CBC mode) or last counter (CTR mode).
+         * This is used e.g. by the CTS mode.
+         */
+        memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize);
+
         qi_cache_free(edesc);
         skcipher_request_complete(req, ecode);
 }
@@ -1405,7 +1449,6 @@ static int skcipher_decrypt(struct skcipher_request *req)
         struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
         struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
         struct caam_request *caam_req = skcipher_request_ctx(req);
-        int ivsize = crypto_skcipher_ivsize(skcipher);
         int ret;
 
         /* allocate extended descriptor */
@@ -1413,13 +1456,6 @@ static int skcipher_decrypt(struct skcipher_request *req)
         if (IS_ERR(edesc))
                 return PTR_ERR(edesc);
 
-        /*
-         * The crypto API expects us to set the IV (req->iv) to the last
-         * ciphertext block.
-         */
-        scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize,
-                                 ivsize, 0);
-
         caam_req->flc = &ctx->flc[DECRYPT];
         caam_req->flc_dma = ctx->flc_dma[DECRYPT];
         caam_req->cbk = skcipher_decrypt_done;
@@ -3380,9 +3416,9 @@ static int ahash_update_ctx(struct ahash_request *req)
 
         if (to_hash) {
                 struct dpaa2_sg_entry *sg_table;
+                int src_len = req->nbytes - *next_buflen;
 
-                src_nents = sg_nents_for_len(req->src,
-                                             req->nbytes - (*next_buflen));
+                src_nents = sg_nents_for_len(req->src, src_len);
                 if (src_nents < 0) {
                         dev_err(ctx->dev, "Invalid number of src SG.\n");
                         return src_nents;
@@ -3409,7 +3445,7 @@ static int ahash_update_ctx(struct ahash_request *req)
 
                 edesc->src_nents = src_nents;
                 qm_sg_src_index = 1 + (*buflen ? 1 : 0);
-                qm_sg_bytes = (qm_sg_src_index + mapped_nents) *
+                qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
                               sizeof(*sg_table);
                 sg_table = &edesc->sgt[0];
 
@@ -3423,7 +3459,7 @@ static int ahash_update_ctx(struct ahash_request *req)
                         goto unmap_ctx;
 
                 if (mapped_nents) {
-                        sg_to_qm_sg_last(req->src, mapped_nents,
+                        sg_to_qm_sg_last(req->src, src_len,
                                          sg_table + qm_sg_src_index, 0);
                         if (*next_buflen)
                                 scatterwalk_map_and_copy(next_buf, req->src,
@@ -3494,7 +3530,7 @@ static int ahash_final_ctx(struct ahash_request *req)
         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                       GFP_KERNEL : GFP_ATOMIC;
         int buflen = *current_buflen(state);
-        int qm_sg_bytes, qm_sg_src_index;
+        int qm_sg_bytes;
         int digestsize = crypto_ahash_digestsize(ahash);
         struct ahash_edesc *edesc;
         struct dpaa2_sg_entry *sg_table;
@@ -3505,8 +3541,7 @@ static int ahash_final_ctx(struct ahash_request *req)
         if (!edesc)
                 return -ENOMEM;
 
-        qm_sg_src_index = 1 + (buflen ? 1 : 0);
-        qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table);
+        qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table);
         sg_table = &edesc->sgt[0];
 
         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
@@ -3518,7 +3553,7 @@ static int ahash_final_ctx(struct ahash_request *req)
         if (ret)
                 goto unmap_ctx;
 
-        dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true);
+        dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true);
 
         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
                                           DMA_TO_DEVICE);
@@ -3599,7 +3634,8 @@ static int ahash_finup_ctx(struct ahash_request *req)
 
         edesc->src_nents = src_nents;
         qm_sg_src_index = 1 + (buflen ? 1 : 0);
-        qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table);
+        qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) *
+                      sizeof(*sg_table);
         sg_table = &edesc->sgt[0];
 
         ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table,
@@ -3611,7 +3647,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
         if (ret)
                 goto unmap_ctx;
 
-        sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0);
+        sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0);
 
         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
                                           DMA_TO_DEVICE);
@@ -3696,8 +3732,8 @@ static int ahash_digest(struct ahash_request *req)
                 int qm_sg_bytes;
                 struct dpaa2_sg_entry *sg_table = &edesc->sgt[0];
 
-                qm_sg_bytes = mapped_nents * sizeof(*sg_table);
-                sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0);
+                qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table);
+                sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0);
                 edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
                                                   qm_sg_bytes, DMA_TO_DEVICE);
                 if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) {
@@ -3840,9 +3876,9 @@ static int ahash_update_no_ctx(struct ahash_request *req)
 
         if (to_hash) {
                 struct dpaa2_sg_entry *sg_table;
+                int src_len = req->nbytes - *next_buflen;
 
-                src_nents = sg_nents_for_len(req->src,
-                                             req->nbytes - *next_buflen);
+                src_nents = sg_nents_for_len(req->src, src_len);
                 if (src_nents < 0) {
                         dev_err(ctx->dev, "Invalid number of src SG.\n");
                         return src_nents;
@@ -3868,14 +3904,15 @@ static int ahash_update_no_ctx(struct ahash_request *req)
                 }
 
                 edesc->src_nents = src_nents;
-                qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table);
+                qm_sg_bytes = pad_sg_nents(1 + mapped_nents) *
+                              sizeof(*sg_table);
                 sg_table = &edesc->sgt[0];
 
                 ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
                 if (ret)
                         goto unmap_ctx;
 
-                sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0);
+                sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0);
 
                 if (*next_buflen)
                         scatterwalk_map_and_copy(next_buf, req->src,
@@ -3987,14 +4024,14 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
         }
 
         edesc->src_nents = src_nents;
-        qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table);
+        qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table);
         sg_table = &edesc->sgt[0];
 
         ret = buf_map_to_qm_sg(ctx->dev, sg_table, state);
         if (ret)
                 goto unmap;
 
-        sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0);
+        sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0);
 
         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes,
                                           DMA_TO_DEVICE);
@@ -4064,9 +4101,9 @@ static int ahash_update_first(struct ahash_request *req)
 
         if (to_hash) {
                 struct dpaa2_sg_entry *sg_table;
+                int src_len = req->nbytes - *next_buflen;
 
-                src_nents = sg_nents_for_len(req->src,
-                                             req->nbytes - (*next_buflen));
+                src_nents = sg_nents_for_len(req->src, src_len);
                 if (src_nents < 0) {
                         dev_err(ctx->dev, "Invalid number of src SG.\n");
                         return src_nents;
@@ -4101,8 +4138,9 @@ static int ahash_update_first(struct ahash_request *req)
                 if (mapped_nents > 1) {
                         int qm_sg_bytes;
 
-                        sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0);
-                        qm_sg_bytes = mapped_nents * sizeof(*sg_table);
+                        sg_to_qm_sg_last(req->src, src_len, sg_table, 0);
+                        qm_sg_bytes = pad_sg_nents(mapped_nents) *
+                                      sizeof(*sg_table);
                         edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table,
                                                           qm_sg_bytes,
                                                           DMA_TO_DEVICE);
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 7205d9f4029e..e4ac5d591ad6 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -82,14 +82,6 @@
 #define HASH_MSG_LEN                    8
 #define MAX_CTX_LEN                     (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
 
-#ifdef DEBUG
-/* for print_hex_dumps with line references */
-#define debug(format, arg...) printk(format, arg)
-#else
-#define debug(format, arg...)
-#endif
-
-
 static struct list_head hash_list;
 
 /* ahash per-session context */
@@ -243,11 +235,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
                           ctx->ctx_len, true, ctrlpriv->era);
         dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
                                    desc_bytes(desc), ctx->dir);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "ahash update shdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+
+        print_hex_dump_debug("ahash update shdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 
         /* ahash_update_first shared descriptor */
         desc = ctx->sh_desc_update_first;
@@ -255,11 +246,9 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
                           ctx->ctx_len, false, ctrlpriv->era);
         dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
                                    desc_bytes(desc), ctx->dir);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "ahash update first shdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("ahash update first shdesc@"__stringify(__LINE__)
+                             ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
+                             desc_bytes(desc), 1);
 
         /* ahash_final shared descriptor */
         desc = ctx->sh_desc_fin;
@@ -267,11 +256,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
                           ctx->ctx_len, true, ctrlpriv->era);
         dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
                                    desc_bytes(desc), ctx->dir);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-                       desc_bytes(desc), 1);
-#endif
+
+        print_hex_dump_debug("ahash final shdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc,
+                             desc_bytes(desc), 1);
 
         /* ahash_digest shared descriptor */
         desc = ctx->sh_desc_digest;
@@ -279,12 +267,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
                           ctx->ctx_len, false, ctrlpriv->era);
         dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
                                    desc_bytes(desc), ctx->dir);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR,
-                       "ahash digest shdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc,
-                       desc_bytes(desc), 1);
-#endif
+
+        print_hex_dump_debug("ahash digest shdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc,
+                             desc_bytes(desc), 1);
 
         return 0;
 }
@@ -328,9 +314,9 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash)
                             ctx->ctx_len, ctx->key_dma);
         dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
                                    desc_bytes(desc), ctx->dir);
-        print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)" : ",
-                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
-                             1);
+        print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)
+                             " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
+                             desc_bytes(desc), 1);
 
         /* shared descriptor for ahash_digest */
         desc = ctx->sh_desc_digest;
@@ -377,8 +363,8 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash)
                             ctx->ctx_len, 0);
         dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
                                    desc_bytes(desc), ctx->dir);
-        print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)" : ",
-                             DUMP_PREFIX_ADDRESS, 16, 4, desc,
+        print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)
+                             " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
                              desc_bytes(desc), 1);
 
         /* shared descriptor for ahash_digest */
@@ -429,12 +415,11 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
         append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
                          LDST_SRCDST_BYTE_CONTEXT);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
-        print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("key_in@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1);
+        print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 
         result.err = 0;
         init_completion(&result.completion);
@@ -444,11 +429,10 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key,
                 /* in progress */
                 wait_for_completion(&result.completion);
                 ret = result.err;
-#ifdef DEBUG
-                print_hex_dump(KERN_ERR,
-                               "digested key@"__stringify(__LINE__)": ",
-                               DUMP_PREFIX_ADDRESS, 16, 4, key, digestsize, 1);
-#endif
+
+                print_hex_dump_debug("digested key@"__stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, key,
+                                     digestsize, 1);
         }
         dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL);
 
@@ -463,15 +447,14 @@ static int ahash_setkey(struct crypto_ahash *ahash,
                         const u8 *key, unsigned int keylen)
 {
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+        struct device *jrdev = ctx->jrdev;
         int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
         int digestsize = crypto_ahash_digestsize(ahash);
         struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
         int ret;
         u8 *hashed_key = NULL;
 
-#ifdef DEBUG
-        printk(KERN_ERR "keylen %d\n", keylen);
-#endif
+        dev_dbg(jrdev, "keylen %d\n", keylen);
 
         if (keylen > blocksize) {
                 hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
@@ -600,11 +583,9 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
         int digestsize = crypto_ahash_digestsize(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
-#ifdef DEBUG
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 
-        dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+        dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 
         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
         if (err)
@@ -614,11 +595,9 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
         memcpy(req->result, state->caam_ctx, digestsize);
         kfree(edesc);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
-                       ctx->ctx_len, 1);
-#endif
+        print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+                             ctx->ctx_len, 1);
 
         req->base.complete(&req->base, err);
 }
@@ -631,11 +610,9 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
-#ifdef DEBUG
         int digestsize = crypto_ahash_digestsize(ahash);
 
-        dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+        dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 
         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
         if (err)
@@ -645,15 +622,13 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
         switch_buf(state);
         kfree(edesc);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
-                       ctx->ctx_len, 1);
+        print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+                             ctx->ctx_len, 1);
         if (req->result)
-                print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
-                               DUMP_PREFIX_ADDRESS, 16, 4, req->result,
-                               digestsize, 1);
-#endif
+                print_hex_dump_debug("result@"__stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
+                                     digestsize, 1);
 
         req->base.complete(&req->base, err);
 }
@@ -666,11 +641,9 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
         int digestsize = crypto_ahash_digestsize(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
-#ifdef DEBUG
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 
-        dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+        dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 
         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
         if (err)
@@ -680,11 +653,9 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
         memcpy(req->result, state->caam_ctx, digestsize);
         kfree(edesc);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
-                       ctx->ctx_len, 1);
-#endif
+        print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+                             ctx->ctx_len, 1);
 
         req->base.complete(&req->base, err);
 }
@@ -697,11 +668,9 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
         struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
         struct caam_hash_state *state = ahash_request_ctx(req);
-#ifdef DEBUG
         int digestsize = crypto_ahash_digestsize(ahash);
 
-        dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+        dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 
         edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
         if (err)
@@ -711,15 +680,13 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
         switch_buf(state);
         kfree(edesc);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
-                       ctx->ctx_len, 1);
+        print_hex_dump_debug("ctx@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
+                             ctx->ctx_len, 1);
         if (req->result)
-                print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
-                               DUMP_PREFIX_ADDRESS, 16, 4, req->result,
-                               digestsize, 1);
-#endif
+                print_hex_dump_debug("result@"__stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, req->result,
+                                     digestsize, 1);
 
         req->base.complete(&req->base, err);
 }
@@ -759,9 +726,10 @@ static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
 
         if (nents > 1 || first_sg) {
                 struct sec4_sg_entry *sg = edesc->sec4_sg;
-                unsigned int sgsize = sizeof(*sg) * (first_sg + nents);
+                unsigned int sgsize = sizeof(*sg) *
+                                      pad_sg_nents(first_sg + nents);
 
-                sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0);
+                sg_to_sec4_sg_last(req->src, to_hash, sg + first_sg, 0);
 
                 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
                 if (dma_mapping_error(ctx->jrdev, src_dma)) {
@@ -819,8 +787,10 @@ static int ahash_update_ctx(struct ahash_request *req)
         }
 
         if (to_hash) {
-                src_nents = sg_nents_for_len(req->src,
-                                             req->nbytes - (*next_buflen));
+                int pad_nents;
+                int src_len = req->nbytes - *next_buflen;
+
+                src_nents = sg_nents_for_len(req->src, src_len);
                 if (src_nents < 0) {
                         dev_err(jrdev, "Invalid number of src SG.\n");
                         return src_nents;
@@ -838,15 +808,14 @@ static int ahash_update_ctx(struct ahash_request *req)
                 }
 
                 sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
-                sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
-                                 sizeof(struct sec4_sg_entry);
+                pad_nents = pad_sg_nents(sec4_sg_src_index + mapped_nents);
+                sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
 
                 /*
                  * allocate space for base edesc and hw desc commands,
                  * link tables
                  */
-                edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
-                                          ctx->sh_desc_update,
+                edesc = ahash_edesc_alloc(ctx, pad_nents, ctx->sh_desc_update,
                                           ctx->sh_desc_update_dma, flags);
                 if (!edesc) {
                         dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
@@ -866,7 +835,7 @@ static int ahash_update_ctx(struct ahash_request *req)
                         goto unmap_ctx;
 
                 if (mapped_nents)
-                        sg_to_sec4_sg_last(req->src, mapped_nents,
+                        sg_to_sec4_sg_last(req->src, src_len,
                                            edesc->sec4_sg + sec4_sg_src_index,
                                            0);
                 else
@@ -893,11 +862,9 @@ static int ahash_update_ctx(struct ahash_request *req)
 
                 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
 
-#ifdef DEBUG
-                print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
-                               DUMP_PREFIX_ADDRESS, 16, 4, desc,
-                               desc_bytes(desc), 1);
-#endif
+                print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, desc,
+                                     desc_bytes(desc), 1);
 
                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
                 if (ret)
@@ -910,13 +877,12 @@ static int ahash_update_ctx(struct ahash_request *req)
                 *buflen = *next_buflen;
                 *next_buflen = last_buflen;
         }
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
-        print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
-                       *next_buflen, 1);
-#endif
+
+        print_hex_dump_debug("buf@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
+        print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
+                             *next_buflen, 1);
 
         return ret;
 unmap_ctx:
@@ -935,18 +901,17 @@ static int ahash_final_ctx(struct ahash_request *req)
                        GFP_KERNEL : GFP_ATOMIC;
         int buflen = *current_buflen(state);
         u32 *desc;
-        int sec4_sg_bytes, sec4_sg_src_index;
+        int sec4_sg_bytes;
         int digestsize = crypto_ahash_digestsize(ahash);
         struct ahash_edesc *edesc;
         int ret;
 
-        sec4_sg_src_index = 1 + (buflen ? 1 : 0);
-        sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry);
+        sec4_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) *
+                        sizeof(struct sec4_sg_entry);
 
         /* allocate space for base edesc and hw desc commands, link tables */
-        edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index,
-                                  ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
-                                  flags);
+        edesc = ahash_edesc_alloc(ctx, 4, ctx->sh_desc_fin,
+                                  ctx->sh_desc_fin_dma, flags);
         if (!edesc)
                 return -ENOMEM;
 
@@ -963,7 +928,7 @@ static int ahash_final_ctx(struct ahash_request *req)
         if (ret)
                 goto unmap_ctx;
 
-        sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1);
+        sg_to_sec4_set_last(edesc->sec4_sg + (buflen ? 1 : 0));
 
         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
                                             sec4_sg_bytes, DMA_TO_DEVICE);
@@ -977,10 +942,9 @@ static int ahash_final_ctx(struct ahash_request *req)
                           LDST_SGF);
         append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 
         ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
         if (ret)
@@ -1058,10 +1022,9 @@ static int ahash_finup_ctx(struct ahash_request *req)
 
         append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 
         ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
         if (ret)
@@ -1135,10 +1098,9 @@ static int ahash_digest(struct ahash_request *req)
                 return -ENOMEM;
         }
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 
         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
         if (!ret) {
@@ -1190,10 +1152,9 @@ static int ahash_final_no_ctx(struct ahash_request *req)
         if (ret)
                 goto unmap;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 
         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
         if (!ret) {
@@ -1246,8 +1207,10 @@ static int ahash_update_no_ctx(struct ahash_request *req)
         }
 
         if (to_hash) {
-                src_nents = sg_nents_for_len(req->src,
-                                             req->nbytes - *next_buflen);
+                int pad_nents;
+                int src_len = req->nbytes - *next_buflen;
+
+                src_nents = sg_nents_for_len(req->src, src_len);
                 if (src_nents < 0) {
                         dev_err(jrdev, "Invalid number of src SG.\n");
                         return src_nents;
@@ -1264,14 +1227,14 @@ static int ahash_update_no_ctx(struct ahash_request *req)
                         mapped_nents = 0;
                 }
 
-                sec4_sg_bytes = (1 + mapped_nents) *
-                                sizeof(struct sec4_sg_entry);
+                pad_nents = pad_sg_nents(1 + mapped_nents);
+                sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry);
 
                 /*
                  * allocate space for base edesc and hw desc commands,
                  * link tables
                  */
-                edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents,
+                edesc = ahash_edesc_alloc(ctx, pad_nents,
                                           ctx->sh_desc_update_first,
                                           ctx->sh_desc_update_first_dma,
                                           flags);
@@ -1287,8 +1250,7 @@ static int ahash_update_no_ctx(struct ahash_request *req)
                 if (ret)
                         goto unmap_ctx;
 
-                sg_to_sec4_sg_last(req->src, mapped_nents,
-                                   edesc->sec4_sg + 1, 0);
+                sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + 1, 0);
 
                 if (*next_buflen) {
                         scatterwalk_map_and_copy(next_buf, req->src,
@@ -1313,11 +1275,9 @@ static int ahash_update_no_ctx(struct ahash_request *req)
                 if (ret)
                         goto unmap_ctx;
 
-#ifdef DEBUG
-                print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
-                               DUMP_PREFIX_ADDRESS, 16, 4, desc,
-                               desc_bytes(desc), 1);
-#endif
+                print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, desc,
+                                     desc_bytes(desc), 1);
 
                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
                 if (ret)
@@ -1333,13 +1293,12 @@ static int ahash_update_no_ctx(struct ahash_request *req)
                 *buflen = *next_buflen;
                 *next_buflen = 0;
         }
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
-        print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
-                       *next_buflen, 1);
-#endif
+
+        print_hex_dump_debug("buf@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
+        print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
+                             1);
 
         return ret;
  unmap_ctx:
@@ -1414,10 +1373,9 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
         if (ret)
                 goto unmap;
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 
         ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
         if (!ret) {
@@ -1517,11 +1475,9 @@ static int ahash_update_first(struct ahash_request *req)
                 if (ret)
                         goto unmap_ctx;
 
-#ifdef DEBUG
-                print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
-                               DUMP_PREFIX_ADDRESS, 16, 4, desc,
-                               desc_bytes(desc), 1);
-#endif
+                print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, desc,
+                                     desc_bytes(desc), 1);
 
                 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
                 if (ret)
@@ -1539,11 +1495,10 @@ static int ahash_update_first(struct ahash_request *req)
                                          req->nbytes, 0);
                 switch_buf(state);
         }
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
-                       *next_buflen, 1);
-#endif
+
+        print_hex_dump_debug("next buf@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen,
+                             1);
 
         return ret;
  unmap_ctx:
@@ -1930,7 +1885,7 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm)
         caam_jr_free(ctx->jrdev);
 }
 
-static void __exit caam_algapi_hash_exit(void)
+void caam_algapi_hash_exit(void)
 {
         struct caam_hash_alg *t_alg, *n;
 
@@ -1988,40 +1943,13 @@ caam_hash_alloc(struct caam_hash_template *template,
         return t_alg;
 }
 
-static int __init caam_algapi_hash_init(void)
+int caam_algapi_hash_init(struct device *ctrldev)
 {
-        struct device_node *dev_node;
-        struct platform_device *pdev;
         int i = 0, err = 0;
-        struct caam_drv_private *priv;
+        struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
         unsigned int md_limit = SHA512_DIGEST_SIZE;
         u32 md_inst, md_vid;
 
-        dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
-        if (!dev_node) {
-                dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
-                if (!dev_node)
-                        return -ENODEV;
-        }
-
-        pdev = of_find_device_by_node(dev_node);
-        if (!pdev) {
-                of_node_put(dev_node);
-                return -ENODEV;
-        }
-
-        priv = dev_get_drvdata(&pdev->dev);
-        of_node_put(dev_node);
-
-        /*
-         * If priv is NULL, it's probably because the caam driver wasn't
-         * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
-         */
-        if (!priv) {
-                err = -ENODEV;
-                goto out_put_dev;
-        }
-
         /*
          * Register crypto algorithms the device supports.  First, identify
          * presence and attributes of MD block.
@@ -2042,10 +1970,8 @@ static int __init caam_algapi_hash_init(void)
          * Skip registration of any hashing algorithms if MD block
          * is not present.
          */
-        if (!md_inst) {
-                err = -ENODEV;
-                goto out_put_dev;
-        }
+        if (!md_inst)
+                return -ENODEV;
 
         /* Limit digest size based on LP256 */
         if (md_vid == CHA_VER_VID_MD_LP256)
@@ -2102,14 +2028,5 @@ static int __init caam_algapi_hash_init(void)
                         list_add_tail(&t_alg->entry, &hash_list);
         }
 
-out_put_dev:
-        put_device(&pdev->dev);
         return err;
 }
-
-module_init(caam_algapi_hash_init);
-module_exit(caam_algapi_hash_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API");
-MODULE_AUTHOR("Freescale Semiconductor - NMG");
diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c
index fe24485274e1..80574106af29 100644
--- a/drivers/crypto/caam/caampkc.c
+++ b/drivers/crypto/caam/caampkc.c
@@ -3,7 +3,7 @@
  * caam - Freescale FSL CAAM support for Public Key Cryptography
  *
  * Copyright 2016 Freescale Semiconductor, Inc.
- * Copyright 2018 NXP
+ * Copyright 2018-2019 NXP
  *
  * There is no Shared Descriptor for PKC so that the Job Descriptor must carry
  * all the desired key parameters, input and output pointers.
@@ -24,12 +24,18 @@
                                  sizeof(struct rsa_priv_f2_pdb))
 #define DESC_RSA_PRIV_F3_LEN    (2 * CAAM_CMD_SZ + \
                                  sizeof(struct rsa_priv_f3_pdb))
+#define CAAM_RSA_MAX_INPUT_SIZE 512 /* for a 4096-bit modulus */
+
+/* buffer filled with zeros, used for padding */
+static u8 *zero_buffer;
 
 static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc,
                          struct akcipher_request *req)
 {
+        struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
+
         dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE);
-        dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
+        dma_unmap_sg(dev, req_ctx->fixup_src, edesc->src_nents, DMA_TO_DEVICE);
 
         if (edesc->sec4_sg_bytes)
                 dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes,
@@ -168,6 +174,13 @@ static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err,
         akcipher_request_complete(req, err);
 }
 
+/**
+ * Count leading zeros, need it to strip, from a given scatterlist
+ *
+ * @sgl   : scatterlist to count zeros from
+ * @nbytes: number of zeros, in bytes, to strip
+ * @flags : operation flags
+ */
 static int caam_rsa_count_leading_zeros(struct scatterlist *sgl,
                                         unsigned int nbytes,
                                         unsigned int flags)
@@ -187,7 +200,8 @@ static int caam_rsa_count_leading_zeros(struct scatterlist *sgl,
         lzeros = 0;
         len = 0;
         while (nbytes > 0) {
-                while (len && !*buff) {
+                /* do not strip more than given bytes */
+                while (len && !*buff && lzeros < nbytes) {
                         lzeros++;
                         len--;
                         buff++;
@@ -218,6 +232,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
         struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
         struct device *dev = ctx->dev;
         struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
+        struct caam_rsa_key *key = &ctx->key;
         struct rsa_edesc *edesc;
         gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
                        GFP_KERNEL : GFP_ATOMIC;
@@ -225,22 +240,45 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
         int sgc;
         int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
         int src_nents, dst_nents;
+        unsigned int diff_size = 0;
         int lzeros;
 
-        lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len, sg_flags);
-        if (lzeros < 0)
-                return ERR_PTR(lzeros);
-
-        req->src_len -= lzeros;
-        req->src = scatterwalk_ffwd(req_ctx->src, req->src, lzeros);
+        if (req->src_len > key->n_sz) {
+                /*
+                 * strip leading zeros and
+                 * return the number of zeros to skip
+                 */
+                lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len -
+                                                      key->n_sz, sg_flags);
+                if (lzeros < 0)
+                        return ERR_PTR(lzeros);
+
+                req_ctx->fixup_src = scatterwalk_ffwd(req_ctx->src, req->src,
+                                                      lzeros);
+                req_ctx->fixup_src_len = req->src_len - lzeros;
+        } else {
+                /*
+                 * input src is less then n key modulus,
+                 * so there will be zero padding
+                 */
+                diff_size = key->n_sz - req->src_len;
+                req_ctx->fixup_src = req->src;
+                req_ctx->fixup_src_len = req->src_len;
+        }
 
-        src_nents = sg_nents_for_len(req->src, req->src_len);
+        src_nents = sg_nents_for_len(req_ctx->fixup_src,
+                                     req_ctx->fixup_src_len);
         dst_nents = sg_nents_for_len(req->dst, req->dst_len);
 
-        if (src_nents > 1)
-                sec4_sg_len = src_nents;
+        if (!diff_size && src_nents == 1)
+                sec4_sg_len = 0; /* no need for an input hw s/g table */
+        else
+                sec4_sg_len = src_nents + !!diff_size;
+        sec4_sg_index = sec4_sg_len;
         if (dst_nents > 1)
-                sec4_sg_len += dst_nents;
+                sec4_sg_len += pad_sg_nents(dst_nents);
+        else
+                sec4_sg_len = pad_sg_nents(sec4_sg_len);
 
         sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
 
@@ -250,7 +288,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
         if (!edesc)
                 return ERR_PTR(-ENOMEM);
 
-        sgc = dma_map_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
+        sgc = dma_map_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE);
         if (unlikely(!sgc)) {
                 dev_err(dev, "unable to map source\n");
                 goto src_fail;
@@ -263,14 +301,16 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
         }
 
         edesc->sec4_sg = (void *)edesc + sizeof(*edesc) + desclen;
+        if (diff_size)
+                dma_to_sec4_sg_one(edesc->sec4_sg, ctx->padding_dma, diff_size,
+                                   0);
+
+        if (sec4_sg_index)
+                sg_to_sec4_sg_last(req_ctx->fixup_src, req_ctx->fixup_src_len,
+                                   edesc->sec4_sg + !!diff_size, 0);
 
-        sec4_sg_index = 0;
-        if (src_nents > 1) {
-                sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
-                sec4_sg_index += src_nents;
-        }
         if (dst_nents > 1)
-                sg_to_sec4_sg_last(req->dst, dst_nents,
+                sg_to_sec4_sg_last(req->dst, req->dst_len,
                                    edesc->sec4_sg + sec4_sg_index, 0);
 
         /* Save nents for later use in Job Descriptor */
@@ -289,12 +329,16 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req,
 
         edesc->sec4_sg_bytes = sec4_sg_bytes;
 
+        print_hex_dump_debug("caampkc sec4_sg@" __stringify(__LINE__) ": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
+                             edesc->sec4_sg_bytes, 1);
+
         return edesc;
 
 sec4_sg_fail:
         dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE);
 dst_fail:
-        dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE);
+        dma_unmap_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE);
 src_fail:
         kfree(edesc);
         return ERR_PTR(-ENOMEM);
@@ -304,6 +348,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req,
                            struct rsa_edesc *edesc)
 {
         struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+        struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
         struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
         struct caam_rsa_key *key = &ctx->key;
         struct device *dev = ctx->dev;
@@ -328,7 +373,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req,
                 pdb->f_dma = edesc->sec4_sg_dma;
                 sec4_sg_index += edesc->src_nents;
         } else {
-                pdb->f_dma = sg_dma_address(req->src);
+                pdb->f_dma = sg_dma_address(req_ctx->fixup_src);
         }
 
         if (edesc->dst_nents > 1) {
@@ -340,7 +385,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req,
         }
 
         pdb->sgf |= (key->e_sz << RSA_PDB_E_SHIFT) | key->n_sz;
-        pdb->f_len = req->src_len;
+        pdb->f_len = req_ctx->fixup_src_len;
 
         return 0;
 }
@@ -373,7 +418,9 @@ static int set_rsa_priv_f1_pdb(struct akcipher_request *req,
                 pdb->g_dma = edesc->sec4_sg_dma;
                 sec4_sg_index += edesc->src_nents;
         } else {
-                pdb->g_dma = sg_dma_address(req->src);
+                struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
+
+                pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
         }
 
         if (edesc->dst_nents > 1) {
@@ -436,7 +483,9 @@ static int set_rsa_priv_f2_pdb(struct akcipher_request *req,
                 pdb->g_dma = edesc->sec4_sg_dma;
                 sec4_sg_index += edesc->src_nents;
         } else {
-                pdb->g_dma = sg_dma_address(req->src);
+                struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
+
+                pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
         }
 
         if (edesc->dst_nents > 1) {
@@ -523,7 +572,9 @@ static int set_rsa_priv_f3_pdb(struct akcipher_request *req,
                 pdb->g_dma = edesc->sec4_sg_dma;
                 sec4_sg_index += edesc->src_nents;
         } else {
-                pdb->g_dma = sg_dma_address(req->src);
+                struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req);
+
+                pdb->g_dma = sg_dma_address(req_ctx->fixup_src);
         }
 
         if (edesc->dst_nents > 1) {
@@ -978,6 +1029,15 @@ static int caam_rsa_init_tfm(struct crypto_akcipher *tfm)
                 return PTR_ERR(ctx->dev);
         }
 
+        ctx->padding_dma = dma_map_single(ctx->dev, zero_buffer,
+                                          CAAM_RSA_MAX_INPUT_SIZE - 1,
+                                          DMA_TO_DEVICE);
+        if (dma_mapping_error(ctx->dev, ctx->padding_dma)) {
+                dev_err(ctx->dev, "unable to map padding\n");
+                caam_jr_free(ctx->dev);
+                return -ENOMEM;
+        }
+
         return 0;
 }
 
@@ -987,6 +1047,8 @@ static void caam_rsa_exit_tfm(struct crypto_akcipher *tfm)
         struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
         struct caam_rsa_key *key = &ctx->key;
 
+        dma_unmap_single(ctx->dev, ctx->padding_dma, CAAM_RSA_MAX_INPUT_SIZE -
+                         1, DMA_TO_DEVICE);
         caam_rsa_free_key(key);
         caam_jr_free(ctx->dev);
 }
@@ -1010,41 +1072,12 @@ static struct akcipher_alg caam_rsa = {
 };
 
 /* Public Key Cryptography module initialization handler */
-static int __init caam_pkc_init(void)
+int caam_pkc_init(struct device *ctrldev)
 {
-        struct device_node *dev_node;
-        struct platform_device *pdev;
-        struct device *ctrldev;
-        struct caam_drv_private *priv;
+        struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
         u32 pk_inst;
         int err;
 
-        dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
-        if (!dev_node) {
-                dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
-                if (!dev_node)
-                        return -ENODEV;
-        }
-
-        pdev = of_find_device_by_node(dev_node);
-        if (!pdev) {
-                of_node_put(dev_node);
-                return -ENODEV;
-        }
-
-        ctrldev = &pdev->dev;
-        priv = dev_get_drvdata(ctrldev);
-        of_node_put(dev_node);
-
-        /*
-         * If priv is NULL, it's probably because the caam driver wasn't
-         * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
-         */
-        if (!priv) {
-                err = -ENODEV;
-                goto out_put_dev;
-        }
-
         /* Determine public key hardware accelerator presence. */
         if (priv->era < 10)
                 pk_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
@@ -1053,31 +1086,29 @@ static int __init caam_pkc_init(void)
                 pk_inst = rd_reg32(&priv->ctrl->vreg.pkha) & CHA_VER_NUM_MASK;
 
         /* Do not register algorithms if PKHA is not present. */
-        if (!pk_inst) {
-                err =  -ENODEV;
-                goto out_put_dev;
-        }
+        if (!pk_inst)
+                return 0;
+
+        /* allocate zero buffer, used for padding input */
+        zero_buffer = kzalloc(CAAM_RSA_MAX_INPUT_SIZE - 1, GFP_DMA |
+                              GFP_KERNEL);
+        if (!zero_buffer)
+                return -ENOMEM;
 
         err = crypto_register_akcipher(&caam_rsa);
-        if (err)
+        if (err) {
+                kfree(zero_buffer);
                 dev_warn(ctrldev, "%s alg registration failed\n",
                          caam_rsa.base.cra_driver_name);
-        else
+        } else {
                 dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n");
+        }
 
-out_put_dev:
-        put_device(ctrldev);
         return err;
 }
 
-static void __exit caam_pkc_exit(void)
+void caam_pkc_exit(void)
 {
+        kfree(zero_buffer);
         crypto_unregister_akcipher(&caam_rsa);
 }
-
-module_init(caam_pkc_init);
-module_exit(caam_pkc_exit);
-
-MODULE_LICENSE("Dual BSD/GPL");
-MODULE_DESCRIPTION("FSL CAAM support for PKC functions of crypto API");
-MODULE_AUTHOR("Freescale Semiconductor");
diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h
index 82645bcf8b27..2c488c9a3812 100644
--- a/drivers/crypto/caam/caampkc.h
+++ b/drivers/crypto/caam/caampkc.h
@@ -89,18 +89,25 @@ struct caam_rsa_key {
  * caam_rsa_ctx - per session context.
  * @key         : RSA key in DMA zone
  * @dev         : device structure
+ * @padding_dma : dma address of padding, for adding it to the input
  */
 struct caam_rsa_ctx {
         struct caam_rsa_key key;
         struct device *dev;
+        dma_addr_t padding_dma;
+
 };
 
 /**
  * caam_rsa_req_ctx - per request context.
- * @src: input scatterlist (stripped of leading zeros)
+ * @src           : input scatterlist (stripped of leading zeros)
+ * @fixup_src     : input scatterlist (that might be stripped of leading zeros)
+ * @fixup_src_len : length of the fixup_src input scatterlist
  */
 struct caam_rsa_req_ctx {
         struct scatterlist src[2];
+        struct scatterlist *fixup_src;
+        unsigned int fixup_src_len;
 };
 
 /**
diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c
index 95eb5402c59f..561bcb535184 100644
--- a/drivers/crypto/caam/caamrng.c
+++ b/drivers/crypto/caam/caamrng.c
@@ -3,7 +3,7 @@
  * caam - Freescale FSL CAAM support for hw_random
  *
  * Copyright 2011 Freescale Semiconductor, Inc.
- * Copyright 2018 NXP
+ * Copyright 2018-2019 NXP
  *
  * Based on caamalg.c crypto API driver.
  *
@@ -113,10 +113,8 @@ static void rng_done(struct device *jrdev, u32 *desc, u32 err, void *context)
         /* Buffer refilled, invalidate cache */
         dma_sync_single_for_cpu(jrdev, bd->addr, RN_BUF_SIZE, DMA_FROM_DEVICE);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "rng refreshed buf@: ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, bd->buf, RN_BUF_SIZE, 1);
-#endif
+        print_hex_dump_debug("rng refreshed buf@: ", DUMP_PREFIX_ADDRESS, 16, 4,
+                             bd->buf, RN_BUF_SIZE, 1);
 }
 
 static inline int submit_job(struct caam_rng_ctx *ctx, int to_current)
@@ -209,10 +207,10 @@ static inline int rng_create_sh_desc(struct caam_rng_ctx *ctx)
                 dev_err(jrdev, "unable to map shared descriptor\n");
                 return -ENOMEM;
         }
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
-                       desc, desc_bytes(desc), 1);
-#endif
+
+        print_hex_dump_debug("rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
+                             desc, desc_bytes(desc), 1);
+
         return 0;
 }
 
@@ -233,10 +231,10 @@ static inline int rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id)
         }
 
         append_seq_out_ptr_intlen(desc, bd->addr, RN_BUF_SIZE, 0);
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
-                       desc, desc_bytes(desc), 1);
-#endif
+
+        print_hex_dump_debug("rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4,
+                             desc, desc_bytes(desc), 1);
+
         return 0;
 }
 
@@ -296,47 +294,20 @@ static struct hwrng caam_rng = {
         .read           = caam_read,
 };
 
-static void __exit caam_rng_exit(void)
+void caam_rng_exit(void)
 {
         caam_jr_free(rng_ctx->jrdev);
         hwrng_unregister(&caam_rng);
         kfree(rng_ctx);
 }
 
-static int __init caam_rng_init(void)
+int caam_rng_init(struct device *ctrldev)
 {
         struct device *dev;
-        struct device_node *dev_node;
-        struct platform_device *pdev;
-        struct caam_drv_private *priv;
         u32 rng_inst;
+        struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
         int err;
 
-        dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
-        if (!dev_node) {
-                dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
-                if (!dev_node)
-                        return -ENODEV;
-        }
-
-        pdev = of_find_device_by_node(dev_node);
-        if (!pdev) {
-                of_node_put(dev_node);
-                return -ENODEV;
-        }
-
-        priv = dev_get_drvdata(&pdev->dev);
-        of_node_put(dev_node);
-
-        /*
-         * If priv is NULL, it's probably because the caam driver wasn't
-         * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
-         */
-        if (!priv) {
-                err = -ENODEV;
-                goto out_put_dev;
-        }
-
         /* Check for an instantiated RNG before registration */
         if (priv->era < 10)
                 rng_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) &
@@ -344,16 +315,13 @@ static int __init caam_rng_init(void)
         else
                 rng_inst = rd_reg32(&priv->ctrl->vreg.rng) & CHA_VER_NUM_MASK;
 
-        if (!rng_inst) {
-                err = -ENODEV;
-                goto out_put_dev;
-        }
+        if (!rng_inst)
+                return 0;
 
         dev = caam_jr_alloc();
         if (IS_ERR(dev)) {
                 pr_err("Job Ring Device allocation for transform failed\n");
-                err = PTR_ERR(dev);
-                goto out_put_dev;
+                return PTR_ERR(dev);
         }
         rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA | GFP_KERNEL);
         if (!rng_ctx) {
@@ -364,7 +332,6 @@ static int __init caam_rng_init(void)
         if (err)
                 goto free_rng_ctx;
 
-        put_device(&pdev->dev);
         dev_info(dev, "registering rng-caam\n");
         return hwrng_register(&caam_rng);
 
@@ -372,14 +339,5 @@ free_rng_ctx:
         kfree(rng_ctx);
 free_caam_alloc:
         caam_jr_free(dev);
-out_put_dev:
-        put_device(&pdev->dev);
         return err;
 }
-
-module_init(caam_rng_init);
-module_exit(caam_rng_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("FSL CAAM support for hw_random API");
-MODULE_AUTHOR("Freescale Semiconductor - NMG");
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index fec39c35c877..4e43ca4d3656 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -3,7 +3,7 @@
  * Controller-level driver, kernel property detection, initialization
  *
  * Copyright 2008-2012 Freescale Semiconductor, Inc.
- * Copyright 2018 NXP
+ * Copyright 2018-2019 NXP
  */
 
 #include <linux/device.h>
@@ -323,8 +323,8 @@ static int caam_remove(struct platform_device *pdev)
         of_platform_depopulate(ctrldev);
 
 #ifdef CONFIG_CAAM_QI
-        if (ctrlpriv->qidev)
-                caam_qi_shutdown(ctrlpriv->qidev);
+        if (ctrlpriv->qi_init)
+                caam_qi_shutdown(ctrldev);
 #endif
 
         /*
@@ -540,7 +540,8 @@ static int caam_probe(struct platform_device *pdev)
         ctrlpriv->caam_ipg = clk;
 
         if (!of_machine_is_compatible("fsl,imx7d") &&
-            !of_machine_is_compatible("fsl,imx7s")) {
+            !of_machine_is_compatible("fsl,imx7s") &&
+            !of_machine_is_compatible("fsl,imx7ulp")) {
                 clk = caam_drv_identify_clk(&pdev->dev, "mem");
                 if (IS_ERR(clk)) {
                         ret = PTR_ERR(clk);
@@ -562,7 +563,8 @@ static int caam_probe(struct platform_device *pdev)
 
         if (!of_machine_is_compatible("fsl,imx6ul") &&
             !of_machine_is_compatible("fsl,imx7d") &&
-            !of_machine_is_compatible("fsl,imx7s")) {
+            !of_machine_is_compatible("fsl,imx7s") &&
+            !of_machine_is_compatible("fsl,imx7ulp")) {
                 clk = caam_drv_identify_clk(&pdev->dev, "emi_slow");
                 if (IS_ERR(clk)) {
                         ret = PTR_ERR(clk);
@@ -702,12 +704,7 @@ static int caam_probe(struct platform_device *pdev)
         }
 
         ctrlpriv->era = caam_get_era(ctrl);
-
-        ret = of_platform_populate(nprop, caam_match, NULL, dev);
-        if (ret) {
-                dev_err(dev, "JR platform devices creation error\n");
-                goto iounmap_ctrl;
-        }
+        ctrlpriv->domain = iommu_get_domain_for_dev(dev);
 
 #ifdef CONFIG_DEBUG_FS
         /*
@@ -721,19 +718,6 @@ static int caam_probe(struct platform_device *pdev)
         ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root);
 #endif
 
-        ring = 0;
-        for_each_available_child_of_node(nprop, np)
-                if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
-                    of_device_is_compatible(np, "fsl,sec4.0-job-ring")) {
-                        ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
-                                             ((__force uint8_t *)ctrl +
-                                             (ring + JR_BLOCK_NUMBER) *
-                                              BLOCK_OFFSET
-                                             );
-                        ctrlpriv->total_jobrs++;
-                        ring++;
-                }
-
         /* Check to see if (DPAA 1.x) QI present. If so, enable */
         ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK);
         if (ctrlpriv->qi_present && !caam_dpaa2) {
@@ -752,6 +736,25 @@ static int caam_probe(struct platform_device *pdev)
 #endif
         }
 
+        ret = of_platform_populate(nprop, caam_match, NULL, dev);
+        if (ret) {
+                dev_err(dev, "JR platform devices creation error\n");
+                goto shutdown_qi;
+        }
+
+        ring = 0;
+        for_each_available_child_of_node(nprop, np)
+                if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") ||
+                    of_device_is_compatible(np, "fsl,sec4.0-job-ring")) {
+                        ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *)
+                                             ((__force uint8_t *)ctrl +
+                                             (ring + JR_BLOCK_NUMBER) *
+                                              BLOCK_OFFSET
+                                             );
+                        ctrlpriv->total_jobrs++;
+                        ring++;
+                }
+
         /* If no QI and no rings specified, quit and go home */
         if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) {
                 dev_err(dev, "no queues configured, terminating\n");
@@ -898,6 +901,11 @@ caam_remove:
         caam_remove(pdev);
         return ret;
 
+shutdown_qi:
+#ifdef CONFIG_CAAM_QI
+        if (ctrlpriv->qi_init)
+                caam_qi_shutdown(dev);
+#endif
 iounmap_ctrl:
         iounmap(ctrl);
 disable_caam_emi_slow:
diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h
index 2980b8ef1fb1..5988a26a2441 100644
--- a/drivers/crypto/caam/desc_constr.h
+++ b/drivers/crypto/caam/desc_constr.h
@@ -3,6 +3,7 @@
  * caam descriptor construction helper functions
  *
  * Copyright 2008-2012 Freescale Semiconductor, Inc.
+ * Copyright 2019 NXP
  */
 
 #ifndef DESC_CONSTR_H
@@ -37,6 +38,16 @@
 
 extern bool caam_little_end;
 
+/*
+ * HW fetches 4 S/G table entries at a time, irrespective of how many entries
+ * are in the table. It's SW's responsibility to make sure these accesses
+ * do not have side effects.
+ */
+static inline int pad_sg_nents(int sg_nents)
+{
+        return ALIGN(sg_nents, 4);
+}
+
 static inline int desc_len(u32 * const desc)
 {
         return caam32_to_cpu(*desc) & HDR_DESCLEN_MASK;
diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c
index 4da844e4b61d..4f0d45865aa2 100644
--- a/drivers/crypto/caam/error.c
+++ b/drivers/crypto/caam/error.c
@@ -13,7 +13,7 @@
 #ifdef DEBUG
 #include <linux/highmem.h>
 
-void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
+void caam_dump_sg(const char *prefix_str, int prefix_type,
                   int rowsize, int groupsize, struct scatterlist *sg,
                   size_t tlen, bool ascii)
 {
@@ -35,15 +35,15 @@ void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
 
                 buf = it_page + it->offset;
                 len = min_t(size_t, tlen, it->length);
-                print_hex_dump(level, prefix_str, prefix_type, rowsize,
-                               groupsize, buf, len, ascii);
+                print_hex_dump_debug(prefix_str, prefix_type, rowsize,
+                                     groupsize, buf, len, ascii);
                 tlen -= len;
 
                 kunmap_atomic(it_page);
         }
 }
 #else
-void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
+void caam_dump_sg(const char *prefix_str, int prefix_type,
                   int rowsize, int groupsize, struct scatterlist *sg,
                   size_t tlen, bool ascii)
 {}
diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h
index 8c6b83e02a70..d9726e66edbf 100644
--- a/drivers/crypto/caam/error.h
+++ b/drivers/crypto/caam/error.h
@@ -17,7 +17,7 @@ void caam_strstatus(struct device *dev, u32 status, bool qi_v2);
 #define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false)
 #define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true)
 
-void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type,
+void caam_dump_sg(const char *prefix_str, int prefix_type,
                   int rowsize, int groupsize, struct scatterlist *sg,
                   size_t tlen, bool ascii);
 
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index 3392615dc91b..6af84bbc612c 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -4,7 +4,7 @@
  * Private/internal definitions between modules
  *
  * Copyright 2008-2011 Freescale Semiconductor, Inc.
- *
+ * Copyright 2019 NXP
  */
 
 #ifndef INTERN_H
@@ -63,10 +63,6 @@ struct caam_drv_private_jr {
  * Driver-private storage for a single CAAM block instance
  */
 struct caam_drv_private {
-#ifdef CONFIG_CAAM_QI
-        struct device *qidev;
-#endif
-
         /* Physical-presence section */
         struct caam_ctrl __iomem *ctrl; /* controller region */
         struct caam_deco __iomem *deco; /* DECO/CCB views */
@@ -74,12 +70,17 @@ struct caam_drv_private {
         struct caam_queue_if __iomem *qi; /* QI control region */
         struct caam_job_ring __iomem *jr[4];    /* JobR's register space */
 
+        struct iommu_domain *domain;
+
         /*
          * Detected geometry block. Filled in from device tree if powerpc,
          * or from register-based version detection code
          */
         u8 total_jobrs;         /* Total Job Rings in device */
         u8 qi_present;          /* Nonzero if QI present in device */
+#ifdef CONFIG_CAAM_QI
+        u8 qi_init;             /* Nonzero if QI has been initialized */
+#endif
         u8 mc_en;               /* Nonzero if MC f/w is active */
         int secvio_irq;         /* Security violation interrupt number */
         int virt_en;            /* Virtualization enabled in CAAM */
@@ -107,8 +108,95 @@ struct caam_drv_private {
 #endif
 };
 
-void caam_jr_algapi_init(struct device *dev);
-void caam_jr_algapi_remove(struct device *dev);
+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API
+
+int caam_algapi_init(struct device *dev);
+void caam_algapi_exit(void);
+
+#else
+
+static inline int caam_algapi_init(struct device *dev)
+{
+        return 0;
+}
+
+static inline void caam_algapi_exit(void)
+{
+}
+
+#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API */
+
+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API
+
+int caam_algapi_hash_init(struct device *dev);
+void caam_algapi_hash_exit(void);
+
+#else
+
+static inline int caam_algapi_hash_init(struct device *dev)
+{
+        return 0;
+}
+
+static inline void caam_algapi_hash_exit(void)
+{
+}
+
+#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API */
+
+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API
+
+int caam_pkc_init(struct device *dev);
+void caam_pkc_exit(void);
+
+#else
+
+static inline int caam_pkc_init(struct device *dev)
+{
+        return 0;
+}
+
+static inline void caam_pkc_exit(void)
+{
+}
+
+#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API */
+
+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API
+
+int caam_rng_init(struct device *dev);
+void caam_rng_exit(void);
+
+#else
+
+static inline int caam_rng_init(struct device *dev)
+{
+        return 0;
+}
+
+static inline void caam_rng_exit(void)
+{
+}
+
+#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API */
+
+#ifdef CONFIG_CAAM_QI
+
+int caam_qi_algapi_init(struct device *dev);
+void caam_qi_algapi_exit(void);
+
+#else
+
+static inline int caam_qi_algapi_init(struct device *dev)
+{
+        return 0;
+}
+
+static inline void caam_qi_algapi_exit(void)
+{
+}
+
+#endif /* CONFIG_CAAM_QI */
 
 #ifdef CONFIG_DEBUG_FS
 static int caam_debugfs_u64_get(void *data, u64 *val)
diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c
index 1de2562d0982..cea811fed320 100644
--- a/drivers/crypto/caam/jr.c
+++ b/drivers/crypto/caam/jr.c
@@ -4,6 +4,7 @@
  * JobR backend functionality
  *
  * Copyright 2008-2012 Freescale Semiconductor, Inc.
+ * Copyright 2019 NXP
  */
 
 #include <linux/of_irq.h>
@@ -23,6 +24,43 @@ struct jr_driver_data {
 } ____cacheline_aligned;
 
 static struct jr_driver_data driver_data;
+static DEFINE_MUTEX(algs_lock);
+static unsigned int active_devs;
+
+static void register_algs(struct device *dev)
+{
+        mutex_lock(&algs_lock);
+
+        if (++active_devs != 1)
+                goto algs_unlock;
+
+        caam_algapi_init(dev);
+        caam_algapi_hash_init(dev);
+        caam_pkc_init(dev);
+        caam_rng_init(dev);
+        caam_qi_algapi_init(dev);
+
+algs_unlock:
+        mutex_unlock(&algs_lock);
+}
+
+static void unregister_algs(void)
+{
+        mutex_lock(&algs_lock);
+
+        if (--active_devs != 0)
+                goto algs_unlock;
+
+        caam_qi_algapi_exit();
+
+        caam_rng_exit();
+        caam_pkc_exit();
+        caam_algapi_hash_exit();
+        caam_algapi_exit();
+
+algs_unlock:
+        mutex_unlock(&algs_lock);
+}
 
 static int caam_reset_hw_jr(struct device *dev)
 {
@@ -109,6 +147,9 @@ static int caam_jr_remove(struct platform_device *pdev)
                 return -EBUSY;
         }
 
+        /* Unregister JR-based RNG & crypto algorithms */
+        unregister_algs();
+
         /* Remove the node from Physical JobR list maintained by driver */
         spin_lock(&driver_data.jr_alloc_lock);
         list_del(&jrpriv->list_node);
@@ -541,6 +582,8 @@ static int caam_jr_probe(struct platform_device *pdev)
 
         atomic_set(&jrpriv->tfm_count, 0);
 
+        register_algs(jrdev->parent);
+
         return 0;
 }
 
diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c
index 8d0713fae6ac..48dd3536060d 100644
--- a/drivers/crypto/caam/key_gen.c
+++ b/drivers/crypto/caam/key_gen.c
@@ -16,9 +16,7 @@ void split_key_done(struct device *dev, u32 *desc, u32 err,
 {
         struct split_key_result *res = context;
 
-#ifdef DEBUG
-        dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
-#endif
+        dev_dbg(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 
         if (err)
                 caam_jr_strstatus(dev, err);
@@ -55,12 +53,10 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
         adata->keylen_pad = split_key_pad_len(adata->algtype &
                                               OP_ALG_ALGSEL_MASK);
 
-#ifdef DEBUG
-        dev_err(jrdev, "split keylen %d split keylen padded %d\n",
+        dev_dbg(jrdev, "split keylen %d split keylen padded %d\n",
                 adata->keylen, adata->keylen_pad);
-        print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
-#endif
+        print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
 
         if (adata->keylen_pad > max_keylen)
                 return -EINVAL;
@@ -102,10 +98,9 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
         append_fifo_store(desc, dma_addr, adata->keylen,
                           LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
 
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
-#endif
+        print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ",
+                             DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
+                             1);
 
         result.err = 0;
         init_completion(&result.completion);
@@ -115,11 +110,10 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
                 /* in progress */
                 wait_for_completion(&result.completion);
                 ret = result.err;
-#ifdef DEBUG
-                print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
-                               DUMP_PREFIX_ADDRESS, 16, 4, key_out,
-                               adata->keylen_pad, 1);
-#endif
+
+                print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
+                                     DUMP_PREFIX_ADDRESS, 16, 4, key_out,
+                                     adata->keylen_pad, 1);
         }
 
         dma_unmap_single(jrdev, dma_addr, adata->keylen_pad, DMA_BIDIRECTIONAL);
diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c
index 9f08f84cca59..0fe618e3804a 100644
--- a/drivers/crypto/caam/qi.c
+++ b/drivers/crypto/caam/qi.c
@@ -4,7 +4,7 @@
  * Queue Interface backend functionality
  *
  * Copyright 2013-2016 Freescale Semiconductor, Inc.
- * Copyright 2016-2017 NXP
+ * Copyright 2016-2017, 2019 NXP
  */
 
 #include <linux/cpumask.h>
@@ -18,6 +18,7 @@
 #include "desc_constr.h"
 
 #define PREHDR_RSLS_SHIFT       31
+#define PREHDR_ABS              BIT(25)
 
 /*
  * Use a reasonable backlog of frames (per CPU) as congestion threshold,
@@ -58,11 +59,9 @@ static DEFINE_PER_CPU(int, last_cpu);
 /*
  * caam_qi_priv - CAAM QI backend private params
  * @cgr: QMan congestion group
- * @qi_pdev: platform device for QI backend
  */
 struct caam_qi_priv {
         struct qman_cgr cgr;
-        struct platform_device *qi_pdev;
 };
 
 static struct caam_qi_priv qipriv ____cacheline_aligned;
@@ -95,6 +94,16 @@ static u64 times_congested;
  */
 static struct kmem_cache *qi_cache;
 
+static void *caam_iova_to_virt(struct iommu_domain *domain,
+                               dma_addr_t iova_addr)
+{
+        phys_addr_t phys_addr;
+
+        phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;
+
+        return phys_to_virt(phys_addr);
+}
+
 int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req)
 {
         struct qm_fd fd;
@@ -135,6 +144,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
         const struct qm_fd *fd;
         struct caam_drv_req *drv_req;
         struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
+        struct caam_drv_private *priv = dev_get_drvdata(qidev);
 
         fd = &msg->ern.fd;
 
@@ -143,7 +153,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq,
                 return;
         }
 
-        drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
+        drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
         if (!drv_req) {
                 dev_err(qidev,
                         "Can't find original request for CAAM response\n");
@@ -346,6 +356,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc)
          */
         drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
                                            num_words);
+        drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
         memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
         dma_sync_single_for_device(qidev, drv_ctx->context_a,
                                    sizeof(drv_ctx->sh_desc) +
@@ -401,6 +412,7 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev,
          */
         drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) |
                                            num_words);
+        drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS);
         memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc));
         size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc);
         hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size,
@@ -488,7 +500,7 @@ EXPORT_SYMBOL(caam_drv_ctx_rel);
 void caam_qi_shutdown(struct device *qidev)
 {
         int i;
-        struct caam_qi_priv *priv = dev_get_drvdata(qidev);
+        struct caam_qi_priv *priv = &qipriv;
         const cpumask_t *cpus = qman_affine_cpus();
 
         for_each_cpu(i, cpus) {
@@ -506,8 +518,6 @@ void caam_qi_shutdown(struct device *qidev)
         qman_release_cgrid(priv->cgr.cgrid);
 
         kmem_cache_destroy(qi_cache);
-
-        platform_device_unregister(priv->qi_pdev);
 }
 
 static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested)
@@ -550,6 +560,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
         struct caam_drv_req *drv_req;
         const struct qm_fd *fd;
         struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev);
+        struct caam_drv_private *priv = dev_get_drvdata(qidev);
         u32 status;
 
         if (caam_qi_napi_schedule(p, caam_napi))
@@ -572,7 +583,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p,
                 return qman_cb_dqrr_consume;
         }
 
-        drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd));
+        drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd));
         if (unlikely(!drv_req)) {
                 dev_err(qidev,
                         "Can't find original request for caam response\n");
@@ -692,33 +703,17 @@ static void free_rsp_fqs(void)
 int caam_qi_init(struct platform_device *caam_pdev)
 {
         int err, i;
-        struct platform_device *qi_pdev;
         struct device *ctrldev = &caam_pdev->dev, *qidev;
         struct caam_drv_private *ctrlpriv;
         const cpumask_t *cpus = qman_affine_cpus();
-        static struct platform_device_info qi_pdev_info = {
-                .name = "caam_qi",
-                .id = PLATFORM_DEVID_NONE
-        };
-
-        qi_pdev_info.parent = ctrldev;
-        qi_pdev_info.dma_mask = dma_get_mask(ctrldev);
-        qi_pdev = platform_device_register_full(&qi_pdev_info);
-        if (IS_ERR(qi_pdev))
-                return PTR_ERR(qi_pdev);
-        set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev));
 
         ctrlpriv = dev_get_drvdata(ctrldev);
-        qidev = &qi_pdev->dev;
-
-        qipriv.qi_pdev = qi_pdev;
-        dev_set_drvdata(qidev, &qipriv);
+        qidev = ctrldev;
 
         /* Initialize the congestion detection */
         err = init_cgr(qidev);
         if (err) {
                 dev_err(qidev, "CGR initialization failed: %d\n", err);
-                platform_device_unregister(qi_pdev);
                 return err;
         }
 
@@ -727,7 +722,6 @@ int caam_qi_init(struct platform_device *caam_pdev)
         if (err) {
                 dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err);
                 free_rsp_fqs();
-                platform_device_unregister(qi_pdev);
                 return err;
         }
 
@@ -750,15 +744,11 @@ int caam_qi_init(struct platform_device *caam_pdev)
                 napi_enable(irqtask);
         }
 
-        /* Hook up QI device to parent controlling caam device */
-        ctrlpriv->qidev = qidev;
-
         qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0,
                                      SLAB_CACHE_DMA, NULL);
         if (!qi_cache) {
                 dev_err(qidev, "Can't allocate CAAM cache\n");
                 free_rsp_fqs();
-                platform_device_unregister(qi_pdev);
                 return -ENOMEM;
         }
 
@@ -766,6 +756,8 @@ int caam_qi_init(struct platform_device *caam_pdev)
         debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl,
                             &times_congested, &caam_fops_u64_ro);
 #endif
+
+        ctrlpriv->qi_init = 1;
         dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n");
         return 0;
 }
diff --git a/drivers/crypto/caam/sg_sw_qm.h b/drivers/crypto/caam/sg_sw_qm.h
index b3e1aaaeffea..d56cc7efbc13 100644
--- a/drivers/crypto/caam/sg_sw_qm.h
+++ b/drivers/crypto/caam/sg_sw_qm.h
@@ -54,15 +54,19 @@ static inline void dma_to_qm_sg_one_last_ext(struct qm_sg_entry *qm_sg_ptr,
  * but does not have final bit; instead, returns last entry
  */
 static inline struct qm_sg_entry *
-sg_to_qm_sg(struct scatterlist *sg, int sg_count,
+sg_to_qm_sg(struct scatterlist *sg, int len,
             struct qm_sg_entry *qm_sg_ptr, u16 offset)
 {
-        while (sg_count && sg) {
-                dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg),
-                                 sg_dma_len(sg), offset);
+        int ent_len;
+
+        while (len) {
+                ent_len = min_t(int, sg_dma_len(sg), len);
+
+                dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len,
+                                 offset);
                 qm_sg_ptr++;
                 sg = sg_next(sg);
-                sg_count--;
+                len -= ent_len;
         }
         return qm_sg_ptr - 1;
 }
@@ -71,10 +75,10 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count,
  * convert scatterlist to h/w link table format
  * scatterlist must have been previously dma mapped
  */
-static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count,
+static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len,
                                     struct qm_sg_entry *qm_sg_ptr, u16 offset)
 {
-        qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset);
+        qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset);
         qm_sg_entry_set_f(qm_sg_ptr, qm_sg_entry_get_len(qm_sg_ptr));
 }
 
diff --git a/drivers/crypto/caam/sg_sw_qm2.h b/drivers/crypto/caam/sg_sw_qm2.h
index c9378402a5f8..b8b737d2b0ea 100644
--- a/drivers/crypto/caam/sg_sw_qm2.h
+++ b/drivers/crypto/caam/sg_sw_qm2.h
@@ -25,15 +25,19 @@ static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr,
  * but does not have final bit; instead, returns last entry
  */
 static inline struct dpaa2_sg_entry *
-sg_to_qm_sg(struct scatterlist *sg, int sg_count,
+sg_to_qm_sg(struct scatterlist *sg, int len,
             struct dpaa2_sg_entry *qm_sg_ptr, u16 offset)
 {
-        while (sg_count && sg) {
-                dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg),
-                                 sg_dma_len(sg), offset);
+        int ent_len;
+
+        while (len) {
+                ent_len = min_t(int, sg_dma_len(sg), len);
+
+                dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len,
+                                 offset);
                 qm_sg_ptr++;
                 sg = sg_next(sg);
-                sg_count--;
+                len -= ent_len;
         }
         return qm_sg_ptr - 1;
 }
@@ -42,11 +46,11 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count,
  * convert scatterlist to h/w link table format
  * scatterlist must have been previously dma mapped
  */
-static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count,
+static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len,
                                     struct dpaa2_sg_entry *qm_sg_ptr,
                                     u16 offset)
 {
-        qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset);
+        qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset);
         dpaa2_sg_set_final(qm_sg_ptr, true);
 }
 
diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h
index dbfa9fce33e0..07e1ee99273b 100644
--- a/drivers/crypto/caam/sg_sw_sec4.h
+++ b/drivers/crypto/caam/sg_sw_sec4.h
@@ -35,11 +35,9 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr,
                 sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset &
                                                          SEC4_SG_OFFSET_MASK);
         }
-#ifdef DEBUG
-        print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ",
-                       DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr,
-                       sizeof(struct sec4_sg_entry), 1);
-#endif
+
+        print_hex_dump_debug("sec4_sg_ptr@: ", DUMP_PREFIX_ADDRESS, 16, 4,
+                             sec4_sg_ptr, sizeof(struct sec4_sg_entry), 1);
 }
 
 /*
@@ -47,15 +45,19 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr,
  * but does not have final bit; instead, returns last entry
  */
 static inline struct sec4_sg_entry *
-sg_to_sec4_sg(struct scatterlist *sg, int sg_count,
+sg_to_sec4_sg(struct scatterlist *sg, int len,
               struct sec4_sg_entry *sec4_sg_ptr, u16 offset)
 {
-        while (sg_count) {
-                dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg),
-                                   sg_dma_len(sg), offset);
+        int ent_len;
+
+        while (len) {
+                ent_len = min_t(int, sg_dma_len(sg), len);
+
+                dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), ent_len,
+                                   offset);
                 sec4_sg_ptr++;
                 sg = sg_next(sg);
-                sg_count--;
+                len -= ent_len;
         }
         return sec4_sg_ptr - 1;
 }
@@ -72,11 +74,11 @@ static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr)
  * convert scatterlist to h/w link table format
  * scatterlist must have been previously dma mapped
  */
-static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count,
+static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int len,
                                       struct sec4_sg_entry *sec4_sg_ptr,
                                       u16 offset)
 {
-        sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset);
+        sec4_sg_ptr = sg_to_sec4_sg(sg, len, sec4_sg_ptr, offset);
         sg_to_sec4_set_last(sec4_sg_ptr);
 }
 
diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c
index e9f4704494fb..ff3cb1f8f2b6 100644
--- a/drivers/crypto/cavium/cpt/cptvf_algs.c
+++ b/drivers/crypto/cavium/cpt/cptvf_algs.c
@@ -7,7 +7,6 @@
 #include <crypto/aes.h>
 #include <crypto/algapi.h>
 #include <crypto/authenc.h>
-#include <crypto/crypto_wq.h>
 #include <crypto/des.h>
 #include <crypto/xts.h>
 #include <linux/crypto.h>
diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.h b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h
index f177b79bbab0..09c4cf2513fb 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_debugfs.h
+++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 #ifndef __NITROX_DEBUGFS_H
 #define __NITROX_DEBUGFS_H
 
diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.h b/drivers/crypto/cavium/nitrox/nitrox_mbx.h
index 5008399775a9..7c93d0282174 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_mbx.h
+++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 #ifndef __NITROX_MBX_H
 #define __NITROX_MBX_H
 
diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c
index ea3d6de55ff6..58c6dddfc5e1 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes.c
@@ -2,7 +2,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) AES crypto API support
  *
- * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
  */
@@ -76,8 +76,7 @@ static int ccp_aes_crypt(struct ablkcipher_request *req, bool encrypt)
                 return -EINVAL;
 
         if (((ctx->u.aes.mode == CCP_AES_MODE_ECB) ||
-             (ctx->u.aes.mode == CCP_AES_MODE_CBC) ||
-             (ctx->u.aes.mode == CCP_AES_MODE_CFB)) &&
+             (ctx->u.aes.mode == CCP_AES_MODE_CBC)) &&
             (req->nbytes & (AES_BLOCK_SIZE - 1)))
                 return -EINVAL;
 
@@ -288,7 +287,7 @@ static struct ccp_aes_def aes_algs[] = {
                 .version        = CCP_VERSION(3, 0),
                 .name           = "cfb(aes)",
                 .driver_name    = "cfb-aes-ccp",
-                .blocksize      = AES_BLOCK_SIZE,
+                .blocksize      = 1,
                 .ivsize         = AES_BLOCK_SIZE,
                 .alg_defaults   = &ccp_aes_defaults,
         },
diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c
index cc3e96c4f5fb..f79eede71c62 100644
--- a/drivers/crypto/ccp/ccp-dev.c
+++ b/drivers/crypto/ccp/ccp-dev.c
@@ -32,56 +32,62 @@ struct ccp_tasklet_data {
 };
 
 /* Human-readable error strings */
+#define CCP_MAX_ERROR_CODE      64
 static char *ccp_error_codes[] = {
         "",
-        "ERR 01: ILLEGAL_ENGINE",
-        "ERR 02: ILLEGAL_KEY_ID",
-        "ERR 03: ILLEGAL_FUNCTION_TYPE",
-        "ERR 04: ILLEGAL_FUNCTION_MODE",
-        "ERR 05: ILLEGAL_FUNCTION_ENCRYPT",
-        "ERR 06: ILLEGAL_FUNCTION_SIZE",
-        "ERR 07: Zlib_MISSING_INIT_EOM",
-        "ERR 08: ILLEGAL_FUNCTION_RSVD",
-        "ERR 09: ILLEGAL_BUFFER_LENGTH",
-        "ERR 10: VLSB_FAULT",
-        "ERR 11: ILLEGAL_MEM_ADDR",
-        "ERR 12: ILLEGAL_MEM_SEL",
-        "ERR 13: ILLEGAL_CONTEXT_ID",
-        "ERR 14: ILLEGAL_KEY_ADDR",
-        "ERR 15: 0xF Reserved",
-        "ERR 16: Zlib_ILLEGAL_MULTI_QUEUE",
-        "ERR 17: Zlib_ILLEGAL_JOBID_CHANGE",
-        "ERR 18: CMD_TIMEOUT",
-        "ERR 19: IDMA0_AXI_SLVERR",
-        "ERR 20: IDMA0_AXI_DECERR",
-        "ERR 21: 0x15 Reserved",
-        "ERR 22: IDMA1_AXI_SLAVE_FAULT",
-        "ERR 23: IDMA1_AIXI_DECERR",
-        "ERR 24: 0x18 Reserved",
-        "ERR 25: ZLIBVHB_AXI_SLVERR",
-        "ERR 26: ZLIBVHB_AXI_DECERR",
-        "ERR 27: 0x1B Reserved",
-        "ERR 27: ZLIB_UNEXPECTED_EOM",
-        "ERR 27: ZLIB_EXTRA_DATA",
-        "ERR 30: ZLIB_BTYPE",
-        "ERR 31: ZLIB_UNDEFINED_SYMBOL",
-        "ERR 32: ZLIB_UNDEFINED_DISTANCE_S",
-        "ERR 33: ZLIB_CODE_LENGTH_SYMBOL",
-        "ERR 34: ZLIB _VHB_ILLEGAL_FETCH",
-        "ERR 35: ZLIB_UNCOMPRESSED_LEN",
-        "ERR 36: ZLIB_LIMIT_REACHED",
-        "ERR 37: ZLIB_CHECKSUM_MISMATCH0",
-        "ERR 38: ODMA0_AXI_SLVERR",
-        "ERR 39: ODMA0_AXI_DECERR",
-        "ERR 40: 0x28 Reserved",
-        "ERR 41: ODMA1_AXI_SLVERR",
-        "ERR 42: ODMA1_AXI_DECERR",
-        "ERR 43: LSB_PARITY_ERR",
+        "ILLEGAL_ENGINE",
+        "ILLEGAL_KEY_ID",
+        "ILLEGAL_FUNCTION_TYPE",
+        "ILLEGAL_FUNCTION_MODE",
+        "ILLEGAL_FUNCTION_ENCRYPT",
+        "ILLEGAL_FUNCTION_SIZE",
+        "Zlib_MISSING_INIT_EOM",
+        "ILLEGAL_FUNCTION_RSVD",
+        "ILLEGAL_BUFFER_LENGTH",
+        "VLSB_FAULT",
+        "ILLEGAL_MEM_ADDR",
+        "ILLEGAL_MEM_SEL",
+        "ILLEGAL_CONTEXT_ID",
+        "ILLEGAL_KEY_ADDR",
+        "0xF Reserved",
+        "Zlib_ILLEGAL_MULTI_QUEUE",
+        "Zlib_ILLEGAL_JOBID_CHANGE",
+        "CMD_TIMEOUT",
+        "IDMA0_AXI_SLVERR",
+        "IDMA0_AXI_DECERR",
+        "0x15 Reserved",
+        "IDMA1_AXI_SLAVE_FAULT",
+        "IDMA1_AIXI_DECERR",
+        "0x18 Reserved",
+        "ZLIBVHB_AXI_SLVERR",
+        "ZLIBVHB_AXI_DECERR",
+        "0x1B Reserved",
+        "ZLIB_UNEXPECTED_EOM",
+        "ZLIB_EXTRA_DATA",
+        "ZLIB_BTYPE",
+        "ZLIB_UNDEFINED_SYMBOL",
+        "ZLIB_UNDEFINED_DISTANCE_S",
+        "ZLIB_CODE_LENGTH_SYMBOL",
+        "ZLIB _VHB_ILLEGAL_FETCH",
+        "ZLIB_UNCOMPRESSED_LEN",
+        "ZLIB_LIMIT_REACHED",
+        "ZLIB_CHECKSUM_MISMATCH0",
+        "ODMA0_AXI_SLVERR",
+        "ODMA0_AXI_DECERR",
+        "0x28 Reserved",
+        "ODMA1_AXI_SLVERR",
+        "ODMA1_AXI_DECERR",
 };
 
-void ccp_log_error(struct ccp_device *d, int e)
+void ccp_log_error(struct ccp_device *d, unsigned int e)
 {
-        dev_err(d->dev, "CCP error: %s (0x%x)\n", ccp_error_codes[e], e);
+        if (WARN_ON(e >= CCP_MAX_ERROR_CODE))
+                return;
+
+        if (e < ARRAY_SIZE(ccp_error_codes))
+                dev_err(d->dev, "CCP error %d: %s\n", e, ccp_error_codes[e]);
+        else
+                dev_err(d->dev, "CCP error %d: Unknown Error\n", e);
 }
 
 /* List of CCPs, CCP count, read-write access lock, and access functions
diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h
index 90523a069bff..5e624920fd99 100644
--- a/drivers/crypto/ccp/ccp-dev.h
+++ b/drivers/crypto/ccp/ccp-dev.h
@@ -629,7 +629,7 @@ struct ccp5_desc {
 void ccp_add_device(struct ccp_device *ccp);
 void ccp_del_device(struct ccp_device *ccp);
 
-extern void ccp_log_error(struct ccp_device *, int);
+extern void ccp_log_error(struct ccp_device *, unsigned int);
 
 struct ccp_device *ccp_alloc_struct(struct sp_device *sp);
 bool ccp_queues_suspended(struct ccp_device *ccp);
diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c
index db8de89d990f..866b2e05ca77 100644
--- a/drivers/crypto/ccp/ccp-ops.c
+++ b/drivers/crypto/ccp/ccp-ops.c
@@ -2,7 +2,7 @@
 /*
  * AMD Cryptographic Coprocessor (CCP) driver
  *
- * Copyright (C) 2013,2018 Advanced Micro Devices, Inc.
+ * Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
  *
  * Author: Tom Lendacky <thomas.lendacky@amd.com>
  * Author: Gary R Hook <gary.hook@amd.com>
@@ -890,8 +890,7 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
                 return -EINVAL;
 
         if (((aes->mode == CCP_AES_MODE_ECB) ||
-             (aes->mode == CCP_AES_MODE_CBC) ||
-             (aes->mode == CCP_AES_MODE_CFB)) &&
+             (aes->mode == CCP_AES_MODE_CBC)) &&
             (aes->src_len & (AES_BLOCK_SIZE - 1)))
                 return -EINVAL;
 
@@ -1264,6 +1263,9 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
         int ret;
 
         /* Error checks */
+        if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
+                return -EINVAL;
+
         if (!cmd_q->ccp->vdata->perform->des3)
                 return -EINVAL;
 
@@ -1346,8 +1348,6 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
          * passthru option to convert from big endian to little endian.
          */
         if (des3->mode != CCP_DES3_MODE_ECB) {
-                u32 load_mode;
-
                 op.sb_ctx = cmd_q->sb_ctx;
 
                 ret = ccp_init_dm_workarea(&ctx, cmd_q,
@@ -1363,12 +1363,8 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
                 if (ret)
                         goto e_ctx;
 
-                if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
-                        load_mode = CCP_PASSTHRU_BYTESWAP_NOOP;
-                else
-                        load_mode = CCP_PASSTHRU_BYTESWAP_256BIT;
                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
-                                     load_mode);
+                                     CCP_PASSTHRU_BYTESWAP_256BIT);
                 if (ret) {
                         cmd->engine_error = cmd_q->cmd_error;
                         goto e_ctx;
@@ -1430,10 +1426,6 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
                 }
 
                 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
-                if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
-                        dm_offset = CCP_SB_BYTES - des3->iv_len;
-                else
-                        dm_offset = 0;
                 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
                                 DES3_EDE_BLOCK_SIZE);
         }
diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c
index 86ac7b443355..980aa04b655b 100644
--- a/drivers/crypto/ccree/cc_driver.c
+++ b/drivers/crypto/ccree/cc_driver.c
@@ -48,6 +48,7 @@ struct cc_hw_data {
 };
 
 #define CC_NUM_IDRS 4
+#define CC_HW_RESET_LOOP_COUNT 10
 
 /* Note: PIDR3 holds CMOD/Rev so ignored for HW identification purposes */
 static const u32 pidr_0124_offsets[CC_NUM_IDRS] = {
@@ -133,6 +134,9 @@ static irqreturn_t cc_isr(int irq, void *dev_id)
         u32 imr;
 
         /* STAT_OP_TYPE_GENERIC STAT_PHASE_0: Interrupt */
+        /* if driver suspended return, probebly shared interrupt */
+        if (cc_pm_is_dev_suspended(dev))
+                return IRQ_NONE;
 
         /* read the interrupt status */
         irr = cc_ioread(drvdata, CC_REG(HOST_IRR));
@@ -188,6 +192,31 @@ static irqreturn_t cc_isr(int irq, void *dev_id)
         return IRQ_HANDLED;
 }
 
+bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata)
+{
+        unsigned int val;
+        unsigned int i;
+
+        /* 712/710/63 has no reset completion indication, always return true */
+        if (drvdata->hw_rev <= CC_HW_REV_712)
+                return true;
+
+        for (i = 0; i < CC_HW_RESET_LOOP_COUNT; i++) {
+                /* in cc7x3 NVM_IS_IDLE indicates that CC reset is
+                 *  completed and device is fully functional
+                 */
+                val = cc_ioread(drvdata, CC_REG(NVM_IS_IDLE));
+                if (val & CC_NVM_IS_IDLE_MASK) {
+                        /* hw indicate reset completed */
+                        return true;
+                }
+                /* allow scheduling other process on the processor */
+                schedule();
+        }
+        /* reset not completed */
+        return false;
+}
+
 int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe)
 {
         unsigned int val, cache_params;
@@ -315,15 +344,6 @@ static int init_cc_resources(struct platform_device *plat_dev)
                 return new_drvdata->irq;
         }
 
-        rc = devm_request_irq(dev, new_drvdata->irq, cc_isr,
-                              IRQF_SHARED, "ccree", new_drvdata);
-        if (rc) {
-                dev_err(dev, "Could not register to interrupt %d\n",
-                        new_drvdata->irq);
-                return rc;
-        }
-        dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq);
-
         init_completion(&new_drvdata->hw_queue_avail);
 
         if (!plat_dev->dev.dma_mask)
@@ -352,6 +372,11 @@ static int init_cc_resources(struct platform_device *plat_dev)
 
         new_drvdata->sec_disabled = cc_sec_disable;
 
+        /* wait for Crytpcell reset completion */
+        if (!cc_wait_for_reset_completion(new_drvdata)) {
+                dev_err(dev, "Cryptocell reset not completed");
+        }
+
         if (hw_rev->rev <= CC_HW_REV_712) {
                 /* Verify correct mapping */
                 val = cc_ioread(new_drvdata, new_drvdata->sig_offset);
@@ -383,6 +408,24 @@ static int init_cc_resources(struct platform_device *plat_dev)
                 }
                 sig_cidr = val;
 
+                /* Check HW engine configuration */
+                val = cc_ioread(new_drvdata, CC_REG(HOST_REMOVE_INPUT_PINS));
+                switch (val) {
+                case CC_PINS_FULL:
+                        /* This is fine */
+                        break;
+                case CC_PINS_SLIM:
+                        if (new_drvdata->std_bodies & CC_STD_NIST) {
+                                dev_warn(dev, "703 mode forced due to HW configuration.\n");
+                                new_drvdata->std_bodies = CC_STD_OSCCA;
+                        }
+                        break;
+                default:
+                        dev_err(dev, "Unsupported engines configration.\n");
+                        rc = -EINVAL;
+                        goto post_clk_err;
+                }
+
                 /* Check security disable state */
                 val = cc_ioread(new_drvdata, CC_REG(SECURITY_DISABLED));
                 val &= CC_SECURITY_DISABLED_MASK;
@@ -401,6 +444,15 @@ static int init_cc_resources(struct platform_device *plat_dev)
         /* Display HW versions */
         dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X/0x%8X, Driver version %s\n",
                  hw_rev->name, hw_rev_pidr, sig_cidr, DRV_MODULE_VERSION);
+        /* register the driver isr function */
+        rc = devm_request_irq(dev, new_drvdata->irq, cc_isr,
+                              IRQF_SHARED, "ccree", new_drvdata);
+        if (rc) {
+                dev_err(dev, "Could not register to interrupt %d\n",
+                        new_drvdata->irq);
+                goto post_clk_err;
+        }
+        dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq);
 
         rc = init_cc_regs(new_drvdata, true);
         if (rc) {
diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h
index b76181335c08..7cd99380bf1f 100644
--- a/drivers/crypto/ccree/cc_driver.h
+++ b/drivers/crypto/ccree/cc_driver.h
@@ -53,6 +53,9 @@ enum cc_std_body {
 
 #define CC_COHERENT_CACHE_PARAMS 0xEEE
 
+#define CC_PINS_FULL    0x0
+#define CC_PINS_SLIM    0x9F
+
 /* Maximum DMA mask supported by IP */
 #define DMA_BIT_MASK_LEN 48
 
@@ -67,6 +70,8 @@ enum cc_std_body {
 
 #define CC_SECURITY_DISABLED_MASK BIT(CC_SECURITY_DISABLED_VALUE_BIT_SHIFT)
 
+#define CC_NVM_IS_IDLE_MASK BIT(CC_NVM_IS_IDLE_VALUE_BIT_SHIFT)
+
 #define AXIM_MON_COMP_VALUE GENMASK(CC_AXIM_MON_COMP_VALUE_BIT_SIZE + \
                                     CC_AXIM_MON_COMP_VALUE_BIT_SHIFT, \
                                     CC_AXIM_MON_COMP_VALUE_BIT_SHIFT)
@@ -216,6 +221,7 @@ static inline void dump_byte_array(const char *name, const u8 *the_array,
                 __dump_byte_array(name, the_array, size);
 }
 
+bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata);
 int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe);
 void fini_cc_regs(struct cc_drvdata *drvdata);
 int cc_clk_on(struct cc_drvdata *drvdata);
diff --git a/drivers/crypto/ccree/cc_host_regs.h b/drivers/crypto/ccree/cc_host_regs.h
index d0764147573f..efe3e1d8b87b 100644
--- a/drivers/crypto/ccree/cc_host_regs.h
+++ b/drivers/crypto/ccree/cc_host_regs.h
@@ -114,6 +114,9 @@
 #define CC_HOST_ICR_DSCRPTR_WATERMARK_QUEUE0_CLEAR_BIT_SIZE     0x1UL
 #define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SHIFT       0x17UL
 #define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SIZE        0x1UL
+#define CC_NVM_IS_IDLE_REG_OFFSET       0x0A10UL
+#define CC_NVM_IS_IDLE_VALUE_BIT_SHIFT  0x0UL
+#define CC_NVM_IS_IDLE_VALUE_BIT_SIZE   0x1UL
 #define CC_SECURITY_DISABLED_REG_OFFSET         0x0A1CUL
 #define CC_SECURITY_DISABLED_VALUE_BIT_SHIFT    0x0UL
 #define CC_SECURITY_DISABLED_VALUE_BIT_SIZE     0x1UL
@@ -203,6 +206,23 @@
 #define CC_HOST_POWER_DOWN_EN_REG_OFFSET        0xA78UL
 #define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SHIFT   0x0UL
 #define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SIZE    0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REG_OFFSET    0x0A7CUL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_ENGINE_BIT_SHIFT   0x0UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_ENGINE_BIT_SIZE    0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_MAC_ENGINE_BIT_SHIFT       0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_MAC_ENGINE_BIT_SIZE        0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_GHASH_ENGINE_BIT_SHIFT 0x2UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_GHASH_ENGINE_BIT_SIZE  0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_DES_ENGINE_BIT_SHIFT   0x3UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_DES_ENGINE_BIT_SIZE    0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_HASH_ENGINE_BIT_SHIFT  0x4UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_HASH_ENGINE_BIT_SIZE   0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM3_ENGINE_BIT_SHIFT   0x5UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM3_ENGINE_BIT_SIZE    0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM4_ENGINE_BIT_SHIFT   0x6UL
+#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM4_ENGINE_BIT_SIZE    0x1UL
+#define CC_HOST_REMOVE_INPUT_PINS_OTP_DISCONNECTED_BIT_SHIFT    0x7UL
+#define CC_HOST_REMOVE_INPUT_PINS_OTP_DISCONNECTED_BIT_SIZE     0x1UL
 // --------------------------------------
 // BLOCK: ID_REGISTERS
 // --------------------------------------
diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c
index 2dad9c9543c6..899a52f05b7a 100644
--- a/drivers/crypto/ccree/cc_pm.c
+++ b/drivers/crypto/ccree/cc_pm.c
@@ -49,6 +49,11 @@ int cc_pm_resume(struct device *dev)
                 dev_err(dev, "failed getting clock back on. We're toast.\n");
                 return rc;
         }
+        /* wait for Crytpcell reset completion */
+        if (!cc_wait_for_reset_completion(drvdata)) {
+                dev_err(dev, "Cryptocell reset not completed");
+                return -EBUSY;
+        }
 
         cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_DISABLE);
         rc = init_cc_regs(drvdata, false);
@@ -101,6 +106,12 @@ int cc_pm_put_suspend(struct device *dev)
         return rc;
 }
 
+bool cc_pm_is_dev_suspended(struct device *dev)
+{
+        /* check device state using runtime api */
+        return pm_runtime_suspended(dev);
+}
+
 int cc_pm_init(struct cc_drvdata *drvdata)
 {
         struct device *dev = drvdata_to_dev(drvdata);
diff --git a/drivers/crypto/ccree/cc_pm.h b/drivers/crypto/ccree/cc_pm.h
index 6190cdba5dad..a7d98a5da2e1 100644
--- a/drivers/crypto/ccree/cc_pm.h
+++ b/drivers/crypto/ccree/cc_pm.h
@@ -22,6 +22,7 @@ int cc_pm_suspend(struct device *dev);
 int cc_pm_resume(struct device *dev);
 int cc_pm_get(struct device *dev);
 int cc_pm_put_suspend(struct device *dev);
+bool cc_pm_is_dev_suspended(struct device *dev);
 
 #else
 
@@ -54,6 +55,12 @@ static inline int cc_pm_put_suspend(struct device *dev)
         return 0;
 }
 
+static inline bool cc_pm_is_dev_suspended(struct device *dev)
+{
+        /* if PM not supported device is never suspend */
+        return false;
+}
+
 #endif
 
 #endif /*__POWER_MGR_H__*/
diff --git a/drivers/crypto/hisilicon/sec/sec_drv.h b/drivers/crypto/hisilicon/sec/sec_drv.h
index 2d2f186674ba..4d9063a8b10b 100644
--- a/drivers/crypto/hisilicon/sec/sec_drv.h
+++ b/drivers/crypto/hisilicon/sec/sec_drv.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /* Copyright (c) 2016-2017 Hisilicon Limited. */
 
 #ifndef _SEC_DRV_H_
diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c
index 86c699c14f84..df43a2c6933b 100644
--- a/drivers/crypto/inside-secure/safexcel.c
+++ b/drivers/crypto/inside-secure/safexcel.c
@@ -398,6 +398,12 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
 
                 /* Processing Engine configuration */
 
+                /* Token & context configuration */
+                val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES |
+                      EIP197_PE_EIP96_TOKEN_CTRL_REUSE_CTX |
+                      EIP197_PE_EIP96_TOKEN_CTRL_POST_REUSE_CTX;
+                writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe));
+
                 /* H/W capabilities selection */
                 val = EIP197_FUNCTION_RSVD;
                 val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY;
@@ -589,9 +595,9 @@ inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
         if (rdesc->result_data.error_code & 0x407f) {
                 /* Fatal error (bits 0-7, 14) */
                 dev_err(priv->dev,
-                        "cipher: result: result descriptor error (%d)\n",
+                        "cipher: result: result descriptor error (0x%x)\n",
                         rdesc->result_data.error_code);
-                return -EIO;
+                return -EINVAL;
         } else if (rdesc->result_data.error_code == BIT(9)) {
                 /* Authentication failed */
                 return -EBADMSG;
@@ -720,11 +726,10 @@ handle_results:
         }
 
 acknowledge:
-        if (i) {
+        if (i)
                 writel(EIP197_xDR_PROC_xD_PKT(i) |
                        EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset),
                        EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
-        }
 
         /* If the number of requests overflowed the counter, try to proceed more
          * requests.
diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h
index 65624a81f0fd..e0c202f33674 100644
--- a/drivers/crypto/inside-secure/safexcel.h
+++ b/drivers/crypto/inside-secure/safexcel.h
@@ -118,6 +118,7 @@
 #define EIP197_PE_ICE_SCRATCH_CTRL(n)           (0x0d04 + (0x2000 * (n)))
 #define EIP197_PE_ICE_FPP_CTRL(n)               (0x0d80 + (0x2000 * (n)))
 #define EIP197_PE_ICE_RAM_CTRL(n)               (0x0ff0 + (0x2000 * (n)))
+#define EIP197_PE_EIP96_TOKEN_CTRL(n)           (0x1000 + (0x2000 * (n)))
 #define EIP197_PE_EIP96_FUNCTION_EN(n)          (0x1004 + (0x2000 * (n)))
 #define EIP197_PE_EIP96_CONTEXT_CTRL(n)         (0x1008 + (0x2000 * (n)))
 #define EIP197_PE_EIP96_CONTEXT_STAT(n)         (0x100c + (0x2000 * (n)))
@@ -249,6 +250,11 @@
 #define EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN      BIT(0)
 #define EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN      BIT(1)
 
+/* EIP197_PE_EIP96_TOKEN_CTRL */
+#define EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES          BIT(16)
+#define EIP197_PE_EIP96_TOKEN_CTRL_REUSE_CTX            BIT(19)
+#define EIP197_PE_EIP96_TOKEN_CTRL_POST_REUSE_CTX       BIT(20)
+
 /* EIP197_PE_EIP96_FUNCTION_EN */
 #define EIP197_FUNCTION_RSVD                    (BIT(6) | BIT(15) | BIT(20) | BIT(23))
 #define EIP197_PROTOCOL_HASH_ONLY               BIT(0)
@@ -333,6 +339,7 @@ struct safexcel_context_record {
 #define CONTEXT_CONTROL_IV3                     BIT(8)
 #define CONTEXT_CONTROL_DIGEST_CNT              BIT(9)
 #define CONTEXT_CONTROL_COUNTER_MODE            BIT(10)
+#define CONTEXT_CONTROL_CRYPTO_STORE            BIT(12)
 #define CONTEXT_CONTROL_HASH_STORE              BIT(19)
 
 /* The hash counter given to the engine in the context has a granularity of
@@ -425,6 +432,10 @@ struct safexcel_token {
 
 #define EIP197_TOKEN_HASH_RESULT_VERIFY         BIT(16)
 
+#define EIP197_TOKEN_CTX_OFFSET(x)              (x)
+#define EIP197_TOKEN_DIRECTION_EXTERNAL         BIT(11)
+#define EIP197_TOKEN_EXEC_IF_SUCCESSFUL         (0x1 << 12)
+
 #define EIP197_TOKEN_STAT_LAST_HASH             BIT(0)
 #define EIP197_TOKEN_STAT_LAST_PACKET           BIT(1)
 #define EIP197_TOKEN_OPCODE_DIRECTION           0x0
@@ -432,6 +443,7 @@ struct safexcel_token {
 #define EIP197_TOKEN_OPCODE_NOOP                EIP197_TOKEN_OPCODE_INSERT
 #define EIP197_TOKEN_OPCODE_RETRIEVE            0x4
 #define EIP197_TOKEN_OPCODE_VERIFY              0xd
+#define EIP197_TOKEN_OPCODE_CTX_ACCESS          0xe
 #define EIP197_TOKEN_OPCODE_BYPASS              GENMASK(3, 0)
 
 static inline void eip197_noop_token(struct safexcel_token *token)
@@ -442,6 +454,8 @@ static inline void eip197_noop_token(struct safexcel_token *token)
 
 /* Instructions */
 #define EIP197_TOKEN_INS_INSERT_HASH_DIGEST     0x1c
+#define EIP197_TOKEN_INS_ORIGIN_IV0             0x14
+#define EIP197_TOKEN_INS_ORIGIN_LEN(x)          ((x) << 5)
 #define EIP197_TOKEN_INS_TYPE_OUTPUT            BIT(5)
 #define EIP197_TOKEN_INS_TYPE_HASH              BIT(6)
 #define EIP197_TOKEN_INS_TYPE_CRYTO             BIT(7)
@@ -468,6 +482,7 @@ struct safexcel_control_data_desc {
 
 #define EIP197_OPTION_MAGIC_VALUE       BIT(0)
 #define EIP197_OPTION_64BIT_CTX         BIT(1)
+#define EIP197_OPTION_RC_AUTO           (0x2 << 3)
 #define EIP197_OPTION_CTX_CTRL_IN_CMD   BIT(8)
 #define EIP197_OPTION_2_TOKEN_IV_CMD    GENMASK(11, 10)
 #define EIP197_OPTION_4_TOKEN_IV_CMD    GENMASK(11, 9)
@@ -629,7 +644,7 @@ struct safexcel_ahash_export_state {
         u32 digest;
 
         u32 state[SHA512_DIGEST_SIZE / sizeof(u32)];
-        u8 cache[SHA512_BLOCK_SIZE];
+        u8 cache[SHA512_BLOCK_SIZE << 1];
 };
 
 /*
diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c
index de4be10b172f..8cdbdbe35681 100644
--- a/drivers/crypto/inside-secure/safexcel_cipher.c
+++ b/drivers/crypto/inside-secure/safexcel_cipher.c
@@ -51,6 +51,8 @@ struct safexcel_cipher_ctx {
 
 struct safexcel_cipher_req {
         enum safexcel_cipher_direction direction;
+        /* Number of result descriptors associated to the request */
+        unsigned int rdescs;
         bool needs_inv;
 };
 
@@ -59,27 +61,26 @@ static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
                                     u32 length)
 {
         struct safexcel_token *token;
-        unsigned offset = 0;
+        u32 offset = 0, block_sz = 0;
 
         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
                 switch (ctx->alg) {
                 case SAFEXCEL_DES:
-                        offset = DES_BLOCK_SIZE / sizeof(u32);
-                        memcpy(cdesc->control_data.token, iv, DES_BLOCK_SIZE);
+                        block_sz = DES_BLOCK_SIZE;
                         cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
                         break;
                 case SAFEXCEL_3DES:
-                        offset = DES3_EDE_BLOCK_SIZE / sizeof(u32);
-                        memcpy(cdesc->control_data.token, iv, DES3_EDE_BLOCK_SIZE);
+                        block_sz = DES3_EDE_BLOCK_SIZE;
                         cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
                         break;
-
                 case SAFEXCEL_AES:
-                        offset = AES_BLOCK_SIZE / sizeof(u32);
-                        memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
+                        block_sz = AES_BLOCK_SIZE;
                         cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
                         break;
                 }
+
+                offset = block_sz / sizeof(u32);
+                memcpy(cdesc->control_data.token, iv, block_sz);
         }
 
         token = (struct safexcel_token *)(cdesc->control_data.token + offset);
@@ -91,6 +92,25 @@ static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
         token[0].instructions = EIP197_TOKEN_INS_LAST |
                                 EIP197_TOKEN_INS_TYPE_CRYTO |
                                 EIP197_TOKEN_INS_TYPE_OUTPUT;
+
+        if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
+                u32 last = (EIP197_MAX_TOKENS - 1) - offset;
+
+                token[last].opcode = EIP197_TOKEN_OPCODE_CTX_ACCESS;
+                token[last].packet_length = EIP197_TOKEN_DIRECTION_EXTERNAL |
+                                            EIP197_TOKEN_EXEC_IF_SUCCESSFUL|
+                                            EIP197_TOKEN_CTX_OFFSET(0x2);
+                token[last].stat = EIP197_TOKEN_STAT_LAST_HASH |
+                        EIP197_TOKEN_STAT_LAST_PACKET;
+                token[last].instructions =
+                        EIP197_TOKEN_INS_ORIGIN_LEN(block_sz / sizeof(u32)) |
+                        EIP197_TOKEN_INS_ORIGIN_IV0;
+
+                /* Store the updated IV values back in the internal context
+                 * registers.
+                 */
+                cdesc->control_data.control1 |= CONTEXT_CONTROL_CRYPTO_STORE;
+        }
 }
 
 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
@@ -333,7 +353,10 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
 
         *ret = 0;
 
-        do {
+        if (unlikely(!sreq->rdescs))
+                return 0;
+
+        while (sreq->rdescs--) {
                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
                 if (IS_ERR(rdesc)) {
                         dev_err(priv->dev,
@@ -346,21 +369,15 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 
                 ndesc++;
-        } while (!rdesc->last_seg);
+        }
 
         safexcel_complete(priv, ring);
 
         if (src == dst) {
-                dma_unmap_sg(priv->dev, src,
-                             sg_nents_for_len(src, cryptlen),
-                             DMA_BIDIRECTIONAL);
+                dma_unmap_sg(priv->dev, src, sg_nents(src), DMA_BIDIRECTIONAL);
         } else {
-                dma_unmap_sg(priv->dev, src,
-                             sg_nents_for_len(src, cryptlen),
-                             DMA_TO_DEVICE);
-                dma_unmap_sg(priv->dev, dst,
-                             sg_nents_for_len(dst, cryptlen),
-                             DMA_FROM_DEVICE);
+                dma_unmap_sg(priv->dev, src, sg_nents(src), DMA_TO_DEVICE);
+                dma_unmap_sg(priv->dev, dst, sg_nents(dst), DMA_FROM_DEVICE);
         }
 
         *should_complete = true;
@@ -385,26 +402,21 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring,
         int i, ret = 0;
 
         if (src == dst) {
-                nr_src = dma_map_sg(priv->dev, src,
-                                    sg_nents_for_len(src, totlen),
+                nr_src = dma_map_sg(priv->dev, src, sg_nents(src),
                                     DMA_BIDIRECTIONAL);
                 nr_dst = nr_src;
                 if (!nr_src)
                         return -EINVAL;
         } else {
-                nr_src = dma_map_sg(priv->dev, src,
-                                    sg_nents_for_len(src, totlen),
+                nr_src = dma_map_sg(priv->dev, src, sg_nents(src),
                                     DMA_TO_DEVICE);
                 if (!nr_src)
                         return -EINVAL;
 
-                nr_dst = dma_map_sg(priv->dev, dst,
-                                    sg_nents_for_len(dst, totlen),
+                nr_dst = dma_map_sg(priv->dev, dst, sg_nents(dst),
                                     DMA_FROM_DEVICE);
                 if (!nr_dst) {
-                        dma_unmap_sg(priv->dev, src,
-                                     sg_nents_for_len(src, totlen),
-                                     DMA_TO_DEVICE);
+                        dma_unmap_sg(priv->dev, src, nr_src, DMA_TO_DEVICE);
                         return -EINVAL;
                 }
         }
@@ -454,7 +466,7 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring,
 
         /* result descriptors */
         for_each_sg(dst, sg, nr_dst, i) {
-                bool first = !i, last = (i == nr_dst - 1);
+                bool first = !i, last = sg_is_last(sg);
                 u32 len = sg_dma_len(sg);
 
                 rdesc = safexcel_add_rdesc(priv, ring, first, last,
@@ -483,16 +495,10 @@ cdesc_rollback:
                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 
         if (src == dst) {
-                dma_unmap_sg(priv->dev, src,
-                             sg_nents_for_len(src, totlen),
-                             DMA_BIDIRECTIONAL);
+                dma_unmap_sg(priv->dev, src, nr_src, DMA_BIDIRECTIONAL);
         } else {
-                dma_unmap_sg(priv->dev, src,
-                             sg_nents_for_len(src, totlen),
-                             DMA_TO_DEVICE);
-                dma_unmap_sg(priv->dev, dst,
-                             sg_nents_for_len(dst, totlen),
-                             DMA_FROM_DEVICE);
+                dma_unmap_sg(priv->dev, src, nr_src, DMA_TO_DEVICE);
+                dma_unmap_sg(priv->dev, dst, nr_dst, DMA_FROM_DEVICE);
         }
 
         return ret;
@@ -501,6 +507,7 @@ cdesc_rollback:
 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
                                       int ring,
                                       struct crypto_async_request *base,
+                                      struct safexcel_cipher_req *sreq,
                                       bool *should_complete, int *ret)
 {
         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
@@ -509,7 +516,10 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 
         *ret = 0;
 
-        do {
+        if (unlikely(!sreq->rdescs))
+                return 0;
+
+        while (sreq->rdescs--) {
                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
                 if (IS_ERR(rdesc)) {
                         dev_err(priv->dev,
@@ -522,7 +532,7 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 
                 ndesc++;
-        } while (!rdesc->last_seg);
+        }
 
         safexcel_complete(priv, ring);
 
@@ -560,16 +570,35 @@ static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv,
 {
         struct skcipher_request *req = skcipher_request_cast(async);
         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
+        struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(async->tfm);
         int err;
 
         if (sreq->needs_inv) {
                 sreq->needs_inv = false;
-                err = safexcel_handle_inv_result(priv, ring, async,
+                err = safexcel_handle_inv_result(priv, ring, async, sreq,
                                                  should_complete, ret);
         } else {
                 err = safexcel_handle_req_result(priv, ring, async, req->src,
                                                  req->dst, req->cryptlen, sreq,
                                                  should_complete, ret);
+
+                if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
+                        u32 block_sz = 0;
+
+                        switch (ctx->alg) {
+                        case SAFEXCEL_DES:
+                                block_sz = DES_BLOCK_SIZE;
+                                break;
+                        case SAFEXCEL_3DES:
+                                block_sz = DES3_EDE_BLOCK_SIZE;
+                                break;
+                        case SAFEXCEL_AES:
+                                block_sz = AES_BLOCK_SIZE;
+                                break;
+                        }
+
+                        memcpy(req->iv, ctx->base.ctxr->data, block_sz);
+                }
         }
 
         return err;
@@ -587,7 +616,7 @@ static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv,
 
         if (sreq->needs_inv) {
                 sreq->needs_inv = false;
-                err = safexcel_handle_inv_result(priv, ring, async,
+                err = safexcel_handle_inv_result(priv, ring, async, sreq,
                                                  should_complete, ret);
         } else {
                 err = safexcel_handle_req_result(priv, ring, async, req->src,
@@ -633,6 +662,8 @@ static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
                 ret = safexcel_send_req(async, ring, sreq, req->src,
                                         req->dst, req->cryptlen, 0, 0, req->iv,
                                         commands, results);
+
+        sreq->rdescs = *results;
         return ret;
 }
 
@@ -655,6 +686,7 @@ static int safexcel_aead_send(struct crypto_async_request *async, int ring,
                                         req->cryptlen, req->assoclen,
                                         crypto_aead_authsize(tfm), req->iv,
                                         commands, results);
+        sreq->rdescs = *results;
         return ret;
 }
 
diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c
index ac9282c1a5ec..a80a5e757b1f 100644
--- a/drivers/crypto/inside-secure/safexcel_hash.c
+++ b/drivers/crypto/inside-secure/safexcel_hash.c
@@ -41,19 +41,21 @@ struct safexcel_ahash_req {
         u64 len[2];
         u64 processed[2];
 
-        u8 cache[SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
+        u8 cache[SHA512_BLOCK_SIZE << 1] __aligned(sizeof(u32));
         dma_addr_t cache_dma;
         unsigned int cache_sz;
 
-        u8 cache_next[SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
+        u8 cache_next[SHA512_BLOCK_SIZE << 1] __aligned(sizeof(u32));
 };
 
 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)
 {
-        if (req->len[1] > req->processed[1])
-                return 0xffffffff - (req->len[0] - req->processed[0]);
+        u64 len, processed;
 
-        return req->len[0] - req->processed[0];
+        len = (0xffffffff * req->len[1]) + req->len[0];
+        processed = (0xffffffff * req->processed[1]) + req->processed[0];
+
+        return len - processed;
 }
 
 static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
@@ -87,6 +89,9 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
         cdesc->control_data.control0 |= ctx->alg;
         cdesc->control_data.control0 |= req->digest;
 
+        if (!req->finish)
+                cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH;
+
         if (req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) {
                 if (req->processed[0] || req->processed[1]) {
                         if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
@@ -105,9 +110,6 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
                         cdesc->control_data.control0 |= CONTEXT_CONTROL_RESTART_HASH;
                 }
 
-                if (!req->finish)
-                        cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH;
-
                 /*
                  * Copy the input digest if needed, and setup the context
                  * fields. Do this now as we need it to setup the first command
@@ -183,6 +185,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
                 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz,
                                  DMA_TO_DEVICE);
                 sreq->cache_dma = 0;
+                sreq->cache_sz = 0;
         }
 
         if (sreq->finish)
@@ -209,11 +212,15 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
         struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
         struct safexcel_result_desc *rdesc;
         struct scatterlist *sg;
-        int i, extra, n_cdesc = 0, ret = 0;
-        u64 queued, len, cache_len;
+        int i, extra = 0, n_cdesc = 0, ret = 0;
+        u64 queued, len, cache_len, cache_max;
+
+        cache_max = crypto_ahash_blocksize(ahash);
+        if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC)
+                cache_max <<= 1;
 
         queued = len = safexcel_queued_len(req);
-        if (queued <= crypto_ahash_blocksize(ahash))
+        if (queued <= cache_max)
                 cache_len = queued;
         else
                 cache_len = queued - areq->nbytes;
@@ -223,26 +230,23 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
                  * fit into full blocks, cache it for the next send() call.
                  */
                 extra = queued & (crypto_ahash_blocksize(ahash) - 1);
+
+                if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC &&
+                    extra < crypto_ahash_blocksize(ahash))
+                        extra += crypto_ahash_blocksize(ahash);
+
+                /* If this is not the last request and the queued data
+                 * is a multiple of a block, cache the last one for now.
+                 */
                 if (!extra)
-                        /* If this is not the last request and the queued data
-                         * is a multiple of a block, cache the last one for now.
-                         */
                         extra = crypto_ahash_blocksize(ahash);
 
-                if (extra) {
-                        sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
-                                           req->cache_next, extra,
-                                           areq->nbytes - extra);
-
-                        queued -= extra;
-                        len -= extra;
+                sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
+                                   req->cache_next, extra,
+                                   areq->nbytes - extra);
 
-                        if (!queued) {
-                                *commands = 0;
-                                *results = 0;
-                                return 0;
-                        }
-                }
+                queued -= extra;
+                len -= extra;
         }
 
         /* Add a command descriptor for the cached data, if any */
@@ -269,8 +273,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
         }
 
         /* Now handle the current ahash request buffer(s) */
-        req->nents = dma_map_sg(priv->dev, areq->src,
-                                sg_nents_for_len(areq->src, areq->nbytes),
+        req->nents = dma_map_sg(priv->dev, areq->src, sg_nents(areq->src),
                                 DMA_TO_DEVICE);
         if (!req->nents) {
                 ret = -ENOMEM;
@@ -345,6 +348,7 @@ unmap_cache:
         if (req->cache_dma) {
                 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,
                                  DMA_TO_DEVICE);
+                req->cache_dma = 0;
                 req->cache_sz = 0;
         }
 
@@ -486,7 +490,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
         struct safexcel_inv_result result = {};
         int ring = ctx->base.ring;
 
-        memset(req, 0, sizeof(struct ahash_request));
+        memset(req, 0, EIP197_AHASH_REQ_SIZE);
 
         /* create invalidation request */
         init_completion(&result.completion);
@@ -519,10 +523,9 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
 /* safexcel_ahash_cache: cache data until at least one request can be sent to
  * the engine, aka. when there is at least 1 block size in the pipe.
  */
-static int safexcel_ahash_cache(struct ahash_request *areq)
+static int safexcel_ahash_cache(struct ahash_request *areq, u32 cache_max)
 {
         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
-        struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
         u64 queued, cache_len;
 
         /* queued: everything accepted by the driver which will be handled by
@@ -539,7 +542,7 @@ static int safexcel_ahash_cache(struct ahash_request *areq)
          * In case there isn't enough bytes to proceed (less than a
          * block size), cache the data until we have enough.
          */
-        if (cache_len + areq->nbytes <= crypto_ahash_blocksize(ahash)) {
+        if (cache_len + areq->nbytes <= cache_max) {
                 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
                                    req->cache + cache_len,
                                    areq->nbytes, 0);
@@ -599,6 +602,7 @@ static int safexcel_ahash_update(struct ahash_request *areq)
 {
         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
+        u32 cache_max;
 
         /* If the request is 0 length, do nothing */
         if (!areq->nbytes)
@@ -608,7 +612,11 @@ static int safexcel_ahash_update(struct ahash_request *areq)
         if (req->len[0] < areq->nbytes)
                 req->len[1]++;
 
-        safexcel_ahash_cache(areq);
+        cache_max = crypto_ahash_blocksize(ahash);
+        if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC)
+                cache_max <<= 1;
+
+        safexcel_ahash_cache(areq, cache_max);
 
         /*
          * We're not doing partial updates when performing an hmac request.
@@ -621,7 +629,7 @@ static int safexcel_ahash_update(struct ahash_request *areq)
                 return safexcel_ahash_enqueue(areq);
 
         if (!req->last_req &&
-            safexcel_queued_len(req) > crypto_ahash_blocksize(ahash))
+            safexcel_queued_len(req) > cache_max)
                 return safexcel_ahash_enqueue(areq);
 
         return 0;
@@ -678,6 +686,11 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out)
         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
         struct safexcel_ahash_export_state *export = out;
+        u32 cache_sz;
+
+        cache_sz = crypto_ahash_blocksize(ahash);
+        if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC)
+                cache_sz <<= 1;
 
         export->len[0] = req->len[0];
         export->len[1] = req->len[1];
@@ -687,7 +700,7 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out)
         export->digest = req->digest;
 
         memcpy(export->state, req->state, req->state_sz);
-        memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash));
+        memcpy(export->cache, req->cache, cache_sz);
 
         return 0;
 }
@@ -697,12 +710,17 @@ static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
         struct safexcel_ahash_req *req = ahash_request_ctx(areq);
         const struct safexcel_ahash_export_state *export = in;
+        u32 cache_sz;
         int ret;
 
         ret = crypto_ahash_init(areq);
         if (ret)
                 return ret;
 
+        cache_sz = crypto_ahash_blocksize(ahash);
+        if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC)
+                cache_sz <<= 1;
+
         req->len[0] = export->len[0];
         req->len[1] = export->len[1];
         req->processed[0] = export->processed[0];
@@ -710,7 +728,7 @@ static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
 
         req->digest = export->digest;
 
-        memcpy(req->cache, export->cache, crypto_ahash_blocksize(ahash));
+        memcpy(req->cache, export->cache, cache_sz);
         memcpy(req->state, export->state, req->state_sz);
 
         return 0;
diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c
index eb75fa684876..142bc3f5c45c 100644
--- a/drivers/crypto/inside-secure/safexcel_ring.c
+++ b/drivers/crypto/inside-secure/safexcel_ring.c
@@ -145,6 +145,9 @@ struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *pr
                         (lower_32_bits(context) & GENMASK(31, 2)) >> 2;
                 cdesc->control_data.context_hi = upper_32_bits(context);
 
+                if (priv->version == EIP197B || priv->version == EIP197D)
+                        cdesc->control_data.options |= EIP197_OPTION_RC_AUTO;
+
                 /* TODO: large xform HMAC with SHA-384/512 uses refresh = 3 */
                 cdesc->control_data.refresh = 2;
 
diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c
index e5cf3a59c420..acedafe3fa98 100644
--- a/drivers/crypto/ixp4xx_crypto.c
+++ b/drivers/crypto/ixp4xx_crypto.c
@@ -100,7 +100,7 @@ struct buffer_desc {
         u16 pkt_len;
         u16 buf_len;
 #endif
-        u32 phys_addr;
+        dma_addr_t phys_addr;
         u32 __reserved[4];
         struct buffer_desc *next;
         enum dma_data_direction dir;
@@ -117,9 +117,9 @@ struct crypt_ctl {
         u8 mode;                /* NPE_OP_*  operation mode */
 #endif
         u8 iv[MAX_IVLEN];       /* IV for CBC mode or CTR IV for CTR mode */
-        u32 icv_rev_aes;        /* icv or rev aes */
-        u32 src_buf;
-        u32 dst_buf;
+        dma_addr_t icv_rev_aes; /* icv or rev aes */
+        dma_addr_t src_buf;
+        dma_addr_t dst_buf;
 #ifdef __ARMEB__
         u16 auth_offs;          /* Authentication start offset */
         u16 auth_len;           /* Authentication data length */
@@ -320,7 +320,8 @@ static struct crypt_ctl *get_crypt_desc_emerg(void)
         }
 }
 
-static void free_buf_chain(struct device *dev, struct buffer_desc *buf,u32 phys)
+static void free_buf_chain(struct device *dev, struct buffer_desc *buf,
+                           dma_addr_t phys)
 {
         while (buf) {
                 struct buffer_desc *buf1;
@@ -602,7 +603,7 @@ static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target,
         struct buffer_desc *buf;
         int i;
         u8 *pad;
-        u32 pad_phys, buf_phys;
+        dma_addr_t pad_phys, buf_phys;
 
         BUILD_BUG_ON(NPE_CTX_LEN < HMAC_PAD_BLOCKLEN);
         pad = dma_pool_alloc(ctx_pool, GFP_KERNEL, &pad_phys);
@@ -787,7 +788,7 @@ static struct buffer_desc *chainup_buffers(struct device *dev,
         for (; nbytes > 0; sg = sg_next(sg)) {
                 unsigned len = min(nbytes, sg->length);
                 struct buffer_desc *next_buf;
-                u32 next_buf_phys;
+                dma_addr_t next_buf_phys;
                 void *ptr;
 
                 nbytes -= len;
diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c
index bdc4c42d3ac8..f1fa637cb029 100644
--- a/drivers/crypto/mxs-dcp.c
+++ b/drivers/crypto/mxs-dcp.c
@@ -986,8 +986,6 @@ static int mxs_dcp_probe(struct platform_device *pdev)
         struct device *dev = &pdev->dev;
         struct dcp *sdcp = NULL;
         int i, ret;
-
-        struct resource *iores;
         int dcp_vmi_irq, dcp_irq;
 
         if (global_sdcp) {
@@ -995,7 +993,6 @@ static int mxs_dcp_probe(struct platform_device *pdev)
                 return -ENODEV;
         }
 
-        iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
         dcp_vmi_irq = platform_get_irq(pdev, 0);
         if (dcp_vmi_irq < 0) {
                 dev_err(dev, "Failed to get IRQ: (%d)!\n", dcp_vmi_irq);
@@ -1013,7 +1010,7 @@ static int mxs_dcp_probe(struct platform_device *pdev)
                 return -ENOMEM;
 
         sdcp->dev = dev;
-        sdcp->base = devm_ioremap_resource(dev, iores);
+        sdcp->base = devm_platform_ioremap_resource(pdev, 0);
         if (IS_ERR(sdcp->base))
                 return PTR_ERR(sdcp->base);
 
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index 4acbc47973e9..e78ff5c65ed6 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -27,8 +27,6 @@ MODULE_ALIAS_CRYPTO("842-nx");
 #define WORKMEM_ALIGN   (CRB_ALIGN)
 #define CSB_WAIT_MAX    (5000) /* ms */
 #define VAS_RETRIES     (10)
-/* # of requests allowed per RxFIFO at a time. 0 for unlimited */
-#define MAX_CREDITS_PER_RXFIFO  (1024)
 
 struct nx842_workmem {
         /* Below fields must be properly aligned */
@@ -812,7 +810,11 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
         rxattr.lnotify_lpid = lpid;
         rxattr.lnotify_pid = pid;
         rxattr.lnotify_tid = tid;
-        rxattr.wcreds_max = MAX_CREDITS_PER_RXFIFO;
+        /*
+         * Maximum RX window credits can not be more than #CRBs in
+         * RxFIFO. Otherwise, can get checkstop if RxFIFO overruns.
+         */
+        rxattr.wcreds_max = fifo_size / CRB_SIZE;
 
         /*
          * Open a VAS receice window which is used to configure RxFIFO
diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c
index 428c273a1ab6..28817880c76d 100644
--- a/drivers/crypto/nx/nx.c
+++ b/drivers/crypto/nx/nx.c
@@ -569,9 +569,7 @@ static int nx_register_algs(void)
 
         memset(&nx_driver.stats, 0, sizeof(struct nx_stats));
 
-        rc = NX_DEBUGFS_INIT(&nx_driver);
-        if (rc)
-                goto out;
+        NX_DEBUGFS_INIT(&nx_driver);
 
         nx_driver.of.status = NX_OKAY;
 
diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h
index c3e54af18645..c6b5a3be02be 100644
--- a/drivers/crypto/nx/nx.h
+++ b/drivers/crypto/nx/nx.h
@@ -76,20 +76,12 @@ struct nx_stats {
         atomic_t last_error_pid;
 };
 
-struct nx_debugfs {
-        struct dentry *dfs_root;
-        struct dentry *dfs_aes_ops, *dfs_aes_bytes;
-        struct dentry *dfs_sha256_ops, *dfs_sha256_bytes;
-        struct dentry *dfs_sha512_ops, *dfs_sha512_bytes;
-        struct dentry *dfs_errors, *dfs_last_error, *dfs_last_error_pid;
-};
-
 struct nx_crypto_driver {
         struct nx_stats    stats;
         struct nx_of       of;
         struct vio_dev    *viodev;
         struct vio_driver  viodriver;
-        struct nx_debugfs  dfs;
+        struct dentry     *dfs_root;
 };
 
 #define NX_GCM4106_NONCE_LEN            (4)
@@ -177,7 +169,7 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *, unsigned int,
 #define NX_DEBUGFS_INIT(drv)    nx_debugfs_init(drv)
 #define NX_DEBUGFS_FINI(drv)    nx_debugfs_fini(drv)
 
-int nx_debugfs_init(struct nx_crypto_driver *);
+void nx_debugfs_init(struct nx_crypto_driver *);
 void nx_debugfs_fini(struct nx_crypto_driver *);
 #else
 #define NX_DEBUGFS_INIT(drv)    (0)
diff --git a/drivers/crypto/nx/nx_debugfs.c b/drivers/crypto/nx/nx_debugfs.c
index 03e4f0363c6a..e0d44a5512ab 100644
--- a/drivers/crypto/nx/nx_debugfs.c
+++ b/drivers/crypto/nx/nx_debugfs.c
@@ -30,62 +30,37 @@
  * Documentation/ABI/testing/debugfs-pfo-nx-crypto
  */
 
-int nx_debugfs_init(struct nx_crypto_driver *drv)
+void nx_debugfs_init(struct nx_crypto_driver *drv)
 {
-        struct nx_debugfs *dfs = &drv->dfs;
+        struct dentry *root;
 
-        dfs->dfs_root = debugfs_create_dir(NX_NAME, NULL);
+        root = debugfs_create_dir(NX_NAME, NULL);
+        drv->dfs_root = root;
 
-        dfs->dfs_aes_ops =
-                debugfs_create_u32("aes_ops",
-                                   S_IRUSR | S_IRGRP | S_IROTH,
-                                   dfs->dfs_root, (u32 *)&drv->stats.aes_ops);
-        dfs->dfs_sha256_ops =
-                debugfs_create_u32("sha256_ops",
-                                   S_IRUSR | S_IRGRP | S_IROTH,
-                                   dfs->dfs_root,
-                                   (u32 *)&drv->stats.sha256_ops);
-        dfs->dfs_sha512_ops =
-                debugfs_create_u32("sha512_ops",
-                                   S_IRUSR | S_IRGRP | S_IROTH,
-                                   dfs->dfs_root,
-                                   (u32 *)&drv->stats.sha512_ops);
-        dfs->dfs_aes_bytes =
-                debugfs_create_u64("aes_bytes",
-                                   S_IRUSR | S_IRGRP | S_IROTH,
-                                   dfs->dfs_root,
-                                   (u64 *)&drv->stats.aes_bytes);
-        dfs->dfs_sha256_bytes =
-                debugfs_create_u64("sha256_bytes",
-                                   S_IRUSR | S_IRGRP | S_IROTH,
-                                   dfs->dfs_root,
-                                   (u64 *)&drv->stats.sha256_bytes);
-        dfs->dfs_sha512_bytes =
-                debugfs_create_u64("sha512_bytes",
-                                   S_IRUSR | S_IRGRP | S_IROTH,
-                                   dfs->dfs_root,
-                                   (u64 *)&drv->stats.sha512_bytes);
-        dfs->dfs_errors =
-                debugfs_create_u32("errors",
-                                   S_IRUSR | S_IRGRP | S_IROTH,
-                                   dfs->dfs_root, (u32 *)&drv->stats.errors);
-        dfs->dfs_last_error =
-                debugfs_create_u32("last_error",
-                                   S_IRUSR | S_IRGRP | S_IROTH,
-                                   dfs->dfs_root,
-                                   (u32 *)&drv->stats.last_error);
-        dfs->dfs_last_error_pid =
-                debugfs_create_u32("last_error_pid",
-                                   S_IRUSR | S_IRGRP | S_IROTH,
-                                   dfs->dfs_root,
-                                   (u32 *)&drv->stats.last_error_pid);
-        return 0;
+        debugfs_create_u32("aes_ops", S_IRUSR | S_IRGRP | S_IROTH,
+                           root, (u32 *)&drv->stats.aes_ops);
+        debugfs_create_u32("sha256_ops", S_IRUSR | S_IRGRP | S_IROTH,
+                           root, (u32 *)&drv->stats.sha256_ops);
+        debugfs_create_u32("sha512_ops", S_IRUSR | S_IRGRP | S_IROTH,
+                           root, (u32 *)&drv->stats.sha512_ops);
+        debugfs_create_u64("aes_bytes", S_IRUSR | S_IRGRP | S_IROTH,
+                           root, (u64 *)&drv->stats.aes_bytes);
+        debugfs_create_u64("sha256_bytes", S_IRUSR | S_IRGRP | S_IROTH,
+                           root, (u64 *)&drv->stats.sha256_bytes);
+        debugfs_create_u64("sha512_bytes", S_IRUSR | S_IRGRP | S_IROTH,
+                           root, (u64 *)&drv->stats.sha512_bytes);
+        debugfs_create_u32("errors", S_IRUSR | S_IRGRP | S_IROTH,
+                           root, (u32 *)&drv->stats.errors);
+        debugfs_create_u32("last_error", S_IRUSR | S_IRGRP | S_IROTH,
+                           root, (u32 *)&drv->stats.last_error);
+        debugfs_create_u32("last_error_pid", S_IRUSR | S_IRGRP | S_IROTH,
+                           root, (u32 *)&drv->stats.last_error_pid);
 }
 
 void
 nx_debugfs_fini(struct nx_crypto_driver *drv)
 {
-        debugfs_remove_recursive(drv->dfs.dfs_root);
+        debugfs_remove_recursive(drv->dfs_root);
 }
 
 #endif
diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c
index c8d401646902..b50eb55f8f57 100644
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_algs.c
@@ -131,7 +131,6 @@ struct qat_alg_ablkcipher_ctx {
         struct icp_qat_fw_la_bulk_req dec_fw_req;
         struct qat_crypto_instance *inst;
         struct crypto_tfm *tfm;
-        spinlock_t lock;        /* protects qat_alg_ablkcipher_ctx struct */
 };
 
 static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
@@ -223,6 +222,9 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
                 return -EFAULT;
 
         offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
+        if (offset < 0)
+                return -EFAULT;
+
         hash_state_out = (__be32 *)(hash->sha.state1 + offset);
         hash512_state_out = (__be64 *)hash_state_out;
 
@@ -253,7 +255,24 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
         return 0;
 }
 
-static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
+static void qat_alg_init_hdr_iv_updt(struct icp_qat_fw_comn_req_hdr *header)
+{
+        ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
+                                           ICP_QAT_FW_CIPH_IV_64BIT_PTR);
+        ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
+                                       ICP_QAT_FW_LA_UPDATE_STATE);
+}
+
+static void qat_alg_init_hdr_no_iv_updt(struct icp_qat_fw_comn_req_hdr *header)
+{
+        ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
+                                           ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
+        ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
+                                       ICP_QAT_FW_LA_NO_UPDATE_STATE);
+}
+
+static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header,
+                                    int aead)
 {
         header->hdr_flags =
                 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
@@ -263,12 +282,12 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
                                             QAT_COMN_PTR_TYPE_SGL);
         ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
                                   ICP_QAT_FW_LA_PARTIAL_NONE);
-        ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
-                                           ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
+        if (aead)
+                qat_alg_init_hdr_no_iv_updt(header);
+        else
+                qat_alg_init_hdr_iv_updt(header);
         ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
                                 ICP_QAT_FW_LA_NO_PROTO);
-        ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
-                                       ICP_QAT_FW_LA_NO_UPDATE_STATE);
 }
 
 static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
@@ -303,7 +322,7 @@ static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
                 return -EFAULT;
 
         /* Request setup */
-        qat_alg_init_common_hdr(header);
+        qat_alg_init_common_hdr(header, 1);
         header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
         ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
                                            ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
@@ -390,7 +409,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
                 return -EFAULT;
 
         /* Request setup */
-        qat_alg_init_common_hdr(header);
+        qat_alg_init_common_hdr(header, 1);
         header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
         ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
                                            ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
@@ -454,7 +473,7 @@ static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx,
         struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
 
         memcpy(cd->aes.key, key, keylen);
-        qat_alg_init_common_hdr(header);
+        qat_alg_init_common_hdr(header, 0);
         header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
         cd_pars->u.s.content_desc_params_sz =
                                 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
@@ -576,45 +595,52 @@ bad_key:
         return -EINVAL;
 }
 
-static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key,
+static int qat_alg_aead_rekey(struct crypto_aead *tfm, const uint8_t *key,
+                              unsigned int keylen)
+{
+        struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
+
+        memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
+        memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
+        memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
+        memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
+
+        return qat_alg_aead_init_sessions(tfm, key, keylen,
+                                          ICP_QAT_HW_CIPHER_CBC_MODE);
+}
+
+static int qat_alg_aead_newkey(struct crypto_aead *tfm, const uint8_t *key,
                                unsigned int keylen)
 {
         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
+        struct qat_crypto_instance *inst = NULL;
+        int node = get_current_node();
         struct device *dev;
+        int ret;
 
-        if (ctx->enc_cd) {
-                /* rekeying */
-                dev = &GET_DEV(ctx->inst->accel_dev);
-                memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
-                memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
-                memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
-                memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
-        } else {
-                /* new key */
-                int node = get_current_node();
-                struct qat_crypto_instance *inst =
-                                qat_crypto_get_instance_node(node);
-                if (!inst) {
-                        return -EINVAL;
-                }
-
-                dev = &GET_DEV(inst->accel_dev);
-                ctx->inst = inst;
-                ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
-                                                 &ctx->enc_cd_paddr,
-                                                 GFP_ATOMIC);
-                if (!ctx->enc_cd) {
-                        return -ENOMEM;
-                }
-                ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
-                                                 &ctx->dec_cd_paddr,
-                                                 GFP_ATOMIC);
-                if (!ctx->dec_cd) {
-                        goto out_free_enc;
-                }
+        inst = qat_crypto_get_instance_node(node);
+        if (!inst)
+                return -EINVAL;
+        dev = &GET_DEV(inst->accel_dev);
+        ctx->inst = inst;
+        ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
+                                         &ctx->enc_cd_paddr,
+                                         GFP_ATOMIC);
+        if (!ctx->enc_cd) {
+                ret = -ENOMEM;
+                goto out_free_inst;
+        }
+        ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
+                                         &ctx->dec_cd_paddr,
+                                         GFP_ATOMIC);
+        if (!ctx->dec_cd) {
+                ret = -ENOMEM;
+                goto out_free_enc;
         }
-        if (qat_alg_aead_init_sessions(tfm, key, keylen,
-                                       ICP_QAT_HW_CIPHER_CBC_MODE))
+
+        ret = qat_alg_aead_init_sessions(tfm, key, keylen,
+                                         ICP_QAT_HW_CIPHER_CBC_MODE);
+        if (ret)
                 goto out_free_all;
 
         return 0;
@@ -629,7 +655,21 @@ out_free_enc:
         dma_free_coherent(dev, sizeof(struct qat_alg_cd),
                           ctx->enc_cd, ctx->enc_cd_paddr);
         ctx->enc_cd = NULL;
-        return -ENOMEM;
+out_free_inst:
+        ctx->inst = NULL;
+        qat_crypto_put_instance(inst);
+        return ret;
+}
+
+static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key,
+                               unsigned int keylen)
+{
+        struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
+
+        if (ctx->enc_cd)
+                return qat_alg_aead_rekey(tfm, key, keylen);
+        else
+                return qat_alg_aead_newkey(tfm, key, keylen);
 }
 
 static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
@@ -677,8 +717,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
         dma_addr_t blp;
         dma_addr_t bloutp = 0;
         struct scatterlist *sg;
-        size_t sz_out, sz = sizeof(struct qat_alg_buf_list) +
-                        ((1 + n) * sizeof(struct qat_alg_buf));
+        size_t sz_out, sz = struct_size(bufl, bufers, n + 1);
 
         if (unlikely(!n))
                 return -EINVAL;
@@ -715,8 +754,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
                 struct qat_alg_buf *bufers;
 
                 n = sg_nents(sglout);
-                sz_out = sizeof(struct qat_alg_buf_list) +
-                        ((1 + n) * sizeof(struct qat_alg_buf));
+                sz_out = struct_size(buflout, bufers, n + 1);
                 sg_nctr = 0;
                 buflout = kzalloc_node(sz_out, GFP_ATOMIC,
                                        dev_to_node(&GET_DEV(inst->accel_dev)));
@@ -801,11 +839,17 @@ static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
         struct qat_crypto_instance *inst = ctx->inst;
         struct ablkcipher_request *areq = qat_req->ablkcipher_req;
         uint8_t stat_filed = qat_resp->comn_resp.comn_status;
+        struct device *dev = &GET_DEV(ctx->inst->accel_dev);
         int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
 
         qat_alg_free_bufl(inst, qat_req);
         if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
                 res = -EINVAL;
+
+        memcpy(areq->info, qat_req->iv, AES_BLOCK_SIZE);
+        dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
+                          qat_req->iv_paddr);
+
         areq->base.complete(&areq->base, res);
 }
 
@@ -905,50 +949,49 @@ static int qat_alg_aead_enc(struct aead_request *areq)
         return -EINPROGRESS;
 }
 
-static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
+static int qat_alg_ablkcipher_rekey(struct qat_alg_ablkcipher_ctx *ctx,
+                                    const u8 *key, unsigned int keylen,
+                                    int mode)
+{
+        memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
+        memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
+        memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
+        memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
+
+        return qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode);
+}
+
+static int qat_alg_ablkcipher_newkey(struct qat_alg_ablkcipher_ctx *ctx,
                                      const u8 *key, unsigned int keylen,
                                      int mode)
 {
-        struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+        struct qat_crypto_instance *inst = NULL;
         struct device *dev;
+        int node = get_current_node();
+        int ret;
 
-        spin_lock(&ctx->lock);
-        if (ctx->enc_cd) {
-                /* rekeying */
-                dev = &GET_DEV(ctx->inst->accel_dev);
-                memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
-                memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
-                memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
-                memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
-        } else {
-                /* new key */
-                int node = get_current_node();
-                struct qat_crypto_instance *inst =
-                                qat_crypto_get_instance_node(node);
-                if (!inst) {
-                        spin_unlock(&ctx->lock);
-                        return -EINVAL;
-                }
-
-                dev = &GET_DEV(inst->accel_dev);
-                ctx->inst = inst;
-                ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
-                                                 &ctx->enc_cd_paddr,
-                                                 GFP_ATOMIC);
-                if (!ctx->enc_cd) {
-                        spin_unlock(&ctx->lock);
-                        return -ENOMEM;
-                }
-                ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
-                                                 &ctx->dec_cd_paddr,
-                                                 GFP_ATOMIC);
-                if (!ctx->dec_cd) {
-                        spin_unlock(&ctx->lock);
-                        goto out_free_enc;
-                }
+        inst = qat_crypto_get_instance_node(node);
+        if (!inst)
+                return -EINVAL;
+        dev = &GET_DEV(inst->accel_dev);
+        ctx->inst = inst;
+        ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
+                                         &ctx->enc_cd_paddr,
+                                         GFP_ATOMIC);
+        if (!ctx->enc_cd) {
+                ret = -ENOMEM;
+                goto out_free_instance;
+        }
+        ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
+                                         &ctx->dec_cd_paddr,
+                                         GFP_ATOMIC);
+        if (!ctx->dec_cd) {
+                ret = -ENOMEM;
+                goto out_free_enc;
         }
-        spin_unlock(&ctx->lock);
-        if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode))
+
+        ret = qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode);
+        if (ret)
                 goto out_free_all;
 
         return 0;
@@ -963,7 +1006,22 @@ out_free_enc:
         dma_free_coherent(dev, sizeof(*ctx->enc_cd),
                           ctx->enc_cd, ctx->enc_cd_paddr);
         ctx->enc_cd = NULL;
-        return -ENOMEM;
+out_free_instance:
+        ctx->inst = NULL;
+        qat_crypto_put_instance(inst);
+        return ret;
+}
+
+static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
+                                     const u8 *key, unsigned int keylen,
+                                     int mode)
+{
+        struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+
+        if (ctx->enc_cd)
+                return qat_alg_ablkcipher_rekey(ctx, key, keylen, mode);
+        else
+                return qat_alg_ablkcipher_newkey(ctx, key, keylen, mode);
 }
 
 static int qat_alg_ablkcipher_cbc_setkey(struct crypto_ablkcipher *tfm,
@@ -995,11 +1053,23 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)
         struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
         struct icp_qat_fw_la_cipher_req_params *cipher_param;
         struct icp_qat_fw_la_bulk_req *msg;
+        struct device *dev = &GET_DEV(ctx->inst->accel_dev);
         int ret, ctr = 0;
 
+        if (req->nbytes == 0)
+                return 0;
+
+        qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE,
+                                         &qat_req->iv_paddr, GFP_ATOMIC);
+        if (!qat_req->iv)
+                return -ENOMEM;
+
         ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
-        if (unlikely(ret))
+        if (unlikely(ret)) {
+                dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
+                                  qat_req->iv_paddr);
                 return ret;
+        }
 
         msg = &qat_req->req;
         *msg = ctx->enc_fw_req;
@@ -1012,18 +1082,29 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)
         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
         cipher_param->cipher_length = req->nbytes;
         cipher_param->cipher_offset = 0;
-        memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
+        cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr;
+        memcpy(qat_req->iv, req->info, AES_BLOCK_SIZE);
         do {
                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
         } while (ret == -EAGAIN && ctr++ < 10);
 
         if (ret == -EAGAIN) {
                 qat_alg_free_bufl(ctx->inst, qat_req);
+                dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
+                                  qat_req->iv_paddr);
                 return -EBUSY;
         }
         return -EINPROGRESS;
 }
 
+static int qat_alg_ablkcipher_blk_encrypt(struct ablkcipher_request *req)
+{
+        if (req->nbytes % AES_BLOCK_SIZE != 0)
+                return -EINVAL;
+
+        return qat_alg_ablkcipher_encrypt(req);
+}
+
 static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
 {
         struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
@@ -1032,11 +1113,23 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
         struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
         struct icp_qat_fw_la_cipher_req_params *cipher_param;
         struct icp_qat_fw_la_bulk_req *msg;
+        struct device *dev = &GET_DEV(ctx->inst->accel_dev);
         int ret, ctr = 0;
 
+        if (req->nbytes == 0)
+                return 0;
+
+        qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE,
+                                         &qat_req->iv_paddr, GFP_ATOMIC);
+        if (!qat_req->iv)
+                return -ENOMEM;
+
         ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
-        if (unlikely(ret))
+        if (unlikely(ret)) {
+                dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
+                                  qat_req->iv_paddr);
                 return ret;
+        }
 
         msg = &qat_req->req;
         *msg = ctx->dec_fw_req;
@@ -1049,18 +1142,28 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
         cipher_param->cipher_length = req->nbytes;
         cipher_param->cipher_offset = 0;
-        memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
+        cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr;
+        memcpy(qat_req->iv, req->info, AES_BLOCK_SIZE);
         do {
                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
         } while (ret == -EAGAIN && ctr++ < 10);
 
         if (ret == -EAGAIN) {
                 qat_alg_free_bufl(ctx->inst, qat_req);
+                dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv,
+                                  qat_req->iv_paddr);
                 return -EBUSY;
         }
         return -EINPROGRESS;
 }
 
+static int qat_alg_ablkcipher_blk_decrypt(struct ablkcipher_request *req)
+{
+        if (req->nbytes % AES_BLOCK_SIZE != 0)
+                return -EINVAL;
+
+        return qat_alg_ablkcipher_decrypt(req);
+}
 static int qat_alg_aead_init(struct crypto_aead *tfm,
                              enum icp_qat_hw_auth_algo hash,
                              const char *hash_name)
@@ -1119,7 +1222,6 @@ static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm)
 {
         struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
 
-        spin_lock_init(&ctx->lock);
         tfm->crt_ablkcipher.reqsize = sizeof(struct qat_crypto_request);
         ctx->tfm = tfm;
         return 0;
@@ -1221,8 +1323,8 @@ static struct crypto_alg qat_algs[] = { {
         .cra_u = {
                 .ablkcipher = {
                         .setkey = qat_alg_ablkcipher_cbc_setkey,
-                        .decrypt = qat_alg_ablkcipher_decrypt,
-                        .encrypt = qat_alg_ablkcipher_encrypt,
+                        .decrypt = qat_alg_ablkcipher_blk_decrypt,
+                        .encrypt = qat_alg_ablkcipher_blk_encrypt,
                         .min_keysize = AES_MIN_KEY_SIZE,
                         .max_keysize = AES_MAX_KEY_SIZE,
                         .ivsize = AES_BLOCK_SIZE,
@@ -1233,7 +1335,7 @@ static struct crypto_alg qat_algs[] = { {
         .cra_driver_name = "qat_aes_ctr",
         .cra_priority = 4001,
         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
-        .cra_blocksize = AES_BLOCK_SIZE,
+        .cra_blocksize = 1,
         .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
         .cra_alignmask = 0,
         .cra_type = &crypto_ablkcipher_type,
@@ -1265,8 +1367,8 @@ static struct crypto_alg qat_algs[] = { {
         .cra_u = {
                 .ablkcipher = {
                         .setkey = qat_alg_ablkcipher_xts_setkey,
-                        .decrypt = qat_alg_ablkcipher_decrypt,
-                        .encrypt = qat_alg_ablkcipher_encrypt,
+                        .decrypt = qat_alg_ablkcipher_blk_decrypt,
+                        .encrypt = qat_alg_ablkcipher_blk_encrypt,
                         .min_keysize = 2 * AES_MIN_KEY_SIZE,
                         .max_keysize = 2 * AES_MAX_KEY_SIZE,
                         .ivsize = AES_BLOCK_SIZE,
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h
index dc0273fe3620..c77a80020cde 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.h
+++ b/drivers/crypto/qat/qat_common/qat_crypto.h
@@ -88,6 +88,8 @@ struct qat_crypto_request {
         struct qat_crypto_request_buffs buf;
         void (*cb)(struct icp_qat_fw_la_resp *resp,
                    struct qat_crypto_request *req);
+        void *iv;
+        dma_addr_t iv_paddr;
 };
 
 #endif
diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c
index 6b498a90181e..b0b8e3d48aef 100644
--- a/drivers/crypto/sahara.c
+++ b/drivers/crypto/sahara.c
@@ -1384,7 +1384,6 @@ MODULE_DEVICE_TABLE(of, sahara_dt_ids);
 static int sahara_probe(struct platform_device *pdev)
 {
         struct sahara_dev *dev;
-        struct resource *res;
         u32 version;
         int irq;
         int err;
@@ -1398,8 +1397,7 @@ static int sahara_probe(struct platform_device *pdev)
         platform_set_drvdata(pdev, dev);
 
         /* Get the base address */
-        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-        dev->regs_base = devm_ioremap_resource(&pdev->dev, res);
+        dev->regs_base = devm_platform_ioremap_resource(pdev, 0);
         if (IS_ERR(dev->regs_base))
                 return PTR_ERR(dev->regs_base);
 
diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile
index ce77e38c77e0..518e0e0b11a9 100644
--- a/drivers/crypto/stm32/Makefile
+++ b/drivers/crypto/stm32/Makefile
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32_crc32.o
+obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32-crc32.o
 obj-$(CONFIG_CRYPTO_DEV_STM32_HASH) += stm32-hash.o
 obj-$(CONFIG_CRYPTO_DEV_STM32_CRYP) += stm32-cryp.o
diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32-crc32.c
index 440c9f1bd006..440c9f1bd006 100644
--- a/drivers/crypto/stm32/stm32_crc32.c
+++ b/drivers/crypto/stm32/stm32-crc32.c
diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c
index 29519d1c403f..23061f2bc74b 100644
--- a/drivers/crypto/stm32/stm32-hash.c
+++ b/drivers/crypto/stm32/stm32-hash.c
@@ -349,7 +349,7 @@ static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev,
                 return -ETIMEDOUT;
 
         if ((hdev->flags & HASH_FLAGS_HMAC) &&
-            (hdev->flags & ~HASH_FLAGS_HMAC_KEY)) {
+            (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) {
                 hdev->flags |= HASH_FLAGS_HMAC_KEY;
                 stm32_hash_write_key(hdev);
                 if (stm32_hash_wait_busy(hdev))
@@ -447,8 +447,8 @@ static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev,
 
         dma_async_issue_pending(hdev->dma_lch);
 
-        if (!wait_for_completion_interruptible_timeout(&hdev->dma_completion,
-                                                       msecs_to_jiffies(100)))
+        if (!wait_for_completion_timeout(&hdev->dma_completion,
+                                         msecs_to_jiffies(100)))
                 err = -ETIMEDOUT;
 
         if (dma_async_is_tx_complete(hdev->dma_lch, cookie,
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
index 7b0c42882830..4ab14d58e85b 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c
@@ -12,7 +12,7 @@
  */
 #include "sun4i-ss.h"
 
-static int sun4i_ss_opti_poll(struct skcipher_request *areq)
+static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq)
 {
         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
         struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
@@ -114,6 +114,29 @@ release_ss:
         return err;
 }
 
+
+static int noinline_for_stack sun4i_ss_cipher_poll_fallback(struct skcipher_request *areq)
+{
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
+        struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm);
+        struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq);
+        SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, op->fallback_tfm);
+        int err;
+
+        skcipher_request_set_sync_tfm(subreq, op->fallback_tfm);
+        skcipher_request_set_callback(subreq, areq->base.flags, NULL,
+                                      NULL);
+        skcipher_request_set_crypt(subreq, areq->src, areq->dst,
+                                   areq->cryptlen, areq->iv);
+        if (ctx->mode & SS_DECRYPTION)
+                err = crypto_skcipher_decrypt(subreq);
+        else
+                err = crypto_skcipher_encrypt(subreq);
+        skcipher_request_zero(subreq);
+
+        return err;
+}
+
 /* Generic function that support SG with size not multiple of 4 */
 static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
 {
@@ -140,8 +163,6 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
         unsigned int todo;
         struct sg_mapping_iter mi, mo;
         unsigned int oi, oo;    /* offset for in and out */
-        char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */
-        char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */
         unsigned int ob = 0;    /* offset in buf */
         unsigned int obo = 0;   /* offset in bufo*/
         unsigned int obl = 0;   /* length of data in bufo */
@@ -178,20 +199,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
         if (no_chunk == 1 && !need_fallback)
                 return sun4i_ss_opti_poll(areq);
 
-        if (need_fallback) {
-                SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, op->fallback_tfm);
-                skcipher_request_set_sync_tfm(subreq, op->fallback_tfm);
-                skcipher_request_set_callback(subreq, areq->base.flags, NULL,
-                                              NULL);
-                skcipher_request_set_crypt(subreq, areq->src, areq->dst,
-                                           areq->cryptlen, areq->iv);
-                if (ctx->mode & SS_DECRYPTION)
-                        err = crypto_skcipher_decrypt(subreq);
-                else
-                        err = crypto_skcipher_encrypt(subreq);
-                skcipher_request_zero(subreq);
-                return err;
-        }
+        if (need_fallback)
+                return sun4i_ss_cipher_poll_fallback(areq);
 
         spin_lock_irqsave(&ss->slock, flags);
 
@@ -224,6 +233,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
 
         while (oleft) {
                 if (ileft) {
+                        char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */
+
                         /*
                          * todo is the number of consecutive 4byte word that we
                          * can read from current SG
@@ -281,6 +292,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq)
                                 oo = 0;
                         }
                 } else {
+                        char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */
+
                         /*
                          * read obl bytes in bufo, we read at maximum for
                          * emptying the device
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index fbc7bf9d7380..c9d686a0e805 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -265,11 +265,11 @@ static int init_device(struct device *dev)
  * callback must check err and feedback in descriptor header
  * for device processing status.
  */
-int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
-                   void (*callback)(struct device *dev,
-                                    struct talitos_desc *desc,
-                                    void *context, int error),
-                   void *context)
+static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
+                          void (*callback)(struct device *dev,
+                                           struct talitos_desc *desc,
+                                           void *context, int error),
+                          void *context)
 {
         struct talitos_private *priv = dev_get_drvdata(dev);
         struct talitos_request *request;
@@ -319,7 +319,21 @@ int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
 
         return -EINPROGRESS;
 }
-EXPORT_SYMBOL(talitos_submit);
+
+static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1)
+{
+        struct talitos_edesc *edesc;
+
+        if (!is_sec1)
+                return request->desc->hdr;
+
+        if (!request->desc->next_desc)
+                return request->desc->hdr1;
+
+        edesc = container_of(request->desc, struct talitos_edesc, desc);
+
+        return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1;
+}
 
 /*
  * process what was done, notify callback of error if not
@@ -342,12 +356,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 
                 /* descriptors with their done bits set don't get the error */
                 rmb();
-                if (!is_sec1)
-                        hdr = request->desc->hdr;
-                else if (request->desc->next_desc)
-                        hdr = (request->desc + 1)->hdr1;
-                else
-                        hdr = request->desc->hdr1;
+                hdr = get_request_hdr(request, is_sec1);
 
                 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
                         status = 0;
@@ -477,8 +486,14 @@ static u32 current_desc_hdr(struct device *dev, int ch)
                 }
         }
 
-        if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc)
-                return (priv->chan[ch].fifo[iter].desc + 1)->hdr;
+        if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) {
+                struct talitos_edesc *edesc;
+
+                edesc = container_of(priv->chan[ch].fifo[iter].desc,
+                                     struct talitos_edesc, desc);
+                return ((struct talitos_desc *)
+                        (edesc->buf + edesc->dma_len))->hdr;
+        }
 
         return priv->chan[ch].fifo[iter].desc->hdr;
 }
@@ -824,7 +839,11 @@ static void talitos_unregister_rng(struct device *dev)
  * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
  */
 #define TALITOS_CRA_PRIORITY_AEAD_HSNA  (TALITOS_CRA_PRIORITY - 1)
+#ifdef CONFIG_CRYPTO_DEV_TALITOS2
 #define TALITOS_MAX_KEY_SIZE            (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
+#else
+#define TALITOS_MAX_KEY_SIZE            (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE)
+#endif
 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 
 struct talitos_ctx {
@@ -948,36 +967,6 @@ badkey:
         goto out;
 }
 
-/*
- * talitos_edesc - s/w-extended descriptor
- * @src_nents: number of segments in input scatterlist
- * @dst_nents: number of segments in output scatterlist
- * @icv_ool: whether ICV is out-of-line
- * @iv_dma: dma address of iv for checking continuity and link table
- * @dma_len: length of dma mapped link_tbl space
- * @dma_link_tbl: bus physical address of link_tbl/buf
- * @desc: h/w descriptor
- * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
- * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
- *
- * if decrypting (with authcheck), or either one of src_nents or dst_nents
- * is greater than 1, an integrity check value is concatenated to the end
- * of link_tbl data
- */
-struct talitos_edesc {
-        int src_nents;
-        int dst_nents;
-        bool icv_ool;
-        dma_addr_t iv_dma;
-        int dma_len;
-        dma_addr_t dma_link_tbl;
-        struct talitos_desc desc;
-        union {
-                struct talitos_ptr link_tbl[0];
-                u8 buf[0];
-        };
-};
-
 static void talitos_sg_unmap(struct device *dev,
                              struct talitos_edesc *edesc,
                              struct scatterlist *src,
@@ -1008,11 +997,13 @@ static void talitos_sg_unmap(struct device *dev,
 
 static void ipsec_esp_unmap(struct device *dev,
                             struct talitos_edesc *edesc,
-                            struct aead_request *areq)
+                            struct aead_request *areq, bool encrypt)
 {
         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
         unsigned int ivsize = crypto_aead_ivsize(aead);
+        unsigned int authsize = crypto_aead_authsize(aead);
+        unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
         bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;
         struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];
 
@@ -1021,8 +1012,8 @@ static void ipsec_esp_unmap(struct device *dev,
                                          DMA_FROM_DEVICE);
         unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE);
 
-        talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
-                         areq->assoclen);
+        talitos_sg_unmap(dev, edesc, areq->src, areq->dst,
+                         cryptlen + authsize, areq->assoclen);
 
         if (edesc->dma_len)
                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
@@ -1032,7 +1023,7 @@ static void ipsec_esp_unmap(struct device *dev,
                 unsigned int dst_nents = edesc->dst_nents ? : 1;
 
                 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
-                                   areq->assoclen + areq->cryptlen - ivsize);
+                                   areq->assoclen + cryptlen - ivsize);
         }
 }
 
@@ -1043,31 +1034,14 @@ static void ipsec_esp_encrypt_done(struct device *dev,
                                    struct talitos_desc *desc, void *context,
                                    int err)
 {
-        struct talitos_private *priv = dev_get_drvdata(dev);
-        bool is_sec1 = has_ftr_sec1(priv);
         struct aead_request *areq = context;
         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
-        unsigned int authsize = crypto_aead_authsize(authenc);
         unsigned int ivsize = crypto_aead_ivsize(authenc);
         struct talitos_edesc *edesc;
-        struct scatterlist *sg;
-        void *icvdata;
 
         edesc = container_of(desc, struct talitos_edesc, desc);
 
-        ipsec_esp_unmap(dev, edesc, areq);
-
-        /* copy the generated ICV to dst */
-        if (edesc->icv_ool) {
-                if (is_sec1)
-                        icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
-                else
-                        icvdata = &edesc->link_tbl[edesc->src_nents +
-                                                   edesc->dst_nents + 2];
-                sg = sg_last(areq->dst, edesc->dst_nents);
-                memcpy((char *)sg_virt(sg) + sg->length - authsize,
-                       icvdata, authsize);
-        }
+        ipsec_esp_unmap(dev, edesc, areq, true);
 
         dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
 
@@ -1084,32 +1058,16 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev,
         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
         unsigned int authsize = crypto_aead_authsize(authenc);
         struct talitos_edesc *edesc;
-        struct scatterlist *sg;
         char *oicv, *icv;
-        struct talitos_private *priv = dev_get_drvdata(dev);
-        bool is_sec1 = has_ftr_sec1(priv);
 
         edesc = container_of(desc, struct talitos_edesc, desc);
 
-        ipsec_esp_unmap(dev, edesc, req);
+        ipsec_esp_unmap(dev, edesc, req, false);
 
         if (!err) {
                 /* auth check */
-                sg = sg_last(req->dst, edesc->dst_nents ? : 1);
-                icv = (char *)sg_virt(sg) + sg->length - authsize;
-
-                if (edesc->dma_len) {
-                        if (is_sec1)
-                                oicv = (char *)&edesc->dma_link_tbl +
-                                               req->assoclen + req->cryptlen;
-                        else
-                                oicv = (char *)
-                                       &edesc->link_tbl[edesc->src_nents +
-                                                        edesc->dst_nents + 2];
-                        if (edesc->icv_ool)
-                                icv = oicv + authsize;
-                } else
-                        oicv = (char *)&edesc->link_tbl[0];
+                oicv = edesc->buf + edesc->dma_len;
+                icv = oicv - authsize;
 
                 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
         }
@@ -1128,7 +1086,7 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
 
         edesc = container_of(desc, struct talitos_edesc, desc);
 
-        ipsec_esp_unmap(dev, edesc, req);
+        ipsec_esp_unmap(dev, edesc, req, false);
 
         /* check ICV auth status */
         if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
@@ -1145,11 +1103,12 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
  * stop at cryptlen bytes
  */
 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
-                                 unsigned int offset, int cryptlen,
+                                 unsigned int offset, int datalen, int elen,
                                  struct talitos_ptr *link_tbl_ptr)
 {
-        int n_sg = sg_count;
+        int n_sg = elen ? sg_count + 1 : sg_count;
         int count = 0;
+        int cryptlen = datalen + elen;
 
         while (cryptlen && sg && n_sg--) {
                 unsigned int len = sg_dma_len(sg);
@@ -1164,11 +1123,20 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
                 if (len > cryptlen)
                         len = cryptlen;
 
+                if (datalen > 0 && len > datalen) {
+                        to_talitos_ptr(link_tbl_ptr + count,
+                                       sg_dma_address(sg) + offset, datalen, 0);
+                        to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
+                        count++;
+                        len -= datalen;
+                        offset += datalen;
+                }
                 to_talitos_ptr(link_tbl_ptr + count,
                                sg_dma_address(sg) + offset, len, 0);
                 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
                 count++;
                 cryptlen -= len;
+                datalen -= len;
                 offset = 0;
 
 next:
@@ -1178,7 +1146,7 @@ next:
         /* tag end of link table */
         if (count > 0)
                 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
-                                       DESC_PTR_LNKTBL_RETURN, 0);
+                                       DESC_PTR_LNKTBL_RET, 0);
 
         return count;
 }
@@ -1186,7 +1154,8 @@ next:
 static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
                               unsigned int len, struct talitos_edesc *edesc,
                               struct talitos_ptr *ptr, int sg_count,
-                              unsigned int offset, int tbl_off, int elen)
+                              unsigned int offset, int tbl_off, int elen,
+                              bool force)
 {
         struct talitos_private *priv = dev_get_drvdata(dev);
         bool is_sec1 = has_ftr_sec1(priv);
@@ -1196,7 +1165,7 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
                 return 1;
         }
         to_talitos_ptr_ext_set(ptr, elen, is_sec1);
-        if (sg_count == 1) {
+        if (sg_count == 1 && !force) {
                 to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1);
                 return sg_count;
         }
@@ -1204,9 +1173,9 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
                 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1);
                 return sg_count;
         }
-        sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len + elen,
+        sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen,
                                          &edesc->link_tbl[tbl_off]);
-        if (sg_count == 1) {
+        if (sg_count == 1 && !force) {
                 /* Only one segment now, so no link tbl needed*/
                 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
                 return sg_count;
@@ -1224,13 +1193,14 @@ static int talitos_sg_map(struct device *dev, struct scatterlist *src,
                           unsigned int offset, int tbl_off)
 {
         return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,
-                                  tbl_off, 0);
+                                  tbl_off, 0, false);
 }
 
 /*
  * fill in and submit ipsec_esp descriptor
  */
 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
+                     bool encrypt,
                      void (*callback)(struct device *dev,
                                       struct talitos_desc *desc,
                                       void *context, int error))
@@ -1240,7 +1210,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
         struct device *dev = ctx->dev;
         struct talitos_desc *desc = &edesc->desc;
-        unsigned int cryptlen = areq->cryptlen;
+        unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
         unsigned int ivsize = crypto_aead_ivsize(aead);
         int tbl_off = 0;
         int sg_count, ret;
@@ -1251,6 +1221,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
         bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP;
         struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3];
         struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2];
+        dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize;
 
         /* hmac key */
         to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1);
@@ -1290,7 +1261,8 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
                 elen = authsize;
 
         ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],
-                                 sg_count, areq->assoclen, tbl_off, elen);
+                                 sg_count, areq->assoclen, tbl_off, elen,
+                                 false);
 
         if (ret > 1) {
                 tbl_off += ret;
@@ -1304,55 +1276,32 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
                         dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
         }
 
-        ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
-                             sg_count, areq->assoclen, tbl_off);
-
-        if (is_ipsec_esp)
-                to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
-
-        /* ICV data */
-        if (ret > 1) {
-                tbl_off += ret;
-                edesc->icv_ool = true;
-                sync_needed = true;
-
-                if (is_ipsec_esp) {
-                        struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
-                        int offset = (edesc->src_nents + edesc->dst_nents + 2) *
-                                     sizeof(struct talitos_ptr) + authsize;
-
-                        /* Add an entry to the link table for ICV data */
-                        to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
-                        to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
-                                               is_sec1);
+        if (is_ipsec_esp && encrypt)
+                elen = authsize;
+        else
+                elen = 0;
+        ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
+                                 sg_count, areq->assoclen, tbl_off, elen,
+                                 is_ipsec_esp && !encrypt);
+        tbl_off += ret;
 
-                        /* icv data follows link tables */
-                        to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
-                                       authsize, is_sec1);
-                } else {
-                        dma_addr_t addr = edesc->dma_link_tbl;
+        if (!encrypt && is_ipsec_esp) {
+                struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
 
-                        if (is_sec1)
-                                addr += areq->assoclen + cryptlen;
-                        else
-                                addr += sizeof(struct talitos_ptr) * tbl_off;
+                /* Add an entry to the link table for ICV data */
+                to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
+                to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1);
 
-                        to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1);
-                }
+                /* icv data follows link tables */
+                to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1);
+                to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
+                sync_needed = true;
+        } else if (!encrypt) {
+                to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1);
+                sync_needed = true;
         } else if (!is_ipsec_esp) {
-                ret = talitos_sg_map(dev, areq->dst, authsize, edesc,
-                                     &desc->ptr[6], sg_count, areq->assoclen +
-                                                              cryptlen,
-                                     tbl_off);
-                if (ret > 1) {
-                        tbl_off += ret;
-                        edesc->icv_ool = true;
-                        sync_needed = true;
-                } else {
-                        edesc->icv_ool = false;
-                }
-        } else {
-                edesc->icv_ool = false;
+                talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6],
+                               sg_count, areq->assoclen + cryptlen, tbl_off);
         }
 
         /* iv out */
@@ -1367,7 +1316,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
 
         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
         if (ret != -EINPROGRESS) {
-                ipsec_esp_unmap(dev, edesc, areq);
+                ipsec_esp_unmap(dev, edesc, areq, encrypt);
                 kfree(edesc);
         }
         return ret;
@@ -1435,18 +1384,18 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
          * and space for two sets of ICVs (stashed and generated)
          */
         alloc_len = sizeof(struct talitos_edesc);
-        if (src_nents || dst_nents) {
+        if (src_nents || dst_nents || !encrypt) {
                 if (is_sec1)
                         dma_len = (src_nents ? src_len : 0) +
-                                  (dst_nents ? dst_len : 0);
+                                  (dst_nents ? dst_len : 0) + authsize;
                 else
                         dma_len = (src_nents + dst_nents + 2) *
-                                  sizeof(struct talitos_ptr) + authsize * 2;
+                                  sizeof(struct talitos_ptr) + authsize;
                 alloc_len += dma_len;
         } else {
                 dma_len = 0;
-                alloc_len += icv_stashing ? authsize : 0;
         }
+        alloc_len += icv_stashing ? authsize : 0;
 
         /* if its a ahash, add space for a second desc next to the first one */
         if (is_sec1 && !dst)
@@ -1466,15 +1415,11 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
         edesc->dst_nents = dst_nents;
         edesc->iv_dma = iv_dma;
         edesc->dma_len = dma_len;
-        if (dma_len) {
-                void *addr = &edesc->link_tbl[0];
-
-                if (is_sec1 && !dst)
-                        addr += sizeof(struct talitos_desc);
-                edesc->dma_link_tbl = dma_map_single(dev, addr,
+        if (dma_len)
+                edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
                                                      edesc->dma_len,
                                                      DMA_BIDIRECTIONAL);
-        }
+
         return edesc;
 }
 
@@ -1485,9 +1430,10 @@ static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
         unsigned int authsize = crypto_aead_authsize(authenc);
         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
         unsigned int ivsize = crypto_aead_ivsize(authenc);
+        unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize);
 
         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
-                                   iv, areq->assoclen, areq->cryptlen,
+                                   iv, areq->assoclen, cryptlen,
                                    authsize, ivsize, icv_stashing,
                                    areq->base.flags, encrypt);
 }
@@ -1506,7 +1452,7 @@ static int aead_encrypt(struct aead_request *req)
         /* set encrypt */
         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
 
-        return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
+        return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done);
 }
 
 static int aead_decrypt(struct aead_request *req)
@@ -1516,17 +1462,15 @@ static int aead_decrypt(struct aead_request *req)
         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
         struct talitos_edesc *edesc;
-        struct scatterlist *sg;
         void *icvdata;
 
-        req->cryptlen -= authsize;
-
         /* allocate extended descriptor */
         edesc = aead_edesc_alloc(req, req->iv, 1, false);
         if (IS_ERR(edesc))
                 return PTR_ERR(edesc);
 
-        if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
+        if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) &&
+            (priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
             ((!edesc->src_nents && !edesc->dst_nents) ||
              priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
 
@@ -1537,24 +1481,20 @@ static int aead_decrypt(struct aead_request *req)
 
                 /* reset integrity check result bits */
 
-                return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
+                return ipsec_esp(edesc, req, false,
+                                 ipsec_esp_decrypt_hwauth_done);
         }
 
         /* Have to check the ICV with software */
         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
 
         /* stash incoming ICV for later cmp with ICV generated by the h/w */
-        if (edesc->dma_len)
-                icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
-                                                   edesc->dst_nents + 2];
-        else
-                icvdata = &edesc->link_tbl[0];
+        icvdata = edesc->buf + edesc->dma_len;
 
-        sg = sg_last(req->src, edesc->src_nents ? : 1);
+        sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize,
+                           req->assoclen + req->cryptlen - authsize);
 
-        memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
-
-        return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
+        return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done);
 }
 
 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
@@ -1605,6 +1545,18 @@ static int ablkcipher_des3_setkey(struct crypto_ablkcipher *cipher,
         return ablkcipher_setkey(cipher, key, keylen);
 }
 
+static int ablkcipher_aes_setkey(struct crypto_ablkcipher *cipher,
+                                  const u8 *key, unsigned int keylen)
+{
+        if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 ||
+            keylen == AES_KEYSIZE_256)
+                return ablkcipher_setkey(cipher, key, keylen);
+
+        crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+
+        return -EINVAL;
+}
+
 static void common_nonsnoop_unmap(struct device *dev,
                                   struct talitos_edesc *edesc,
                                   struct ablkcipher_request *areq)
@@ -1624,11 +1576,15 @@ static void ablkcipher_done(struct device *dev,
                             int err)
 {
         struct ablkcipher_request *areq = context;
+        struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
+        struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
+        unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
         struct talitos_edesc *edesc;
 
         edesc = container_of(desc, struct talitos_edesc, desc);
 
         common_nonsnoop_unmap(dev, edesc, areq);
+        memcpy(areq->info, ctx->iv, ivsize);
 
         kfree(edesc);
 
@@ -1723,6 +1679,14 @@ static int ablkcipher_encrypt(struct ablkcipher_request *areq)
         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
         struct talitos_edesc *edesc;
+        unsigned int blocksize =
+                        crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher));
+
+        if (!areq->nbytes)
+                return 0;
+
+        if (areq->nbytes % blocksize)
+                return -EINVAL;
 
         /* allocate extended descriptor */
         edesc = ablkcipher_edesc_alloc(areq, true);
@@ -1740,6 +1704,14 @@ static int ablkcipher_decrypt(struct ablkcipher_request *areq)
         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
         struct talitos_edesc *edesc;
+        unsigned int blocksize =
+                        crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher));
+
+        if (!areq->nbytes)
+                return 0;
+
+        if (areq->nbytes % blocksize)
+                return -EINVAL;
 
         /* allocate extended descriptor */
         edesc = ablkcipher_edesc_alloc(areq, false);
@@ -1759,14 +1731,16 @@ static void common_nonsnoop_hash_unmap(struct device *dev,
         struct talitos_private *priv = dev_get_drvdata(dev);
         bool is_sec1 = has_ftr_sec1(priv);
         struct talitos_desc *desc = &edesc->desc;
-        struct talitos_desc *desc2 = desc + 1;
+        struct talitos_desc *desc2 = (struct talitos_desc *)
+                                     (edesc->buf + edesc->dma_len);
 
         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
         if (desc->next_desc &&
             desc->ptr[5].ptr != desc2->ptr[5].ptr)
                 unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE);
 
-        talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
+        if (req_ctx->psrc)
+                talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
 
         /* When using hashctx-in, must unmap it. */
         if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
@@ -1833,7 +1807,6 @@ static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
 
 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
                                 struct ahash_request *areq, unsigned int length,
-                                unsigned int offset,
                                 void (*callback) (struct device *dev,
                                                   struct talitos_desc *desc,
                                                   void *context, int error))
@@ -1872,9 +1845,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
 
         sg_count = edesc->src_nents ?: 1;
         if (is_sec1 && sg_count > 1)
-                sg_pcopy_to_buffer(req_ctx->psrc, sg_count,
-                                   edesc->buf + sizeof(struct talitos_desc),
-                                   length, req_ctx->nbuf);
+                sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length);
         else if (length)
                 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
                                       DMA_TO_DEVICE);
@@ -1887,7 +1858,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
                                        DMA_TO_DEVICE);
         } else {
                 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
-                                          &desc->ptr[3], sg_count, offset, 0);
+                                          &desc->ptr[3], sg_count, 0, 0);
                 if (sg_count > 1)
                         sync_needed = true;
         }
@@ -1911,7 +1882,8 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
                 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
 
         if (is_sec1 && req_ctx->nbuf && length) {
-                struct talitos_desc *desc2 = desc + 1;
+                struct talitos_desc *desc2 = (struct talitos_desc *)
+                                             (edesc->buf + edesc->dma_len);
                 dma_addr_t next_desc;
 
                 memset(desc2, 0, sizeof(*desc2));
@@ -1932,7 +1904,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
                                                       DMA_TO_DEVICE);
                 copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1);
                 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
-                                          &desc2->ptr[3], sg_count, offset, 0);
+                                          &desc2->ptr[3], sg_count, 0, 0);
                 if (sg_count > 1)
                         sync_needed = true;
                 copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1);
@@ -2043,7 +2015,6 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
         struct device *dev = ctx->dev;
         struct talitos_private *priv = dev_get_drvdata(dev);
         bool is_sec1 = has_ftr_sec1(priv);
-        int offset = 0;
         u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
 
         if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
@@ -2083,6 +2054,8 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
                         sg_chain(req_ctx->bufsl, 2, areq->src);
                 req_ctx->psrc = req_ctx->bufsl;
         } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
+                int offset;
+
                 if (nbytes_to_hash > blocksize)
                         offset = blocksize - req_ctx->nbuf;
                 else
@@ -2095,7 +2068,8 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
                 sg_copy_to_buffer(areq->src, nents,
                                   ctx_buf + req_ctx->nbuf, offset);
                 req_ctx->nbuf += offset;
-                req_ctx->psrc = areq->src;
+                req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src,
+                                                 offset);
         } else
                 req_ctx->psrc = areq->src;
 
@@ -2135,8 +2109,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
         if (ctx->keylen && (req_ctx->first || req_ctx->last))
                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
 
-        return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset,
-                                    ahash_done);
+        return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done);
 }
 
 static int ahash_update(struct ahash_request *areq)
@@ -2339,7 +2312,7 @@ static struct talitos_alg_template driver_algs[] = {
                         .base = {
                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
                                 .cra_driver_name = "authenc-hmac-sha1-"
-                                                   "cbc-aes-talitos",
+                                                   "cbc-aes-talitos-hsna",
                                 .cra_blocksize = AES_BLOCK_SIZE,
                                 .cra_flags = CRYPTO_ALG_ASYNC,
                         },
@@ -2384,7 +2357,7 @@ static struct talitos_alg_template driver_algs[] = {
                                 .cra_name = "authenc(hmac(sha1),"
                                             "cbc(des3_ede))",
                                 .cra_driver_name = "authenc-hmac-sha1-"
-                                                   "cbc-3des-talitos",
+                                                   "cbc-3des-talitos-hsna",
                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
                                 .cra_flags = CRYPTO_ALG_ASYNC,
                         },
@@ -2427,7 +2400,7 @@ static struct talitos_alg_template driver_algs[] = {
                         .base = {
                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
                                 .cra_driver_name = "authenc-hmac-sha224-"
-                                                   "cbc-aes-talitos",
+                                                   "cbc-aes-talitos-hsna",
                                 .cra_blocksize = AES_BLOCK_SIZE,
                                 .cra_flags = CRYPTO_ALG_ASYNC,
                         },
@@ -2472,7 +2445,7 @@ static struct talitos_alg_template driver_algs[] = {
                                 .cra_name = "authenc(hmac(sha224),"
                                             "cbc(des3_ede))",
                                 .cra_driver_name = "authenc-hmac-sha224-"
-                                                   "cbc-3des-talitos",
+                                                   "cbc-3des-talitos-hsna",
                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
                                 .cra_flags = CRYPTO_ALG_ASYNC,
                         },
@@ -2515,7 +2488,7 @@ static struct talitos_alg_template driver_algs[] = {
                         .base = {
                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
                                 .cra_driver_name = "authenc-hmac-sha256-"
-                                                   "cbc-aes-talitos",
+                                                   "cbc-aes-talitos-hsna",
                                 .cra_blocksize = AES_BLOCK_SIZE,
                                 .cra_flags = CRYPTO_ALG_ASYNC,
                         },
@@ -2560,7 +2533,7 @@ static struct talitos_alg_template driver_algs[] = {
                                 .cra_name = "authenc(hmac(sha256),"
                                             "cbc(des3_ede))",
                                 .cra_driver_name = "authenc-hmac-sha256-"
-                                                   "cbc-3des-talitos",
+                                                   "cbc-3des-talitos-hsna",
                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
                                 .cra_flags = CRYPTO_ALG_ASYNC,
                         },
@@ -2689,7 +2662,7 @@ static struct talitos_alg_template driver_algs[] = {
                         .base = {
                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
                                 .cra_driver_name = "authenc-hmac-md5-"
-                                                   "cbc-aes-talitos",
+                                                   "cbc-aes-talitos-hsna",
                                 .cra_blocksize = AES_BLOCK_SIZE,
                                 .cra_flags = CRYPTO_ALG_ASYNC,
                         },
@@ -2732,7 +2705,7 @@ static struct talitos_alg_template driver_algs[] = {
                         .base = {
                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
                                 .cra_driver_name = "authenc-hmac-md5-"
-                                                   "cbc-3des-talitos",
+                                                   "cbc-3des-talitos-hsna",
                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
                                 .cra_flags = CRYPTO_ALG_ASYNC,
                         },
@@ -2760,7 +2733,7 @@ static struct talitos_alg_template driver_algs[] = {
                         .cra_ablkcipher = {
                                 .min_keysize = AES_MIN_KEY_SIZE,
                                 .max_keysize = AES_MAX_KEY_SIZE,
-                                .ivsize = AES_BLOCK_SIZE,
+                                .setkey = ablkcipher_aes_setkey,
                         }
                 },
                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2777,6 +2750,7 @@ static struct talitos_alg_template driver_algs[] = {
                                 .min_keysize = AES_MIN_KEY_SIZE,
                                 .max_keysize = AES_MAX_KEY_SIZE,
                                 .ivsize = AES_BLOCK_SIZE,
+                                .setkey = ablkcipher_aes_setkey,
                         }
                 },
                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
@@ -2787,13 +2761,14 @@ static struct talitos_alg_template driver_algs[] = {
                 .alg.crypto = {
                         .cra_name = "ctr(aes)",
                         .cra_driver_name = "ctr-aes-talitos",
-                        .cra_blocksize = AES_BLOCK_SIZE,
+                        .cra_blocksize = 1,
                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
                                      CRYPTO_ALG_ASYNC,
                         .cra_ablkcipher = {
                                 .min_keysize = AES_MIN_KEY_SIZE,
                                 .max_keysize = AES_MAX_KEY_SIZE,
                                 .ivsize = AES_BLOCK_SIZE,
+                                .setkey = ablkcipher_aes_setkey,
                         }
                 },
                 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
@@ -2810,7 +2785,6 @@ static struct talitos_alg_template driver_algs[] = {
                         .cra_ablkcipher = {
                                 .min_keysize = DES_KEY_SIZE,
                                 .max_keysize = DES_KEY_SIZE,
-                                .ivsize = DES_BLOCK_SIZE,
                                 .setkey = ablkcipher_des_setkey,
                         }
                 },
@@ -2845,7 +2819,6 @@ static struct talitos_alg_template driver_algs[] = {
                         .cra_ablkcipher = {
                                 .min_keysize = DES3_EDE_KEY_SIZE,
                                 .max_keysize = DES3_EDE_KEY_SIZE,
-                                .ivsize = DES3_EDE_BLOCK_SIZE,
                                 .setkey = ablkcipher_des3_setkey,
                         }
                 },
@@ -3270,7 +3243,10 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
                 alg->cra_priority = t_alg->algt.priority;
         else
                 alg->cra_priority = TALITOS_CRA_PRIORITY;
-        alg->cra_alignmask = 0;
+        if (has_ftr_sec1(priv))
+                alg->cra_alignmask = 3;
+        else
+                alg->cra_alignmask = 0;
         alg->cra_ctxsize = sizeof(struct talitos_ctx);
         alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
 
@@ -3418,7 +3394,7 @@ static int talitos_probe(struct platform_device *ofdev)
         if (err)
                 goto err_out;
 
-        if (of_device_is_compatible(np, "fsl,sec1.0")) {
+        if (has_ftr_sec1(priv)) {
                 if (priv->num_channels == 1)
                         tasklet_init(&priv->done_task[0], talitos1_done_ch0,
                                      (unsigned long)dev);
diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h
index a65a63e0d6c1..1469b956948a 100644
--- a/drivers/crypto/talitos.h
+++ b/drivers/crypto/talitos.h
@@ -1,31 +1,8 @@
+/* SPDX-License-Identifier: BSD-3-Clause */
 /*
  * Freescale SEC (talitos) device register and descriptor header defines
  *
  * Copyright (c) 2006-2011 Freescale Semiconductor, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote products
- *    derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
  */
 
 #define TALITOS_TIMEOUT 100000
@@ -65,6 +42,34 @@ struct talitos_desc {
 
 #define TALITOS_DESC_SIZE       (sizeof(struct talitos_desc) - sizeof(__be32))
 
+/*
+ * talitos_edesc - s/w-extended descriptor
+ * @src_nents: number of segments in input scatterlist
+ * @dst_nents: number of segments in output scatterlist
+ * @iv_dma: dma address of iv for checking continuity and link table
+ * @dma_len: length of dma mapped link_tbl space
+ * @dma_link_tbl: bus physical address of link_tbl/buf
+ * @desc: h/w descriptor
+ * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
+ * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
+ *
+ * if decrypting (with authcheck), or either one of src_nents or dst_nents
+ * is greater than 1, an integrity check value is concatenated to the end
+ * of link_tbl data
+ */
+struct talitos_edesc {
+        int src_nents;
+        int dst_nents;
+        dma_addr_t iv_dma;
+        int dma_len;
+        dma_addr_t dma_link_tbl;
+        struct talitos_desc desc;
+        union {
+                struct talitos_ptr link_tbl[0];
+                u8 buf[0];
+        };
+};
+
 /**
  * talitos_request - descriptor submission request
  * @desc: descriptor pointer (kernel virtual)
@@ -150,12 +155,6 @@ struct talitos_private {
         bool rng_registered;
 };
 
-extern int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
-                          void (*callback)(struct device *dev,
-                                           struct talitos_desc *desc,
-                                           void *context, int error),
-                          void *context);
-
 /* .features flag */
 #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
 #define TALITOS_FTR_HW_AUTH_CHECK 0x00000002
@@ -170,13 +169,11 @@ extern int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
  */
 static inline bool has_ftr_sec1(struct talitos_private *priv)
 {
-#if defined(CONFIG_CRYPTO_DEV_TALITOS1) && defined(CONFIG_CRYPTO_DEV_TALITOS2)
-        return priv->features & TALITOS_FTR_SEC1 ? true : false;
-#elif defined(CONFIG_CRYPTO_DEV_TALITOS1)
-        return true;
-#else
-        return false;
-#endif
+        if (IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS1) &&
+            IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS2))
+                return priv->features & TALITOS_FTR_SEC1;
+
+        return IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS1);
 }
 
 /*
@@ -412,5 +409,5 @@ static inline bool has_ftr_sec1(struct talitos_private *priv)
 
 /* link table extent field bits */
 #define DESC_PTR_LNKTBL_JUMP                    0x80
-#define DESC_PTR_LNKTBL_RETURN                  0x02
+#define DESC_PTR_LNKTBL_RET                     0x02
 #define DESC_PTR_LNKTBL_NEXT                    0x01
diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c
index c7e515a1bc97..d88084447f1c 100644
--- a/drivers/crypto/vmx/aes_cbc.c
+++ b/drivers/crypto/vmx/aes_cbc.c
@@ -7,64 +7,52 @@
  * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
  */
 
-#include <linux/types.h>
-#include <linux/err.h>
-#include <linux/crypto.h>
-#include <linux/delay.h>
 #include <asm/simd.h>
 #include <asm/switch_to.h>
 #include <crypto/aes.h>
 #include <crypto/internal/simd.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/skcipher.h>
+#include <crypto/internal/skcipher.h>
 
 #include "aesp8-ppc.h"
 
 struct p8_aes_cbc_ctx {
-        struct crypto_sync_skcipher *fallback;
+        struct crypto_skcipher *fallback;
         struct aes_key enc_key;
         struct aes_key dec_key;
 };
 
-static int p8_aes_cbc_init(struct crypto_tfm *tfm)
+static int p8_aes_cbc_init(struct crypto_skcipher *tfm)
 {
-        const char *alg = crypto_tfm_alg_name(tfm);
-        struct crypto_sync_skcipher *fallback;
-        struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
-
-        fallback = crypto_alloc_sync_skcipher(alg, 0,
-                                              CRYPTO_ALG_NEED_FALLBACK);
+        struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+        struct crypto_skcipher *fallback;
 
+        fallback = crypto_alloc_skcipher("cbc(aes)", 0,
+                                         CRYPTO_ALG_NEED_FALLBACK |
+                                         CRYPTO_ALG_ASYNC);
         if (IS_ERR(fallback)) {
-                printk(KERN_ERR
-                       "Failed to allocate transformation for '%s': %ld\n",
-                       alg, PTR_ERR(fallback));
+                pr_err("Failed to allocate cbc(aes) fallback: %ld\n",
+                       PTR_ERR(fallback));
                 return PTR_ERR(fallback);
         }
 
-        crypto_sync_skcipher_set_flags(
-                fallback,
-                crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
+        crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
+                                    crypto_skcipher_reqsize(fallback));
         ctx->fallback = fallback;
-
         return 0;
 }
 
-static void p8_aes_cbc_exit(struct crypto_tfm *tfm)
+static void p8_aes_cbc_exit(struct crypto_skcipher *tfm)
 {
-        struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 
-        if (ctx->fallback) {
-                crypto_free_sync_skcipher(ctx->fallback);
-                ctx->fallback = NULL;
-        }
+        crypto_free_skcipher(ctx->fallback);
 }
 
-static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key,
+static int p8_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
                              unsigned int keylen)
 {
+        struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
         int ret;
-        struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
 
         preempt_disable();
         pagefault_disable();
@@ -75,108 +63,71 @@ static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key,
         pagefault_enable();
         preempt_enable();
 
-        ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
+        ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen);
 
         return ret ? -EINVAL : 0;
 }
 
-static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc,
-                              struct scatterlist *dst,
-                              struct scatterlist *src, unsigned int nbytes)
+static int p8_aes_cbc_crypt(struct skcipher_request *req, int enc)
 {
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+        const struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+        struct skcipher_walk walk;
+        unsigned int nbytes;
         int ret;
-        struct blkcipher_walk walk;
-        struct p8_aes_cbc_ctx *ctx =
-                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
 
         if (!crypto_simd_usable()) {
-                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
-                skcipher_request_set_sync_tfm(req, ctx->fallback);
-                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
-                skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
-                ret = crypto_skcipher_encrypt(req);
-                skcipher_request_zero(req);
-        } else {
-                blkcipher_walk_init(&walk, dst, src, nbytes);
-                ret = blkcipher_walk_virt(desc, &walk);
-                while ((nbytes = walk.nbytes)) {
-                        preempt_disable();
-                        pagefault_disable();
-                        enable_kernel_vsx();
-                        aes_p8_cbc_encrypt(walk.src.virt.addr,
-                                           walk.dst.virt.addr,
-                                           nbytes & AES_BLOCK_MASK,
-                                           &ctx->enc_key, walk.iv, 1);
-                        disable_kernel_vsx();
-                        pagefault_enable();
-                        preempt_enable();
-
-                        nbytes &= AES_BLOCK_SIZE - 1;
-                        ret = blkcipher_walk_done(desc, &walk, nbytes);
-                }
+                struct skcipher_request *subreq = skcipher_request_ctx(req);
+
+                *subreq = *req;
+                skcipher_request_set_tfm(subreq, ctx->fallback);
+                return enc ? crypto_skcipher_encrypt(subreq) :
+                             crypto_skcipher_decrypt(subreq);
         }
 
+        ret = skcipher_walk_virt(&walk, req, false);
+        while ((nbytes = walk.nbytes) != 0) {
+                preempt_disable();
+                pagefault_disable();
+                enable_kernel_vsx();
+                aes_p8_cbc_encrypt(walk.src.virt.addr,
+                                   walk.dst.virt.addr,
+                                   round_down(nbytes, AES_BLOCK_SIZE),
+                                   enc ? &ctx->enc_key : &ctx->dec_key,
+                                   walk.iv, enc);
+                disable_kernel_vsx();
+                pagefault_enable();
+                preempt_enable();
+
+                ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
+        }
         return ret;
 }
 
-static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc,
-                              struct scatterlist *dst,
-                              struct scatterlist *src, unsigned int nbytes)
+static int p8_aes_cbc_encrypt(struct skcipher_request *req)
 {
-        int ret;
-        struct blkcipher_walk walk;
-        struct p8_aes_cbc_ctx *ctx =
-                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
-
-        if (!crypto_simd_usable()) {
-                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
-                skcipher_request_set_sync_tfm(req, ctx->fallback);
-                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
-                skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
-                ret = crypto_skcipher_decrypt(req);
-                skcipher_request_zero(req);
-        } else {
-                blkcipher_walk_init(&walk, dst, src, nbytes);
-                ret = blkcipher_walk_virt(desc, &walk);
-                while ((nbytes = walk.nbytes)) {
-                        preempt_disable();
-                        pagefault_disable();
-                        enable_kernel_vsx();
-                        aes_p8_cbc_encrypt(walk.src.virt.addr,
-                                           walk.dst.virt.addr,
-                                           nbytes & AES_BLOCK_MASK,
-                                           &ctx->dec_key, walk.iv, 0);
-                        disable_kernel_vsx();
-                        pagefault_enable();
-                        preempt_enable();
-
-                        nbytes &= AES_BLOCK_SIZE - 1;
-                        ret = blkcipher_walk_done(desc, &walk, nbytes);
-                }
-        }
-
-        return ret;
+        return p8_aes_cbc_crypt(req, 1);
 }
 
+static int p8_aes_cbc_decrypt(struct skcipher_request *req)
+{
+        return p8_aes_cbc_crypt(req, 0);
+}
 
-struct crypto_alg p8_aes_cbc_alg = {
-        .cra_name = "cbc(aes)",
-        .cra_driver_name = "p8_aes_cbc",
-        .cra_module = THIS_MODULE,
-        .cra_priority = 2000,
-        .cra_type = &crypto_blkcipher_type,
-        .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
-        .cra_alignmask = 0,
-        .cra_blocksize = AES_BLOCK_SIZE,
-        .cra_ctxsize = sizeof(struct p8_aes_cbc_ctx),
-        .cra_init = p8_aes_cbc_init,
-        .cra_exit = p8_aes_cbc_exit,
-        .cra_blkcipher = {
-                          .ivsize = AES_BLOCK_SIZE,
-                          .min_keysize = AES_MIN_KEY_SIZE,
-                          .max_keysize = AES_MAX_KEY_SIZE,
-                          .setkey = p8_aes_cbc_setkey,
-                          .encrypt = p8_aes_cbc_encrypt,
-                          .decrypt = p8_aes_cbc_decrypt,
-        },
+struct skcipher_alg p8_aes_cbc_alg = {
+        .base.cra_name = "cbc(aes)",
+        .base.cra_driver_name = "p8_aes_cbc",
+        .base.cra_module = THIS_MODULE,
+        .base.cra_priority = 2000,
+        .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+        .base.cra_blocksize = AES_BLOCK_SIZE,
+        .base.cra_ctxsize = sizeof(struct p8_aes_cbc_ctx),
+        .setkey = p8_aes_cbc_setkey,
+        .encrypt = p8_aes_cbc_encrypt,
+        .decrypt = p8_aes_cbc_decrypt,
+        .init = p8_aes_cbc_init,
+        .exit = p8_aes_cbc_exit,
+        .min_keysize = AES_MIN_KEY_SIZE,
+        .max_keysize = AES_MAX_KEY_SIZE,
+        .ivsize = AES_BLOCK_SIZE,
 };
diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c
index dd017ef42fa9..79ba062ee1c1 100644
--- a/drivers/crypto/vmx/aes_ctr.c
+++ b/drivers/crypto/vmx/aes_ctr.c
@@ -7,62 +7,51 @@
  * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
  */
 
-#include <linux/types.h>
-#include <linux/err.h>
-#include <linux/crypto.h>
-#include <linux/delay.h>
 #include <asm/simd.h>
 #include <asm/switch_to.h>
 #include <crypto/aes.h>
 #include <crypto/internal/simd.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/skcipher.h>
+#include <crypto/internal/skcipher.h>
 
 #include "aesp8-ppc.h"
 
 struct p8_aes_ctr_ctx {
-        struct crypto_sync_skcipher *fallback;
+        struct crypto_skcipher *fallback;
         struct aes_key enc_key;
 };
 
-static int p8_aes_ctr_init(struct crypto_tfm *tfm)
+static int p8_aes_ctr_init(struct crypto_skcipher *tfm)
 {
-        const char *alg = crypto_tfm_alg_name(tfm);
-        struct crypto_sync_skcipher *fallback;
-        struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
+        struct crypto_skcipher *fallback;
 
-        fallback = crypto_alloc_sync_skcipher(alg, 0,
-                                              CRYPTO_ALG_NEED_FALLBACK);
+        fallback = crypto_alloc_skcipher("ctr(aes)", 0,
+                                         CRYPTO_ALG_NEED_FALLBACK |
+                                         CRYPTO_ALG_ASYNC);
         if (IS_ERR(fallback)) {
-                printk(KERN_ERR
-                       "Failed to allocate transformation for '%s': %ld\n",
-                       alg, PTR_ERR(fallback));
+                pr_err("Failed to allocate ctr(aes) fallback: %ld\n",
+                       PTR_ERR(fallback));
                 return PTR_ERR(fallback);
         }
 
-        crypto_sync_skcipher_set_flags(
-                fallback,
-                crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
+        crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
+                                    crypto_skcipher_reqsize(fallback));
         ctx->fallback = fallback;
-
         return 0;
 }
 
-static void p8_aes_ctr_exit(struct crypto_tfm *tfm)
+static void p8_aes_ctr_exit(struct crypto_skcipher *tfm)
 {
-        struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
 
-        if (ctx->fallback) {
-                crypto_free_sync_skcipher(ctx->fallback);
-                ctx->fallback = NULL;
-        }
+        crypto_free_skcipher(ctx->fallback);
 }
 
-static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key,
+static int p8_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key,
                              unsigned int keylen)
 {
+        struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
         int ret;
-        struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
 
         preempt_disable();
         pagefault_disable();
@@ -72,13 +61,13 @@ static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key,
         pagefault_enable();
         preempt_enable();
 
-        ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
+        ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen);
 
         return ret ? -EINVAL : 0;
 }
 
-static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx,
-                             struct blkcipher_walk *walk)
+static void p8_aes_ctr_final(const struct p8_aes_ctr_ctx *ctx,
+                             struct skcipher_walk *walk)
 {
         u8 *ctrblk = walk->iv;
         u8 keystream[AES_BLOCK_SIZE];
@@ -98,77 +87,63 @@ static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx,
         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 }
 
-static int p8_aes_ctr_crypt(struct blkcipher_desc *desc,
-                            struct scatterlist *dst,
-                            struct scatterlist *src, unsigned int nbytes)
+static int p8_aes_ctr_crypt(struct skcipher_request *req)
 {
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+        const struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm);
+        struct skcipher_walk walk;
+        unsigned int nbytes;
         int ret;
-        u64 inc;
-        struct blkcipher_walk walk;
-        struct p8_aes_ctr_ctx *ctx =
-                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
 
         if (!crypto_simd_usable()) {
-                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
-                skcipher_request_set_sync_tfm(req, ctx->fallback);
-                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
-                skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
-                ret = crypto_skcipher_encrypt(req);
-                skcipher_request_zero(req);
-        } else {
-                blkcipher_walk_init(&walk, dst, src, nbytes);
-                ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
-                while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
-                        preempt_disable();
-                        pagefault_disable();
-                        enable_kernel_vsx();
-                        aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr,
-                                                    walk.dst.virt.addr,
-                                                    (nbytes &
-                                                     AES_BLOCK_MASK) /
-                                                    AES_BLOCK_SIZE,
-                                                    &ctx->enc_key,
-                                                    walk.iv);
-                        disable_kernel_vsx();
-                        pagefault_enable();
-                        preempt_enable();
-
-                        /* We need to update IV mostly for last bytes/round */
-                        inc = (nbytes & AES_BLOCK_MASK) / AES_BLOCK_SIZE;
-                        if (inc > 0)
-                                while (inc--)
-                                        crypto_inc(walk.iv, AES_BLOCK_SIZE);
-
-                        nbytes &= AES_BLOCK_SIZE - 1;
-                        ret = blkcipher_walk_done(desc, &walk, nbytes);
-                }
-                if (walk.nbytes) {
-                        p8_aes_ctr_final(ctx, &walk);
-                        ret = blkcipher_walk_done(desc, &walk, 0);
-                }
+                struct skcipher_request *subreq = skcipher_request_ctx(req);
+
+                *subreq = *req;
+                skcipher_request_set_tfm(subreq, ctx->fallback);
+                return crypto_skcipher_encrypt(subreq);
         }
 
+        ret = skcipher_walk_virt(&walk, req, false);
+        while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
+                preempt_disable();
+                pagefault_disable();
+                enable_kernel_vsx();
+                aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr,
+                                            walk.dst.virt.addr,
+                                            nbytes / AES_BLOCK_SIZE,
+                                            &ctx->enc_key, walk.iv);
+                disable_kernel_vsx();
+                pagefault_enable();
+                preempt_enable();
+
+                do {
+                        crypto_inc(walk.iv, AES_BLOCK_SIZE);
+                } while ((nbytes -= AES_BLOCK_SIZE) >= AES_BLOCK_SIZE);
+
+                ret = skcipher_walk_done(&walk, nbytes);
+        }
+        if (nbytes) {
+                p8_aes_ctr_final(ctx, &walk);
+                ret = skcipher_walk_done(&walk, 0);
+        }
         return ret;
 }
 
-struct crypto_alg p8_aes_ctr_alg = {
-        .cra_name = "ctr(aes)",
-        .cra_driver_name = "p8_aes_ctr",
-        .cra_module = THIS_MODULE,
-        .cra_priority = 2000,
-        .cra_type = &crypto_blkcipher_type,
-        .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
-        .cra_alignmask = 0,
-        .cra_blocksize = 1,
-        .cra_ctxsize = sizeof(struct p8_aes_ctr_ctx),
-        .cra_init = p8_aes_ctr_init,
-        .cra_exit = p8_aes_ctr_exit,
-        .cra_blkcipher = {
-                          .ivsize = AES_BLOCK_SIZE,
-                          .min_keysize = AES_MIN_KEY_SIZE,
-                          .max_keysize = AES_MAX_KEY_SIZE,
-                          .setkey = p8_aes_ctr_setkey,
-                          .encrypt = p8_aes_ctr_crypt,
-                          .decrypt = p8_aes_ctr_crypt,
-        },
+struct skcipher_alg p8_aes_ctr_alg = {
+        .base.cra_name = "ctr(aes)",
+        .base.cra_driver_name = "p8_aes_ctr",
+        .base.cra_module = THIS_MODULE,
+        .base.cra_priority = 2000,
+        .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+        .base.cra_blocksize = 1,
+        .base.cra_ctxsize = sizeof(struct p8_aes_ctr_ctx),
+        .setkey = p8_aes_ctr_setkey,
+        .encrypt = p8_aes_ctr_crypt,
+        .decrypt = p8_aes_ctr_crypt,
+        .init = p8_aes_ctr_init,
+        .exit = p8_aes_ctr_exit,
+        .min_keysize = AES_MIN_KEY_SIZE,
+        .max_keysize = AES_MAX_KEY_SIZE,
+        .ivsize = AES_BLOCK_SIZE,
+        .chunksize = AES_BLOCK_SIZE,
 };
diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c
index 536167e737a0..49f7258045fa 100644
--- a/drivers/crypto/vmx/aes_xts.c
+++ b/drivers/crypto/vmx/aes_xts.c
@@ -7,67 +7,56 @@
  * Author: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com>
  */
 
-#include <linux/types.h>
-#include <linux/err.h>
-#include <linux/crypto.h>
-#include <linux/delay.h>
 #include <asm/simd.h>
 #include <asm/switch_to.h>
 #include <crypto/aes.h>
 #include <crypto/internal/simd.h>
-#include <crypto/scatterwalk.h>
+#include <crypto/internal/skcipher.h>
 #include <crypto/xts.h>
-#include <crypto/skcipher.h>
 
 #include "aesp8-ppc.h"
 
 struct p8_aes_xts_ctx {
-        struct crypto_sync_skcipher *fallback;
+        struct crypto_skcipher *fallback;
         struct aes_key enc_key;
         struct aes_key dec_key;
         struct aes_key tweak_key;
 };
 
-static int p8_aes_xts_init(struct crypto_tfm *tfm)
+static int p8_aes_xts_init(struct crypto_skcipher *tfm)
 {
-        const char *alg = crypto_tfm_alg_name(tfm);
-        struct crypto_sync_skcipher *fallback;
-        struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+        struct crypto_skcipher *fallback;
 
-        fallback = crypto_alloc_sync_skcipher(alg, 0,
-                                              CRYPTO_ALG_NEED_FALLBACK);
+        fallback = crypto_alloc_skcipher("xts(aes)", 0,
+                                         CRYPTO_ALG_NEED_FALLBACK |
+                                         CRYPTO_ALG_ASYNC);
         if (IS_ERR(fallback)) {
-                printk(KERN_ERR
-                        "Failed to allocate transformation for '%s': %ld\n",
-                        alg, PTR_ERR(fallback));
+                pr_err("Failed to allocate xts(aes) fallback: %ld\n",
+                       PTR_ERR(fallback));
                 return PTR_ERR(fallback);
         }
 
-        crypto_sync_skcipher_set_flags(
-                fallback,
-                crypto_skcipher_get_flags((struct crypto_skcipher *)tfm));
+        crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) +
+                                    crypto_skcipher_reqsize(fallback));
         ctx->fallback = fallback;
-
         return 0;
 }
 
-static void p8_aes_xts_exit(struct crypto_tfm *tfm)
+static void p8_aes_xts_exit(struct crypto_skcipher *tfm)
 {
-        struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+        struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 
-        if (ctx->fallback) {
-                crypto_free_sync_skcipher(ctx->fallback);
-                ctx->fallback = NULL;
-        }
+        crypto_free_skcipher(ctx->fallback);
 }
 
-static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
+static int p8_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
                              unsigned int keylen)
 {
+        struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
         int ret;
-        struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 
-        ret = xts_check_key(tfm, key, keylen);
+        ret = xts_verify_key(tfm, key, keylen);
         if (ret)
                 return ret;
 
@@ -81,100 +70,90 @@ static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key,
         pagefault_enable();
         preempt_enable();
 
-        ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen);
+        ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen);
 
         return ret ? -EINVAL : 0;
 }
 
-static int p8_aes_xts_crypt(struct blkcipher_desc *desc,
-                            struct scatterlist *dst,
-                            struct scatterlist *src,
-                            unsigned int nbytes, int enc)
+static int p8_aes_xts_crypt(struct skcipher_request *req, int enc)
 {
-        int ret;
+        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+        const struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+        struct skcipher_walk walk;
+        unsigned int nbytes;
         u8 tweak[AES_BLOCK_SIZE];
-        u8 *iv;
-        struct blkcipher_walk walk;
-        struct p8_aes_xts_ctx *ctx =
-                crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm));
+        int ret;
 
         if (!crypto_simd_usable()) {
-                SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback);
-                skcipher_request_set_sync_tfm(req, ctx->fallback);
-                skcipher_request_set_callback(req, desc->flags, NULL, NULL);
-                skcipher_request_set_crypt(req, src, dst, nbytes, desc->info);
-                ret = enc? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);
-                skcipher_request_zero(req);
-        } else {
-                blkcipher_walk_init(&walk, dst, src, nbytes);
+                struct skcipher_request *subreq = skcipher_request_ctx(req);
+
+                *subreq = *req;
+                skcipher_request_set_tfm(subreq, ctx->fallback);
+                return enc ? crypto_skcipher_encrypt(subreq) :
+                             crypto_skcipher_decrypt(subreq);
+        }
+
+        ret = skcipher_walk_virt(&walk, req, false);
+        if (ret)
+                return ret;
+
+        preempt_disable();
+        pagefault_disable();
+        enable_kernel_vsx();
 
-                ret = blkcipher_walk_virt(desc, &walk);
+        aes_p8_encrypt(walk.iv, tweak, &ctx->tweak_key);
+
+        disable_kernel_vsx();
+        pagefault_enable();
+        preempt_enable();
 
+        while ((nbytes = walk.nbytes) != 0) {
                 preempt_disable();
                 pagefault_disable();
                 enable_kernel_vsx();
-
-                iv = walk.iv;
-                memset(tweak, 0, AES_BLOCK_SIZE);
-                aes_p8_encrypt(iv, tweak, &ctx->tweak_key);
-
+                if (enc)
+                        aes_p8_xts_encrypt(walk.src.virt.addr,
+                                           walk.dst.virt.addr,
+                                           round_down(nbytes, AES_BLOCK_SIZE),
+                                           &ctx->enc_key, NULL, tweak);
+                else
+                        aes_p8_xts_decrypt(walk.src.virt.addr,
+                                           walk.dst.virt.addr,
+                                           round_down(nbytes, AES_BLOCK_SIZE),
+                                           &ctx->dec_key, NULL, tweak);
                 disable_kernel_vsx();
                 pagefault_enable();
                 preempt_enable();
 
-                while ((nbytes = walk.nbytes)) {
-                        preempt_disable();
-                        pagefault_disable();
-                        enable_kernel_vsx();
-                        if (enc)
-                                aes_p8_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
-                                                nbytes & AES_BLOCK_MASK, &ctx->enc_key, NULL, tweak);
-                        else
-                                aes_p8_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
-                                                nbytes & AES_BLOCK_MASK, &ctx->dec_key, NULL, tweak);
-                        disable_kernel_vsx();
-                        pagefault_enable();
-                        preempt_enable();
-
-                        nbytes &= AES_BLOCK_SIZE - 1;
-                        ret = blkcipher_walk_done(desc, &walk, nbytes);
-                }
+                ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
         }
         return ret;
 }
 
-static int p8_aes_xts_encrypt(struct blkcipher_desc *desc,
-                              struct scatterlist *dst,
-                              struct scatterlist *src, unsigned int nbytes)
+static int p8_aes_xts_encrypt(struct skcipher_request *req)
 {
-        return p8_aes_xts_crypt(desc, dst, src, nbytes, 1);
+        return p8_aes_xts_crypt(req, 1);
 }
 
-static int p8_aes_xts_decrypt(struct blkcipher_desc *desc,
-                              struct scatterlist *dst,
-                              struct scatterlist *src, unsigned int nbytes)
+static int p8_aes_xts_decrypt(struct skcipher_request *req)
 {
-        return p8_aes_xts_crypt(desc, dst, src, nbytes, 0);
+        return p8_aes_xts_crypt(req, 0);
 }
 
-struct crypto_alg p8_aes_xts_alg = {
-        .cra_name = "xts(aes)",
-        .cra_driver_name = "p8_aes_xts",
-        .cra_module = THIS_MODULE,
-        .cra_priority = 2000,
-        .cra_type = &crypto_blkcipher_type,
-        .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK,
-        .cra_alignmask = 0,
-        .cra_blocksize = AES_BLOCK_SIZE,
-        .cra_ctxsize = sizeof(struct p8_aes_xts_ctx),
-        .cra_init = p8_aes_xts_init,
-        .cra_exit = p8_aes_xts_exit,
-        .cra_blkcipher = {
-                        .ivsize = AES_BLOCK_SIZE,
-                        .min_keysize = 2 * AES_MIN_KEY_SIZE,
-                        .max_keysize = 2 * AES_MAX_KEY_SIZE,
-                        .setkey  = p8_aes_xts_setkey,
-                        .encrypt = p8_aes_xts_encrypt,
-                        .decrypt = p8_aes_xts_decrypt,
-        }
+struct skcipher_alg p8_aes_xts_alg = {
+        .base.cra_name = "xts(aes)",
+        .base.cra_driver_name = "p8_aes_xts",
+        .base.cra_module = THIS_MODULE,
+        .base.cra_priority = 2000,
+        .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
+        .base.cra_blocksize = AES_BLOCK_SIZE,
+        .base.cra_ctxsize = sizeof(struct p8_aes_xts_ctx),
+        .setkey = p8_aes_xts_setkey,
+        .encrypt = p8_aes_xts_encrypt,
+        .decrypt = p8_aes_xts_decrypt,
+        .init = p8_aes_xts_init,
+        .exit = p8_aes_xts_exit,
+        .min_keysize = 2 * AES_MIN_KEY_SIZE,
+        .max_keysize = 2 * AES_MAX_KEY_SIZE,
+        .ivsize = AES_BLOCK_SIZE,
 };
diff --git a/drivers/crypto/vmx/aesp8-ppc.h b/drivers/crypto/vmx/aesp8-ppc.h
index 349646b73754..01774a4d26a2 100644
--- a/drivers/crypto/vmx/aesp8-ppc.h
+++ b/drivers/crypto/vmx/aesp8-ppc.h
@@ -2,8 +2,6 @@
 #include <linux/types.h>
 #include <crypto/aes.h>
 
-#define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
-
 struct aes_key {
         u8 key[AES_MAX_KEYLENGTH];
         int rounds;
diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl
index 9c6b5c1d6a1a..db874367b602 100644
--- a/drivers/crypto/vmx/aesp8-ppc.pl
+++ b/drivers/crypto/vmx/aesp8-ppc.pl
@@ -1286,6 +1286,24 @@ ___
 
 #########################################################################
 {{{     # CTR procedure[s]                                              #
+
+####################### WARNING: Here be dragons! #######################
+#
+# This code is written as 'ctr32', based on a 32-bit counter used
+# upstream. The kernel does *not* use a 32-bit counter. The kernel uses
+# a 128-bit counter.
+#
+# This leads to subtle changes from the upstream code: the counter
+# is incremented with vaddu_q_m rather than vaddu_w_m. This occurs in
+# both the bulk (8 blocks at a time) path, and in the individual block
+# path. Be aware of this when doing updates.
+#
+# See:
+# 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug")
+# 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword")
+# https://github.com/openssl/openssl/pull/8942
+#
+#########################################################################
 my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
 my ($rndkey0,$rndkey1,$inout,$tmp)=             map("v$_",(0..3));
 my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
@@ -1357,7 +1375,7 @@ Loop_ctr32_enc:
         addi            $idx,$idx,16
         bdnz            Loop_ctr32_enc
 
-        vadduqm         $ivec,$ivec,$one
+        vadduqm         $ivec,$ivec,$one        # Kernel change for 128-bit
          vmr            $dat,$inptail
          lvx            $inptail,0,$inp
          addi           $inp,$inp,16
@@ -1501,7 +1519,7 @@ Load_ctr32_enc_key:
         $SHL            $len,$len,4
 
         vadduqm         $out1,$ivec,$one        # counter values ...
-        vadduqm         $out2,$ivec,$two
+        vadduqm         $out2,$ivec,$two        # (do all ctr adds as 128-bit)
         vxor            $out0,$ivec,$rndkey0    # ... xored with rndkey[0]
          le?li          $idx,8
         vadduqm         $out3,$out1,$two
diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c
index 6c4c77f4e159..3e0335fb406c 100644
--- a/drivers/crypto/vmx/vmx.c
+++ b/drivers/crypto/vmx/vmx.c
@@ -15,54 +15,58 @@
 #include <linux/crypto.h>
 #include <asm/cputable.h>
 #include <crypto/internal/hash.h>
+#include <crypto/internal/skcipher.h>
 
 extern struct shash_alg p8_ghash_alg;
 extern struct crypto_alg p8_aes_alg;
-extern struct crypto_alg p8_aes_cbc_alg;
-extern struct crypto_alg p8_aes_ctr_alg;
-extern struct crypto_alg p8_aes_xts_alg;
-static struct crypto_alg *algs[] = {
-        &p8_aes_alg,
-        &p8_aes_cbc_alg,
-        &p8_aes_ctr_alg,
-        &p8_aes_xts_alg,
-        NULL,
-};
+extern struct skcipher_alg p8_aes_cbc_alg;
+extern struct skcipher_alg p8_aes_ctr_alg;
+extern struct skcipher_alg p8_aes_xts_alg;
 
 static int __init p8_init(void)
 {
-        int ret = 0;
-        struct crypto_alg **alg_it;
+        int ret;
 
-        for (alg_it = algs; *alg_it; alg_it++) {
-                ret = crypto_register_alg(*alg_it);
-                printk(KERN_INFO "crypto_register_alg '%s' = %d\n",
-                       (*alg_it)->cra_name, ret);
-                if (ret) {
-                        for (alg_it--; alg_it >= algs; alg_it--)
-                                crypto_unregister_alg(*alg_it);
-                        break;
-                }
-        }
+        ret = crypto_register_shash(&p8_ghash_alg);
         if (ret)
-                return ret;
+                goto err;
 
-        ret = crypto_register_shash(&p8_ghash_alg);
-        if (ret) {
-                for (alg_it = algs; *alg_it; alg_it++)
-                        crypto_unregister_alg(*alg_it);
-        }
+        ret = crypto_register_alg(&p8_aes_alg);
+        if (ret)
+                goto err_unregister_ghash;
+
+        ret = crypto_register_skcipher(&p8_aes_cbc_alg);
+        if (ret)
+                goto err_unregister_aes;
+
+        ret = crypto_register_skcipher(&p8_aes_ctr_alg);
+        if (ret)
+                goto err_unregister_aes_cbc;
+
+        ret = crypto_register_skcipher(&p8_aes_xts_alg);
+        if (ret)
+                goto err_unregister_aes_ctr;
+
+        return 0;
+
+err_unregister_aes_ctr:
+        crypto_unregister_skcipher(&p8_aes_ctr_alg);
+err_unregister_aes_cbc:
+        crypto_unregister_skcipher(&p8_aes_cbc_alg);
+err_unregister_aes:
+        crypto_unregister_alg(&p8_aes_alg);
+err_unregister_ghash:
+        crypto_unregister_shash(&p8_ghash_alg);
+err:
         return ret;
 }
 
 static void __exit p8_exit(void)
 {
-        struct crypto_alg **alg_it;
-
-        for (alg_it = algs; *alg_it; alg_it++) {
-                printk(KERN_INFO "Removing '%s'\n", (*alg_it)->cra_name);
-                crypto_unregister_alg(*alg_it);
-        }
+        crypto_unregister_skcipher(&p8_aes_xts_alg);
+        crypto_unregister_skcipher(&p8_aes_ctr_alg);
+        crypto_unregister_skcipher(&p8_aes_cbc_alg);
+        crypto_unregister_alg(&p8_aes_alg);
         crypto_unregister_shash(&p8_ghash_alg);
 }
 
diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c
index d84095591e45..f86065e16772 100644
--- a/drivers/i2c/i2c-core-acpi.c
+++ b/drivers/i2c/i2c-core-acpi.c
@@ -111,8 +111,7 @@ static int i2c_acpi_do_lookup(struct acpi_device *adev,
         struct list_head resource_list;
         int ret;
 
-        if (acpi_bus_get_status(adev) || !adev->status.present ||
-            acpi_device_enumerated(adev))
+        if (acpi_bus_get_status(adev) || !adev->status.present)
                 return -EINVAL;
 
         if (acpi_match_device_ids(adev, i2c_acpi_ignored_device_ids) == 0)
@@ -147,6 +146,9 @@ static int i2c_acpi_get_info(struct acpi_device *adev,
         lookup.info = info;
         lookup.index = -1;
 
+        if (acpi_device_enumerated(adev))
+                return -EINVAL;
+
         ret = i2c_acpi_do_lookup(adev, &lookup);
         if (ret)
                 return ret;
diff --git a/drivers/net/ppp/Kconfig b/drivers/net/ppp/Kconfig
index bf395df3bb37..1a2e2f7629f3 100644
--- a/drivers/net/ppp/Kconfig
+++ b/drivers/net/ppp/Kconfig
@@ -87,8 +87,7 @@ config PPP_MPPE
         depends on PPP
         select CRYPTO
         select CRYPTO_SHA1
-        select CRYPTO_ARC4
-        select CRYPTO_ECB
+        select CRYPTO_LIB_ARC4
         ---help---
           Support for the MPPE Encryption protocol, as employed by the
           Microsoft Point-to-Point Tunneling Protocol.
diff --git a/drivers/net/ppp/ppp_mppe.c b/drivers/net/ppp/ppp_mppe.c
index 66c8e65f6872..bd3c80b0bc77 100644
--- a/drivers/net/ppp/ppp_mppe.c
+++ b/drivers/net/ppp/ppp_mppe.c
@@ -42,9 +42,10 @@
  *                    deprecated in 2.6
  */
 
+#include <crypto/arc4.h>
 #include <crypto/hash.h>
-#include <crypto/skcipher.h>
 #include <linux/err.h>
+#include <linux/fips.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
@@ -66,13 +67,6 @@ MODULE_ALIAS("ppp-compress-" __stringify(CI_MPPE));
 MODULE_SOFTDEP("pre: arc4");
 MODULE_VERSION("1.0.2");
 
-static unsigned int
-setup_sg(struct scatterlist *sg, const void *address, unsigned int length)
-{
-        sg_set_buf(sg, address, length);
-        return length;
-}
-
 #define SHA1_PAD_SIZE 40
 
 /*
@@ -96,7 +90,7 @@ static inline void sha_pad_init(struct sha_pad *shapad)
  * State for an MPPE (de)compressor.
  */
 struct ppp_mppe_state {
-        struct crypto_sync_skcipher *arc4;
+        struct arc4_ctx arc4;
         struct shash_desc *sha1;
         unsigned char *sha1_digest;
         unsigned char master_key[MPPE_MAX_KEY_LEN];
@@ -155,24 +149,11 @@ static void get_new_key_from_sha(struct ppp_mppe_state * state)
  */
 static void mppe_rekey(struct ppp_mppe_state * state, int initial_key)
 {
-        struct scatterlist sg_in[1], sg_out[1];
-        SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4);
-
-        skcipher_request_set_sync_tfm(req, state->arc4);
-        skcipher_request_set_callback(req, 0, NULL, NULL);
-
         get_new_key_from_sha(state);
         if (!initial_key) {
-                crypto_sync_skcipher_setkey(state->arc4, state->sha1_digest,
-                                            state->keylen);
-                sg_init_table(sg_in, 1);
-                sg_init_table(sg_out, 1);
-                setup_sg(sg_in, state->sha1_digest, state->keylen);
-                setup_sg(sg_out, state->session_key, state->keylen);
-                skcipher_request_set_crypt(req, sg_in, sg_out, state->keylen,
-                                           NULL);
-                if (crypto_skcipher_encrypt(req))
-                    printk(KERN_WARNING "mppe_rekey: cipher_encrypt failed\n");
+                arc4_setkey(&state->arc4, state->sha1_digest, state->keylen);
+                arc4_crypt(&state->arc4, state->session_key, state->sha1_digest,
+                           state->keylen);
         } else {
                 memcpy(state->session_key, state->sha1_digest, state->keylen);
         }
@@ -182,9 +163,7 @@ static void mppe_rekey(struct ppp_mppe_state * state, int initial_key)
                 state->session_key[1] = 0x26;
                 state->session_key[2] = 0x9e;
         }
-        crypto_sync_skcipher_setkey(state->arc4, state->session_key,
-                                    state->keylen);
-        skcipher_request_zero(req);
+        arc4_setkey(&state->arc4, state->session_key, state->keylen);
 }
 
 /*
@@ -197,7 +176,8 @@ static void *mppe_alloc(unsigned char *options, int optlen)
         unsigned int digestsize;
 
         if (optlen != CILEN_MPPE + sizeof(state->master_key) ||
-            options[0] != CI_MPPE || options[1] != CILEN_MPPE)
+            options[0] != CI_MPPE || options[1] != CILEN_MPPE ||
+            fips_enabled)
                 goto out;
 
         state = kzalloc(sizeof(*state), GFP_KERNEL);
@@ -205,12 +185,6 @@ static void *mppe_alloc(unsigned char *options, int optlen)
                 goto out;
 
 
-        state->arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0);
-        if (IS_ERR(state->arc4)) {
-                state->arc4 = NULL;
-                goto out_free;
-        }
-
         shash = crypto_alloc_shash("sha1", 0, 0);
         if (IS_ERR(shash))
                 goto out_free;
@@ -251,7 +225,6 @@ out_free:
                 crypto_free_shash(state->sha1->tfm);
                 kzfree(state->sha1);
         }
-        crypto_free_sync_skcipher(state->arc4);
         kfree(state);
 out:
         return NULL;
@@ -267,8 +240,7 @@ static void mppe_free(void *arg)
                 kfree(state->sha1_digest);
                 crypto_free_shash(state->sha1->tfm);
                 kzfree(state->sha1);
-                crypto_free_sync_skcipher(state->arc4);
-                kfree(state);
+                kzfree(state);
         }
 }
 
@@ -367,10 +339,7 @@ mppe_compress(void *arg, unsigned char *ibuf, unsigned char *obuf,
               int isize, int osize)
 {
         struct ppp_mppe_state *state = (struct ppp_mppe_state *) arg;
-        SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4);
         int proto;
-        int err;
-        struct scatterlist sg_in[1], sg_out[1];
 
         /*
          * Check that the protocol is in the range we handle.
@@ -421,21 +390,7 @@ mppe_compress(void *arg, unsigned char *ibuf, unsigned char *obuf,
         ibuf += 2;              /* skip to proto field */
         isize -= 2;
 
-        /* Encrypt packet */
-        sg_init_table(sg_in, 1);
-        sg_init_table(sg_out, 1);
-        setup_sg(sg_in, ibuf, isize);
-        setup_sg(sg_out, obuf, osize);
-
-        skcipher_request_set_sync_tfm(req, state->arc4);
-        skcipher_request_set_callback(req, 0, NULL, NULL);
-        skcipher_request_set_crypt(req, sg_in, sg_out, isize, NULL);
-        err = crypto_skcipher_encrypt(req);
-        skcipher_request_zero(req);
-        if (err) {
-                printk(KERN_DEBUG "crypto_cypher_encrypt failed\n");
-                return -1;
-        }
+        arc4_crypt(&state->arc4, obuf, ibuf, isize);
 
         state->stats.unc_bytes += isize;
         state->stats.unc_packets++;
@@ -481,10 +436,8 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf,
                 int osize)
 {
         struct ppp_mppe_state *state = (struct ppp_mppe_state *) arg;
-        SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4);
         unsigned ccount;
         int flushed = MPPE_BITS(ibuf) & MPPE_BIT_FLUSHED;
-        struct scatterlist sg_in[1], sg_out[1];
 
         if (isize <= PPP_HDRLEN + MPPE_OVHD) {
                 if (state->debug)
@@ -611,19 +564,7 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf,
          * Decrypt the first byte in order to check if it is
          * a compressed or uncompressed protocol field.
          */
-        sg_init_table(sg_in, 1);
-        sg_init_table(sg_out, 1);
-        setup_sg(sg_in, ibuf, 1);
-        setup_sg(sg_out, obuf, 1);
-
-        skcipher_request_set_sync_tfm(req, state->arc4);
-        skcipher_request_set_callback(req, 0, NULL, NULL);
-        skcipher_request_set_crypt(req, sg_in, sg_out, 1, NULL);
-        if (crypto_skcipher_decrypt(req)) {
-                printk(KERN_DEBUG "crypto_cypher_decrypt failed\n");
-                osize = DECOMP_ERROR;
-                goto out_zap_req;
-        }
+        arc4_crypt(&state->arc4, obuf, ibuf, 1);
 
         /*
          * Do PFC decompression.
@@ -638,14 +579,7 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf,
         }
 
         /* And finally, decrypt the rest of the packet. */
-        setup_sg(sg_in, ibuf + 1, isize - 1);
-        setup_sg(sg_out, obuf + 1, osize - 1);
-        skcipher_request_set_crypt(req, sg_in, sg_out, isize - 1, NULL);
-        if (crypto_skcipher_decrypt(req)) {
-                printk(KERN_DEBUG "crypto_cypher_decrypt failed\n");
-                osize = DECOMP_ERROR;
-                goto out_zap_req;
-        }
+        arc4_crypt(&state->arc4, obuf + 1, ibuf + 1, isize - 1);
 
         state->stats.unc_bytes += osize;
         state->stats.unc_packets++;
@@ -655,8 +589,6 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf,
         /* good packet credit */
         state->sanity_errors >>= 1;
 
-out_zap_req:
-        skcipher_request_zero(req);
         return osize;
 
 sanity_error:
@@ -729,8 +661,7 @@ static struct compressor ppp_mppe = {
 static int __init ppp_mppe_init(void)
 {
         int answer;
-        if (!(crypto_has_skcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC) &&
-              crypto_has_ahash("sha1", 0, CRYPTO_ALG_ASYNC)))
+        if (fips_enabled || !crypto_has_ahash("sha1", 0, CRYPTO_ALG_ASYNC))
                 return -ENODEV;
 
         sha_pad = kmalloc(sizeof(struct sha_pad), GFP_KERNEL);
diff --git a/fs/cifs/Kconfig b/fs/cifs/Kconfig
index aae2b8b2adf5..523e9ea78a28 100644
--- a/fs/cifs/Kconfig
+++ b/fs/cifs/Kconfig
@@ -10,7 +10,7 @@ config CIFS
         select CRYPTO_SHA512
         select CRYPTO_CMAC
         select CRYPTO_HMAC
-        select CRYPTO_ARC4
+        select CRYPTO_LIB_ARC4
         select CRYPTO_AEAD2
         select CRYPTO_CCM
         select CRYPTO_ECB
diff --git a/fs/cifs/cifsencrypt.c b/fs/cifs/cifsencrypt.c
index d2a05e46d6f5..97b7497c13ef 100644
--- a/fs/cifs/cifsencrypt.c
+++ b/fs/cifs/cifsencrypt.c
@@ -33,7 +33,8 @@
 #include <linux/ctype.h>
 #include <linux/random.h>
 #include <linux/highmem.h>
-#include <crypto/skcipher.h>
+#include <linux/fips.h>
+#include <crypto/arc4.h>
 #include <crypto/aead.h>
 
 int __cifs_calc_signature(struct smb_rqst *rqst,
@@ -772,63 +773,32 @@ setup_ntlmv2_rsp_ret:
 int
 calc_seckey(struct cifs_ses *ses)
 {
-        int rc;
-        struct crypto_skcipher *tfm_arc4;
-        struct scatterlist sgin, sgout;
-        struct skcipher_request *req;
-        unsigned char *sec_key;
+        unsigned char sec_key[CIFS_SESS_KEY_SIZE]; /* a nonce */
+        struct arc4_ctx *ctx_arc4;
 
-        sec_key = kmalloc(CIFS_SESS_KEY_SIZE, GFP_KERNEL);
-        if (sec_key == NULL)
-                return -ENOMEM;
+        if (fips_enabled)
+                return -ENODEV;
 
         get_random_bytes(sec_key, CIFS_SESS_KEY_SIZE);
 
-        tfm_arc4 = crypto_alloc_skcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC);
-        if (IS_ERR(tfm_arc4)) {
-                rc = PTR_ERR(tfm_arc4);
-                cifs_dbg(VFS, "could not allocate crypto API arc4\n");
-                goto out;
-        }
-
-        rc = crypto_skcipher_setkey(tfm_arc4, ses->auth_key.response,
-                                        CIFS_SESS_KEY_SIZE);
-        if (rc) {
-                cifs_dbg(VFS, "%s: Could not set response as a key\n",
-                         __func__);
-                goto out_free_cipher;
-        }
-
-        req = skcipher_request_alloc(tfm_arc4, GFP_KERNEL);
-        if (!req) {
-                rc = -ENOMEM;
-                cifs_dbg(VFS, "could not allocate crypto API arc4 request\n");
-                goto out_free_cipher;
+        ctx_arc4 = kmalloc(sizeof(*ctx_arc4), GFP_KERNEL);
+        if (!ctx_arc4) {
+                cifs_dbg(VFS, "could not allocate arc4 context\n");
+                return -ENOMEM;
         }
 
-        sg_init_one(&sgin, sec_key, CIFS_SESS_KEY_SIZE);
-        sg_init_one(&sgout, ses->ntlmssp->ciphertext, CIFS_CPHTXT_SIZE);
-
-        skcipher_request_set_callback(req, 0, NULL, NULL);
-        skcipher_request_set_crypt(req, &sgin, &sgout, CIFS_CPHTXT_SIZE, NULL);
-
-        rc = crypto_skcipher_encrypt(req);
-        skcipher_request_free(req);
-        if (rc) {
-                cifs_dbg(VFS, "could not encrypt session key rc: %d\n", rc);
-                goto out_free_cipher;
-        }
+        arc4_setkey(ctx_arc4, ses->auth_key.response, CIFS_SESS_KEY_SIZE);
+        arc4_crypt(ctx_arc4, ses->ntlmssp->ciphertext, sec_key,
+                   CIFS_CPHTXT_SIZE);
 
         /* make secondary_key/nonce as session key */
         memcpy(ses->auth_key.response, sec_key, CIFS_SESS_KEY_SIZE);
         /* and make len as that of session key only */
         ses->auth_key.len = CIFS_SESS_KEY_SIZE;
 
-out_free_cipher:
-        crypto_free_skcipher(tfm_arc4);
-out:
-        kfree(sec_key);
-        return rc;
+        memzero_explicit(sec_key, CIFS_SESS_KEY_SIZE);
+        kzfree(ctx_arc4);
+        return 0;
 }
 
 void
diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c
index 65d9771e49f9..72db1c89bf5a 100644
--- a/fs/cifs/cifsfs.c
+++ b/fs/cifs/cifsfs.c
@@ -1591,7 +1591,6 @@ MODULE_DESCRIPTION
         ("VFS to access SMB3 servers e.g. Samba, Macs, Azure and Windows (and "
         "also older servers complying with the SNIA CIFS Specification)");
 MODULE_VERSION(CIFS_VERSION);
-MODULE_SOFTDEP("pre: arc4");
 MODULE_SOFTDEP("pre: des");
 MODULE_SOFTDEP("pre: ecb");
 MODULE_SOFTDEP("pre: hmac");
diff --git a/include/crypto/aead.h b/include/crypto/aead.h
index 61bb10490492..3c245b1859e7 100644
--- a/include/crypto/aead.h
+++ b/include/crypto/aead.h
@@ -317,21 +317,7 @@ static inline struct crypto_aead *crypto_aead_reqtfm(struct aead_request *req)
  *
  * Return: 0 if the cipher operation was successful; < 0 if an error occurred
  */
-static inline int crypto_aead_encrypt(struct aead_request *req)
-{
-        struct crypto_aead *aead = crypto_aead_reqtfm(req);
-        struct crypto_alg *alg = aead->base.__crt_alg;
-        unsigned int cryptlen = req->cryptlen;
-        int ret;
-
-        crypto_stats_get(alg);
-        if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
-                ret = -ENOKEY;
-        else
-                ret = crypto_aead_alg(aead)->encrypt(req);
-        crypto_stats_aead_encrypt(cryptlen, alg, ret);
-        return ret;
-}
+int crypto_aead_encrypt(struct aead_request *req);
 
 /**
  * crypto_aead_decrypt() - decrypt ciphertext
@@ -355,23 +341,7 @@ static inline int crypto_aead_encrypt(struct aead_request *req)
  *         integrity of the ciphertext or the associated data was violated);
  *         < 0 if an error occurred.
  */
-static inline int crypto_aead_decrypt(struct aead_request *req)
-{
-        struct crypto_aead *aead = crypto_aead_reqtfm(req);
-        struct crypto_alg *alg = aead->base.__crt_alg;
-        unsigned int cryptlen = req->cryptlen;
-        int ret;
-
-        crypto_stats_get(alg);
-        if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY)
-                ret = -ENOKEY;
-        else if (req->cryptlen < crypto_aead_authsize(aead))
-                ret = -EINVAL;
-        else
-                ret = crypto_aead_alg(aead)->decrypt(req);
-        crypto_stats_aead_decrypt(cryptlen, alg, ret);
-        return ret;
-}
+int crypto_aead_decrypt(struct aead_request *req);
 
 /**
  * DOC: Asynchronous AEAD Request Handle
diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h
index 743d626479ef..dc1106af95c3 100644
--- a/include/crypto/algapi.h
+++ b/include/crypto/algapi.h
@@ -189,7 +189,6 @@ void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen);
 int crypto_enqueue_request(struct crypto_queue *queue,
                            struct crypto_async_request *request);
 struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue);
-int crypto_tfm_in_queue(struct crypto_queue *queue, struct crypto_tfm *tfm);
 static inline unsigned int crypto_queue_len(struct crypto_queue *queue)
 {
         return queue->qlen;
@@ -371,12 +370,6 @@ static inline void *ablkcipher_request_ctx(struct ablkcipher_request *req)
         return req->__ctx;
 }
 
-static inline int ablkcipher_tfm_in_queue(struct crypto_queue *queue,
-                                          struct crypto_ablkcipher *tfm)
-{
-        return crypto_tfm_in_queue(queue, crypto_ablkcipher_tfm(tfm));
-}
-
 static inline struct crypto_alg *crypto_get_attr_alg(struct rtattr **tb,
                                                      u32 type, u32 mask)
 {
diff --git a/include/crypto/arc4.h b/include/crypto/arc4.h
index 5b2c24ab0139..f3c22fe01704 100644
--- a/include/crypto/arc4.h
+++ b/include/crypto/arc4.h
@@ -6,8 +6,18 @@
 #ifndef _CRYPTO_ARC4_H
 #define _CRYPTO_ARC4_H
 
+#include <linux/types.h>
+
 #define ARC4_MIN_KEY_SIZE       1
 #define ARC4_MAX_KEY_SIZE       256
 #define ARC4_BLOCK_SIZE         1
 
+struct arc4_ctx {
+        u32 S[256];
+        u32 x, y;
+};
+
+int arc4_setkey(struct arc4_ctx *ctx, const u8 *in_key, unsigned int key_len);
+void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in, unsigned int len);
+
 #endif /* _CRYPTO_ARC4_H */
diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h
index 1fc70a69d550..d1e723c6a37d 100644
--- a/include/crypto/chacha.h
+++ b/include/crypto/chacha.h
@@ -41,7 +41,7 @@ static inline void chacha20_block(u32 *state, u8 *stream)
 }
 void hchacha_block(const u32 *in, u32 *out, int nrounds);
 
-void crypto_chacha_init(u32 *state, struct chacha_ctx *ctx, u8 *iv);
+void crypto_chacha_init(u32 *state, const struct chacha_ctx *ctx, const u8 *iv);
 
 int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key,
                            unsigned int keysize);
diff --git a/include/crypto/crypto_wq.h b/include/crypto/crypto_wq.h
deleted file mode 100644
index 23114746ac08..000000000000
--- a/include/crypto/crypto_wq.h
+++ /dev/null
@@ -1,8 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef CRYPTO_WQ_H
-#define CRYPTO_WQ_H
-
-#include <linux/workqueue.h>
-
-extern struct workqueue_struct *kcrypto_wq;
-#endif
diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h
index 3fb581bf3b87..8c9af21efce1 100644
--- a/include/crypto/drbg.h
+++ b/include/crypto/drbg.h
@@ -129,6 +129,8 @@ struct drbg_state {
 
         bool seeded;            /* DRBG fully seeded? */
         bool pr;                /* Prediction resistance enabled? */
+        bool fips_primed;       /* Continuous test primed? */
+        unsigned char *prev;    /* FIPS 140-2 continuous test value */
         struct work_struct seed_work;   /* asynchronous seeding support */
         struct crypto_rng *jent;
         const struct drbg_state_ops *d_ops;
diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h
index 31e0662fa429..bfc9db7b100d 100644
--- a/include/crypto/internal/hash.h
+++ b/include/crypto/internal/hash.h
@@ -196,12 +196,6 @@ static inline struct ahash_request *ahash_dequeue_request(
         return ahash_request_cast(crypto_dequeue_request(queue));
 }
 
-static inline int ahash_tfm_in_queue(struct crypto_queue *queue,
-                                          struct crypto_ahash *tfm)
-{
-        return crypto_tfm_in_queue(queue, crypto_ahash_tfm(tfm));
-}
-
 static inline void *crypto_shash_ctx(struct crypto_shash *tfm)
 {
         return crypto_tfm_ctx(&tfm->base);
diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h
index fe0376d5a471..d68faa5759ad 100644
--- a/include/crypto/internal/skcipher.h
+++ b/include/crypto/internal/skcipher.h
@@ -200,6 +200,66 @@ static inline unsigned int crypto_skcipher_alg_max_keysize(
         return alg->max_keysize;
 }
 
+static inline unsigned int crypto_skcipher_alg_chunksize(
+        struct skcipher_alg *alg)
+{
+        if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) ==
+            CRYPTO_ALG_TYPE_BLKCIPHER)
+                return alg->base.cra_blocksize;
+
+        if (alg->base.cra_ablkcipher.encrypt)
+                return alg->base.cra_blocksize;
+
+        return alg->chunksize;
+}
+
+static inline unsigned int crypto_skcipher_alg_walksize(
+        struct skcipher_alg *alg)
+{
+        if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) ==
+            CRYPTO_ALG_TYPE_BLKCIPHER)
+                return alg->base.cra_blocksize;
+
+        if (alg->base.cra_ablkcipher.encrypt)
+                return alg->base.cra_blocksize;
+
+        return alg->walksize;
+}
+
+/**
+ * crypto_skcipher_chunksize() - obtain chunk size
+ * @tfm: cipher handle
+ *
+ * The block size is set to one for ciphers such as CTR.  However,
+ * you still need to provide incremental updates in multiples of
+ * the underlying block size as the IV does not have sub-block
+ * granularity.  This is known in this API as the chunk size.
+ *
+ * Return: chunk size in bytes
+ */
+static inline unsigned int crypto_skcipher_chunksize(
+        struct crypto_skcipher *tfm)
+{
+        return crypto_skcipher_alg_chunksize(crypto_skcipher_alg(tfm));
+}
+
+/**
+ * crypto_skcipher_walksize() - obtain walk size
+ * @tfm: cipher handle
+ *
+ * In some cases, algorithms can only perform optimally when operating on
+ * multiple blocks in parallel. This is reflected by the walksize, which
+ * must be a multiple of the chunksize (or equal if the concern does not
+ * apply)
+ *
+ * Return: walk size in bytes
+ */
+static inline unsigned int crypto_skcipher_walksize(
+        struct crypto_skcipher *tfm)
+{
+        return crypto_skcipher_alg_walksize(crypto_skcipher_alg(tfm));
+}
+
 /* Helpers for simple block cipher modes of operation */
 struct skcipher_ctx_simple {
         struct crypto_cipher *cipher;   /* underlying block cipher */
diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h
index ce7fa0973580..37c164234d97 100644
--- a/include/crypto/skcipher.h
+++ b/include/crypto/skcipher.h
@@ -288,66 +288,6 @@ static inline unsigned int crypto_sync_skcipher_ivsize(
         return crypto_skcipher_ivsize(&tfm->base);
 }
 
-static inline unsigned int crypto_skcipher_alg_chunksize(
-        struct skcipher_alg *alg)
-{
-        if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) ==
-            CRYPTO_ALG_TYPE_BLKCIPHER)
-                return alg->base.cra_blocksize;
-
-        if (alg->base.cra_ablkcipher.encrypt)
-                return alg->base.cra_blocksize;
-
-        return alg->chunksize;
-}
-
-static inline unsigned int crypto_skcipher_alg_walksize(
-        struct skcipher_alg *alg)
-{
-        if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) ==
-            CRYPTO_ALG_TYPE_BLKCIPHER)
-                return alg->base.cra_blocksize;
-
-        if (alg->base.cra_ablkcipher.encrypt)
-                return alg->base.cra_blocksize;
-
-        return alg->walksize;
-}
-
-/**
- * crypto_skcipher_chunksize() - obtain chunk size
- * @tfm: cipher handle
- *
- * The block size is set to one for ciphers such as CTR.  However,
- * you still need to provide incremental updates in multiples of
- * the underlying block size as the IV does not have sub-block
- * granularity.  This is known in this API as the chunk size.
- *
- * Return: chunk size in bytes
- */
-static inline unsigned int crypto_skcipher_chunksize(
-        struct crypto_skcipher *tfm)
-{
-        return crypto_skcipher_alg_chunksize(crypto_skcipher_alg(tfm));
-}
-
-/**
- * crypto_skcipher_walksize() - obtain walk size
- * @tfm: cipher handle
- *
- * In some cases, algorithms can only perform optimally when operating on
- * multiple blocks in parallel. This is reflected by the walksize, which
- * must be a multiple of the chunksize (or equal if the concern does not
- * apply)
- *
- * Return: walk size in bytes
- */
-static inline unsigned int crypto_skcipher_walksize(
-        struct crypto_skcipher *tfm)
-{
-        return crypto_skcipher_alg_walksize(crypto_skcipher_alg(tfm));
-}
-
 /**
  * crypto_skcipher_blocksize() - obtain block size of cipher
  * @tfm: cipher handle
@@ -479,21 +419,7 @@ static inline struct crypto_sync_skcipher *crypto_sync_skcipher_reqtfm(
  *
  * Return: 0 if the cipher operation was successful; < 0 if an error occurred
  */
-static inline int crypto_skcipher_encrypt(struct skcipher_request *req)
-{
-        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-        struct crypto_alg *alg = tfm->base.__crt_alg;
-        unsigned int cryptlen = req->cryptlen;
-        int ret;
-
-        crypto_stats_get(alg);
-        if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
-                ret = -ENOKEY;
-        else
-                ret = tfm->encrypt(req);
-        crypto_stats_skcipher_encrypt(cryptlen, ret, alg);
-        return ret;
-}
+int crypto_skcipher_encrypt(struct skcipher_request *req);
 
 /**
  * crypto_skcipher_decrypt() - decrypt ciphertext
@@ -506,21 +432,7 @@ static inline int crypto_skcipher_encrypt(struct skcipher_request *req)
  *
  * Return: 0 if the cipher operation was successful; < 0 if an error occurred
  */
-static inline int crypto_skcipher_decrypt(struct skcipher_request *req)
-{
-        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-        struct crypto_alg *alg = tfm->base.__crt_alg;
-        unsigned int cryptlen = req->cryptlen;
-        int ret;
-
-        crypto_stats_get(alg);
-        if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
-                ret = -ENOKEY;
-        else
-                ret = tfm->decrypt(req);
-        crypto_stats_skcipher_decrypt(cryptlen, ret, alg);
-        return ret;
-}
+int crypto_skcipher_decrypt(struct skcipher_request *req);
 
 /**
  * DOC: Symmetric Key Cipher Request Handle
diff --git a/include/linux/crypto.h b/include/linux/crypto.h
index 9cf8f3ce0e50..19ea3a371d7b 100644
--- a/include/linux/crypto.h
+++ b/include/linux/crypto.h
@@ -49,7 +49,6 @@
 #define CRYPTO_ALG_TYPE_SCOMPRESS       0x0000000b
 #define CRYPTO_ALG_TYPE_RNG             0x0000000c
 #define CRYPTO_ALG_TYPE_AKCIPHER        0x0000000d
-#define CRYPTO_ALG_TYPE_DIGEST          0x0000000e
 #define CRYPTO_ALG_TYPE_HASH            0x0000000e
 #define CRYPTO_ALG_TYPE_SHASH           0x0000000e
 #define CRYPTO_ALG_TYPE_AHASH           0x0000000f
@@ -323,6 +322,17 @@ struct cipher_alg {
         void (*cia_decrypt)(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
 };
 
+/**
+ * struct compress_alg - compression/decompression algorithm
+ * @coa_compress: Compress a buffer of specified length, storing the resulting
+ *                data in the specified buffer. Return the length of the
+ *                compressed data in dlen.
+ * @coa_decompress: Decompress the source buffer, storing the uncompressed
+ *                  data in the specified buffer. The length of the data is
+ *                  returned in dlen.
+ *
+ * All fields are mandatory.
+ */
 struct compress_alg {
         int (*coa_compress)(struct crypto_tfm *tfm, const u8 *src,
                             unsigned int slen, u8 *dst, unsigned int *dlen);
diff --git a/lib/Makefile b/lib/Makefile
index fb7697031a79..d3daedf93c5a 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -102,7 +102,7 @@ endif
 obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o
 CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any)
 
-obj-y += math/
+obj-y += math/ crypto/
 
 obj-$(CONFIG_GENERIC_IOMAP) += iomap.o
 obj-$(CONFIG_GENERIC_PCI_IOMAP) += pci_iomap.o
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
new file mode 100644
index 000000000000..88195c34932d
--- /dev/null
+++ b/lib/crypto/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_CRYPTO_LIB_ARC4) += libarc4.o
+libarc4-y := arc4.o
diff --git a/lib/crypto/arc4.c b/lib/crypto/arc4.c
new file mode 100644
index 000000000000..c2020f19c652
--- /dev/null
+++ b/lib/crypto/arc4.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Cryptographic API
+ *
+ * ARC4 Cipher Algorithm
+ *
+ * Jon Oberheide <jon@oberheide.org>
+ */
+
+#include <crypto/arc4.h>
+#include <linux/module.h>
+
+int arc4_setkey(struct arc4_ctx *ctx, const u8 *in_key, unsigned int key_len)
+{
+        int i, j = 0, k = 0;
+
+        ctx->x = 1;
+        ctx->y = 0;
+
+        for (i = 0; i < 256; i++)
+                ctx->S[i] = i;
+
+        for (i = 0; i < 256; i++) {
+                u32 a = ctx->S[i];
+
+                j = (j + in_key[k] + a) & 0xff;
+                ctx->S[i] = ctx->S[j];
+                ctx->S[j] = a;
+                if (++k >= key_len)
+                        k = 0;
+        }
+
+        return 0;
+}
+EXPORT_SYMBOL(arc4_setkey);
+
+void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in, unsigned int len)
+{
+        u32 *const S = ctx->S;
+        u32 x, y, a, b;
+        u32 ty, ta, tb;
+
+        if (len == 0)
+                return;
+
+        x = ctx->x;
+        y = ctx->y;
+
+        a = S[x];
+        y = (y + a) & 0xff;
+        b = S[y];
+
+        do {
+                S[y] = a;
+                a = (a + b) & 0xff;
+                S[x] = b;
+                x = (x + 1) & 0xff;
+                ta = S[x];
+                ty = (y + ta) & 0xff;
+                tb = S[ty];
+                *out++ = *in++ ^ S[a];
+                if (--len == 0)
+                        break;
+                y = ty;
+                a = ta;
+                b = tb;
+        } while (true);
+
+        ctx->x = x;
+        ctx->y = y;
+}
+EXPORT_SYMBOL(arc4_crypt);
+
+MODULE_LICENSE("GPL");
diff --git a/lib/scatterlist.c b/lib/scatterlist.c
index 2882d9ba6607..eacb82468437 100644
--- a/lib/scatterlist.c
+++ b/lib/scatterlist.c
@@ -676,17 +676,18 @@ static bool sg_miter_get_next_page(struct sg_mapping_iter *miter)
 {
         if (!miter->__remaining) {
                 struct scatterlist *sg;
-                unsigned long pgoffset;
 
                 if (!__sg_page_iter_next(&miter->piter))
                         return false;
 
                 sg = miter->piter.sg;
-                pgoffset = miter->piter.sg_pgoffset;
 
-                miter->__offset = pgoffset ? 0 : sg->offset;
+                miter->__offset = miter->piter.sg_pgoffset ? 0 : sg->offset;
+                miter->piter.sg_pgoffset += miter->__offset >> PAGE_SHIFT;
+                miter->__offset &= PAGE_SIZE - 1;
                 miter->__remaining = sg->offset + sg->length -
-                                (pgoffset << PAGE_SHIFT) - miter->__offset;
+                                     (miter->piter.sg_pgoffset << PAGE_SHIFT) -
+                                     miter->__offset;
                 miter->__remaining = min_t(unsigned long, miter->__remaining,
                                            PAGE_SIZE - miter->__offset);
         }
diff --git a/net/mac80211/Kconfig b/net/mac80211/Kconfig
index 0227cce9685e..0c93b1b7a826 100644
--- a/net/mac80211/Kconfig
+++ b/net/mac80211/Kconfig
@@ -3,7 +3,7 @@ config MAC80211
         tristate "Generic IEEE 802.11 Networking Stack (mac80211)"
         depends on CFG80211
         select CRYPTO
-        select CRYPTO_ARC4
+        select CRYPTO_LIB_ARC4
         select CRYPTO_AES
         select CRYPTO_CCM
         select CRYPTO_GCM
diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c
index a1973a26c7fc..3fae902937fd 100644
--- a/net/mac80211/cfg.c
+++ b/net/mac80211/cfg.c
@@ -14,6 +14,7 @@
 #include <linux/slab.h>
 #include <net/net_namespace.h>
 #include <linux/rcupdate.h>
+#include <linux/fips.h>
 #include <linux/if_ether.h>
 #include <net/cfg80211.h>
 #include "ieee80211_i.h"
@@ -402,9 +403,8 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev,
         case WLAN_CIPHER_SUITE_WEP40:
         case WLAN_CIPHER_SUITE_TKIP:
         case WLAN_CIPHER_SUITE_WEP104:
-                if (IS_ERR(local->wep_tx_tfm))
+                if (WARN_ON_ONCE(fips_enabled))
                         return -EINVAL;
-                break;
         case WLAN_CIPHER_SUITE_CCMP:
         case WLAN_CIPHER_SUITE_CCMP_256:
         case WLAN_CIPHER_SUITE_AES_CMAC:
diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h
index 6396d46a9a71..004e2e3adb88 100644
--- a/net/mac80211/ieee80211_i.h
+++ b/net/mac80211/ieee80211_i.h
@@ -1255,8 +1255,8 @@ struct ieee80211_local {
 
         struct rate_control_ref *rate_ctrl;
 
-        struct crypto_cipher *wep_tx_tfm;
-        struct crypto_cipher *wep_rx_tfm;
+        struct arc4_ctx wep_tx_ctx;
+        struct arc4_ctx wep_rx_ctx;
         u32 wep_iv;
 
         /* see iface.c */
diff --git a/net/mac80211/key.h b/net/mac80211/key.h
index be118c39433f..b8b9cd743bf4 100644
--- a/net/mac80211/key.h
+++ b/net/mac80211/key.h
@@ -11,6 +11,7 @@
 #include <linux/list.h>
 #include <linux/crypto.h>
 #include <linux/rcupdate.h>
+#include <crypto/arc4.h>
 #include <net/mac80211.h>
 
 #define NUM_DEFAULT_KEYS 4
diff --git a/net/mac80211/main.c b/net/mac80211/main.c
index 55583b71ffaf..087c90e461b5 100644
--- a/net/mac80211/main.c
+++ b/net/mac80211/main.c
@@ -10,6 +10,7 @@
 
 #include <net/mac80211.h>
 #include <linux/module.h>
+#include <linux/fips.h>
 #include <linux/init.h>
 #include <linux/netdevice.h>
 #include <linux/types.h>
@@ -730,8 +731,7 @@ EXPORT_SYMBOL(ieee80211_alloc_hw_nm);
 
 static int ieee80211_init_cipher_suites(struct ieee80211_local *local)
 {
-        bool have_wep = !(IS_ERR(local->wep_tx_tfm) ||
-                          IS_ERR(local->wep_rx_tfm));
+        bool have_wep = !fips_enabled; /* FIPS does not permit the use of RC4 */
         bool have_mfp = ieee80211_hw_check(&local->hw, MFP_CAPABLE);
         int n_suites = 0, r = 0, w = 0;
         u32 *suites;
@@ -1298,7 +1298,6 @@ int ieee80211_register_hw(struct ieee80211_hw *hw)
  fail_rate:
         rtnl_unlock();
         ieee80211_led_exit(local);
-        ieee80211_wep_free(local);
  fail_flows:
         destroy_workqueue(local->workqueue);
  fail_workqueue:
@@ -1355,7 +1354,6 @@ void ieee80211_unregister_hw(struct ieee80211_hw *hw)
 
         destroy_workqueue(local->workqueue);
         wiphy_unregister(local->hw.wiphy);
-        ieee80211_wep_free(local);
         ieee80211_led_exit(local);
         kfree(local->int_scan_req);
 }
diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c
index 379d2ab6d327..a00b703bfdfd 100644
--- a/net/mac80211/mlme.c
+++ b/net/mac80211/mlme.c
@@ -12,6 +12,7 @@
  */
 
 #include <linux/delay.h>
+#include <linux/fips.h>
 #include <linux/if_ether.h>
 #include <linux/skbuff.h>
 #include <linux/if_arp.h>
@@ -5045,7 +5046,7 @@ int ieee80211_mgd_auth(struct ieee80211_sub_if_data *sdata,
                 auth_alg = WLAN_AUTH_OPEN;
                 break;
         case NL80211_AUTHTYPE_SHARED_KEY:
-                if (IS_ERR(local->wep_tx_tfm))
+                if (fips_enabled)
                         return -EOPNOTSUPP;
                 auth_alg = WLAN_AUTH_SHARED_KEY;
                 break;
diff --git a/net/mac80211/tkip.c b/net/mac80211/tkip.c
index 7914b8e3ce8c..727dc9f3f3b3 100644
--- a/net/mac80211/tkip.c
+++ b/net/mac80211/tkip.c
@@ -219,7 +219,7 @@ EXPORT_SYMBOL(ieee80211_get_tkip_p2k);
  * @payload_len is the length of payload (_not_ including IV/ICV length).
  * @ta is the transmitter addresses.
  */
-int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm,
+int ieee80211_tkip_encrypt_data(struct arc4_ctx *ctx,
                                 struct ieee80211_key *key,
                                 struct sk_buff *skb,
                                 u8 *payload, size_t payload_len)
@@ -228,7 +228,7 @@ int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm,
 
         ieee80211_get_tkip_p2k(&key->conf, skb, rc4key);
 
-        return ieee80211_wep_encrypt_data(tfm, rc4key, 16,
+        return ieee80211_wep_encrypt_data(ctx, rc4key, 16,
                                           payload, payload_len);
 }
 
@@ -236,7 +236,7 @@ int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm,
  * beginning of the buffer containing IEEE 802.11 header payload, i.e.,
  * including IV, Ext. IV, real data, Michael MIC, ICV. @payload_len is the
  * length of payload, including IV, Ext. IV, MIC, ICV.  */
-int ieee80211_tkip_decrypt_data(struct crypto_cipher *tfm,
+int ieee80211_tkip_decrypt_data(struct arc4_ctx *ctx,
                                 struct ieee80211_key *key,
                                 u8 *payload, size_t payload_len, u8 *ta,
                                 u8 *ra, int only_iv, int queue,
@@ -294,7 +294,7 @@ int ieee80211_tkip_decrypt_data(struct crypto_cipher *tfm,
 
         tkip_mixing_phase2(tk, &rx_ctx->ctx, iv16, rc4key);
 
-        res = ieee80211_wep_decrypt_data(tfm, rc4key, 16, pos, payload_len - 12);
+        res = ieee80211_wep_decrypt_data(ctx, rc4key, 16, pos, payload_len - 12);
  done:
         if (res == TKIP_DECRYPT_OK) {
                 /*
diff --git a/net/mac80211/tkip.h b/net/mac80211/tkip.h
index 676a7babdf5d..9d2f8bd36cc7 100644
--- a/net/mac80211/tkip.h
+++ b/net/mac80211/tkip.h
@@ -10,7 +10,7 @@
 #include <linux/crypto.h>
 #include "key.h"
 
-int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm,
+int ieee80211_tkip_encrypt_data(struct arc4_ctx *ctx,
                                 struct ieee80211_key *key,
                                 struct sk_buff *skb,
                                 u8 *payload, size_t payload_len);
@@ -21,7 +21,7 @@ enum {
         TKIP_DECRYPT_INVALID_KEYIDX = -2,
         TKIP_DECRYPT_REPLAY = -3,
 };
-int ieee80211_tkip_decrypt_data(struct crypto_cipher *tfm,
+int ieee80211_tkip_decrypt_data(struct arc4_ctx *ctx,
                                 struct ieee80211_key *key,
                                 u8 *payload, size_t payload_len, u8 *ta,
                                 u8 *ra, int only_iv, int queue,
diff --git a/net/mac80211/wep.c b/net/mac80211/wep.c
index 3d9e92867ef0..b75c2c54e665 100644
--- a/net/mac80211/wep.c
+++ b/net/mac80211/wep.c
@@ -27,30 +27,9 @@ int ieee80211_wep_init(struct ieee80211_local *local)
         /* start WEP IV from a random value */
         get_random_bytes(&local->wep_iv, IEEE80211_WEP_IV_LEN);
 
-        local->wep_tx_tfm = crypto_alloc_cipher("arc4", 0, 0);
-        if (IS_ERR(local->wep_tx_tfm)) {
-                local->wep_rx_tfm = ERR_PTR(-EINVAL);
-                return PTR_ERR(local->wep_tx_tfm);
-        }
-
-        local->wep_rx_tfm = crypto_alloc_cipher("arc4", 0, 0);
-        if (IS_ERR(local->wep_rx_tfm)) {
-                crypto_free_cipher(local->wep_tx_tfm);
-                local->wep_tx_tfm = ERR_PTR(-EINVAL);
-                return PTR_ERR(local->wep_rx_tfm);
-        }
-
         return 0;
 }
 
-void ieee80211_wep_free(struct ieee80211_local *local)
-{
-        if (!IS_ERR(local->wep_tx_tfm))
-                crypto_free_cipher(local->wep_tx_tfm);
-        if (!IS_ERR(local->wep_rx_tfm))
-                crypto_free_cipher(local->wep_rx_tfm);
-}
-
 static inline bool ieee80211_wep_weak_iv(u32 iv, int keylen)
 {
         /*
@@ -128,21 +107,17 @@ static void ieee80211_wep_remove_iv(struct ieee80211_local *local,
 /* Perform WEP encryption using given key. data buffer must have tailroom
  * for 4-byte ICV. data_len must not include this ICV. Note: this function
  * does _not_ add IV. data = RC4(data | CRC32(data)) */
-int ieee80211_wep_encrypt_data(struct crypto_cipher *tfm, u8 *rc4key,
+int ieee80211_wep_encrypt_data(struct arc4_ctx *ctx, u8 *rc4key,
                                size_t klen, u8 *data, size_t data_len)
 {
         __le32 icv;
-        int i;
-
-        if (IS_ERR(tfm))
-                return -1;
 
         icv = cpu_to_le32(~crc32_le(~0, data, data_len));
         put_unaligned(icv, (__le32 *)(data + data_len));
 
-        crypto_cipher_setkey(tfm, rc4key, klen);
-        for (i = 0; i < data_len + IEEE80211_WEP_ICV_LEN; i++)
-                crypto_cipher_encrypt_one(tfm, data + i, data + i);
+        arc4_setkey(ctx, rc4key, klen);
+        arc4_crypt(ctx, data, data, data_len + IEEE80211_WEP_ICV_LEN);
+        memzero_explicit(ctx, sizeof(*ctx));
 
         return 0;
 }
@@ -181,7 +156,7 @@ int ieee80211_wep_encrypt(struct ieee80211_local *local,
         /* Add room for ICV */
         skb_put(skb, IEEE80211_WEP_ICV_LEN);
 
-        return ieee80211_wep_encrypt_data(local->wep_tx_tfm, rc4key, keylen + 3,
+        return ieee80211_wep_encrypt_data(&local->wep_tx_ctx, rc4key, keylen + 3,
                                           iv + IEEE80211_WEP_IV_LEN, len);
 }
 
@@ -189,18 +164,14 @@ int ieee80211_wep_encrypt(struct ieee80211_local *local,
 /* Perform WEP decryption using given key. data buffer includes encrypted
  * payload, including 4-byte ICV, but _not_ IV. data_len must not include ICV.
  * Return 0 on success and -1 on ICV mismatch. */
-int ieee80211_wep_decrypt_data(struct crypto_cipher *tfm, u8 *rc4key,
+int ieee80211_wep_decrypt_data(struct arc4_ctx *ctx, u8 *rc4key,
                                size_t klen, u8 *data, size_t data_len)
 {
         __le32 crc;
-        int i;
-
-        if (IS_ERR(tfm))
-                return -1;
 
-        crypto_cipher_setkey(tfm, rc4key, klen);
-        for (i = 0; i < data_len + IEEE80211_WEP_ICV_LEN; i++)
-                crypto_cipher_decrypt_one(tfm, data + i, data + i);
+        arc4_setkey(ctx, rc4key, klen);
+        arc4_crypt(ctx, data, data, data_len + IEEE80211_WEP_ICV_LEN);
+        memzero_explicit(ctx, sizeof(*ctx));
 
         crc = cpu_to_le32(~crc32_le(~0, data, data_len));
         if (memcmp(&crc, data + data_len, IEEE80211_WEP_ICV_LEN) != 0)
@@ -253,7 +224,7 @@ static int ieee80211_wep_decrypt(struct ieee80211_local *local,
         /* Copy rest of the WEP key (the secret part) */
         memcpy(rc4key + 3, key->conf.key, key->conf.keylen);
 
-        if (ieee80211_wep_decrypt_data(local->wep_rx_tfm, rc4key, klen,
+        if (ieee80211_wep_decrypt_data(&local->wep_rx_ctx, rc4key, klen,
                                        skb->data + hdrlen +
                                        IEEE80211_WEP_IV_LEN, len))
                 ret = -1;
diff --git a/net/mac80211/wep.h b/net/mac80211/wep.h
index 866a6798c9ef..997a034233c2 100644
--- a/net/mac80211/wep.h
+++ b/net/mac80211/wep.h
@@ -14,13 +14,12 @@
 #include "key.h"
 
 int ieee80211_wep_init(struct ieee80211_local *local);
-void ieee80211_wep_free(struct ieee80211_local *local);
-int ieee80211_wep_encrypt_data(struct crypto_cipher *tfm, u8 *rc4key,
+int ieee80211_wep_encrypt_data(struct arc4_ctx *ctx, u8 *rc4key,
                                 size_t klen, u8 *data, size_t data_len);
 int ieee80211_wep_encrypt(struct ieee80211_local *local,
                           struct sk_buff *skb,
                           const u8 *key, int keylen, int keyidx);
-int ieee80211_wep_decrypt_data(struct crypto_cipher *tfm, u8 *rc4key,
+int ieee80211_wep_decrypt_data(struct arc4_ctx *ctx, u8 *rc4key,
                                size_t klen, u8 *data, size_t data_len);
 
 ieee80211_rx_result
diff --git a/net/mac80211/wpa.c b/net/mac80211/wpa.c
index a51c7909366e..ee72779729e5 100644
--- a/net/mac80211/wpa.c
+++ b/net/mac80211/wpa.c
@@ -239,7 +239,7 @@ static int tkip_encrypt_skb(struct ieee80211_tx_data *tx, struct sk_buff *skb)
         /* Add room for ICV */
         skb_put(skb, IEEE80211_TKIP_ICV_LEN);
 
-        return ieee80211_tkip_encrypt_data(tx->local->wep_tx_tfm,
+        return ieee80211_tkip_encrypt_data(&tx->local->wep_tx_ctx,
                                            key, skb, pos, len);
 }
 
@@ -290,7 +290,7 @@ ieee80211_crypto_tkip_decrypt(struct ieee80211_rx_data *rx)
         if (status->flag & RX_FLAG_DECRYPTED)
                 hwaccel = 1;
 
-        res = ieee80211_tkip_decrypt_data(rx->local->wep_rx_tfm,
+        res = ieee80211_tkip_decrypt_data(&rx->local->wep_rx_ctx,
                                           key, skb->data + hdrlen,
                                           skb->len - hdrlen, rx->sta->sta.addr,
                                           hdr->addr1, hwaccel, rx->security_idx,
diff --git a/net/wireless/Kconfig b/net/wireless/Kconfig
index 6310ddede220..578cce4fbe6c 100644
--- a/net/wireless/Kconfig
+++ b/net/wireless/Kconfig
@@ -213,12 +213,14 @@ config LIB80211
 
 config LIB80211_CRYPT_WEP
         tristate
+        select CRYPTO_LIB_ARC4
 
 config LIB80211_CRYPT_CCMP
         tristate
 
 config LIB80211_CRYPT_TKIP
         tristate
+        select CRYPTO_LIB_ARC4
 
 config LIB80211_DEBUG
         bool "lib80211 debugging messages"
diff --git a/net/wireless/lib80211_crypt_tkip.c b/net/wireless/lib80211_crypt_tkip.c
index 62edf5b01953..f5e842ba7673 100644
--- a/net/wireless/lib80211_crypt_tkip.c
+++ b/net/wireless/lib80211_crypt_tkip.c
@@ -9,6 +9,7 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/err.h>
+#include <linux/fips.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/slab.h>
@@ -25,6 +26,7 @@
 #include <linux/ieee80211.h>
 #include <net/iw_handler.h>
 
+#include <crypto/arc4.h>
 #include <crypto/hash.h>
 #include <linux/crypto.h>
 #include <linux/crc32.h>
@@ -60,9 +62,9 @@ struct lib80211_tkip_data {
 
         int key_idx;
 
-        struct crypto_cipher *rx_tfm_arc4;
+        struct arc4_ctx rx_ctx_arc4;
+        struct arc4_ctx tx_ctx_arc4;
         struct crypto_shash *rx_tfm_michael;
-        struct crypto_cipher *tx_tfm_arc4;
         struct crypto_shash *tx_tfm_michael;
 
         /* scratch buffers for virt_to_page() (crypto API) */
@@ -89,30 +91,21 @@ static void *lib80211_tkip_init(int key_idx)
 {
         struct lib80211_tkip_data *priv;
 
+        if (fips_enabled)
+                return NULL;
+
         priv = kzalloc(sizeof(*priv), GFP_ATOMIC);
         if (priv == NULL)
                 goto fail;
 
         priv->key_idx = key_idx;
 
-        priv->tx_tfm_arc4 = crypto_alloc_cipher("arc4", 0, 0);
-        if (IS_ERR(priv->tx_tfm_arc4)) {
-                priv->tx_tfm_arc4 = NULL;
-                goto fail;
-        }
-
         priv->tx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0);
         if (IS_ERR(priv->tx_tfm_michael)) {
                 priv->tx_tfm_michael = NULL;
                 goto fail;
         }
 
-        priv->rx_tfm_arc4 = crypto_alloc_cipher("arc4", 0, 0);
-        if (IS_ERR(priv->rx_tfm_arc4)) {
-                priv->rx_tfm_arc4 = NULL;
-                goto fail;
-        }
-
         priv->rx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0);
         if (IS_ERR(priv->rx_tfm_michael)) {
                 priv->rx_tfm_michael = NULL;
@@ -124,9 +117,7 @@ static void *lib80211_tkip_init(int key_idx)
       fail:
         if (priv) {
                 crypto_free_shash(priv->tx_tfm_michael);
-                crypto_free_cipher(priv->tx_tfm_arc4);
                 crypto_free_shash(priv->rx_tfm_michael);
-                crypto_free_cipher(priv->rx_tfm_arc4);
                 kfree(priv);
         }
 
@@ -138,11 +129,9 @@ static void lib80211_tkip_deinit(void *priv)
         struct lib80211_tkip_data *_priv = priv;
         if (_priv) {
                 crypto_free_shash(_priv->tx_tfm_michael);
-                crypto_free_cipher(_priv->tx_tfm_arc4);
                 crypto_free_shash(_priv->rx_tfm_michael);
-                crypto_free_cipher(_priv->rx_tfm_arc4);
         }
-        kfree(priv);
+        kzfree(priv);
 }
 
 static inline u16 RotR1(u16 val)
@@ -341,7 +330,6 @@ static int lib80211_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
         int len;
         u8 rc4key[16], *pos, *icv;
         u32 crc;
-        int i;
 
         if (tkey->flags & IEEE80211_CRYPTO_TKIP_COUNTERMEASURES) {
                 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
@@ -366,9 +354,9 @@ static int lib80211_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
         icv[2] = crc >> 16;
         icv[3] = crc >> 24;
 
-        crypto_cipher_setkey(tkey->tx_tfm_arc4, rc4key, 16);
-        for (i = 0; i < len + 4; i++)
-                crypto_cipher_encrypt_one(tkey->tx_tfm_arc4, pos + i, pos + i);
+        arc4_setkey(&tkey->tx_ctx_arc4, rc4key, 16);
+        arc4_crypt(&tkey->tx_ctx_arc4, pos, pos, len + 4);
+
         return 0;
 }
 
@@ -396,7 +384,6 @@ static int lib80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
         u8 icv[4];
         u32 crc;
         int plen;
-        int i;
 
         hdr = (struct ieee80211_hdr *)skb->data;
 
@@ -449,9 +436,8 @@ static int lib80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
 
         plen = skb->len - hdr_len - 12;
 
-        crypto_cipher_setkey(tkey->rx_tfm_arc4, rc4key, 16);
-        for (i = 0; i < plen + 4; i++)
-                crypto_cipher_decrypt_one(tkey->rx_tfm_arc4, pos + i, pos + i);
+        arc4_setkey(&tkey->rx_ctx_arc4, rc4key, 16);
+        arc4_crypt(&tkey->rx_ctx_arc4, pos, pos, plen + 4);
 
         crc = ~crc32_le(~0, pos, plen);
         icv[0] = crc;
@@ -636,17 +622,17 @@ static int lib80211_tkip_set_key(void *key, int len, u8 * seq, void *priv)
         struct lib80211_tkip_data *tkey = priv;
         int keyidx;
         struct crypto_shash *tfm = tkey->tx_tfm_michael;
-        struct crypto_cipher *tfm2 = tkey->tx_tfm_arc4;
+        struct arc4_ctx *tfm2 = &tkey->tx_ctx_arc4;
         struct crypto_shash *tfm3 = tkey->rx_tfm_michael;
-        struct crypto_cipher *tfm4 = tkey->rx_tfm_arc4;
+        struct arc4_ctx *tfm4 = &tkey->rx_ctx_arc4;
 
         keyidx = tkey->key_idx;
         memset(tkey, 0, sizeof(*tkey));
         tkey->key_idx = keyidx;
         tkey->tx_tfm_michael = tfm;
-        tkey->tx_tfm_arc4 = tfm2;
+        tkey->tx_ctx_arc4 = *tfm2;
         tkey->rx_tfm_michael = tfm3;
-        tkey->rx_tfm_arc4 = tfm4;
+        tkey->rx_ctx_arc4 = *tfm4;
         if (len == TKIP_KEY_LEN) {
                 memcpy(tkey->key, key, TKIP_KEY_LEN);
                 tkey->key_set = 1;
diff --git a/net/wireless/lib80211_crypt_wep.c b/net/wireless/lib80211_crypt_wep.c
index e127b6f7fc9f..dafc6f3571db 100644
--- a/net/wireless/lib80211_crypt_wep.c
+++ b/net/wireless/lib80211_crypt_wep.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/err.h>
+#include <linux/fips.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/slab.h>
@@ -18,7 +19,7 @@
 
 #include <net/lib80211.h>
 
-#include <linux/crypto.h>
+#include <crypto/arc4.h>
 #include <linux/crc32.h>
 
 MODULE_AUTHOR("Jouni Malinen");
@@ -31,52 +32,31 @@ struct lib80211_wep_data {
         u8 key[WEP_KEY_LEN + 1];
         u8 key_len;
         u8 key_idx;
-        struct crypto_cipher *tx_tfm;
-        struct crypto_cipher *rx_tfm;
+        struct arc4_ctx tx_ctx;
+        struct arc4_ctx rx_ctx;
 };
 
 static void *lib80211_wep_init(int keyidx)
 {
         struct lib80211_wep_data *priv;
 
+        if (fips_enabled)
+                return NULL;
+
         priv = kzalloc(sizeof(*priv), GFP_ATOMIC);
         if (priv == NULL)
-                goto fail;
+                return NULL;
         priv->key_idx = keyidx;
 
-        priv->tx_tfm = crypto_alloc_cipher("arc4", 0, 0);
-        if (IS_ERR(priv->tx_tfm)) {
-                priv->tx_tfm = NULL;
-                goto fail;
-        }
-
-        priv->rx_tfm = crypto_alloc_cipher("arc4", 0, 0);
-        if (IS_ERR(priv->rx_tfm)) {
-                priv->rx_tfm = NULL;
-                goto fail;
-        }
         /* start WEP IV from a random value */
         get_random_bytes(&priv->iv, 4);
 
         return priv;
-
-      fail:
-        if (priv) {
-                crypto_free_cipher(priv->tx_tfm);
-                crypto_free_cipher(priv->rx_tfm);
-                kfree(priv);
-        }
-        return NULL;
 }
 
 static void lib80211_wep_deinit(void *priv)
 {
-        struct lib80211_wep_data *_priv = priv;
-        if (_priv) {
-                crypto_free_cipher(_priv->tx_tfm);
-                crypto_free_cipher(_priv->rx_tfm);
-        }
-        kfree(priv);
+        kzfree(priv);
 }
 
 /* Add WEP IV/key info to a frame that has at least 4 bytes of headroom */
@@ -128,7 +108,6 @@ static int lib80211_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
         u32 crc, klen, len;
         u8 *pos, *icv;
         u8 key[WEP_KEY_LEN + 3];
-        int i;
 
         /* other checks are in lib80211_wep_build_iv */
         if (skb_tailroom(skb) < 4)
@@ -156,10 +135,8 @@ static int lib80211_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv)
         icv[2] = crc >> 16;
         icv[3] = crc >> 24;
 
-        crypto_cipher_setkey(wep->tx_tfm, key, klen);
-
-        for (i = 0; i < len + 4; i++)
-                crypto_cipher_encrypt_one(wep->tx_tfm, pos + i, pos + i);
+        arc4_setkey(&wep->tx_ctx, key, klen);
+        arc4_crypt(&wep->tx_ctx, pos, pos, len + 4);
 
         return 0;
 }
@@ -177,7 +154,6 @@ static int lib80211_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
         u32 crc, klen, plen;
         u8 key[WEP_KEY_LEN + 3];
         u8 keyidx, *pos, icv[4];
-        int i;
 
         if (skb->len < hdr_len + 8)
                 return -1;
@@ -198,9 +174,8 @@ static int lib80211_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv)
         /* Apply RC4 to data and compute CRC32 over decrypted data */
         plen = skb->len - hdr_len - 8;
 
-        crypto_cipher_setkey(wep->rx_tfm, key, klen);
-        for (i = 0; i < plen + 4; i++)
-                crypto_cipher_decrypt_one(wep->rx_tfm, pos + i, pos + i);
+        arc4_setkey(&wep->rx_ctx, key, klen);
+        arc4_crypt(&wep->rx_ctx, pos, pos, plen + 4);
 
         crc = ~crc32_le(~0, pos, plen);
         icv[0] = crc;