crypto: s390 - add System z hardware support for CTR mode

This patch adds System z hardware acceleration support for AES, DES
and 3DES in CTR mode. The hardware support is available starting with
System z196.

Signed-off-by: Gerald Schaefer <gerald.schaefer@de.ibm.com>
Signed-off-by: Jan Glauber <jang@linux.vnet.ibm.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

1 files changed, 168 insertions, 1 deletions

diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c
index c36a01d70e04..a52bfd124d86 100644
--- a/arch/s390/crypto/des_s390.c
+++ b/arch/s390/crypto/des_s390.c
@@ -3,7 +3,7 @@
  *
  * s390 implementation of the DES Cipher Algorithm.
  *
- * Copyright IBM Corp. 2003,2007
+ * Copyright IBM Corp. 2003,2011
  * Author(s): Thomas Spatzier
  *            Jan Glauber (jan.glauber@de.ibm.com)
  *
@@ -24,6 +24,8 @@
 
 #define DES3_KEY_SIZE   (3 * DES_KEY_SIZE)
 
+static u8 *ctrblk;
+
 struct s390_des_ctx {
         u8 iv[DES_BLOCK_SIZE];
         u8 key[DES3_KEY_SIZE];
@@ -370,6 +372,143 @@ static struct crypto_alg cbc_des3_alg = {
         }
 };
 
+static int ctr_desall_crypt(struct blkcipher_desc *desc, long func,
+                            struct s390_des_ctx *ctx, struct blkcipher_walk *walk)
+{
+        int ret = blkcipher_walk_virt_block(desc, walk, DES_BLOCK_SIZE);
+        unsigned int i, n, nbytes;
+        u8 buf[DES_BLOCK_SIZE];
+        u8 *out, *in;
+
+        memcpy(ctrblk, walk->iv, DES_BLOCK_SIZE);
+        while ((nbytes = walk->nbytes) >= DES_BLOCK_SIZE) {
+                out = walk->dst.virt.addr;
+                in = walk->src.virt.addr;
+                while (nbytes >= DES_BLOCK_SIZE) {
+                        /* align to block size, max. PAGE_SIZE */
+                        n = (nbytes > PAGE_SIZE) ? PAGE_SIZE :
+                                nbytes & ~(DES_BLOCK_SIZE - 1);
+                        for (i = DES_BLOCK_SIZE; i < n; i += DES_BLOCK_SIZE) {
+                                memcpy(ctrblk + i, ctrblk + i - DES_BLOCK_SIZE,
+                                       DES_BLOCK_SIZE);
+                                crypto_inc(ctrblk + i, DES_BLOCK_SIZE);
+                        }
+                        ret = crypt_s390_kmctr(func, ctx->key, out, in, n, ctrblk);
+                        BUG_ON((ret < 0) || (ret != n));
+                        if (n > DES_BLOCK_SIZE)
+                                memcpy(ctrblk, ctrblk + n - DES_BLOCK_SIZE,
+                                       DES_BLOCK_SIZE);
+                        crypto_inc(ctrblk, DES_BLOCK_SIZE);
+                        out += n;
+                        in += n;
+                        nbytes -= n;
+                }
+                ret = blkcipher_walk_done(desc, walk, nbytes);
+        }
+
+        /* final block may be < DES_BLOCK_SIZE, copy only nbytes */
+        if (nbytes) {
+                out = walk->dst.virt.addr;
+                in = walk->src.virt.addr;
+                ret = crypt_s390_kmctr(func, ctx->key, buf, in,
+                                       DES_BLOCK_SIZE, ctrblk);
+                BUG_ON(ret < 0 || ret != DES_BLOCK_SIZE);
+                memcpy(out, buf, nbytes);
+                crypto_inc(ctrblk, DES_BLOCK_SIZE);
+                ret = blkcipher_walk_done(desc, walk, 0);
+        }
+        memcpy(walk->iv, ctrblk, DES_BLOCK_SIZE);
+        return ret;
+}
+
+static int ctr_des_encrypt(struct blkcipher_desc *desc,
+                           struct scatterlist *dst, struct scatterlist *src,
+                           unsigned int nbytes)
+{
+        struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+        struct blkcipher_walk walk;
+
+        blkcipher_walk_init(&walk, dst, src, nbytes);
+        return ctr_desall_crypt(desc, KMCTR_DEA_ENCRYPT, ctx, &walk);
+}
+
+static int ctr_des_decrypt(struct blkcipher_desc *desc,
+                           struct scatterlist *dst, struct scatterlist *src,
+                           unsigned int nbytes)
+{
+        struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+        struct blkcipher_walk walk;
+
+        blkcipher_walk_init(&walk, dst, src, nbytes);
+        return ctr_desall_crypt(desc, KMCTR_DEA_DECRYPT, ctx, &walk);
+}
+
+static struct crypto_alg ctr_des_alg = {
+        .cra_name               =       "ctr(des)",
+        .cra_driver_name        =       "ctr-des-s390",
+        .cra_priority           =       CRYPT_S390_COMPOSITE_PRIORITY,
+        .cra_flags              =       CRYPTO_ALG_TYPE_BLKCIPHER,
+        .cra_blocksize          =       1,
+        .cra_ctxsize            =       sizeof(struct s390_des_ctx),
+        .cra_type               =       &crypto_blkcipher_type,
+        .cra_module             =       THIS_MODULE,
+        .cra_list               =       LIST_HEAD_INIT(ctr_des_alg.cra_list),
+        .cra_u                  =       {
+                .blkcipher = {
+                        .min_keysize            =       DES_KEY_SIZE,
+                        .max_keysize            =       DES_KEY_SIZE,
+                        .ivsize                 =       DES_BLOCK_SIZE,
+                        .setkey                 =       des_setkey,
+                        .encrypt                =       ctr_des_encrypt,
+                        .decrypt                =       ctr_des_decrypt,
+                }
+        }
+};
+
+static int ctr_des3_encrypt(struct blkcipher_desc *desc,
+                            struct scatterlist *dst, struct scatterlist *src,
+                            unsigned int nbytes)
+{
+        struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+        struct blkcipher_walk walk;
+
+        blkcipher_walk_init(&walk, dst, src, nbytes);
+        return ctr_desall_crypt(desc, KMCTR_TDEA_192_ENCRYPT, ctx, &walk);
+}
+
+static int ctr_des3_decrypt(struct blkcipher_desc *desc,
+                            struct scatterlist *dst, struct scatterlist *src,
+                            unsigned int nbytes)
+{
+        struct s390_des_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+        struct blkcipher_walk walk;
+
+        blkcipher_walk_init(&walk, dst, src, nbytes);
+        return ctr_desall_crypt(desc, KMCTR_TDEA_192_DECRYPT, ctx, &walk);
+}
+
+static struct crypto_alg ctr_des3_alg = {
+        .cra_name               =       "ctr(des3_ede)",
+        .cra_driver_name        =       "ctr-des3_ede-s390",
+        .cra_priority           =       CRYPT_S390_COMPOSITE_PRIORITY,
+        .cra_flags              =       CRYPTO_ALG_TYPE_BLKCIPHER,
+        .cra_blocksize          =       1,
+        .cra_ctxsize            =       sizeof(struct s390_des_ctx),
+        .cra_type               =       &crypto_blkcipher_type,
+        .cra_module             =       THIS_MODULE,
+        .cra_list               =       LIST_HEAD_INIT(ctr_des3_alg.cra_list),
+        .cra_u                  =       {
+                .blkcipher = {
+                        .min_keysize            =       DES3_KEY_SIZE,
+                        .max_keysize            =       DES3_KEY_SIZE,
+                        .ivsize                 =       DES_BLOCK_SIZE,
+                        .setkey                 =       des3_setkey,
+                        .encrypt                =       ctr_des3_encrypt,
+                        .decrypt                =       ctr_des3_decrypt,
+                }
+        }
+};
+
 static int __init des_s390_init(void)
 {
         int ret;
@@ -396,9 +535,32 @@ static int __init des_s390_init(void)
         ret = crypto_register_alg(&cbc_des3_alg);
         if (ret)
                 goto cbc_des3_err;
+
+        if (crypt_s390_func_available(KMCTR_DEA_ENCRYPT,
+                        CRYPT_S390_MSA | CRYPT_S390_MSA4) &&
+            crypt_s390_func_available(KMCTR_TDEA_192_ENCRYPT,
+                        CRYPT_S390_MSA | CRYPT_S390_MSA4)) {
+                ret = crypto_register_alg(&ctr_des_alg);
+                if (ret)
+                        goto ctr_des_err;
+                ret = crypto_register_alg(&ctr_des3_alg);
+                if (ret)
+                        goto ctr_des3_err;
+                ctrblk = (u8 *) __get_free_page(GFP_KERNEL);
+                if (!ctrblk) {
+                        ret = -ENOMEM;
+                        goto ctr_mem_err;
+                }
+        }
 out:
         return ret;
 
+ctr_mem_err:
+        crypto_unregister_alg(&ctr_des3_alg);
+ctr_des3_err:
+        crypto_unregister_alg(&ctr_des_alg);
+ctr_des_err:
+        crypto_unregister_alg(&cbc_des3_alg);
 cbc_des3_err:
         crypto_unregister_alg(&ecb_des3_alg);
 ecb_des3_err:
@@ -415,6 +577,11 @@ des_err:
 
 static void __exit des_s390_exit(void)
 {
+        if (ctrblk) {
+                crypto_unregister_alg(&ctr_des_alg);
+                crypto_unregister_alg(&ctr_des3_alg);
+                free_page((unsigned long) ctrblk);
+        }
         crypto_unregister_alg(&cbc_des3_alg);
         crypto_unregister_alg(&ecb_des3_alg);
         crypto_unregister_alg(&des3_alg);