crypto: x86/glue_helper - use le128 instead of u128 for CTR mode

'u128' currently used for CTR mode is on little-endian 'long long' swapped
and would require extra swap operations by SSE/AVX code. Use of le128
instead of u128 allows IV calculations to be done with vector registers
easier.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

9 files changed, 64 insertions, 58 deletions

diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c
index 42ffd2bbab5b..021a0086186b 100644
--- a/arch/x86/crypto/camellia_glue.c
+++ b/arch/x86/crypto/camellia_glue.c
@@ -1317,21 +1317,21 @@ static void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src)
         u128_xor(&dst[1], &dst[1], &iv);
 }
 
-static void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
+static void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 {
         be128 ctrblk;
 
         if (dst != src)
                 *dst = *src;
 
-        u128_to_be128(&ctrblk, iv);
-        u128_inc(iv);
+        le128_to_be128(&ctrblk, iv);
+        le128_inc(iv);
 
         camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)&ctrblk);
 }
 
 static void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src,
-                                    u128 *iv)
+                                    le128 *iv)
 {
         be128 ctrblks[2];
 
@@ -1340,10 +1340,10 @@ static void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src,
                 dst[1] = src[1];
         }
 
-        u128_to_be128(&ctrblks[0], iv);
-        u128_inc(iv);
-        u128_to_be128(&ctrblks[1], iv);
-        u128_inc(iv);
+        le128_to_be128(&ctrblks[0], iv);
+        le128_inc(iv);
+        le128_to_be128(&ctrblks[1], iv);
+        le128_inc(iv);
 
         camellia_enc_blk_xor_2way(ctx, (u8 *)dst, (u8 *)ctrblks);
 }
diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c
index 15e5f85a5011..1dfd33b5b4fb 100644
--- a/arch/x86/crypto/cast6_avx_glue.c
+++ b/arch/x86/crypto/cast6_avx_glue.c
@@ -78,19 +78,19 @@ static void cast6_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
                 u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
 }
 
-static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
+static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 {
         be128 ctrblk;
 
-        u128_to_be128(&ctrblk, iv);
-        u128_inc(iv);
+        le128_to_be128(&ctrblk, iv);
+        le128_inc(iv);
 
         __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
         u128_xor(dst, src, (u128 *)&ctrblk);
 }
 
 static void cast6_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
-                                   u128 *iv)
+                                 le128 *iv)
 {
         be128 ctrblks[CAST6_PARALLEL_BLOCKS];
         unsigned int i;
@@ -99,8 +99,8 @@ static void cast6_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
                 if (dst != src)
                         dst[i] = src[i];
 
-                u128_to_be128(&ctrblks[i], iv);
-                u128_inc(iv);
+                le128_to_be128(&ctrblks[i], iv);
+                le128_inc(iv);
         }
 
         cast6_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c
index 30b3927bd733..22ce4f683e55 100644
--- a/arch/x86/crypto/glue_helper.c
+++ b/arch/x86/crypto/glue_helper.c
@@ -221,16 +221,16 @@ static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr,
         u8 *src = (u8 *)walk->src.virt.addr;
         u8 *dst = (u8 *)walk->dst.virt.addr;
         unsigned int nbytes = walk->nbytes;
-        u128 ctrblk;
+        le128 ctrblk;
         u128 tmp;
 
-        be128_to_u128(&ctrblk, (be128 *)walk->iv);
+        be128_to_le128(&ctrblk, (be128 *)walk->iv);
 
         memcpy(&tmp, src, nbytes);
         fn_ctr(ctx, &tmp, &tmp, &ctrblk);
         memcpy(dst, &tmp, nbytes);
 
-        u128_to_be128((be128 *)walk->iv, &ctrblk);
+        le128_to_be128((be128 *)walk->iv, &ctrblk);
 }
 EXPORT_SYMBOL_GPL(glue_ctr_crypt_final_128bit);
 
@@ -243,11 +243,11 @@ static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
         unsigned int nbytes = walk->nbytes;
         u128 *src = (u128 *)walk->src.virt.addr;
         u128 *dst = (u128 *)walk->dst.virt.addr;
-        u128 ctrblk;
+        le128 ctrblk;
         unsigned int num_blocks, func_bytes;
         unsigned int i;
 
-        be128_to_u128(&ctrblk, (be128 *)walk->iv);
+        be128_to_le128(&ctrblk, (be128 *)walk->iv);
 
         /* Process multi-block batch */
         for (i = 0; i < gctx->num_funcs; i++) {
@@ -269,7 +269,7 @@ static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
         }
 
 done:
-        u128_to_be128((be128 *)walk->iv, &ctrblk);
+        le128_to_be128((be128 *)walk->iv, &ctrblk);
         return nbytes;
 }
 
diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c
index 3f543a04cf1e..2aa31ade1e68 100644
--- a/arch/x86/crypto/serpent_avx_glue.c
+++ b/arch/x86/crypto/serpent_avx_glue.c
@@ -56,19 +56,19 @@ static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
                 u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
 }
 
-static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
+static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
 {
         be128 ctrblk;
 
-        u128_to_be128(&ctrblk, iv);
-        u128_inc(iv);
+        le128_to_be128(&ctrblk, iv);
+        le128_inc(iv);
 
         __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
         u128_xor(dst, src, (u128 *)&ctrblk);
 }