aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm64
diff options
context:
space:
mode:
authorArd Biesheuvel <ard.biesheuvel@linaro.org>2014-03-26 15:53:05 -0400
committerArd Biesheuvel <ard.biesheuvel@linaro.org>2014-05-14 13:04:07 -0400
commitfdd2389457b209a9723c3be818fcf301f35db906 (patch)
tree086cdabe6082cf998ef5ca07fb9235ad04c1d8d6 /arch/arm64
parent6ba6c74dfc6bcf43312ef572592f7d4ebb3aedfa (diff)
arm64/crypto: GHASH secure hash using ARMv8 Crypto Extensions
This is a port to ARMv8 (Crypto Extensions) of the Intel implementation of the GHASH Secure Hash (used in the Galois/Counter chaining mode). It relies on the optional PMULL/PMULL2 instruction (polynomial multiply long, what Intel call carry-less multiply). Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
Diffstat (limited to 'arch/arm64')
-rw-r--r--arch/arm64/crypto/Kconfig6
-rw-r--r--arch/arm64/crypto/Makefile3
-rw-r--r--arch/arm64/crypto/ghash-ce-core.S95
-rw-r--r--arch/arm64/crypto/ghash-ce-glue.c155
4 files changed, 259 insertions, 0 deletions
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index eb1e99770c21..0c50859ee7b9 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -18,4 +18,10 @@ config CRYPTO_SHA2_ARM64_CE
18 depends on ARM64 && KERNEL_MODE_NEON 18 depends on ARM64 && KERNEL_MODE_NEON
19 select CRYPTO_HASH 19 select CRYPTO_HASH
20 20
21
22config CRYPTO_GHASH_ARM64_CE
23 tristate "GHASH (for GCM chaining mode) using ARMv8 Crypto Extensions"
24 depends on ARM64 && KERNEL_MODE_NEON
25 select CRYPTO_HASH
26
21endif 27endif
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index 0b3885a60d43..e8c81a068868 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -13,3 +13,6 @@ sha1-ce-y := sha1-ce-glue.o sha1-ce-core.o
13 13
14obj-$(CONFIG_CRYPTO_SHA2_ARM64_CE) += sha2-ce.o 14obj-$(CONFIG_CRYPTO_SHA2_ARM64_CE) += sha2-ce.o
15sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o 15sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o
16
17obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o
18ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S
new file mode 100644
index 000000000000..b9e6eaf41c9b
--- /dev/null
+++ b/arch/arm64/crypto/ghash-ce-core.S
@@ -0,0 +1,95 @@
1/*
2 * Accelerated GHASH implementation with ARMv8 PMULL instructions.
3 *
4 * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * Based on arch/x86/crypto/ghash-pmullni-intel_asm.S
7 *
8 * Copyright (c) 2009 Intel Corp.
9 * Author: Huang Ying <ying.huang@intel.com>
10 * Vinodh Gopal
11 * Erdinc Ozturk
12 * Deniz Karakoyunlu
13 *
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU General Public License version 2 as published
16 * by the Free Software Foundation.
17 */
18
19#include <linux/linkage.h>
20#include <asm/assembler.h>
21
22 DATA .req v0
23 SHASH .req v1
24 IN1 .req v2
25 T1 .req v2
26 T2 .req v3
27 T3 .req v4
28 VZR .req v5
29
30 .text
31 .arch armv8-a+crypto
32
33 /*
34 * void pmull_ghash_update(int blocks, u64 dg[], const char *src,
35 * struct ghash_key const *k, const char *head)
36 */
37ENTRY(pmull_ghash_update)
38 ld1 {DATA.16b}, [x1]
39 ld1 {SHASH.16b}, [x3]
40 eor VZR.16b, VZR.16b, VZR.16b
41
42 /* do the head block first, if supplied */
43 cbz x4, 0f
44 ld1 {IN1.2d}, [x4]
45 b 1f
46
470: ld1 {IN1.2d}, [x2], #16
48 sub w0, w0, #1
491: ext IN1.16b, IN1.16b, IN1.16b, #8
50CPU_LE( rev64 IN1.16b, IN1.16b )
51 eor DATA.16b, DATA.16b, IN1.16b
52
53 /* multiply DATA by SHASH in GF(2^128) */
54 ext T2.16b, DATA.16b, DATA.16b, #8
55 ext T3.16b, SHASH.16b, SHASH.16b, #8
56 eor T2.16b, T2.16b, DATA.16b
57 eor T3.16b, T3.16b, SHASH.16b
58
59 pmull2 T1.1q, SHASH.2d, DATA.2d // a1 * b1
60 pmull DATA.1q, SHASH.1d, DATA.1d // a0 * b0
61 pmull T2.1q, T2.1d, T3.1d // (a1 + a0)(b1 + b0)
62 eor T2.16b, T2.16b, T1.16b // (a0 * b1) + (a1 * b0)
63 eor T2.16b, T2.16b, DATA.16b
64
65 ext T3.16b, VZR.16b, T2.16b, #8
66 ext T2.16b, T2.16b, VZR.16b, #8
67 eor DATA.16b, DATA.16b, T3.16b
68 eor T1.16b, T1.16b, T2.16b // <T1:DATA> is result of
69 // carry-less multiplication
70
71 /* first phase of the reduction */
72 shl T3.2d, DATA.2d, #1
73 eor T3.16b, T3.16b, DATA.16b
74 shl T3.2d, T3.2d, #5
75 eor T3.16b, T3.16b, DATA.16b
76 shl T3.2d, T3.2d, #57
77 ext T2.16b, VZR.16b, T3.16b, #8
78 ext T3.16b, T3.16b, VZR.16b, #8
79 eor DATA.16b, DATA.16b, T2.16b
80 eor T1.16b, T1.16b, T3.16b
81
82 /* second phase of the reduction */
83 ushr T2.2d, DATA.2d, #5
84 eor T2.16b, T2.16b, DATA.16b
85 ushr T2.2d, T2.2d, #1
86 eor T2.16b, T2.16b, DATA.16b
87 ushr T2.2d, T2.2d, #1
88 eor T1.16b, T1.16b, T2.16b
89 eor DATA.16b, DATA.16b, T1.16b
90
91 cbnz w0, 0b
92
93 st1 {DATA.16b}, [x1]
94 ret
95ENDPROC(pmull_ghash_update)
diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c
new file mode 100644
index 000000000000..b92baf3f68c7
--- /dev/null
+++ b/arch/arm64/crypto/ghash-ce-glue.c
@@ -0,0 +1,155 @@
1/*
2 * Accelerated GHASH implementation with ARMv8 PMULL instructions.
3 *
4 * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 */
10
11#include <asm/neon.h>
12#include <asm/unaligned.h>
13#include <crypto/internal/hash.h>
14#include <linux/cpufeature.h>
15#include <linux/crypto.h>
16#include <linux/module.h>
17
18MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions");
19MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
20MODULE_LICENSE("GPL v2");
21
22#define GHASH_BLOCK_SIZE 16
23#define GHASH_DIGEST_SIZE 16
24
25struct ghash_key {
26 u64 a;
27 u64 b;
28};
29
30struct ghash_desc_ctx {
31 u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
32 u8 buf[GHASH_BLOCK_SIZE];
33 u32 count;
34};
35
36asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src,
37 struct ghash_key const *k, const char *head);
38
39static int ghash_init(struct shash_desc *desc)
40{
41 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
42
43 *ctx = (struct ghash_desc_ctx){};
44 return 0;
45}
46
47static int ghash_update(struct shash_desc *desc, const u8 *src,
48 unsigned int len)
49{
50 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
51 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
52
53 ctx->count += len;
54
55 if ((partial + len) >= GHASH_BLOCK_SIZE) {
56 struct ghash_key *key = crypto_shash_ctx(desc->tfm);
57 int blocks;
58
59 if (partial) {
60 int p = GHASH_BLOCK_SIZE - partial;
61
62 memcpy(ctx->buf + partial, src, p);
63 src += p;
64 len -= p;
65 }
66
67 blocks = len / GHASH_BLOCK_SIZE;
68 len %= GHASH_BLOCK_SIZE;
69
70 kernel_neon_begin_partial(6);
71 pmull_ghash_update(blocks, ctx->digest, src, key,
72 partial ? ctx->buf : NULL);
73 kernel_neon_end();
74 src += blocks * GHASH_BLOCK_SIZE;
75 }
76 if (len)
77 memcpy(ctx->buf + partial, src, len);
78 return 0;
79}
80
81static int ghash_final(struct shash_desc *desc, u8 *dst)
82{
83 struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
84 unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
85
86 if (partial) {
87 struct ghash_key *key = crypto_shash_ctx(desc->tfm);
88
89 memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
90
91 kernel_neon_begin_partial(6);
92 pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL);
93 kernel_neon_end();
94 }
95 put_unaligned_be64(ctx->digest[1], dst);
96 put_unaligned_be64(ctx->digest[0], dst + 8);
97
98 *ctx = (struct ghash_desc_ctx){};
99 return 0;
100}
101
102static int ghash_setkey(struct crypto_shash *tfm,
103 const u8 *inkey, unsigned int keylen)
104{
105 struct ghash_key *key = crypto_shash_ctx(tfm);
106 u64 a, b;
107
108 if (keylen != GHASH_BLOCK_SIZE) {
109 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
110 return -EINVAL;
111 }
112
113 /* perform multiplication by 'x' in GF(2^128) */
114 b = get_unaligned_be64(inkey);
115 a = get_unaligned_be64(inkey + 8);
116
117 key->a = (a << 1) | (b >> 63);
118 key->b = (b << 1) | (a >> 63);
119
120 if (b >> 63)
121 key->b ^= 0xc200000000000000UL;
122
123 return 0;
124}
125
126static struct shash_alg ghash_alg = {
127 .digestsize = GHASH_DIGEST_SIZE,
128 .init = ghash_init,
129 .update = ghash_update,
130 .final = ghash_final,
131 .setkey = ghash_setkey,
132 .descsize = sizeof(struct ghash_desc_ctx),
133 .base = {
134 .cra_name = "ghash",
135 .cra_driver_name = "ghash-ce",
136 .cra_priority = 200,
137 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
138 .cra_blocksize = GHASH_BLOCK_SIZE,
139 .cra_ctxsize = sizeof(struct ghash_key),
140 .cra_module = THIS_MODULE,
141 },
142};
143
144static int __init ghash_ce_mod_init(void)
145{
146 return crypto_register_shash(&ghash_alg);
147}
148
149static void __exit ghash_ce_mod_exit(void)
150{
151 crypto_unregister_shash(&ghash_alg);
152}
153
154module_cpu_feature_match(PMULL, ghash_ce_mod_init);
155module_exit(ghash_ce_mod_exit);
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-30 03:19:03 -0400