aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorArd Biesheuvel <ard.biesheuvel@linaro.org>2018-01-19 07:04:39 -0500
committerHerbert Xu <herbert@gondor.apana.org.au>2018-01-25 09:10:35 -0500
commit140aa50d68ea01e7caea6900f6f51882d4dd65c5 (patch)
treebd885fab91d34f380acd75321c9f465913f55cd3
parent15d5910e92614e642e7485bb9e89d46e4d1d65d9 (diff)
crypto: arm64/sm3 - new v8.2 Crypto Extensions implementation
Implement the Chinese SM3 secure hash algorithm using the new special instructions that have been introduced as an optional extension in ARMv8.2. Tested-by: Steve Capper <steve.capper@arm.com> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Diffstat
-rw-r--r--arch/arm64/crypto/Kconfig6
-rw-r--r--arch/arm64/crypto/Makefile3
-rw-r--r--arch/arm64/crypto/sm3-ce-core.S141
-rw-r--r--arch/arm64/crypto/sm3-ce-glue.c92
4 files changed, 242 insertions, 0 deletions
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 3321b2c9a2b5..285c36c7b408 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -41,6 +41,12 @@ config CRYPTO_SHA3_ARM64
41 select CRYPTO_HASH 41 select CRYPTO_HASH
42 select CRYPTO_SHA3 42 select CRYPTO_SHA3
43 43
44config CRYPTO_SM3_ARM64_CE
45 tristate "SM3 digest algorithm (ARMv8.2 Crypto Extensions)"
46 depends on KERNEL_MODE_NEON
47 select CRYPTO_HASH
48 select CRYPTO_SM3
49
44config CRYPTO_GHASH_ARM64_CE 50config CRYPTO_GHASH_ARM64_CE
45 tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions" 51 tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions"
46 depends on KERNEL_MODE_NEON 52 depends on KERNEL_MODE_NEON
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index 4ca2d146e213..cee9b8d9830b 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -20,6 +20,9 @@ sha512-ce-y := sha512-ce-glue.o sha512-ce-core.o
20obj-$(CONFIG_CRYPTO_SHA3_ARM64) += sha3-ce.o 20obj-$(CONFIG_CRYPTO_SHA3_ARM64) += sha3-ce.o
21sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o 21sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o
22 22
23obj-$(CONFIG_CRYPTO_SM3_ARM64_CE) += sm3-ce.o
24sm3-ce-y := sm3-ce-glue.o sm3-ce-core.o
25
23obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o 26obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o
24ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o 27ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
25 28
diff --git a/arch/arm64/crypto/sm3-ce-core.S b/arch/arm64/crypto/sm3-ce-core.S
new file mode 100644
index 000000000000..27169fe07a68
--- /dev/null
+++ b/arch/arm64/crypto/sm3-ce-core.S
@@ -0,0 +1,141 @@
1/*
2 * sm3-ce-core.S - SM3 secure hash using ARMv8.2 Crypto Extensions
3 *
4 * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/linkage.h>
12#include <asm/assembler.h>
13
14 .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
15 .set .Lv\b\().4s, \b
16 .endr
17
18 .macro sm3partw1, rd, rn, rm
19 .inst 0xce60c000 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
20 .endm
21
22 .macro sm3partw2, rd, rn, rm
23 .inst 0xce60c400 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
24 .endm
25
26 .macro sm3ss1, rd, rn, rm, ra
27 .inst 0xce400000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
28 .endm
29
30 .macro sm3tt1a, rd, rn, rm, imm2
31 .inst 0xce408000 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
32 .endm
33
34 .macro sm3tt1b, rd, rn, rm, imm2
35 .inst 0xce408400 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
36 .endm
37
38 .macro sm3tt2a, rd, rn, rm, imm2
39 .inst 0xce408800 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
40 .endm
41
42 .macro sm3tt2b, rd, rn, rm, imm2
43 .inst 0xce408c00 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
44 .endm
45
46 .macro round, ab, s0, t0, t1, i
47 sm3ss1 v5.4s, v8.4s, \t0\().4s, v9.4s
48 shl \t1\().4s, \t0\().4s, #1
49 sri \t1\().4s, \t0\().4s, #31
50 sm3tt1\ab v8.4s, v5.4s, v10.4s, \i
51 sm3tt2\ab v9.4s, v5.4s, \s0\().4s, \i
52 .endm
53
54 .macro qround, ab, s0, s1, s2, s3, s4
55 .ifnb \s4
56 ext \s4\().16b, \s1\().16b, \s2\().16b, #12
57 ext v6.16b, \s0\().16b, \s1\().16b, #12
58 ext v7.16b, \s2\().16b, \s3\().16b, #8
59 sm3partw1 \s4\().4s, \s0\().4s, \s3\().4s
60 .endif
61
62 eor v10.16b, \s0\().16b, \s1\().16b
63
64 round \ab, \s0, v11, v12, 0
65 round \ab, \s0, v12, v11, 1
66 round \ab, \s0, v11, v12, 2
67 round \ab, \s0, v12, v11, 3
68
69 .ifnb \s4
70 sm3partw2 \s4\().4s, v7.4s, v6.4s
71 .endif
72 .endm
73
74 /*
75 * void sm3_ce_transform(struct sm3_state *sst, u8 const *src,
76 * int blocks)
77 */
78 .text
79ENTRY(sm3_ce_transform)
80 /* load state */
81 ld1 {v8.4s-v9.4s}, [x0]
82 rev64 v8.4s, v8.4s
83 rev64 v9.4s, v9.4s
84 ext v8.16b, v8.16b, v8.16b, #8
85 ext v9.16b, v9.16b, v9.16b, #8
86
87 adr_l x8, .Lt
88 ldp s13, s14, [x8]
89
90 /* load input */
910: ld1 {v0.16b-v3.16b}, [x1], #64
92 sub w2, w2, #1
93
94 mov v15.16b, v8.16b
95 mov v16.16b, v9.16b
96
97CPU_LE( rev32 v0.16b, v0.16b )
98CPU_LE( rev32 v1.16b, v1.16b )
99CPU_LE( rev32 v2.16b, v2.16b )
100CPU_LE( rev32 v3.16b, v3.16b )
101
102 ext v11.16b, v13.16b, v13.16b, #4
103
104 qround a, v0, v1, v2, v3, v4
105 qround a, v1, v2, v3, v4, v0
106 qround a, v2, v3, v4, v0, v1
107 qround a, v3, v4, v0, v1, v2
108
109 ext v11.16b, v14.16b, v14.16b, #4
110
111 qround b, v4, v0, v1, v2, v3
112 qround b, v0, v1, v2, v3, v4
113 qround b, v1, v2, v3, v4, v0
114 qround b, v2, v3, v4, v0, v1
115 qround b, v3, v4, v0, v1, v2
116 qround b, v4, v0, v1, v2, v3
117 qround b, v0, v1, v2, v3, v4
118 qround b, v1, v2, v3, v4, v0
119 qround b, v2, v3, v4, v0, v1
120 qround b, v3, v4
121 qround b, v4, v0
122 qround b, v0, v1
123
124 eor v8.16b, v8.16b, v15.16b
125 eor v9.16b, v9.16b, v16.16b
126
127 /* handled all input blocks? */
128 cbnz w2, 0b
129
130 /* save state */
131 rev64 v8.4s, v8.4s
132 rev64 v9.4s, v9.4s
133 ext v8.16b, v8.16b, v8.16b, #8
134 ext v9.16b, v9.16b, v9.16b, #8
135 st1 {v8.4s-v9.4s}, [x0]
136 ret
137ENDPROC(sm3_ce_transform)
138
139 .section ".rodata", "a"
140 .align 3
141.Lt: .word 0x79cc4519, 0x9d8a7a87
diff --git a/arch/arm64/crypto/sm3-ce-glue.c b/arch/arm64/crypto/sm3-ce-glue.c
new file mode 100644
index 000000000000..3b4948f7e26f
--- /dev/null
+++ b/arch/arm64/crypto/sm3-ce-glue.c
@@ -0,0 +1,92 @@
1/*
2 * sm3-ce-glue.c - SM3 secure hash using ARMv8.2 Crypto Extensions
3 *
4 * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <asm/neon.h>
12#include <asm/simd.h>
13#include <asm/unaligned.h>
14#include <crypto/internal/hash.h>
15#include <crypto/sm3.h>
16#include <crypto/sm3_base.h>
17#include <linux/cpufeature.h>
18#include <linux/crypto.h>
19#include <linux/module.h>
20
21MODULE_DESCRIPTION("SM3 secure hash using ARMv8 Crypto Extensions");
22MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
23MODULE_LICENSE("GPL v2");
24
25asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src,
26 int blocks);
27
28static int sm3_ce_update(struct shash_desc *desc, const u8 *data,
29 unsigned int len)
30{
31 if (!may_use_simd())
32 return crypto_sm3_update(desc, data, len);
33
34 kernel_neon_begin();
35 sm3_base_do_update(desc, data, len, sm3_ce_transform);
36 kernel_neon_end();
37
38 return 0;
39}
40
41static int sm3_ce_final(struct shash_desc *desc, u8 *out)
42{
43 if (!may_use_simd())
44 return crypto_sm3_finup(desc, NULL, 0, out);
45
46 kernel_neon_begin();
47 sm3_base_do_finalize(desc, sm3_ce_transform);
48 kernel_neon_end();
49
50 return sm3_base_finish(desc, out);
51}
52
53static int sm3_ce_finup(struct shash_desc *desc, const u8 *data,
54 unsigned int len, u8 *out)
55{
56 if (!may_use_simd())
57 return crypto_sm3_finup(desc, data, len, out);
58
59 kernel_neon_begin();
60 sm3_base_do_update(desc, data, len, sm3_ce_transform);
61 kernel_neon_end();
62
63 return sm3_ce_final(desc, out);
64}
65
66static struct shash_alg sm3_alg = {
67 .digestsize = SM3_DIGEST_SIZE,
68 .init = sm3_base_init,
69 .update = sm3_ce_update,
70 .final = sm3_ce_final,
71 .finup = sm3_ce_finup,
72 .descsize = sizeof(struct sm3_state),
73 .base.cra_name = "sm3",
74 .base.cra_driver_name = "sm3-ce",
75 .base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
76 .base.cra_blocksize = SM3_BLOCK_SIZE,
77 .base.cra_module = THIS_MODULE,
78 .base.cra_priority = 200,
79};
80
81static int __init sm3_ce_mod_init(void)
82{
83 return crypto_register_shash(&sm3_alg);
84}
85
86static void __exit sm3_ce_mod_fini(void)
87{
88 crypto_unregister_shash(&sm3_alg);
89}
90
91module_cpu_feature_match(SM3, sm3_ce_mod_init);
92module_exit(sm3_ce_mod_fini);
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-30 11:27:54 -0400