aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/arm/crypto/Kconfig10
-rw-r--r--arch/arm/crypto/Makefile2
-rw-r--r--arch/arm/crypto/sha1-ce-core.S134
-rw-r--r--arch/arm/crypto/sha1-ce-glue.c150
4 files changed, 296 insertions, 0 deletions
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index 66fe82857e99..d7bc10beb8ac 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -27,6 +27,16 @@ config CRYPTO_SHA1_ARM_NEON
27 using optimized ARM NEON assembly, when NEON instructions are 27 using optimized ARM NEON assembly, when NEON instructions are
28 available. 28 available.
29 29
30config CRYPTO_SHA1_ARM_CE
31 tristate "SHA1 digest algorithm (ARM v8 Crypto Extensions)"
32 depends on KERNEL_MODE_NEON
33 select CRYPTO_SHA1_ARM
34 select CRYPTO_SHA1
35 select CRYPTO_HASH
36 help
37 SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
38 using special ARMv8 Crypto Extensions.
39
30config CRYPTO_SHA512_ARM_NEON 40config CRYPTO_SHA512_ARM_NEON
31 tristate "SHA384 and SHA512 digest algorithm (ARM NEON)" 41 tristate "SHA384 and SHA512 digest algorithm (ARM NEON)"
32 depends on KERNEL_MODE_NEON 42 depends on KERNEL_MODE_NEON
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index b48fa341648d..d92d05ba646e 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -7,12 +7,14 @@ obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o
7obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o 7obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o
8obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o 8obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o
9obj-$(CONFIG_CRYPTO_SHA512_ARM_NEON) += sha512-arm-neon.o 9obj-$(CONFIG_CRYPTO_SHA512_ARM_NEON) += sha512-arm-neon.o
10obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
10 11
11aes-arm-y := aes-armv4.o aes_glue.o 12aes-arm-y := aes-armv4.o aes_glue.o
12aes-arm-bs-y := aesbs-core.o aesbs-glue.o 13aes-arm-bs-y := aesbs-core.o aesbs-glue.o
13sha1-arm-y := sha1-armv4-large.o sha1_glue.o 14sha1-arm-y := sha1-armv4-large.o sha1_glue.o
14sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o 15sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o
15sha512-arm-neon-y := sha512-armv7-neon.o sha512_neon_glue.o 16sha512-arm-neon-y := sha512-armv7-neon.o sha512_neon_glue.o
17sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o
16 18
17quiet_cmd_perl = PERL $@ 19quiet_cmd_perl = PERL $@
18 cmd_perl = $(PERL) $(<) > $(@) 20 cmd_perl = $(PERL) $(<) > $(@)
diff --git a/arch/arm/crypto/sha1-ce-core.S b/arch/arm/crypto/sha1-ce-core.S
new file mode 100644
index 000000000000..4aad520935d8
--- /dev/null
+++ b/arch/arm/crypto/sha1-ce-core.S
@@ -0,0 +1,134 @@
1/*
2 * sha1-ce-core.S - SHA-1 secure hash using ARMv8 Crypto Extensions
3 *
4 * Copyright (C) 2015 Linaro Ltd.
5 * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11
12#include <linux/linkage.h>
13#include <asm/assembler.h>
14
15 .text
16 .fpu crypto-neon-fp-armv8
17
18 k0 .req q0
19 k1 .req q1
20 k2 .req q2
21 k3 .req q3
22
23 ta0 .req q4
24 ta1 .req q5
25 tb0 .req q5
26 tb1 .req q4
27
28 dga .req q6
29 dgb .req q7
30 dgbs .req s28
31
32 dg0 .req q12
33 dg1a0 .req q13
34 dg1a1 .req q14
35 dg1b0 .req q14
36 dg1b1 .req q13
37
38 .macro add_only, op, ev, rc, s0, dg1
39 .ifnb \s0
40 vadd.u32 tb\ev, q\s0, \rc
41 .endif
42 sha1h.32 dg1b\ev, dg0
43 .ifb \dg1
44 sha1\op\().32 dg0, dg1a\ev, ta\ev
45 .else
46 sha1\op\().32 dg0, \dg1, ta\ev
47 .endif
48 .endm
49
50 .macro add_update, op, ev, rc, s0, s1, s2, s3, dg1
51 sha1su0.32 q\s0, q\s1, q\s2
52 add_only \op, \ev, \rc, \s1, \dg1
53 sha1su1.32 q\s0, q\s3
54 .endm
55
56 .align 6
57.Lsha1_rcon:
58 .word 0x5a827999, 0x5a827999, 0x5a827999, 0x5a827999
59 .word 0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1
60 .word 0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc
61 .word 0xca62c1d6, 0xca62c1d6, 0xca62c1d6, 0xca62c1d6
62
63 /*
64 * void sha1_ce_transform(int blocks, u8 const *src, u32 *state,
65 * u8 *head);
66 */
67ENTRY(sha1_ce_transform)
68 /* load round constants */
69 adr ip, .Lsha1_rcon
70 vld1.32 {k0-k1}, [ip, :128]!
71 vld1.32 {k2-k3}, [ip, :128]
72
73 /* load state */
74 vld1.32 {dga}, [r2]
75 vldr dgbs, [r2, #16]
76
77 /* load partial input (if supplied) */
78 teq r3, #0
79 beq 0f
80 vld1.32 {q8-q9}, [r3]!
81 vld1.32 {q10-q11}, [r3]
82 teq r0, #0
83 b 1f
84
85 /* load input */
860: vld1.32 {q8-q9}, [r1]!
87 vld1.32 {q10-q11}, [r1]!
88 subs r0, r0, #1
89
901:
91#ifndef CONFIG_CPU_BIG_ENDIAN
92 vrev32.8 q8, q8
93 vrev32.8 q9, q9
94 vrev32.8 q10, q10
95 vrev32.8 q11, q11
96#endif
97
98 vadd.u32 ta0, q8, k0
99 vmov dg0, dga
100
101 add_update c, 0, k0, 8, 9, 10, 11, dgb
102 add_update c, 1, k0, 9, 10, 11, 8
103 add_update c, 0, k0, 10, 11, 8, 9
104 add_update c, 1, k0, 11, 8, 9, 10
105 add_update c, 0, k1, 8, 9, 10, 11
106
107 add_update p, 1, k1, 9, 10, 11, 8
108 add_update p, 0, k1, 10, 11, 8, 9
109 add_update p, 1, k1, 11, 8, 9, 10
110 add_update p, 0, k1, 8, 9, 10, 11
111 add_update p, 1, k2, 9, 10, 11, 8
112
113 add_update m, 0, k2, 10, 11, 8, 9
114 add_update m, 1, k2, 11, 8, 9, 10
115 add_update m, 0, k2, 8, 9, 10, 11
116 add_update m, 1, k2, 9, 10, 11, 8
117 add_update m, 0, k3, 10, 11, 8, 9
118
119 add_update p, 1, k3, 11, 8, 9, 10
120 add_only p, 0, k3, 9
121 add_only p, 1, k3, 10
122 add_only p, 0, k3, 11
123 add_only p, 1
124
125 /* update state */
126 vadd.u32 dga, dga, dg0
127 vadd.u32 dgb, dgb, dg1a0
128 bne 0b
129
130 /* store new state */
131 vst1.32 {dga}, [r2]
132 vstr dgbs, [r2, #16]
133 bx lr
134ENDPROC(sha1_ce_transform)
diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c
new file mode 100644
index 000000000000..a9dd90df9fd7
--- /dev/null
+++ b/arch/arm/crypto/sha1-ce-glue.c
@@ -0,0 +1,150 @@
1/*
2 * sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions
3 *
4 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <crypto/internal/hash.h>
12#include <crypto/sha.h>
13#include <linux/crypto.h>
14#include <linux/module.h>
15
16#include <asm/crypto/sha1.h>
17#include <asm/hwcap.h>
18#include <asm/neon.h>
19#include <asm/simd.h>
20#include <asm/unaligned.h>
21
22MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions");
23MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
24MODULE_LICENSE("GPL v2");
25
26asmlinkage void sha1_ce_transform(int blocks, u8 const *src, u32 *state,
27 u8 *head);
28
29static int sha1_init(struct shash_desc *desc)
30{
31 struct sha1_state *sctx = shash_desc_ctx(desc);
32
33 *sctx = (struct sha1_state){
34 .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
35 };
36 return 0;
37}
38
39static int sha1_update(struct shash_desc *desc, const u8 *data,
40 unsigned int len)
41{
42 struct sha1_state *sctx = shash_desc_ctx(desc);
43 unsigned int partial;
44
45 if (!may_use_simd())
46 return sha1_update_arm(desc, data, len);
47
48 partial = sctx->count % SHA1_BLOCK_SIZE;
49 sctx->count += len;
50
51 if ((partial + len) >= SHA1_BLOCK_SIZE) {
52 int blocks;
53
54 if (partial) {
55 int p = SHA1_BLOCK_SIZE - partial;
56
57 memcpy(sctx->buffer + partial, data, p);
58 data += p;
59 len -= p;
60 }
61
62 blocks = len / SHA1_BLOCK_SIZE;
63 len %= SHA1_BLOCK_SIZE;
64
65 kernel_neon_begin();
66 sha1_ce_transform(blocks, data, sctx->state,
67 partial ? sctx->buffer : NULL);
68 kernel_neon_end();
69
70 data += blocks * SHA1_BLOCK_SIZE;
71 partial = 0;
72 }
73 if (len)
74 memcpy(sctx->buffer + partial, data, len);
75 return 0;
76}
77
78static int sha1_final(struct shash_desc *desc, u8 *out)
79{
80 static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, };
81
82 struct sha1_state *sctx = shash_desc_ctx(desc);
83 __be64 bits = cpu_to_be64(sctx->count << 3);
84 __be32 *dst = (__be32 *)out;
85 int i;
86
87 u32 padlen = SHA1_BLOCK_SIZE
88 - ((sctx->count + sizeof(bits)) % SHA1_BLOCK_SIZE);
89
90 sha1_update(desc, padding, padlen);
91 sha1_update(desc, (const u8 *)&bits, sizeof(bits));
92
93 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++)
94 put_unaligned_be32(sctx->state[i], dst++);
95
96 *sctx = (struct sha1_state){};
97 return 0;
98}
99
100static int sha1_export(struct shash_desc *desc, void *out)
101{
102 struct sha1_state *sctx = shash_desc_ctx(desc);
103 struct sha1_state *dst = out;
104
105 *dst = *sctx;
106 return 0;
107}
108
109static int sha1_import(struct shash_desc *desc, const void *in)
110{
111 struct sha1_state *sctx = shash_desc_ctx(desc);
112 struct sha1_state const *src = in;
113
114 *sctx = *src;
115 return 0;
116}
117
118static struct shash_alg alg = {
119 .init = sha1_init,
120 .update = sha1_update,
121 .final = sha1_final,
122 .export = sha1_export,
123 .import = sha1_import,
124 .descsize = sizeof(struct sha1_state),
125 .digestsize = SHA1_DIGEST_SIZE,
126 .statesize = sizeof(struct sha1_state),
127 .base = {
128 .cra_name = "sha1",
129 .cra_driver_name = "sha1-ce",
130 .cra_priority = 200,
131 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
132 .cra_blocksize = SHA1_BLOCK_SIZE,
133 .cra_module = THIS_MODULE,
134 }
135};
136
137static int __init sha1_ce_mod_init(void)
138{
139 if (!(elf_hwcap2 & HWCAP2_SHA1))
140 return -ENODEV;
141 return crypto_register_shash(&alg);
142}
143
144static void __exit sha1_ce_mod_fini(void)
145{
146 crypto_unregister_shash(&alg);
147}
148
149module_init(sha1_ce_mod_init);
150module_exit(sha1_ce_mod_fini);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-29 09:20:20 -0500