diff options
Diffstat (limited to 'arch/arm')
-rw-r--r-- | arch/arm/crypto/sha1_neon_glue.c | 135 |
1 files changed, 24 insertions, 111 deletions
diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c index 5d9a1b4aac73..4e22f122f966 100644 --- a/arch/arm/crypto/sha1_neon_glue.c +++ b/arch/arm/crypto/sha1_neon_glue.c | |||
@@ -25,7 +25,7 @@ | |||
25 | #include <linux/cryptohash.h> | 25 | #include <linux/cryptohash.h> |
26 | #include <linux/types.h> | 26 | #include <linux/types.h> |
27 | #include <crypto/sha.h> | 27 | #include <crypto/sha.h> |
28 | #include <asm/byteorder.h> | 28 | #include <crypto/sha1_base.h> |
29 | #include <asm/neon.h> | 29 | #include <asm/neon.h> |
30 | #include <asm/simd.h> | 30 | #include <asm/simd.h> |
31 | 31 | ||
@@ -34,138 +34,51 @@ | |||
34 | asmlinkage void sha1_transform_neon(void *state_h, const char *data, | 34 | asmlinkage void sha1_transform_neon(void *state_h, const char *data, |
35 | unsigned int rounds); | 35 | unsigned int rounds); |
36 | 36 | ||
37 | |||
38 | static int sha1_neon_init(struct shash_desc *desc) | ||
39 | { | ||
40 | struct sha1_state *sctx = shash_desc_ctx(desc); | ||
41 | |||
42 | *sctx = (struct sha1_state){ | ||
43 | .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, | ||
44 | }; | ||
45 | |||
46 | return 0; | ||
47 | } | ||
48 | |||
49 | static int __sha1_neon_update(struct shash_desc *desc, const u8 *data, | ||
50 | unsigned int len, unsigned int partial) | ||
51 | { | ||
52 | struct sha1_state *sctx = shash_desc_ctx(desc); | ||
53 | unsigned int done = 0; | ||
54 | |||
55 | sctx->count += len; | ||
56 | |||
57 | if (partial) { | ||
58 | done = SHA1_BLOCK_SIZE - partial; | ||
59 | memcpy(sctx->buffer + partial, data, done); | ||
60 | sha1_transform_neon(sctx->state, sctx->buffer, 1); | ||
61 | } | ||
62 | |||
63 | if (len - done >= SHA1_BLOCK_SIZE) { | ||
64 | const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; | ||
65 | |||
66 | sha1_transform_neon(sctx->state, data + done, rounds); | ||
67 | done += rounds * SHA1_BLOCK_SIZE; | ||
68 | } | ||
69 | |||
70 | memcpy(sctx->buffer, data + done, len - done); | ||
71 | |||
72 | return 0; | ||
73 | } | ||
74 | |||
75 | static int sha1_neon_update(struct shash_desc *desc, const u8 *data, | 37 | static int sha1_neon_update(struct shash_desc *desc, const u8 *data, |
76 | unsigned int len) | 38 | unsigned int len) |
77 | { | 39 | { |
78 | struct sha1_state *sctx = shash_desc_ctx(desc); | 40 | struct sha1_state *sctx = shash_desc_ctx(desc); |
79 | unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; | ||
80 | int res; | ||
81 | 41 | ||
82 | /* Handle the fast case right here */ | 42 | if (!may_use_simd() || |
83 | if (partial + len < SHA1_BLOCK_SIZE) { | 43 | (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) |
84 | sctx->count += len; | 44 | return sha1_update_arm(desc, data, len); |
85 | memcpy(sctx->buffer + partial, data, len); | ||
86 | 45 | ||
87 | return 0; | 46 | kernel_neon_begin(); |
88 | } | 47 | sha1_base_do_update(desc, data, len, |
89 | 48 | (sha1_block_fn *)sha1_transform_neon); | |
90 | if (!may_use_simd()) { | 49 | kernel_neon_end(); |
91 | res = sha1_update_arm(desc, data, len); | ||
92 | } else { | ||
93 | kernel_neon_begin(); | ||
94 | res = __sha1_neon_update(desc, data, len, partial); | ||
95 | kernel_neon_end(); | ||
96 | } | ||
97 | |||
98 | return res; | ||
99 | } | ||
100 | |||
101 | |||
102 | /* Add padding and return the message digest. */ | ||
103 | static int sha1_neon_final(struct shash_desc *desc, u8 *out) | ||
104 | { | ||
105 | struct sha1_state *sctx = shash_desc_ctx(desc); | ||
106 | unsigned int i, index, padlen; | ||
107 | __be32 *dst = (__be32 *)out; | ||
108 | __be64 bits; | ||
109 | static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; | ||
110 | |||
111 | bits = cpu_to_be64(sctx->count << 3); | ||
112 | |||
113 | /* Pad out to 56 mod 64 and append length */ | ||
114 | index = sctx->count % SHA1_BLOCK_SIZE; | ||
115 | padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); | ||
116 | if (!may_use_simd()) { | ||
117 | sha1_update_arm(desc, padding, padlen); | ||
118 | sha1_update_arm(desc, (const u8 *)&bits, sizeof(bits)); | ||
119 | } else { | ||
120 | kernel_neon_begin(); | ||
121 | /* We need to fill a whole block for __sha1_neon_update() */ | ||
122 | if (padlen <= 56) { | ||
123 | sctx->count += padlen; | ||
124 | memcpy(sctx->buffer + index, padding, padlen); | ||
125 | } else { | ||
126 | __sha1_neon_update(desc, padding, padlen, index); | ||
127 | } | ||
128 | __sha1_neon_update(desc, (const u8 *)&bits, sizeof(bits), 56); | ||
129 | kernel_neon_end(); | ||
130 | } | ||
131 | |||
132 | /* Store state in digest */ | ||
133 | for (i = 0; i < 5; i++) | ||
134 | dst[i] = cpu_to_be32(sctx->state[i]); | ||
135 | |||
136 | /* Wipe context */ | ||
137 | memset(sctx, 0, sizeof(*sctx)); | ||
138 | 50 | ||
139 | return 0; | 51 | return 0; |
140 | } | 52 | } |
141 | 53 | ||
142 | static int sha1_neon_export(struct shash_desc *desc, void *out) | 54 | static int sha1_neon_finup(struct shash_desc *desc, const u8 *data, |
55 | unsigned int len, u8 *out) | ||
143 | { | 56 | { |
144 | struct sha1_state *sctx = shash_desc_ctx(desc); | 57 | if (!may_use_simd()) |
58 | return sha1_finup_arm(desc, data, len, out); | ||
145 | 59 | ||
146 | memcpy(out, sctx, sizeof(*sctx)); | 60 | kernel_neon_begin(); |
61 | if (len) | ||
62 | sha1_base_do_update(desc, data, len, | ||
63 | (sha1_block_fn *)sha1_transform_neon); | ||
64 | sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_transform_neon); | ||
65 | kernel_neon_end(); | ||
147 | 66 | ||
148 | return 0; | 67 | return sha1_base_finish(desc, out); |
149 | } | 68 | } |
150 | 69 | ||
151 | static int sha1_neon_import(struct shash_desc *desc, const void *in) | 70 | static int sha1_neon_final(struct shash_desc *desc, u8 *out) |
152 | { | 71 | { |
153 | struct sha1_state *sctx = shash_desc_ctx(desc); | 72 | return sha1_neon_finup(desc, NULL, 0, out); |
154 | |||
155 | memcpy(sctx, in, sizeof(*sctx)); | ||
156 | |||
157 | return 0; | ||
158 | } | 73 | } |
159 | 74 | ||
160 | static struct shash_alg alg = { | 75 | static struct shash_alg alg = { |
161 | .digestsize = SHA1_DIGEST_SIZE, | 76 | .digestsize = SHA1_DIGEST_SIZE, |
162 | .init = sha1_neon_init, | 77 | .init = sha1_base_init, |
163 | .update = sha1_neon_update, | 78 | .update = sha1_neon_update, |
164 | .final = sha1_neon_final, | 79 | .final = sha1_neon_final, |
165 | .export = sha1_neon_export, | 80 | .finup = sha1_neon_finup, |
166 | .import = sha1_neon_import, | ||
167 | .descsize = sizeof(struct sha1_state), | 81 | .descsize = sizeof(struct sha1_state), |
168 | .statesize = sizeof(struct sha1_state), | ||
169 | .base = { | 82 | .base = { |
170 | .cra_name = "sha1", | 83 | .cra_name = "sha1", |
171 | .cra_driver_name = "sha1-neon", | 84 | .cra_driver_name = "sha1-neon", |