diff options
-rw-r--r-- | crypto/Kconfig | 2 | ||||
-rw-r--r-- | crypto/gcm.c | 580 |
2 files changed, 408 insertions, 174 deletions
diff --git a/crypto/Kconfig b/crypto/Kconfig index 0ae170e01051..5105cf15834f 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig | |||
@@ -157,7 +157,7 @@ config CRYPTO_GCM | |||
157 | tristate "GCM/GMAC support" | 157 | tristate "GCM/GMAC support" |
158 | select CRYPTO_CTR | 158 | select CRYPTO_CTR |
159 | select CRYPTO_AEAD | 159 | select CRYPTO_AEAD |
160 | select CRYPTO_GF128MUL | 160 | select CRYPTO_GHASH |
161 | help | 161 | help |
162 | Support for Galois/Counter Mode (GCM) and Galois Message | 162 | Support for Galois/Counter Mode (GCM) and Galois Message |
163 | Authentication Code (GMAC). Required for IPSec. | 163 | Authentication Code (GMAC). Required for IPSec. |
diff --git a/crypto/gcm.c b/crypto/gcm.c index e70afd0c73dd..5fc3292483ef 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c | |||
@@ -11,7 +11,10 @@ | |||
11 | #include <crypto/gf128mul.h> | 11 | #include <crypto/gf128mul.h> |
12 | #include <crypto/internal/aead.h> | 12 | #include <crypto/internal/aead.h> |
13 | #include <crypto/internal/skcipher.h> | 13 | #include <crypto/internal/skcipher.h> |
14 | #include <crypto/internal/hash.h> | ||
14 | #include <crypto/scatterwalk.h> | 15 | #include <crypto/scatterwalk.h> |
16 | #include <crypto/hash.h> | ||
17 | #include "internal.h" | ||
15 | #include <linux/completion.h> | 18 | #include <linux/completion.h> |
16 | #include <linux/err.h> | 19 | #include <linux/err.h> |
17 | #include <linux/init.h> | 20 | #include <linux/init.h> |
@@ -21,11 +24,12 @@ | |||
21 | 24 | ||
22 | struct gcm_instance_ctx { | 25 | struct gcm_instance_ctx { |
23 | struct crypto_skcipher_spawn ctr; | 26 | struct crypto_skcipher_spawn ctr; |
27 | struct crypto_ahash_spawn ghash; | ||
24 | }; | 28 | }; |
25 | 29 | ||
26 | struct crypto_gcm_ctx { | 30 | struct crypto_gcm_ctx { |
27 | struct crypto_ablkcipher *ctr; | 31 | struct crypto_ablkcipher *ctr; |
28 | struct gf128mul_4k *gf128; | 32 | struct crypto_ahash *ghash; |
29 | }; | 33 | }; |
30 | 34 | ||
31 | struct crypto_rfc4106_ctx { | 35 | struct crypto_rfc4106_ctx { |
@@ -34,10 +38,9 @@ struct crypto_rfc4106_ctx { | |||
34 | }; | 38 | }; |
35 | 39 | ||
36 | struct crypto_gcm_ghash_ctx { | 40 | struct crypto_gcm_ghash_ctx { |
37 | u32 bytes; | 41 | unsigned int cryptlen; |
38 | u32 flags; | 42 | struct scatterlist *src; |
39 | struct gf128mul_4k *gf128; | 43 | crypto_completion_t complete; |
40 | u8 buffer[16]; | ||
41 | }; | 44 | }; |
42 | 45 | ||
43 | struct crypto_gcm_req_priv_ctx { | 46 | struct crypto_gcm_req_priv_ctx { |
@@ -45,8 +48,11 @@ struct crypto_gcm_req_priv_ctx { | |||
45 | u8 iauth_tag[16]; | 48 | u8 iauth_tag[16]; |
46 | struct scatterlist src[2]; | 49 | struct scatterlist src[2]; |
47 | struct scatterlist dst[2]; | 50 | struct scatterlist dst[2]; |
48 | struct crypto_gcm_ghash_ctx ghash; | 51 | struct crypto_gcm_ghash_ctx ghash_ctx; |
49 | struct ablkcipher_request abreq; | 52 | union { |
53 | struct ahash_request ahreq; | ||
54 | struct ablkcipher_request abreq; | ||
55 | } u; | ||
50 | }; | 56 | }; |
51 | 57 | ||
52 | struct crypto_gcm_setkey_result { | 58 | struct crypto_gcm_setkey_result { |
@@ -54,6 +60,8 @@ struct crypto_gcm_setkey_result { | |||
54 | struct completion completion; | 60 | struct completion completion; |
55 | }; | 61 | }; |
56 | 62 | ||
63 | static void *gcm_zeroes; | ||
64 | |||
57 | static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( | 65 | static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( |
58 | struct aead_request *req) | 66 | struct aead_request *req) |
59 | { | 67 | { |
@@ -62,113 +70,6 @@ static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( | |||
62 | return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); | 70 | return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); |
63 | } | 71 | } |
64 | 72 | ||
65 | static void crypto_gcm_ghash_init(struct crypto_gcm_ghash_ctx *ctx, u32 flags, | ||
66 | struct gf128mul_4k *gf128) | ||
67 | { | ||
68 | ctx->bytes = 0; | ||
69 | ctx->flags = flags; | ||
70 | ctx->gf128 = gf128; | ||
71 | memset(ctx->buffer, 0, 16); | ||
72 | } | ||
73 | |||
74 | static void crypto_gcm_ghash_update(struct crypto_gcm_ghash_ctx *ctx, | ||
75 | const u8 *src, unsigned int srclen) | ||
76 | { | ||
77 | u8 *dst = ctx->buffer; | ||
78 | |||
79 | if (ctx->bytes) { | ||
80 | int n = min(srclen, ctx->bytes); | ||
81 | u8 *pos = dst + (16 - ctx->bytes); | ||
82 | |||
83 | ctx->bytes -= n; | ||
84 | srclen -= n; | ||
85 | |||
86 | while (n--) | ||
87 | *pos++ ^= *src++; | ||
88 | |||
89 | if (!ctx->bytes) | ||
90 | gf128mul_4k_lle((be128 *)dst, ctx->gf128); | ||
91 | } | ||
92 | |||
93 | while (srclen >= 16) { | ||
94 | crypto_xor(dst, src, 16); | ||
95 | gf128mul_4k_lle((be128 *)dst, ctx->gf128); | ||
96 | src += 16; | ||
97 | srclen -= 16; | ||
98 | } | ||
99 | |||
100 | if (srclen) { | ||
101 | ctx->bytes = 16 - srclen; | ||
102 | while (srclen--) | ||
103 | *dst++ ^= *src++; | ||
104 | } | ||
105 | } | ||
106 | |||
107 | static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx *ctx, | ||
108 | struct scatterlist *sg, int len) | ||
109 | { | ||
110 | struct scatter_walk walk; | ||
111 | u8 *src; | ||
112 | int n; | ||
113 | |||
114 | if (!len) | ||
115 | return; | ||
116 | |||
117 | scatterwalk_start(&walk, sg); | ||
118 | |||
119 | while (len) { | ||
120 | n = scatterwalk_clamp(&walk, len); | ||
121 | |||
122 | if (!n) { | ||
123 | scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg)); | ||
124 | n = scatterwalk_clamp(&walk, len); | ||
125 | } | ||
126 | |||
127 | src = scatterwalk_map(&walk, 0); | ||
128 | |||
129 | crypto_gcm_ghash_update(ctx, src, n); | ||
130 | len -= n; | ||
131 | |||
132 | scatterwalk_unmap(src, 0); | ||
133 | scatterwalk_advance(&walk, n); | ||
134 | scatterwalk_done(&walk, 0, len); | ||
135 | if (len) | ||
136 | crypto_yield(ctx->flags); | ||
137 | } | ||
138 | } | ||
139 | |||
140 | static void crypto_gcm_ghash_flush(struct crypto_gcm_ghash_ctx *ctx) | ||
141 | { | ||
142 | u8 *dst = ctx->buffer; | ||
143 | |||
144 | if (ctx->bytes) { | ||
145 | u8 *tmp = dst + (16 - ctx->bytes); | ||
146 | |||
147 | while (ctx->bytes--) | ||
148 | *tmp++ ^= 0; | ||
149 | |||
150 | gf128mul_4k_lle((be128 *)dst, ctx->gf128); | ||
151 | } | ||
152 | |||
153 | ctx->bytes = 0; | ||
154 | } | ||
155 | |||
156 | static void crypto_gcm_ghash_final_xor(struct crypto_gcm_ghash_ctx *ctx, | ||
157 | unsigned int authlen, | ||
158 | unsigned int cryptlen, u8 *dst) | ||
159 | { | ||
160 | u8 *buf = ctx->buffer; | ||
161 | u128 lengths; | ||
162 | |||
163 | lengths.a = cpu_to_be64(authlen * 8); | ||
164 | lengths.b = cpu_to_be64(cryptlen * 8); | ||
165 | |||
166 | crypto_gcm_ghash_flush(ctx); | ||
167 | crypto_xor(buf, (u8 *)&lengths, 16); | ||
168 | gf128mul_4k_lle((be128 *)buf, ctx->gf128); | ||
169 | crypto_xor(dst, buf, 16); | ||
170 | } | ||
171 | |||
172 | static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err) | 73 | static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err) |
173 | { | 74 | { |
174 | struct crypto_gcm_setkey_result *result = req->data; | 75 | struct crypto_gcm_setkey_result *result = req->data; |
@@ -184,6 +85,7 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, | |||
184 | unsigned int keylen) | 85 | unsigned int keylen) |
185 | { | 86 | { |
186 | struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); | 87 | struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); |
88 | struct crypto_ahash *ghash = ctx->ghash; | ||
187 | struct crypto_ablkcipher *ctr = ctx->ctr; | 89 | struct crypto_ablkcipher *ctr = ctx->ctr; |
188 | struct { | 90 | struct { |
189 | be128 hash; | 91 | be128 hash; |
@@ -233,13 +135,12 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, | |||
233 | if (err) | 135 | if (err) |
234 | goto out; | 136 | goto out; |
235 | 137 | ||
236 | if (ctx->gf128 != NULL) | 138 | crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); |
237 | gf128mul_free_4k(ctx->gf128); | 139 | crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & |
238 | 140 | CRYPTO_TFM_REQ_MASK); | |
239 | ctx->gf128 = gf128mul_init_4k_lle(&data->hash); | 141 | err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); |
240 | 142 | crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) & | |
241 | if (ctx->gf128 == NULL) | 143 | CRYPTO_TFM_RES_MASK); |
242 | err = -ENOMEM; | ||
243 | 144 | ||
244 | out: | 145 | out: |
245 | kfree(data); | 146 | kfree(data); |
@@ -272,8 +173,6 @@ static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req, | |||
272 | struct crypto_aead *aead = crypto_aead_reqtfm(req); | 173 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
273 | struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); | 174 | struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); |
274 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | 175 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
275 | u32 flags = req->base.tfm->crt_flags; | ||
276 | struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; | ||
277 | struct scatterlist *dst; | 176 | struct scatterlist *dst; |
278 | __be32 counter = cpu_to_be32(1); | 177 | __be32 counter = cpu_to_be32(1); |
279 | 178 | ||
@@ -296,108 +195,398 @@ static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req, | |||
296 | ablkcipher_request_set_crypt(ablk_req, pctx->src, dst, | 195 | ablkcipher_request_set_crypt(ablk_req, pctx->src, dst, |
297 | cryptlen + sizeof(pctx->auth_tag), | 196 | cryptlen + sizeof(pctx->auth_tag), |
298 | req->iv); | 197 | req->iv); |
198 | } | ||
199 | |||
200 | static inline unsigned int gcm_remain(unsigned int len) | ||
201 | { | ||
202 | len &= 0xfU; | ||
203 | return len ? 16 - len : 0; | ||
204 | } | ||
205 | |||
206 | static void gcm_hash_len_done(struct crypto_async_request *areq, int err); | ||
207 | static void gcm_hash_final_done(struct crypto_async_request *areq, int err); | ||
299 | 208 | ||
300 | crypto_gcm_ghash_init(ghash, flags, ctx->gf128); | 209 | static int gcm_hash_update(struct aead_request *req, |
210 | struct crypto_gcm_req_priv_ctx *pctx, | ||
211 | crypto_completion_t complete, | ||
212 | struct scatterlist *src, | ||
213 | unsigned int len) | ||
214 | { | ||
215 | struct ahash_request *ahreq = &pctx->u.ahreq; | ||
301 | 216 | ||
302 | crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen); | 217 | ahash_request_set_callback(ahreq, aead_request_flags(req), |
303 | crypto_gcm_ghash_flush(ghash); | 218 | complete, req); |
219 | ahash_request_set_crypt(ahreq, src, NULL, len); | ||
220 | |||
221 | return crypto_ahash_update(ahreq); | ||
304 | } | 222 | } |
305 | 223 | ||
306 | static int crypto_gcm_hash(struct aead_request *req) | 224 | static int gcm_hash_remain(struct aead_request *req, |
225 | struct crypto_gcm_req_priv_ctx *pctx, | ||
226 | unsigned int remain, | ||
227 | crypto_completion_t complete) | ||
307 | { | 228 | { |
308 | struct crypto_aead *aead = crypto_aead_reqtfm(req); | 229 | struct ahash_request *ahreq = &pctx->u.ahreq; |
230 | |||
231 | ahash_request_set_callback(ahreq, aead_request_flags(req), | ||
232 | complete, req); | ||
233 | sg_init_one(pctx->src, gcm_zeroes, remain); | ||
234 | ahash_request_set_crypt(ahreq, pctx->src, NULL, remain); | ||
235 | |||
236 | return crypto_ahash_update(ahreq); | ||
237 | } | ||
238 | |||
239 | static int gcm_hash_len(struct aead_request *req, | ||
240 | struct crypto_gcm_req_priv_ctx *pctx) | ||
241 | { | ||
242 | struct ahash_request *ahreq = &pctx->u.ahreq; | ||
243 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; | ||
244 | u128 lengths; | ||
245 | |||
246 | lengths.a = cpu_to_be64(req->assoclen * 8); | ||
247 | lengths.b = cpu_to_be64(gctx->cryptlen * 8); | ||
248 | memcpy(pctx->iauth_tag, &lengths, 16); | ||
249 | sg_init_one(pctx->src, pctx->iauth_tag, 16); | ||
250 | ahash_request_set_callback(ahreq, aead_request_flags(req), | ||
251 | gcm_hash_len_done, req); | ||
252 | ahash_request_set_crypt(ahreq, pctx->src, | ||
253 | NULL, sizeof(lengths)); | ||
254 | |||
255 | return crypto_ahash_update(ahreq); | ||
256 | } | ||
257 | |||
258 | static int gcm_hash_final(struct aead_request *req, | ||
259 | struct crypto_gcm_req_priv_ctx *pctx) | ||
260 | { | ||
261 | struct ahash_request *ahreq = &pctx->u.ahreq; | ||
262 | |||
263 | ahash_request_set_callback(ahreq, aead_request_flags(req), | ||
264 | gcm_hash_final_done, req); | ||
265 | ahash_request_set_crypt(ahreq, NULL, pctx->iauth_tag, 0); | ||
266 | |||
267 | return crypto_ahash_final(ahreq); | ||
268 | } | ||
269 | |||
270 | static void gcm_hash_final_done(struct crypto_async_request *areq, | ||
271 | int err) | ||
272 | { | ||
273 | struct aead_request *req = areq->data; | ||
309 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | 274 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
310 | u8 *auth_tag = pctx->auth_tag; | 275 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; |
311 | struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; | 276 | |
277 | if (!err) | ||
278 | crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); | ||
312 | 279 | ||
313 | crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen); | 280 | gctx->complete(areq, err); |
314 | crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen, | 281 | } |
315 | auth_tag); | 282 | |
283 | static void gcm_hash_len_done(struct crypto_async_request *areq, | ||
284 | int err) | ||
285 | { | ||
286 | struct aead_request *req = areq->data; | ||
287 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | ||
288 | |||
289 | if (!err) { | ||
290 | err = gcm_hash_final(req, pctx); | ||
291 | if (err == -EINPROGRESS || err == -EBUSY) | ||
292 | return; | ||
293 | } | ||
294 | |||
295 | gcm_hash_final_done(areq, err); | ||
296 | } | ||
297 | |||
298 | static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq, | ||
299 | int err) | ||
300 | { | ||
301 | struct aead_request *req = areq->data; | ||
302 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | ||
303 | |||
304 | if (!err) { | ||
305 | err = gcm_hash_len(req, pctx); | ||
306 | if (err == -EINPROGRESS || err == -EBUSY) | ||
307 | return; | ||
308 | } | ||
309 | |||
310 | gcm_hash_len_done(areq, err); | ||
311 | } | ||
312 | |||
313 | static void gcm_hash_crypt_done(struct crypto_async_request *areq, | ||
314 | int err) | ||
315 | { | ||
316 | struct aead_request *req = areq->data; | ||
317 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | ||
318 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; | ||
319 | unsigned int remain; | ||
320 | |||
321 | if (!err) { | ||
322 | remain = gcm_remain(gctx->cryptlen); | ||
323 | BUG_ON(!remain); | ||
324 | err = gcm_hash_remain(req, pctx, remain, | ||
325 | gcm_hash_crypt_remain_done); | ||
326 | if (err == -EINPROGRESS || err == -EBUSY) | ||
327 | return; | ||
328 | } | ||
329 | |||
330 | gcm_hash_crypt_remain_done(areq, err); | ||
331 | } | ||
332 | |||
333 | static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq, | ||
334 | int err) | ||
335 | { | ||
336 | struct aead_request *req = areq->data; | ||
337 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | ||
338 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; | ||
339 | crypto_completion_t complete; | ||
340 | unsigned int remain = 0; | ||
341 | |||
342 | if (!err && gctx->cryptlen) { | ||
343 | remain = gcm_remain(gctx->cryptlen); | ||
344 | complete = remain ? gcm_hash_crypt_done : | ||
345 | gcm_hash_crypt_remain_done; | ||
346 | err = gcm_hash_update(req, pctx, complete, | ||
347 | gctx->src, gctx->cryptlen); | ||
348 | if (err == -EINPROGRESS || err == -EBUSY) | ||
349 | return; | ||
350 | } | ||
351 | |||
352 | if (remain) | ||
353 | gcm_hash_crypt_done(areq, err); | ||
354 | else | ||
355 | gcm_hash_crypt_remain_done(areq, err); | ||
356 | } | ||
357 | |||
358 | static void gcm_hash_assoc_done(struct crypto_async_request *areq, | ||
359 | int err) | ||
360 | { | ||
361 | struct aead_request *req = areq->data; | ||
362 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | ||
363 | unsigned int remain; | ||
364 | |||
365 | if (!err) { | ||
366 | remain = gcm_remain(req->assoclen); | ||
367 | BUG_ON(!remain); | ||
368 | err = gcm_hash_remain(req, pctx, remain, | ||
369 | gcm_hash_assoc_remain_done); | ||
370 | if (err == -EINPROGRESS || err == -EBUSY) | ||
371 | return; | ||
372 | } | ||
373 | |||
374 | gcm_hash_assoc_remain_done(areq, err); | ||
375 | } | ||
376 | |||
377 | static void gcm_hash_init_done(struct crypto_async_request *areq, | ||
378 | int err) | ||
379 | { | ||
380 | struct aead_request *req = areq->data; | ||
381 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | ||
382 | crypto_completion_t complete; | ||
383 | unsigned int remain = 0; | ||
384 | |||
385 | if (!err && req->assoclen) { | ||
386 | remain = gcm_remain(req->assoclen); | ||
387 | complete = remain ? gcm_hash_assoc_done : | ||
388 | gcm_hash_assoc_remain_done; | ||
389 | err = gcm_hash_update(req, pctx, complete, | ||
390 | req->assoc, req->assoclen); | ||
391 | if (err == -EINPROGRESS || err == -EBUSY) | ||
392 | return; | ||
393 | } | ||
394 | |||
395 | if (remain) | ||
396 | gcm_hash_assoc_done(areq, err); | ||
397 | else | ||
398 | gcm_hash_assoc_remain_done(areq, err); | ||
399 | } | ||
400 | |||
401 | static int gcm_hash(struct aead_request *req, | ||
402 | struct crypto_gcm_req_priv_ctx *pctx) | ||
403 | { | ||
404 | struct ahash_request *ahreq = &pctx->u.ahreq; | ||
405 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; | ||
406 | struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(req->base.tfm); | ||
407 | unsigned int remain; | ||
408 | crypto_completion_t complete; | ||
409 | int err; | ||
410 | |||
411 | ahash_request_set_tfm(ahreq, ctx->ghash); | ||
412 | |||
413 | ahash_request_set_callback(ahreq, aead_request_flags(req), | ||
414 | gcm_hash_init_done, req); | ||
415 | err = crypto_ahash_init(ahreq); | ||
416 | if (err) | ||
417 | return err; | ||
418 | remain = gcm_remain(req->assoclen); | ||
419 | complete = remain ? gcm_hash_assoc_done : gcm_hash_assoc_remain_done; | ||
420 | err = gcm_hash_update(req, pctx, complete, req->assoc, req->assoclen); | ||
421 | if (err) | ||
422 | return err; | ||
423 | if (remain) { | ||
424 | err = gcm_hash_remain(req, pctx, remain, | ||
425 | gcm_hash_assoc_remain_done); | ||
426 | if (err) | ||
427 | return err; | ||
428 | } | ||
429 | remain = gcm_remain(gctx->cryptlen); | ||
430 | complete = remain ? gcm_hash_crypt_done : gcm_hash_crypt_remain_done; | ||
431 | err = gcm_hash_update(req, pctx, complete, gctx->src, gctx->cryptlen); | ||
432 | if (err) | ||
433 | return err; | ||
434 | if (remain) { | ||
435 | err = gcm_hash_remain(req, pctx, remain, | ||
436 | gcm_hash_crypt_remain_done); | ||
437 | if (err) | ||
438 | return err; | ||
439 | } | ||
440 | err = gcm_hash_len(req, pctx); | ||
441 | if (err) | ||
442 | return err; | ||
443 | err = gcm_hash_final(req, pctx); | ||
444 | if (err) | ||
445 | return err; | ||
446 | |||
447 | return 0; | ||
448 | } | ||
449 | |||
450 | static void gcm_enc_copy_hash(struct aead_request *req, | ||
451 | struct crypto_gcm_req_priv_ctx *pctx) | ||
452 | { | ||
453 | struct crypto_aead *aead = crypto_aead_reqtfm(req); | ||
454 | u8 *auth_tag = pctx->auth_tag; | ||
316 | 455 | ||
317 | scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen, | 456 | scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen, |
318 | crypto_aead_authsize(aead), 1); | 457 | crypto_aead_authsize(aead), 1); |
319 | return 0; | ||
320 | } | 458 | } |
321 | 459 | ||
322 | static void crypto_gcm_encrypt_done(struct crypto_async_request *areq, int err) | 460 | static void gcm_enc_hash_done(struct crypto_async_request *areq, |
461 | int err) | ||
323 | { | 462 | { |
324 | struct aead_request *req = areq->data; | 463 | struct aead_request *req = areq->data; |
464 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | ||
325 | 465 | ||
326 | if (!err) | 466 | if (!err) |
327 | err = crypto_gcm_hash(req); | 467 | gcm_enc_copy_hash(req, pctx); |
328 | 468 | ||
329 | aead_request_complete(req, err); | 469 | aead_request_complete(req, err); |
330 | } | 470 | } |
331 | 471 | ||
472 | static void gcm_encrypt_done(struct crypto_async_request *areq, | ||
473 | int err) | ||
474 | { | ||
475 | struct aead_request *req = areq->data; | ||
476 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | ||
477 | |||
478 | if (!err) { | ||
479 | err = gcm_hash(req, pctx); | ||
480 | if (err == -EINPROGRESS || err == -EBUSY) | ||
481 | return; | ||
482 | } | ||
483 | |||
484 | gcm_enc_hash_done(areq, err); | ||
485 | } | ||
486 | |||
332 | static int crypto_gcm_encrypt(struct aead_request *req) | 487 | static int crypto_gcm_encrypt(struct aead_request *req) |
333 | { | 488 | { |
334 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | 489 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
335 | struct ablkcipher_request *abreq = &pctx->abreq; | 490 | struct ablkcipher_request *abreq = &pctx->u.abreq; |
491 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; | ||
336 | int err; | 492 | int err; |
337 | 493 | ||
338 | crypto_gcm_init_crypt(abreq, req, req->cryptlen); | 494 | crypto_gcm_init_crypt(abreq, req, req->cryptlen); |
339 | ablkcipher_request_set_callback(abreq, aead_request_flags(req), | 495 | ablkcipher_request_set_callback(abreq, aead_request_flags(req), |
340 | crypto_gcm_encrypt_done, req); | 496 | gcm_encrypt_done, req); |
497 | |||
498 | gctx->src = req->dst; | ||
499 | gctx->cryptlen = req->cryptlen; | ||
500 | gctx->complete = gcm_enc_hash_done; | ||
341 | 501 | ||
342 | err = crypto_ablkcipher_encrypt(abreq); | 502 | err = crypto_ablkcipher_encrypt(abreq); |
343 | if (err) | 503 | if (err) |
344 | return err; | 504 | return err; |
345 | 505 | ||
346 | return crypto_gcm_hash(req); | 506 | err = gcm_hash(req, pctx); |
507 | if (err) | ||
508 | return err; | ||
509 | |||
510 | crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); | ||
511 | gcm_enc_copy_hash(req, pctx); | ||
512 | |||
513 | return 0; | ||
347 | } | 514 | } |
348 | 515 | ||
349 | static int crypto_gcm_verify(struct aead_request *req) | 516 | static int crypto_gcm_verify(struct aead_request *req, |
517 | struct crypto_gcm_req_priv_ctx *pctx) | ||
350 | { | 518 | { |
351 | struct crypto_aead *aead = crypto_aead_reqtfm(req); | 519 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
352 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | ||
353 | struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; | ||
354 | u8 *auth_tag = pctx->auth_tag; | 520 | u8 *auth_tag = pctx->auth_tag; |
355 | u8 *iauth_tag = pctx->iauth_tag; | 521 | u8 *iauth_tag = pctx->iauth_tag; |
356 | unsigned int authsize = crypto_aead_authsize(aead); | 522 | unsigned int authsize = crypto_aead_authsize(aead); |
357 | unsigned int cryptlen = req->cryptlen - authsize; | 523 | unsigned int cryptlen = req->cryptlen - authsize; |
358 | 524 | ||
359 | crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag); | 525 | crypto_xor(auth_tag, iauth_tag, 16); |
360 | |||
361 | authsize = crypto_aead_authsize(aead); | ||
362 | scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0); | 526 | scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0); |
363 | return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; | 527 | return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; |
364 | } | 528 | } |
365 | 529 | ||
366 | static void crypto_gcm_decrypt_done(struct crypto_async_request *areq, int err) | 530 | static void gcm_decrypt_done(struct crypto_async_request *areq, int err) |
367 | { | 531 | { |
368 | struct aead_request *req = areq->data; | 532 | struct aead_request *req = areq->data; |
533 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | ||
369 | 534 | ||
370 | if (!err) | 535 | if (!err) |
371 | err = crypto_gcm_verify(req); | 536 | err = crypto_gcm_verify(req, pctx); |
372 | 537 | ||
373 | aead_request_complete(req, err); | 538 | aead_request_complete(req, err); |
374 | } | 539 | } |
375 | 540 | ||
541 | static void gcm_dec_hash_done(struct crypto_async_request *areq, int err) | ||
542 | { | ||
543 | struct aead_request *req = areq->data; | ||
544 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | ||
545 | struct ablkcipher_request *abreq = &pctx->u.abreq; | ||
546 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; | ||
547 | |||
548 | if (!err) { | ||
549 | ablkcipher_request_set_callback(abreq, aead_request_flags(req), | ||
550 | gcm_decrypt_done, req); | ||
551 | crypto_gcm_init_crypt(abreq, req, gctx->cryptlen); | ||
552 | err = crypto_ablkcipher_decrypt(abreq); | ||
553 | if (err == -EINPROGRESS || err == -EBUSY) | ||
554 | return; | ||
555 | } | ||
556 | |||
557 | gcm_decrypt_done(areq, err); | ||
558 | } | ||
559 | |||
376 | static int crypto_gcm_decrypt(struct aead_request *req) | 560 | static int crypto_gcm_decrypt(struct aead_request *req) |
377 | { | 561 | { |
378 | struct crypto_aead *aead = crypto_aead_reqtfm(req); | 562 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
379 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); | 563 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
380 | struct ablkcipher_request *abreq = &pctx->abreq; | 564 | struct ablkcipher_request *abreq = &pctx->u.abreq; |
381 | struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; | 565 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; |
382 | unsigned int cryptlen = req->cryptlen; | ||
383 | unsigned int authsize = crypto_aead_authsize(aead); | 566 | unsigned int authsize = crypto_aead_authsize(aead); |
567 | unsigned int cryptlen = req->cryptlen; | ||
384 | int err; | 568 | int err; |
385 | 569 | ||
386 | if (cryptlen < authsize) | 570 | if (cryptlen < authsize) |
387 | return -EINVAL; | 571 | return -EINVAL; |
388 | cryptlen -= authsize; | 572 | cryptlen -= authsize; |
389 | 573 | ||
390 | crypto_gcm_init_crypt(abreq, req, cryptlen); | 574 | gctx->src = req->src; |
391 | ablkcipher_request_set_callback(abreq, aead_request_flags(req), | 575 | gctx->cryptlen = cryptlen; |
392 | crypto_gcm_decrypt_done, req); | 576 | gctx->complete = gcm_dec_hash_done; |
393 | 577 | ||
394 | crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen); | 578 | err = gcm_hash(req, pctx); |
579 | if (err) | ||
580 | return err; | ||
395 | 581 | ||
582 | ablkcipher_request_set_callback(abreq, aead_request_flags(req), | ||
583 | gcm_decrypt_done, req); | ||
584 | crypto_gcm_init_crypt(abreq, req, cryptlen); | ||
396 | err = crypto_ablkcipher_decrypt(abreq); | 585 | err = crypto_ablkcipher_decrypt(abreq); |
397 | if (err) | 586 | if (err) |
398 | return err; | 587 | return err; |
399 | 588 | ||
400 | return crypto_gcm_verify(req); | 589 | return crypto_gcm_verify(req, pctx); |
401 | } | 590 | } |
402 | 591 | ||
403 | static int crypto_gcm_init_tfm(struct crypto_tfm *tfm) | 592 | static int crypto_gcm_init_tfm(struct crypto_tfm *tfm) |
@@ -406,43 +595,56 @@ static int crypto_gcm_init_tfm(struct crypto_tfm *tfm) | |||
406 | struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst); | 595 | struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst); |
407 | struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); | 596 | struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); |
408 | struct crypto_ablkcipher *ctr; | 597 | struct crypto_ablkcipher *ctr; |
598 | struct crypto_ahash *ghash; | ||
409 | unsigned long align; | 599 | unsigned long align; |
410 | int err; | 600 | int err; |
411 | 601 | ||
602 | ghash = crypto_spawn_ahash(&ictx->ghash); | ||
603 | if (IS_ERR(ghash)) | ||
604 | return PTR_ERR(ghash); | ||
605 | |||
412 | ctr = crypto_spawn_skcipher(&ictx->ctr); | 606 | ctr = crypto_spawn_skcipher(&ictx->ctr); |
413 | err = PTR_ERR(ctr); | 607 | err = PTR_ERR(ctr); |
414 | if (IS_ERR(ctr)) | 608 | if (IS_ERR(ctr)) |
415 | return err; | 609 | goto err_free_hash; |
416 | 610 | ||
417 | ctx->ctr = ctr; | 611 | ctx->ctr = ctr; |
418 | ctx->gf128 = NULL; | 612 | ctx->ghash = ghash; |
419 | 613 | ||
420 | align = crypto_tfm_alg_alignmask(tfm); | 614 | align = crypto_tfm_alg_alignmask(tfm); |
421 | align &= ~(crypto_tfm_ctx_alignment() - 1); | 615 | align &= ~(crypto_tfm_ctx_alignment() - 1); |
422 | tfm->crt_aead.reqsize = align + | 616 | tfm->crt_aead.reqsize = align + |
423 | sizeof(struct crypto_gcm_req_priv_ctx) + | 617 | offsetof(struct crypto_gcm_req_priv_ctx, u) + |
424 | crypto_ablkcipher_reqsize(ctr); | 618 | max(sizeof(struct ablkcipher_request) + |
619 | crypto_ablkcipher_reqsize(ctr), | ||
620 | sizeof(struct ahash_request) + | ||
621 | crypto_ahash_reqsize(ghash)); | ||
425 | 622 | ||
426 | return 0; | 623 | return 0; |
624 | |||
625 | err_free_hash: | ||
626 | crypto_free_ahash(ghash); | ||
627 | return err; | ||
427 | } | 628 | } |
428 | 629 | ||
429 | static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm) | 630 | static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm) |
430 | { | 631 | { |
431 | struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); | 632 | struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); |
432 | 633 | ||
433 | if (ctx->gf128 != NULL) | 634 | crypto_free_ahash(ctx->ghash); |
434 | gf128mul_free_4k(ctx->gf128); | ||
435 | |||
436 | crypto_free_ablkcipher(ctx->ctr); | 635 | crypto_free_ablkcipher(ctx->ctr); |
437 | } | 636 | } |
438 | 637 | ||
439 | static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, | 638 | static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, |
440 | const char *full_name, | 639 | const char *full_name, |
441 | const char *ctr_name) | 640 | const char *ctr_name, |
641 | const char *ghash_name) | ||
442 | { | 642 | { |
443 | struct crypto_attr_type *algt; | 643 | struct crypto_attr_type *algt; |
444 | struct crypto_instance *inst; | 644 | struct crypto_instance *inst; |
445 | struct crypto_alg *ctr; | 645 | struct crypto_alg *ctr; |
646 | struct crypto_alg *ghash_alg; | ||
647 | struct ahash_alg *ghash_ahash_alg; | ||
446 | struct gcm_instance_ctx *ctx; | 648 | struct gcm_instance_ctx *ctx; |
447 | int err; | 649 | int err; |
448 | 650 | ||
@@ -454,17 +656,31 @@ static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, | |||
454 | if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) | 656 | if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) |
455 | return ERR_PTR(-EINVAL); | 657 | return ERR_PTR(-EINVAL); |
456 | 658 | ||
659 | ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type, | ||
660 | CRYPTO_ALG_TYPE_HASH, | ||
661 | CRYPTO_ALG_TYPE_AHASH_MASK); | ||
662 | err = PTR_ERR(ghash_alg); | ||
663 | if (IS_ERR(ghash_alg)) | ||
664 | return ERR_PTR(err); | ||
665 | |||
666 | err = -ENOMEM; | ||
457 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); | 667 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
458 | if (!inst) | 668 | if (!inst) |
459 | return ERR_PTR(-ENOMEM); | 669 | goto out_put_ghash; |
460 | 670 | ||
461 | ctx = crypto_instance_ctx(inst); | 671 | ctx = crypto_instance_ctx(inst); |
672 | ghash_ahash_alg = container_of(ghash_alg, struct ahash_alg, halg.base); | ||
673 | err = crypto_init_ahash_spawn(&ctx->ghash, &ghash_ahash_alg->halg, | ||
674 | inst); | ||
675 | if (err) | ||
676 | goto err_free_inst; | ||
677 | |||
462 | crypto_set_skcipher_spawn(&ctx->ctr, inst); | 678 | crypto_set_skcipher_spawn(&ctx->ctr, inst); |
463 | err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0, | 679 | err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0, |
464 | crypto_requires_sync(algt->type, | 680 | crypto_requires_sync(algt->type, |
465 | algt->mask)); | 681 | algt->mask)); |
466 | if (err) | 682 | if (err) |
467 | goto err_free_inst; | 683 | goto err_drop_ghash; |
468 | 684 | ||
469 | ctr = crypto_skcipher_spawn_alg(&ctx->ctr); | 685 | ctr = crypto_skcipher_spawn_alg(&ctx->ctr); |
470 | 686 | ||
@@ -479,7 +695,8 @@ static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, | |||
479 | 695 | ||
480 | err = -ENAMETOOLONG; | 696 | err = -ENAMETOOLONG; |
481 | if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, | 697 | if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, |
482 | "gcm_base(%s)", ctr->cra_driver_name) >= | 698 | "gcm_base(%s,%s)", ctr->cra_driver_name, |
699 | ghash_alg->cra_driver_name) >= | ||
483 | CRYPTO_MAX_ALG_NAME) | 700 | CRYPTO_MAX_ALG_NAME) |
484 | goto out_put_ctr; | 701 | goto out_put_ctr; |
485 | 702 | ||
@@ -502,12 +719,16 @@ static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, | |||
502 | inst->alg.cra_aead.decrypt = crypto_gcm_decrypt; | 719 | inst->alg.cra_aead.decrypt = crypto_gcm_decrypt; |
503 | 720 | ||
504 | out: | 721 | out: |
722 | crypto_mod_put(ghash_alg); | ||
505 | return inst; | 723 | return inst; |
506 | 724 | ||
507 | out_put_ctr: | 725 | out_put_ctr: |
508 | crypto_drop_skcipher(&ctx->ctr); | 726 | crypto_drop_skcipher(&ctx->ctr); |
727 | err_drop_ghash: | ||
728 | crypto_drop_ahash(&ctx->ghash); | ||
509 | err_free_inst: | 729 | err_free_inst: |
510 | kfree(inst); | 730 | kfree(inst); |
731 | out_put_ghash: | ||
511 | inst = ERR_PTR(err); | 732 | inst = ERR_PTR(err); |
512 | goto out; | 733 | goto out; |
513 | } | 734 | } |
@@ -532,7 +753,7 @@ static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb) | |||
532 | CRYPTO_MAX_ALG_NAME) | 753 | CRYPTO_MAX_ALG_NAME) |
533 | return ERR_PTR(-ENAMETOOLONG); | 754 | return ERR_PTR(-ENAMETOOLONG); |
534 | 755 | ||
535 | return crypto_gcm_alloc_common(tb, full_name, ctr_name); | 756 | return crypto_gcm_alloc_common(tb, full_name, ctr_name, "ghash"); |
536 | } | 757 | } |
537 | 758 | ||
538 | static void crypto_gcm_free(struct crypto_instance *inst) | 759 | static void crypto_gcm_free(struct crypto_instance *inst) |
@@ -540,6 +761,7 @@ static void crypto_gcm_free(struct crypto_instance *inst) | |||
540 | struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst); | 761 | struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst); |
541 | 762 | ||
542 | crypto_drop_skcipher(&ctx->ctr); | 763 | crypto_drop_skcipher(&ctx->ctr); |
764 | crypto_drop_ahash(&ctx->ghash); | ||
543 | kfree(inst); | 765 | kfree(inst); |
544 | } | 766 | } |
545 | 767 | ||
@@ -554,6 +776,7 @@ static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb) | |||
554 | { | 776 | { |
555 | int err; | 777 | int err; |
556 | const char *ctr_name; | 778 | const char *ctr_name; |
779 | const char *ghash_name; | ||
557 | char full_name[CRYPTO_MAX_ALG_NAME]; | 780 | char full_name[CRYPTO_MAX_ALG_NAME]; |
558 | 781 | ||
559 | ctr_name = crypto_attr_alg_name(tb[1]); | 782 | ctr_name = crypto_attr_alg_name(tb[1]); |
@@ -561,11 +784,16 @@ static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb) | |||
561 | if (IS_ERR(ctr_name)) | 784 | if (IS_ERR(ctr_name)) |
562 | return ERR_PTR(err); | 785 | return ERR_PTR(err); |
563 | 786 | ||
564 | if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s)", | 787 | ghash_name = crypto_attr_alg_name(tb[2]); |
565 | ctr_name) >= CRYPTO_MAX_ALG_NAME) | 788 | err = PTR_ERR(ghash_name); |
789 | if (IS_ERR(ghash_name)) | ||
790 | return ERR_PTR(err); | ||
791 | |||
792 | if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)", | ||
793 | ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME) | ||
566 | return ERR_PTR(-ENAMETOOLONG); | 794 | return ERR_PTR(-ENAMETOOLONG); |
567 | 795 | ||
568 | return crypto_gcm_alloc_common(tb, full_name, ctr_name); | 796 | return crypto_gcm_alloc_common(tb, full_name, ctr_name, ghash_name); |
569 | } | 797 | } |
570 | 798 | ||
571 | static struct crypto_template crypto_gcm_base_tmpl = { | 799 | static struct crypto_template crypto_gcm_base_tmpl = { |
@@ -784,6 +1012,10 @@ static int __init crypto_gcm_module_init(void) | |||
784 | { | 1012 | { |
785 | int err; | 1013 | int err; |
786 | 1014 | ||
1015 | gcm_zeroes = kzalloc(16, GFP_KERNEL); | ||
1016 | if (!gcm_zeroes) | ||
1017 | return -ENOMEM; | ||
1018 | |||
787 | err = crypto_register_template(&crypto_gcm_base_tmpl); | 1019 | err = crypto_register_template(&crypto_gcm_base_tmpl); |
788 | if (err) | 1020 | if (err) |
789 | goto out; | 1021 | goto out; |
@@ -796,18 +1028,20 @@ static int __init crypto_gcm_module_init(void) | |||
796 | if (err) | 1028 | if (err) |
797 | goto out_undo_gcm; | 1029 | goto out_undo_gcm; |
798 | 1030 | ||
799 | out: | 1031 | return 0; |
800 | return err; | ||
801 | 1032 | ||
802 | out_undo_gcm: | 1033 | out_undo_gcm: |
803 | crypto_unregister_template(&crypto_gcm_tmpl); | 1034 | crypto_unregister_template(&crypto_gcm_tmpl); |
804 | out_undo_base: | 1035 | out_undo_base: |
805 | crypto_unregister_template(&crypto_gcm_base_tmpl); | 1036 | crypto_unregister_template(&crypto_gcm_base_tmpl); |
806 | goto out; | 1037 | out: |
1038 | kfree(gcm_zeroes); | ||
1039 | return err; | ||
807 | } | 1040 | } |
808 | 1041 | ||
809 | static void __exit crypto_gcm_module_exit(void) | 1042 | static void __exit crypto_gcm_module_exit(void) |
810 | { | 1043 | { |
1044 | kfree(gcm_zeroes); | ||
811 | crypto_unregister_template(&crypto_rfc4106_tmpl); | 1045 | crypto_unregister_template(&crypto_rfc4106_tmpl); |
812 | crypto_unregister_template(&crypto_gcm_tmpl); | 1046 | crypto_unregister_template(&crypto_gcm_tmpl); |
813 | crypto_unregister_template(&crypto_gcm_base_tmpl); | 1047 | crypto_unregister_template(&crypto_gcm_base_tmpl); |