aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGilad Ben-Yossef <gilad@benyossef.com>2018-01-22 04:27:03 -0500
committerHerbert Xu <herbert@gondor.apana.org.au>2018-02-15 10:26:42 -0500
commitff27e85a85bbde19589f775297db92ff925e5981 (patch)
tree9ae7d4858b55898b41f48833c9e02136975b7602
parent63893811b0fcb52f6eaf9811cc08bddd46f81c3e (diff)
crypto: ccree - add AEAD support
Add CryptoCell AEAD support Signed-off-by: Gilad Ben-Yossef <gilad@benyossef.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Diffstat
-rw-r--r--drivers/crypto/ccree/Makefile2
-rw-r--r--drivers/crypto/ccree/cc_aead.c2702
-rw-r--r--drivers/crypto/ccree/cc_aead.h109
-rw-r--r--drivers/crypto/ccree/cc_buffer_mgr.c882
-rw-r--r--drivers/crypto/ccree/cc_buffer_mgr.h4
-rw-r--r--drivers/crypto/ccree/cc_driver.c10
-rw-r--r--drivers/crypto/ccree/cc_driver.h2
7 files changed, 3710 insertions, 1 deletions
diff --git a/drivers/crypto/ccree/Makefile b/drivers/crypto/ccree/Makefile
index 11094809c313..7cb308295188 100644
--- a/drivers/crypto/ccree/Makefile
+++ b/drivers/crypto/ccree/Makefile
@@ -1,6 +1,6 @@
1# SPDX-License-Identifier: GPL-2.0 1# SPDX-License-Identifier: GPL-2.0
2 2
3obj-$(CONFIG_CRYPTO_DEV_CCREE) := ccree.o 3obj-$(CONFIG_CRYPTO_DEV_CCREE) := ccree.o
4ccree-y := cc_driver.o cc_buffer_mgr.o cc_request_mgr.o cc_cipher.o cc_hash.o cc_ivgen.o cc_sram_mgr.o 4ccree-y := cc_driver.o cc_buffer_mgr.o cc_request_mgr.o cc_cipher.o cc_hash.o cc_aead.o cc_ivgen.o cc_sram_mgr.o
5ccree-$(CONFIG_DEBUG_FS) += cc_debugfs.o 5ccree-$(CONFIG_DEBUG_FS) += cc_debugfs.o
6ccree-$(CONFIG_PM) += cc_pm.o 6ccree-$(CONFIG_PM) += cc_pm.o
diff --git a/drivers/crypto/ccree/cc_aead.c b/drivers/crypto/ccree/cc_aead.c
new file mode 100644
index 000000000000..3e1046a87a01
--- /dev/null
+++ b/drivers/crypto/ccree/cc_aead.c
@@ -0,0 +1,2702 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
3
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <crypto/algapi.h>
7#include <crypto/internal/aead.h>
8#include <crypto/authenc.h>
9#include <crypto/des.h>
10#include <linux/rtnetlink.h>
11#include "cc_driver.h"
12#include "cc_buffer_mgr.h"
13#include "cc_aead.h"
14#include "cc_request_mgr.h"
15#include "cc_hash.h"
16#include "cc_sram_mgr.h"
17
18#define template_aead template_u.aead
19
20#define MAX_AEAD_SETKEY_SEQ 12
21#define MAX_AEAD_PROCESS_SEQ 23
22
23#define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE)
24#define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE)
25
26#define AES_CCM_RFC4309_NONCE_SIZE 3
27#define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE
28
29/* Value of each ICV_CMP byte (of 8) in case of success */
30#define ICV_VERIF_OK 0x01
31
32struct cc_aead_handle {
33 cc_sram_addr_t sram_workspace_addr;
34 struct list_head aead_list;
35};
36
37struct cc_hmac_s {
38 u8 *padded_authkey;
39 u8 *ipad_opad; /* IPAD, OPAD*/
40 dma_addr_t padded_authkey_dma_addr;
41 dma_addr_t ipad_opad_dma_addr;
42};
43
44struct cc_xcbc_s {
45 u8 *xcbc_keys; /* K1,K2,K3 */
46 dma_addr_t xcbc_keys_dma_addr;
47};
48
49struct cc_aead_ctx {
50 struct cc_drvdata *drvdata;
51 u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */
52 u8 *enckey;
53 dma_addr_t enckey_dma_addr;
54 union {
55 struct cc_hmac_s hmac;
56 struct cc_xcbc_s xcbc;
57 } auth_state;
58 unsigned int enc_keylen;
59 unsigned int auth_keylen;
60 unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */
61 enum drv_cipher_mode cipher_mode;
62 enum cc_flow_mode flow_mode;
63 enum drv_hash_mode auth_mode;
64};
65
66static inline bool valid_assoclen(struct aead_request *req)
67{
68 return ((req->assoclen == 16) || (req->assoclen == 20));
69}
70
71static void cc_aead_exit(struct crypto_aead *tfm)
72{
73 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
74 struct device *dev = drvdata_to_dev(ctx->drvdata);
75
76 dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm),
77 crypto_tfm_alg_name(&tfm->base));
78
79 /* Unmap enckey buffer */
80 if (ctx->enckey) {
81 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey,
82 ctx->enckey_dma_addr);
83 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n",
84 &ctx->enckey_dma_addr);
85 ctx->enckey_dma_addr = 0;
86 ctx->enckey = NULL;
87 }
88
89 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
90 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
91
92 if (xcbc->xcbc_keys) {
93 dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3,
94 xcbc->xcbc_keys,
95 xcbc->xcbc_keys_dma_addr);
96 }
97 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n",
98 &xcbc->xcbc_keys_dma_addr);
99 xcbc->xcbc_keys_dma_addr = 0;
100 xcbc->xcbc_keys = NULL;
101 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */
102 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
103
104 if (hmac->ipad_opad) {
105 dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE,
106 hmac->ipad_opad,
107 hmac->ipad_opad_dma_addr);
108 dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n",
109 &hmac->ipad_opad_dma_addr);
110 hmac->ipad_opad_dma_addr = 0;
111 hmac->ipad_opad = NULL;
112 }
113 if (hmac->padded_authkey) {
114 dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE,
115 hmac->padded_authkey,
116 hmac->padded_authkey_dma_addr);
117 dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n",
118 &hmac->padded_authkey_dma_addr);
119 hmac->padded_authkey_dma_addr = 0;
120 hmac->padded_authkey = NULL;
121 }
122 }
123}
124
125static int cc_aead_init(struct crypto_aead *tfm)
126{
127 struct aead_alg *alg = crypto_aead_alg(tfm);
128 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
129 struct cc_crypto_alg *cc_alg =
130 container_of(alg, struct cc_crypto_alg, aead_alg);
131 struct device *dev = drvdata_to_dev(cc_alg->drvdata);
132
133 dev_dbg(dev, "Initializing context @%p for %s\n", ctx,
134 crypto_tfm_alg_name(&tfm->base));
135
136 /* Initialize modes in instance */
137 ctx->cipher_mode = cc_alg->cipher_mode;
138 ctx->flow_mode = cc_alg->flow_mode;
139 ctx->auth_mode = cc_alg->auth_mode;
140 ctx->drvdata = cc_alg->drvdata;
141 crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx));
142
143 /* Allocate key buffer, cache line aligned */
144 ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE,
145 &ctx->enckey_dma_addr, GFP_KERNEL);
146 if (!ctx->enckey) {
147 dev_err(dev, "Failed allocating key buffer\n");
148 goto init_failed;
149 }
150 dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n",
151 ctx->enckey);
152
153 /* Set default authlen value */
154
155 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
156 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
157 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3;
158
159 /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */
160 /* (and temporary for user key - up to 256b) */
161 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size,
162 &xcbc->xcbc_keys_dma_addr,
163 GFP_KERNEL);
164 if (!xcbc->xcbc_keys) {
165 dev_err(dev, "Failed allocating buffer for XCBC keys\n");
166 goto init_failed;
167 }
168 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */
169 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
170 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE;
171 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr;
172
173 /* Allocate dma-coherent buffer for IPAD + OPAD */
174 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size,
175 &hmac->ipad_opad_dma_addr,
176 GFP_KERNEL);
177
178 if (!hmac->ipad_opad) {
179 dev_err(dev, "Failed allocating IPAD/OPAD buffer\n");
180 goto init_failed;
181 }
182
183 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n",
184 hmac->ipad_opad);
185
186 hmac->padded_authkey = dma_alloc_coherent(dev,
187 MAX_HMAC_BLOCK_SIZE,
188 pkey_dma,
189 GFP_KERNEL);
190
191 if (!hmac->padded_authkey) {
192 dev_err(dev, "failed to allocate padded_authkey\n");
193 goto init_failed;
194 }
195 } else {
196 ctx->auth_state.hmac.ipad_opad = NULL;
197 ctx->auth_state.hmac.padded_authkey = NULL;
198 }
199
200 return 0;
201
202init_failed:
203 cc_aead_exit(tfm);
204 return -ENOMEM;
205}
206
207static void cc_aead_complete(struct device *dev, void *cc_req, int err)
208{
209 struct aead_request *areq = (struct aead_request *)cc_req;
210 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
211 struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req);
212 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
213
214 cc_unmap_aead_request(dev, areq);
215
216 /* Restore ordinary iv pointer */
217 areq->iv = areq_ctx->backup_iv;
218
219 if (err)
220 goto done;
221
222 if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
223 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr,
224 ctx->authsize) != 0) {
225 dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n",
226 ctx->authsize, ctx->cipher_mode);
227 /* In case of payload authentication failure, MUST NOT
228 * revealed the decrypted message --> zero its memory.
229 */
230 cc_zero_sgl(areq->dst, areq_ctx->cryptlen);
231 err = -EBADMSG;
232 }
233 } else { /*ENCRYPT*/
234 if (areq_ctx->is_icv_fragmented) {
235 u32 skip = areq->cryptlen + areq_ctx->dst_offset;
236
237 cc_copy_sg_portion(dev, areq_ctx->mac_buf,
238 areq_ctx->dst_sgl, skip,
239 (skip + ctx->authsize),
240 CC_SG_FROM_BUF);
241 }
242
243 /* If an IV was generated, copy it back to the user provided
244 * buffer.
245 */
246 if (areq_ctx->backup_giv) {
247 if (ctx->cipher_mode == DRV_CIPHER_CTR)
248 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
249 CTR_RFC3686_NONCE_SIZE,
250 CTR_RFC3686_IV_SIZE);
251 else if (ctx->cipher_mode == DRV_CIPHER_CCM)
252 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
253 CCM_BLOCK_IV_OFFSET, CCM_BLOCK_IV_SIZE);
254 }
255 }
256done:
257 aead_request_complete(areq, err);
258}
259
260static unsigned int xcbc_setkey(struct cc_hw_desc *desc,
261 struct cc_aead_ctx *ctx)
262{
263 /* Load the AES key */
264 hw_desc_init(&desc[0]);
265 /* We are using for the source/user key the same buffer
266 * as for the output keys, * because after this key loading it
267 * is not needed anymore
268 */
269 set_din_type(&desc[0], DMA_DLLI,
270 ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen,
271 NS_BIT);
272 set_cipher_mode(&desc[0], DRV_CIPHER_ECB);
273 set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT);
274 set_key_size_aes(&desc[0], ctx->auth_keylen);
275 set_flow_mode(&desc[0], S_DIN_to_AES);
276 set_setup_mode(&desc[0], SETUP_LOAD_KEY0);
277
278 hw_desc_init(&desc[1]);
279 set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE);
280 set_flow_mode(&desc[1], DIN_AES_DOUT);
281 set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr,
282 AES_KEYSIZE_128, NS_BIT, 0);
283
284 hw_desc_init(&desc[2]);
285 set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE);
286 set_flow_mode(&desc[2], DIN_AES_DOUT);
287 set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
288 + AES_KEYSIZE_128),
289 AES_KEYSIZE_128, NS_BIT, 0);
290
291 hw_desc_init(&desc[3]);
292 set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE);
293 set_flow_mode(&desc[3], DIN_AES_DOUT);
294 set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
295 + 2 * AES_KEYSIZE_128),
296 AES_KEYSIZE_128, NS_BIT, 0);
297
298 return 4;
299}
300
301static int hmac_setkey(struct cc_hw_desc *desc, struct cc_aead_ctx *ctx)
302{
303 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
304 unsigned int digest_ofs = 0;
305 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
306 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
307 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
308 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
309 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
310
311 unsigned int idx = 0;
312 int i;
313
314 /* calc derived HMAC key */
315 for (i = 0; i < 2; i++) {
316 /* Load hash initial state */
317 hw_desc_init(&desc[idx]);
318 set_cipher_mode(&desc[idx], hash_mode);
319 set_din_sram(&desc[idx],
320 cc_larval_digest_addr(ctx->drvdata,
321 ctx->auth_mode),
322 digest_size);
323 set_flow_mode(&desc[idx], S_DIN_to_HASH);
324 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
325 idx++;
326
327 /* Load the hash current length*/
328 hw_desc_init(&desc[idx]);
329 set_cipher_mode(&desc[idx], hash_mode);
330 set_din_const(&desc[idx], 0, HASH_LEN_SIZE);
331 set_flow_mode(&desc[idx], S_DIN_to_HASH);
332 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
333 idx++;
334
335 /* Prepare ipad key */
336 hw_desc_init(&desc[idx]);
337 set_xor_val(&desc[idx], hmac_pad_const[i]);
338 set_cipher_mode(&desc[idx], hash_mode);
339 set_flow_mode(&desc[idx], S_DIN_to_HASH);
340 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
341 idx++;
342
343 /* Perform HASH update */
344 hw_desc_init(&desc[idx]);
345 set_din_type(&desc[idx], DMA_DLLI,
346 hmac->padded_authkey_dma_addr,
347 SHA256_BLOCK_SIZE, NS_BIT);
348 set_cipher_mode(&desc[idx], hash_mode);
349 set_xor_active(&desc[idx]);
350 set_flow_mode(&desc[idx], DIN_HASH);
351 idx++;
352
353 /* Get the digset */
354 hw_desc_init(&desc[idx]);
355 set_cipher_mode(&desc[idx], hash_mode);
356 set_dout_dlli(&desc[idx],
357 (hmac->ipad_opad_dma_addr + digest_ofs),
358 digest_size, NS_BIT, 0);
359 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
360 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
361 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
362 idx++;
363
364 digest_ofs += digest_size;
365 }
366
367 return idx;
368}
369
370static int validate_keys_sizes(struct cc_aead_ctx *ctx)
371{
372 struct device *dev = drvdata_to_dev(ctx->drvdata);
373
374 dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n",
375 ctx->enc_keylen, ctx->auth_keylen);
376
377 switch (ctx->auth_mode) {
378 case DRV_HASH_SHA1:
379 case DRV_HASH_SHA256:
380 break;
381 case DRV_HASH_XCBC_MAC:
382 if (ctx->auth_keylen != AES_KEYSIZE_128 &&
383 ctx->auth_keylen != AES_KEYSIZE_192 &&
384 ctx->auth_keylen != AES_KEYSIZE_256)
385 return -ENOTSUPP;
386 break;
387 case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */
388 if (ctx->auth_keylen > 0)
389 return -EINVAL;
390 break;
391 default:
392 dev_err(dev, "Invalid auth_mode=%d\n", ctx->auth_mode);
393 return -EINVAL;
394 }
395 /* Check cipher key size */
396 if (ctx->flow_mode == S_DIN_to_DES) {
397 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) {
398 dev_err(dev, "Invalid cipher(3DES) key size: %u\n",
399 ctx->enc_keylen);
400 return -EINVAL;
401 }
402 } else { /* Default assumed to be AES ciphers */
403 if (ctx->enc_keylen != AES_KEYSIZE_128 &&
404 ctx->enc_keylen != AES_KEYSIZE_192 &&
405 ctx->enc_keylen != AES_KEYSIZE_256) {
406 dev_err(dev, "Invalid cipher(AES) key size: %u\n",
407 ctx->enc_keylen);
408 return -EINVAL;
409 }
410 }
411
412 return 0; /* All tests of keys sizes passed */
413}
414
415/* This function prepers the user key so it can pass to the hmac processing
416 * (copy to intenral buffer or hash in case of key longer than block
417 */
418static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *key,
419 unsigned int keylen)
420{
421 dma_addr_t key_dma_addr = 0;
422 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
423 struct device *dev = drvdata_to_dev(ctx->drvdata);
424 u32 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->auth_mode);
425 struct cc_crypto_req cc_req = {};
426 unsigned int blocksize;
427 unsigned int digestsize;
428 unsigned int hashmode;
429 unsigned int idx = 0;
430 int rc = 0;
431 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
432 dma_addr_t padded_authkey_dma_addr =
433 ctx->auth_state.hmac.padded_authkey_dma_addr;
434
435 switch (ctx->auth_mode) { /* auth_key required and >0 */
436 case DRV_HASH_SHA1:
437 blocksize = SHA1_BLOCK_SIZE;
438 digestsize = SHA1_DIGEST_SIZE;
439 hashmode = DRV_HASH_HW_SHA1;
440 break;
441 case DRV_HASH_SHA256:
442 default:
443 blocksize = SHA256_BLOCK_SIZE;
444 digestsize = SHA256_DIGEST_SIZE;
445 hashmode = DRV_HASH_HW_SHA256;
446 }
447
448 if (keylen != 0) {
449 key_dma_addr = dma_map_single(dev, (void *)key, keylen,
450 DMA_TO_DEVICE);
451 if (dma_mapping_error(dev, key_dma_addr)) {
452 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
453 key, keylen);
454 return -ENOMEM;
455 }
456 if (keylen > blocksize) {
457 /* Load hash initial state */
458 hw_desc_init(&desc[idx]);
459 set_cipher_mode(&desc[idx], hashmode);
460 set_din_sram(&desc[idx], larval_addr, digestsize);
461 set_flow_mode(&desc[idx], S_DIN_to_HASH);
462 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
463 idx++;
464
465 /* Load the hash current length*/
466 hw_desc_init(&desc[idx]);
467 set_cipher_mode(&desc[idx], hashmode);
468 set_din_const(&desc[idx], 0, HASH_LEN_SIZE);
469 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
470 set_flow_mode(&desc[idx], S_DIN_to_HASH);
471 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
472 idx++;
473
474 hw_desc_init(&desc[idx]);
475 set_din_type(&desc[idx], DMA_DLLI,
476 key_dma_addr, keylen, NS_BIT);
477 set_flow_mode(&desc[idx], DIN_HASH);
478 idx++;
479
480 /* Get hashed key */
481 hw_desc_init(&desc[idx]);
482 set_cipher_mode(&desc[idx], hashmode);
483 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
484 digestsize, NS_BIT, 0);
485 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
486 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
487 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
488 set_cipher_config0(&desc[idx],
489 HASH_DIGEST_RESULT_LITTLE_ENDIAN);
490 idx++;
491
492 hw_desc_init(&desc[idx]);
493 set_din_const(&desc[idx], 0, (blocksize - digestsize));
494 set_flow_mode(&desc[idx], BYPASS);
495 set_dout_dlli(&desc[idx], (padded_authkey_dma_addr +
496 digestsize), (blocksize - digestsize),
497 NS_BIT, 0);
498 idx++;
499 } else {
500 hw_desc_init(&desc[idx]);
501 set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
502 keylen, NS_BIT);
503 set_flow_mode(&desc[idx], BYPASS);
504 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
505 keylen, NS_BIT, 0);
506 idx++;
507
508 if ((blocksize - keylen) != 0) {
509 hw_desc_init(&desc[idx]);
510 set_din_const(&desc[idx], 0,
511 (blocksize - keylen));
512 set_flow_mode(&desc[idx], BYPASS);
513 set_dout_dlli(&desc[idx],
514 (padded_authkey_dma_addr +
515 keylen),
516 (blocksize - keylen), NS_BIT, 0);
517 idx++;
518 }
519 }
520 } else {
521 hw_desc_init(&desc[idx]);
522 set_din_const(&desc[idx], 0, (blocksize - keylen));
523 set_flow_mode(&desc[idx], BYPASS);
524 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
525 blocksize, NS_BIT, 0);
526 idx++;
527 }
528
529 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
530 if (rc)
531 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
532
533 if (key_dma_addr)
534 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE);
535
536 return rc;
537}
538
539static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key,
540 unsigned int keylen)
541{
542 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
543 struct rtattr *rta = (struct rtattr *)key;
544 struct cc_crypto_req cc_req = {};
545 struct crypto_authenc_key_param *param;
546 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
547 int rc = -EINVAL;
548 unsigned int seq_len = 0;
549 struct device *dev = drvdata_to_dev(ctx->drvdata);
550
551 dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n",
552 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen);
553
554 /* STAT_PHASE_0: Init and sanity checks */
555
556 if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */
557 if (!RTA_OK(rta, keylen))
558 goto badkey;
559 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
560 goto badkey;
561 if (RTA_PAYLOAD(rta) < sizeof(*param))
562 goto badkey;
563 param = RTA_DATA(rta);
564 ctx->enc_keylen = be32_to_cpu(param->enckeylen);
565 key += RTA_ALIGN(rta->rta_len);
566 keylen -= RTA_ALIGN(rta->rta_len);
567 if (keylen < ctx->enc_keylen)
568 goto badkey;
569 ctx->auth_keylen = keylen - ctx->enc_keylen;
570
571 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
572 /* the nonce is stored in bytes at end of key */
573 if (ctx->enc_keylen <
574 (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE))
575 goto badkey;
576 /* Copy nonce from last 4 bytes in CTR key to
577 * first 4 bytes in CTR IV
578 */
579 memcpy(ctx->ctr_nonce, key + ctx->auth_keylen +
580 ctx->enc_keylen - CTR_RFC3686_NONCE_SIZE,
581 CTR_RFC3686_NONCE_SIZE);
582 /* Set CTR key size */
583 ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE;
584 }
585 } else { /* non-authenc - has just one key */
586 ctx->enc_keylen = keylen;
587 ctx->auth_keylen = 0;
588 }
589
590 rc = validate_keys_sizes(ctx);
591 if (rc)
592 goto badkey;
593
594 /* STAT_PHASE_1: Copy key to ctx */
595
596 /* Get key material */
597 memcpy(ctx->enckey, key + ctx->auth_keylen, ctx->enc_keylen);
598 if (ctx->enc_keylen == 24)
599 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24);
600 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
601 memcpy(ctx->auth_state.xcbc.xcbc_keys, key, ctx->auth_keylen);
602 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */
603 rc = cc_get_plain_hmac_key(tfm, key, ctx->auth_keylen);
604 if (rc)
605 goto badkey;
606 }
607
608 /* STAT_PHASE_2: Create sequence */
609
610 switch (ctx->auth_mode) {
611 case DRV_HASH_SHA1:
612 case DRV_HASH_SHA256:
613 seq_len = hmac_setkey(desc, ctx);
614 break;
615 case DRV_HASH_XCBC_MAC:
616 seq_len = xcbc_setkey(desc, ctx);
617 break;
618 case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */
619 break; /* No auth. key setup */
620 default:
621 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
622 rc = -ENOTSUPP;
623 goto badkey;
624 }
625
626 /* STAT_PHASE_3: Submit sequence to HW */
627
628 if (seq_len > 0) { /* For CCM there is no sequence to setup the key */
629 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len);
630 if (rc) {
631 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
632 goto setkey_error;
633 }
634 }
635
636 /* Update STAT_PHASE_3 */
637 return rc;
638
639badkey:
640 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
641
642setkey_error:
643 return rc;
644}
645
646static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key,
647 unsigned int keylen)
648{
649 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
650
651 if (keylen < 3)
652 return -EINVAL;
653
654 keylen -= 3;
655 memcpy(ctx->ctr_nonce, key + keylen, 3);
656
657 return cc_aead_setkey(tfm, key, keylen);
658}
659
660static int cc_aead_setauthsize(struct crypto_aead *authenc,
661 unsigned int authsize)
662{
663 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
664 struct device *dev = drvdata_to_dev(ctx->drvdata);
665
666 /* Unsupported auth. sizes */
667 if (authsize == 0 ||
668 authsize > crypto_aead_maxauthsize(authenc)) {
669 return -ENOTSUPP;
670 }
671
672 ctx->authsize = authsize;
673 dev_dbg(dev, "authlen=%d\n", ctx->authsize);
674
675 return 0;
676}
677
678static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc,
679 unsigned int authsize)
680{
681 switch (authsize) {
682 case 8:
683 case 12:
684 case 16:
685 break;
686 default:
687 return -EINVAL;
688 }
689
690 return cc_aead_setauthsize(authenc, authsize);
691}
692
693static int cc_ccm_setauthsize(struct crypto_aead *authenc,
694 unsigned int authsize)
695{
696 switch (authsize) {
697 case 4:
698 case 6:
699 case 8:
700 case 10:
701 case 12:
702 case 14:
703 case 16:
704 break;
705 default:
706 return -EINVAL;
707 }
708
709 return cc_aead_setauthsize(authenc, authsize);
710}
711
712static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode,
713 struct cc_hw_desc desc[], unsigned int *seq_size)
714{
715 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
716 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
717 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
718 enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type;
719 unsigned int idx = *seq_size;
720 struct device *dev = drvdata_to_dev(ctx->drvdata);
721
722 switch (assoc_dma_type) {
723 case CC_DMA_BUF_DLLI:
724 dev_dbg(dev, "ASSOC buffer type DLLI\n");
725 hw_desc_init(&desc[idx]);
726 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src),
727 areq->assoclen, NS_BIT);
728 set_flow_mode(&desc[idx], flow_mode);
729 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
730 areq_ctx->cryptlen > 0)
731 set_din_not_last_indication(&desc[idx]);
732 break;
733 case CC_DMA_BUF_MLLI:
734 dev_dbg(dev, "ASSOC buffer type MLLI\n");
735 hw_desc_init(&desc[idx]);
736 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr,
737 areq_ctx->assoc.mlli_nents, NS_BIT);
738 set_flow_mode(&desc[idx], flow_mode);
739 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
740 areq_ctx->cryptlen > 0)
741 set_din_not_last_indication(&desc[idx]);
742 break;
743 case CC_DMA_BUF_NULL:
744 default:
745 dev_err(dev, "Invalid ASSOC buffer type\n");
746 }
747
748 *seq_size = (++idx);
749}
750
751static void cc_proc_authen_desc(struct aead_request *areq,
752 unsigned int flow_mode,
753 struct cc_hw_desc desc[],
754 unsigned int *seq_size, int direct)
755{
756 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
757 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
758 unsigned int idx = *seq_size;
759 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
760 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
761 struct device *dev = drvdata_to_dev(ctx->drvdata);
762
763 switch (data_dma_type) {
764 case CC_DMA_BUF_DLLI:
765 {
766 struct scatterlist *cipher =
767 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
768 areq_ctx->dst_sgl : areq_ctx->src_sgl;
769
770 unsigned int offset =
771 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
772 areq_ctx->dst_offset : areq_ctx->src_offset;
773 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n");
774 hw_desc_init(&desc[idx]);
775 set_din_type(&desc[idx], DMA_DLLI,
776 (sg_dma_address(cipher) + offset),
777 areq_ctx->cryptlen, NS_BIT);
778 set_flow_mode(&desc[idx], flow_mode);
779 break;
780 }
781 case CC_DMA_BUF_MLLI:
782 {
783 /* DOUBLE-PASS flow (as default)
784 * assoc. + iv + data -compact in one table
785 * if assoclen is ZERO only IV perform
786 */
787 cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr;
788 u32 mlli_nents = areq_ctx->assoc.mlli_nents;
789
790 if (areq_ctx->is_single_pass) {
791 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
792 mlli_addr = areq_ctx->dst.sram_addr;
793 mlli_nents = areq_ctx->dst.mlli_nents;
794 } else {
795 mlli_addr = areq_ctx->src.sram_addr;
796 mlli_nents = areq_ctx->src.mlli_nents;
797 }
798 }
799
800 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n");
801 hw_desc_init(&desc[idx]);
802 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents,
803 NS_BIT);
804 set_flow_mode(&desc[idx], flow_mode);
805 break;
806 }
807 case CC_DMA_BUF_NULL:
808 default:
809 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n");
810 }
811
812 *seq_size = (++idx);
813}
814
815static void cc_proc_cipher_desc(struct aead_request *areq,
816 unsigned int flow_mode,
817 struct cc_hw_desc desc[],
818 unsigned int *seq_size)
819{
820 unsigned int idx = *seq_size;
821 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
822 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
823 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
824 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
825 struct device *dev = drvdata_to_dev(ctx->drvdata);
826
827 if (areq_ctx->cryptlen == 0)
828 return; /*null processing*/
829
830 switch (data_dma_type) {
831 case CC_DMA_BUF_DLLI:
832 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n");
833 hw_desc_init(&desc[idx]);
834 set_din_type(&desc[idx], DMA_DLLI,
835 (sg_dma_address(areq_ctx->src_sgl) +
836 areq_ctx->src_offset), areq_ctx->cryptlen,
837 NS_BIT);
838 set_dout_dlli(&desc[idx],
839 (sg_dma_address(areq_ctx->dst_sgl) +
840 areq_ctx->dst_offset),
841 areq_ctx->cryptlen, NS_BIT, 0);
842 set_flow_mode(&desc[idx], flow_mode);
843 break;
844 case CC_DMA_BUF_MLLI:
845 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n");
846 hw_desc_init(&desc[idx]);
847 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr,
848 areq_ctx->src.mlli_nents, NS_BIT);
849 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr,
850 areq_ctx->dst.mlli_nents, NS_BIT, 0);
851 set_flow_mode(&desc[idx], flow_mode);
852 break;
853 case CC_DMA_BUF_NULL:
854 default:
855 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n");
856 }
857
858 *seq_size = (++idx);
859}
860
861static void cc_proc_digest_desc(struct aead_request *req,
862 struct cc_hw_desc desc[],
863 unsigned int *seq_size)
864{
865 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
866 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
867 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
868 unsigned int idx = *seq_size;
869 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
870 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
871 int direct = req_ctx->gen_ctx.op_type;
872
873 /* Get final ICV result */
874 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
875 hw_desc_init(&desc[idx]);
876 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
877 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
878 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize,
879 NS_BIT, 1);
880 set_queue_last_ind(&desc[idx]);
881 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
882 set_aes_not_hash_mode(&desc[idx]);
883 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
884 } else {
885 set_cipher_config0(&desc[idx],
886 HASH_DIGEST_RESULT_LITTLE_ENDIAN);
887 set_cipher_mode(&desc[idx], hash_mode);
888 }
889 } else { /*Decrypt*/
890 /* Get ICV out from hardware */
891 hw_desc_init(&desc[idx]);
892 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
893 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
894 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr,
895 ctx->authsize, NS_BIT, 1);
896 set_queue_last_ind(&desc[idx]);
897 set_cipher_config0(&desc[idx],
898 HASH_DIGEST_RESULT_LITTLE_ENDIAN);
899 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
900 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
901 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
902 set_aes_not_hash_mode(&desc[idx]);
903 } else {
904 set_cipher_mode(&desc[idx], hash_mode);
905 }
906 }
907
908 *seq_size = (++idx);
909}
910
911static void cc_set_cipher_desc(struct aead_request *req,
912 struct cc_hw_desc desc[],
913 unsigned int *seq_size)
914{
915 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
916 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
917 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
918 unsigned int hw_iv_size = req_ctx->hw_iv_size;
919 unsigned int idx = *seq_size;
920 int direct = req_ctx->gen_ctx.op_type;
921
922 /* Setup cipher state */
923 hw_desc_init(&desc[idx]);
924 set_cipher_config0(&desc[idx], direct);
925 set_flow_mode(&desc[idx], ctx->flow_mode);
926 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr,
927 hw_iv_size, NS_BIT);
928 if (ctx->cipher_mode == DRV_CIPHER_CTR)
929 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
930 else
931 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
932 set_cipher_mode(&desc[idx], ctx->cipher_mode);
933 idx++;
934
935 /* Setup enc. key */
936 hw_desc_init(&desc[idx]);
937 set_cipher_config0(&desc[idx], direct);
938 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
939 set_flow_mode(&desc[idx], ctx->flow_mode);
940 if (ctx->flow_mode == S_DIN_to_AES) {
941 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
942 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
943 ctx->enc_keylen), NS_BIT);
944 set_key_size_aes(&desc[idx], ctx->enc_keylen);
945 } else {
946 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
947 ctx->enc_keylen, NS_BIT);
948 set_key_size_des(&desc[idx], ctx->enc_keylen);
949 }
950 set_cipher_mode(&desc[idx], ctx->cipher_mode);
951 idx++;
952
953 *seq_size = idx;
954}
955
956static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[],
957 unsigned int *seq_size, unsigned int data_flow_mode)
958{
959 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
960 int direct = req_ctx->gen_ctx.op_type;
961 unsigned int idx = *seq_size;
962
963 if (req_ctx->cryptlen == 0)
964 return; /*null processing*/
965
966 cc_set_cipher_desc(req, desc, &idx);
967 cc_proc_cipher_desc(req, data_flow_mode, desc, &idx);
968 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
969 /* We must wait for DMA to write all cipher */
970 hw_desc_init(&desc[idx]);
971 set_din_no_dma(&desc[idx], 0, 0xfffff0);
972 set_dout_no_dma(&desc[idx], 0, 0, 1);
973 idx++;
974 }
975
976 *seq_size = idx;
977}
978
979static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[],
980 unsigned int *seq_size)
981{
982 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
983 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
984 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
985 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
986 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
987 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
988 unsigned int idx = *seq_size;
989
990 /* Loading hash ipad xor key state */
991 hw_desc_init(&desc[idx]);
992 set_cipher_mode(&desc[idx], hash_mode);
993 set_din_type(&desc[idx], DMA_DLLI,
994 ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size,
995 NS_BIT);
996 set_flow_mode(&desc[idx], S_DIN_to_HASH);
997 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
998 idx++;
999
1000 /* Load init. digest len (64 bytes) */
1001 hw_desc_init(&desc[idx]);
1002 set_cipher_mode(&desc[idx], hash_mode);
1003 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1004 HASH_LEN_SIZE);
1005 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1006 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1007 idx++;
1008
1009 *seq_size = idx;
1010}
1011
1012static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[],
1013 unsigned int *seq_size)
1014{
1015 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1016 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1017 unsigned int idx = *seq_size;
1018
1019 /* Loading MAC state */
1020 hw_desc_init(&desc[idx]);
1021 set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE);
1022 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1023 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1024 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1025 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1026 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1027 set_aes_not_hash_mode(&desc[idx]);
1028 idx++;
1029
1030 /* Setup XCBC MAC K1 */
1031 hw_desc_init(&desc[idx]);
1032 set_din_type(&desc[idx], DMA_DLLI,
1033 ctx->auth_state.xcbc.xcbc_keys_dma_addr,
1034 AES_KEYSIZE_128, NS_BIT);
1035 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1036 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1037 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1038 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1039 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1040 set_aes_not_hash_mode(&desc[idx]);
1041 idx++;
1042
1043 /* Setup XCBC MAC K2 */
1044 hw_desc_init(&desc[idx]);
1045 set_din_type(&desc[idx], DMA_DLLI,
1046 (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1047 AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1048 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1049 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1050 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1051 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1052 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1053 set_aes_not_hash_mode(&desc[idx]);
1054 idx++;
1055
1056 /* Setup XCBC MAC K3 */
1057 hw_desc_init(&desc[idx]);
1058 set_din_type(&desc[idx], DMA_DLLI,
1059 (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1060 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1061 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);
1062 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1063 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1064 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1065 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1066 set_aes_not_hash_mode(&desc[idx]);
1067 idx++;
1068
1069 *seq_size = idx;
1070}
1071
1072static void cc_proc_header_desc(struct aead_request *req,
1073 struct cc_hw_desc desc[],
1074 unsigned int *seq_size)
1075{
1076 unsigned int idx = *seq_size;
1077 /* Hash associated data */
1078 if (req->assoclen > 0)
1079 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1080
1081 /* Hash IV */
1082 *seq_size = idx;
1083}
1084
1085static void cc_proc_scheme_desc(struct aead_request *req,
1086 struct cc_hw_desc desc[],
1087 unsigned int *seq_size)
1088{
1089 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1090 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1091 struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle;
1092 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
1093 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
1094 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
1095 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
1096 unsigned int idx = *seq_size;
1097
1098 hw_desc_init(&desc[idx]);
1099 set_cipher_mode(&desc[idx], hash_mode);
1100 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1101 HASH_LEN_SIZE);
1102 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1103 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
1104 set_cipher_do(&desc[idx], DO_PAD);
1105 idx++;
1106
1107 /* Get final ICV result */
1108 hw_desc_init(&desc[idx]);
1109 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1110 digest_size);
1111 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1112 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1113 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1114 set_cipher_mode(&desc[idx], hash_mode);
1115 idx++;
1116
1117 /* Loading hash opad xor key state */
1118 hw_desc_init(&desc[idx]);
1119 set_cipher_mode(&desc[idx], hash_mode);
1120 set_din_type(&desc[idx], DMA_DLLI,
1121 (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size),
1122 digest_size, NS_BIT);
1123 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1124 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1125 idx++;
1126
1127 /* Load init. digest len (64 bytes) */
1128 hw_desc_init(&desc[idx]);
1129 set_cipher_mode(&desc[idx], hash_mode);
1130 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1131 HASH_LEN_SIZE);
1132 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1133 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1134 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1135 idx++;
1136
1137 /* Perform HASH update */
1138 hw_desc_init(&desc[idx]);
1139 set_din_sram(&desc[idx], aead_handle->sram_workspace_addr,
1140 digest_size);
1141 set_flow_mode(&desc[idx], DIN_HASH);
1142 idx++;
1143
1144 *seq_size = idx;
1145}
1146
1147static void cc_mlli_to_sram(struct aead_request *req,
1148 struct cc_hw_desc desc[], unsigned int *seq_size)
1149{
1150 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1151 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1152 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1153 struct device *dev = drvdata_to_dev(ctx->drvdata);
1154
1155 if (req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
1156 req_ctx->data_buff_type == CC_DMA_BUF_MLLI ||
1157 !req_ctx->is_single_pass) {
1158 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n",
1159 (unsigned int)ctx->drvdata->mlli_sram_addr,
1160 req_ctx->mlli_params.mlli_len);
1161 /* Copy MLLI table host-to-sram */
1162 hw_desc_init(&desc[*seq_size]);
1163 set_din_type(&desc[*seq_size], DMA_DLLI,
1164 req_ctx->mlli_params.mlli_dma_addr,
1165 req_ctx->mlli_params.mlli_len, NS_BIT);
1166 set_dout_sram(&desc[*seq_size],
1167 ctx->drvdata->mlli_sram_addr,
1168 req_ctx->mlli_params.mlli_len);
1169 set_flow_mode(&desc[*seq_size], BYPASS);
1170 (*seq_size)++;
1171 }
1172}
1173
1174static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct,
1175 enum cc_flow_mode setup_flow_mode,
1176 bool is_single_pass)
1177{
1178 enum cc_flow_mode data_flow_mode;
1179
1180 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1181 if (setup_flow_mode == S_DIN_to_AES)
1182 data_flow_mode = is_single_pass ?
1183 AES_to_HASH_and_DOUT : DIN_AES_DOUT;
1184 else
1185 data_flow_mode = is_single_pass ?
1186 DES_to_HASH_and_DOUT : DIN_DES_DOUT;
1187 } else { /* Decrypt */
1188 if (setup_flow_mode == S_DIN_to_AES)
1189 data_flow_mode = is_single_pass ?
1190 AES_and_HASH : DIN_AES_DOUT;
1191 else
1192 data_flow_mode = is_single_pass ?
1193 DES_and_HASH : DIN_DES_DOUT;
1194 }
1195
1196 return data_flow_mode;
1197}
1198
1199static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1200 unsigned int *seq_size)
1201{
1202 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1203 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1204 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1205 int direct = req_ctx->gen_ctx.op_type;
1206 unsigned int data_flow_mode =
1207 cc_get_data_flow(direct, ctx->flow_mode,
1208 req_ctx->is_single_pass);
1209
1210 if (req_ctx->is_single_pass) {
1211 /**
1212 * Single-pass flow
1213 */
1214 cc_set_hmac_desc(req, desc, seq_size);
1215 cc_set_cipher_desc(req, desc, seq_size);
1216 cc_proc_header_desc(req, desc, seq_size);
1217 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1218 cc_proc_scheme_desc(req, desc, seq_size);
1219 cc_proc_digest_desc(req, desc, seq_size);
1220 return;
1221 }
1222
1223 /**
1224 * Double-pass flow
1225 * Fallback for unsupported single-pass modes,
1226 * i.e. using assoc. data of non-word-multiple
1227 */
1228 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1229 /* encrypt first.. */
1230 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1231 /* authenc after..*/
1232 cc_set_hmac_desc(req, desc, seq_size);
1233 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1234 cc_proc_scheme_desc(req, desc, seq_size);
1235 cc_proc_digest_desc(req, desc, seq_size);
1236
1237 } else { /*DECRYPT*/
1238 /* authenc first..*/
1239 cc_set_hmac_desc(req, desc, seq_size);
1240 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1241 cc_proc_scheme_desc(req, desc, seq_size);
1242 /* decrypt after.. */
1243 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1244 /* read the digest result with setting the completion bit
1245 * must be after the cipher operation
1246 */
1247 cc_proc_digest_desc(req, desc, seq_size);
1248 }
1249}
1250
1251static void
1252cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1253 unsigned int *seq_size)
1254{
1255 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1256 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1257 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1258 int direct = req_ctx->gen_ctx.op_type;
1259 unsigned int data_flow_mode =
1260 cc_get_data_flow(direct, ctx->flow_mode,
1261 req_ctx->is_single_pass);
1262
1263 if (req_ctx->is_single_pass) {
1264 /**
1265 * Single-pass flow
1266 */
1267 cc_set_xcbc_desc(req, desc, seq_size);
1268 cc_set_cipher_desc(req, desc, seq_size);
1269 cc_proc_header_desc(req, desc, seq_size);
1270 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1271 cc_proc_digest_desc(req, desc, seq_size);
1272 return;
1273 }
1274
1275 /**
1276 * Double-pass flow
1277 * Fallback for unsupported single-pass modes,
1278 * i.e. using assoc. data of non-word-multiple
1279 */
1280 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1281 /* encrypt first.. */
1282 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1283 /* authenc after.. */
1284 cc_set_xcbc_desc(req, desc, seq_size);
1285 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1286 cc_proc_digest_desc(req, desc, seq_size);
1287 } else { /*DECRYPT*/
1288 /* authenc first.. */
1289 cc_set_xcbc_desc(req, desc, seq_size);
1290 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1291 /* decrypt after..*/
1292 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1293 /* read the digest result with setting the completion bit
1294 * must be after the cipher operation
1295 */
1296 cc_proc_digest_desc(req, desc, seq_size);
1297 }
1298}
1299
1300static int validate_data_size(struct cc_aead_ctx *ctx,
1301 enum drv_crypto_direction direct,
1302 struct aead_request *req)
1303{
1304 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1305 struct device *dev = drvdata_to_dev(ctx->drvdata);
1306 unsigned int assoclen = req->assoclen;
1307 unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ?
1308 (req->cryptlen - ctx->authsize) : req->cryptlen;
1309
1310 if (direct == DRV_CRYPTO_DIRECTION_DECRYPT &&
1311 req->cryptlen < ctx->authsize)
1312 goto data_size_err;
1313
1314 areq_ctx->is_single_pass = true; /*defaulted to fast flow*/
1315
1316 switch (ctx->flow_mode) {
1317 case S_DIN_to_AES:
1318 if (ctx->cipher_mode == DRV_CIPHER_CBC &&
1319 !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE))
1320 goto data_size_err;
1321 if (ctx->cipher_mode == DRV_CIPHER_CCM)
1322 break;
1323 if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1324 if (areq_ctx->plaintext_authenticate_only)
1325 areq_ctx->is_single_pass = false;
1326 break;
1327 }
1328
1329 if (!IS_ALIGNED(assoclen, sizeof(u32)))
1330 areq_ctx->is_single_pass = false;
1331
1332 if (ctx->cipher_mode == DRV_CIPHER_CTR &&
1333 !IS_ALIGNED(cipherlen, sizeof(u32)))
1334 areq_ctx->is_single_pass = false;
1335
1336 break;
1337 case S_DIN_to_DES:
1338 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE))
1339 goto data_size_err;
1340 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE))
1341 areq_ctx->is_single_pass = false;
1342 break;
1343 default:
1344 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode);
1345 goto data_size_err;
1346 }
1347
1348 return 0;
1349
1350data_size_err:
1351 return -EINVAL;
1352}
1353
1354static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size)
1355{
1356 unsigned int len = 0;
1357
1358 if (header_size == 0)
1359 return 0;
1360
1361 if (header_size < ((1UL << 16) - (1UL << 8))) {
1362 len = 2;
1363
1364 pa0_buff[0] = (header_size >> 8) & 0xFF;
1365 pa0_buff[1] = header_size & 0xFF;
1366 } else {
1367 len = 6;
1368
1369 pa0_buff[0] = 0xFF;
1370 pa0_buff[1] = 0xFE;
1371 pa0_buff[2] = (header_size >> 24) & 0xFF;
1372 pa0_buff[3] = (header_size >> 16) & 0xFF;
1373 pa0_buff[4] = (header_size >> 8) & 0xFF;
1374 pa0_buff[5] = header_size & 0xFF;
1375 }
1376
1377 return len;
1378}
1379
1380static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize)
1381{
1382 __be32 data;
1383
1384 memset(block, 0, csize);
1385 block += csize;
1386
1387 if (csize >= 4)
1388 csize = 4;
1389 else if (msglen > (1 << (8 * csize)))
1390 return -EOVERFLOW;
1391
1392 data = cpu_to_be32(msglen);
1393 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
1394
1395 return 0;
1396}
1397
1398static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[],
1399 unsigned int *seq_size)
1400{
1401 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1402 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1403 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1404 unsigned int idx = *seq_size;
1405 unsigned int cipher_flow_mode;
1406 dma_addr_t mac_result;
1407
1408 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1409 cipher_flow_mode = AES_to_HASH_and_DOUT;
1410 mac_result = req_ctx->mac_buf_dma_addr;
1411 } else { /* Encrypt */
1412 cipher_flow_mode = AES_and_HASH;
1413 mac_result = req_ctx->icv_dma_addr;
1414 }
1415
1416 /* load key */
1417 hw_desc_init(&desc[idx]);
1418 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1419 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1420 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
1421 ctx->enc_keylen), NS_BIT);
1422 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1423 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1424 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1425 set_flow_mode(&desc[idx], S_DIN_to_AES);
1426 idx++;
1427
1428 /* load ctr state */
1429 hw_desc_init(&desc[idx]);
1430 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1431 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1432 set_din_type(&desc[idx], DMA_DLLI,
1433 req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT);
1434 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1435 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1436 set_flow_mode(&desc[idx], S_DIN_to_AES);
1437 idx++;
1438
1439 /* load MAC key */
1440 hw_desc_init(&desc[idx]);
1441 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1442 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1443 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
1444 ctx->enc_keylen), NS_BIT);
1445 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1446 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1447 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1448 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1449 set_aes_not_hash_mode(&desc[idx]);
1450 idx++;
1451
1452 /* load MAC state */
1453 hw_desc_init(&desc[idx]);
1454 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1455 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1456 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1457 AES_BLOCK_SIZE, NS_BIT);
1458 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1459 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1460 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1461 set_aes_not_hash_mode(&desc[idx]);
1462 idx++;
1463
1464 /* process assoc data */
1465 if (req->assoclen > 0) {
1466 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1467 } else {
1468 hw_desc_init(&desc[idx]);
1469 set_din_type(&desc[idx], DMA_DLLI,
1470 sg_dma_address(&req_ctx->ccm_adata_sg),
1471 AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT);
1472 set_flow_mode(&desc[idx], DIN_HASH);
1473 idx++;
1474 }
1475
1476 /* process the cipher */
1477 if (req_ctx->cryptlen)
1478 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx);
1479
1480 /* Read temporal MAC */
1481 hw_desc_init(&desc[idx]);
1482 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1483 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize,
1484 NS_BIT, 0);
1485 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1486 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1487 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1488 set_aes_not_hash_mode(&desc[idx]);
1489 idx++;
1490
1491 /* load AES-CTR state (for last MAC calculation)*/
1492 hw_desc_init(&desc[idx]);
1493 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1494 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1495 set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr,
1496 AES_BLOCK_SIZE, NS_BIT);
1497 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1498 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1499 set_flow_mode(&desc[idx], S_DIN_to_AES);
1500 idx++;
1501
1502 hw_desc_init(&desc[idx]);
1503 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1504 set_dout_no_dma(&desc[idx], 0, 0, 1);
1505 idx++;
1506
1507 /* encrypt the "T" value and store MAC in mac_state */
1508 hw_desc_init(&desc[idx]);
1509 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1510 ctx->authsize, NS_BIT);
1511 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1512 set_queue_last_ind(&desc[idx]);
1513 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1514 idx++;
1515
1516 *seq_size = idx;
1517 return 0;
1518}
1519
1520static int config_ccm_adata(struct aead_request *req)
1521{
1522 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1523 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1524 struct device *dev = drvdata_to_dev(ctx->drvdata);
1525 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1526 //unsigned int size_of_a = 0, rem_a_size = 0;
1527 unsigned int lp = req->iv[0];
1528 /* Note: The code assume that req->iv[0] already contains the value
1529 * of L' of RFC3610
1530 */
1531 unsigned int l = lp + 1; /* This is L' of RFC 3610. */
1532 unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */
1533 u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET;
1534 u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET;
1535 u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET;
1536 unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1537 DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1538 req->cryptlen :
1539 (req->cryptlen - ctx->authsize);
1540 int rc;
1541
1542 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1543 memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3);
1544
1545 /* taken from crypto/ccm.c */
1546 /* 2 <= L <= 8, so 1 <= L' <= 7. */
1547 if (l < 2 || l > 8) {
1548 dev_err(dev, "illegal iv value %X\n", req->iv[0]);
1549 return -EINVAL;
1550 }
1551 memcpy(b0, req->iv, AES_BLOCK_SIZE);
1552
1553 /* format control info per RFC 3610 and
1554 * NIST Special Publication 800-38C
1555 */
1556 *b0 |= (8 * ((m - 2) / 2));
1557 if (req->assoclen > 0)
1558 *b0 |= 64; /* Enable bit 6 if Adata exists. */
1559
1560 rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */
1561 if (rc) {
1562 dev_err(dev, "message len overflow detected");
1563 return rc;
1564 }
1565 /* END of "taken from crypto/ccm.c" */
1566
1567 /* l(a) - size of associated data. */
1568 req_ctx->ccm_hdr_size = format_ccm_a0(a0, req->assoclen);
1569
1570 memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1);
1571 req->iv[15] = 1;
1572
1573 memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE);
1574 ctr_count_0[15] = 0;
1575
1576 return 0;
1577}
1578
1579static void cc_proc_rfc4309_ccm(struct aead_request *req)
1580{
1581 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1582 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1583 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1584
1585 /* L' */
1586 memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE);
1587 /* For RFC 4309, always use 4 bytes for message length
1588 * (at most 2^32-1 bytes).
1589 */
1590 areq_ctx->ctr_iv[0] = 3;
1591
1592 /* In RFC 4309 there is an 11-bytes nonce+IV part,
1593 * that we build here.
1594 */
1595 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce,
1596 CCM_BLOCK_NONCE_SIZE);
1597 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv,
1598 CCM_BLOCK_IV_SIZE);
1599 req->iv = areq_ctx->ctr_iv;
1600 req->assoclen -= CCM_BLOCK_IV_SIZE;
1601}
1602
1603static void cc_set_ghash_desc(struct aead_request *req,
1604 struct cc_hw_desc desc[], unsigned int *seq_size)
1605{
1606 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1607 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1608 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1609 unsigned int idx = *seq_size;
1610
1611 /* load key to AES*/
1612 hw_desc_init(&desc[idx]);
1613 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
1614 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1615 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1616 ctx->enc_keylen, NS_BIT);
1617 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1618 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1619 set_flow_mode(&desc[idx], S_DIN_to_AES);
1620 idx++;
1621
1622 /* process one zero block to generate hkey */
1623 hw_desc_init(&desc[idx]);
1624 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1625 set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE,
1626 NS_BIT, 0);
1627 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1628 idx++;
1629
1630 /* Memory Barrier */
1631 hw_desc_init(&desc[idx]);
1632 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1633 set_dout_no_dma(&desc[idx], 0, 0, 1);
1634 idx++;
1635
1636 /* Load GHASH subkey */
1637 hw_desc_init(&desc[idx]);
1638 set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr,
1639 AES_BLOCK_SIZE, NS_BIT);
1640 set_dout_no_dma(&desc[idx], 0, 0, 1);
1641 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1642 set_aes_not_hash_mode(&desc[idx]);
1643 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1644 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1645 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1646 idx++;
1647
1648 /* Configure Hash Engine to work with GHASH.
1649 * Since it was not possible to extend HASH submodes to add GHASH,
1650 * The following command is necessary in order to
1651 * select GHASH (according to HW designers)
1652 */
1653 hw_desc_init(&desc[idx]);
1654 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1655 set_dout_no_dma(&desc[idx], 0, 0, 1);
1656 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1657 set_aes_not_hash_mode(&desc[idx]);
1658 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1659 set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK
1660 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1661 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1662 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1663 idx++;
1664
1665 /* Load GHASH initial STATE (which is 0). (for any hash there is an
1666 * initial state)
1667 */
1668 hw_desc_init(&desc[idx]);
1669 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1670 set_dout_no_dma(&desc[idx], 0, 0, 1);
1671 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1672 set_aes_not_hash_mode(&desc[idx]);
1673 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1674 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1675 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1676 idx++;
1677
1678 *seq_size = idx;
1679}
1680
1681static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[],
1682 unsigned int *seq_size)
1683{
1684 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1685 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1686 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1687 unsigned int idx = *seq_size;
1688
1689 /* load key to AES*/
1690 hw_desc_init(&desc[idx]);
1691 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1692 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1693 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1694 ctx->enc_keylen, NS_BIT);
1695 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1696 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1697 set_flow_mode(&desc[idx], S_DIN_to_AES);
1698 idx++;
1699
1700 if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) {
1701 /* load AES/CTR initial CTR value inc by 2*/
1702 hw_desc_init(&desc[idx]);
1703 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1704 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1705 set_din_type(&desc[idx], DMA_DLLI,
1706 req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE,
1707 NS_BIT);
1708 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1709 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1710 set_flow_mode(&desc[idx], S_DIN_to_AES);
1711 idx++;
1712 }
1713
1714 *seq_size = idx;
1715}
1716
1717static void cc_proc_gcm_result(struct aead_request *req,
1718 struct cc_hw_desc desc[],
1719 unsigned int *seq_size)
1720{
1721 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1722 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1723 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1724 dma_addr_t mac_result;
1725 unsigned int idx = *seq_size;
1726
1727 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1728 mac_result = req_ctx->mac_buf_dma_addr;
1729 } else { /* Encrypt */
1730 mac_result = req_ctx->icv_dma_addr;
1731 }
1732
1733 /* process(ghash) gcm_block_len */
1734 hw_desc_init(&desc[idx]);
1735 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr,
1736 AES_BLOCK_SIZE, NS_BIT);
1737 set_flow_mode(&desc[idx], DIN_HASH);
1738 idx++;
1739
1740 /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */
1741 hw_desc_init(&desc[idx]);
1742 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1743 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1744 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE,
1745 NS_BIT, 0);
1746 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1747 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1748 set_aes_not_hash_mode(&desc[idx]);
1749
1750 idx++;
1751
1752 /* load AES/CTR initial CTR value inc by 1*/
1753 hw_desc_init(&desc[idx]);
1754 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1755 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1756 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr,
1757 AES_BLOCK_SIZE, NS_BIT);
1758 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1759 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1760 set_flow_mode(&desc[idx], S_DIN_to_AES);
1761 idx++;
1762
1763 /* Memory Barrier */
1764 hw_desc_init(&desc[idx]);
1765 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1766 set_dout_no_dma(&desc[idx], 0, 0, 1);
1767 idx++;
1768
1769 /* process GCTR on stored GHASH and store MAC in mac_state*/
1770 hw_desc_init(&desc[idx]);
1771 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1772 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1773 AES_BLOCK_SIZE, NS_BIT);
1774 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1775 set_queue_last_ind(&desc[idx]);
1776 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1777 idx++;
1778
1779 *seq_size = idx;
1780}
1781
1782static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[],
1783 unsigned int *seq_size)
1784{
1785 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1786 unsigned int cipher_flow_mode;
1787
1788 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1789 cipher_flow_mode = AES_and_HASH;
1790 } else { /* Encrypt */
1791 cipher_flow_mode = AES_to_HASH_and_DOUT;
1792 }
1793
1794 //in RFC4543 no data to encrypt. just copy data from src to dest.
1795 if (req_ctx->plaintext_authenticate_only) {
1796 cc_proc_cipher_desc(req, BYPASS, desc, seq_size);
1797 cc_set_ghash_desc(req, desc, seq_size);
1798 /* process(ghash) assoc data */
1799 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1800 cc_set_gctr_desc(req, desc, seq_size);
1801 cc_proc_gcm_result(req, desc, seq_size);
1802 return 0;
1803 }
1804
1805 // for gcm and rfc4106.
1806 cc_set_ghash_desc(req, desc, seq_size);
1807 /* process(ghash) assoc data */
1808 if (req->assoclen > 0)
1809 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1810 cc_set_gctr_desc(req, desc, seq_size);
1811 /* process(gctr+ghash) */
1812 if (req_ctx->cryptlen)
1813 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size);
1814 cc_proc_gcm_result(req, desc, seq_size);
1815
1816 return 0;
1817}
1818
1819static int config_gcm_context(struct aead_request *req)
1820{
1821 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1822 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1823 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1824 struct device *dev = drvdata_to_dev(ctx->drvdata);
1825
1826 unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1827 DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1828 req->cryptlen :
1829 (req->cryptlen - ctx->authsize);
1830 __be32 counter = cpu_to_be32(2);
1831
1832 dev_dbg(dev, "%s() cryptlen = %d, req->assoclen = %d ctx->authsize = %d\n",
1833 __func__, cryptlen, req->assoclen, ctx->authsize);
1834
1835 memset(req_ctx->hkey, 0, AES_BLOCK_SIZE);
1836
1837 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1838
1839 memcpy(req->iv + 12, &counter, 4);
1840 memcpy(req_ctx->gcm_iv_inc2, req->iv, 16);
1841
1842 counter = cpu_to_be32(1);
1843 memcpy(req->iv + 12, &counter, 4);
1844 memcpy(req_ctx->gcm_iv_inc1, req->iv, 16);
1845
1846 if (!req_ctx->plaintext_authenticate_only) {
1847 __be64 temp64;
1848
1849 temp64 = cpu_to_be64(req->assoclen * 8);
1850 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1851 temp64 = cpu_to_be64(cryptlen * 8);
1852 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1853 } else {
1854 /* rfc4543=> all data(AAD,IV,Plain) are considered additional
1855 * data that is nothing is encrypted.
1856 */
1857 __be64 temp64;
1858
1859 temp64 = cpu_to_be64((req->assoclen + GCM_BLOCK_RFC4_IV_SIZE +
1860 cryptlen) * 8);
1861 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1862 temp64 = 0;
1863 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1864 }
1865
1866 return 0;
1867}
1868
1869static void cc_proc_rfc4_gcm(struct aead_request *req)
1870{
1871 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1872 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1873 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1874
1875 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET,
1876 ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE);
1877 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv,
1878 GCM_BLOCK_RFC4_IV_SIZE);
1879 req->iv = areq_ctx->ctr_iv;
1880 req->assoclen -= GCM_BLOCK_RFC4_IV_SIZE;
1881}
1882
1883static int cc_proc_aead(struct aead_request *req,
1884 enum drv_crypto_direction direct)
1885{
1886 int rc = 0;
1887 int seq_len = 0;
1888 struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ];
1889 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1890 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1891 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1892 struct device *dev = drvdata_to_dev(ctx->drvdata);
1893 struct cc_crypto_req cc_req = {};
1894
1895 dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n",
1896 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"),
1897 ctx, req, req->iv, sg_virt(req->src), req->src->offset,
1898 sg_virt(req->dst), req->dst->offset, req->cryptlen);
1899
1900 /* STAT_PHASE_0: Init and sanity checks */
1901
1902 /* Check data length according to mode */
1903 if (validate_data_size(ctx, direct, req)) {
1904 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n",
1905 req->cryptlen, req->assoclen);
1906 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN);
1907 return -EINVAL;
1908 }
1909
1910 /* Setup request structure */
1911 cc_req.user_cb = (void *)cc_aead_complete;
1912 cc_req.user_arg = (void *)req;
1913
1914 /* Setup request context */
1915 areq_ctx->gen_ctx.op_type = direct;
1916 areq_ctx->req_authsize = ctx->authsize;
1917 areq_ctx->cipher_mode = ctx->cipher_mode;
1918
1919 /* STAT_PHASE_1: Map buffers */
1920
1921 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
1922 /* Build CTR IV - Copy nonce from last 4 bytes in
1923 * CTR key to first 4 bytes in CTR IV
1924 */
1925 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce,
1926 CTR_RFC3686_NONCE_SIZE);
1927 if (!areq_ctx->backup_giv) /*User none-generated IV*/
1928 memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE,
1929 req->iv, CTR_RFC3686_IV_SIZE);
1930 /* Initialize counter portion of counter block */
1931 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE +
1932 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1933
1934 /* Replace with counter iv */
1935 req->iv = areq_ctx->ctr_iv;
1936 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE;
1937 } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) ||
1938 (ctx->cipher_mode == DRV_CIPHER_GCTR)) {
1939 areq_ctx->hw_iv_size = AES_BLOCK_SIZE;
1940 if (areq_ctx->ctr_iv != req->iv) {
1941 memcpy(areq_ctx->ctr_iv, req->iv,
1942 crypto_aead_ivsize(tfm));
1943 req->iv = areq_ctx->ctr_iv;
1944 }
1945 } else {
1946 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm);
1947 }
1948
1949 if (ctx->cipher_mode == DRV_CIPHER_CCM) {
1950 rc = config_ccm_adata(req);
1951 if (rc) {
1952 dev_dbg(dev, "config_ccm_adata() returned with a failure %d!",
1953 rc);
1954 goto exit;
1955 }
1956 } else {
1957 areq_ctx->ccm_hdr_size = ccm_header_size_null;
1958 }
1959
1960 if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1961 rc = config_gcm_context(req);
1962 if (rc) {
1963 dev_dbg(dev, "config_gcm_context() returned with a failure %d!",
1964 rc);
1965 goto exit;
1966 }
1967 }
1968
1969 rc = cc_map_aead_request(ctx->drvdata, req);
1970 if (rc) {
1971 dev_err(dev, "map_request() failed\n");
1972 goto exit;
1973 }
1974
1975 /* do we need to generate IV? */
1976 if (areq_ctx->backup_giv) {
1977 /* set the DMA mapped IV address*/
1978 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
1979 cc_req.ivgen_dma_addr[0] =
1980 areq_ctx->gen_ctx.iv_dma_addr +
1981 CTR_RFC3686_NONCE_SIZE;
1982 cc_req.ivgen_dma_addr_len = 1;
1983 } else if (ctx->cipher_mode == DRV_CIPHER_CCM) {
1984 /* In ccm, the IV needs to exist both inside B0 and
1985 * inside the counter.It is also copied to iv_dma_addr
1986 * for other reasons (like returning it to the user).
1987 * So, using 3 (identical) IV outputs.
1988 */
1989 cc_req.ivgen_dma_addr[0] =
1990 areq_ctx->gen_ctx.iv_dma_addr +
1991 CCM_BLOCK_IV_OFFSET;
1992 cc_req.ivgen_dma_addr[1] =
1993 sg_dma_address(&areq_ctx->ccm_adata_sg) +
1994 CCM_B0_OFFSET + CCM_BLOCK_IV_OFFSET;
1995 cc_req.ivgen_dma_addr[2] =
1996 sg_dma_address(&areq_ctx->ccm_adata_sg) +
1997 CCM_CTR_COUNT_0_OFFSET + CCM_BLOCK_IV_OFFSET;
1998 cc_req.ivgen_dma_addr_len = 3;
1999 } else {
2000 cc_req.ivgen_dma_addr[0] =
2001 areq_ctx->gen_ctx.iv_dma_addr;
2002 cc_req.ivgen_dma_addr_len = 1;
2003 }
2004
2005 /* set the IV size (8/16 B long)*/
2006 cc_req.ivgen_size = crypto_aead_ivsize(tfm);
2007 }
2008
2009 /* STAT_PHASE_2: Create sequence */
2010
2011 /* Load MLLI tables to SRAM if necessary */
2012 cc_mlli_to_sram(req, desc, &seq_len);
2013
2014 /*TODO: move seq len by reference */
2015 switch (ctx->auth_mode) {
2016 case DRV_HASH_SHA1:
2017 case DRV_HASH_SHA256:
2018 cc_hmac_authenc(req, desc, &seq_len);
2019 break;
2020 case DRV_HASH_XCBC_MAC:
2021 cc_xcbc_authenc(req, desc, &seq_len);
2022 break;
2023 case DRV_HASH_NULL:
2024 if (ctx->cipher_mode == DRV_CIPHER_CCM)
2025 cc_ccm(req, desc, &seq_len);
2026 if (ctx->cipher_mode == DRV_CIPHER_GCTR)
2027 cc_gcm(req, desc, &seq_len);
2028 break;
2029 default:
2030 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
2031 cc_unmap_aead_request(dev, req);
2032 rc = -ENOTSUPP;
2033 goto exit;
2034 }
2035
2036 /* STAT_PHASE_3: Lock HW and push sequence */
2037
2038 rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base);
2039
2040 if (rc != -EINPROGRESS && rc != -EBUSY) {
2041 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
2042 cc_unmap_aead_request(dev, req);
2043 }
2044
2045exit:
2046 return rc;
2047}
2048
2049static int cc_aead_encrypt(struct aead_request *req)
2050{
2051 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2052 int rc;
2053
2054 /* No generated IV required */
2055 areq_ctx->backup_iv = req->iv;
2056 areq_ctx->backup_giv = NULL;
2057 areq_ctx->is_gcm4543 = false;
2058
2059 areq_ctx->plaintext_authenticate_only = false;
2060
2061 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2062 if (rc != -EINPROGRESS && rc != -EBUSY)
2063 req->iv = areq_ctx->backup_iv;
2064
2065 return rc;
2066}
2067
2068static int cc_rfc4309_ccm_encrypt(struct aead_request *req)
2069{
2070 /* Very similar to cc_aead_encrypt() above. */
2071
2072 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2073 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2074 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2075 struct device *dev = drvdata_to_dev(ctx->drvdata);
2076 int rc = -EINVAL;
2077
2078 if (!valid_assoclen(req)) {
2079 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2080 goto out;
2081 }
2082
2083 /* No generated IV required */
2084 areq_ctx->backup_iv = req->iv;
2085 areq_ctx->backup_giv = NULL;
2086 areq_ctx->is_gcm4543 = true;
2087
2088 cc_proc_rfc4309_ccm(req);
2089
2090 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2091 if (rc != -EINPROGRESS && rc != -EBUSY)
2092 req->iv = areq_ctx->backup_iv;
2093out:
2094 return rc;
2095}
2096
2097static int cc_aead_decrypt(struct aead_request *req)
2098{
2099 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2100 int rc;
2101
2102 /* No generated IV required */
2103 areq_ctx->backup_iv = req->iv;
2104 areq_ctx->backup_giv = NULL;
2105 areq_ctx->is_gcm4543 = false;
2106
2107 areq_ctx->plaintext_authenticate_only = false;
2108
2109 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2110 if (rc != -EINPROGRESS && rc != -EBUSY)
2111 req->iv = areq_ctx->backup_iv;
2112
2113 return rc;
2114}
2115
2116static int cc_rfc4309_ccm_decrypt(struct aead_request *req)
2117{
2118 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2119 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2120 struct device *dev = drvdata_to_dev(ctx->drvdata);
2121 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2122 int rc = -EINVAL;
2123
2124 if (!valid_assoclen(req)) {
2125 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2126 goto out;
2127 }
2128
2129 /* No generated IV required */
2130 areq_ctx->backup_iv = req->iv;
2131 areq_ctx->backup_giv = NULL;
2132
2133 areq_ctx->is_gcm4543 = true;
2134 cc_proc_rfc4309_ccm(req);
2135
2136 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2137 if (rc != -EINPROGRESS && rc != -EBUSY)
2138 req->iv = areq_ctx->backup_iv;
2139
2140out:
2141 return rc;
2142}
2143
2144static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2145 unsigned int keylen)
2146{
2147 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2148 struct device *dev = drvdata_to_dev(ctx->drvdata);
2149
2150 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key);
2151
2152 if (keylen < 4)
2153 return -EINVAL;
2154
2155 keylen -= 4;
2156 memcpy(ctx->ctr_nonce, key + keylen, 4);
2157
2158 return cc_aead_setkey(tfm, key, keylen);
2159}
2160
2161static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2162 unsigned int keylen)
2163{
2164 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2165 struct device *dev = drvdata_to_dev(ctx->drvdata);
2166
2167 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key);
2168
2169 if (keylen < 4)
2170 return -EINVAL;
2171
2172 keylen -= 4;
2173 memcpy(ctx->ctr_nonce, key + keylen, 4);
2174
2175 return cc_aead_setkey(tfm, key, keylen);
2176}
2177
2178static int cc_gcm_setauthsize(struct crypto_aead *authenc,
2179 unsigned int authsize)
2180{
2181 switch (authsize) {
2182 case 4:
2183 case 8:
2184 case 12:
2185 case 13:
2186 case 14:
2187 case 15:
2188 case 16:
2189 break;
2190 default:
2191 return -EINVAL;
2192 }
2193
2194 return cc_aead_setauthsize(authenc, authsize);
2195}
2196
2197static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc,
2198 unsigned int authsize)
2199{
2200 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2201 struct device *dev = drvdata_to_dev(ctx->drvdata);
2202
2203 dev_dbg(dev, "authsize %d\n", authsize);
2204
2205 switch (authsize) {
2206 case 8:
2207 case 12:
2208 case 16:
2209 break;
2210 default:
2211 return -EINVAL;
2212 }
2213
2214 return cc_aead_setauthsize(authenc, authsize);
2215}
2216
2217static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc,
2218 unsigned int authsize)
2219{
2220 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2221 struct device *dev = drvdata_to_dev(ctx->drvdata);
2222
2223 dev_dbg(dev, "authsize %d\n", authsize);
2224
2225 if (authsize != 16)
2226 return -EINVAL;
2227
2228 return cc_aead_setauthsize(authenc, authsize);
2229}
2230
2231static int cc_rfc4106_gcm_encrypt(struct aead_request *req)
2232{
2233 /* Very similar to cc_aead_encrypt() above. */
2234
2235 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2236 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2237 struct device *dev = drvdata_to_dev(ctx->drvdata);
2238 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2239 int rc = -EINVAL;
2240
2241 if (!valid_assoclen(req)) {
2242 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2243 goto out;
2244 }
2245
2246 /* No generated IV required */
2247 areq_ctx->backup_iv = req->iv;
2248 areq_ctx->backup_giv = NULL;
2249
2250 areq_ctx->plaintext_authenticate_only = false;
2251
2252 cc_proc_rfc4_gcm(req);
2253 areq_ctx->is_gcm4543 = true;
2254
2255 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2256 if (rc != -EINPROGRESS && rc != -EBUSY)
2257 req->iv = areq_ctx->backup_iv;
2258out:
2259 return rc;
2260}
2261
2262static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
2263{
2264 /* Very similar to cc_aead_encrypt() above. */
2265
2266 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2267 int rc;
2268
2269 //plaintext is not encryped with rfc4543
2270 areq_ctx->plaintext_authenticate_only = true;
2271
2272 /* No generated IV required */
2273 areq_ctx->backup_iv = req->iv;
2274 areq_ctx->backup_giv = NULL;
2275
2276 cc_proc_rfc4_gcm(req);
2277 areq_ctx->is_gcm4543 = true;
2278
2279 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2280 if (rc != -EINPROGRESS && rc != -EBUSY)
2281 req->iv = areq_ctx->backup_iv;
2282
2283 return rc;
2284}
2285
2286static int cc_rfc4106_gcm_decrypt(struct aead_request *req)
2287{
2288 /* Very similar to cc_aead_decrypt() above. */
2289
2290 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2291 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2292 struct device *dev = drvdata_to_dev(ctx->drvdata);
2293 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2294 int rc = -EINVAL;
2295
2296 if (!valid_assoclen(req)) {
2297 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2298 goto out;
2299 }
2300
2301 /* No generated IV required */
2302 areq_ctx->backup_iv = req->iv;
2303 areq_ctx->backup_giv = NULL;
2304
2305 areq_ctx->plaintext_authenticate_only = false;
2306
2307 cc_proc_rfc4_gcm(req);
2308 areq_ctx->is_gcm4543 = true;
2309
2310 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2311 if (rc != -EINPROGRESS && rc != -EBUSY)
2312 req->iv = areq_ctx->backup_iv;
2313out:
2314 return rc;
2315}
2316
2317static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
2318{
2319 /* Very similar to cc_aead_decrypt() above. */
2320
2321 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2322 int rc;
2323
2324 //plaintext is not decryped with rfc4543
2325 areq_ctx->plaintext_authenticate_only = true;
2326
2327 /* No generated IV required */
2328 areq_ctx->backup_iv = req->iv;
2329 areq_ctx->backup_giv = NULL;
2330
2331 cc_proc_rfc4_gcm(req);
2332 areq_ctx->is_gcm4543 = true;
2333
2334 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2335 if (rc != -EINPROGRESS && rc != -EBUSY)
2336 req->iv = areq_ctx->backup_iv;
2337
2338 return rc;
2339}
2340
2341/* aead alg */
2342static struct cc_alg_template aead_algs[] = {
2343 {
2344 .name = "authenc(hmac(sha1),cbc(aes))",
2345 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree",
2346 .blocksize = AES_BLOCK_SIZE,
2347 .type = CRYPTO_ALG_TYPE_AEAD,
2348 .template_aead = {
2349 .setkey = cc_aead_setkey,
2350 .setauthsize = cc_aead_setauthsize,
2351 .encrypt = cc_aead_encrypt,
2352 .decrypt = cc_aead_decrypt,
2353 .init = cc_aead_init,
2354 .exit = cc_aead_exit,
2355 .ivsize = AES_BLOCK_SIZE,
2356 .maxauthsize = SHA1_DIGEST_SIZE,
2357 },
2358 .cipher_mode = DRV_CIPHER_CBC,
2359 .flow_mode = S_DIN_to_AES,
2360 .auth_mode = DRV_HASH_SHA1,
2361 },
2362 {
2363 .name = "authenc(hmac(sha1),cbc(des3_ede))",
2364 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree",
2365 .blocksize = DES3_EDE_BLOCK_SIZE,
2366 .type = CRYPTO_ALG_TYPE_AEAD,
2367 .template_aead = {
2368 .setkey = cc_aead_setkey,
2369 .setauthsize = cc_aead_setauthsize,
2370 .encrypt = cc_aead_encrypt,
2371 .decrypt = cc_aead_decrypt,
2372 .init = cc_aead_init,
2373 .exit = cc_aead_exit,
2374 .ivsize = DES3_EDE_BLOCK_SIZE,
2375 .maxauthsize = SHA1_DIGEST_SIZE,
2376 },
2377 .cipher_mode = DRV_CIPHER_CBC,
2378 .flow_mode = S_DIN_to_DES,
2379 .auth_mode = DRV_HASH_SHA1,
2380 },
2381 {
2382 .name = "authenc(hmac(sha256),cbc(aes))",
2383 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree",
2384 .blocksize = AES_BLOCK_SIZE,
2385 .type = CRYPTO_ALG_TYPE_AEAD,
2386 .template_aead = {
2387 .setkey = cc_aead_setkey,
2388 .setauthsize = cc_aead_setauthsize,
2389 .encrypt = cc_aead_encrypt,
2390 .decrypt = cc_aead_decrypt,
2391 .init = cc_aead_init,
2392 .exit = cc_aead_exit,
2393 .ivsize = AES_BLOCK_SIZE,
2394 .maxauthsize = SHA256_DIGEST_SIZE,
2395 },
2396 .cipher_mode = DRV_CIPHER_CBC,
2397 .flow_mode = S_DIN_to_AES,
2398 .auth_mode = DRV_HASH_SHA256,
2399 },
2400 {
2401 .name = "authenc(hmac(sha256),cbc(des3_ede))",
2402 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree",
2403 .blocksize = DES3_EDE_BLOCK_SIZE,
2404 .type = CRYPTO_ALG_TYPE_AEAD,
2405 .template_aead = {
2406 .setkey = cc_aead_setkey,
2407 .setauthsize = cc_aead_setauthsize,
2408 .encrypt = cc_aead_encrypt,
2409 .decrypt = cc_aead_decrypt,
2410 .init = cc_aead_init,
2411 .exit = cc_aead_exit,
2412 .ivsize = DES3_EDE_BLOCK_SIZE,
2413 .maxauthsize = SHA256_DIGEST_SIZE,
2414 },
2415 .cipher_mode = DRV_CIPHER_CBC,
2416 .flow_mode = S_DIN_to_DES,
2417 .auth_mode = DRV_HASH_SHA256,
2418 },
2419 {
2420 .name = "authenc(xcbc(aes),cbc(aes))",
2421 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree",
2422 .blocksize = AES_BLOCK_SIZE,
2423 .type = CRYPTO_ALG_TYPE_AEAD,
2424 .template_aead = {
2425 .setkey = cc_aead_setkey,
2426 .setauthsize = cc_aead_setauthsize,
2427 .encrypt = cc_aead_encrypt,
2428 .decrypt = cc_aead_decrypt,
2429 .init = cc_aead_init,
2430 .exit = cc_aead_exit,
2431 .ivsize = AES_BLOCK_SIZE,
2432 .maxauthsize = AES_BLOCK_SIZE,
2433 },
2434 .cipher_mode = DRV_CIPHER_CBC,
2435 .flow_mode = S_DIN_to_AES,
2436 .auth_mode = DRV_HASH_XCBC_MAC,
2437 },
2438 {
2439 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
2440 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree",
2441 .blocksize = 1,
2442 .type = CRYPTO_ALG_TYPE_AEAD,
2443 .template_aead = {
2444 .setkey = cc_aead_setkey,
2445 .setauthsize = cc_aead_setauthsize,
2446 .encrypt = cc_aead_encrypt,
2447 .decrypt = cc_aead_decrypt,
2448 .init = cc_aead_init,
2449 .exit = cc_aead_exit,
2450 .ivsize = CTR_RFC3686_IV_SIZE,
2451 .maxauthsize = SHA1_DIGEST_SIZE,
2452 },
2453 .cipher_mode = DRV_CIPHER_CTR,
2454 .flow_mode = S_DIN_to_AES,
2455 .auth_mode = DRV_HASH_SHA1,
2456 },
2457 {
2458 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
2459 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree",
2460 .blocksize = 1,
2461 .type = CRYPTO_ALG_TYPE_AEAD,
2462 .template_aead = {
2463 .setkey = cc_aead_setkey,
2464 .setauthsize = cc_aead_setauthsize,
2465 .encrypt = cc_aead_encrypt,
2466 .decrypt = cc_aead_decrypt,
2467 .init = cc_aead_init,
2468 .exit = cc_aead_exit,
2469 .ivsize = CTR_RFC3686_IV_SIZE,
2470 .maxauthsize = SHA256_DIGEST_SIZE,
2471 },
2472 .cipher_mode = DRV_CIPHER_CTR,
2473 .flow_mode = S_DIN_to_AES,
2474 .auth_mode = DRV_HASH_SHA256,
2475 },
2476 {
2477 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))",
2478 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree",
2479 .blocksize = 1,
2480 .type = CRYPTO_ALG_TYPE_AEAD,
2481 .template_aead = {
2482 .setkey = cc_aead_setkey,
2483 .setauthsize = cc_aead_setauthsize,
2484 .encrypt = cc_aead_encrypt,
2485 .decrypt = cc_aead_decrypt,
2486 .init = cc_aead_init,
2487 .exit = cc_aead_exit,
2488 .ivsize = CTR_RFC3686_IV_SIZE,
2489 .maxauthsize = AES_BLOCK_SIZE,
2490 },
2491 .cipher_mode = DRV_CIPHER_CTR,
2492 .flow_mode = S_DIN_to_AES,
2493 .auth_mode = DRV_HASH_XCBC_MAC,
2494 },
2495 {
2496 .name = "ccm(aes)",
2497 .driver_name = "ccm-aes-ccree",
2498 .blocksize = 1,
2499 .type = CRYPTO_ALG_TYPE_AEAD,
2500 .template_aead = {
2501 .setkey = cc_aead_setkey,
2502 .setauthsize = cc_ccm_setauthsize,
2503 .encrypt = cc_aead_encrypt,
2504 .decrypt = cc_aead_decrypt,
2505 .init = cc_aead_init,
2506 .exit = cc_aead_exit,
2507 .ivsize = AES_BLOCK_SIZE,
2508 .maxauthsize = AES_BLOCK_SIZE,
2509 },
2510 .cipher_mode = DRV_CIPHER_CCM,
2511 .flow_mode = S_DIN_to_AES,
2512 .auth_mode = DRV_HASH_NULL,
2513 },
2514 {
2515 .name = "rfc4309(ccm(aes))",
2516 .driver_name = "rfc4309-ccm-aes-ccree",
2517 .blocksize = 1,
2518 .type = CRYPTO_ALG_TYPE_AEAD,
2519 .template_aead = {
2520 .setkey = cc_rfc4309_ccm_setkey,
2521 .setauthsize = cc_rfc4309_ccm_setauthsize,
2522 .encrypt = cc_rfc4309_ccm_encrypt,
2523 .decrypt = cc_rfc4309_ccm_decrypt,
2524 .init = cc_aead_init,
2525 .exit = cc_aead_exit,
2526 .ivsize = CCM_BLOCK_IV_SIZE,
2527 .maxauthsize = AES_BLOCK_SIZE,
2528 },
2529 .cipher_mode = DRV_CIPHER_CCM,
2530 .flow_mode = S_DIN_to_AES,
2531 .auth_mode = DRV_HASH_NULL,
2532 },
2533 {
2534 .name = "gcm(aes)",
2535 .driver_name = "gcm-aes-ccree",
2536 .blocksize = 1,
2537 .type = CRYPTO_ALG_TYPE_AEAD,
2538 .template_aead = {
2539 .setkey = cc_aead_setkey,
2540 .setauthsize = cc_gcm_setauthsize,
2541 .encrypt = cc_aead_encrypt,
2542 .decrypt = cc_aead_decrypt,
2543 .init = cc_aead_init,
2544 .exit = cc_aead_exit,
2545 .ivsize = 12,
2546 .maxauthsize = AES_BLOCK_SIZE,
2547 },
2548 .cipher_mode = DRV_CIPHER_GCTR,
2549 .flow_mode = S_DIN_to_AES,
2550 .auth_mode = DRV_HASH_NULL,
2551 },
2552 {
2553 .name = "rfc4106(gcm(aes))",
2554 .driver_name = "rfc4106-gcm-aes-ccree",
2555 .blocksize = 1,
2556 .type = CRYPTO_ALG_TYPE_AEAD,
2557 .template_aead = {
2558 .setkey = cc_rfc4106_gcm_setkey,
2559 .setauthsize = cc_rfc4106_gcm_setauthsize,
2560 .encrypt = cc_rfc4106_gcm_encrypt,
2561 .decrypt = cc_rfc4106_gcm_decrypt,
2562 .init = cc_aead_init,
2563 .exit = cc_aead_exit,
2564 .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2565 .maxauthsize = AES_BLOCK_SIZE,
2566 },
2567 .cipher_mode = DRV_CIPHER_GCTR,
2568 .flow_mode = S_DIN_to_AES,
2569 .auth_mode = DRV_HASH_NULL,
2570 },
2571 {
2572 .name = "rfc4543(gcm(aes))",
2573 .driver_name = "rfc4543-gcm-aes-ccree",
2574 .blocksize = 1,
2575 .type = CRYPTO_ALG_TYPE_AEAD,
2576 .template_aead = {
2577 .setkey = cc_rfc4543_gcm_setkey,
2578 .setauthsize = cc_rfc4543_gcm_setauthsize,
2579 .encrypt = cc_rfc4543_gcm_encrypt,
2580 .decrypt = cc_rfc4543_gcm_decrypt,
2581 .init = cc_aead_init,
2582 .exit = cc_aead_exit,
2583 .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2584 .maxauthsize = AES_BLOCK_SIZE,
2585 },
2586 .cipher_mode = DRV_CIPHER_GCTR,
2587 .flow_mode = S_DIN_to_AES,
2588 .auth_mode = DRV_HASH_NULL,
2589 },
2590};
2591
2592static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl,
2593 struct device *dev)
2594{
2595 struct cc_crypto_alg *t_alg;
2596 struct aead_alg *alg;
2597
2598 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
2599 if (!t_alg)
2600 return ERR_PTR(-ENOMEM);
2601
2602 alg = &tmpl->template_aead;
2603
2604 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
2605 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
2606 tmpl->driver_name);
2607 alg->base.cra_module = THIS_MODULE;
2608 alg->base.cra_priority = CC_CRA_PRIO;
2609
2610 alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx);
2611 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
2612 tmpl->type;
2613 alg->init = cc_aead_init;
2614 alg->exit = cc_aead_exit;
2615
2616 t_alg->aead_alg = *alg;
2617
2618 t_alg->cipher_mode = tmpl->cipher_mode;
2619 t_alg->flow_mode = tmpl->flow_mode;
2620 t_alg->auth_mode = tmpl->auth_mode;
2621
2622 return t_alg;
2623}
2624
2625int cc_aead_free(struct cc_drvdata *drvdata)
2626{
2627 struct cc_crypto_alg *t_alg, *n;
2628 struct cc_aead_handle *aead_handle =
2629 (struct cc_aead_handle *)drvdata->aead_handle;
2630
2631 if (aead_handle) {
2632 /* Remove registered algs */
2633 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list,
2634 entry) {
2635 crypto_unregister_aead(&t_alg->aead_alg);
2636 list_del(&t_alg->entry);
2637 kfree(t_alg);
2638 }
2639 kfree(aead_handle);
2640 drvdata->aead_handle = NULL;
2641 }
2642
2643 return 0;
2644}
2645
2646int cc_aead_alloc(struct cc_drvdata *drvdata)
2647{
2648 struct cc_aead_handle *aead_handle;
2649 struct cc_crypto_alg *t_alg;
2650 int rc = -ENOMEM;
2651 int alg;
2652 struct device *dev = drvdata_to_dev(drvdata);
2653
2654 aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL);
2655 if (!aead_handle) {
2656 rc = -ENOMEM;
2657 goto fail0;
2658 }
2659
2660 INIT_LIST_HEAD(&aead_handle->aead_list);
2661 drvdata->aead_handle = aead_handle;
2662
2663 aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata,
2664 MAX_HMAC_DIGEST_SIZE);
2665
2666 if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) {
2667 dev_err(dev, "SRAM pool exhausted\n");
2668 rc = -ENOMEM;
2669 goto fail1;
2670 }
2671
2672 /* Linux crypto */
2673 for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) {
2674 t_alg = cc_create_aead_alg(&aead_algs[alg], dev);
2675 if (IS_ERR(t_alg)) {
2676 rc = PTR_ERR(t_alg);
2677 dev_err(dev, "%s alg allocation failed\n",
2678 aead_algs[alg].driver_name);
2679 goto fail1;
2680 }
2681 t_alg->drvdata = drvdata;
2682 rc = crypto_register_aead(&t_alg->aead_alg);
2683 if (rc) {
2684 dev_err(dev, "%s alg registration failed\n",
2685 t_alg->aead_alg.base.cra_driver_name);
2686 goto fail2;
2687 } else {
2688 list_add_tail(&t_alg->entry, &aead_handle->aead_list);
2689 dev_dbg(dev, "Registered %s\n",
2690 t_alg->aead_alg.base.cra_driver_name);
2691 }
2692 }
2693
2694 return 0;
2695
2696fail2:
2697 kfree(t_alg);
2698fail1:
2699 cc_aead_free(drvdata);
2700fail0:
2701 return rc;
2702}
diff --git a/drivers/crypto/ccree/cc_aead.h b/drivers/crypto/ccree/cc_aead.h
new file mode 100644
index 000000000000..5edf3b351fa4
--- /dev/null
+++ b/drivers/crypto/ccree/cc_aead.h
@@ -0,0 +1,109 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
3
4/* \file cc_aead.h
5 * ARM CryptoCell AEAD Crypto API
6 */
7
8#ifndef __CC_AEAD_H__
9#define __CC_AEAD_H__
10
11#include <linux/kernel.h>
12#include <crypto/algapi.h>
13#include <crypto/ctr.h>
14
15/* mac_cmp - HW writes 8 B but all bytes hold the same value */
16#define ICV_CMP_SIZE 8
17#define CCM_CONFIG_BUF_SIZE (AES_BLOCK_SIZE * 3)
18#define MAX_MAC_SIZE SHA256_DIGEST_SIZE
19
20/* defines for AES GCM configuration buffer */
21#define GCM_BLOCK_LEN_SIZE 8
22
23#define GCM_BLOCK_RFC4_IV_OFFSET 4
24#define GCM_BLOCK_RFC4_IV_SIZE 8 /* IV size for rfc's */
25#define GCM_BLOCK_RFC4_NONCE_OFFSET 0
26#define GCM_BLOCK_RFC4_NONCE_SIZE 4
27
28/* Offsets into AES CCM configuration buffer */
29#define CCM_B0_OFFSET 0
30#define CCM_A0_OFFSET 16
31#define CCM_CTR_COUNT_0_OFFSET 32
32/* CCM B0 and CTR_COUNT constants. */
33#define CCM_BLOCK_NONCE_OFFSET 1 /* Nonce offset inside B0 and CTR_COUNT */
34#define CCM_BLOCK_NONCE_SIZE 3 /* Nonce size inside B0 and CTR_COUNT */
35#define CCM_BLOCK_IV_OFFSET 4 /* IV offset inside B0 and CTR_COUNT */
36#define CCM_BLOCK_IV_SIZE 8 /* IV size inside B0 and CTR_COUNT */
37
38enum aead_ccm_header_size {
39 ccm_header_size_null = -1,
40 ccm_header_size_zero = 0,
41 ccm_header_size_2 = 2,
42 ccm_header_size_6 = 6,
43 ccm_header_size_max = S32_MAX
44};
45
46struct aead_req_ctx {
47 /* Allocate cache line although only 4 bytes are needed to
48 * assure next field falls @ cache line
49 * Used for both: digest HW compare and CCM/GCM MAC value
50 */
51 u8 mac_buf[MAX_MAC_SIZE] ____cacheline_aligned;
52 u8 ctr_iv[AES_BLOCK_SIZE] ____cacheline_aligned;
53
54 //used in gcm
55 u8 gcm_iv_inc1[AES_BLOCK_SIZE] ____cacheline_aligned;
56 u8 gcm_iv_inc2[AES_BLOCK_SIZE] ____cacheline_aligned;
57 u8 hkey[AES_BLOCK_SIZE] ____cacheline_aligned;
58 struct {
59 u8 len_a[GCM_BLOCK_LEN_SIZE] ____cacheline_aligned;
60 u8 len_c[GCM_BLOCK_LEN_SIZE];
61 } gcm_len_block;
62
63 u8 ccm_config[CCM_CONFIG_BUF_SIZE] ____cacheline_aligned;
64 /* HW actual size input */
65 unsigned int hw_iv_size ____cacheline_aligned;
66 /* used to prevent cache coherence problem */
67 u8 backup_mac[MAX_MAC_SIZE];
68 u8 *backup_iv; /*store iv for generated IV flow*/
69 u8 *backup_giv; /*store iv for rfc3686(ctr) flow*/
70 dma_addr_t mac_buf_dma_addr; /* internal ICV DMA buffer */
71 /* buffer for internal ccm configurations */
72 dma_addr_t ccm_iv0_dma_addr;
73 dma_addr_t icv_dma_addr; /* Phys. address of ICV */
74
75 //used in gcm
76 /* buffer for internal gcm configurations */
77 dma_addr_t gcm_iv_inc1_dma_addr;
78 /* buffer for internal gcm configurations */
79 dma_addr_t gcm_iv_inc2_dma_addr;
80 dma_addr_t hkey_dma_addr; /* Phys. address of hkey */
81 dma_addr_t gcm_block_len_dma_addr; /* Phys. address of gcm block len */
82 bool is_gcm4543;
83
84 u8 *icv_virt_addr; /* Virt. address of ICV */
85 struct async_gen_req_ctx gen_ctx;
86 struct cc_mlli assoc;
87 struct cc_mlli src;
88 struct cc_mlli dst;
89 struct scatterlist *src_sgl;
90 struct scatterlist *dst_sgl;
91 unsigned int src_offset;
92 unsigned int dst_offset;
93 enum cc_req_dma_buf_type assoc_buff_type;
94 enum cc_req_dma_buf_type data_buff_type;
95 struct mlli_params mlli_params;
96 unsigned int cryptlen;
97 struct scatterlist ccm_adata_sg;
98 enum aead_ccm_header_size ccm_hdr_size;
99 unsigned int req_authsize;
100 enum drv_cipher_mode cipher_mode;
101 bool is_icv_fragmented;
102 bool is_single_pass;
103 bool plaintext_authenticate_only; //for gcm_rfc4543
104};
105
106int cc_aead_alloc(struct cc_drvdata *drvdata);
107int cc_aead_free(struct cc_drvdata *drvdata);
108
109#endif /*__CC_AEAD_H__*/
diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c
index bb306b4efa4c..b32577477b4c 100644
--- a/drivers/crypto/ccree/cc_buffer_mgr.c
+++ b/drivers/crypto/ccree/cc_buffer_mgr.c
@@ -1,6 +1,7 @@
1// SPDX-License-Identifier: GPL-2.0 1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */ 2/* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
3 3
4#include <crypto/internal/aead.h>
4#include <crypto/authenc.h> 5#include <crypto/authenc.h>
5#include <crypto/scatterwalk.h> 6#include <crypto/scatterwalk.h>
6#include <linux/dmapool.h> 7#include <linux/dmapool.h>
@@ -10,6 +11,7 @@
10#include "cc_lli_defs.h" 11#include "cc_lli_defs.h"
11#include "cc_cipher.h" 12#include "cc_cipher.h"
12#include "cc_hash.h" 13#include "cc_hash.h"
14#include "cc_aead.h"
13 15
14enum dma_buffer_type { 16enum dma_buffer_type {
15 DMA_NULL_TYPE = -1, 17 DMA_NULL_TYPE = -1,
@@ -52,6 +54,27 @@ static inline char *cc_dma_buf_type(enum cc_req_dma_buf_type type)
52} 54}
53 55
54/** 56/**
57 * cc_copy_mac() - Copy MAC to temporary location
58 *
59 * @dev: device object
60 * @req: aead request object
61 * @dir: [IN] copy from/to sgl
62 */
63static void cc_copy_mac(struct device *dev, struct aead_request *req,
64 enum cc_sg_cpy_direct dir)
65{
66 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
67 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
68 u32 skip = req->assoclen + req->cryptlen;
69
70 if (areq_ctx->is_gcm4543)
71 skip += crypto_aead_ivsize(tfm);
72
73 cc_copy_sg_portion(dev, areq_ctx->backup_mac, req->src,
74 (skip - areq_ctx->req_authsize), skip, dir);
75}
76
77/**
55 * cc_get_sgl_nents() - Get scatterlist number of entries. 78 * cc_get_sgl_nents() - Get scatterlist number of entries.
56 * 79 *
57 * @sg_list: SG list 80 * @sg_list: SG list
@@ -246,6 +269,27 @@ build_mlli_exit:
246 return rc; 269 return rc;
247} 270}
248 271
272static void cc_add_buffer_entry(struct device *dev,
273 struct buffer_array *sgl_data,
274 dma_addr_t buffer_dma, unsigned int buffer_len,
275 bool is_last_entry, u32 *mlli_nents)
276{
277 unsigned int index = sgl_data->num_of_buffers;
278
279 dev_dbg(dev, "index=%u single_buff=%pad buffer_len=0x%08X is_last=%d\n",
280 index, &buffer_dma, buffer_len, is_last_entry);
281 sgl_data->nents[index] = 1;
282 sgl_data->entry[index].buffer_dma = buffer_dma;
283 sgl_data->offset[index] = 0;
284 sgl_data->total_data_len[index] = buffer_len;
285 sgl_data->type[index] = DMA_BUFF_TYPE;
286 sgl_data->is_last[index] = is_last_entry;
287 sgl_data->mlli_nents[index] = mlli_nents;
288 if (sgl_data->mlli_nents[index])
289 *sgl_data->mlli_nents[index] = 0;
290 sgl_data->num_of_buffers++;
291}
292
249static void cc_add_sg_entry(struct device *dev, struct buffer_array *sgl_data, 293static void cc_add_sg_entry(struct device *dev, struct buffer_array *sgl_data,
250 unsigned int nents, struct scatterlist *sgl, 294 unsigned int nents, struct scatterlist *sgl,
251 unsigned int data_len, unsigned int data_offset, 295 unsigned int data_len, unsigned int data_offset,
@@ -349,6 +393,33 @@ static int cc_map_sg(struct device *dev, struct scatterlist *sg,
349 return 0; 393 return 0;
350} 394}
351 395
396static int
397cc_set_aead_conf_buf(struct device *dev, struct aead_req_ctx *areq_ctx,
398 u8 *config_data, struct buffer_array *sg_data,
399 unsigned int assoclen)
400{
401 dev_dbg(dev, " handle additional data config set to DLLI\n");
402 /* create sg for the current buffer */
403 sg_init_one(&areq_ctx->ccm_adata_sg, config_data,
404 AES_BLOCK_SIZE + areq_ctx->ccm_hdr_size);
405 if (dma_map_sg(dev, &areq_ctx->ccm_adata_sg, 1, DMA_TO_DEVICE) != 1) {
406 dev_err(dev, "dma_map_sg() config buffer failed\n");
407 return -ENOMEM;
408 }
409 dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n",
410 &sg_dma_address(&areq_ctx->ccm_adata_sg),
411 sg_page(&areq_ctx->ccm_adata_sg),
412 sg_virt(&areq_ctx->ccm_adata_sg),
413 areq_ctx->ccm_adata_sg.offset, areq_ctx->ccm_adata_sg.length);
414 /* prepare for case of MLLI */
415 if (assoclen > 0) {
416 cc_add_sg_entry(dev, sg_data, 1, &areq_ctx->ccm_adata_sg,
417 (AES_BLOCK_SIZE + areq_ctx->ccm_hdr_size),
418 0, false, NULL);
419 }
420 return 0;
421}
422
352static int cc_set_hash_buf(struct device *dev, struct ahash_req_ctx *areq_ctx, 423static int cc_set_hash_buf(struct device *dev, struct ahash_req_ctx *areq_ctx,
353 u8 *curr_buff, u32 curr_buff_cnt, 424 u8 *curr_buff, u32 curr_buff_cnt,
354 struct buffer_array *sg_data) 425 struct buffer_array *sg_data)
@@ -497,6 +568,817 @@ cipher_exit:
497 return rc; 568 return rc;
498} 569}
499 570
571void cc_unmap_aead_request(struct device *dev, struct aead_request *req)
572{
573 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
574 unsigned int hw_iv_size = areq_ctx->hw_iv_size;
575 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
576 struct cc_drvdata *drvdata = dev_get_drvdata(dev);
577 u32 dummy;
578 bool chained;
579 u32 size_to_unmap = 0;
580
581 if (areq_ctx->mac_buf_dma_addr) {
582 dma_unmap_single(dev, areq_ctx->mac_buf_dma_addr,
583 MAX_MAC_SIZE, DMA_BIDIRECTIONAL);
584 }
585
586 if (areq_ctx->cipher_mode == DRV_CIPHER_GCTR) {
587 if (areq_ctx->hkey_dma_addr) {
588 dma_unmap_single(dev, areq_ctx->hkey_dma_addr,
589 AES_BLOCK_SIZE, DMA_BIDIRECTIONAL);
590 }
591
592 if (areq_ctx->gcm_block_len_dma_addr) {
593 dma_unmap_single(dev, areq_ctx->gcm_block_len_dma_addr,
594 AES_BLOCK_SIZE, DMA_TO_DEVICE);
595 }
596
597 if (areq_ctx->gcm_iv_inc1_dma_addr) {
598 dma_unmap_single(dev, areq_ctx->gcm_iv_inc1_dma_addr,
599 AES_BLOCK_SIZE, DMA_TO_DEVICE);
600 }
601
602 if (areq_ctx->gcm_iv_inc2_dma_addr) {
603 dma_unmap_single(dev, areq_ctx->gcm_iv_inc2_dma_addr,
604 AES_BLOCK_SIZE, DMA_TO_DEVICE);
605 }
606 }
607
608 if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
609 if (areq_ctx->ccm_iv0_dma_addr) {
610 dma_unmap_single(dev, areq_ctx->ccm_iv0_dma_addr,
611 AES_BLOCK_SIZE, DMA_TO_DEVICE);
612 }
613
614 dma_unmap_sg(dev, &areq_ctx->ccm_adata_sg, 1, DMA_TO_DEVICE);
615 }
616 if (areq_ctx->gen_ctx.iv_dma_addr) {
617 dma_unmap_single(dev, areq_ctx->gen_ctx.iv_dma_addr,
618 hw_iv_size, DMA_BIDIRECTIONAL);
619 }
620
621 /*In case a pool was set, a table was
622 *allocated and should be released
623 */
624 if (areq_ctx->mlli_params.curr_pool) {
625 dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%pK\n",
626 &areq_ctx->mlli_params.mlli_dma_addr,
627 areq_ctx->mlli_params.mlli_virt_addr);
628 dma_pool_free(areq_ctx->mlli_params.curr_pool,
629 areq_ctx->mlli_params.mlli_virt_addr,
630 areq_ctx->mlli_params.mlli_dma_addr);
631 }
632
633 dev_dbg(dev, "Unmapping src sgl: req->src=%pK areq_ctx->src.nents=%u areq_ctx->assoc.nents=%u assoclen:%u cryptlen=%u\n",
634 sg_virt(req->src), areq_ctx->src.nents, areq_ctx->assoc.nents,
635 req->assoclen, req->cryptlen);
636 size_to_unmap = req->assoclen + req->cryptlen;
637 if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_ENCRYPT)
638 size_to_unmap += areq_ctx->req_authsize;
639 if (areq_ctx->is_gcm4543)
640 size_to_unmap += crypto_aead_ivsize(tfm);
641
642 dma_unmap_sg(dev, req->src,
643 cc_get_sgl_nents(dev, req->src, size_to_unmap,
644 &dummy, &chained),
645 DMA_BIDIRECTIONAL);
646 if (req->src != req->dst) {
647 dev_dbg(dev, "Unmapping dst sgl: req->dst=%pK\n",
648 sg_virt(req->dst));
649 dma_unmap_sg(dev, req->dst,
650 cc_get_sgl_nents(dev, req->dst, size_to_unmap,
651 &dummy, &chained),
652 DMA_BIDIRECTIONAL);
653 }
654 if (drvdata->coherent &&
655 areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT &&
656 req->src == req->dst) {
657 /* copy back mac from temporary location to deal with possible
658 * data memory overriding that caused by cache coherence
659 * problem.
660 */
661 cc_copy_mac(dev, req, CC_SG_FROM_BUF);
662 }
663}
664
665static int cc_get_aead_icv_nents(struct device *dev, struct scatterlist *sgl,
666 unsigned int sgl_nents, unsigned int authsize,
667 u32 last_entry_data_size,
668 bool *is_icv_fragmented)
669{
670 unsigned int icv_max_size = 0;
671 unsigned int icv_required_size = authsize > last_entry_data_size ?
672 (authsize - last_entry_data_size) :
673 authsize;
674 unsigned int nents;
675 unsigned int i;
676
677 if (sgl_nents < MAX_ICV_NENTS_SUPPORTED) {
678 *is_icv_fragmented = false;
679 return 0;
680 }
681
682 for (i = 0 ; i < (sgl_nents - MAX_ICV_NENTS_SUPPORTED) ; i++) {
683 if (!sgl)
684 break;
685 sgl = sg_next(sgl);
686 }
687
688 if (sgl)
689 icv_max_size = sgl->length;
690
691 if (last_entry_data_size > authsize) {
692 /* ICV attached to data in last entry (not fragmented!) */
693 nents = 0;
694 *is_icv_fragmented = false;
695 } else if (last_entry_data_size == authsize) {
696 /* ICV placed in whole last entry (not fragmented!) */
697 nents = 1;
698 *is_icv_fragmented = false;
699 } else if (icv_max_size > icv_required_size) {
700 nents = 1;
701 *is_icv_fragmented = true;
702 } else if (icv_max_size == icv_required_size) {
703 nents = 2;
704 *is_icv_fragmented = true;
705 } else {
706 dev_err(dev, "Unsupported num. of ICV fragments (> %d)\n",
707 MAX_ICV_NENTS_SUPPORTED);
708 nents = -1; /*unsupported*/
709 }
710 dev_dbg(dev, "is_frag=%s icv_nents=%u\n",
711 (*is_icv_fragmented ? "true" : "false"), nents);
712
713 return nents;
714}
715
716static int cc_aead_chain_iv(struct cc_drvdata *drvdata,
717 struct aead_request *req,
718 struct buffer_array *sg_data,
719 bool is_last, bool do_chain)
720{
721 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
722 unsigned int hw_iv_size = areq_ctx->hw_iv_size;
723 struct device *dev = drvdata_to_dev(drvdata);
724 int rc = 0;
725
726 if (!req->iv) {
727 areq_ctx->gen_ctx.iv_dma_addr = 0;
728 goto chain_iv_exit;
729 }
730
731 areq_ctx->gen_ctx.iv_dma_addr = dma_map_single(dev, req->iv,
732 hw_iv_size,
733 DMA_BIDIRECTIONAL);
734 if (dma_mapping_error(dev, areq_ctx->gen_ctx.iv_dma_addr)) {
735 dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n",
736 hw_iv_size, req->iv);
737 rc = -ENOMEM;
738 goto chain_iv_exit;
739 }
740
741 dev_dbg(dev, "Mapped iv %u B at va=%pK to dma=%pad\n",
742 hw_iv_size, req->iv, &areq_ctx->gen_ctx.iv_dma_addr);
743 // TODO: what about CTR?? ask Ron
744 if (do_chain && areq_ctx->plaintext_authenticate_only) {
745 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
746 unsigned int iv_size_to_authenc = crypto_aead_ivsize(tfm);
747 unsigned int iv_ofs = GCM_BLOCK_RFC4_IV_OFFSET;
748 /* Chain to given list */
749 cc_add_buffer_entry(dev, sg_data,
750 (areq_ctx->gen_ctx.iv_dma_addr + iv_ofs),
751 iv_size_to_authenc, is_last,
752 &areq_ctx->assoc.mlli_nents);
753 areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI;
754 }
755
756chain_iv_exit:
757 return rc;
758}
759
760static int cc_aead_chain_assoc(struct cc_drvdata *drvdata,
761 struct aead_request *req,
762 struct buffer_array *sg_data,
763 bool is_last, bool do_chain)
764{
765 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
766 int rc = 0;
767 u32 mapped_nents = 0;
768 struct scatterlist *current_sg = req->src;
769 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
770 unsigned int sg_index = 0;
771 u32 size_of_assoc = req->assoclen;
772 struct device *dev = drvdata_to_dev(drvdata);
773
774 if (areq_ctx->is_gcm4543)
775 size_of_assoc += crypto_aead_ivsize(tfm);
776
777 if (!sg_data) {
778 rc = -EINVAL;
779 goto chain_assoc_exit;
780 }
781
782 if (req->assoclen == 0) {
783 areq_ctx->assoc_buff_type = CC_DMA_BUF_NULL;
784 areq_ctx->assoc.nents = 0;
785 areq_ctx->assoc.mlli_nents = 0;
786 dev_dbg(dev, "Chain assoc of length 0: buff_type=%s nents=%u\n",
787 cc_dma_buf_type(areq_ctx->assoc_buff_type),
788 areq_ctx->assoc.nents);
789 goto chain_assoc_exit;
790 }
791
792 //iterate over the sgl to see how many entries are for associated data
793 //it is assumed that if we reach here , the sgl is already mapped
794 sg_index = current_sg->length;
795 //the first entry in the scatter list contains all the associated data
796 if (sg_index > size_of_assoc) {
797 mapped_nents++;
798 } else {
799 while (sg_index <= size_of_assoc) {
800 current_sg = sg_next(current_sg);
801 /* if have reached the end of the sgl, then this is
802 * unexpected
803 */
804 if (!current_sg) {
805 dev_err(dev, "reached end of sg list. unexpected\n");
806 return -EINVAL;
807 }
808 sg_index += current_sg->length;
809 mapped_nents++;
810 }
811 }
812 if (mapped_nents > LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) {
813 dev_err(dev, "Too many fragments. current %d max %d\n",
814 mapped_nents, LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES);
815 return -ENOMEM;
816 }
817 areq_ctx->assoc.nents = mapped_nents;
818
819 /* in CCM case we have additional entry for
820 * ccm header configurations
821 */
822 if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
823 if ((mapped_nents + 1) > LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) {
824 dev_err(dev, "CCM case.Too many fragments. Current %d max %d\n",
825 (areq_ctx->assoc.nents + 1),
826 LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES);
827 rc = -ENOMEM;
828 goto chain_assoc_exit;
829 }
830 }
831
832 if (mapped_nents == 1 && areq_ctx->ccm_hdr_size == ccm_header_size_null)
833 areq_ctx->assoc_buff_type = CC_DMA_BUF_DLLI;
834 else
835 areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI;
836
837 if (do_chain || areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI) {
838 dev_dbg(dev, "Chain assoc: buff_type=%s nents=%u\n",
839 cc_dma_buf_type(areq_ctx->assoc_buff_type),
840 areq_ctx->assoc.nents);
841 cc_add_sg_entry(dev, sg_data, areq_ctx->assoc.nents, req->src,
842 req->assoclen, 0, is_last,
843 &areq_ctx->assoc.mlli_nents);
844 areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI;
845 }
846
847chain_assoc_exit:
848 return rc;
849}
850
851static void cc_prepare_aead_data_dlli(struct aead_request *req,
852 u32 *src_last_bytes, u32 *dst_last_bytes)
853{
854 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
855 enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
856 unsigned int authsize = areq_ctx->req_authsize;
857
858 areq_ctx->is_icv_fragmented = false;
859 if (req->src == req->dst) {
860 /*INPLACE*/
861 areq_ctx->icv_dma_addr = sg_dma_address(areq_ctx->src_sgl) +
862 (*src_last_bytes - authsize);
863 areq_ctx->icv_virt_addr = sg_virt(areq_ctx->src_sgl) +
864 (*src_last_bytes - authsize);
865 } else if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) {
866 /*NON-INPLACE and DECRYPT*/
867 areq_ctx->icv_dma_addr = sg_dma_address(areq_ctx->src_sgl) +
868 (*src_last_bytes - authsize);
869 areq_ctx->icv_virt_addr = sg_virt(areq_ctx->src_sgl) +
870 (*src_last_bytes - authsize);
871 } else {
872 /*NON-INPLACE and ENCRYPT*/
873 areq_ctx->icv_dma_addr = sg_dma_address(areq_ctx->dst_sgl) +
874 (*dst_last_bytes - authsize);
875 areq_ctx->icv_virt_addr = sg_virt(areq_ctx->dst_sgl) +
876 (*dst_last_bytes - authsize);
877 }
878}
879
880static int cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata,
881 struct aead_request *req,
882 struct buffer_array *sg_data,
883 u32 *src_last_bytes, u32 *dst_last_bytes,
884 bool is_last_table)
885{
886 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
887 enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
888 unsigned int authsize = areq_ctx->req_authsize;
889 int rc = 0, icv_nents;
890 struct device *dev = drvdata_to_dev(drvdata);
891 struct scatterlist *sg;
892
893 if (req->src == req->dst) {
894 /*INPLACE*/
895 cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents,
896 areq_ctx->src_sgl, areq_ctx->cryptlen,
897 areq_ctx->src_offset, is_last_table,
898 &areq_ctx->src.mlli_nents);
899
900 icv_nents = cc_get_aead_icv_nents(dev, areq_ctx->src_sgl,
901 areq_ctx->src.nents,
902 authsize, *src_last_bytes,
903 &areq_ctx->is_icv_fragmented);
904 if (icv_nents < 0) {
905 rc = -ENOTSUPP;
906 goto prepare_data_mlli_exit;
907 }
908
909 if (areq_ctx->is_icv_fragmented) {
910 /* Backup happens only when ICV is fragmented, ICV
911 * verification is made by CPU compare in order to
912 * simplify MAC verification upon request completion
913 */
914 if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) {
915 /* In coherent platforms (e.g. ACP)
916 * already copying ICV for any
917 * INPLACE-DECRYPT operation, hence
918 * we must neglect this code.
919 */
920 if (!drvdata->coherent)
921 cc_copy_mac(dev, req, CC_SG_TO_BUF);
922
923 areq_ctx->icv_virt_addr = areq_ctx->backup_mac;
924 } else {
925 areq_ctx->icv_virt_addr = areq_ctx->mac_buf;
926 areq_ctx->icv_dma_addr =
927 areq_ctx->mac_buf_dma_addr;
928 }
929 } else { /* Contig. ICV */
930 sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1];
931 /*Should hanlde if the sg is not contig.*/
932 areq_ctx->icv_dma_addr = sg_dma_address(sg) +
933 (*src_last_bytes - authsize);
934 areq_ctx->icv_virt_addr = sg_virt(sg) +
935 (*src_last_bytes - authsize);
936 }
937
938 } else if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) {
939 /*NON-INPLACE and DECRYPT*/
940 cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents,
941 areq_ctx->src_sgl, areq_ctx->cryptlen,
942 areq_ctx->src_offset, is_last_table,
943 &areq_ctx->src.mlli_nents);
944 cc_add_sg_entry(dev, sg_data, areq_ctx->dst.nents,
945 areq_ctx->dst_sgl, areq_ctx->cryptlen,
946 areq_ctx->dst_offset, is_last_table,
947 &areq_ctx->dst.mlli_nents);
948
949 icv_nents = cc_get_aead_icv_nents(dev, areq_ctx->src_sgl,
950 areq_ctx->src.nents,
951 authsize, *src_last_bytes,
952 &areq_ctx->is_icv_fragmented);
953 if (icv_nents < 0) {
954 rc = -ENOTSUPP;
955 goto prepare_data_mlli_exit;
956 }
957
958 /* Backup happens only when ICV is fragmented, ICV
959 * verification is made by CPU compare in order to simplify
960 * MAC verification upon request completion
961 */
962 if (areq_ctx->is_icv_fragmented) {
963 cc_copy_mac(dev, req, CC_SG_TO_BUF);
964 areq_ctx->icv_virt_addr = areq_ctx->backup_mac;
965
966 } else { /* Contig. ICV */
967 sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1];
968 /*Should hanlde if the sg is not contig.*/
969 areq_ctx->icv_dma_addr = sg_dma_address(sg) +
970 (*src_last_bytes - authsize);
971 areq_ctx->icv_virt_addr = sg_virt(sg) +
972 (*src_last_bytes - authsize);
973 }
974
975 } else {
976 /*NON-INPLACE and ENCRYPT*/
977 cc_add_sg_entry(dev, sg_data, areq_ctx->dst.nents,
978 areq_ctx->dst_sgl, areq_ctx->cryptlen,
979 areq_ctx->dst_offset, is_last_table,
980 &areq_ctx->dst.mlli_nents);
981 cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents,
982 areq_ctx->src_sgl, areq_ctx->cryptlen,
983 areq_ctx->src_offset, is_last_table,
984 &areq_ctx->src.mlli_nents);
985
986 icv_nents = cc_get_aead_icv_nents(dev, areq_ctx->dst_sgl,
987 areq_ctx->dst.nents,
988 authsize, *dst_last_bytes,
989 &areq_ctx->is_icv_fragmented);
990 if (icv_nents < 0) {
991 rc = -ENOTSUPP;
992 goto prepare_data_mlli_exit;
993 }
994
995 if (!areq_ctx->is_icv_fragmented) {
996 sg = &areq_ctx->dst_sgl[areq_ctx->dst.nents - 1];
997 /* Contig. ICV */
998 areq_ctx->icv_dma_addr = sg_dma_address(sg) +
999 (*dst_last_bytes - authsize);
1000 areq_ctx->icv_virt_addr = sg_virt(sg) +
1001 (*dst_last_bytes - authsize);
1002 } else {
1003 areq_ctx->icv_dma_addr = areq_ctx->mac_buf_dma_addr;
1004 areq_ctx->icv_virt_addr = areq_ctx->mac_buf;
1005 }
1006 }
1007
1008prepare_data_mlli_exit:
1009 return rc;
1010}
1011
1012static int cc_aead_chain_data(struct cc_drvdata *drvdata,
1013 struct aead_request *req,
1014 struct buffer_array *sg_data,
1015 bool is_last_table, bool do_chain)
1016{
1017 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1018 struct device *dev = drvdata_to_dev(drvdata);
1019 enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type;
1020 unsigned int authsize = areq_ctx->req_authsize;
1021 unsigned int src_last_bytes = 0, dst_last_bytes = 0;
1022 int rc = 0;
1023 u32 src_mapped_nents = 0, dst_mapped_nents = 0;
1024 u32 offset = 0;
1025 /* non-inplace mode */
1026 unsigned int size_for_map = req->assoclen + req->cryptlen;
1027 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1028 u32 sg_index = 0;
1029 bool chained = false;
1030 bool is_gcm4543 = areq_ctx->is_gcm4543;
1031 u32 size_to_skip = req->assoclen;
1032
1033 if (is_gcm4543)
1034 size_to_skip += crypto_aead_ivsize(tfm);
1035
1036 offset = size_to_skip;
1037
1038 if (!sg_data)
1039 return -EINVAL;
1040
1041 areq_ctx->src_sgl = req->src;
1042 areq_ctx->dst_sgl = req->dst;
1043
1044 if (is_gcm4543)
1045 size_for_map += crypto_aead_ivsize(tfm);
1046
1047 size_for_map += (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1048 authsize : 0;
1049 src_mapped_nents = cc_get_sgl_nents(dev, req->src, size_for_map,
1050 &src_last_bytes, &chained);
1051 sg_index = areq_ctx->src_sgl->length;
1052 //check where the data starts
1053 while (sg_index <= size_to_skip) {
1054 offset -= areq_ctx->src_sgl->length;
1055 areq_ctx->src_sgl = sg_next(areq_ctx->src_sgl);
1056 //if have reached the end of the sgl, then this is unexpected
1057 if (!areq_ctx->src_sgl) {
1058 dev_err(dev, "reached end of sg list. unexpected\n");
1059 return -EINVAL;
1060 }
1061 sg_index += areq_ctx->src_sgl->length;
1062 src_mapped_nents--;
1063 }
1064 if (src_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) {
1065 dev_err(dev, "Too many fragments. current %d max %d\n",
1066 src_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES);
1067 return -ENOMEM;
1068 }
1069
1070 areq_ctx->src.nents = src_mapped_nents;
1071
1072 areq_ctx->src_offset = offset;
1073
1074 if (req->src != req->dst) {
1075 size_for_map = req->assoclen + req->cryptlen;
1076 size_for_map += (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1077 authsize : 0;
1078 if (is_gcm4543)
1079 size_for_map += crypto_aead_ivsize(tfm);
1080
1081 rc = cc_map_sg(dev, req->dst, size_for_map, DMA_BIDIRECTIONAL,
1082 &areq_ctx->dst.nents,
1083 LLI_MAX_NUM_OF_DATA_ENTRIES, &dst_last_bytes,
1084 &dst_mapped_nents);
1085 if (rc) {
1086 rc = -ENOMEM;
1087 goto chain_data_exit;
1088 }
1089 }
1090
1091 dst_mapped_nents = cc_get_sgl_nents(dev, req->dst, size_for_map,
1092 &dst_last_bytes, &chained);
1093 sg_index = areq_ctx->dst_sgl->length;
1094 offset = size_to_skip;
1095
1096 //check where the data starts
1097 while (sg_index <= size_to_skip) {
1098 offset -= areq_ctx->dst_sgl->length;
1099 areq_ctx->dst_sgl = sg_next(areq_ctx->dst_sgl);
1100 //if have reached the end of the sgl, then this is unexpected
1101 if (!areq_ctx->dst_sgl) {
1102 dev_err(dev, "reached end of sg list. unexpected\n");
1103 return -EINVAL;
1104 }
1105 sg_index += areq_ctx->dst_sgl->length;
1106 dst_mapped_nents--;
1107 }
1108 if (dst_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) {
1109 dev_err(dev, "Too many fragments. current %d max %d\n",
1110 dst_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES);
1111 return -ENOMEM;
1112 }
1113 areq_ctx->dst.nents = dst_mapped_nents;
1114 areq_ctx->dst_offset = offset;
1115 if (src_mapped_nents > 1 ||
1116 dst_mapped_nents > 1 ||
1117 do_chain) {
1118 areq_ctx->data_buff_type = CC_DMA_BUF_MLLI;
1119 rc = cc_prepare_aead_data_mlli(drvdata, req, sg_data,
1120 &src_last_bytes,
1121 &dst_last_bytes, is_last_table);
1122 } else {
1123 areq_ctx->data_buff_type = CC_DMA_BUF_DLLI;
1124 cc_prepare_aead_data_dlli(req, &src_last_bytes,
1125 &dst_last_bytes);
1126 }
1127
1128chain_data_exit:
1129 return rc;
1130}
1131
1132static void cc_update_aead_mlli_nents(struct cc_drvdata *drvdata,
1133 struct aead_request *req)
1134{
1135 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1136 u32 curr_mlli_size = 0;
1137
1138 if (areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI) {
1139 areq_ctx->assoc.sram_addr = drvdata->mlli_sram_addr;
1140 curr_mlli_size = areq_ctx->assoc.mlli_nents *
1141 LLI_ENTRY_BYTE_SIZE;
1142 }
1143
1144 if (areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) {
1145 /*Inplace case dst nents equal to src nents*/
1146 if (req->src == req->dst) {
1147 areq_ctx->dst.mlli_nents = areq_ctx->src.mlli_nents;
1148 areq_ctx->src.sram_addr = drvdata->mlli_sram_addr +
1149 curr_mlli_size;
1150 areq_ctx->dst.sram_addr = areq_ctx->src.sram_addr;
1151 if (!areq_ctx->is_single_pass)
1152 areq_ctx->assoc.mlli_nents +=
1153 areq_ctx->src.mlli_nents;
1154 } else {
1155 if (areq_ctx->gen_ctx.op_type ==
1156 DRV_CRYPTO_DIRECTION_DECRYPT) {
1157 areq_ctx->src.sram_addr =
1158 drvdata->mlli_sram_addr +
1159 curr_mlli_size;
1160 areq_ctx->dst.sram_addr =
1161 areq_ctx->src.sram_addr +
1162 areq_ctx->src.mlli_nents *
1163 LLI_ENTRY_BYTE_SIZE;
1164 if (!areq_ctx->is_single_pass)
1165 areq_ctx->assoc.mlli_nents +=
1166 areq_ctx->src.mlli_nents;
1167 } else {
1168 areq_ctx->dst.sram_addr =
1169 drvdata->mlli_sram_addr +
1170 curr_mlli_size;
1171 areq_ctx->src.sram_addr =
1172 areq_ctx->dst.sram_addr +
1173 areq_ctx->dst.mlli_nents *
1174 LLI_ENTRY_BYTE_SIZE;
1175 if (!areq_ctx->is_single_pass)
1176 areq_ctx->assoc.mlli_nents +=
1177 areq_ctx->dst.mlli_nents;
1178 }
1179 }
1180 }
1181}
1182
1183int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req)
1184{
1185 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1186 struct mlli_params *mlli_params = &areq_ctx->mlli_params;
1187 struct device *dev = drvdata_to_dev(drvdata);
1188 struct buffer_array sg_data;
1189 unsigned int authsize = areq_ctx->req_authsize;
1190 struct buff_mgr_handle *buff_mgr = drvdata->buff_mgr_handle;
1191 int rc = 0;
1192 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1193 bool is_gcm4543 = areq_ctx->is_gcm4543;
1194 dma_addr_t dma_addr;
1195 u32 mapped_nents = 0;
1196 u32 dummy = 0; /*used for the assoc data fragments */
1197 u32 size_to_map = 0;
1198 gfp_t flags = cc_gfp_flags(&req->base);
1199
1200 mlli_params->curr_pool = NULL;
1201 sg_data.num_of_buffers = 0;
1202
1203 /* copy mac to a temporary location to deal with possible
1204 * data memory overriding that caused by cache coherence problem.
1205 */
1206 if (drvdata->coherent &&
1207 areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT &&
1208 req->src == req->dst)
1209 cc_copy_mac(dev, req, CC_SG_TO_BUF);
1210
1211 /* cacluate the size for cipher remove ICV in decrypt*/
1212 areq_ctx->cryptlen = (areq_ctx->gen_ctx.op_type ==
1213 DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1214 req->cryptlen :
1215 (req->cryptlen - authsize);
1216
1217 dma_addr = dma_map_single(dev, areq_ctx->mac_buf, MAX_MAC_SIZE,
1218 DMA_BIDIRECTIONAL);
1219 if (dma_mapping_error(dev, dma_addr)) {
1220 dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n",
1221 MAX_MAC_SIZE, areq_ctx->mac_buf);
1222 rc = -ENOMEM;
1223 goto aead_map_failure;
1224 }
1225 areq_ctx->mac_buf_dma_addr = dma_addr;
1226
1227 if (areq_ctx->ccm_hdr_size != ccm_header_size_null) {
1228 void *addr = areq_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET;
1229
1230 dma_addr = dma_map_single(dev, addr, AES_BLOCK_SIZE,
1231 DMA_TO_DEVICE);
1232
1233 if (dma_mapping_error(dev, dma_addr)) {
1234 dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n",
1235 AES_BLOCK_SIZE, addr);
1236 areq_ctx->ccm_iv0_dma_addr = 0;
1237 rc = -ENOMEM;
1238 goto aead_map_failure;
1239 }
1240 areq_ctx->ccm_iv0_dma_addr = dma_addr;
1241
1242 if (cc_set_aead_conf_buf(dev, areq_ctx, areq_ctx->ccm_config,
1243 &sg_data, req->assoclen)) {
1244 rc = -ENOMEM;
1245 goto aead_map_failure;
1246 }
1247 }
1248
1249 if (areq_ctx->cipher_mode == DRV_CIPHER_GCTR) {
1250 dma_addr = dma_map_single(dev, areq_ctx->hkey, AES_BLOCK_SIZE,
1251 DMA_BIDIRECTIONAL);
1252 if (dma_mapping_error(dev, dma_addr)) {
1253 dev_err(dev, "Mapping hkey %u B at va=%pK for DMA failed\n",
1254 AES_BLOCK_SIZE, areq_ctx->hkey);
1255 rc = -ENOMEM;
1256 goto aead_map_failure;
1257 }
1258 areq_ctx->hkey_dma_addr = dma_addr;
1259
1260 dma_addr = dma_map_single(dev, &areq_ctx->gcm_len_block,
1261 AES_BLOCK_SIZE, DMA_TO_DEVICE);
1262 if (dma_mapping_error(dev, dma_addr)) {
1263 dev_err(dev, "Mapping gcm_len_block %u B at va=%pK for DMA failed\n",
1264 AES_BLOCK_SIZE, &areq_ctx->gcm_len_block);
1265 rc = -ENOMEM;
1266 goto aead_map_failure;
1267 }
1268 areq_ctx->gcm_block_len_dma_addr = dma_addr;
1269
1270 dma_addr = dma_map_single(dev, areq_ctx->gcm_iv_inc1,
1271 AES_BLOCK_SIZE, DMA_TO_DEVICE);
1272
1273 if (dma_mapping_error(dev, dma_addr)) {
1274 dev_err(dev, "Mapping gcm_iv_inc1 %u B at va=%pK for DMA failed\n",
1275 AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc1));
1276 areq_ctx->gcm_iv_inc1_dma_addr = 0;
1277 rc = -ENOMEM;
1278 goto aead_map_failure;
1279 }
1280 areq_ctx->gcm_iv_inc1_dma_addr = dma_addr;
1281
1282 dma_addr = dma_map_single(dev, areq_ctx->gcm_iv_inc2,
1283 AES_BLOCK_SIZE, DMA_TO_DEVICE);
1284
1285 if (dma_mapping_error(dev, dma_addr)) {
1286 dev_err(dev, "Mapping gcm_iv_inc2 %u B at va=%pK for DMA failed\n",
1287 AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc2));
1288 areq_ctx->gcm_iv_inc2_dma_addr = 0;
1289 rc = -ENOMEM;
1290 goto aead_map_failure;
1291 }
1292 areq_ctx->gcm_iv_inc2_dma_addr = dma_addr;
1293 }
1294
1295 size_to_map = req->cryptlen + req->assoclen;
1296 if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_ENCRYPT)
1297 size_to_map += authsize;
1298
1299 if (is_gcm4543)
1300 size_to_map += crypto_aead_ivsize(tfm);
1301 rc = cc_map_sg(dev, req->src, size_to_map, DMA_BIDIRECTIONAL,
1302 &areq_ctx->src.nents,
1303 (LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES +
1304 LLI_MAX_NUM_OF_DATA_ENTRIES),
1305 &dummy, &mapped_nents);
1306 if (rc) {
1307 rc = -ENOMEM;
1308 goto aead_map_failure;
1309 }
1310
1311 if (areq_ctx->is_single_pass) {
1312 /*
1313 * Create MLLI table for:
1314 * (1) Assoc. data
1315 * (2) Src/Dst SGLs
1316 * Note: IV is contg. buffer (not an SGL)
1317 */
1318 rc = cc_aead_chain_assoc(drvdata, req, &sg_data, true, false);
1319 if (rc)
1320 goto aead_map_failure;
1321 rc = cc_aead_chain_iv(drvdata, req, &sg_data, true, false);
1322 if (rc)
1323 goto aead_map_failure;
1324 rc = cc_aead_chain_data(drvdata, req, &sg_data, true, false);
1325 if (rc)
1326 goto aead_map_failure;
1327 } else { /* DOUBLE-PASS flow */
1328 /*
1329 * Prepare MLLI table(s) in this order:
1330 *
1331 * If ENCRYPT/DECRYPT (inplace):
1332 * (1) MLLI table for assoc
1333 * (2) IV entry (chained right after end of assoc)
1334 * (3) MLLI for src/dst (inplace operation)
1335 *
1336 * If ENCRYPT (non-inplace)
1337 * (1) MLLI table for assoc
1338 * (2) IV entry (chained right after end of assoc)
1339 * (3) MLLI for dst
1340 * (4) MLLI for src
1341 *
1342 * If DECRYPT (non-inplace)
1343 * (1) MLLI table for assoc
1344 * (2) IV entry (chained right after end of assoc)
1345 * (3) MLLI for src
1346 * (4) MLLI for dst
1347 */
1348 rc = cc_aead_chain_assoc(drvdata, req, &sg_data, false, true);
1349 if (rc)
1350 goto aead_map_failure;
1351 rc = cc_aead_chain_iv(drvdata, req, &sg_data, false, true);
1352 if (rc)
1353 goto aead_map_failure;
1354 rc = cc_aead_chain_data(drvdata, req, &sg_data, true, true);
1355 if (rc)
1356 goto aead_map_failure;
1357 }
1358
1359 /* Mlli support -start building the MLLI according to the above
1360 * results
1361 */
1362 if (areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
1363 areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) {
1364 mlli_params->curr_pool = buff_mgr->mlli_buffs_pool;
1365 rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags);
1366 if (rc)
1367 goto aead_map_failure;
1368
1369 cc_update_aead_mlli_nents(drvdata, req);
1370 dev_dbg(dev, "assoc params mn %d\n",
1371 areq_ctx->assoc.mlli_nents);
1372 dev_dbg(dev, "src params mn %d\n", areq_ctx->src.mlli_nents);
1373 dev_dbg(dev, "dst params mn %d\n", areq_ctx->dst.mlli_nents);
1374 }
1375 return 0;
1376
1377aead_map_failure:
1378 cc_unmap_aead_request(dev, req);
1379 return rc;
1380}
1381
500int cc_map_hash_request_final(struct cc_drvdata *drvdata, void *ctx, 1382int cc_map_hash_request_final(struct cc_drvdata *drvdata, void *ctx,
501 struct scatterlist *src, unsigned int nbytes, 1383 struct scatterlist *src, unsigned int nbytes,
502 bool do_update, gfp_t flags) 1384 bool do_update, gfp_t flags)
diff --git a/drivers/crypto/ccree/cc_buffer_mgr.h b/drivers/crypto/ccree/cc_buffer_mgr.h
index 614c5c5b2721..3ec4b4db5247 100644
--- a/drivers/crypto/ccree/cc_buffer_mgr.h
+++ b/drivers/crypto/ccree/cc_buffer_mgr.h
@@ -48,6 +48,10 @@ int cc_map_cipher_request(struct cc_drvdata *drvdata, void *ctx,
48void cc_unmap_cipher_request(struct device *dev, void *ctx, unsigned int ivsize, 48void cc_unmap_cipher_request(struct device *dev, void *ctx, unsigned int ivsize,
49 struct scatterlist *src, struct scatterlist *dst); 49 struct scatterlist *src, struct scatterlist *dst);
50 50
51int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req);
52
53void cc_unmap_aead_request(struct device *dev, struct aead_request *req);
54
51int cc_map_hash_request_final(struct cc_drvdata *drvdata, void *ctx, 55int cc_map_hash_request_final(struct cc_drvdata *drvdata, void *ctx,
52 struct scatterlist *src, unsigned int nbytes, 56 struct scatterlist *src, unsigned int nbytes,
53 bool do_update, gfp_t flags); 57 bool do_update, gfp_t flags);
diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c
index 6e32de90b38f..8a530a45be2e 100644
--- a/drivers/crypto/ccree/cc_driver.c
+++ b/drivers/crypto/ccree/cc_driver.c
@@ -20,6 +20,7 @@
20#include "cc_buffer_mgr.h" 20#include "cc_buffer_mgr.h"
21#include "cc_debugfs.h" 21#include "cc_debugfs.h"
22#include "cc_cipher.h" 22#include "cc_cipher.h"
23#include "cc_aead.h"
23#include "cc_hash.h" 24#include "cc_hash.h"
24#include "cc_ivgen.h" 25#include "cc_ivgen.h"
25#include "cc_sram_mgr.h" 26#include "cc_sram_mgr.h"
@@ -294,8 +295,16 @@ static int init_cc_resources(struct platform_device *plat_dev)
294 goto post_cipher_err; 295 goto post_cipher_err;
295 } 296 }
296 297
298 rc = cc_aead_alloc(new_drvdata);
299 if (rc) {
300 dev_err(dev, "cc_aead_alloc failed\n");
301 goto post_hash_err;
302 }
303
297 return 0; 304 return 0;
298 305
306post_hash_err:
307 cc_hash_free(new_drvdata);
299post_cipher_err: 308post_cipher_err:
300 cc_cipher_free(new_drvdata); 309 cc_cipher_free(new_drvdata);
301post_ivgen_err: 310post_ivgen_err:
@@ -328,6 +337,7 @@ static void cleanup_cc_resources(struct platform_device *plat_dev)
328 struct cc_drvdata *drvdata = 337 struct cc_drvdata *drvdata =
329 (struct cc_drvdata *)platform_get_drvdata(plat_dev); 338 (struct cc_drvdata *)platform_get_drvdata(plat_dev);
330 339
340 cc_aead_free(drvdata);
331 cc_hash_free(drvdata); 341 cc_hash_free(drvdata);
332 cc_cipher_free(drvdata); 342 cc_cipher_free(drvdata);
333 cc_ivgen_fini(drvdata); 343 cc_ivgen_fini(drvdata);
diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h
index a7098f5fde40..5862e2bf0795 100644
--- a/drivers/crypto/ccree/cc_driver.h
+++ b/drivers/crypto/ccree/cc_driver.h
@@ -114,6 +114,7 @@ struct cc_drvdata {
114 void *buff_mgr_handle; 114 void *buff_mgr_handle;
115 void *cipher_handle; 115 void *cipher_handle;
116 void *hash_handle; 116 void *hash_handle;
117 void *aead_handle;
117 void *request_mgr_handle; 118 void *request_mgr_handle;
118 void *ivgen_handle; 119 void *ivgen_handle;
119 void *sram_mgr_handle; 120 void *sram_mgr_handle;
@@ -130,6 +131,7 @@ struct cc_crypto_alg {
130 unsigned int data_unit; 131 unsigned int data_unit;
131 struct cc_drvdata *drvdata; 132 struct cc_drvdata *drvdata;
132 struct skcipher_alg skcipher_alg; 133 struct skcipher_alg skcipher_alg;
134 struct aead_alg aead_alg;
133}; 135};
134 136
135struct cc_alg_template { 137struct cc_alg_template {
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-25 10:33:15 -0400