aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc/crypto
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2012-08-29 15:50:16 -0400
committerDavid S. Miller <davem@davemloft.net>2012-08-29 17:22:23 -0400
commit0bdcaf7495726688a93a2f7226e9b4beaeabd2ec (patch)
tree1bd6f45eff4d6f5aea604a44ffb38596dd818426 /arch/sparc/crypto
parent45dfe237a865368929534ec75fe5f26c151c88d9 (diff)
sparc64: Move AES driver over to a methods based implementation.
Instead of testing and branching off of the key size on every encrypt/decrypt call, use method ops assigned at key set time. Reverse the order of float registers used for decryption to make future changes easier. Align all assembler routines on a 32-byte boundary. Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'arch/sparc/crypto')
-rw-r--r--arch/sparc/crypto/aes_asm.S879
-rw-r--r--arch/sparc/crypto/aes_glue.c186
2 files changed, 728 insertions, 337 deletions
diff --git a/arch/sparc/crypto/aes_asm.S b/arch/sparc/crypto/aes_asm.S
index f656dc7a173e..50faae03c592 100644
--- a/arch/sparc/crypto/aes_asm.S
+++ b/arch/sparc/crypto/aes_asm.S
@@ -44,8 +44,8 @@
44 .word 0x85b02307; 44 .word 0x85b02307;
45#define MOVXTOD_O0_F0 \ 45#define MOVXTOD_O0_F0 \
46 .word 0x81b02308; 46 .word 0x81b02308;
47#define MOVXTOD_O1_F2 \ 47#define MOVXTOD_O5_F2 \
48 .word 0x85b02309; 48 .word 0x85b0230d;
49 49
50#define ENCRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \ 50#define ENCRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \
51 AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \ 51 AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \
@@ -86,45 +86,46 @@
86 ENCRYPT_TWO_ROUNDS(KEY_BASE + 40, I0, I1, T0, T1) \ 86 ENCRYPT_TWO_ROUNDS(KEY_BASE + 40, I0, I1, T0, T1) \
87 ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 48, I0, I1, T0, T1) 87 ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 48, I0, I1, T0, T1)
88 88
89#define DECRYPT_TWO_ROUNDS(KEY_TOP, I0, I1, T0, T1) \ 89#define DECRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \
90 AES_DROUND23(KEY_TOP - 2, I0, I1, T1) \ 90 AES_DROUND23(KEY_BASE + 0, I0, I1, T1) \
91 AES_DROUND01(KEY_TOP - 4, I0, I1, T0) \ 91 AES_DROUND01(KEY_BASE + 2, I0, I1, T0) \
92 AES_DROUND23(KEY_TOP - 6, T0, T1, I1) \ 92 AES_DROUND23(KEY_BASE + 4, T0, T1, I1) \
93 AES_DROUND01(KEY_TOP - 8, T0, T1, I0) 93 AES_DROUND01(KEY_BASE + 6, T0, T1, I0)
94 94
95#define DECRYPT_TWO_ROUNDS_LAST(KEY_TOP, I0, I1, T0, T1) \ 95#define DECRYPT_TWO_ROUNDS_LAST(KEY_BASE, I0, I1, T0, T1) \
96 AES_DROUND23(KEY_TOP - 2, I0, I1, T1) \ 96 AES_DROUND23(KEY_BASE + 0, I0, I1, T1) \
97 AES_DROUND01(KEY_TOP - 4, I0, I1, T0) \ 97 AES_DROUND01(KEY_BASE + 2, I0, I1, T0) \
98 AES_DROUND23_L(KEY_TOP - 6, T0, T1, I1) \ 98 AES_DROUND23_L(KEY_BASE + 4, T0, T1, I1) \
99 AES_DROUND01_L(KEY_TOP - 8, T0, T1, I0) 99 AES_DROUND01_L(KEY_BASE + 6, T0, T1, I0)
100 100
101 /* 10 rounds */ 101 /* 10 rounds */
102#define DECRYPT_128(KEY_TOP, I0, I1, T0, T1) \ 102#define DECRYPT_128(KEY_BASE, I0, I1, T0, T1) \
103 DECRYPT_TWO_ROUNDS(KEY_TOP - 0, I0, I1, T0, T1) \ 103 DECRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \
104 DECRYPT_TWO_ROUNDS(KEY_TOP - 8, I0, I1, T0, T1) \ 104 DECRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \
105 DECRYPT_TWO_ROUNDS(KEY_TOP - 16, I0, I1, T0, T1) \ 105 DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
106 DECRYPT_TWO_ROUNDS(KEY_TOP - 24, I0, I1, T0, T1) \ 106 DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
107 DECRYPT_TWO_ROUNDS_LAST(KEY_TOP - 32, I0, I1, T0, T1) 107 DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 32, I0, I1, T0, T1)
108 108
109 /* 12 rounds */ 109 /* 12 rounds */
110#define DECRYPT_192(KEY_TOP, I0, I1, T0, T1) \ 110#define DECRYPT_192(KEY_BASE, I0, I1, T0, T1) \
111 DECRYPT_TWO_ROUNDS(KEY_TOP - 0, I0, I1, T0, T1) \ 111 DECRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \
112 DECRYPT_TWO_ROUNDS(KEY_TOP - 8, I0, I1, T0, T1) \ 112 DECRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \
113 DECRYPT_TWO_ROUNDS(KEY_TOP - 16, I0, I1, T0, T1) \ 113 DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
114 DECRYPT_TWO_ROUNDS(KEY_TOP - 24, I0, I1, T0, T1) \ 114 DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
115 DECRYPT_TWO_ROUNDS(KEY_TOP - 32, I0, I1, T0, T1) \ 115 DECRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
116 DECRYPT_TWO_ROUNDS_LAST(KEY_TOP - 40, I0, I1, T0, T1) 116 DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 40, I0, I1, T0, T1)
117 117
118 /* 14 rounds */ 118 /* 14 rounds */
119#define DECRYPT_256(KEY_TOP, I0, I1, T0, T1) \ 119#define DECRYPT_256(KEY_BASE, I0, I1, T0, T1) \
120 DECRYPT_TWO_ROUNDS(KEY_TOP - 0, I0, I1, T0, T1) \ 120 DECRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \
121 DECRYPT_TWO_ROUNDS(KEY_TOP - 8, I0, I1, T0, T1) \ 121 DECRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \
122 DECRYPT_TWO_ROUNDS(KEY_TOP - 16, I0, I1, T0, T1) \ 122 DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
123 DECRYPT_TWO_ROUNDS(KEY_TOP - 24, I0, I1, T0, T1) \ 123 DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
124 DECRYPT_TWO_ROUNDS(KEY_TOP - 32, I0, I1, T0, T1) \ 124 DECRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
125 DECRYPT_TWO_ROUNDS(KEY_TOP - 40, I0, I1, T0, T1) \ 125 DECRYPT_TWO_ROUNDS(KEY_BASE + 40, I0, I1, T0, T1) \
126 DECRYPT_TWO_ROUNDS_LAST(KEY_TOP - 48, I0, I1, T0, T1) 126 DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 48, I0, I1, T0, T1)
127 127
128 .align 32
128ENTRY(aes_sparc64_key_expand) 129ENTRY(aes_sparc64_key_expand)
129 /* %o0=input_key, %o1=output_key, %o2=key_len */ 130 /* %o0=input_key, %o1=output_key, %o2=key_len */
130 VISEntry 131 VISEntry
@@ -314,34 +315,63 @@ ENTRY(aes_sparc64_key_expand)
314 VISExit 315 VISExit
315ENDPROC(aes_sparc64_key_expand) 316ENDPROC(aes_sparc64_key_expand)
316 317
317ENTRY(aes_sparc64_encrypt) 318 .align 32
318 /* %o0=key, %o1=input, %o2=output, %o3=key_len */ 319ENTRY(aes_sparc64_encrypt_128)
320 /* %o0=key, %o1=input, %o2=output */
319 VISEntry 321 VISEntry
320 ld [%o1 + 0x00], %f4 322 ld [%o1 + 0x00], %f4
321 ld [%o1 + 0x04], %f5 323 ld [%o1 + 0x04], %f5
322 ld [%o1 + 0x08], %f6 324 ld [%o1 + 0x08], %f6
323 ld [%o1 + 0x0c], %f7 325 ld [%o1 + 0x0c], %f7
324
325 ldd [%o0 + 0x00], %f8 326 ldd [%o0 + 0x00], %f8
326 ldd [%o0 + 0x08], %f10 327 ldd [%o0 + 0x08], %f10
327 cmp %o3, 24 328 ldd [%o0 + 0x10], %f12
329 ldd [%o0 + 0x18], %f14
330 ldd [%o0 + 0x20], %f16
331 ldd [%o0 + 0x28], %f18
332 ldd [%o0 + 0x30], %f20
333 ldd [%o0 + 0x38], %f22
334 ldd [%o0 + 0x40], %f24
335 ldd [%o0 + 0x48], %f26
336 ldd [%o0 + 0x50], %f28
337 ldd [%o0 + 0x58], %f30
338 ldd [%o0 + 0x60], %f32
339 ldd [%o0 + 0x68], %f34
340 ldd [%o0 + 0x70], %f36
341 ldd [%o0 + 0x78], %f38
342 ldd [%o0 + 0x80], %f40
343 ldd [%o0 + 0x88], %f42
344 ldd [%o0 + 0x90], %f44
345 ldd [%o0 + 0x98], %f46
346 ldd [%o0 + 0xa0], %f48
347 ldd [%o0 + 0xa8], %f50
328 fxor %f8, %f4, %f4 348 fxor %f8, %f4, %f4
329 bl 2f 349 fxor %f10, %f6, %f6
330 fxor %f10, %f6, %f6 350 ENCRYPT_128(12, 4, 6, 0, 2)
351 st %f4, [%o2 + 0x00]
352 st %f5, [%o2 + 0x04]
353 st %f6, [%o2 + 0x08]
354 st %f7, [%o2 + 0x0c]
355 retl
356 VISExit
357ENDPROC(aes_sparc64_encrypt_128)
331 358
332 be 1f 359 .align 32
333 ldd [%o0 + 0x10], %f8 360ENTRY(aes_sparc64_encrypt_192)
361 /* %o0=key, %o1=input, %o2=output */
362 VISEntry
363 ld [%o1 + 0x00], %f4
364 ld [%o1 + 0x04], %f5
365 ld [%o1 + 0x08], %f6
366 ld [%o1 + 0x0c], %f7
334 367
335 ldd [%o0 + 0x18], %f10 368 ldd [%o0 + 0x00], %f8
336 ldd [%o0 + 0x20], %f12 369 ldd [%o0 + 0x08], %f10
337 ldd [%o0 + 0x28], %f14
338 add %o0, 0x20, %o0
339 370
340 ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2) 371 fxor %f8, %f4, %f4
372 fxor %f10, %f6, %f6
341 373
342 ldd [%o0 + 0x10], %f8 374 ldd [%o0 + 0x10], %f8
343
3441:
345 ldd [%o0 + 0x18], %f10 375 ldd [%o0 + 0x18], %f10
346 ldd [%o0 + 0x20], %f12 376 ldd [%o0 + 0x20], %f12
347 ldd [%o0 + 0x28], %f14 377 ldd [%o0 + 0x28], %f14
@@ -349,7 +379,6 @@ ENTRY(aes_sparc64_encrypt)
349 379
350 ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2) 380 ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
351 381
3522:
353 ldd [%o0 + 0x10], %f12 382 ldd [%o0 + 0x10], %f12
354 ldd [%o0 + 0x18], %f14 383 ldd [%o0 + 0x18], %f14
355 ldd [%o0 + 0x20], %f16 384 ldd [%o0 + 0x20], %f16
@@ -381,66 +410,63 @@ ENTRY(aes_sparc64_encrypt)
381 410
382 retl 411 retl
383 VISExit 412 VISExit
384ENDPROC(aes_sparc64_encrypt) 413ENDPROC(aes_sparc64_encrypt_192)
385 414
386ENTRY(aes_sparc64_decrypt) 415 .align 32
387 /* %o0=key, %o1=input, %o2=output, %o3=key_len, %o4=exp_key_len */ 416ENTRY(aes_sparc64_encrypt_256)
417 /* %o0=key, %o1=input, %o2=output */
388 VISEntry 418 VISEntry
389 ld [%o1 + 0x00], %f4 419 ld [%o1 + 0x00], %f4
390 add %o0, %o4, %o0
391 ld [%o1 + 0x04], %f5 420 ld [%o1 + 0x04], %f5
392 ld [%o1 + 0x08], %f6 421 ld [%o1 + 0x08], %f6
393 ld [%o1 + 0x0c], %f7 422 ld [%o1 + 0x0c], %f7
394
395 ldd [%o0 - 0x08], %f8
396 ldd [%o0 - 0x10], %f10
397 423
398 cmp %o3, 24 424 ldd [%o0 + 0x00], %f8
399 fxor %f10, %f4, %f4 425 ldd [%o0 + 0x08], %f10
400 bl 2f
401 fxor %f8, %f6, %f6
402 426
403 be 1f 427 fxor %f8, %f4, %f4
404 ldd [%o0 - 0x30], %f8 428 fxor %f10, %f6, %f6
405 429
406 ldd [%o0 - 0x28], %f10 430 ldd [%o0 + 0x10], %f8
407 ldd [%o0 - 0x20], %f12
408 ldd [%o0 - 0x18], %f14
409 sub %o0, 0x20, %o0
410 431
411 DECRYPT_TWO_ROUNDS(16, 4, 6, 0, 2) 432 ldd [%o0 + 0x18], %f10
433 ldd [%o0 + 0x20], %f12
434 ldd [%o0 + 0x28], %f14
435 add %o0, 0x20, %o0
412 436
413 ldd [%o0 - 0x30], %f8 437 ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
4141:
415 ldd [%o0 - 0x28], %f10
416 ldd [%o0 - 0x20], %f12
417 ldd [%o0 - 0x18], %f14
418 sub %o0, 0x20, %o0
419 438
420 DECRYPT_TWO_ROUNDS(16, 4, 6, 0, 2) 439 ldd [%o0 + 0x10], %f8
4212: 440
422 ldd [%o0 - 0xb0], %f12 441 ldd [%o0 + 0x18], %f10
423 ldd [%o0 - 0xa8], %f14 442 ldd [%o0 + 0x20], %f12
424 ldd [%o0 - 0xa0], %f16 443 ldd [%o0 + 0x28], %f14
425 ldd [%o0 - 0x98], %f18 444 add %o0, 0x20, %o0
426 ldd [%o0 - 0x90], %f20 445
427 ldd [%o0 - 0x88], %f22 446 ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
428 ldd [%o0 - 0x80], %f24 447
429 ldd [%o0 - 0x78], %f26 448 ldd [%o0 + 0x10], %f12
430 ldd [%o0 - 0x70], %f28 449 ldd [%o0 + 0x18], %f14
431 ldd [%o0 - 0x68], %f30 450 ldd [%o0 + 0x20], %f16
432 ldd [%o0 - 0x60], %f32 451 ldd [%o0 + 0x28], %f18
433 ldd [%o0 - 0x58], %f34 452 ldd [%o0 + 0x30], %f20
434 ldd [%o0 - 0x50], %f36 453 ldd [%o0 + 0x38], %f22
435 ldd [%o0 - 0x48], %f38 454 ldd [%o0 + 0x40], %f24
436 ldd [%o0 - 0x40], %f40 455 ldd [%o0 + 0x48], %f26
437 ldd [%o0 - 0x38], %f42 456 ldd [%o0 + 0x50], %f28
438 ldd [%o0 - 0x30], %f44 457 ldd [%o0 + 0x58], %f30
439 ldd [%o0 - 0x28], %f46 458 ldd [%o0 + 0x60], %f32
440 ldd [%o0 - 0x20], %f48 459 ldd [%o0 + 0x68], %f34
441 ldd [%o0 - 0x18], %f50 460 ldd [%o0 + 0x70], %f36
442 461 ldd [%o0 + 0x78], %f38
443 DECRYPT_128(52, 4, 6, 0, 2) 462 ldd [%o0 + 0x80], %f40
463 ldd [%o0 + 0x88], %f42
464 ldd [%o0 + 0x90], %f44
465 ldd [%o0 + 0x98], %f46
466 ldd [%o0 + 0xa0], %f48
467 ldd [%o0 + 0xa8], %f50
468
469 ENCRYPT_128(12, 4, 6, 0, 2)
444 470
445 st %f4, [%o2 + 0x00] 471 st %f4, [%o2 + 0x00]
446 st %f5, [%o2 + 0x04] 472 st %f5, [%o2 + 0x04]
@@ -449,15 +475,231 @@ ENTRY(aes_sparc64_decrypt)
449 475
450 retl 476 retl
451 VISExit 477 VISExit
452ENDPROC(aes_sparc64_decrypt) 478ENDPROC(aes_sparc64_encrypt_256)
453 479
454ENTRY(aes_sparc64_load_decrypt_keys) 480 .align 32
481ENTRY(aes_sparc64_decrypt_128)
482 /* %o0=key, %o1=input, %o2=output */
483 VISEntry
484 ld [%o1 + 0x00], %f4
485 ld [%o1 + 0x04], %f5
486 ld [%o1 + 0x08], %f6
487 ld [%o1 + 0x0c], %f7
488 ldd [%o0 + 0xa0], %f8
489 ldd [%o0 + 0xa8], %f10
490 ldd [%o0 + 0x98], %f12
491 ldd [%o0 + 0x90], %f14
492 ldd [%o0 + 0x88], %f16
493 ldd [%o0 + 0x80], %f18
494 ldd [%o0 + 0x78], %f20
495 ldd [%o0 + 0x70], %f22
496 ldd [%o0 + 0x68], %f24
497 ldd [%o0 + 0x60], %f26
498 ldd [%o0 + 0x58], %f28
499 ldd [%o0 + 0x50], %f30
500 ldd [%o0 + 0x48], %f32
501 ldd [%o0 + 0x40], %f34
502 ldd [%o0 + 0x38], %f36
503 ldd [%o0 + 0x30], %f38
504 ldd [%o0 + 0x28], %f40
505 ldd [%o0 + 0x20], %f42
506 ldd [%o0 + 0x18], %f44
507 ldd [%o0 + 0x10], %f46
508 ldd [%o0 + 0x08], %f48
509 ldd [%o0 + 0x00], %f50
510 fxor %f8, %f4, %f4
511 fxor %f10, %f6, %f6
512 DECRYPT_128(12, 4, 6, 0, 2)
513 st %f4, [%o2 + 0x00]
514 st %f5, [%o2 + 0x04]
515 st %f6, [%o2 + 0x08]
516 st %f7, [%o2 + 0x0c]
517 retl
518 VISExit
519ENDPROC(aes_sparc64_decrypt_128)
520
521 .align 32
522ENTRY(aes_sparc64_decrypt_192)
523 /* %o0=key, %o1=input, %o2=output */
524 VISEntry
525 ld [%o1 + 0x00], %f4
526 ld [%o1 + 0x04], %f5
527 ld [%o1 + 0x08], %f6
528 ld [%o1 + 0x0c], %f7
529 ldd [%o0 + 0xc0], %f8
530 ldd [%o0 + 0xc8], %f10
531 ldd [%o0 + 0xb8], %f12
532 ldd [%o0 + 0xb0], %f14
533 ldd [%o0 + 0xa8], %f16
534 ldd [%o0 + 0xa0], %f18
535 fxor %f8, %f4, %f4
536 fxor %f10, %f6, %f6
537 ldd [%o0 + 0x98], %f20
538 ldd [%o0 + 0x90], %f22
539 ldd [%o0 + 0x88], %f24
540 ldd [%o0 + 0x80], %f26
541 DECRYPT_TWO_ROUNDS(12, 4, 6, 0, 2)
542 ldd [%o0 + 0x78], %f28
543 ldd [%o0 + 0x70], %f30
544 ldd [%o0 + 0x68], %f32
545 ldd [%o0 + 0x60], %f34
546 ldd [%o0 + 0x58], %f36
547 ldd [%o0 + 0x50], %f38
548 ldd [%o0 + 0x48], %f40
549 ldd [%o0 + 0x40], %f42
550 ldd [%o0 + 0x38], %f44
551 ldd [%o0 + 0x30], %f46
552 ldd [%o0 + 0x28], %f48
553 ldd [%o0 + 0x20], %f50
554 ldd [%o0 + 0x18], %f52
555 ldd [%o0 + 0x10], %f54
556 ldd [%o0 + 0x08], %f56
557 ldd [%o0 + 0x00], %f58
558 DECRYPT_128(20, 4, 6, 0, 2)
559 st %f4, [%o2 + 0x00]
560 st %f5, [%o2 + 0x04]
561 st %f6, [%o2 + 0x08]
562 st %f7, [%o2 + 0x0c]
563 retl
564 VISExit
565ENDPROC(aes_sparc64_decrypt_192)
566
567 .align 32
568ENTRY(aes_sparc64_decrypt_256)
569 /* %o0=key, %o1=input, %o2=output */
570 VISEntry
571 ld [%o1 + 0x00], %f4
572 ld [%o1 + 0x04], %f5
573 ld [%o1 + 0x08], %f6
574 ld [%o1 + 0x0c], %f7
575 ldd [%o0 + 0xe0], %f8
576 ldd [%o0 + 0xe8], %f10
577 ldd [%o0 + 0xd8], %f12
578 ldd [%o0 + 0xd0], %f14
579 ldd [%o0 + 0xc8], %f16
580 fxor %f8, %f4, %f4
581 ldd [%o0 + 0xc0], %f18
582 fxor %f10, %f6, %f6
583 ldd [%o0 + 0xb8], %f20
584 AES_DROUND23(12, 4, 6, 2)
585 ldd [%o0 + 0xb0], %f22
586 AES_DROUND01(14, 4, 6, 0)
587 ldd [%o0 + 0xa8], %f24
588 AES_DROUND23(16, 0, 2, 6)
589 ldd [%o0 + 0xa0], %f26
590 AES_DROUND01(18, 0, 2, 4)
591 ldd [%o0 + 0x98], %f12
592 AES_DROUND23(20, 4, 6, 2)
593 ldd [%o0 + 0x90], %f14
594 AES_DROUND01(22, 4, 6, 0)
595 ldd [%o0 + 0x88], %f16
596 AES_DROUND23(24, 0, 2, 6)
597 ldd [%o0 + 0x80], %f18
598 AES_DROUND01(26, 0, 2, 4)
599 ldd [%o0 + 0x78], %f20
600 AES_DROUND23(12, 4, 6, 2)
601 ldd [%o0 + 0x70], %f22
602 AES_DROUND01(14, 4, 6, 0)
603 ldd [%o0 + 0x68], %f24
604 AES_DROUND23(16, 0, 2, 6)
605 ldd [%o0 + 0x60], %f26
606 AES_DROUND01(18, 0, 2, 4)
607 ldd [%o0 + 0x58], %f28
608 AES_DROUND23(20, 4, 6, 2)
609 ldd [%o0 + 0x50], %f30
610 AES_DROUND01(22, 4, 6, 0)
611 ldd [%o0 + 0x48], %f32
612 AES_DROUND23(24, 0, 2, 6)
613 ldd [%o0 + 0x40], %f34
614 AES_DROUND01(26, 0, 2, 4)
615 ldd [%o0 + 0x38], %f36
616 AES_DROUND23(28, 4, 6, 2)
617 ldd [%o0 + 0x30], %f38
618 AES_DROUND01(30, 4, 6, 0)
619 ldd [%o0 + 0x28], %f40
620 AES_DROUND23(32, 0, 2, 6)
621 ldd [%o0 + 0x20], %f42
622 AES_DROUND01(34, 0, 2, 4)
623 ldd [%o0 + 0x18], %f44
624 AES_DROUND23(36, 4, 6, 2)
625 ldd [%o0 + 0x10], %f46
626 AES_DROUND01(38, 4, 6, 0)
627 ldd [%o0 + 0x08], %f48
628 AES_DROUND23(40, 0, 2, 6)
629 ldd [%o0 + 0x00], %f50
630 AES_DROUND01(42, 0, 2, 4)
631 AES_DROUND23(44, 4, 6, 2)
632 AES_DROUND01(46, 4, 6, 0)
633 AES_DROUND23_L(48, 0, 2, 6)
634 AES_DROUND01_L(50, 0, 2, 4)
635 st %f4, [%o2 + 0x00]
636 st %f5, [%o2 + 0x04]
637 st %f6, [%o2 + 0x08]
638 st %f7, [%o2 + 0x0c]
639 retl
640 VISExit
641ENDPROC(aes_sparc64_decrypt_256)
642
643 .align 32
644ENTRY(aes_sparc64_load_encrypt_keys_128)
455 /* %o0=key */ 645 /* %o0=key */
456 ba,pt %xcc, aes_sparc64_load_encrypt_keys 646 VISEntry
457 sub %o0, 0x10, %o0 647 ldd [%o0 + 0x10], %f8
458ENDPROC(aes_sparc64_load_decrypt_keys) 648 ldd [%o0 + 0x18], %f10
649 ldd [%o0 + 0x20], %f12
650 ldd [%o0 + 0x28], %f14
651 ldd [%o0 + 0x30], %f16
652 ldd [%o0 + 0x38], %f18
653 ldd [%o0 + 0x40], %f20
654 ldd [%o0 + 0x48], %f22
655 ldd [%o0 + 0x50], %f24
656 ldd [%o0 + 0x58], %f26
657 ldd [%o0 + 0x60], %f28
658 ldd [%o0 + 0x68], %f30
659 ldd [%o0 + 0x70], %f32
660 ldd [%o0 + 0x78], %f34
661 ldd [%o0 + 0x80], %f36
662 ldd [%o0 + 0x88], %f38
663 ldd [%o0 + 0x90], %f40
664 ldd [%o0 + 0x98], %f42
665 ldd [%o0 + 0xa0], %f44
666 retl
667 ldd [%o0 + 0xa8], %f46
668ENDPROC(aes_sparc64_load_encrypt_keys_128)
459 669
460ENTRY(aes_sparc64_load_encrypt_keys) 670 .align 32
671ENTRY(aes_sparc64_load_encrypt_keys_192)
672 /* %o0=key */
673 VISEntry
674 ldd [%o0 + 0x10], %f8
675 ldd [%o0 + 0x18], %f10
676 ldd [%o0 + 0x20], %f12
677 ldd [%o0 + 0x28], %f14
678 ldd [%o0 + 0x30], %f16
679 ldd [%o0 + 0x38], %f18
680 ldd [%o0 + 0x40], %f20
681 ldd [%o0 + 0x48], %f22
682 ldd [%o0 + 0x50], %f24
683 ldd [%o0 + 0x58], %f26
684 ldd [%o0 + 0x60], %f28
685 ldd [%o0 + 0x68], %f30
686 ldd [%o0 + 0x70], %f32
687 ldd [%o0 + 0x78], %f34
688 ldd [%o0 + 0x80], %f36
689 ldd [%o0 + 0x88], %f38
690 ldd [%o0 + 0x90], %f40
691 ldd [%o0 + 0x98], %f42
692 ldd [%o0 + 0xa0], %f44
693 ldd [%o0 + 0xa8], %f46
694 ldd [%o0 + 0xb0], %f48
695 ldd [%o0 + 0xb8], %f50
696 ldd [%o0 + 0xc0], %f52
697 retl
698 ldd [%o0 + 0xc8], %f54
699ENDPROC(aes_sparc64_load_encrypt_keys_192)
700
701 .align 32
702ENTRY(aes_sparc64_load_encrypt_keys_256)
461 /* %o0=key */ 703 /* %o0=key */
462 VISEntry 704 VISEntry
463 ldd [%o0 + 0x10], %f8 705 ldd [%o0 + 0x10], %f8
@@ -489,171 +731,241 @@ ENTRY(aes_sparc64_load_encrypt_keys)
489 ldd [%o0 + 0xe0], %f60 731 ldd [%o0 + 0xe0], %f60
490 retl 732 retl
491 ldd [%o0 + 0xe8], %f62 733 ldd [%o0 + 0xe8], %f62
492ENDPROC(aes_sparc64_load_encrypt_keys) 734ENDPROC(aes_sparc64_load_encrypt_keys_256)
735
736 .align 32
737ENTRY(aes_sparc64_load_decrypt_keys_128)
738 /* %o0=key */
739 VISEntry
740 ldd [%o0 + 0x98], %f8
741 ldd [%o0 + 0x90], %f10
742 ldd [%o0 + 0x88], %f12
743 ldd [%o0 + 0x80], %f14
744 ldd [%o0 + 0x78], %f16
745 ldd [%o0 + 0x70], %f18
746 ldd [%o0 + 0x68], %f20
747 ldd [%o0 + 0x60], %f22
748 ldd [%o0 + 0x58], %f24
749 ldd [%o0 + 0x50], %f26
750 ldd [%o0 + 0x48], %f28
751 ldd [%o0 + 0x40], %f30
752 ldd [%o0 + 0x38], %f32
753 ldd [%o0 + 0x30], %f34
754 ldd [%o0 + 0x28], %f36
755 ldd [%o0 + 0x20], %f38
756 ldd [%o0 + 0x18], %f40
757 ldd [%o0 + 0x10], %f42
758 ldd [%o0 + 0x08], %f44
759 retl
760 ldd [%o0 + 0x00], %f46
761ENDPROC(aes_sparc64_load_decrypt_keys_128)
493 762
494ENTRY(aes_sparc64_ecb_encrypt) 763 .align 32
495 /* %o0=key, %o1=input, %o2=output, %o3=key_len, %o4=len */ 764ENTRY(aes_sparc64_load_decrypt_keys_192)
765 /* %o0=key */
766 VISEntry
767 ldd [%o0 + 0xb8], %f8
768 ldd [%o0 + 0xb0], %f10
769 ldd [%o0 + 0xa8], %f12
770 ldd [%o0 + 0xa0], %f14
771 ldd [%o0 + 0x98], %f16
772 ldd [%o0 + 0x90], %f18
773 ldd [%o0 + 0x88], %f20
774 ldd [%o0 + 0x80], %f22
775 ldd [%o0 + 0x78], %f24
776 ldd [%o0 + 0x70], %f26
777 ldd [%o0 + 0x68], %f28
778 ldd [%o0 + 0x60], %f30
779 ldd [%o0 + 0x58], %f32
780 ldd [%o0 + 0x50], %f34
781 ldd [%o0 + 0x48], %f36
782 ldd [%o0 + 0x40], %f38
783 ldd [%o0 + 0x38], %f40
784 ldd [%o0 + 0x30], %f42
785 ldd [%o0 + 0x28], %f44
786 ldd [%o0 + 0x20], %f46
787 ldd [%o0 + 0x18], %f48
788 ldd [%o0 + 0x10], %f50
789 ldd [%o0 + 0x08], %f52
790 retl
791 ldd [%o0 + 0x00], %f54
792ENDPROC(aes_sparc64_load_decrypt_keys_192)
793
794 .align 32
795ENTRY(aes_sparc64_load_decrypt_keys_256)
796 /* %o0=key */
797 VISEntry
798 ldd [%o0 + 0xd8], %f8
799 ldd [%o0 + 0xd0], %f10
800 ldd [%o0 + 0xc8], %f12
801 ldd [%o0 + 0xc0], %f14
802 ldd [%o0 + 0xb8], %f16
803 ldd [%o0 + 0xb0], %f18
804 ldd [%o0 + 0xa8], %f20
805 ldd [%o0 + 0xa0], %f22
806 ldd [%o0 + 0x98], %f24
807 ldd [%o0 + 0x90], %f26
808 ldd [%o0 + 0x88], %f28
809 ldd [%o0 + 0x80], %f30
810 ldd [%o0 + 0x78], %f32
811 ldd [%o0 + 0x70], %f34
812 ldd [%o0 + 0x68], %f36
813 ldd [%o0 + 0x60], %f38
814 ldd [%o0 + 0x58], %f40
815 ldd [%o0 + 0x50], %f42
816 ldd [%o0 + 0x48], %f44
817 ldd [%o0 + 0x40], %f46
818 ldd [%o0 + 0x38], %f48
819 ldd [%o0 + 0x30], %f50
820 ldd [%o0 + 0x28], %f52
821 ldd [%o0 + 0x20], %f54
822 ldd [%o0 + 0x18], %f56
823 ldd [%o0 + 0x10], %f58
824 ldd [%o0 + 0x08], %f60
825 retl
826 ldd [%o0 + 0x00], %f62
827ENDPROC(aes_sparc64_load_decrypt_keys_256)
828
829 .align 32
830ENTRY(aes_sparc64_ecb_encrypt_128)
831 /* %o0=key, %o1=input, %o2=output, %o3=len */
496 ldx [%o0 + 0x00], %g1 832 ldx [%o0 + 0x00], %g1
497 ldx [%o0 + 0x08], %g2 833 ldx [%o0 + 0x08], %g2
498 cmp %o3, 24 8341: ldx [%o1 + 0x00], %g3
499 bl 2f
500 nop
501 be 1f
502 nop
503
5040:
505 /* 256-bit key */
506 ldx [%o1 + 0x00], %g3
507 ldx [%o1 + 0x08], %g7 835 ldx [%o1 + 0x08], %g7
508 add %o1, 0x10, %o1 836 add %o1, 0x10, %o1
509 xor %g1, %g3, %g3 837 xor %g1, %g3, %g3
510 xor %g2, %g7, %g7 838 xor %g2, %g7, %g7
511 MOVXTOD_G3_F4 839 MOVXTOD_G3_F4
512 MOVXTOD_G7_F6 840 MOVXTOD_G7_F6
513 841 ENCRYPT_128(8, 4, 6, 0, 2)
514 ENCRYPT_256(8, 4, 6, 0, 2)
515
516 std %f4, [%o2 + 0x00] 842 std %f4, [%o2 + 0x00]
517 std %f6, [%o2 + 0x08] 843 std %f6, [%o2 + 0x08]
518 subcc %o4, 0x10, %o4 844 subcc %o3, 0x10, %o3
519 bne,pt %xcc, 0b 845 bne,pt %xcc, 1b
520 add %o2, 0x10, %o2 846 add %o2, 0x10, %o2
521
522 retl 847 retl
523 nop 848 nop
849ENDPROC(aes_sparc64_ecb_encrypt_128)
524 850
5251: 851 .align 32
526 /* 192-bit key */ 852ENTRY(aes_sparc64_ecb_encrypt_192)
527 ldx [%o1 + 0x00], %g3 853 /* %o0=key, %o1=input, %o2=output, %o3=len */
854 ldx [%o0 + 0x00], %g1
855 ldx [%o0 + 0x08], %g2
8561: ldx [%o1 + 0x00], %g3
528 ldx [%o1 + 0x08], %g7 857 ldx [%o1 + 0x08], %g7
529 add %o1, 0x10, %o1 858 add %o1, 0x10, %o1
530 xor %g1, %g3, %g3 859 xor %g1, %g3, %g3
531 xor %g2, %g7, %g7 860 xor %g2, %g7, %g7
532 MOVXTOD_G3_F4 861 MOVXTOD_G3_F4
533 MOVXTOD_G7_F6 862 MOVXTOD_G7_F6
534
535 ENCRYPT_192(8, 4, 6, 0, 2) 863 ENCRYPT_192(8, 4, 6, 0, 2)
536
537 std %f4, [%o2 + 0x00] 864 std %f4, [%o2 + 0x00]
538 std %f6, [%o2 + 0x08] 865 std %f6, [%o2 + 0x08]
539 subcc %o4, 0x10, %o4 866 subcc %o3, 0x10, %o3
540 bne,pt %xcc, 1b 867 bne,pt %xcc, 1b
541 add %o2, 0x10, %o2 868 add %o2, 0x10, %o2
542
543 retl 869 retl
544 nop 870 nop
871ENDPROC(aes_sparc64_ecb_encrypt_192)
545 872
5462: 873 .align 32
547 /* 128-bit key */ 874ENTRY(aes_sparc64_ecb_encrypt_256)
548 ldx [%o1 + 0x00], %g3 875 /* %o0=key, %o1=input, %o2=output, %o3=len */
876 ldx [%o0 + 0x00], %g1
877 ldx [%o0 + 0x08], %g2
8781: ldx [%o1 + 0x00], %g3
549 ldx [%o1 + 0x08], %g7 879 ldx [%o1 + 0x08], %g7
550 add %o1, 0x10, %o1 880 add %o1, 0x10, %o1
551 xor %g1, %g3, %g3 881 xor %g1, %g3, %g3
552 xor %g2, %g7, %g7 882 xor %g2, %g7, %g7
553 MOVXTOD_G3_F4 883 MOVXTOD_G3_F4
554 MOVXTOD_G7_F6 884 MOVXTOD_G7_F6
555 885 ENCRYPT_256(8, 4, 6, 0, 2)
556 ENCRYPT_128(8, 4, 6, 0, 2)
557
558 std %f4, [%o2 + 0x00] 886 std %f4, [%o2 + 0x00]
559 std %f6, [%o2 + 0x08] 887 std %f6, [%o2 + 0x08]
560 subcc %o4, 0x10, %o4 888 subcc %o3, 0x10, %o3
561 bne,pt %xcc, 2b 889 bne,pt %xcc, 1b
562 add %o2, 0x10, %o2 890 add %o2, 0x10, %o2
563
564 retl 891 retl
565 nop 892 nop
566ENDPROC(aes_sparc64_ecb_encrypt) 893ENDPROC(aes_sparc64_ecb_encrypt_256)
567 894
568ENTRY(aes_sparc64_ecb_decrypt) 895 .align 32
569 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=key_len, %o4=len, %o5=iv */ 896ENTRY(aes_sparc64_ecb_decrypt_128)
897 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
570 ldx [%o0 - 0x10], %g1 898 ldx [%o0 - 0x10], %g1
571 ldx [%o0 - 0x08], %g2 899 ldx [%o0 - 0x08], %g2
572 cmp %o3, 24 9001: ldx [%o1 + 0x00], %g3
573 bl 2f
574 nop
575 be 1f
576 nop
577
5780:
579 /* 256-bit key */
580 ldx [%o1 + 0x00], %g3
581 ldx [%o1 + 0x08], %g7 901 ldx [%o1 + 0x08], %g7
582 add %o1, 0x10, %o1 902 add %o1, 0x10, %o1
583 xor %g1, %g3, %g3 903 xor %g1, %g3, %g3
584 xor %g2, %g7, %g7 904 xor %g2, %g7, %g7
585 MOVXTOD_G3_F4 905 MOVXTOD_G3_F4
586 MOVXTOD_G7_F6 906 MOVXTOD_G7_F6
587 907 DECRYPT_128(8, 4, 6, 0, 2)
588 DECRYPT_256(64, 4, 6, 0, 2)
589
590 std %f4, [%o2 + 0x00] 908 std %f4, [%o2 + 0x00]
591 std %f6, [%o2 + 0x08] 909 std %f6, [%o2 + 0x08]
592 subcc %o4, 0x10, %o4 910 subcc %o3, 0x10, %o3
593 bne,pt %xcc, 0b 911 bne,pt %xcc, 1b
594 add %o2, 0x10, %o2 912 add %o2, 0x10, %o2
595
596 retl 913 retl
597 nop 914 nop
915ENDPROC(aes_sparc64_ecb_decrypt_128)
598 916
5991: 917 .align 32
600 /* 192-bit key */ 918ENTRY(aes_sparc64_ecb_decrypt_192)
601 ldx [%o1 + 0x00], %g3 919 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
920 ldx [%o0 - 0x10], %g1
921 ldx [%o0 - 0x08], %g2
9221: ldx [%o1 + 0x00], %g3
602 ldx [%o1 + 0x08], %g7 923 ldx [%o1 + 0x08], %g7
603 add %o1, 0x10, %o1 924 add %o1, 0x10, %o1
604 xor %g1, %g3, %g3 925 xor %g1, %g3, %g3
605 xor %g2, %g7, %g7 926 xor %g2, %g7, %g7
606 MOVXTOD_G3_F4 927 MOVXTOD_G3_F4
607 MOVXTOD_G7_F6 928 MOVXTOD_G7_F6
608 929 DECRYPT_192(8, 4, 6, 0, 2)
609 DECRYPT_192(56, 4, 6, 0, 2)
610
611 std %f4, [%o2 + 0x00] 930 std %f4, [%o2 + 0x00]
612 std %f6, [%o2 + 0x08] 931 std %f6, [%o2 + 0x08]
613 subcc %o4, 0x10, %o4 932 subcc %o3, 0x10, %o3
614 bne,pt %xcc, 1b 933 bne,pt %xcc, 1b
615 add %o2, 0x10, %o2 934 add %o2, 0x10, %o2
616
617 retl 935 retl
618 nop 936 nop
937ENDPROC(aes_sparc64_ecb_decrypt_192)
619 938
6202: 939 .align 32
621 /* 128-bit key */ 940ENTRY(aes_sparc64_ecb_decrypt_256)
622 ldx [%o1 + 0x00], %g3 941 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
942 ldx [%o0 - 0x10], %g1
943 ldx [%o0 - 0x08], %g2
9441: ldx [%o1 + 0x00], %g3
623 ldx [%o1 + 0x08], %g7 945 ldx [%o1 + 0x08], %g7
624 add %o1, 0x10, %o1 946 add %o1, 0x10, %o1
625 xor %g1, %g3, %g3 947 xor %g1, %g3, %g3
626 xor %g2, %g7, %g7 948 xor %g2, %g7, %g7
627 MOVXTOD_G3_F4 949 MOVXTOD_G3_F4
628 MOVXTOD_G7_F6 950 MOVXTOD_G7_F6
629 951 DECRYPT_256(8, 4, 6, 0, 2)
630 DECRYPT_128(48, 4, 6, 0, 2)
631
632 std %f4, [%o2 + 0x00] 952 std %f4, [%o2 + 0x00]
633 std %f6, [%o2 + 0x08] 953 std %f6, [%o2 + 0x08]
634 subcc %o4, 0x10, %o4 954 subcc %o3, 0x10, %o3
635 bne,pt %xcc, 2b 955 bne,pt %xcc, 1b
636 add %o2, 0x10, %o2 956 add %o2, 0x10, %o2
637
638 retl 957 retl
639 nop 958 nop
640ENDPROC(aes_sparc64_ecb_decrypt) 959ENDPROC(aes_sparc64_ecb_decrypt_256)
641 960
642ENTRY(aes_sparc64_cbc_encrypt) 961 .align 32
643 /* %o0=key, %o1=input, %o2=output, %o3=key_len, %o4=len */ 962ENTRY(aes_sparc64_cbc_encrypt_128)
644 ldd [%o5 + 0x00], %f4 963 /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
645 ldd [%o5 + 0x08], %f6 964 ldd [%o4 + 0x00], %f4
965 ldd [%o4 + 0x08], %f6
646 ldx [%o0 + 0x00], %g1 966 ldx [%o0 + 0x00], %g1
647 ldx [%o0 + 0x08], %g2 967 ldx [%o0 + 0x08], %g2
648 cmp %o3, 24 9681: ldx [%o1 + 0x00], %g3
649 bl 2f
650 nop
651 be 1f
652 nop
653
6540:
655 /* 256-bit key */
656 ldx [%o1 + 0x00], %g3
657 ldx [%o1 + 0x08], %g7 969 ldx [%o1 + 0x08], %g7
658 add %o1, 0x10, %o1 970 add %o1, 0x10, %o1
659 xor %g1, %g3, %g3 971 xor %g1, %g3, %g3
@@ -662,24 +974,26 @@ ENTRY(aes_sparc64_cbc_encrypt)
662 MOVXTOD_G7_F2 974 MOVXTOD_G7_F2
663 fxor %f4, %f0, %f4 975 fxor %f4, %f0, %f4
664 fxor %f6, %f2, %f6 976 fxor %f6, %f2, %f6
665 977 ENCRYPT_128(8, 4, 6, 0, 2)
666 ENCRYPT_256(8, 4, 6, 0, 2)
667
668 std %f4, [%o2 + 0x00] 978 std %f4, [%o2 + 0x00]
669 std %f6, [%o2 + 0x08] 979 std %f6, [%o2 + 0x08]
670 subcc %o4, 0x10, %o4 980 subcc %o3, 0x10, %o3
671 bne,pt %xcc, 0b 981 bne,pt %xcc, 1b
672 add %o2, 0x10, %o2 982 add %o2, 0x10, %o2
673 983 std %f4, [%o4 + 0x00]
674 std %f4, [%o5 + 0x00] 984 std %f6, [%o4 + 0x08]
675 std %f6, [%o5 + 0x08]
676
677 retl 985 retl
678 nop 986 nop
987ENDPROC(aes_sparc64_cbc_encrypt_128)
679 988
6801: 989 .align 32
681 /* 192-bit key */ 990ENTRY(aes_sparc64_cbc_encrypt_192)
682 ldx [%o1 + 0x00], %g3 991 /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
992 ldd [%o4 + 0x00], %f4
993 ldd [%o4 + 0x08], %f6
994 ldx [%o0 + 0x00], %g1
995 ldx [%o0 + 0x08], %g2
9961: ldx [%o1 + 0x00], %g3
683 ldx [%o1 + 0x08], %g7 997 ldx [%o1 + 0x08], %g7
684 add %o1, 0x10, %o1 998 add %o1, 0x10, %o1
685 xor %g1, %g3, %g3 999 xor %g1, %g3, %g3
@@ -688,24 +1002,26 @@ ENTRY(aes_sparc64_cbc_encrypt)
688 MOVXTOD_G7_F2 1002 MOVXTOD_G7_F2
689 fxor %f4, %f0, %f4 1003 fxor %f4, %f0, %f4
690 fxor %f6, %f2, %f6 1004 fxor %f6, %f2, %f6
691
692 ENCRYPT_192(8, 4, 6, 0, 2) 1005 ENCRYPT_192(8, 4, 6, 0, 2)
693
694 std %f4, [%o2 + 0x00] 1006 std %f4, [%o2 + 0x00]
695 std %f6, [%o2 + 0x08] 1007 std %f6, [%o2 + 0x08]
696 subcc %o4, 0x10, %o4 1008 subcc %o3, 0x10, %o3
697 bne,pt %xcc, 1b 1009 bne,pt %xcc, 1b
698 add %o2, 0x10, %o2 1010 add %o2, 0x10, %o2
699 1011 std %f4, [%o4 + 0x00]
700 std %f4, [%o5 + 0x00] 1012 std %f6, [%o4 + 0x08]
701 std %f6, [%o5 + 0x08]
702
703 retl 1013 retl
704 nop 1014 nop
1015ENDPROC(aes_sparc64_cbc_encrypt_192)
705 1016
7062: 1017 .align 32
707 /* 128-bit key */ 1018ENTRY(aes_sparc64_cbc_encrypt_256)
708 ldx [%o1 + 0x00], %g3 1019 /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
1020 ldd [%o4 + 0x00], %f4
1021 ldd [%o4 + 0x08], %f6
1022 ldx [%o0 + 0x00], %g1
1023 ldx [%o0 + 0x08], %g2
10241: ldx [%o1 + 0x00], %g3
709 ldx [%o1 + 0x08], %g7 1025 ldx [%o1 + 0x08], %g7
710 add %o1, 0x10, %o1 1026 add %o1, 0x10, %o1
711 xor %g1, %g3, %g3 1027 xor %g1, %g3, %g3
@@ -714,123 +1030,110 @@ ENTRY(aes_sparc64_cbc_encrypt)
714 MOVXTOD_G7_F2 1030 MOVXTOD_G7_F2
715 fxor %f4, %f0, %f4 1031 fxor %f4, %f0, %f4
716 fxor %f6, %f2, %f6 1032 fxor %f6, %f2, %f6
717 1033 ENCRYPT_256(8, 4, 6, 0, 2)
718 ENCRYPT_128(8, 4, 6, 0, 2)
719
720 std %f4, [%o2 + 0x00] 1034 std %f4, [%o2 + 0x00]
721 std %f6, [%o2 + 0x08] 1035 std %f6, [%o2 + 0x08]
722 subcc %o4, 0x10, %o4 1036 subcc %o3, 0x10, %o3
723 bne,pt %xcc, 2b 1037 bne,pt %xcc, 1b
724 add %o2, 0x10, %o2 1038 add %o2, 0x10, %o2
725 1039 std %f4, [%o4 + 0x00]
726 std %f4, [%o5 + 0x00] 1040 std %f6, [%o4 + 0x08]
727 std %f6, [%o5 + 0x08]
728
729 retl 1041 retl
730 nop 1042 nop
731ENDPROC(aes_sparc64_cbc_encrypt) 1043ENDPROC(aes_sparc64_cbc_encrypt_256)
732 1044
733ENTRY(aes_sparc64_cbc_decrypt) 1045 .align 32
734 /* %o0=&key[key_len], %o1=key_len, %o2=input, %o3=output, %o4=len, %o5=iv */ 1046ENTRY(aes_sparc64_cbc_decrypt_128)
1047 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
735 ldx [%o0 - 0x10], %g1 1048 ldx [%o0 - 0x10], %g1
736 ldx [%o0 - 0x08], %g2 1049 ldx [%o0 - 0x08], %g2
737 cmp %o1, 24 1050 ldx [%o4 + 0x00], %o0
738 ldx [%o5 + 0x00], %o0 1051 ldx [%o4 + 0x08], %o5
739 bl 2f 10521: ldx [%o1 + 0x00], %g3
740 ldx [%o5 + 0x08], %o1 1053 ldx [%o1 + 0x08], %g7
741 be 1f 1054 add %o1, 0x10, %o1
742 nop
743
7440:
745 /* 256-bit key */
746 ldx [%o2 + 0x00], %g3
747 ldx [%o2 + 0x08], %g7
748 add %o2, 0x10, %o2
749 xor %g1, %g3, %g3 1055 xor %g1, %g3, %g3
750 xor %g2, %g7, %g7 1056 xor %g2, %g7, %g7
751 MOVXTOD_G3_F4 1057 MOVXTOD_G3_F4
752 MOVXTOD_G7_F6 1058 MOVXTOD_G7_F6
753 1059 DECRYPT_128(8, 4, 6, 0, 2)
754 DECRYPT_256(64, 4, 6, 0, 2)
755
756 MOVXTOD_O0_F0 1060 MOVXTOD_O0_F0
757 MOVXTOD_O1_F2 1061 MOVXTOD_O5_F2
758 xor %g1, %g3, %o0 1062 xor %g1, %g3, %o0
759 xor %g2, %g7, %o1 1063 xor %g2, %g7, %o5
760 fxor %f4, %f0, %f4 1064 fxor %f4, %f0, %f4
761 fxor %f6, %f2, %f6 1065 fxor %f6, %f2, %f6
762 1066 std %f4, [%o2 + 0x00]
763 std %f4, [%o3 + 0x00] 1067 std %f6, [%o2 + 0x08]
764 std %f6, [%o3 + 0x08] 1068 subcc %o3, 0x10, %o3
765 subcc %o4, 0x10, %o4 1069 bne,pt %xcc, 1b
766 bne,pt %xcc, 0b 1070 add %o2, 0x10, %o2
767 add %o3, 0x10, %o3 1071 stx %o0, [%o4 + 0x00]
768 1072 stx %o5, [%o4 + 0x08]
769 stx %o0, [%o5 + 0x00]
770 stx %o1, [%o5 + 0x08]
771
772 retl 1073 retl
773 nop 1074 nop
1075ENDPROC(aes_sparc64_cbc_decrypt_128)
774 1076
7751: 1077 .align 32
776 /* 192-bit key */ 1078ENTRY(aes_sparc64_cbc_decrypt_192)
777 ldx [%o2 + 0x00], %g3 1079 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
778 ldx [%o2 + 0x08], %g7 1080 ldx [%o0 - 0x10], %g1
779 add %o2, 0x10, %o2 1081 ldx [%o0 - 0x08], %g2
1082 ldx [%o4 + 0x00], %o0
1083 ldx [%o4 + 0x08], %o5
10841: ldx [%o1 + 0x00], %g3
1085 ldx [%o1 + 0x08], %g7
1086 add %o1, 0x10, %o1
780 xor %g1, %g3, %g3 1087 xor %g1, %g3, %g3
781 xor %g2, %g7, %g7 1088 xor %g2, %g7, %g7
782 MOVXTOD_G3_F4 1089 MOVXTOD_G3_F4
783 MOVXTOD_G7_F6 1090 MOVXTOD_G7_F6
784 1091 DECRYPT_192(8, 4, 6, 0, 2)
785 DECRYPT_192(56, 4, 6, 0, 2)
786
787 MOVXTOD_O0_F0 1092 MOVXTOD_O0_F0
788 MOVXTOD_O1_F2 1093 MOVXTOD_O5_F2
789 xor %g1, %g3, %o0 1094 xor %g1, %g3, %o0
790 xor %g2, %g7, %o1 1095 xor %g2, %g7, %o5
791 fxor %f4, %f0, %f4 1096 fxor %f4, %f0, %f4
792 fxor %f6, %f2, %f6 1097 fxor %f6, %f2, %f6
793 1098 std %f4, [%o2 + 0x00]
794 std %f4, [%o3 + 0x00] 1099 std %f6, [%o2 + 0x08]
795 std %f6, [%o3 + 0x08] 1100 subcc %o3, 0x10, %o3
796 subcc %o4, 0x10, %o4
797 bne,pt %xcc, 1b 1101 bne,pt %xcc, 1b
798 add %o3, 0x10, %o3 1102 add %o2, 0x10, %o2
799 1103 stx %o0, [%o4 + 0x00]
800 stx %o0, [%o5 + 0x00] 1104 stx %o5, [%o4 + 0x08]
801 stx %o1, [%o5 + 0x08]
802
803 retl 1105 retl
804 nop 1106 nop
1107ENDPROC(aes_sparc64_cbc_decrypt_192)
805 1108
8062: 1109 .align 32
807 /* 128-bit key */ 1110ENTRY(aes_sparc64_cbc_decrypt_256)
808 ldx [%o2 + 0x00], %g3 1111 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
809 ldx [%o2 + 0x08], %g7 1112 ldx [%o0 - 0x10], %g1
810 add %o2, 0x10, %o2 1113 ldx [%o0 - 0x08], %g2
1114 ldx [%o4 + 0x00], %o0
1115 ldx [%o4 + 0x08], %o5
11161: ldx [%o1 + 0x00], %g3
1117 ldx [%o1 + 0x08], %g7
1118 add %o1, 0x10, %o1
811 xor %g1, %g3, %g3 1119 xor %g1, %g3, %g3
812 xor %g2, %g7, %g7 1120 xor %g2, %g7, %g7
813 MOVXTOD_G3_F4 1121 MOVXTOD_G3_F4
814 MOVXTOD_G7_F6 1122 MOVXTOD_G7_F6
815 1123 DECRYPT_256(8, 4, 6, 0, 2)
816 DECRYPT_128(48, 4, 6, 0, 2)
817
818 MOVXTOD_O0_F0 1124 MOVXTOD_O0_F0
819 MOVXTOD_O1_F2 1125 MOVXTOD_O5_F2
820 xor %g1, %g3, %o0 1126 xor %g1, %g3, %o0
821 xor %g2, %g7, %o1 1127 xor %g2, %g7, %o5
822 fxor %f4, %f0, %f4 1128 fxor %f4, %f0, %f4
823 fxor %f6, %f2, %f6 1129 fxor %f6, %f2, %f6
824 1130 std %f4, [%o2 + 0x00]
825 std %f4, [%o3 + 0x00] 1131 std %f6, [%o2 + 0x08]
826 std %f6, [%o3 + 0x08] 1132 subcc %o3, 0x10, %o3
827 subcc %o4, 0x10, %o4 1133 bne,pt %xcc, 1b
828 bne,pt %xcc, 2b 1134 add %o2, 0x10, %o2
829 add %o3, 0x10, %o3 1135 stx %o0, [%o4 + 0x00]
830 1136 stx %o5, [%o4 + 0x08]
831 stx %o0, [%o5 + 0x00]
832 stx %o1, [%o5 + 0x08]
833
834 retl 1137 retl
835 nop 1138 nop
836ENDPROC(aes_sparc64_cbc_decrypt) 1139ENDPROC(aes_sparc64_cbc_decrypt_256)
diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
index a87c5fa76e20..0b1de0b470a2 100644
--- a/arch/sparc/crypto/aes_glue.c
+++ b/arch/sparc/crypto/aes_glue.c
@@ -26,12 +26,121 @@
26#include <asm/pstate.h> 26#include <asm/pstate.h>
27#include <asm/elf.h> 27#include <asm/elf.h>
28 28
29struct aes_ops {
30 void (*encrypt)(const u64 *key, const u32 *input, u32 *output);
31 void (*decrypt)(const u64 *key, const u32 *input, u32 *output);
32 void (*load_encrypt_keys)(const u64 *key);
33 void (*load_decrypt_keys)(const u64 *key);
34 void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
35 unsigned int len);
36 void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,
37 unsigned int len);
38 void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,
39 unsigned int len, u64 *iv);
40 void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
41 unsigned int len, u64 *iv);
42};
43
29struct crypto_sparc64_aes_ctx { 44struct crypto_sparc64_aes_ctx {
45 struct aes_ops *ops;
30 u64 key[AES_MAX_KEYLENGTH / sizeof(u64)]; 46 u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];
31 u32 key_length; 47 u32 key_length;
32 u32 expanded_key_length; 48 u32 expanded_key_length;
33}; 49};
34 50
51extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input,
52 u32 *output);
53extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input,
54 u32 *output);
55extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input,
56 u32 *output);
57
58extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input,
59 u32 *output);
60extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input,
61 u32 *output);
62extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input,
63 u32 *output);
64
65extern void aes_sparc64_load_encrypt_keys_128(const u64 *key);
66extern void aes_sparc64_load_encrypt_keys_192(const u64 *key);
67extern void aes_sparc64_load_encrypt_keys_256(const u64 *key);
68
69extern void aes_sparc64_load_decrypt_keys_128(const u64 *key);
70extern void aes_sparc64_load_decrypt_keys_192(const u64 *key);
71extern void aes_sparc64_load_decrypt_keys_256(const u64 *key);
72
73extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input,
74 u64 *output, unsigned int len);
75extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input,
76 u64 *output, unsigned int len);
77extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input,
78 u64 *output, unsigned int len);
79
80extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input,
81 u64 *output, unsigned int len);
82extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input,
83 u64 *output, unsigned int len);
84extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input,
85 u64 *output, unsigned int len);
86
87extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input,
88 u64 *output, unsigned int len,
89 u64 *iv);
90
91extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input,
92 u64 *output, unsigned int len,
93 u64 *iv);
94
95extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input,
96 u64 *output, unsigned int len,
97 u64 *iv);
98
99extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input,
100 u64 *output, unsigned int len,
101 u64 *iv);
102
103extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input,
104 u64 *output, unsigned int len,
105 u64 *iv);
106
107extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input,
108 u64 *output, unsigned int len,
109 u64 *iv);
110
111struct aes_ops aes128_ops = {
112 .encrypt = aes_sparc64_encrypt_128,
113 .decrypt = aes_sparc64_decrypt_128,
114 .load_encrypt_keys = aes_sparc64_load_encrypt_keys_128,
115 .load_decrypt_keys = aes_sparc64_load_decrypt_keys_128,
116 .ecb_encrypt = aes_sparc64_ecb_encrypt_128,
117 .ecb_decrypt = aes_sparc64_ecb_decrypt_128,
118 .cbc_encrypt = aes_sparc64_cbc_encrypt_128,
119 .cbc_decrypt = aes_sparc64_cbc_decrypt_128,
120};
121
122struct aes_ops aes192_ops = {
123 .encrypt = aes_sparc64_encrypt_192,
124 .decrypt = aes_sparc64_decrypt_192,
125 .load_encrypt_keys = aes_sparc64_load_encrypt_keys_192,
126 .load_decrypt_keys = aes_sparc64_load_decrypt_keys_192,
127 .ecb_encrypt = aes_sparc64_ecb_encrypt_192,
128 .ecb_decrypt = aes_sparc64_ecb_decrypt_192,
129 .cbc_encrypt = aes_sparc64_cbc_encrypt_192,
130 .cbc_decrypt = aes_sparc64_cbc_decrypt_192,
131};
132
133struct aes_ops aes256_ops = {
134 .encrypt = aes_sparc64_encrypt_256,
135 .decrypt = aes_sparc64_decrypt_256,
136 .load_encrypt_keys = aes_sparc64_load_encrypt_keys_256,
137 .load_decrypt_keys = aes_sparc64_load_decrypt_keys_256,
138 .ecb_encrypt = aes_sparc64_ecb_encrypt_256,
139 .ecb_decrypt = aes_sparc64_ecb_decrypt_256,
140 .cbc_encrypt = aes_sparc64_cbc_encrypt_256,
141 .cbc_decrypt = aes_sparc64_cbc_decrypt_256,
142};
143
35extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key, 144extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
36 unsigned int key_len); 145 unsigned int key_len);
37 146
@@ -44,14 +153,17 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
44 switch (key_len) { 153 switch (key_len) {
45 case AES_KEYSIZE_128: 154 case AES_KEYSIZE_128:
46 ctx->expanded_key_length = 0xb0; 155 ctx->expanded_key_length = 0xb0;
156 ctx->ops = &aes128_ops;
47 break; 157 break;
48 158
49 case AES_KEYSIZE_192: 159 case AES_KEYSIZE_192:
50 ctx->expanded_key_length = 0xd0; 160 ctx->expanded_key_length = 0xd0;
161 ctx->ops = &aes192_ops;
51 break; 162 break;
52 163
53 case AES_KEYSIZE_256: 164 case AES_KEYSIZE_256:
54 ctx->expanded_key_length = 0xf0; 165 ctx->expanded_key_length = 0xf0;
166 ctx->ops = &aes256_ops;
55 break; 167 break;
56 168
57 default: 169 default:
@@ -65,38 +177,22 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
65 return 0; 177 return 0;
66} 178}
67 179
68extern void aes_sparc64_encrypt(const u64 *key, const u32 *input,
69 u32 *output, unsigned int key_len);
70
71static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 180static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
72{ 181{
73 struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); 182 struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
74 183
75 aes_sparc64_encrypt(&ctx->key[0], (const u32 *) src, 184 ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
76 (u32 *) dst, ctx->key_length);
77} 185}
78 186
79extern void aes_sparc64_decrypt(const u64 *key, const u32 *input,
80 u32 *output, unsigned int key_len,
81 unsigned int expanded_key_len);
82
83static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 187static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
84{ 188{
85 struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); 189 struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
86 190
87 aes_sparc64_decrypt(&ctx->key[0], (const u32 *) src, 191 ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
88 (u32 *) dst, ctx->key_length,
89 ctx->expanded_key_length);
90} 192}
91 193
92extern void aes_sparc64_load_encrypt_keys(u64 *key);
93extern void aes_sparc64_load_decrypt_keys(u64 *key);
94
95#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 194#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1))
96 195
97extern void aes_sparc64_ecb_encrypt(u64 *key, const u32 *input, u32 *output,
98 unsigned int key_len, unsigned int len);
99
100static int ecb_encrypt(struct blkcipher_desc *desc, 196static int ecb_encrypt(struct blkcipher_desc *desc,
101 struct scatterlist *dst, struct scatterlist *src, 197 struct scatterlist *dst, struct scatterlist *src,
102 unsigned int nbytes) 198 unsigned int nbytes)
@@ -108,15 +204,15 @@ static int ecb_encrypt(struct blkcipher_desc *desc,
108 blkcipher_walk_init(&walk, dst, src, nbytes); 204 blkcipher_walk_init(&walk, dst, src, nbytes);
109 err = blkcipher_walk_virt(desc, &walk); 205 err = blkcipher_walk_virt(desc, &walk);
110 206
111 aes_sparc64_load_encrypt_keys(&ctx->key[0]); 207 ctx->ops->load_encrypt_keys(&ctx->key[0]);
112 while ((nbytes = walk.nbytes)) { 208 while ((nbytes = walk.nbytes)) {
113 unsigned int block_len = nbytes & AES_BLOCK_MASK; 209 unsigned int block_len = nbytes & AES_BLOCK_MASK;
114 210
115 if (likely(block_len)) { 211 if (likely(block_len)) {
116 aes_sparc64_ecb_encrypt(&ctx->key[0], 212 ctx->ops->ecb_encrypt(&ctx->key[0],
117 (const u32 *)walk.src.virt.addr, 213 (const u64 *)walk.src.virt.addr,
118 (u32 *) walk.dst.virt.addr, 214 (u64 *) walk.dst.virt.addr,
119 ctx->key_length, block_len); 215 block_len);
120 } 216 }
121 nbytes &= AES_BLOCK_SIZE - 1; 217 nbytes &= AES_BLOCK_SIZE - 1;
122 err = blkcipher_walk_done(desc, &walk, nbytes); 218 err = blkcipher_walk_done(desc, &walk, nbytes);
@@ -125,9 +221,6 @@ static int ecb_encrypt(struct blkcipher_desc *desc,
125 return err; 221 return err;
126} 222}
127 223
128extern void aes_sparc64_ecb_decrypt(u64 *ekey, const u32 *input, u32 *output,
129 unsigned int key_len, unsigned int len);
130
131static int ecb_decrypt(struct blkcipher_desc *desc, 224static int ecb_decrypt(struct blkcipher_desc *desc,
132 struct scatterlist *dst, struct scatterlist *src, 225 struct scatterlist *dst, struct scatterlist *src,
133 unsigned int nbytes) 226 unsigned int nbytes)
@@ -140,14 +233,16 @@ static int ecb_decrypt(struct blkcipher_desc *desc,
140 blkcipher_walk_init(&walk, dst, src, nbytes); 233 blkcipher_walk_init(&walk, dst, src, nbytes);
141 err = blkcipher_walk_virt(desc, &walk); 234 err = blkcipher_walk_virt(desc, &walk);
142 235
143 aes_sparc64_load_decrypt_keys(&ctx->key[0]); 236 ctx->ops->load_decrypt_keys(&ctx->key[0]);
144 key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; 237 key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
145 while ((nbytes = walk.nbytes)) { 238 while ((nbytes = walk.nbytes)) {
146 unsigned int block_len = nbytes & AES_BLOCK_MASK; 239 unsigned int block_len = nbytes & AES_BLOCK_MASK;
147 240
148 aes_sparc64_ecb_decrypt(key_end, (const u32 *) walk.src.virt.addr, 241 if (likely(block_len)) {
149 (u32 *) walk.dst.virt.addr, ctx->key_length, 242 ctx->ops->ecb_decrypt(key_end,
150 block_len); 243 (const u64 *) walk.src.virt.addr,
244 (u64 *) walk.dst.virt.addr, block_len);
245 }
151 nbytes &= AES_BLOCK_SIZE - 1; 246 nbytes &= AES_BLOCK_SIZE - 1;
152 err = blkcipher_walk_done(desc, &walk, nbytes); 247 err = blkcipher_walk_done(desc, &walk, nbytes);
153 } 248 }
@@ -156,10 +251,6 @@ static int ecb_decrypt(struct blkcipher_desc *desc,
156 return err; 251 return err;
157} 252}
158 253
159extern void aes_sparc64_cbc_encrypt(u64 *key, const u32 *input, u32 *output,
160 unsigned int key_len, unsigned int len,
161 u64 *iv);
162
163static int cbc_encrypt(struct blkcipher_desc *desc, 254static int cbc_encrypt(struct blkcipher_desc *desc,
164 struct scatterlist *dst, struct scatterlist *src, 255 struct scatterlist *dst, struct scatterlist *src,
165 unsigned int nbytes) 256 unsigned int nbytes)
@@ -171,16 +262,15 @@ static int cbc_encrypt(struct blkcipher_desc *desc,
171 blkcipher_walk_init(&walk, dst, src, nbytes); 262 blkcipher_walk_init(&walk, dst, src, nbytes);
172 err = blkcipher_walk_virt(desc, &walk); 263 err = blkcipher_walk_virt(desc, &walk);
173 264
174 aes_sparc64_load_encrypt_keys(&ctx->key[0]); 265 ctx->ops->load_encrypt_keys(&ctx->key[0]);
175 while ((nbytes = walk.nbytes)) { 266 while ((nbytes = walk.nbytes)) {
176 unsigned int block_len = nbytes & AES_BLOCK_MASK; 267 unsigned int block_len = nbytes & AES_BLOCK_MASK;
177 268
178 if (likely(block_len)) { 269 if (likely(block_len)) {
179 aes_sparc64_cbc_encrypt(&ctx->key[0], 270 ctx->ops->cbc_encrypt(&ctx->key[0],
180 (const u32 *)walk.src.virt.addr, 271 (const u64 *)walk.src.virt.addr,
181 (u32 *) walk.dst.virt.addr, 272 (u64 *) walk.dst.virt.addr,
182 ctx->key_length, block_len, 273 block_len, (u64 *) walk.iv);
183 (u64 *) walk.iv);
184 } 274 }
185 nbytes &= AES_BLOCK_SIZE - 1; 275 nbytes &= AES_BLOCK_SIZE - 1;
186 err = blkcipher_walk_done(desc, &walk, nbytes); 276 err = blkcipher_walk_done(desc, &walk, nbytes);
@@ -189,10 +279,6 @@ static int cbc_encrypt(struct blkcipher_desc *desc,
189 return err; 279 return err;
190} 280}
191 281
192extern void aes_sparc64_cbc_decrypt(u64 *ekey, unsigned int key_len,
193 const u32 *input, u32 *output,
194 unsigned int len, u64 *iv);
195
196static int cbc_decrypt(struct blkcipher_desc *desc, 282static int cbc_decrypt(struct blkcipher_desc *desc,
197 struct scatterlist *dst, struct scatterlist *src, 283 struct scatterlist *dst, struct scatterlist *src,
198 unsigned int nbytes) 284 unsigned int nbytes)
@@ -205,15 +291,17 @@ static int cbc_decrypt(struct blkcipher_desc *desc,
205 blkcipher_walk_init(&walk, dst, src, nbytes); 291 blkcipher_walk_init(&walk, dst, src, nbytes);
206 err = blkcipher_walk_virt(desc, &walk); 292 err = blkcipher_walk_virt(desc, &walk);
207 293
208 aes_sparc64_load_decrypt_keys(&ctx->key[0]); 294 ctx->ops->load_decrypt_keys(&ctx->key[0]);
209 key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; 295 key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
210 while ((nbytes = walk.nbytes)) { 296 while ((nbytes = walk.nbytes)) {
211 unsigned int block_len = nbytes & AES_BLOCK_MASK; 297 unsigned int block_len = nbytes & AES_BLOCK_MASK;
212 298
213 aes_sparc64_cbc_decrypt(key_end, ctx->key_length, 299 if (likely(block_len)) {
214 (const u32 *) walk.src.virt.addr, 300 ctx->ops->cbc_decrypt(key_end,
215 (u32 *) walk.dst.virt.addr, 301 (const u64 *) walk.src.virt.addr,
216 block_len, (u64 *) walk.iv); 302 (u64 *) walk.dst.virt.addr,
303 block_len, (u64 *) walk.iv);
304 }
217 nbytes &= AES_BLOCK_SIZE - 1; 305 nbytes &= AES_BLOCK_SIZE - 1;
218 err = blkcipher_walk_done(desc, &walk, nbytes); 306 err = blkcipher_walk_done(desc, &walk, nbytes);
219 } 307 }
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-01 04:00:54 -0500