aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/crypto/aesni-intel_glue.c723
1 files changed, 303 insertions, 420 deletions
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index c799352e24fc..20c622016629 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -222,27 +222,6 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
222 } 222 }
223} 223}
224 224
225static struct crypto_alg aesni_alg = {
226 .cra_name = "aes",
227 .cra_driver_name = "aes-aesni",
228 .cra_priority = 300,
229 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
230 .cra_blocksize = AES_BLOCK_SIZE,
231 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
232 .cra_alignmask = 0,
233 .cra_module = THIS_MODULE,
234 .cra_list = LIST_HEAD_INIT(aesni_alg.cra_list),
235 .cra_u = {
236 .cipher = {
237 .cia_min_keysize = AES_MIN_KEY_SIZE,
238 .cia_max_keysize = AES_MAX_KEY_SIZE,
239 .cia_setkey = aes_set_key,
240 .cia_encrypt = aes_encrypt,
241 .cia_decrypt = aes_decrypt
242 }
243 }
244};
245
246static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 225static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
247{ 226{
248 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); 227 struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
@@ -257,27 +236,6 @@ static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
257 aesni_dec(ctx, dst, src); 236 aesni_dec(ctx, dst, src);
258} 237}
259 238
260static struct crypto_alg __aesni_alg = {
261 .cra_name = "__aes-aesni",
262 .cra_driver_name = "__driver-aes-aesni",
263 .cra_priority = 0,
264 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
265 .cra_blocksize = AES_BLOCK_SIZE,
266 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
267 .cra_alignmask = 0,
268 .cra_module = THIS_MODULE,
269 .cra_list = LIST_HEAD_INIT(__aesni_alg.cra_list),
270 .cra_u = {
271 .cipher = {
272 .cia_min_keysize = AES_MIN_KEY_SIZE,
273 .cia_max_keysize = AES_MAX_KEY_SIZE,
274 .cia_setkey = aes_set_key,
275 .cia_encrypt = __aes_encrypt,
276 .cia_decrypt = __aes_decrypt
277 }
278 }
279};
280
281static int ecb_encrypt(struct blkcipher_desc *desc, 239static int ecb_encrypt(struct blkcipher_desc *desc,
282 struct scatterlist *dst, struct scatterlist *src, 240 struct scatterlist *dst, struct scatterlist *src,
283 unsigned int nbytes) 241 unsigned int nbytes)
@@ -326,28 +284,6 @@ static int ecb_decrypt(struct blkcipher_desc *desc,
326 return err; 284 return err;
327} 285}
328 286
329static struct crypto_alg blk_ecb_alg = {
330 .cra_name = "__ecb-aes-aesni",
331 .cra_driver_name = "__driver-ecb-aes-aesni",
332 .cra_priority = 0,
333 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
334 .cra_blocksize = AES_BLOCK_SIZE,
335 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
336 .cra_alignmask = 0,
337 .cra_type = &crypto_blkcipher_type,
338 .cra_module = THIS_MODULE,
339 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
340 .cra_u = {
341 .blkcipher = {
342 .min_keysize = AES_MIN_KEY_SIZE,
343 .max_keysize = AES_MAX_KEY_SIZE,
344 .setkey = aes_set_key,
345 .encrypt = ecb_encrypt,
346 .decrypt = ecb_decrypt,
347 },
348 },
349};
350
351static int cbc_encrypt(struct blkcipher_desc *desc, 287static int cbc_encrypt(struct blkcipher_desc *desc,
352 struct scatterlist *dst, struct scatterlist *src, 288 struct scatterlist *dst, struct scatterlist *src,
353 unsigned int nbytes) 289 unsigned int nbytes)
@@ -396,28 +332,6 @@ static int cbc_decrypt(struct blkcipher_desc *desc,
396 return err; 332 return err;
397} 333}
398 334
399static struct crypto_alg blk_cbc_alg = {
400 .cra_name = "__cbc-aes-aesni",
401 .cra_driver_name = "__driver-cbc-aes-aesni",
402 .cra_priority = 0,
403 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
404 .cra_blocksize = AES_BLOCK_SIZE,
405 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
406 .cra_alignmask = 0,
407 .cra_type = &crypto_blkcipher_type,
408 .cra_module = THIS_MODULE,
409 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
410 .cra_u = {
411 .blkcipher = {
412 .min_keysize = AES_MIN_KEY_SIZE,
413 .max_keysize = AES_MAX_KEY_SIZE,
414 .setkey = aes_set_key,
415 .encrypt = cbc_encrypt,
416 .decrypt = cbc_decrypt,
417 },
418 },
419};
420
421#ifdef CONFIG_X86_64 335#ifdef CONFIG_X86_64
422static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 336static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
423 struct blkcipher_walk *walk) 337 struct blkcipher_walk *walk)
@@ -461,29 +375,6 @@ static int ctr_crypt(struct blkcipher_desc *desc,
461 375
462 return err; 376 return err;
463} 377}
464
465static struct crypto_alg blk_ctr_alg = {
466 .cra_name = "__ctr-aes-aesni",
467 .cra_driver_name = "__driver-ctr-aes-aesni",
468 .cra_priority = 0,
469 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
470 .cra_blocksize = 1,
471 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
472 .cra_alignmask = 0,
473 .cra_type = &crypto_blkcipher_type,
474 .cra_module = THIS_MODULE,
475 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
476 .cra_u = {
477 .blkcipher = {
478 .min_keysize = AES_MIN_KEY_SIZE,
479 .max_keysize = AES_MAX_KEY_SIZE,
480 .ivsize = AES_BLOCK_SIZE,
481 .setkey = aes_set_key,
482 .encrypt = ctr_crypt,
483 .decrypt = ctr_crypt,
484 },
485 },
486};
487#endif 378#endif
488 379
489static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, 380static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
@@ -572,30 +463,6 @@ static int ablk_ecb_init(struct crypto_tfm *tfm)
572 return 0; 463 return 0;
573} 464}
574 465
575static struct crypto_alg ablk_ecb_alg = {
576 .cra_name = "ecb(aes)",
577 .cra_driver_name = "ecb-aes-aesni",
578 .cra_priority = 400,
579 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
580 .cra_blocksize = AES_BLOCK_SIZE,
581 .cra_ctxsize = sizeof(struct async_aes_ctx),
582 .cra_alignmask = 0,
583 .cra_type = &crypto_ablkcipher_type,
584 .cra_module = THIS_MODULE,
585 .cra_list = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
586 .cra_init = ablk_ecb_init,
587 .cra_exit = ablk_exit,
588 .cra_u = {
589 .ablkcipher = {
590 .min_keysize = AES_MIN_KEY_SIZE,
591 .max_keysize = AES_MAX_KEY_SIZE,
592 .setkey = ablk_set_key,
593 .encrypt = ablk_encrypt,
594 .decrypt = ablk_decrypt,
595 },
596 },
597};
598
599static int ablk_cbc_init(struct crypto_tfm *tfm) 466static int ablk_cbc_init(struct crypto_tfm *tfm)
600{ 467{
601 struct cryptd_ablkcipher *cryptd_tfm; 468 struct cryptd_ablkcipher *cryptd_tfm;
@@ -607,31 +474,6 @@ static int ablk_cbc_init(struct crypto_tfm *tfm)
607 return 0; 474 return 0;
608} 475}
609 476
610static struct crypto_alg ablk_cbc_alg = {
611 .cra_name = "cbc(aes)",
612 .cra_driver_name = "cbc-aes-aesni",
613 .cra_priority = 400,
614 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
615 .cra_blocksize = AES_BLOCK_SIZE,
616 .cra_ctxsize = sizeof(struct async_aes_ctx),
617 .cra_alignmask = 0,
618 .cra_type = &crypto_ablkcipher_type,
619 .cra_module = THIS_MODULE,
620 .cra_list = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
621 .cra_init = ablk_cbc_init,
622 .cra_exit = ablk_exit,
623 .cra_u = {
624 .ablkcipher = {
625 .min_keysize = AES_MIN_KEY_SIZE,
626 .max_keysize = AES_MAX_KEY_SIZE,
627 .ivsize = AES_BLOCK_SIZE,
628 .setkey = ablk_set_key,
629 .encrypt = ablk_encrypt,
630 .decrypt = ablk_decrypt,
631 },
632 },
633};
634
635#ifdef CONFIG_X86_64 477#ifdef CONFIG_X86_64
636static int ablk_ctr_init(struct crypto_tfm *tfm) 478static int ablk_ctr_init(struct crypto_tfm *tfm)
637{ 479{
@@ -644,32 +486,6 @@ static int ablk_ctr_init(struct crypto_tfm *tfm)
644 return 0; 486 return 0;
645} 487}
646 488
647static struct crypto_alg ablk_ctr_alg = {
648 .cra_name = "ctr(aes)",
649 .cra_driver_name = "ctr-aes-aesni",
650 .cra_priority = 400,
651 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
652 .cra_blocksize = 1,
653 .cra_ctxsize = sizeof(struct async_aes_ctx),
654 .cra_alignmask = 0,
655 .cra_type = &crypto_ablkcipher_type,
656 .cra_module = THIS_MODULE,
657 .cra_list = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
658 .cra_init = ablk_ctr_init,
659 .cra_exit = ablk_exit,
660 .cra_u = {
661 .ablkcipher = {
662 .min_keysize = AES_MIN_KEY_SIZE,
663 .max_keysize = AES_MAX_KEY_SIZE,
664 .ivsize = AES_BLOCK_SIZE,
665 .setkey = ablk_set_key,
666 .encrypt = ablk_encrypt,
667 .decrypt = ablk_encrypt,
668 .geniv = "chainiv",
669 },
670 },
671};
672
673#ifdef HAS_CTR 489#ifdef HAS_CTR
674static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm) 490static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
675{ 491{
@@ -682,32 +498,6 @@ static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
682 ablk_init_common(tfm, cryptd_tfm); 498 ablk_init_common(tfm, cryptd_tfm);
683 return 0; 499 return 0;
684} 500}
685
686static struct crypto_alg ablk_rfc3686_ctr_alg = {
687 .cra_name = "rfc3686(ctr(aes))",
688 .cra_driver_name = "rfc3686-ctr-aes-aesni",
689 .cra_priority = 400,
690 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
691 .cra_blocksize = 1,
692 .cra_ctxsize = sizeof(struct async_aes_ctx),
693 .cra_alignmask = 0,
694 .cra_type = &crypto_ablkcipher_type,
695 .cra_module = THIS_MODULE,
696 .cra_list = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list),
697 .cra_init = ablk_rfc3686_ctr_init,
698 .cra_exit = ablk_exit,
699 .cra_u = {
700 .ablkcipher = {
701 .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
702 .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
703 .ivsize = CTR_RFC3686_IV_SIZE,
704 .setkey = ablk_set_key,
705 .encrypt = ablk_encrypt,
706 .decrypt = ablk_decrypt,
707 .geniv = "seqiv",
708 },
709 },
710};
711#endif 501#endif
712#endif 502#endif
713 503
@@ -723,31 +513,6 @@ static int ablk_lrw_init(struct crypto_tfm *tfm)
723 ablk_init_common(tfm, cryptd_tfm); 513 ablk_init_common(tfm, cryptd_tfm);
724 return 0; 514 return 0;
725} 515}
726
727static struct crypto_alg ablk_lrw_alg = {
728 .cra_name = "lrw(aes)",
729 .cra_driver_name = "lrw-aes-aesni",
730 .cra_priority = 400,
731 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
732 .cra_blocksize = AES_BLOCK_SIZE,
733 .cra_ctxsize = sizeof(struct async_aes_ctx),
734 .cra_alignmask = 0,
735 .cra_type = &crypto_ablkcipher_type,
736 .cra_module = THIS_MODULE,
737 .cra_list = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
738 .cra_init = ablk_lrw_init,
739 .cra_exit = ablk_exit,
740 .cra_u = {
741 .ablkcipher = {
742 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
743 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
744 .ivsize = AES_BLOCK_SIZE,
745 .setkey = ablk_set_key,
746 .encrypt = ablk_encrypt,
747 .decrypt = ablk_decrypt,
748 },
749 },
750};
751#endif 516#endif
752 517
753#ifdef HAS_PCBC 518#ifdef HAS_PCBC
@@ -762,31 +527,6 @@ static int ablk_pcbc_init(struct crypto_tfm *tfm)
762 ablk_init_common(tfm, cryptd_tfm); 527 ablk_init_common(tfm, cryptd_tfm);
763 return 0; 528 return 0;
764} 529}
765
766static struct crypto_alg ablk_pcbc_alg = {
767 .cra_name = "pcbc(aes)",
768 .cra_driver_name = "pcbc-aes-aesni",
769 .cra_priority = 400,
770 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
771 .cra_blocksize = AES_BLOCK_SIZE,
772 .cra_ctxsize = sizeof(struct async_aes_ctx),
773 .cra_alignmask = 0,
774 .cra_type = &crypto_ablkcipher_type,
775 .cra_module = THIS_MODULE,
776 .cra_list = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list),
777 .cra_init = ablk_pcbc_init,
778 .cra_exit = ablk_exit,
779 .cra_u = {
780 .ablkcipher = {
781 .min_keysize = AES_MIN_KEY_SIZE,
782 .max_keysize = AES_MAX_KEY_SIZE,
783 .ivsize = AES_BLOCK_SIZE,
784 .setkey = ablk_set_key,
785 .encrypt = ablk_encrypt,
786 .decrypt = ablk_decrypt,
787 },
788 },
789};
790#endif 530#endif
791 531
792#ifdef HAS_XTS 532#ifdef HAS_XTS
@@ -801,31 +541,6 @@ static int ablk_xts_init(struct crypto_tfm *tfm)
801 ablk_init_common(tfm, cryptd_tfm); 541 ablk_init_common(tfm, cryptd_tfm);
802 return 0; 542 return 0;
803} 543}
804
805static struct crypto_alg ablk_xts_alg = {
806 .cra_name = "xts(aes)",
807 .cra_driver_name = "xts-aes-aesni",
808 .cra_priority = 400,
809 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
810 .cra_blocksize = AES_BLOCK_SIZE,
811 .cra_ctxsize = sizeof(struct async_aes_ctx),
812 .cra_alignmask = 0,
813 .cra_type = &crypto_ablkcipher_type,
814 .cra_module = THIS_MODULE,
815 .cra_list = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
816 .cra_init = ablk_xts_init,
817 .cra_exit = ablk_exit,
818 .cra_u = {
819 .ablkcipher = {
820 .min_keysize = 2 * AES_MIN_KEY_SIZE,
821 .max_keysize = 2 * AES_MAX_KEY_SIZE,
822 .ivsize = AES_BLOCK_SIZE,
823 .setkey = ablk_set_key,
824 .encrypt = ablk_encrypt,
825 .decrypt = ablk_decrypt,
826 },
827 },
828};
829#endif 544#endif
830 545
831#ifdef CONFIG_X86_64 546#ifdef CONFIG_X86_64
@@ -1050,32 +765,6 @@ static int rfc4106_decrypt(struct aead_request *req)
1050 } 765 }
1051} 766}
1052 767
1053static struct crypto_alg rfc4106_alg = {
1054 .cra_name = "rfc4106(gcm(aes))",
1055 .cra_driver_name = "rfc4106-gcm-aesni",
1056 .cra_priority = 400,
1057 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1058 .cra_blocksize = 1,
1059 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1060 .cra_alignmask = 0,
1061 .cra_type = &crypto_nivaead_type,
1062 .cra_module = THIS_MODULE,
1063 .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
1064 .cra_init = rfc4106_init,
1065 .cra_exit = rfc4106_exit,
1066 .cra_u = {
1067 .aead = {
1068 .setkey = rfc4106_set_key,
1069 .setauthsize = rfc4106_set_authsize,
1070 .encrypt = rfc4106_encrypt,
1071 .decrypt = rfc4106_decrypt,
1072 .geniv = "seqiv",
1073 .ivsize = 8,
1074 .maxauthsize = 16,
1075 },
1076 },
1077};
1078
1079static int __driver_rfc4106_encrypt(struct aead_request *req) 768static int __driver_rfc4106_encrypt(struct aead_request *req)
1080{ 769{
1081 u8 one_entry_in_sg = 0; 770 u8 one_entry_in_sg = 0;
@@ -1233,26 +922,316 @@ static int __driver_rfc4106_decrypt(struct aead_request *req)
1233 } 922 }
1234 return retval; 923 return retval;
1235} 924}
925#endif
1236 926
1237static struct crypto_alg __rfc4106_alg = { 927static struct crypto_alg aesni_algs[] = { {
928 .cra_name = "aes",
929 .cra_driver_name = "aes-aesni",
930 .cra_priority = 300,
931 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
932 .cra_blocksize = AES_BLOCK_SIZE,
933 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
934 AESNI_ALIGN - 1,
935 .cra_alignmask = 0,
936 .cra_module = THIS_MODULE,
937 .cra_u = {
938 .cipher = {
939 .cia_min_keysize = AES_MIN_KEY_SIZE,
940 .cia_max_keysize = AES_MAX_KEY_SIZE,
941 .cia_setkey = aes_set_key,
942 .cia_encrypt = aes_encrypt,
943 .cia_decrypt = aes_decrypt
944 }
945 }
946}, {
947 .cra_name = "__aes-aesni",
948 .cra_driver_name = "__driver-aes-aesni",
949 .cra_priority = 0,
950 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
951 .cra_blocksize = AES_BLOCK_SIZE,
952 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
953 AESNI_ALIGN - 1,
954 .cra_alignmask = 0,
955 .cra_module = THIS_MODULE,
956 .cra_u = {
957 .cipher = {
958 .cia_min_keysize = AES_MIN_KEY_SIZE,
959 .cia_max_keysize = AES_MAX_KEY_SIZE,
960 .cia_setkey = aes_set_key,
961 .cia_encrypt = __aes_encrypt,
962 .cia_decrypt = __aes_decrypt
963 }
964 }
965}, {
966 .cra_name = "__ecb-aes-aesni",
967 .cra_driver_name = "__driver-ecb-aes-aesni",
968 .cra_priority = 0,
969 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
970 .cra_blocksize = AES_BLOCK_SIZE,
971 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
972 AESNI_ALIGN - 1,
973 .cra_alignmask = 0,
974 .cra_type = &crypto_blkcipher_type,
975 .cra_module = THIS_MODULE,
976 .cra_u = {
977 .blkcipher = {
978 .min_keysize = AES_MIN_KEY_SIZE,
979 .max_keysize = AES_MAX_KEY_SIZE,
980 .setkey = aes_set_key,
981 .encrypt = ecb_encrypt,
982 .decrypt = ecb_decrypt,
983 },
984 },
985}, {
986 .cra_name = "__cbc-aes-aesni",
987 .cra_driver_name = "__driver-cbc-aes-aesni",
988 .cra_priority = 0,
989 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
990 .cra_blocksize = AES_BLOCK_SIZE,
991 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
992 AESNI_ALIGN - 1,
993 .cra_alignmask = 0,
994 .cra_type = &crypto_blkcipher_type,
995 .cra_module = THIS_MODULE,
996 .cra_u = {
997 .blkcipher = {
998 .min_keysize = AES_MIN_KEY_SIZE,
999 .max_keysize = AES_MAX_KEY_SIZE,
1000 .setkey = aes_set_key,
1001 .encrypt = cbc_encrypt,
1002 .decrypt = cbc_decrypt,
1003 },
1004 },
1005}, {
1006 .cra_name = "ecb(aes)",
1007 .cra_driver_name = "ecb-aes-aesni",
1008 .cra_priority = 400,
1009 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1010 .cra_blocksize = AES_BLOCK_SIZE,
1011 .cra_ctxsize = sizeof(struct async_aes_ctx),
1012 .cra_alignmask = 0,
1013 .cra_type = &crypto_ablkcipher_type,
1014 .cra_module = THIS_MODULE,
1015 .cra_init = ablk_ecb_init,
1016 .cra_exit = ablk_exit,
1017 .cra_u = {
1018 .ablkcipher = {
1019 .min_keysize = AES_MIN_KEY_SIZE,
1020 .max_keysize = AES_MAX_KEY_SIZE,
1021 .setkey = ablk_set_key,
1022 .encrypt = ablk_encrypt,
1023 .decrypt = ablk_decrypt,
1024 },
1025 },
1026}, {
1027 .cra_name = "cbc(aes)",
1028 .cra_driver_name = "cbc-aes-aesni",
1029 .cra_priority = 400,
1030 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1031 .cra_blocksize = AES_BLOCK_SIZE,
1032 .cra_ctxsize = sizeof(struct async_aes_ctx),
1033 .cra_alignmask = 0,
1034 .cra_type = &crypto_ablkcipher_type,
1035 .cra_module = THIS_MODULE,
1036 .cra_init = ablk_cbc_init,
1037 .cra_exit = ablk_exit,
1038 .cra_u = {
1039 .ablkcipher = {
1040 .min_keysize = AES_MIN_KEY_SIZE,
1041 .max_keysize = AES_MAX_KEY_SIZE,
1042 .ivsize = AES_BLOCK_SIZE,
1043 .setkey = ablk_set_key,
1044 .encrypt = ablk_encrypt,
1045 .decrypt = ablk_decrypt,
1046 },
1047 },
1048#ifdef CONFIG_X86_64
1049}, {
1050 .cra_name = "__ctr-aes-aesni",
1051 .cra_driver_name = "__driver-ctr-aes-aesni",
1052 .cra_priority = 0,
1053 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1054 .cra_blocksize = 1,
1055 .cra_ctxsize = sizeof(struct crypto_aes_ctx) +
1056 AESNI_ALIGN - 1,
1057 .cra_alignmask = 0,
1058 .cra_type = &crypto_blkcipher_type,
1059 .cra_module = THIS_MODULE,
1060 .cra_u = {
1061 .blkcipher = {
1062 .min_keysize = AES_MIN_KEY_SIZE,
1063 .max_keysize = AES_MAX_KEY_SIZE,
1064 .ivsize = AES_BLOCK_SIZE,
1065 .setkey = aes_set_key,
1066 .encrypt = ctr_crypt,
1067 .decrypt = ctr_crypt,
1068 },
1069 },
1070}, {
1071 .cra_name = "ctr(aes)",
1072 .cra_driver_name = "ctr-aes-aesni",
1073 .cra_priority = 400,
1074 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1075 .cra_blocksize = 1,
1076 .cra_ctxsize = sizeof(struct async_aes_ctx),
1077 .cra_alignmask = 0,
1078 .cra_type = &crypto_ablkcipher_type,
1079 .cra_module = THIS_MODULE,
1080 .cra_init = ablk_ctr_init,
1081 .cra_exit = ablk_exit,
1082 .cra_u = {
1083 .ablkcipher = {
1084 .min_keysize = AES_MIN_KEY_SIZE,
1085 .max_keysize = AES_MAX_KEY_SIZE,
1086 .ivsize = AES_BLOCK_SIZE,
1087 .setkey = ablk_set_key,
1088 .encrypt = ablk_encrypt,
1089 .decrypt = ablk_encrypt,
1090 .geniv = "chainiv",
1091 },
1092 },
1093}, {
1238 .cra_name = "__gcm-aes-aesni", 1094 .cra_name = "__gcm-aes-aesni",
1239 .cra_driver_name = "__driver-gcm-aes-aesni", 1095 .cra_driver_name = "__driver-gcm-aes-aesni",
1240 .cra_priority = 0, 1096 .cra_priority = 0,
1241 .cra_flags = CRYPTO_ALG_TYPE_AEAD, 1097 .cra_flags = CRYPTO_ALG_TYPE_AEAD,
1242 .cra_blocksize = 1, 1098 .cra_blocksize = 1,
1243 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, 1099 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) +
1100 AESNI_ALIGN,
1244 .cra_alignmask = 0, 1101 .cra_alignmask = 0,
1245 .cra_type = &crypto_aead_type, 1102 .cra_type = &crypto_aead_type,
1246 .cra_module = THIS_MODULE, 1103 .cra_module = THIS_MODULE,
1247 .cra_list = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
1248 .cra_u = { 1104 .cra_u = {
1249 .aead = { 1105 .aead = {
1250 .encrypt = __driver_rfc4106_encrypt, 1106 .encrypt = __driver_rfc4106_encrypt,
1251 .decrypt = __driver_rfc4106_decrypt, 1107 .decrypt = __driver_rfc4106_decrypt,
1252 }, 1108 },
1253 }, 1109 },
1254}; 1110}, {
1111 .cra_name = "rfc4106(gcm(aes))",
1112 .cra_driver_name = "rfc4106-gcm-aesni",
1113 .cra_priority = 400,
1114 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1115 .cra_blocksize = 1,
1116 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) +
1117 AESNI_ALIGN,
1118 .cra_alignmask = 0,
1119 .cra_type = &crypto_nivaead_type,
1120 .cra_module = THIS_MODULE,
1121 .cra_init = rfc4106_init,
1122 .cra_exit = rfc4106_exit,
1123 .cra_u = {
1124 .aead = {
1125 .setkey = rfc4106_set_key,
1126 .setauthsize = rfc4106_set_authsize,
1127 .encrypt = rfc4106_encrypt,
1128 .decrypt = rfc4106_decrypt,
1129 .geniv = "seqiv",
1130 .ivsize = 8,
1131 .maxauthsize = 16,
1132 },
1133 },
1134#ifdef HAS_CTR
1135}, {
1136 .cra_name = "rfc3686(ctr(aes))",
1137 .cra_driver_name = "rfc3686-ctr-aes-aesni",
1138 .cra_priority = 400,
1139 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1140 .cra_blocksize = 1,
1141 .cra_ctxsize = sizeof(struct async_aes_ctx),
1142 .cra_alignmask = 0,
1143 .cra_type = &crypto_ablkcipher_type,
1144 .cra_module = THIS_MODULE,
1145 .cra_init = ablk_rfc3686_ctr_init,
1146 .cra_exit = ablk_exit,
1147 .cra_u = {
1148 .ablkcipher = {
1149 .min_keysize = AES_MIN_KEY_SIZE +
1150 CTR_RFC3686_NONCE_SIZE,
1151 .max_keysize = AES_MAX_KEY_SIZE +
1152 CTR_RFC3686_NONCE_SIZE,
1153 .ivsize = CTR_RFC3686_IV_SIZE,
1154 .setkey = ablk_set_key,
1155 .encrypt = ablk_encrypt,
1156 .decrypt = ablk_decrypt,
1157 .geniv = "seqiv",
1158 },
1159 },
1160#endif
1161#endif
1162#ifdef HAS_LRW
1163}, {
1164 .cra_name = "lrw(aes)",
1165 .cra_driver_name = "lrw-aes-aesni",
1166 .cra_priority = 400,
1167 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1168 .cra_blocksize = AES_BLOCK_SIZE,
1169 .cra_ctxsize = sizeof(struct async_aes_ctx),
1170 .cra_alignmask = 0,
1171 .cra_type = &crypto_ablkcipher_type,
1172 .cra_module = THIS_MODULE,
1173 .cra_init = ablk_lrw_init,
1174 .cra_exit = ablk_exit,
1175 .cra_u = {
1176 .ablkcipher = {
1177 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1178 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1179 .ivsize = AES_BLOCK_SIZE,
1180 .setkey = ablk_set_key,
1181 .encrypt = ablk_encrypt,
1182 .decrypt = ablk_decrypt,
1183 },
1184 },
1185#endif
1186#ifdef HAS_PCBC
1187}, {
1188 .cra_name = "pcbc(aes)",
1189 .cra_driver_name = "pcbc-aes-aesni",
1190 .cra_priority = 400,
1191 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1192 .cra_blocksize = AES_BLOCK_SIZE,
1193 .cra_ctxsize = sizeof(struct async_aes_ctx),
1194 .cra_alignmask = 0,
1195 .cra_type = &crypto_ablkcipher_type,
1196 .cra_module = THIS_MODULE,
1197 .cra_init = ablk_pcbc_init,
1198 .cra_exit = ablk_exit,
1199 .cra_u = {
1200 .ablkcipher = {
1201 .min_keysize = AES_MIN_KEY_SIZE,
1202 .max_keysize = AES_MAX_KEY_SIZE,
1203 .ivsize = AES_BLOCK_SIZE,
1204 .setkey = ablk_set_key,
1205 .encrypt = ablk_encrypt,
1206 .decrypt = ablk_decrypt,
1207 },
1208 },
1255#endif 1209#endif
1210#ifdef HAS_XTS
1211}, {
1212 .cra_name = "xts(aes)",
1213 .cra_driver_name = "xts-aes-aesni",
1214 .cra_priority = 400,
1215 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1216 .cra_blocksize = AES_BLOCK_SIZE,
1217 .cra_ctxsize = sizeof(struct async_aes_ctx),
1218 .cra_alignmask = 0,
1219 .cra_type = &crypto_ablkcipher_type,
1220 .cra_module = THIS_MODULE,
1221 .cra_init = ablk_xts_init,
1222 .cra_exit = ablk_exit,
1223 .cra_u = {
1224 .ablkcipher = {
1225 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1226 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1227 .ivsize = AES_BLOCK_SIZE,
1228 .setkey = ablk_set_key,
1229 .encrypt = ablk_encrypt,
1230 .decrypt = ablk_decrypt,
1231 },
1232 },
1233#endif
1234} };
1256 1235
1257 1236
1258static const struct x86_cpu_id aesni_cpu_id[] = { 1237static const struct x86_cpu_id aesni_cpu_id[] = {
@@ -1263,120 +1242,24 @@ MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1263 1242
1264static int __init aesni_init(void) 1243static int __init aesni_init(void)
1265{ 1244{
1266 int err; 1245 int err, i;
1267 1246
1268 if (!x86_match_cpu(aesni_cpu_id)) 1247 if (!x86_match_cpu(aesni_cpu_id))
1269 return -ENODEV; 1248 return -ENODEV;
1270 1249
1271 if ((err = crypto_fpu_init())) 1250 err = crypto_fpu_init();
1272 goto fpu_err; 1251 if (err)
1273 if ((err = crypto_register_alg(&aesni_alg))) 1252 return err;
1274 goto aes_err;
1275 if ((err = crypto_register_alg(&__aesni_alg)))
1276 goto __aes_err;
1277 if ((err = crypto_register_alg(&blk_ecb_alg)))
1278 goto blk_ecb_err;
1279 if ((err = crypto_register_alg(&blk_cbc_alg)))
1280 goto blk_cbc_err;
1281 if ((err = crypto_register_alg(&ablk_ecb_alg)))
1282 goto ablk_ecb_err;
1283 if ((err = crypto_register_alg(&ablk_cbc_alg)))
1284 goto ablk_cbc_err;
1285#ifdef CONFIG_X86_64
1286 if ((err = crypto_register_alg(&blk_ctr_alg)))
1287 goto blk_ctr_err;
1288 if ((err = crypto_register_alg(&ablk_ctr_alg)))
1289 goto ablk_ctr_err;
1290 if ((err = crypto_register_alg(&__rfc4106_alg)))
1291 goto __aead_gcm_err;
1292 if ((err = crypto_register_alg(&rfc4106_alg)))
1293 goto aead_gcm_err;
1294#ifdef HAS_CTR
1295 if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
1296 goto ablk_rfc3686_ctr_err;
1297#endif
1298#endif
1299#ifdef HAS_LRW
1300 if ((err = crypto_register_alg(&ablk_lrw_alg)))
1301 goto ablk_lrw_err;
1302#endif
1303#ifdef HAS_PCBC
1304 if ((err = crypto_register_alg(&ablk_pcbc_alg)))
1305 goto ablk_pcbc_err;
1306#endif
1307#ifdef HAS_XTS
1308 if ((err = crypto_register_alg(&ablk_xts_alg)))
1309 goto ablk_xts_err;
1310#endif
1311 return err;
1312 1253
1313#ifdef HAS_XTS 1254 for (i = 0; i < ARRAY_SIZE(aesni_algs); i++)
1314ablk_xts_err: 1255 INIT_LIST_HEAD(&aesni_algs[i].cra_list);
1315#endif 1256
1316#ifdef HAS_PCBC 1257 return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1317 crypto_unregister_alg(&ablk_pcbc_alg);
1318ablk_pcbc_err:
1319#endif
1320#ifdef HAS_LRW
1321 crypto_unregister_alg(&ablk_lrw_alg);
1322ablk_lrw_err:
1323#endif
1324#ifdef CONFIG_X86_64
1325#ifdef HAS_CTR
1326 crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1327ablk_rfc3686_ctr_err:
1328#endif
1329 crypto_unregister_alg(&rfc4106_alg);
1330aead_gcm_err:
1331 crypto_unregister_alg(&__rfc4106_alg);
1332__aead_gcm_err:
1333 crypto_unregister_alg(&ablk_ctr_alg);
1334ablk_ctr_err:
1335 crypto_unregister_alg(&blk_ctr_alg);
1336blk_ctr_err:
1337#endif
1338 crypto_unregister_alg(&ablk_cbc_alg);
1339ablk_cbc_err:
1340 crypto_unregister_alg(&ablk_ecb_alg);
1341ablk_ecb_err:
1342 crypto_unregister_alg(&blk_cbc_alg);
1343blk_cbc_err:
1344 crypto_unregister_alg(&blk_ecb_alg);
1345blk_ecb_err:
1346 crypto_unregister_alg(&__aesni_alg);
1347__aes_err:
1348 crypto_unregister_alg(&aesni_alg);
1349aes_err:
1350fpu_err:
1351 return err;
1352} 1258}
1353 1259
1354static void __exit aesni_exit(void) 1260static void __exit aesni_exit(void)
1355{ 1261{
1356#ifdef HAS_XTS 1262 crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1357 crypto_unregister_alg(&ablk_xts_alg);
1358#endif
1359#ifdef HAS_PCBC
1360 crypto_unregister_alg(&ablk_pcbc_alg);
1361#endif
1362#ifdef HAS_LRW
1363 crypto_unregister_alg(&ablk_lrw_alg);
1364#endif
1365#ifdef CONFIG_X86_64
1366#ifdef HAS_CTR
1367 crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1368#endif
1369 crypto_unregister_alg(&rfc4106_alg);
1370 crypto_unregister_alg(&__rfc4106_alg);
1371 crypto_unregister_alg(&ablk_ctr_alg);
1372 crypto_unregister_alg(&blk_ctr_alg);
1373#endif
1374 crypto_unregister_alg(&ablk_cbc_alg);
1375 crypto_unregister_alg(&ablk_ecb_alg);
1376 crypto_unregister_alg(&blk_cbc_alg);
1377 crypto_unregister_alg(&blk_ecb_alg);
1378 crypto_unregister_alg(&__aesni_alg);
1379 crypto_unregister_alg(&aesni_alg);
1380 1263
1381 crypto_fpu_exit(); 1264 crypto_fpu_exit();
1382} 1265}
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-24 02:02:51 -0500