diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2013-09-07 17:31:18 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2013-09-07 17:31:18 -0400 |
commit | 6be48f2940af9ea8d93c23a0dd8e322672c92efd (patch) | |
tree | 1bdc85a9d3fd0c19e108ea27a29a83ef2b44f5d0 /arch | |
parent | 0ffb01d9def22f1954e99529b7e4ded497b2e88b (diff) | |
parent | 68411521cc6055edc6274e03ab3210a5893533ba (diff) |
Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto update from Herbert Xu:
"Here is the crypto update for 3.12:
- Added MODULE_SOFTDEP to allow pre-loading of modules.
- Reinstated crct10dif driver using the module softdep feature.
- Allow via rng driver to be auto-loaded.
- Split large input data when necessary in nx.
- Handle zero length messages correctly for GCM/XCBC in nx.
- Handle SHA-2 chunks bigger than block size properly in nx.
- Handle unaligned lengths in omap-aes.
- Added SHA384/SHA512 to omap-sham.
- Added OMAP5/AM43XX SHAM support.
- Added OMAP4 TRNG support.
- Misc fixes"
* git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (66 commits)
Reinstate "crypto: crct10dif - Wrap crc_t10dif function all to use crypto transform framework"
hwrng: via - Add MODULE_DEVICE_TABLE
crypto: fcrypt - Fix bitoperation for compilation with clang
crypto: nx - fix SHA-2 for chunks bigger than block size
crypto: nx - fix GCM for zero length messages
crypto: nx - fix XCBC for zero length messages
crypto: nx - fix limits to sg lists for AES-CCM
crypto: nx - fix limits to sg lists for AES-XCBC
crypto: nx - fix limits to sg lists for AES-GCM
crypto: nx - fix limits to sg lists for AES-CTR
crypto: nx - fix limits to sg lists for AES-CBC
crypto: nx - fix limits to sg lists for AES-ECB
crypto: nx - add offset to nx_build_sg_lists()
padata - Register hotcpu notifier after initialization
padata - share code between CPU_ONLINE and CPU_DOWN_FAILED, same to CPU_DOWN_PREPARE and CPU_UP_CANCELED
hwrng: omap - reorder OMAP TRNG driver code
crypto: omap-sham - correct dma burst size
crypto: omap-sham - Enable Polling mode if DMA fails
crypto: tegra-aes - bitwise vs logical and
crypto: sahara - checking the wrong variable
...
Diffstat (limited to 'arch')
-rw-r--r-- | arch/arm/mach-omap2/devices.c | 2 | ||||
-rw-r--r-- | arch/x86/crypto/Makefile | 2 | ||||
-rw-r--r-- | arch/x86/crypto/camellia_glue.c | 64 | ||||
-rw-r--r-- | arch/x86/crypto/crct10dif-pcl-asm_64.S | 643 | ||||
-rw-r--r-- | arch/x86/crypto/crct10dif-pclmul_glue.c | 151 | ||||
-rw-r--r-- | arch/x86/include/asm/xor_avx.h | 4 |
6 files changed, 831 insertions, 35 deletions
diff --git a/arch/arm/mach-omap2/devices.c b/arch/arm/mach-omap2/devices.c index 73ae7536a32b..5c5315ba129b 100644 --- a/arch/arm/mach-omap2/devices.c +++ b/arch/arm/mach-omap2/devices.c | |||
@@ -530,12 +530,12 @@ static int __init omap2_init_devices(void) | |||
530 | omap_init_mcspi(); | 530 | omap_init_mcspi(); |
531 | omap_init_sham(); | 531 | omap_init_sham(); |
532 | omap_init_aes(); | 532 | omap_init_aes(); |
533 | omap_init_rng(); | ||
533 | } else { | 534 | } else { |
534 | /* These can be removed when bindings are done */ | 535 | /* These can be removed when bindings are done */ |
535 | omap_init_wl12xx_of(); | 536 | omap_init_wl12xx_of(); |
536 | } | 537 | } |
537 | omap_init_sti(); | 538 | omap_init_sti(); |
538 | omap_init_rng(); | ||
539 | omap_init_vout(); | 539 | omap_init_vout(); |
540 | 540 | ||
541 | return 0; | 541 | return 0; |
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 6c63c358a7e6..7d6ba9db1be9 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile | |||
@@ -27,6 +27,7 @@ obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o | |||
27 | obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o | 27 | obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o |
28 | obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o | 28 | obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o |
29 | obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o | 29 | obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o |
30 | obj-$(CONFIG_CRYPTO_CRCT10DIF_PCLMUL) += crct10dif-pclmul.o | ||
30 | 31 | ||
31 | # These modules require assembler to support AVX. | 32 | # These modules require assembler to support AVX. |
32 | ifeq ($(avx_supported),yes) | 33 | ifeq ($(avx_supported),yes) |
@@ -81,3 +82,4 @@ crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o | |||
81 | crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o | 82 | crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o |
82 | sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ssse3_glue.o | 83 | sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ssse3_glue.o |
83 | sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o | 84 | sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o |
85 | crct10dif-pclmul-y := crct10dif-pcl-asm_64.o crct10dif-pclmul_glue.o | ||
diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c index 5cb86ccd4acb..c171dcbf192d 100644 --- a/arch/x86/crypto/camellia_glue.c +++ b/arch/x86/crypto/camellia_glue.c | |||
@@ -62,7 +62,7 @@ static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) | |||
62 | } | 62 | } |
63 | 63 | ||
64 | /* camellia sboxes */ | 64 | /* camellia sboxes */ |
65 | const u64 camellia_sp10011110[256] = { | 65 | __visible const u64 camellia_sp10011110[256] = { |
66 | 0x7000007070707000ULL, 0x8200008282828200ULL, 0x2c00002c2c2c2c00ULL, | 66 | 0x7000007070707000ULL, 0x8200008282828200ULL, 0x2c00002c2c2c2c00ULL, |
67 | 0xec0000ecececec00ULL, 0xb30000b3b3b3b300ULL, 0x2700002727272700ULL, | 67 | 0xec0000ecececec00ULL, 0xb30000b3b3b3b300ULL, 0x2700002727272700ULL, |
68 | 0xc00000c0c0c0c000ULL, 0xe50000e5e5e5e500ULL, 0xe40000e4e4e4e400ULL, | 68 | 0xc00000c0c0c0c000ULL, 0xe50000e5e5e5e500ULL, 0xe40000e4e4e4e400ULL, |
@@ -151,7 +151,7 @@ const u64 camellia_sp10011110[256] = { | |||
151 | 0x9e00009e9e9e9e00ULL, | 151 | 0x9e00009e9e9e9e00ULL, |
152 | }; | 152 | }; |
153 | 153 | ||
154 | const u64 camellia_sp22000222[256] = { | 154 | __visible const u64 camellia_sp22000222[256] = { |
155 | 0xe0e0000000e0e0e0ULL, 0x0505000000050505ULL, 0x5858000000585858ULL, | 155 | 0xe0e0000000e0e0e0ULL, 0x0505000000050505ULL, 0x5858000000585858ULL, |
156 | 0xd9d9000000d9d9d9ULL, 0x6767000000676767ULL, 0x4e4e0000004e4e4eULL, | 156 | 0xd9d9000000d9d9d9ULL, 0x6767000000676767ULL, 0x4e4e0000004e4e4eULL, |
157 | 0x8181000000818181ULL, 0xcbcb000000cbcbcbULL, 0xc9c9000000c9c9c9ULL, | 157 | 0x8181000000818181ULL, 0xcbcb000000cbcbcbULL, 0xc9c9000000c9c9c9ULL, |
@@ -240,7 +240,7 @@ const u64 camellia_sp22000222[256] = { | |||
240 | 0x3d3d0000003d3d3dULL, | 240 | 0x3d3d0000003d3d3dULL, |
241 | }; | 241 | }; |
242 | 242 | ||
243 | const u64 camellia_sp03303033[256] = { | 243 | __visible const u64 camellia_sp03303033[256] = { |
244 | 0x0038380038003838ULL, 0x0041410041004141ULL, 0x0016160016001616ULL, | 244 | 0x0038380038003838ULL, 0x0041410041004141ULL, 0x0016160016001616ULL, |
245 | 0x0076760076007676ULL, 0x00d9d900d900d9d9ULL, 0x0093930093009393ULL, | 245 | 0x0076760076007676ULL, 0x00d9d900d900d9d9ULL, 0x0093930093009393ULL, |
246 | 0x0060600060006060ULL, 0x00f2f200f200f2f2ULL, 0x0072720072007272ULL, | 246 | 0x0060600060006060ULL, 0x00f2f200f200f2f2ULL, 0x0072720072007272ULL, |
@@ -329,7 +329,7 @@ const u64 camellia_sp03303033[256] = { | |||
329 | 0x004f4f004f004f4fULL, | 329 | 0x004f4f004f004f4fULL, |
330 | }; | 330 | }; |
331 | 331 | ||
332 | const u64 camellia_sp00444404[256] = { | 332 | __visible const u64 camellia_sp00444404[256] = { |
333 | 0x0000707070700070ULL, 0x00002c2c2c2c002cULL, 0x0000b3b3b3b300b3ULL, | 333 | 0x0000707070700070ULL, 0x00002c2c2c2c002cULL, 0x0000b3b3b3b300b3ULL, |
334 | 0x0000c0c0c0c000c0ULL, 0x0000e4e4e4e400e4ULL, 0x0000575757570057ULL, | 334 | 0x0000c0c0c0c000c0ULL, 0x0000e4e4e4e400e4ULL, 0x0000575757570057ULL, |
335 | 0x0000eaeaeaea00eaULL, 0x0000aeaeaeae00aeULL, 0x0000232323230023ULL, | 335 | 0x0000eaeaeaea00eaULL, 0x0000aeaeaeae00aeULL, 0x0000232323230023ULL, |
@@ -418,7 +418,7 @@ const u64 camellia_sp00444404[256] = { | |||
418 | 0x00009e9e9e9e009eULL, | 418 | 0x00009e9e9e9e009eULL, |
419 | }; | 419 | }; |
420 | 420 | ||
421 | const u64 camellia_sp02220222[256] = { | 421 | __visible const u64 camellia_sp02220222[256] = { |
422 | 0x00e0e0e000e0e0e0ULL, 0x0005050500050505ULL, 0x0058585800585858ULL, | 422 | 0x00e0e0e000e0e0e0ULL, 0x0005050500050505ULL, 0x0058585800585858ULL, |
423 | 0x00d9d9d900d9d9d9ULL, 0x0067676700676767ULL, 0x004e4e4e004e4e4eULL, | 423 | 0x00d9d9d900d9d9d9ULL, 0x0067676700676767ULL, 0x004e4e4e004e4e4eULL, |
424 | 0x0081818100818181ULL, 0x00cbcbcb00cbcbcbULL, 0x00c9c9c900c9c9c9ULL, | 424 | 0x0081818100818181ULL, 0x00cbcbcb00cbcbcbULL, 0x00c9c9c900c9c9c9ULL, |
@@ -507,7 +507,7 @@ const u64 camellia_sp02220222[256] = { | |||
507 | 0x003d3d3d003d3d3dULL, | 507 | 0x003d3d3d003d3d3dULL, |
508 | }; | 508 | }; |
509 | 509 | ||
510 | const u64 camellia_sp30333033[256] = { | 510 | __visible const u64 camellia_sp30333033[256] = { |
511 | 0x3800383838003838ULL, 0x4100414141004141ULL, 0x1600161616001616ULL, | 511 | 0x3800383838003838ULL, 0x4100414141004141ULL, 0x1600161616001616ULL, |
512 | 0x7600767676007676ULL, 0xd900d9d9d900d9d9ULL, 0x9300939393009393ULL, | 512 | 0x7600767676007676ULL, 0xd900d9d9d900d9d9ULL, 0x9300939393009393ULL, |
513 | 0x6000606060006060ULL, 0xf200f2f2f200f2f2ULL, 0x7200727272007272ULL, | 513 | 0x6000606060006060ULL, 0xf200f2f2f200f2f2ULL, 0x7200727272007272ULL, |
@@ -596,7 +596,7 @@ const u64 camellia_sp30333033[256] = { | |||
596 | 0x4f004f4f4f004f4fULL, | 596 | 0x4f004f4f4f004f4fULL, |
597 | }; | 597 | }; |
598 | 598 | ||
599 | const u64 camellia_sp44044404[256] = { | 599 | __visible const u64 camellia_sp44044404[256] = { |
600 | 0x7070007070700070ULL, 0x2c2c002c2c2c002cULL, 0xb3b300b3b3b300b3ULL, | 600 | 0x7070007070700070ULL, 0x2c2c002c2c2c002cULL, 0xb3b300b3b3b300b3ULL, |
601 | 0xc0c000c0c0c000c0ULL, 0xe4e400e4e4e400e4ULL, 0x5757005757570057ULL, | 601 | 0xc0c000c0c0c000c0ULL, 0xe4e400e4e4e400e4ULL, 0x5757005757570057ULL, |
602 | 0xeaea00eaeaea00eaULL, 0xaeae00aeaeae00aeULL, 0x2323002323230023ULL, | 602 | 0xeaea00eaeaea00eaULL, 0xaeae00aeaeae00aeULL, 0x2323002323230023ULL, |
@@ -685,7 +685,7 @@ const u64 camellia_sp44044404[256] = { | |||
685 | 0x9e9e009e9e9e009eULL, | 685 | 0x9e9e009e9e9e009eULL, |
686 | }; | 686 | }; |
687 | 687 | ||
688 | const u64 camellia_sp11101110[256] = { | 688 | __visible const u64 camellia_sp11101110[256] = { |
689 | 0x7070700070707000ULL, 0x8282820082828200ULL, 0x2c2c2c002c2c2c00ULL, | 689 | 0x7070700070707000ULL, 0x8282820082828200ULL, 0x2c2c2c002c2c2c00ULL, |
690 | 0xececec00ececec00ULL, 0xb3b3b300b3b3b300ULL, 0x2727270027272700ULL, | 690 | 0xececec00ececec00ULL, 0xb3b3b300b3b3b300ULL, 0x2727270027272700ULL, |
691 | 0xc0c0c000c0c0c000ULL, 0xe5e5e500e5e5e500ULL, 0xe4e4e400e4e4e400ULL, | 691 | 0xc0c0c000c0c0c000ULL, 0xe5e5e500e5e5e500ULL, 0xe4e4e400e4e4e400ULL, |
@@ -828,8 +828,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
828 | 828 | ||
829 | subRL[1] ^= (subRL[1] & ~subRL[9]) << 32; | 829 | subRL[1] ^= (subRL[1] & ~subRL[9]) << 32; |
830 | /* modified for FLinv(kl2) */ | 830 | /* modified for FLinv(kl2) */ |
831 | dw = (subRL[1] & subRL[9]) >> 32, | 831 | dw = (subRL[1] & subRL[9]) >> 32; |
832 | subRL[1] ^= rol32(dw, 1); | 832 | subRL[1] ^= rol32(dw, 1); |
833 | 833 | ||
834 | /* round 8 */ | 834 | /* round 8 */ |
835 | subRL[11] ^= subRL[1]; | 835 | subRL[11] ^= subRL[1]; |
@@ -840,8 +840,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
840 | 840 | ||
841 | subRL[1] ^= (subRL[1] & ~subRL[17]) << 32; | 841 | subRL[1] ^= (subRL[1] & ~subRL[17]) << 32; |
842 | /* modified for FLinv(kl4) */ | 842 | /* modified for FLinv(kl4) */ |
843 | dw = (subRL[1] & subRL[17]) >> 32, | 843 | dw = (subRL[1] & subRL[17]) >> 32; |
844 | subRL[1] ^= rol32(dw, 1); | 844 | subRL[1] ^= rol32(dw, 1); |
845 | 845 | ||
846 | /* round 14 */ | 846 | /* round 14 */ |
847 | subRL[19] ^= subRL[1]; | 847 | subRL[19] ^= subRL[1]; |
@@ -859,8 +859,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
859 | } else { | 859 | } else { |
860 | subRL[1] ^= (subRL[1] & ~subRL[25]) << 32; | 860 | subRL[1] ^= (subRL[1] & ~subRL[25]) << 32; |
861 | /* modified for FLinv(kl6) */ | 861 | /* modified for FLinv(kl6) */ |
862 | dw = (subRL[1] & subRL[25]) >> 32, | 862 | dw = (subRL[1] & subRL[25]) >> 32; |
863 | subRL[1] ^= rol32(dw, 1); | 863 | subRL[1] ^= rol32(dw, 1); |
864 | 864 | ||
865 | /* round 20 */ | 865 | /* round 20 */ |
866 | subRL[27] ^= subRL[1]; | 866 | subRL[27] ^= subRL[1]; |
@@ -882,8 +882,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
882 | 882 | ||
883 | kw4 ^= (kw4 & ~subRL[24]) << 32; | 883 | kw4 ^= (kw4 & ~subRL[24]) << 32; |
884 | /* modified for FL(kl5) */ | 884 | /* modified for FL(kl5) */ |
885 | dw = (kw4 & subRL[24]) >> 32, | 885 | dw = (kw4 & subRL[24]) >> 32; |
886 | kw4 ^= rol32(dw, 1); | 886 | kw4 ^= rol32(dw, 1); |
887 | } | 887 | } |
888 | 888 | ||
889 | /* round 17 */ | 889 | /* round 17 */ |
@@ -895,8 +895,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
895 | 895 | ||
896 | kw4 ^= (kw4 & ~subRL[16]) << 32; | 896 | kw4 ^= (kw4 & ~subRL[16]) << 32; |
897 | /* modified for FL(kl3) */ | 897 | /* modified for FL(kl3) */ |
898 | dw = (kw4 & subRL[16]) >> 32, | 898 | dw = (kw4 & subRL[16]) >> 32; |
899 | kw4 ^= rol32(dw, 1); | 899 | kw4 ^= rol32(dw, 1); |
900 | 900 | ||
901 | /* round 11 */ | 901 | /* round 11 */ |
902 | subRL[14] ^= kw4; | 902 | subRL[14] ^= kw4; |
@@ -907,8 +907,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
907 | 907 | ||
908 | kw4 ^= (kw4 & ~subRL[8]) << 32; | 908 | kw4 ^= (kw4 & ~subRL[8]) << 32; |
909 | /* modified for FL(kl1) */ | 909 | /* modified for FL(kl1) */ |
910 | dw = (kw4 & subRL[8]) >> 32, | 910 | dw = (kw4 & subRL[8]) >> 32; |
911 | kw4 ^= rol32(dw, 1); | 911 | kw4 ^= rol32(dw, 1); |
912 | 912 | ||
913 | /* round 5 */ | 913 | /* round 5 */ |
914 | subRL[6] ^= kw4; | 914 | subRL[6] ^= kw4; |
@@ -928,8 +928,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
928 | SET_SUBKEY_LR(6, subRL[5] ^ subRL[7]); /* round 5 */ | 928 | SET_SUBKEY_LR(6, subRL[5] ^ subRL[7]); /* round 5 */ |
929 | 929 | ||
930 | tl = (subRL[10] >> 32) ^ (subRL[10] & ~subRL[8]); | 930 | tl = (subRL[10] >> 32) ^ (subRL[10] & ~subRL[8]); |
931 | dw = tl & (subRL[8] >> 32), /* FL(kl1) */ | 931 | dw = tl & (subRL[8] >> 32); /* FL(kl1) */ |
932 | tr = subRL[10] ^ rol32(dw, 1); | 932 | tr = subRL[10] ^ rol32(dw, 1); |
933 | tt = (tr | ((u64)tl << 32)); | 933 | tt = (tr | ((u64)tl << 32)); |
934 | 934 | ||
935 | SET_SUBKEY_LR(7, subRL[6] ^ tt); /* round 6 */ | 935 | SET_SUBKEY_LR(7, subRL[6] ^ tt); /* round 6 */ |
@@ -937,8 +937,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
937 | SET_SUBKEY_LR(9, subRL[9]); /* FLinv(kl2) */ | 937 | SET_SUBKEY_LR(9, subRL[9]); /* FLinv(kl2) */ |
938 | 938 | ||
939 | tl = (subRL[7] >> 32) ^ (subRL[7] & ~subRL[9]); | 939 | tl = (subRL[7] >> 32) ^ (subRL[7] & ~subRL[9]); |
940 | dw = tl & (subRL[9] >> 32), /* FLinv(kl2) */ | 940 | dw = tl & (subRL[9] >> 32); /* FLinv(kl2) */ |
941 | tr = subRL[7] ^ rol32(dw, 1); | 941 | tr = subRL[7] ^ rol32(dw, 1); |
942 | tt = (tr | ((u64)tl << 32)); | 942 | tt = (tr | ((u64)tl << 32)); |
943 | 943 | ||
944 | SET_SUBKEY_LR(10, subRL[11] ^ tt); /* round 7 */ | 944 | SET_SUBKEY_LR(10, subRL[11] ^ tt); /* round 7 */ |
@@ -948,8 +948,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
948 | SET_SUBKEY_LR(14, subRL[13] ^ subRL[15]); /* round 11 */ | 948 | SET_SUBKEY_LR(14, subRL[13] ^ subRL[15]); /* round 11 */ |
949 | 949 | ||
950 | tl = (subRL[18] >> 32) ^ (subRL[18] & ~subRL[16]); | 950 | tl = (subRL[18] >> 32) ^ (subRL[18] & ~subRL[16]); |
951 | dw = tl & (subRL[16] >> 32), /* FL(kl3) */ | 951 | dw = tl & (subRL[16] >> 32); /* FL(kl3) */ |
952 | tr = subRL[18] ^ rol32(dw, 1); | 952 | tr = subRL[18] ^ rol32(dw, 1); |
953 | tt = (tr | ((u64)tl << 32)); | 953 | tt = (tr | ((u64)tl << 32)); |
954 | 954 | ||
955 | SET_SUBKEY_LR(15, subRL[14] ^ tt); /* round 12 */ | 955 | SET_SUBKEY_LR(15, subRL[14] ^ tt); /* round 12 */ |
@@ -957,8 +957,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
957 | SET_SUBKEY_LR(17, subRL[17]); /* FLinv(kl4) */ | 957 | SET_SUBKEY_LR(17, subRL[17]); /* FLinv(kl4) */ |
958 | 958 | ||
959 | tl = (subRL[15] >> 32) ^ (subRL[15] & ~subRL[17]); | 959 | tl = (subRL[15] >> 32) ^ (subRL[15] & ~subRL[17]); |
960 | dw = tl & (subRL[17] >> 32), /* FLinv(kl4) */ | 960 | dw = tl & (subRL[17] >> 32); /* FLinv(kl4) */ |
961 | tr = subRL[15] ^ rol32(dw, 1); | 961 | tr = subRL[15] ^ rol32(dw, 1); |
962 | tt = (tr | ((u64)tl << 32)); | 962 | tt = (tr | ((u64)tl << 32)); |
963 | 963 | ||
964 | SET_SUBKEY_LR(18, subRL[19] ^ tt); /* round 13 */ | 964 | SET_SUBKEY_LR(18, subRL[19] ^ tt); /* round 13 */ |
@@ -972,8 +972,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
972 | SET_SUBKEY_LR(24, subRL[24] ^ subRL[23]); /* kw3 */ | 972 | SET_SUBKEY_LR(24, subRL[24] ^ subRL[23]); /* kw3 */ |
973 | } else { | 973 | } else { |
974 | tl = (subRL[26] >> 32) ^ (subRL[26] & ~subRL[24]); | 974 | tl = (subRL[26] >> 32) ^ (subRL[26] & ~subRL[24]); |
975 | dw = tl & (subRL[24] >> 32), /* FL(kl5) */ | 975 | dw = tl & (subRL[24] >> 32); /* FL(kl5) */ |
976 | tr = subRL[26] ^ rol32(dw, 1); | 976 | tr = subRL[26] ^ rol32(dw, 1); |
977 | tt = (tr | ((u64)tl << 32)); | 977 | tt = (tr | ((u64)tl << 32)); |
978 | 978 | ||
979 | SET_SUBKEY_LR(23, subRL[22] ^ tt); /* round 18 */ | 979 | SET_SUBKEY_LR(23, subRL[22] ^ tt); /* round 18 */ |
@@ -981,8 +981,8 @@ static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max) | |||
981 | SET_SUBKEY_LR(25, subRL[25]); /* FLinv(kl6) */ | 981 | SET_SUBKEY_LR(25, subRL[25]); /* FLinv(kl6) */ |
982 | 982 | ||
983 | tl = (subRL[23] >> 32) ^ (subRL[23] & ~subRL[25]); | 983 | tl = (subRL[23] >> 32) ^ (subRL[23] & ~subRL[25]); |
984 | dw = tl & (subRL[25] >> 32), /* FLinv(kl6) */ | 984 | dw = tl & (subRL[25] >> 32); /* FLinv(kl6) */ |
985 | tr = subRL[23] ^ rol32(dw, 1); | 985 | tr = subRL[23] ^ rol32(dw, 1); |
986 | tt = (tr | ((u64)tl << 32)); | 986 | tt = (tr | ((u64)tl << 32)); |
987 | 987 | ||
988 | SET_SUBKEY_LR(26, subRL[27] ^ tt); /* round 19 */ | 988 | SET_SUBKEY_LR(26, subRL[27] ^ tt); /* round 19 */ |
diff --git a/arch/x86/crypto/crct10dif-pcl-asm_64.S b/arch/x86/crypto/crct10dif-pcl-asm_64.S new file mode 100644 index 000000000000..35e97569d05f --- /dev/null +++ b/arch/x86/crypto/crct10dif-pcl-asm_64.S | |||
@@ -0,0 +1,643 @@ | |||
1 | ######################################################################## | ||
2 | # Implement fast CRC-T10DIF computation with SSE and PCLMULQDQ instructions | ||
3 | # | ||
4 | # Copyright (c) 2013, Intel Corporation | ||
5 | # | ||
6 | # Authors: | ||
7 | # Erdinc Ozturk <erdinc.ozturk@intel.com> | ||
8 | # Vinodh Gopal <vinodh.gopal@intel.com> | ||
9 | # James Guilford <james.guilford@intel.com> | ||
10 | # Tim Chen <tim.c.chen@linux.intel.com> | ||
11 | # | ||
12 | # This software is available to you under a choice of one of two | ||
13 | # licenses. You may choose to be licensed under the terms of the GNU | ||
14 | # General Public License (GPL) Version 2, available from the file | ||
15 | # COPYING in the main directory of this source tree, or the | ||
16 | # OpenIB.org BSD license below: | ||
17 | # | ||
18 | # Redistribution and use in source and binary forms, with or without | ||
19 | # modification, are permitted provided that the following conditions are | ||
20 | # met: | ||
21 | # | ||
22 | # * Redistributions of source code must retain the above copyright | ||
23 | # notice, this list of conditions and the following disclaimer. | ||
24 | # | ||
25 | # * Redistributions in binary form must reproduce the above copyright | ||
26 | # notice, this list of conditions and the following disclaimer in the | ||
27 | # documentation and/or other materials provided with the | ||
28 | # distribution. | ||
29 | # | ||
30 | # * Neither the name of the Intel Corporation nor the names of its | ||
31 | # contributors may be used to endorse or promote products derived from | ||
32 | # this software without specific prior written permission. | ||
33 | # | ||
34 | # | ||
35 | # THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY | ||
36 | # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
37 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||
38 | # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR | ||
39 | # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, | ||
40 | # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | ||
41 | # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR | ||
42 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF | ||
43 | # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING | ||
44 | # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS | ||
45 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
46 | ######################################################################## | ||
47 | # Function API: | ||
48 | # UINT16 crc_t10dif_pcl( | ||
49 | # UINT16 init_crc, //initial CRC value, 16 bits | ||
50 | # const unsigned char *buf, //buffer pointer to calculate CRC on | ||
51 | # UINT64 len //buffer length in bytes (64-bit data) | ||
52 | # ); | ||
53 | # | ||
54 | # Reference paper titled "Fast CRC Computation for Generic | ||
55 | # Polynomials Using PCLMULQDQ Instruction" | ||
56 | # URL: http://www.intel.com/content/dam/www/public/us/en/documents | ||
57 | # /white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf | ||
58 | # | ||
59 | # | ||
60 | |||
61 | #include <linux/linkage.h> | ||
62 | |||
63 | .text | ||
64 | |||
65 | #define arg1 %rdi | ||
66 | #define arg2 %rsi | ||
67 | #define arg3 %rdx | ||
68 | |||
69 | #define arg1_low32 %edi | ||
70 | |||
71 | ENTRY(crc_t10dif_pcl) | ||
72 | .align 16 | ||
73 | |||
74 | # adjust the 16-bit initial_crc value, scale it to 32 bits | ||
75 | shl $16, arg1_low32 | ||
76 | |||
77 | # Allocate Stack Space | ||
78 | mov %rsp, %rcx | ||
79 | sub $16*2, %rsp | ||
80 | # align stack to 16 byte boundary | ||
81 | and $~(0x10 - 1), %rsp | ||
82 | |||
83 | # check if smaller than 256 | ||
84 | cmp $256, arg3 | ||
85 | |||
86 | # for sizes less than 128, we can't fold 64B at a time... | ||
87 | jl _less_than_128 | ||
88 | |||
89 | |||
90 | # load the initial crc value | ||
91 | movd arg1_low32, %xmm10 # initial crc | ||
92 | |||
93 | # crc value does not need to be byte-reflected, but it needs | ||
94 | # to be moved to the high part of the register. | ||
95 | # because data will be byte-reflected and will align with | ||
96 | # initial crc at correct place. | ||
97 | pslldq $12, %xmm10 | ||
98 | |||
99 | movdqa SHUF_MASK(%rip), %xmm11 | ||
100 | # receive the initial 64B data, xor the initial crc value | ||
101 | movdqu 16*0(arg2), %xmm0 | ||
102 | movdqu 16*1(arg2), %xmm1 | ||
103 | movdqu 16*2(arg2), %xmm2 | ||
104 | movdqu 16*3(arg2), %xmm3 | ||
105 | movdqu 16*4(arg2), %xmm4 | ||
106 | movdqu 16*5(arg2), %xmm5 | ||
107 | movdqu 16*6(arg2), %xmm6 | ||
108 | movdqu 16*7(arg2), %xmm7 | ||
109 | |||
110 | pshufb %xmm11, %xmm0 | ||
111 | # XOR the initial_crc value | ||
112 | pxor %xmm10, %xmm0 | ||
113 | pshufb %xmm11, %xmm1 | ||
114 | pshufb %xmm11, %xmm2 | ||
115 | pshufb %xmm11, %xmm3 | ||
116 | pshufb %xmm11, %xmm4 | ||
117 | pshufb %xmm11, %xmm5 | ||
118 | pshufb %xmm11, %xmm6 | ||
119 | pshufb %xmm11, %xmm7 | ||
120 | |||
121 | movdqa rk3(%rip), %xmm10 #xmm10 has rk3 and rk4 | ||
122 | #imm value of pclmulqdq instruction | ||
123 | #will determine which constant to use | ||
124 | |||
125 | ################################################################# | ||
126 | # we subtract 256 instead of 128 to save one instruction from the loop | ||
127 | sub $256, arg3 | ||
128 | |||
129 | # at this section of the code, there is 64*x+y (0<=y<64) bytes of | ||
130 | # buffer. The _fold_64_B_loop will fold 64B at a time | ||
131 | # until we have 64+y Bytes of buffer | ||
132 | |||
133 | |||
134 | # fold 64B at a time. This section of the code folds 4 xmm | ||
135 | # registers in parallel | ||
136 | _fold_64_B_loop: | ||
137 | |||
138 | # update the buffer pointer | ||
139 | add $128, arg2 # buf += 64# | ||
140 | |||
141 | movdqu 16*0(arg2), %xmm9 | ||
142 | movdqu 16*1(arg2), %xmm12 | ||
143 | pshufb %xmm11, %xmm9 | ||
144 | pshufb %xmm11, %xmm12 | ||
145 | movdqa %xmm0, %xmm8 | ||
146 | movdqa %xmm1, %xmm13 | ||
147 | pclmulqdq $0x0 , %xmm10, %xmm0 | ||
148 | pclmulqdq $0x11, %xmm10, %xmm8 | ||
149 | pclmulqdq $0x0 , %xmm10, %xmm1 | ||
150 | pclmulqdq $0x11, %xmm10, %xmm13 | ||
151 | pxor %xmm9 , %xmm0 | ||
152 | xorps %xmm8 , %xmm0 | ||
153 | pxor %xmm12, %xmm1 | ||
154 | xorps %xmm13, %xmm1 | ||
155 | |||
156 | movdqu 16*2(arg2), %xmm9 | ||
157 | movdqu 16*3(arg2), %xmm12 | ||
158 | pshufb %xmm11, %xmm9 | ||
159 | pshufb %xmm11, %xmm12 | ||
160 | movdqa %xmm2, %xmm8 | ||
161 | movdqa %xmm3, %xmm13 | ||
162 | pclmulqdq $0x0, %xmm10, %xmm2 | ||
163 | pclmulqdq $0x11, %xmm10, %xmm8 | ||
164 | pclmulqdq $0x0, %xmm10, %xmm3 | ||
165 | pclmulqdq $0x11, %xmm10, %xmm13 | ||
166 | pxor %xmm9 , %xmm2 | ||
167 | xorps %xmm8 , %xmm2 | ||
168 | pxor %xmm12, %xmm3 | ||
169 | xorps %xmm13, %xmm3 | ||
170 | |||
171 | movdqu 16*4(arg2), %xmm9 | ||
172 | movdqu 16*5(arg2), %xmm12 | ||
173 | pshufb %xmm11, %xmm9 | ||
174 | pshufb %xmm11, %xmm12 | ||
175 | movdqa %xmm4, %xmm8 | ||
176 | movdqa %xmm5, %xmm13 | ||
177 | pclmulqdq $0x0, %xmm10, %xmm4 | ||
178 | pclmulqdq $0x11, %xmm10, %xmm8 | ||
179 | pclmulqdq $0x0, %xmm10, %xmm5 | ||
180 | pclmulqdq $0x11, %xmm10, %xmm13 | ||
181 | pxor %xmm9 , %xmm4 | ||
182 | xorps %xmm8 , %xmm4 | ||
183 | pxor %xmm12, %xmm5 | ||
184 | xorps %xmm13, %xmm5 | ||
185 | |||
186 | movdqu 16*6(arg2), %xmm9 | ||
187 | movdqu 16*7(arg2), %xmm12 | ||
188 | pshufb %xmm11, %xmm9 | ||
189 | pshufb %xmm11, %xmm12 | ||
190 | movdqa %xmm6 , %xmm8 | ||
191 | movdqa %xmm7 , %xmm13 | ||
192 | pclmulqdq $0x0 , %xmm10, %xmm6 | ||
193 | pclmulqdq $0x11, %xmm10, %xmm8 | ||
194 | pclmulqdq $0x0 , %xmm10, %xmm7 | ||
195 | pclmulqdq $0x11, %xmm10, %xmm13 | ||
196 | pxor %xmm9 , %xmm6 | ||
197 | xorps %xmm8 , %xmm6 | ||
198 | pxor %xmm12, %xmm7 | ||
199 | xorps %xmm13, %xmm7 | ||
200 | |||
201 | sub $128, arg3 | ||
202 | |||
203 | # check if there is another 64B in the buffer to be able to fold | ||
204 | jge _fold_64_B_loop | ||
205 | ################################################################## | ||
206 | |||
207 | |||
208 | add $128, arg2 | ||
209 | # at this point, the buffer pointer is pointing at the last y Bytes | ||
210 | # of the buffer the 64B of folded data is in 4 of the xmm | ||
211 | # registers: xmm0, xmm1, xmm2, xmm3 | ||
212 | |||
213 | |||
214 | # fold the 8 xmm registers to 1 xmm register with different constants | ||
215 | |||
216 | movdqa rk9(%rip), %xmm10 | ||
217 | movdqa %xmm0, %xmm8 | ||
218 | pclmulqdq $0x11, %xmm10, %xmm0 | ||
219 | pclmulqdq $0x0 , %xmm10, %xmm8 | ||
220 | pxor %xmm8, %xmm7 | ||
221 | xorps %xmm0, %xmm7 | ||
222 | |||
223 | movdqa rk11(%rip), %xmm10 | ||
224 | movdqa %xmm1, %xmm8 | ||
225 | pclmulqdq $0x11, %xmm10, %xmm1 | ||
226 | pclmulqdq $0x0 , %xmm10, %xmm8 | ||
227 | pxor %xmm8, %xmm7 | ||
228 | xorps %xmm1, %xmm7 | ||
229 | |||
230 | movdqa rk13(%rip), %xmm10 | ||
231 | movdqa %xmm2, %xmm8 | ||
232 | pclmulqdq $0x11, %xmm10, %xmm2 | ||
233 | pclmulqdq $0x0 , %xmm10, %xmm8 | ||
234 | pxor %xmm8, %xmm7 | ||
235 | pxor %xmm2, %xmm7 | ||
236 | |||
237 | movdqa rk15(%rip), %xmm10 | ||
238 | movdqa %xmm3, %xmm8 | ||
239 | pclmulqdq $0x11, %xmm10, %xmm3 | ||
240 | pclmulqdq $0x0 , %xmm10, %xmm8 | ||
241 | pxor %xmm8, %xmm7 | ||
242 | xorps %xmm3, %xmm7 | ||
243 | |||
244 | movdqa rk17(%rip), %xmm10 | ||
245 | movdqa %xmm4, %xmm8 | ||
246 | pclmulqdq $0x11, %xmm10, %xmm4 | ||
247 | pclmulqdq $0x0 , %xmm10, %xmm8 | ||
248 | pxor %xmm8, %xmm7 | ||
249 | pxor %xmm4, %xmm7 | ||
250 | |||
251 | movdqa rk19(%rip), %xmm10 | ||
252 | movdqa %xmm5, %xmm8 | ||
253 | pclmulqdq $0x11, %xmm10, %xmm5 | ||
254 | pclmulqdq $0x0 , %xmm10, %xmm8 | ||
255 | pxor %xmm8, %xmm7 | ||
256 | xorps %xmm5, %xmm7 | ||
257 | |||
258 | movdqa rk1(%rip), %xmm10 #xmm10 has rk1 and rk2 | ||
259 | #imm value of pclmulqdq instruction | ||
260 | #will determine which constant to use | ||
261 | movdqa %xmm6, %xmm8 | ||
262 | pclmulqdq $0x11, %xmm10, %xmm6 | ||
263 | pclmulqdq $0x0 , %xmm10, %xmm8 | ||
264 | pxor %xmm8, %xmm7 | ||
265 | pxor %xmm6, %xmm7 | ||
266 | |||
267 | |||
268 | # instead of 64, we add 48 to the loop counter to save 1 instruction | ||
269 | # from the loop instead of a cmp instruction, we use the negative | ||
270 | # flag with the jl instruction | ||
271 | add $128-16, arg3 | ||
272 | jl _final_reduction_for_128 | ||
273 | |||
274 | # now we have 16+y bytes left to reduce. 16 Bytes is in register xmm7 | ||
275 | # and the rest is in memory. We can fold 16 bytes at a time if y>=16 | ||
276 | # continue folding 16B at a time | ||
277 | |||
278 | _16B_reduction_loop: | ||
279 | movdqa %xmm7, %xmm8 | ||
280 | pclmulqdq $0x11, %xmm10, %xmm7 | ||
281 | pclmulqdq $0x0 , %xmm10, %xmm8 | ||
282 | pxor %xmm8, %xmm7 | ||
283 | movdqu (arg2), %xmm0 | ||
284 | pshufb %xmm11, %xmm0 | ||
285 | pxor %xmm0 , %xmm7 | ||
286 | add $16, arg2 | ||
287 | sub $16, arg3 | ||
288 | # instead of a cmp instruction, we utilize the flags with the | ||
289 | # jge instruction equivalent of: cmp arg3, 16-16 | ||
290 | # check if there is any more 16B in the buffer to be able to fold | ||
291 | jge _16B_reduction_loop | ||
292 | |||
293 | #now we have 16+z bytes left to reduce, where 0<= z < 16. | ||
294 | #first, we reduce the data in the xmm7 register | ||
295 | |||
296 | |||
297 | _final_reduction_for_128: | ||
298 | # check if any more data to fold. If not, compute the CRC of | ||
299 | # the final 128 bits | ||
300 | add $16, arg3 | ||
301 | je _128_done | ||
302 | |||
303 | # here we are getting data that is less than 16 bytes. | ||
304 | # since we know that there was data before the pointer, we can | ||
305 | # offset the input pointer before the actual point, to receive | ||
306 | # exactly 16 bytes. after that the registers need to be adjusted. | ||
307 | _get_last_two_xmms: | ||
308 | movdqa %xmm7, %xmm2 | ||
309 | |||
310 | movdqu -16(arg2, arg3), %xmm1 | ||
311 | pshufb %xmm11, %xmm1 | ||
312 | |||
313 | # get rid of the extra data that was loaded before | ||
314 | # load the shift constant | ||
315 | lea pshufb_shf_table+16(%rip), %rax | ||
316 | sub arg3, %rax | ||
317 | movdqu (%rax), %xmm0 | ||
318 | |||
319 | # shift xmm2 to the left by arg3 bytes | ||
320 | pshufb %xmm0, %xmm2 | ||
321 | |||
322 | # shift xmm7 to the right by 16-arg3 bytes | ||
323 | pxor mask1(%rip), %xmm0 | ||
324 | pshufb %xmm0, %xmm7 | ||
325 | pblendvb %xmm2, %xmm1 #xmm0 is implicit | ||
326 | |||
327 | # fold 16 Bytes | ||
328 | movdqa %xmm1, %xmm2 | ||
329 | movdqa %xmm7, %xmm8 | ||
330 | pclmulqdq $0x11, %xmm10, %xmm7 | ||
331 | pclmulqdq $0x0 , %xmm10, %xmm8 | ||
332 | pxor %xmm8, %xmm7 | ||
333 | pxor %xmm2, %xmm7 | ||
334 | |||
335 | _128_done: | ||
336 | # compute crc of a 128-bit value | ||
337 | movdqa rk5(%rip), %xmm10 # rk5 and rk6 in xmm10 | ||
338 | movdqa %xmm7, %xmm0 | ||
339 | |||
340 | #64b fold | ||
341 | pclmulqdq $0x1, %xmm10, %xmm7 | ||
342 | pslldq $8 , %xmm0 | ||
343 | pxor %xmm0, %xmm7 | ||
344 | |||
345 | #32b fold | ||
346 | movdqa %xmm7, %xmm0 | ||
347 | |||
348 | pand mask2(%rip), %xmm0 | ||
349 | |||
350 | psrldq $12, %xmm7 | ||
351 | pclmulqdq $0x10, %xmm10, %xmm7 | ||
352 | pxor %xmm0, %xmm7 | ||
353 | |||
354 | #barrett reduction | ||
355 | _barrett: | ||
356 | movdqa rk7(%rip), %xmm10 # rk7 and rk8 in xmm10 | ||
357 | movdqa %xmm7, %xmm0 | ||
358 | pclmulqdq $0x01, %xmm10, %xmm7 | ||
359 | pslldq $4, %xmm7 | ||
360 | pclmulqdq $0x11, %xmm10, %xmm7 | ||
361 | |||
362 | pslldq $4, %xmm7 | ||
363 | pxor %xmm0, %xmm7 | ||
364 | pextrd $1, %xmm7, %eax | ||
365 | |||
366 | _cleanup: | ||
367 | # scale the result back to 16 bits | ||
368 | shr $16, %eax | ||
369 | mov %rcx, %rsp | ||
370 | ret | ||
371 | |||
372 | ######################################################################## | ||
373 | |||
374 | .align 16 | ||
375 | _less_than_128: | ||
376 | |||
377 | # check if there is enough buffer to be able to fold 16B at a time | ||
378 | cmp $32, arg3 | ||
379 | jl _less_than_32 | ||
380 | movdqa SHUF_MASK(%rip), %xmm11 | ||
381 | |||
382 | # now if there is, load the constants | ||
383 | movdqa rk1(%rip), %xmm10 # rk1 and rk2 in xmm10 | ||
384 | |||
385 | movd arg1_low32, %xmm0 # get the initial crc value | ||
386 | pslldq $12, %xmm0 # align it to its correct place | ||
387 | movdqu (arg2), %xmm7 # load the plaintext | ||
388 | pshufb %xmm11, %xmm7 # byte-reflect the plaintext | ||
389 | pxor %xmm0, %xmm7 | ||
390 | |||
391 | |||
392 | # update the buffer pointer | ||
393 | add $16, arg2 | ||
394 | |||
395 | # update the counter. subtract 32 instead of 16 to save one | ||
396 | # instruction from the loop | ||
397 | sub $32, arg3 | ||
398 | |||
399 | jmp _16B_reduction_loop | ||
400 | |||
401 | |||
402 | .align 16 | ||
403 | _less_than_32: | ||
404 | # mov initial crc to the return value. this is necessary for | ||
405 | # zero-length buffers. | ||
406 | mov arg1_low32, %eax | ||
407 | test arg3, arg3 | ||
408 | je _cleanup | ||
409 | |||
410 | movdqa SHUF_MASK(%rip), %xmm11 | ||
411 | |||
412 | movd arg1_low32, %xmm0 # get the initial crc value | ||
413 | pslldq $12, %xmm0 # align it to its correct place | ||
414 | |||
415 | cmp $16, arg3 | ||
416 | je _exact_16_left | ||
417 | jl _less_than_16_left | ||
418 | |||
419 | movdqu (arg2), %xmm7 # load the plaintext | ||
420 | pshufb %xmm11, %xmm7 # byte-reflect the plaintext | ||
421 | pxor %xmm0 , %xmm7 # xor the initial crc value | ||
422 | add $16, arg2 | ||
423 | sub $16, arg3 | ||
424 | movdqa rk1(%rip), %xmm10 # rk1 and rk2 in xmm10 | ||
425 | jmp _get_last_two_xmms | ||
426 | |||
427 | |||
428 | .align 16 | ||
429 | _less_than_16_left: | ||
430 | # use stack space to load data less than 16 bytes, zero-out | ||
431 | # the 16B in memory first. | ||
432 | |||
433 | pxor %xmm1, %xmm1 | ||
434 | mov %rsp, %r11 | ||
435 | movdqa %xmm1, (%r11) | ||
436 | |||
437 | cmp $4, arg3 | ||
438 | jl _only_less_than_4 | ||
439 | |||
440 | # backup the counter value | ||
441 | mov arg3, %r9 | ||
442 | cmp $8, arg3 | ||
443 | jl _less_than_8_left | ||
444 | |||
445 | # load 8 Bytes | ||
446 | mov (arg2), %rax | ||
447 | mov %rax, (%r11) | ||
448 | add $8, %r11 | ||
449 | sub $8, arg3 | ||
450 | add $8, arg2 | ||
451 | _less_than_8_left: | ||
452 | |||
453 | cmp $4, arg3 | ||
454 | jl _less_than_4_left | ||
455 | |||
456 | # load 4 Bytes | ||
457 | mov (arg2), %eax | ||
458 | mov %eax, (%r11) | ||
459 | add $4, %r11 | ||
460 | sub $4, arg3 | ||
461 | add $4, arg2 | ||
462 | _less_than_4_left: | ||
463 | |||
464 | cmp $2, arg3 | ||
465 | jl _less_than_2_left | ||
466 | |||
467 | # load 2 Bytes | ||
468 | mov (arg2), %ax | ||
469 | mov %ax, (%r11) | ||
470 | add $2, %r11 | ||
471 | sub $2, arg3 | ||
472 | add $2, arg2 | ||
473 | _less_than_2_left: | ||
474 | cmp $1, arg3 | ||
475 | jl _zero_left | ||
476 | |||
477 | # load 1 Byte | ||
478 | mov (arg2), %al | ||
479 | mov %al, (%r11) | ||
480 | _zero_left: | ||
481 | movdqa (%rsp), %xmm7 | ||
482 | pshufb %xmm11, %xmm7 | ||
483 | pxor %xmm0 , %xmm7 # xor the initial crc value | ||
484 | |||
485 | # shl r9, 4 | ||
486 | lea pshufb_shf_table+16(%rip), %rax | ||
487 | sub %r9, %rax | ||
488 | movdqu (%rax), %xmm0 | ||
489 | pxor mask1(%rip), %xmm0 | ||
490 | |||
491 | pshufb %xmm0, %xmm7 | ||
492 | jmp _128_done | ||
493 | |||
494 | .align 16 | ||
495 | _exact_16_left: | ||
496 | movdqu (arg2), %xmm7 | ||
497 | pshufb %xmm11, %xmm7 | ||
498 | pxor %xmm0 , %xmm7 # xor the initial crc value | ||
499 | |||
500 | jmp _128_done | ||
501 | |||
502 | _only_less_than_4: | ||
503 | cmp $3, arg3 | ||
504 | jl _only_less_than_3 | ||
505 | |||
506 | # load 3 Bytes | ||
507 | mov (arg2), %al | ||
508 | mov %al, (%r11) | ||
509 | |||
510 | mov 1(arg2), %al | ||
511 | mov %al, 1(%r11) | ||
512 | |||
513 | mov 2(arg2), %al | ||
514 | mov %al, 2(%r11) | ||
515 | |||
516 | movdqa (%rsp), %xmm7 | ||
517 | pshufb %xmm11, %xmm7 | ||
518 | pxor %xmm0 , %xmm7 # xor the initial crc value | ||
519 | |||
520 | psrldq $5, %xmm7 | ||
521 | |||
522 | jmp _barrett | ||
523 | _only_less_than_3: | ||
524 | cmp $2, arg3 | ||
525 | jl _only_less_than_2 | ||
526 | |||
527 | # load 2 Bytes | ||
528 | mov (arg2), %al | ||
529 | mov %al, (%r11) | ||
530 | |||
531 | mov 1(arg2), %al | ||
532 | mov %al, 1(%r11) | ||
533 | |||
534 | movdqa (%rsp), %xmm7 | ||
535 | pshufb %xmm11, %xmm7 | ||
536 | pxor %xmm0 , %xmm7 # xor the initial crc value | ||
537 | |||
538 | psrldq $6, %xmm7 | ||
539 | |||
540 | jmp _barrett | ||
541 | _only_less_than_2: | ||
542 | |||
543 | # load 1 Byte | ||
544 | mov (arg2), %al | ||
545 | mov %al, (%r11) | ||
546 | |||
547 | movdqa (%rsp), %xmm7 | ||
548 | pshufb %xmm11, %xmm7 | ||
549 | pxor %xmm0 , %xmm7 # xor the initial crc value | ||
550 | |||
551 | psrldq $7, %xmm7 | ||
552 | |||
553 | jmp _barrett | ||
554 | |||
555 | ENDPROC(crc_t10dif_pcl) | ||
556 | |||
557 | .data | ||
558 | |||
559 | # precomputed constants | ||
560 | # these constants are precomputed from the poly: | ||
561 | # 0x8bb70000 (0x8bb7 scaled to 32 bits) | ||
562 | .align 16 | ||
563 | # Q = 0x18BB70000 | ||
564 | # rk1 = 2^(32*3) mod Q << 32 | ||
565 | # rk2 = 2^(32*5) mod Q << 32 | ||
566 | # rk3 = 2^(32*15) mod Q << 32 | ||
567 | # rk4 = 2^(32*17) mod Q << 32 | ||
568 | # rk5 = 2^(32*3) mod Q << 32 | ||
569 | # rk6 = 2^(32*2) mod Q << 32 | ||
570 | # rk7 = floor(2^64/Q) | ||
571 | # rk8 = Q | ||
572 | rk1: | ||
573 | .quad 0x2d56000000000000 | ||
574 | rk2: | ||
575 | .quad 0x06df000000000000 | ||
576 | rk3: | ||
577 | .quad 0x9d9d000000000000 | ||
578 | rk4: | ||
579 | .quad 0x7cf5000000000000 | ||
580 | rk5: | ||
581 | .quad 0x2d56000000000000 | ||
582 | rk6: | ||
583 | .quad 0x1368000000000000 | ||
584 | rk7: | ||
585 | .quad 0x00000001f65a57f8 | ||
586 | rk8: | ||
587 | .quad 0x000000018bb70000 | ||
588 | |||
589 | rk9: | ||
590 | .quad 0xceae000000000000 | ||
591 | rk10: | ||
592 | .quad 0xbfd6000000000000 | ||
593 | rk11: | ||
594 | .quad 0x1e16000000000000 | ||
595 | rk12: | ||
596 | .quad 0x713c000000000000 | ||
597 | rk13: | ||
598 | .quad 0xf7f9000000000000 | ||
599 | rk14: | ||
600 | .quad 0x80a6000000000000 | ||
601 | rk15: | ||
602 | .quad 0x044c000000000000 | ||
603 | rk16: | ||
604 | .quad 0xe658000000000000 | ||
605 | rk17: | ||
606 | .quad 0xad18000000000000 | ||
607 | rk18: | ||
608 | .quad 0xa497000000000000 | ||
609 | rk19: | ||
610 | .quad 0x6ee3000000000000 | ||
611 | rk20: | ||
612 | .quad 0xe7b5000000000000 | ||
613 | |||
614 | |||
615 | |||
616 | mask1: | ||
617 | .octa 0x80808080808080808080808080808080 | ||
618 | mask2: | ||
619 | .octa 0x00000000FFFFFFFFFFFFFFFFFFFFFFFF | ||
620 | |||
621 | SHUF_MASK: | ||
622 | .octa 0x000102030405060708090A0B0C0D0E0F | ||
623 | |||
624 | pshufb_shf_table: | ||
625 | # use these values for shift constants for the pshufb instruction | ||
626 | # different alignments result in values as shown: | ||
627 | # DDQ 0x008f8e8d8c8b8a898887868584838281 # shl 15 (16-1) / shr1 | ||
628 | # DDQ 0x01008f8e8d8c8b8a8988878685848382 # shl 14 (16-3) / shr2 | ||
629 | # DDQ 0x0201008f8e8d8c8b8a89888786858483 # shl 13 (16-4) / shr3 | ||
630 | # DDQ 0x030201008f8e8d8c8b8a898887868584 # shl 12 (16-4) / shr4 | ||
631 | # DDQ 0x04030201008f8e8d8c8b8a8988878685 # shl 11 (16-5) / shr5 | ||
632 | # DDQ 0x0504030201008f8e8d8c8b8a89888786 # shl 10 (16-6) / shr6 | ||
633 | # DDQ 0x060504030201008f8e8d8c8b8a898887 # shl 9 (16-7) / shr7 | ||
634 | # DDQ 0x07060504030201008f8e8d8c8b8a8988 # shl 8 (16-8) / shr8 | ||
635 | # DDQ 0x0807060504030201008f8e8d8c8b8a89 # shl 7 (16-9) / shr9 | ||
636 | # DDQ 0x090807060504030201008f8e8d8c8b8a # shl 6 (16-10) / shr10 | ||
637 | # DDQ 0x0a090807060504030201008f8e8d8c8b # shl 5 (16-11) / shr11 | ||
638 | # DDQ 0x0b0a090807060504030201008f8e8d8c # shl 4 (16-12) / shr12 | ||
639 | # DDQ 0x0c0b0a090807060504030201008f8e8d # shl 3 (16-13) / shr13 | ||
640 | # DDQ 0x0d0c0b0a090807060504030201008f8e # shl 2 (16-14) / shr14 | ||
641 | # DDQ 0x0e0d0c0b0a090807060504030201008f # shl 1 (16-15) / shr15 | ||
642 | .octa 0x8f8e8d8c8b8a89888786858483828100 | ||
643 | .octa 0x000e0d0c0b0a09080706050403020100 | ||
diff --git a/arch/x86/crypto/crct10dif-pclmul_glue.c b/arch/x86/crypto/crct10dif-pclmul_glue.c new file mode 100644 index 000000000000..7845d7fd54c0 --- /dev/null +++ b/arch/x86/crypto/crct10dif-pclmul_glue.c | |||
@@ -0,0 +1,151 @@ | |||
1 | /* | ||
2 | * Cryptographic API. | ||
3 | * | ||
4 | * T10 Data Integrity Field CRC16 Crypto Transform using PCLMULQDQ Instructions | ||
5 | * | ||
6 | * Copyright (C) 2013 Intel Corporation | ||
7 | * Author: Tim Chen <tim.c.chen@linux.intel.com> | ||
8 | * | ||
9 | * This program is free software; you can redistribute it and/or modify it | ||
10 | * under the terms of the GNU General Public License as published by the Free | ||
11 | * Software Foundation; either version 2 of the License, or (at your option) | ||
12 | * any later version. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | ||
15 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | ||
16 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | ||
17 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | ||
18 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | ||
19 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | ||
20 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
21 | * SOFTWARE. | ||
22 | * | ||
23 | */ | ||
24 | |||
25 | #include <linux/types.h> | ||
26 | #include <linux/module.h> | ||
27 | #include <linux/crc-t10dif.h> | ||
28 | #include <crypto/internal/hash.h> | ||
29 | #include <linux/init.h> | ||
30 | #include <linux/string.h> | ||
31 | #include <linux/kernel.h> | ||
32 | #include <asm/i387.h> | ||
33 | #include <asm/cpufeature.h> | ||
34 | #include <asm/cpu_device_id.h> | ||
35 | |||
36 | asmlinkage __u16 crc_t10dif_pcl(__u16 crc, const unsigned char *buf, | ||
37 | size_t len); | ||
38 | |||
39 | struct chksum_desc_ctx { | ||
40 | __u16 crc; | ||
41 | }; | ||
42 | |||
43 | /* | ||
44 | * Steps through buffer one byte at at time, calculates reflected | ||
45 | * crc using table. | ||
46 | */ | ||
47 | |||
48 | static int chksum_init(struct shash_desc *desc) | ||
49 | { | ||
50 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); | ||
51 | |||
52 | ctx->crc = 0; | ||
53 | |||
54 | return 0; | ||
55 | } | ||
56 | |||
57 | static int chksum_update(struct shash_desc *desc, const u8 *data, | ||
58 | unsigned int length) | ||
59 | { | ||
60 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); | ||
61 | |||
62 | if (irq_fpu_usable()) { | ||
63 | kernel_fpu_begin(); | ||
64 | ctx->crc = crc_t10dif_pcl(ctx->crc, data, length); | ||
65 | kernel_fpu_end(); | ||
66 | } else | ||
67 | ctx->crc = crc_t10dif_generic(ctx->crc, data, length); | ||
68 | return 0; | ||
69 | } | ||
70 | |||
71 | static int chksum_final(struct shash_desc *desc, u8 *out) | ||
72 | { | ||
73 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); | ||
74 | |||
75 | *(__u16 *)out = ctx->crc; | ||
76 | return 0; | ||
77 | } | ||
78 | |||
79 | static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len, | ||
80 | u8 *out) | ||
81 | { | ||
82 | if (irq_fpu_usable()) { | ||
83 | kernel_fpu_begin(); | ||
84 | *(__u16 *)out = crc_t10dif_pcl(*crcp, data, len); | ||
85 | kernel_fpu_end(); | ||
86 | } else | ||
87 | *(__u16 *)out = crc_t10dif_generic(*crcp, data, len); | ||
88 | return 0; | ||
89 | } | ||
90 | |||
91 | static int chksum_finup(struct shash_desc *desc, const u8 *data, | ||
92 | unsigned int len, u8 *out) | ||
93 | { | ||
94 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); | ||
95 | |||
96 | return __chksum_finup(&ctx->crc, data, len, out); | ||
97 | } | ||
98 | |||
99 | static int chksum_digest(struct shash_desc *desc, const u8 *data, | ||
100 | unsigned int length, u8 *out) | ||
101 | { | ||
102 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); | ||
103 | |||
104 | return __chksum_finup(&ctx->crc, data, length, out); | ||
105 | } | ||
106 | |||
107 | static struct shash_alg alg = { | ||
108 | .digestsize = CRC_T10DIF_DIGEST_SIZE, | ||
109 | .init = chksum_init, | ||
110 | .update = chksum_update, | ||
111 | .final = chksum_final, | ||
112 | .finup = chksum_finup, | ||
113 | .digest = chksum_digest, | ||
114 | .descsize = sizeof(struct chksum_desc_ctx), | ||
115 | .base = { | ||
116 | .cra_name = "crct10dif", | ||
117 | .cra_driver_name = "crct10dif-pclmul", | ||
118 | .cra_priority = 200, | ||
119 | .cra_blocksize = CRC_T10DIF_BLOCK_SIZE, | ||
120 | .cra_module = THIS_MODULE, | ||
121 | } | ||
122 | }; | ||
123 | |||
124 | static const struct x86_cpu_id crct10dif_cpu_id[] = { | ||
125 | X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ), | ||
126 | {} | ||
127 | }; | ||
128 | MODULE_DEVICE_TABLE(x86cpu, crct10dif_cpu_id); | ||
129 | |||
130 | static int __init crct10dif_intel_mod_init(void) | ||
131 | { | ||
132 | if (!x86_match_cpu(crct10dif_cpu_id)) | ||
133 | return -ENODEV; | ||
134 | |||
135 | return crypto_register_shash(&alg); | ||
136 | } | ||
137 | |||
138 | static void __exit crct10dif_intel_mod_fini(void) | ||
139 | { | ||
140 | crypto_unregister_shash(&alg); | ||
141 | } | ||
142 | |||
143 | module_init(crct10dif_intel_mod_init); | ||
144 | module_exit(crct10dif_intel_mod_fini); | ||
145 | |||
146 | MODULE_AUTHOR("Tim Chen <tim.c.chen@linux.intel.com>"); | ||
147 | MODULE_DESCRIPTION("T10 DIF CRC calculation accelerated with PCLMULQDQ."); | ||
148 | MODULE_LICENSE("GPL"); | ||
149 | |||
150 | MODULE_ALIAS("crct10dif"); | ||
151 | MODULE_ALIAS("crct10dif-pclmul"); | ||
diff --git a/arch/x86/include/asm/xor_avx.h b/arch/x86/include/asm/xor_avx.h index 7ea79c5fa1f2..492b29802f57 100644 --- a/arch/x86/include/asm/xor_avx.h +++ b/arch/x86/include/asm/xor_avx.h | |||
@@ -167,12 +167,12 @@ static struct xor_block_template xor_block_avx = { | |||
167 | 167 | ||
168 | #define AVX_XOR_SPEED \ | 168 | #define AVX_XOR_SPEED \ |
169 | do { \ | 169 | do { \ |
170 | if (cpu_has_avx) \ | 170 | if (cpu_has_avx && cpu_has_osxsave) \ |
171 | xor_speed(&xor_block_avx); \ | 171 | xor_speed(&xor_block_avx); \ |
172 | } while (0) | 172 | } while (0) |
173 | 173 | ||
174 | #define AVX_SELECT(FASTEST) \ | 174 | #define AVX_SELECT(FASTEST) \ |
175 | (cpu_has_avx ? &xor_block_avx : FASTEST) | 175 | (cpu_has_avx && cpu_has_osxsave ? &xor_block_avx : FASTEST) |
176 | 176 | ||
177 | #else | 177 | #else |
178 | 178 | ||