diff options
Diffstat (limited to 'arch/x86')
-rw-r--r-- | arch/x86/crypto/aesni-intel_asm.S | 5 | ||||
-rw-r--r-- | arch/x86/crypto/aesni-intel_glue.c | 14 |
2 files changed, 16 insertions, 3 deletions
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index adcf794b22e2..be6d9e365a80 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S | |||
@@ -1612,6 +1612,7 @@ _zero_cipher_left_encrypt: | |||
1612 | movdqa SHUF_MASK(%rip), %xmm10 | 1612 | movdqa SHUF_MASK(%rip), %xmm10 |
1613 | PSHUFB_XMM %xmm10, %xmm0 | 1613 | PSHUFB_XMM %xmm10, %xmm0 |
1614 | 1614 | ||
1615 | |||
1615 | ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # Encrypt(K, Yn) | 1616 | ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # Encrypt(K, Yn) |
1616 | sub $16, %r11 | 1617 | sub $16, %r11 |
1617 | add %r13, %r11 | 1618 | add %r13, %r11 |
@@ -1634,7 +1635,9 @@ _zero_cipher_left_encrypt: | |||
1634 | # GHASH computation for the last <16 byte block | 1635 | # GHASH computation for the last <16 byte block |
1635 | sub %r13, %r11 | 1636 | sub %r13, %r11 |
1636 | add $16, %r11 | 1637 | add $16, %r11 |
1637 | PSHUFB_XMM %xmm10, %xmm1 | 1638 | |
1639 | movdqa SHUF_MASK(%rip), %xmm10 | ||
1640 | PSHUFB_XMM %xmm10, %xmm0 | ||
1638 | 1641 | ||
1639 | # shuffle xmm0 back to output as ciphertext | 1642 | # shuffle xmm0 back to output as ciphertext |
1640 | 1643 | ||
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index e0e6340c8dad..2577613fb32b 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c | |||
@@ -828,9 +828,15 @@ static int rfc4106_init(struct crypto_tfm *tfm) | |||
828 | struct cryptd_aead *cryptd_tfm; | 828 | struct cryptd_aead *cryptd_tfm; |
829 | struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) | 829 | struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *) |
830 | PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); | 830 | PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); |
831 | struct crypto_aead *cryptd_child; | ||
832 | struct aesni_rfc4106_gcm_ctx *child_ctx; | ||
831 | cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); | 833 | cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); |
832 | if (IS_ERR(cryptd_tfm)) | 834 | if (IS_ERR(cryptd_tfm)) |
833 | return PTR_ERR(cryptd_tfm); | 835 | return PTR_ERR(cryptd_tfm); |
836 | |||
837 | cryptd_child = cryptd_aead_child(cryptd_tfm); | ||
838 | child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child); | ||
839 | memcpy(child_ctx, ctx, sizeof(*ctx)); | ||
834 | ctx->cryptd_tfm = cryptd_tfm; | 840 | ctx->cryptd_tfm = cryptd_tfm; |
835 | tfm->crt_aead.reqsize = sizeof(struct aead_request) | 841 | tfm->crt_aead.reqsize = sizeof(struct aead_request) |
836 | + crypto_aead_reqsize(&cryptd_tfm->base); | 842 | + crypto_aead_reqsize(&cryptd_tfm->base); |
@@ -923,6 +929,9 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, | |||
923 | int ret = 0; | 929 | int ret = 0; |
924 | struct crypto_tfm *tfm = crypto_aead_tfm(parent); | 930 | struct crypto_tfm *tfm = crypto_aead_tfm(parent); |
925 | struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); | 931 | struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); |
932 | struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); | ||
933 | struct aesni_rfc4106_gcm_ctx *child_ctx = | ||
934 | aesni_rfc4106_gcm_ctx_get(cryptd_child); | ||
926 | u8 *new_key_mem = NULL; | 935 | u8 *new_key_mem = NULL; |
927 | 936 | ||
928 | if (key_len < 4) { | 937 | if (key_len < 4) { |
@@ -966,6 +975,7 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, | |||
966 | goto exit; | 975 | goto exit; |
967 | } | 976 | } |
968 | ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); | 977 | ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); |
978 | memcpy(child_ctx, ctx, sizeof(*ctx)); | ||
969 | exit: | 979 | exit: |
970 | kfree(new_key_mem); | 980 | kfree(new_key_mem); |
971 | return ret; | 981 | return ret; |
@@ -997,7 +1007,6 @@ static int rfc4106_encrypt(struct aead_request *req) | |||
997 | int ret; | 1007 | int ret; |
998 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); | 1008 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
999 | struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); | 1009 | struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); |
1000 | struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); | ||
1001 | 1010 | ||
1002 | if (!irq_fpu_usable()) { | 1011 | if (!irq_fpu_usable()) { |
1003 | struct aead_request *cryptd_req = | 1012 | struct aead_request *cryptd_req = |
@@ -1006,6 +1015,7 @@ static int rfc4106_encrypt(struct aead_request *req) | |||
1006 | aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); | 1015 | aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); |
1007 | return crypto_aead_encrypt(cryptd_req); | 1016 | return crypto_aead_encrypt(cryptd_req); |
1008 | } else { | 1017 | } else { |
1018 | struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); | ||
1009 | kernel_fpu_begin(); | 1019 | kernel_fpu_begin(); |
1010 | ret = cryptd_child->base.crt_aead.encrypt(req); | 1020 | ret = cryptd_child->base.crt_aead.encrypt(req); |
1011 | kernel_fpu_end(); | 1021 | kernel_fpu_end(); |
@@ -1018,7 +1028,6 @@ static int rfc4106_decrypt(struct aead_request *req) | |||
1018 | int ret; | 1028 | int ret; |
1019 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); | 1029 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); |
1020 | struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); | 1030 | struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); |
1021 | struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); | ||
1022 | 1031 | ||
1023 | if (!irq_fpu_usable()) { | 1032 | if (!irq_fpu_usable()) { |
1024 | struct aead_request *cryptd_req = | 1033 | struct aead_request *cryptd_req = |
@@ -1027,6 +1036,7 @@ static int rfc4106_decrypt(struct aead_request *req) | |||
1027 | aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); | 1036 | aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); |
1028 | return crypto_aead_decrypt(cryptd_req); | 1037 | return crypto_aead_decrypt(cryptd_req); |
1029 | } else { | 1038 | } else { |
1039 | struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); | ||
1030 | kernel_fpu_begin(); | 1040 | kernel_fpu_begin(); |
1031 | ret = cryptd_child->base.crt_aead.decrypt(req); | 1041 | ret = cryptd_child->base.crt_aead.decrypt(req); |
1032 | kernel_fpu_end(); | 1042 | kernel_fpu_end(); |