aboutsummaryrefslogtreecommitdiffstats
path: root/crypto
diff options
context:
space:
mode:
Diffstat (limited to 'crypto')
-rw-r--r--crypto/Kconfig26
-rw-r--r--crypto/Makefile9
-rw-r--r--crypto/ablk_helper.c150
-rw-r--r--crypto/ablkcipher.c21
-rw-r--r--crypto/algif_hash.c5
-rw-r--r--crypto/algif_skcipher.c4
-rw-r--r--crypto/ansi_cprng.c4
-rw-r--r--crypto/asymmetric_keys/Kconfig4
-rw-r--r--crypto/asymmetric_keys/asymmetric_type.c1
-rw-r--r--crypto/asymmetric_keys/public_key.c66
-rw-r--r--crypto/asymmetric_keys/public_key.h6
-rw-r--r--crypto/asymmetric_keys/rsa.c19
-rw-r--r--crypto/asymmetric_keys/x509_cert_parser.c35
-rw-r--r--crypto/asymmetric_keys/x509_parser.h18
-rw-r--r--crypto/asymmetric_keys/x509_public_key.c157
-rw-r--r--crypto/async_tx/async_memcpy.c37
-rw-r--r--crypto/async_tx/async_pq.c174
-rw-r--r--crypto/async_tx/async_raid6_recov.c61
-rw-r--r--crypto/async_tx/async_tx.c4
-rw-r--r--crypto/async_tx/async_xor.c123
-rw-r--r--crypto/async_tx/raid6test.c10
-rw-r--r--crypto/authenc.c61
-rw-r--r--crypto/authencesn.c34
-rw-r--r--crypto/ccm.c7
-rw-r--r--crypto/gcm.c2
-rw-r--r--crypto/hash_info.c56
-rw-r--r--crypto/memneq.c138
-rw-r--r--crypto/tcrypt.c4
-rw-r--r--crypto/testmgr.c26
29 files changed, 843 insertions, 419 deletions
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 71f337aefa39..7bcb70d216e1 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -174,9 +174,8 @@ config CRYPTO_TEST
174 help 174 help
175 Quick & dirty crypto test module. 175 Quick & dirty crypto test module.
176 176
177config CRYPTO_ABLK_HELPER_X86 177config CRYPTO_ABLK_HELPER
178 tristate 178 tristate
179 depends on X86
180 select CRYPTO_CRYPTD 179 select CRYPTO_CRYPTD
181 180
182config CRYPTO_GLUE_HELPER_X86 181config CRYPTO_GLUE_HELPER_X86
@@ -695,7 +694,7 @@ config CRYPTO_AES_NI_INTEL
695 select CRYPTO_AES_X86_64 if 64BIT 694 select CRYPTO_AES_X86_64 if 64BIT
696 select CRYPTO_AES_586 if !64BIT 695 select CRYPTO_AES_586 if !64BIT
697 select CRYPTO_CRYPTD 696 select CRYPTO_CRYPTD
698 select CRYPTO_ABLK_HELPER_X86 697 select CRYPTO_ABLK_HELPER
699 select CRYPTO_ALGAPI 698 select CRYPTO_ALGAPI
700 select CRYPTO_GLUE_HELPER_X86 if 64BIT 699 select CRYPTO_GLUE_HELPER_X86 if 64BIT
701 select CRYPTO_LRW 700 select CRYPTO_LRW
@@ -895,7 +894,7 @@ config CRYPTO_CAMELLIA_AESNI_AVX_X86_64
895 depends on CRYPTO 894 depends on CRYPTO
896 select CRYPTO_ALGAPI 895 select CRYPTO_ALGAPI
897 select CRYPTO_CRYPTD 896 select CRYPTO_CRYPTD
898 select CRYPTO_ABLK_HELPER_X86 897 select CRYPTO_ABLK_HELPER
899 select CRYPTO_GLUE_HELPER_X86 898 select CRYPTO_GLUE_HELPER_X86
900 select CRYPTO_CAMELLIA_X86_64 899 select CRYPTO_CAMELLIA_X86_64
901 select CRYPTO_LRW 900 select CRYPTO_LRW
@@ -917,7 +916,7 @@ config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64
917 depends on CRYPTO 916 depends on CRYPTO
918 select CRYPTO_ALGAPI 917 select CRYPTO_ALGAPI
919 select CRYPTO_CRYPTD 918 select CRYPTO_CRYPTD
920 select CRYPTO_ABLK_HELPER_X86 919 select CRYPTO_ABLK_HELPER
921 select CRYPTO_GLUE_HELPER_X86 920 select CRYPTO_GLUE_HELPER_X86
922 select CRYPTO_CAMELLIA_X86_64 921 select CRYPTO_CAMELLIA_X86_64
923 select CRYPTO_CAMELLIA_AESNI_AVX_X86_64 922 select CRYPTO_CAMELLIA_AESNI_AVX_X86_64
@@ -969,7 +968,7 @@ config CRYPTO_CAST5_AVX_X86_64
969 depends on X86 && 64BIT 968 depends on X86 && 64BIT
970 select CRYPTO_ALGAPI 969 select CRYPTO_ALGAPI
971 select CRYPTO_CRYPTD 970 select CRYPTO_CRYPTD
972 select CRYPTO_ABLK_HELPER_X86 971 select CRYPTO_ABLK_HELPER
973 select CRYPTO_CAST_COMMON 972 select CRYPTO_CAST_COMMON
974 select CRYPTO_CAST5 973 select CRYPTO_CAST5
975 help 974 help
@@ -992,7 +991,7 @@ config CRYPTO_CAST6_AVX_X86_64
992 depends on X86 && 64BIT 991 depends on X86 && 64BIT
993 select CRYPTO_ALGAPI 992 select CRYPTO_ALGAPI
994 select CRYPTO_CRYPTD 993 select CRYPTO_CRYPTD
995 select CRYPTO_ABLK_HELPER_X86 994 select CRYPTO_ABLK_HELPER
996 select CRYPTO_GLUE_HELPER_X86 995 select CRYPTO_GLUE_HELPER_X86
997 select CRYPTO_CAST_COMMON 996 select CRYPTO_CAST_COMMON
998 select CRYPTO_CAST6 997 select CRYPTO_CAST6
@@ -1110,7 +1109,7 @@ config CRYPTO_SERPENT_SSE2_X86_64
1110 depends on X86 && 64BIT 1109 depends on X86 && 64BIT
1111 select CRYPTO_ALGAPI 1110 select CRYPTO_ALGAPI
1112 select CRYPTO_CRYPTD 1111 select CRYPTO_CRYPTD
1113 select CRYPTO_ABLK_HELPER_X86 1112 select CRYPTO_ABLK_HELPER
1114 select CRYPTO_GLUE_HELPER_X86 1113 select CRYPTO_GLUE_HELPER_X86
1115 select CRYPTO_SERPENT 1114 select CRYPTO_SERPENT
1116 select CRYPTO_LRW 1115 select CRYPTO_LRW
@@ -1132,7 +1131,7 @@ config CRYPTO_SERPENT_SSE2_586
1132 depends on X86 && !64BIT 1131 depends on X86 && !64BIT
1133 select CRYPTO_ALGAPI 1132 select CRYPTO_ALGAPI
1134 select CRYPTO_CRYPTD 1133 select CRYPTO_CRYPTD
1135 select CRYPTO_ABLK_HELPER_X86 1134 select CRYPTO_ABLK_HELPER
1136 select CRYPTO_GLUE_HELPER_X86 1135 select CRYPTO_GLUE_HELPER_X86
1137 select CRYPTO_SERPENT 1136 select CRYPTO_SERPENT
1138 select CRYPTO_LRW 1137 select CRYPTO_LRW
@@ -1154,7 +1153,7 @@ config CRYPTO_SERPENT_AVX_X86_64
1154 depends on X86 && 64BIT 1153 depends on X86 && 64BIT
1155 select CRYPTO_ALGAPI 1154 select CRYPTO_ALGAPI
1156 select CRYPTO_CRYPTD 1155 select CRYPTO_CRYPTD
1157 select CRYPTO_ABLK_HELPER_X86 1156 select CRYPTO_ABLK_HELPER
1158 select CRYPTO_GLUE_HELPER_X86 1157 select CRYPTO_GLUE_HELPER_X86
1159 select CRYPTO_SERPENT 1158 select CRYPTO_SERPENT
1160 select CRYPTO_LRW 1159 select CRYPTO_LRW
@@ -1176,7 +1175,7 @@ config CRYPTO_SERPENT_AVX2_X86_64
1176 depends on X86 && 64BIT 1175 depends on X86 && 64BIT
1177 select CRYPTO_ALGAPI 1176 select CRYPTO_ALGAPI
1178 select CRYPTO_CRYPTD 1177 select CRYPTO_CRYPTD
1179 select CRYPTO_ABLK_HELPER_X86 1178 select CRYPTO_ABLK_HELPER
1180 select CRYPTO_GLUE_HELPER_X86 1179 select CRYPTO_GLUE_HELPER_X86
1181 select CRYPTO_SERPENT 1180 select CRYPTO_SERPENT
1182 select CRYPTO_SERPENT_AVX_X86_64 1181 select CRYPTO_SERPENT_AVX_X86_64
@@ -1292,7 +1291,7 @@ config CRYPTO_TWOFISH_AVX_X86_64
1292 depends on X86 && 64BIT 1291 depends on X86 && 64BIT
1293 select CRYPTO_ALGAPI 1292 select CRYPTO_ALGAPI
1294 select CRYPTO_CRYPTD 1293 select CRYPTO_CRYPTD
1295 select CRYPTO_ABLK_HELPER_X86 1294 select CRYPTO_ABLK_HELPER
1296 select CRYPTO_GLUE_HELPER_X86 1295 select CRYPTO_GLUE_HELPER_X86
1297 select CRYPTO_TWOFISH_COMMON 1296 select CRYPTO_TWOFISH_COMMON
1298 select CRYPTO_TWOFISH_X86_64 1297 select CRYPTO_TWOFISH_X86_64
@@ -1402,6 +1401,9 @@ config CRYPTO_USER_API_SKCIPHER
1402 This option enables the user-spaces interface for symmetric 1401 This option enables the user-spaces interface for symmetric
1403 key cipher algorithms. 1402 key cipher algorithms.
1404 1403
1404config CRYPTO_HASH_INFO
1405 bool
1406
1405source "drivers/crypto/Kconfig" 1407source "drivers/crypto/Kconfig"
1406source crypto/asymmetric_keys/Kconfig 1408source crypto/asymmetric_keys/Kconfig
1407 1409
diff --git a/crypto/Makefile b/crypto/Makefile
index 80019ba8da3a..989c510da8cc 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -2,8 +2,13 @@
2# Cryptographic API 2# Cryptographic API
3# 3#
4 4
5# memneq MUST be built with -Os or -O0 to prevent early-return optimizations
6# that will defeat memneq's actual purpose to prevent timing attacks.
7CFLAGS_REMOVE_memneq.o := -O1 -O2 -O3
8CFLAGS_memneq.o := -Os
9
5obj-$(CONFIG_CRYPTO) += crypto.o 10obj-$(CONFIG_CRYPTO) += crypto.o
6crypto-y := api.o cipher.o compress.o 11crypto-y := api.o cipher.o compress.o memneq.o
7 12
8obj-$(CONFIG_CRYPTO_WORKQUEUE) += crypto_wq.o 13obj-$(CONFIG_CRYPTO_WORKQUEUE) += crypto_wq.o
9 14
@@ -104,3 +109,5 @@ obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o
104obj-$(CONFIG_XOR_BLOCKS) += xor.o 109obj-$(CONFIG_XOR_BLOCKS) += xor.o
105obj-$(CONFIG_ASYNC_CORE) += async_tx/ 110obj-$(CONFIG_ASYNC_CORE) += async_tx/
106obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/ 111obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/
112obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o
113obj-$(CONFIG_CRYPTO_ABLK_HELPER) += ablk_helper.o
diff --git a/crypto/ablk_helper.c b/crypto/ablk_helper.c
new file mode 100644
index 000000000000..ffe7278d4bd8
--- /dev/null
+++ b/crypto/ablk_helper.c
@@ -0,0 +1,150 @@
1/*
2 * Shared async block cipher helpers
3 *
4 * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 *
6 * Based on aesni-intel_glue.c by:
7 * Copyright (C) 2008, Intel Corp.
8 * Author: Huang Ying <ying.huang@intel.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
23 * USA
24 *
25 */
26
27#include <linux/kernel.h>
28#include <linux/crypto.h>
29#include <linux/init.h>
30#include <linux/module.h>
31#include <linux/hardirq.h>
32#include <crypto/algapi.h>
33#include <crypto/cryptd.h>
34#include <crypto/ablk_helper.h>
35#include <asm/simd.h>
36
37int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
38 unsigned int key_len)
39{
40 struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
41 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
42 int err;
43
44 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
45 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
46 & CRYPTO_TFM_REQ_MASK);
47 err = crypto_ablkcipher_setkey(child, key, key_len);
48 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
49 & CRYPTO_TFM_RES_MASK);
50 return err;
51}
52EXPORT_SYMBOL_GPL(ablk_set_key);
53
54int __ablk_encrypt(struct ablkcipher_request *req)
55{
56 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
57 struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
58 struct blkcipher_desc desc;
59
60 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
61 desc.info = req->info;
62 desc.flags = 0;
63
64 return crypto_blkcipher_crt(desc.tfm)->encrypt(
65 &desc, req->dst, req->src, req->nbytes);
66}
67EXPORT_SYMBOL_GPL(__ablk_encrypt);
68
69int ablk_encrypt(struct ablkcipher_request *req)
70{
71 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
72 struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
73
74 if (!may_use_simd()) {
75 struct ablkcipher_request *cryptd_req =
76 ablkcipher_request_ctx(req);
77
78 *cryptd_req = *req;
79 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
80
81 return crypto_ablkcipher_encrypt(cryptd_req);
82 } else {
83 return __ablk_encrypt(req);
84 }
85}
86EXPORT_SYMBOL_GPL(ablk_encrypt);
87
88int ablk_decrypt(struct ablkcipher_request *req)
89{
90 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
91 struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
92
93 if (!may_use_simd()) {
94 struct ablkcipher_request *cryptd_req =
95 ablkcipher_request_ctx(req);
96
97 *cryptd_req = *req;
98 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
99
100 return crypto_ablkcipher_decrypt(cryptd_req);
101 } else {
102 struct blkcipher_desc desc;
103
104 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
105 desc.info = req->info;
106 desc.flags = 0;
107
108 return crypto_blkcipher_crt(desc.tfm)->decrypt(
109 &desc, req->dst, req->src, req->nbytes);
110 }
111}
112EXPORT_SYMBOL_GPL(ablk_decrypt);
113
114void ablk_exit(struct crypto_tfm *tfm)
115{
116 struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm);
117
118 cryptd_free_ablkcipher(ctx->cryptd_tfm);
119}
120EXPORT_SYMBOL_GPL(ablk_exit);
121
122int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name)
123{
124 struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm);
125 struct cryptd_ablkcipher *cryptd_tfm;
126
127 cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0);
128 if (IS_ERR(cryptd_tfm))
129 return PTR_ERR(cryptd_tfm);
130
131 ctx->cryptd_tfm = cryptd_tfm;
132 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
133 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
134
135 return 0;
136}
137EXPORT_SYMBOL_GPL(ablk_init_common);
138
139int ablk_init(struct crypto_tfm *tfm)
140{
141 char drv_name[CRYPTO_MAX_ALG_NAME];
142
143 snprintf(drv_name, sizeof(drv_name), "__driver-%s",
144 crypto_tfm_alg_driver_name(tfm));
145
146 return ablk_init_common(tfm, drv_name);
147}
148EXPORT_SYMBOL_GPL(ablk_init);
149
150MODULE_LICENSE("GPL");
diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c
index 7d4a8d28277e..40886c489903 100644
--- a/crypto/ablkcipher.c
+++ b/crypto/ablkcipher.c
@@ -16,9 +16,7 @@
16#include <crypto/internal/skcipher.h> 16#include <crypto/internal/skcipher.h>
17#include <linux/cpumask.h> 17#include <linux/cpumask.h>
18#include <linux/err.h> 18#include <linux/err.h>
19#include <linux/init.h>
20#include <linux/kernel.h> 19#include <linux/kernel.h>
21#include <linux/module.h>
22#include <linux/rtnetlink.h> 20#include <linux/rtnetlink.h>
23#include <linux/sched.h> 21#include <linux/sched.h>
24#include <linux/slab.h> 22#include <linux/slab.h>
@@ -30,8 +28,6 @@
30 28
31#include "internal.h" 29#include "internal.h"
32 30
33static const char *skcipher_default_geniv __read_mostly;
34
35struct ablkcipher_buffer { 31struct ablkcipher_buffer {
36 struct list_head entry; 32 struct list_head entry;
37 struct scatter_walk dst; 33 struct scatter_walk dst;
@@ -527,8 +523,7 @@ const char *crypto_default_geniv(const struct crypto_alg *alg)
527 alg->cra_blocksize) 523 alg->cra_blocksize)
528 return "chainiv"; 524 return "chainiv";
529 525
530 return alg->cra_flags & CRYPTO_ALG_ASYNC ? 526 return "eseqiv";
531 "eseqiv" : skcipher_default_geniv;
532} 527}
533 528
534static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask) 529static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask)
@@ -709,17 +704,3 @@ err:
709 return ERR_PTR(err); 704 return ERR_PTR(err);
710} 705}
711EXPORT_SYMBOL_GPL(crypto_alloc_ablkcipher); 706EXPORT_SYMBOL_GPL(crypto_alloc_ablkcipher);
712
713static int __init skcipher_module_init(void)
714{
715 skcipher_default_geniv = num_possible_cpus() > 1 ?
716 "eseqiv" : "chainiv";
717 return 0;
718}
719
720static void skcipher_module_exit(void)
721{
722}
723
724module_init(skcipher_module_init);
725module_exit(skcipher_module_exit);
diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c
index 0262210cad38..850246206b12 100644
--- a/crypto/algif_hash.c
+++ b/crypto/algif_hash.c
@@ -114,6 +114,9 @@ static ssize_t hash_sendpage(struct socket *sock, struct page *page,
114 struct hash_ctx *ctx = ask->private; 114 struct hash_ctx *ctx = ask->private;
115 int err; 115 int err;
116 116
117 if (flags & MSG_SENDPAGE_NOTLAST)
118 flags |= MSG_MORE;
119
117 lock_sock(sk); 120 lock_sock(sk);
118 sg_init_table(ctx->sgl.sg, 1); 121 sg_init_table(ctx->sgl.sg, 1);
119 sg_set_page(ctx->sgl.sg, page, size, offset); 122 sg_set_page(ctx->sgl.sg, page, size, offset);
@@ -161,8 +164,6 @@ static int hash_recvmsg(struct kiocb *unused, struct socket *sock,
161 else if (len < ds) 164 else if (len < ds)
162 msg->msg_flags |= MSG_TRUNC; 165 msg->msg_flags |= MSG_TRUNC;
163 166
164 msg->msg_namelen = 0;
165
166 lock_sock(sk); 167 lock_sock(sk);
167 if (ctx->more) { 168 if (ctx->more) {
168 ctx->more = 0; 169 ctx->more = 0;
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
index a1c4f0a55583..a19c027b29bd 100644
--- a/crypto/algif_skcipher.c
+++ b/crypto/algif_skcipher.c
@@ -378,6 +378,9 @@ static ssize_t skcipher_sendpage(struct socket *sock, struct page *page,
378 struct skcipher_sg_list *sgl; 378 struct skcipher_sg_list *sgl;
379 int err = -EINVAL; 379 int err = -EINVAL;
380 380
381 if (flags & MSG_SENDPAGE_NOTLAST)
382 flags |= MSG_MORE;
383
381 lock_sock(sk); 384 lock_sock(sk);
382 if (!ctx->more && ctx->used) 385 if (!ctx->more && ctx->used)
383 goto unlock; 386 goto unlock;
@@ -432,7 +435,6 @@ static int skcipher_recvmsg(struct kiocb *unused, struct socket *sock,
432 long copied = 0; 435 long copied = 0;
433 436
434 lock_sock(sk); 437 lock_sock(sk);
435 msg->msg_namelen = 0;
436 for (iov = msg->msg_iov, iovlen = msg->msg_iovlen; iovlen > 0; 438 for (iov = msg->msg_iov, iovlen = msg->msg_iovlen; iovlen > 0;
437 iovlen--, iov++) { 439 iovlen--, iov++) {
438 unsigned long seglen = iov->iov_len; 440 unsigned long seglen = iov->iov_len;
diff --git a/crypto/ansi_cprng.c b/crypto/ansi_cprng.c
index c0bb3778f1ae..666f1962a160 100644
--- a/crypto/ansi_cprng.c
+++ b/crypto/ansi_cprng.c
@@ -230,11 +230,11 @@ remainder:
230 */ 230 */
231 if (byte_count < DEFAULT_BLK_SZ) { 231 if (byte_count < DEFAULT_BLK_SZ) {
232empty_rbuf: 232empty_rbuf:
233 for (; ctx->rand_data_valid < DEFAULT_BLK_SZ; 233 while (ctx->rand_data_valid < DEFAULT_BLK_SZ) {
234 ctx->rand_data_valid++) {
235 *ptr = ctx->rand_data[ctx->rand_data_valid]; 234 *ptr = ctx->rand_data[ctx->rand_data_valid];
236 ptr++; 235 ptr++;
237 byte_count--; 236 byte_count--;
237 ctx->rand_data_valid++;
238 if (byte_count == 0) 238 if (byte_count == 0)
239 goto done; 239 goto done;
240 } 240 }
diff --git a/crypto/asymmetric_keys/Kconfig b/crypto/asymmetric_keys/Kconfig
index 6d2c2ea12559..03a6eb95ab50 100644
--- a/crypto/asymmetric_keys/Kconfig
+++ b/crypto/asymmetric_keys/Kconfig
@@ -12,6 +12,8 @@ if ASYMMETRIC_KEY_TYPE
12config ASYMMETRIC_PUBLIC_KEY_SUBTYPE 12config ASYMMETRIC_PUBLIC_KEY_SUBTYPE
13 tristate "Asymmetric public-key crypto algorithm subtype" 13 tristate "Asymmetric public-key crypto algorithm subtype"
14 select MPILIB 14 select MPILIB
15 select PUBLIC_KEY_ALGO_RSA
16 select CRYPTO_HASH_INFO
15 help 17 help
16 This option provides support for asymmetric public key type handling. 18 This option provides support for asymmetric public key type handling.
17 If signature generation and/or verification are to be used, 19 If signature generation and/or verification are to be used,
@@ -20,8 +22,8 @@ config ASYMMETRIC_PUBLIC_KEY_SUBTYPE
20 22
21config PUBLIC_KEY_ALGO_RSA 23config PUBLIC_KEY_ALGO_RSA
22 tristate "RSA public-key algorithm" 24 tristate "RSA public-key algorithm"
23 depends on ASYMMETRIC_PUBLIC_KEY_SUBTYPE
24 select MPILIB_EXTRA 25 select MPILIB_EXTRA
26 select MPILIB
25 help 27 help
26 This option enables support for the RSA algorithm (PKCS#1, RFC3447). 28 This option enables support for the RSA algorithm (PKCS#1, RFC3447).
27 29
diff --git a/crypto/asymmetric_keys/asymmetric_type.c b/crypto/asymmetric_keys/asymmetric_type.c
index cf807654d221..b77eb5304788 100644
--- a/crypto/asymmetric_keys/asymmetric_type.c
+++ b/crypto/asymmetric_keys/asymmetric_type.c
@@ -209,6 +209,7 @@ struct key_type key_type_asymmetric = {
209 .match = asymmetric_key_match, 209 .match = asymmetric_key_match,
210 .destroy = asymmetric_key_destroy, 210 .destroy = asymmetric_key_destroy,
211 .describe = asymmetric_key_describe, 211 .describe = asymmetric_key_describe,
212 .def_lookup_type = KEYRING_SEARCH_LOOKUP_ITERATE,
212}; 213};
213EXPORT_SYMBOL_GPL(key_type_asymmetric); 214EXPORT_SYMBOL_GPL(key_type_asymmetric);
214 215
diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c
index cb2e29180a87..97eb001960b9 100644
--- a/crypto/asymmetric_keys/public_key.c
+++ b/crypto/asymmetric_keys/public_key.c
@@ -22,29 +22,25 @@
22 22
23MODULE_LICENSE("GPL"); 23MODULE_LICENSE("GPL");
24 24
25const char *const pkey_algo[PKEY_ALGO__LAST] = { 25const char *const pkey_algo_name[PKEY_ALGO__LAST] = {
26 [PKEY_ALGO_DSA] = "DSA", 26 [PKEY_ALGO_DSA] = "DSA",
27 [PKEY_ALGO_RSA] = "RSA", 27 [PKEY_ALGO_RSA] = "RSA",
28}; 28};
29EXPORT_SYMBOL_GPL(pkey_algo); 29EXPORT_SYMBOL_GPL(pkey_algo_name);
30 30
31const char *const pkey_hash_algo[PKEY_HASH__LAST] = { 31const struct public_key_algorithm *pkey_algo[PKEY_ALGO__LAST] = {
32 [PKEY_HASH_MD4] = "md4", 32#if defined(CONFIG_PUBLIC_KEY_ALGO_RSA) || \
33 [PKEY_HASH_MD5] = "md5", 33 defined(CONFIG_PUBLIC_KEY_ALGO_RSA_MODULE)
34 [PKEY_HASH_SHA1] = "sha1", 34 [PKEY_ALGO_RSA] = &RSA_public_key_algorithm,
35 [PKEY_HASH_RIPE_MD_160] = "rmd160", 35#endif
36 [PKEY_HASH_SHA256] = "sha256",
37 [PKEY_HASH_SHA384] = "sha384",
38 [PKEY_HASH_SHA512] = "sha512",
39 [PKEY_HASH_SHA224] = "sha224",
40}; 36};
41EXPORT_SYMBOL_GPL(pkey_hash_algo); 37EXPORT_SYMBOL_GPL(pkey_algo);
42 38
43const char *const pkey_id_type[PKEY_ID_TYPE__LAST] = { 39const char *const pkey_id_type_name[PKEY_ID_TYPE__LAST] = {
44 [PKEY_ID_PGP] = "PGP", 40 [PKEY_ID_PGP] = "PGP",
45 [PKEY_ID_X509] = "X509", 41 [PKEY_ID_X509] = "X509",
46}; 42};
47EXPORT_SYMBOL_GPL(pkey_id_type); 43EXPORT_SYMBOL_GPL(pkey_id_type_name);
48 44
49/* 45/*
50 * Provide a part of a description of the key for /proc/keys. 46 * Provide a part of a description of the key for /proc/keys.
@@ -56,7 +52,7 @@ static void public_key_describe(const struct key *asymmetric_key,
56 52
57 if (key) 53 if (key)
58 seq_printf(m, "%s.%s", 54 seq_printf(m, "%s.%s",
59 pkey_id_type[key->id_type], key->algo->name); 55 pkey_id_type_name[key->id_type], key->algo->name);
60} 56}
61 57
62/* 58/*
@@ -78,21 +74,45 @@ EXPORT_SYMBOL_GPL(public_key_destroy);
78/* 74/*
79 * Verify a signature using a public key. 75 * Verify a signature using a public key.
80 */ 76 */
81static int public_key_verify_signature(const struct key *key, 77int public_key_verify_signature(const struct public_key *pk,
82 const struct public_key_signature *sig) 78 const struct public_key_signature *sig)
83{ 79{
84 const struct public_key *pk = key->payload.data; 80 const struct public_key_algorithm *algo;
81
82 BUG_ON(!pk);
83 BUG_ON(!pk->mpi[0]);
84 BUG_ON(!pk->mpi[1]);
85 BUG_ON(!sig);
86 BUG_ON(!sig->digest);
87 BUG_ON(!sig->mpi[0]);
88
89 algo = pk->algo;
90 if (!algo) {
91 if (pk->pkey_algo >= PKEY_ALGO__LAST)
92 return -ENOPKG;
93 algo = pkey_algo[pk->pkey_algo];
94 if (!algo)
95 return -ENOPKG;
96 }
85 97
86 if (!pk->algo->verify_signature) 98 if (!algo->verify_signature)
87 return -ENOTSUPP; 99 return -ENOTSUPP;
88 100
89 if (sig->nr_mpi != pk->algo->n_sig_mpi) { 101 if (sig->nr_mpi != algo->n_sig_mpi) {
90 pr_debug("Signature has %u MPI not %u\n", 102 pr_debug("Signature has %u MPI not %u\n",
91 sig->nr_mpi, pk->algo->n_sig_mpi); 103 sig->nr_mpi, algo->n_sig_mpi);
92 return -EINVAL; 104 return -EINVAL;
93 } 105 }
94 106
95 return pk->algo->verify_signature(pk, sig); 107 return algo->verify_signature(pk, sig);
108}
109EXPORT_SYMBOL_GPL(public_key_verify_signature);
110
111static int public_key_verify_signature_2(const struct key *key,
112 const struct public_key_signature *sig)
113{
114 const struct public_key *pk = key->payload.data;
115 return public_key_verify_signature(pk, sig);
96} 116}
97 117
98/* 118/*
@@ -103,6 +123,6 @@ struct asymmetric_key_subtype public_key_subtype = {
103 .name = "public_key", 123 .name = "public_key",
104 .describe = public_key_describe, 124 .describe = public_key_describe,
105 .destroy = public_key_destroy, 125 .destroy = public_key_destroy,
106 .verify_signature = public_key_verify_signature, 126 .verify_signature = public_key_verify_signature_2,
107}; 127};
108EXPORT_SYMBOL_GPL(public_key_subtype); 128EXPORT_SYMBOL_GPL(public_key_subtype);
diff --git a/crypto/asymmetric_keys/public_key.h b/crypto/asymmetric_keys/public_key.h
index 5e5e35626899..5c37a22a0637 100644
--- a/crypto/asymmetric_keys/public_key.h
+++ b/crypto/asymmetric_keys/public_key.h
@@ -28,3 +28,9 @@ struct public_key_algorithm {
28}; 28};
29 29
30extern const struct public_key_algorithm RSA_public_key_algorithm; 30extern const struct public_key_algorithm RSA_public_key_algorithm;
31
32/*
33 * public_key.c
34 */
35extern int public_key_verify_signature(const struct public_key *pk,
36 const struct public_key_signature *sig);
diff --git a/crypto/asymmetric_keys/rsa.c b/crypto/asymmetric_keys/rsa.c
index 4a6a0696f8a3..459cf97a75e2 100644
--- a/crypto/asymmetric_keys/rsa.c
+++ b/crypto/asymmetric_keys/rsa.c
@@ -13,6 +13,7 @@
13#include <linux/module.h> 13#include <linux/module.h>
14#include <linux/kernel.h> 14#include <linux/kernel.h>
15#include <linux/slab.h> 15#include <linux/slab.h>
16#include <crypto/algapi.h>
16#include "public_key.h" 17#include "public_key.h"
17 18
18MODULE_LICENSE("GPL"); 19MODULE_LICENSE("GPL");
@@ -73,13 +74,13 @@ static const struct {
73 size_t size; 74 size_t size;
74} RSA_ASN1_templates[PKEY_HASH__LAST] = { 75} RSA_ASN1_templates[PKEY_HASH__LAST] = {
75#define _(X) { RSA_digest_info_##X, sizeof(RSA_digest_info_##X) } 76#define _(X) { RSA_digest_info_##X, sizeof(RSA_digest_info_##X) }
76 [PKEY_HASH_MD5] = _(MD5), 77 [HASH_ALGO_MD5] = _(MD5),
77 [PKEY_HASH_SHA1] = _(SHA1), 78 [HASH_ALGO_SHA1] = _(SHA1),
78 [PKEY_HASH_RIPE_MD_160] = _(RIPE_MD_160), 79 [HASH_ALGO_RIPE_MD_160] = _(RIPE_MD_160),
79 [PKEY_HASH_SHA256] = _(SHA256), 80 [HASH_ALGO_SHA256] = _(SHA256),
80 [PKEY_HASH_SHA384] = _(SHA384), 81 [HASH_ALGO_SHA384] = _(SHA384),
81 [PKEY_HASH_SHA512] = _(SHA512), 82 [HASH_ALGO_SHA512] = _(SHA512),
82 [PKEY_HASH_SHA224] = _(SHA224), 83 [HASH_ALGO_SHA224] = _(SHA224),
83#undef _ 84#undef _
84}; 85};
85 86
@@ -189,12 +190,12 @@ static int RSA_verify(const u8 *H, const u8 *EM, size_t k, size_t hash_size,
189 } 190 }
190 } 191 }
191 192
192 if (memcmp(asn1_template, EM + T_offset, asn1_size) != 0) { 193 if (crypto_memneq(asn1_template, EM + T_offset, asn1_size) != 0) {
193 kleave(" = -EBADMSG [EM[T] ASN.1 mismatch]"); 194 kleave(" = -EBADMSG [EM[T] ASN.1 mismatch]");
194 return -EBADMSG; 195 return -EBADMSG;
195 } 196 }
196 197
197 if (memcmp(H, EM + T_offset + asn1_size, hash_size) != 0) { 198 if (crypto_memneq(H, EM + T_offset + asn1_size, hash_size) != 0) {
198 kleave(" = -EKEYREJECTED [EM[T] hash mismatch]"); 199 kleave(" = -EKEYREJECTED [EM[T] hash mismatch]");
199 return -EKEYREJECTED; 200 return -EKEYREJECTED;
200 } 201 }
diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c
index facbf26bc6bb..29893162497c 100644
--- a/crypto/asymmetric_keys/x509_cert_parser.c
+++ b/crypto/asymmetric_keys/x509_cert_parser.c
@@ -47,6 +47,8 @@ void x509_free_certificate(struct x509_certificate *cert)
47 kfree(cert->subject); 47 kfree(cert->subject);
48 kfree(cert->fingerprint); 48 kfree(cert->fingerprint);
49 kfree(cert->authority); 49 kfree(cert->authority);
50 kfree(cert->sig.digest);
51 mpi_free(cert->sig.rsa.s);
50 kfree(cert); 52 kfree(cert);
51 } 53 }
52} 54}
@@ -152,33 +154,33 @@ int x509_note_pkey_algo(void *context, size_t hdrlen,
152 return -ENOPKG; /* Unsupported combination */ 154 return -ENOPKG; /* Unsupported combination */
153 155
154 case OID_md4WithRSAEncryption: 156 case OID_md4WithRSAEncryption:
155 ctx->cert->sig_hash_algo = PKEY_HASH_MD5; 157 ctx->cert->sig.pkey_hash_algo = HASH_ALGO_MD5;
156 ctx->cert->sig_pkey_algo = PKEY_ALGO_RSA; 158 ctx->cert->sig.pkey_algo = PKEY_ALGO_RSA;
157 break; 159 break;
158 160
159 case OID_sha1WithRSAEncryption: 161 case OID_sha1WithRSAEncryption:
160 ctx->cert->sig_hash_algo = PKEY_HASH_SHA1; 162 ctx->cert->sig.pkey_hash_algo = HASH_ALGO_SHA1;
161 ctx->cert->sig_pkey_algo = PKEY_ALGO_RSA; 163 ctx->cert->sig.pkey_algo = PKEY_ALGO_RSA;
162 break; 164 break;
163 165
164 case OID_sha256WithRSAEncryption: 166 case OID_sha256WithRSAEncryption:
165 ctx->cert->sig_hash_algo = PKEY_HASH_SHA256; 167 ctx->cert->sig.pkey_hash_algo = HASH_ALGO_SHA256;
166 ctx->cert->sig_pkey_algo = PKEY_ALGO_RSA; 168 ctx->cert->sig.pkey_algo = PKEY_ALGO_RSA;
167 break; 169 break;
168 170
169 case OID_sha384WithRSAEncryption: 171 case OID_sha384WithRSAEncryption:
170 ctx->cert->sig_hash_algo = PKEY_HASH_SHA384; 172 ctx->cert->sig.pkey_hash_algo = HASH_ALGO_SHA384;
171 ctx->cert->sig_pkey_algo = PKEY_ALGO_RSA; 173 ctx->cert->sig.pkey_algo = PKEY_ALGO_RSA;
172 break; 174 break;
173 175
174 case OID_sha512WithRSAEncryption: 176 case OID_sha512WithRSAEncryption:
175 ctx->cert->sig_hash_algo = PKEY_HASH_SHA512; 177 ctx->cert->sig.pkey_hash_algo = HASH_ALGO_SHA512;
176 ctx->cert->sig_pkey_algo = PKEY_ALGO_RSA; 178 ctx->cert->sig.pkey_algo = PKEY_ALGO_RSA;
177 break; 179 break;
178 180
179 case OID_sha224WithRSAEncryption: 181 case OID_sha224WithRSAEncryption:
180 ctx->cert->sig_hash_algo = PKEY_HASH_SHA224; 182 ctx->cert->sig.pkey_hash_algo = HASH_ALGO_SHA224;
181 ctx->cert->sig_pkey_algo = PKEY_ALGO_RSA; 183 ctx->cert->sig.pkey_algo = PKEY_ALGO_RSA;
182 break; 184 break;
183 } 185 }
184 186
@@ -203,8 +205,8 @@ int x509_note_signature(void *context, size_t hdrlen,
203 return -EINVAL; 205 return -EINVAL;
204 } 206 }
205 207
206 ctx->cert->sig = value; 208 ctx->cert->raw_sig = value;
207 ctx->cert->sig_size = vlen; 209 ctx->cert->raw_sig_size = vlen;
208 return 0; 210 return 0;
209} 211}
210 212
@@ -343,8 +345,9 @@ int x509_extract_key_data(void *context, size_t hdrlen,
343 if (ctx->last_oid != OID_rsaEncryption) 345 if (ctx->last_oid != OID_rsaEncryption)
344 return -ENOPKG; 346 return -ENOPKG;
345 347
346 /* There seems to be an extraneous 0 byte on the front of the data */ 348 ctx->cert->pub->pkey_algo = PKEY_ALGO_RSA;
347 ctx->cert->pkey_algo = PKEY_ALGO_RSA; 349
350 /* Discard the BIT STRING metadata */
348 ctx->key = value + 1; 351 ctx->key = value + 1;
349 ctx->key_size = vlen - 1; 352 ctx->key_size = vlen - 1;
350 return 0; 353 return 0;
diff --git a/crypto/asymmetric_keys/x509_parser.h b/crypto/asymmetric_keys/x509_parser.h
index f86dc5fcc4ad..87d9cc26f630 100644
--- a/crypto/asymmetric_keys/x509_parser.h
+++ b/crypto/asymmetric_keys/x509_parser.h
@@ -9,6 +9,7 @@
9 * 2 of the Licence, or (at your option) any later version. 9 * 2 of the Licence, or (at your option) any later version.
10 */ 10 */
11 11
12#include <linux/time.h>
12#include <crypto/public_key.h> 13#include <crypto/public_key.h>
13 14
14struct x509_certificate { 15struct x509_certificate {
@@ -20,13 +21,11 @@ struct x509_certificate {
20 char *authority; /* Authority key fingerprint as hex */ 21 char *authority; /* Authority key fingerprint as hex */
21 struct tm valid_from; 22 struct tm valid_from;
22 struct tm valid_to; 23 struct tm valid_to;
23 enum pkey_algo pkey_algo : 8; /* Public key algorithm */
24 enum pkey_algo sig_pkey_algo : 8; /* Signature public key algorithm */
25 enum pkey_hash_algo sig_hash_algo : 8; /* Signature hash algorithm */
26 const void *tbs; /* Signed data */ 24 const void *tbs; /* Signed data */
27 size_t tbs_size; /* Size of signed data */ 25 unsigned tbs_size; /* Size of signed data */
28 const void *sig; /* Signature data */ 26 unsigned raw_sig_size; /* Size of sigature */
29 size_t sig_size; /* Size of sigature */ 27 const void *raw_sig; /* Signature data */
28 struct public_key_signature sig; /* Signature parameters */
30}; 29};
31 30
32/* 31/*
@@ -34,3 +33,10 @@ struct x509_certificate {
34 */ 33 */
35extern void x509_free_certificate(struct x509_certificate *cert); 34extern void x509_free_certificate(struct x509_certificate *cert);
36extern struct x509_certificate *x509_cert_parse(const void *data, size_t datalen); 35extern struct x509_certificate *x509_cert_parse(const void *data, size_t datalen);
36
37/*
38 * x509_public_key.c
39 */
40extern int x509_get_sig_params(struct x509_certificate *cert);
41extern int x509_check_signature(const struct public_key *pub,
42 struct x509_certificate *cert);
diff --git a/crypto/asymmetric_keys/x509_public_key.c b/crypto/asymmetric_keys/x509_public_key.c
index 06007f0e880c..382ef0d2ff2e 100644
--- a/crypto/asymmetric_keys/x509_public_key.c
+++ b/crypto/asymmetric_keys/x509_public_key.c
@@ -23,82 +23,84 @@
23#include "public_key.h" 23#include "public_key.h"
24#include "x509_parser.h" 24#include "x509_parser.h"
25 25
26static const
27struct public_key_algorithm *x509_public_key_algorithms[PKEY_ALGO__LAST] = {
28 [PKEY_ALGO_DSA] = NULL,
29#if defined(CONFIG_PUBLIC_KEY_ALGO_RSA) || \
30 defined(CONFIG_PUBLIC_KEY_ALGO_RSA_MODULE)
31 [PKEY_ALGO_RSA] = &RSA_public_key_algorithm,
32#endif
33};
34
35/* 26/*
36 * Check the signature on a certificate using the provided public key 27 * Set up the signature parameters in an X.509 certificate. This involves
28 * digesting the signed data and extracting the signature.
37 */ 29 */
38static int x509_check_signature(const struct public_key *pub, 30int x509_get_sig_params(struct x509_certificate *cert)
39 const struct x509_certificate *cert)
40{ 31{
41 struct public_key_signature *sig;
42 struct crypto_shash *tfm; 32 struct crypto_shash *tfm;
43 struct shash_desc *desc; 33 struct shash_desc *desc;
44 size_t digest_size, desc_size; 34 size_t digest_size, desc_size;
35 void *digest;
45 int ret; 36 int ret;
46 37
47 pr_devel("==>%s()\n", __func__); 38 pr_devel("==>%s()\n", __func__);
48 39
40 if (cert->sig.rsa.s)
41 return 0;
42
43 cert->sig.rsa.s = mpi_read_raw_data(cert->raw_sig, cert->raw_sig_size);
44 if (!cert->sig.rsa.s)
45 return -ENOMEM;
46 cert->sig.nr_mpi = 1;
47
49 /* Allocate the hashing algorithm we're going to need and find out how 48 /* Allocate the hashing algorithm we're going to need and find out how
50 * big the hash operational data will be. 49 * big the hash operational data will be.
51 */ 50 */
52 tfm = crypto_alloc_shash(pkey_hash_algo[cert->sig_hash_algo], 0, 0); 51 tfm = crypto_alloc_shash(hash_algo_name[cert->sig.pkey_hash_algo], 0, 0);
53 if (IS_ERR(tfm)) 52 if (IS_ERR(tfm))
54 return (PTR_ERR(tfm) == -ENOENT) ? -ENOPKG : PTR_ERR(tfm); 53 return (PTR_ERR(tfm) == -ENOENT) ? -ENOPKG : PTR_ERR(tfm);
55 54
56 desc_size = crypto_shash_descsize(tfm) + sizeof(*desc); 55 desc_size = crypto_shash_descsize(tfm) + sizeof(*desc);
57 digest_size = crypto_shash_digestsize(tfm); 56 digest_size = crypto_shash_digestsize(tfm);
58 57
59 /* We allocate the hash operational data storage on the end of our 58 /* We allocate the hash operational data storage on the end of the
60 * context data. 59 * digest storage space.
61 */ 60 */
62 ret = -ENOMEM; 61 ret = -ENOMEM;
63 sig = kzalloc(sizeof(*sig) + desc_size + digest_size, GFP_KERNEL); 62 digest = kzalloc(digest_size + desc_size, GFP_KERNEL);
64 if (!sig) 63 if (!digest)
65 goto error_no_sig; 64 goto error;
66 65
67 sig->pkey_hash_algo = cert->sig_hash_algo; 66 cert->sig.digest = digest;
68 sig->digest = (u8 *)sig + sizeof(*sig) + desc_size; 67 cert->sig.digest_size = digest_size;
69 sig->digest_size = digest_size;
70 68
71 desc = (void *)sig + sizeof(*sig); 69 desc = digest + digest_size;
72 desc->tfm = tfm; 70 desc->tfm = tfm;
73 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; 71 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
74 72
75 ret = crypto_shash_init(desc); 73 ret = crypto_shash_init(desc);
76 if (ret < 0) 74 if (ret < 0)
77 goto error; 75 goto error;
76 might_sleep();
77 ret = crypto_shash_finup(desc, cert->tbs, cert->tbs_size, digest);
78error:
79 crypto_free_shash(tfm);
80 pr_devel("<==%s() = %d\n", __func__, ret);
81 return ret;
82}
83EXPORT_SYMBOL_GPL(x509_get_sig_params);
78 84
79 ret = -ENOMEM; 85/*
80 sig->rsa.s = mpi_read_raw_data(cert->sig, cert->sig_size); 86 * Check the signature on a certificate using the provided public key
81 if (!sig->rsa.s) 87 */
82 goto error; 88int x509_check_signature(const struct public_key *pub,
89 struct x509_certificate *cert)
90{
91 int ret;
83 92
84 ret = crypto_shash_finup(desc, cert->tbs, cert->tbs_size, sig->digest); 93 pr_devel("==>%s()\n", __func__);
85 if (ret < 0)
86 goto error_mpi;
87 94
88 ret = pub->algo->verify_signature(pub, sig); 95 ret = x509_get_sig_params(cert);
96 if (ret < 0)
97 return ret;
89 98
99 ret = public_key_verify_signature(pub, &cert->sig);
90 pr_debug("Cert Verification: %d\n", ret); 100 pr_debug("Cert Verification: %d\n", ret);
91
92error_mpi:
93 mpi_free(sig->rsa.s);
94error:
95 kfree(sig);
96error_no_sig:
97 crypto_free_shash(tfm);
98
99 pr_devel("<==%s() = %d\n", __func__, ret);
100 return ret; 101 return ret;
101} 102}
103EXPORT_SYMBOL_GPL(x509_check_signature);
102 104
103/* 105/*
104 * Attempt to parse a data blob for a key as an X509 certificate. 106 * Attempt to parse a data blob for a key as an X509 certificate.
@@ -106,7 +108,6 @@ error_no_sig:
106static int x509_key_preparse(struct key_preparsed_payload *prep) 108static int x509_key_preparse(struct key_preparsed_payload *prep)
107{ 109{
108 struct x509_certificate *cert; 110 struct x509_certificate *cert;
109 struct tm now;
110 size_t srlen, sulen; 111 size_t srlen, sulen;
111 char *desc = NULL; 112 char *desc = NULL;
112 int ret; 113 int ret;
@@ -117,7 +118,18 @@ static int x509_key_preparse(struct key_preparsed_payload *prep)
117 118
118 pr_devel("Cert Issuer: %s\n", cert->issuer); 119 pr_devel("Cert Issuer: %s\n", cert->issuer);
119 pr_devel("Cert Subject: %s\n", cert->subject); 120 pr_devel("Cert Subject: %s\n", cert->subject);
120 pr_devel("Cert Key Algo: %s\n", pkey_algo[cert->pkey_algo]); 121
122 if (cert->pub->pkey_algo >= PKEY_ALGO__LAST ||
123 cert->sig.pkey_algo >= PKEY_ALGO__LAST ||
124 cert->sig.pkey_hash_algo >= PKEY_HASH__LAST ||
125 !pkey_algo[cert->pub->pkey_algo] ||
126 !pkey_algo[cert->sig.pkey_algo] ||
127 !hash_algo_name[cert->sig.pkey_hash_algo]) {
128 ret = -ENOPKG;
129 goto error_free_cert;
130 }
131
132 pr_devel("Cert Key Algo: %s\n", pkey_algo_name[cert->pub->pkey_algo]);
121 pr_devel("Cert Valid From: %04ld-%02d-%02d %02d:%02d:%02d\n", 133 pr_devel("Cert Valid From: %04ld-%02d-%02d %02d:%02d:%02d\n",
122 cert->valid_from.tm_year + 1900, cert->valid_from.tm_mon + 1, 134 cert->valid_from.tm_year + 1900, cert->valid_from.tm_mon + 1,
123 cert->valid_from.tm_mday, cert->valid_from.tm_hour, 135 cert->valid_from.tm_mday, cert->valid_from.tm_hour,
@@ -127,58 +139,22 @@ static int x509_key_preparse(struct key_preparsed_payload *prep)
127 cert->valid_to.tm_mday, cert->valid_to.tm_hour, 139 cert->valid_to.tm_mday, cert->valid_to.tm_hour,
128 cert->valid_to.tm_min, cert->valid_to.tm_sec); 140 cert->valid_to.tm_min, cert->valid_to.tm_sec);
129 pr_devel("Cert Signature: %s + %s\n", 141 pr_devel("Cert Signature: %s + %s\n",
130 pkey_algo[cert->sig_pkey_algo], 142 pkey_algo_name[cert->sig.pkey_algo],
131 pkey_hash_algo[cert->sig_hash_algo]); 143 hash_algo_name[cert->sig.pkey_hash_algo]);
132 144
133 if (!cert->fingerprint || !cert->authority) { 145 if (!cert->fingerprint) {
134 pr_warn("Cert for '%s' must have SubjKeyId and AuthKeyId extensions\n", 146 pr_warn("Cert for '%s' must have a SubjKeyId extension\n",
135 cert->subject); 147 cert->subject);
136 ret = -EKEYREJECTED; 148 ret = -EKEYREJECTED;
137 goto error_free_cert; 149 goto error_free_cert;
138 } 150 }
139 151
140 time_to_tm(CURRENT_TIME.tv_sec, 0, &now); 152 cert->pub->algo = pkey_algo[cert->pub->pkey_algo];
141 pr_devel("Now: %04ld-%02d-%02d %02d:%02d:%02d\n",
142 now.tm_year + 1900, now.tm_mon + 1, now.tm_mday,
143 now.tm_hour, now.tm_min, now.tm_sec);
144 if (now.tm_year < cert->valid_from.tm_year ||
145 (now.tm_year == cert->valid_from.tm_year &&
146 (now.tm_mon < cert->valid_from.tm_mon ||
147 (now.tm_mon == cert->valid_from.tm_mon &&
148 (now.tm_mday < cert->valid_from.tm_mday ||
149 (now.tm_mday == cert->valid_from.tm_mday &&
150 (now.tm_hour < cert->valid_from.tm_hour ||
151 (now.tm_hour == cert->valid_from.tm_hour &&
152 (now.tm_min < cert->valid_from.tm_min ||
153 (now.tm_min == cert->valid_from.tm_min &&
154 (now.tm_sec < cert->valid_from.tm_sec
155 ))))))))))) {
156 pr_warn("Cert %s is not yet valid\n", cert->fingerprint);
157 ret = -EKEYREJECTED;
158 goto error_free_cert;
159 }
160 if (now.tm_year > cert->valid_to.tm_year ||
161 (now.tm_year == cert->valid_to.tm_year &&
162 (now.tm_mon > cert->valid_to.tm_mon ||
163 (now.tm_mon == cert->valid_to.tm_mon &&
164 (now.tm_mday > cert->valid_to.tm_mday ||
165 (now.tm_mday == cert->valid_to.tm_mday &&
166 (now.tm_hour > cert->valid_to.tm_hour ||
167 (now.tm_hour == cert->valid_to.tm_hour &&
168 (now.tm_min > cert->valid_to.tm_min ||
169 (now.tm_min == cert->valid_to.tm_min &&
170 (now.tm_sec > cert->valid_to.tm_sec
171 ))))))))))) {
172 pr_warn("Cert %s has expired\n", cert->fingerprint);
173 ret = -EKEYEXPIRED;
174 goto error_free_cert;
175 }
176
177 cert->pub->algo = x509_public_key_algorithms[cert->pkey_algo];
178 cert->pub->id_type = PKEY_ID_X509; 153 cert->pub->id_type = PKEY_ID_X509;
179 154
180 /* Check the signature on the key */ 155 /* Check the signature on the key if it appears to be self-signed */
181 if (strcmp(cert->fingerprint, cert->authority) == 0) { 156 if (!cert->authority ||
157 strcmp(cert->fingerprint, cert->authority) == 0) {
182 ret = x509_check_signature(cert->pub, cert); 158 ret = x509_check_signature(cert->pub, cert);
183 if (ret < 0) 159 if (ret < 0)
184 goto error_free_cert; 160 goto error_free_cert;
@@ -237,3 +213,6 @@ static void __exit x509_key_exit(void)
237 213
238module_init(x509_key_init); 214module_init(x509_key_init);
239module_exit(x509_key_exit); 215module_exit(x509_key_exit);
216
217MODULE_DESCRIPTION("X.509 certificate parser");
218MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_memcpy.c b/crypto/async_tx/async_memcpy.c
index 9e62feffb374..f8c0b8dbeb75 100644
--- a/crypto/async_tx/async_memcpy.c
+++ b/crypto/async_tx/async_memcpy.c
@@ -50,33 +50,36 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
50 &dest, 1, &src, 1, len); 50 &dest, 1, &src, 1, len);
51 struct dma_device *device = chan ? chan->device : NULL; 51 struct dma_device *device = chan ? chan->device : NULL;
52 struct dma_async_tx_descriptor *tx = NULL; 52 struct dma_async_tx_descriptor *tx = NULL;
53 struct dmaengine_unmap_data *unmap = NULL;
53 54
54 if (device && is_dma_copy_aligned(device, src_offset, dest_offset, len)) { 55 if (device)
55 dma_addr_t dma_dest, dma_src; 56 unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOIO);
57
58 if (unmap && is_dma_copy_aligned(device, src_offset, dest_offset, len)) {
56 unsigned long dma_prep_flags = 0; 59 unsigned long dma_prep_flags = 0;
57 60
58 if (submit->cb_fn) 61 if (submit->cb_fn)
59 dma_prep_flags |= DMA_PREP_INTERRUPT; 62 dma_prep_flags |= DMA_PREP_INTERRUPT;
60 if (submit->flags & ASYNC_TX_FENCE) 63 if (submit->flags & ASYNC_TX_FENCE)
61 dma_prep_flags |= DMA_PREP_FENCE; 64 dma_prep_flags |= DMA_PREP_FENCE;
62 dma_dest = dma_map_page(device->dev, dest, dest_offset, len, 65
63 DMA_FROM_DEVICE); 66 unmap->to_cnt = 1;
64 67 unmap->addr[0] = dma_map_page(device->dev, src, src_offset, len,
65 dma_src = dma_map_page(device->dev, src, src_offset, len, 68 DMA_TO_DEVICE);
66 DMA_TO_DEVICE); 69 unmap->from_cnt = 1;
67 70 unmap->addr[1] = dma_map_page(device->dev, dest, dest_offset, len,
68 tx = device->device_prep_dma_memcpy(chan, dma_dest, dma_src, 71 DMA_FROM_DEVICE);
69 len, dma_prep_flags); 72 unmap->len = len;
70 if (!tx) { 73
71 dma_unmap_page(device->dev, dma_dest, len, 74 tx = device->device_prep_dma_memcpy(chan, unmap->addr[1],
72 DMA_FROM_DEVICE); 75 unmap->addr[0], len,
73 dma_unmap_page(device->dev, dma_src, len, 76 dma_prep_flags);
74 DMA_TO_DEVICE);
75 }
76 } 77 }
77 78
78 if (tx) { 79 if (tx) {
79 pr_debug("%s: (async) len: %zu\n", __func__, len); 80 pr_debug("%s: (async) len: %zu\n", __func__, len);
81
82 dma_set_unmap(tx, unmap);
80 async_tx_submit(chan, tx, submit); 83 async_tx_submit(chan, tx, submit);
81 } else { 84 } else {
82 void *dest_buf, *src_buf; 85 void *dest_buf, *src_buf;
@@ -96,6 +99,8 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
96 async_tx_sync_epilog(submit); 99 async_tx_sync_epilog(submit);
97 } 100 }
98 101
102 dmaengine_unmap_put(unmap);
103
99 return tx; 104 return tx;
100} 105}
101EXPORT_SYMBOL_GPL(async_memcpy); 106EXPORT_SYMBOL_GPL(async_memcpy);
diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c
index 91d5d385899e..d05327caf69d 100644
--- a/crypto/async_tx/async_pq.c
+++ b/crypto/async_tx/async_pq.c
@@ -46,49 +46,24 @@ static struct page *pq_scribble_page;
46 * do_async_gen_syndrome - asynchronously calculate P and/or Q 46 * do_async_gen_syndrome - asynchronously calculate P and/or Q
47 */ 47 */
48static __async_inline struct dma_async_tx_descriptor * 48static __async_inline struct dma_async_tx_descriptor *
49do_async_gen_syndrome(struct dma_chan *chan, struct page **blocks, 49do_async_gen_syndrome(struct dma_chan *chan,
50 const unsigned char *scfs, unsigned int offset, int disks, 50 const unsigned char *scfs, int disks,
51 size_t len, dma_addr_t *dma_src, 51 struct dmaengine_unmap_data *unmap,
52 enum dma_ctrl_flags dma_flags,
52 struct async_submit_ctl *submit) 53 struct async_submit_ctl *submit)
53{ 54{
54 struct dma_async_tx_descriptor *tx = NULL; 55 struct dma_async_tx_descriptor *tx = NULL;
55 struct dma_device *dma = chan->device; 56 struct dma_device *dma = chan->device;
56 enum dma_ctrl_flags dma_flags = 0;
57 enum async_tx_flags flags_orig = submit->flags; 57 enum async_tx_flags flags_orig = submit->flags;
58 dma_async_tx_callback cb_fn_orig = submit->cb_fn; 58 dma_async_tx_callback cb_fn_orig = submit->cb_fn;
59 dma_async_tx_callback cb_param_orig = submit->cb_param; 59 dma_async_tx_callback cb_param_orig = submit->cb_param;
60 int src_cnt = disks - 2; 60 int src_cnt = disks - 2;
61 unsigned char coefs[src_cnt];
62 unsigned short pq_src_cnt; 61 unsigned short pq_src_cnt;
63 dma_addr_t dma_dest[2]; 62 dma_addr_t dma_dest[2];
64 int src_off = 0; 63 int src_off = 0;
65 int idx;
66 int i;
67 64
68 /* DMAs use destinations as sources, so use BIDIRECTIONAL mapping */ 65 if (submit->flags & ASYNC_TX_FENCE)
69 if (P(blocks, disks)) 66 dma_flags |= DMA_PREP_FENCE;
70 dma_dest[0] = dma_map_page(dma->dev, P(blocks, disks), offset,
71 len, DMA_BIDIRECTIONAL);
72 else
73 dma_flags |= DMA_PREP_PQ_DISABLE_P;
74 if (Q(blocks, disks))
75 dma_dest[1] = dma_map_page(dma->dev, Q(blocks, disks), offset,
76 len, DMA_BIDIRECTIONAL);
77 else
78 dma_flags |= DMA_PREP_PQ_DISABLE_Q;
79
80 /* convert source addresses being careful to collapse 'empty'
81 * sources and update the coefficients accordingly
82 */
83 for (i = 0, idx = 0; i < src_cnt; i++) {
84 if (blocks[i] == NULL)
85 continue;
86 dma_src[idx] = dma_map_page(dma->dev, blocks[i], offset, len,
87 DMA_TO_DEVICE);
88 coefs[idx] = scfs[i];
89 idx++;
90 }
91 src_cnt = idx;
92 67
93 while (src_cnt > 0) { 68 while (src_cnt > 0) {
94 submit->flags = flags_orig; 69 submit->flags = flags_orig;
@@ -100,28 +75,25 @@ do_async_gen_syndrome(struct dma_chan *chan, struct page **blocks,
100 if (src_cnt > pq_src_cnt) { 75 if (src_cnt > pq_src_cnt) {
101 submit->flags &= ~ASYNC_TX_ACK; 76 submit->flags &= ~ASYNC_TX_ACK;
102 submit->flags |= ASYNC_TX_FENCE; 77 submit->flags |= ASYNC_TX_FENCE;
103 dma_flags |= DMA_COMPL_SKIP_DEST_UNMAP;
104 submit->cb_fn = NULL; 78 submit->cb_fn = NULL;
105 submit->cb_param = NULL; 79 submit->cb_param = NULL;
106 } else { 80 } else {
107 dma_flags &= ~DMA_COMPL_SKIP_DEST_UNMAP;
108 submit->cb_fn = cb_fn_orig; 81 submit->cb_fn = cb_fn_orig;
109 submit->cb_param = cb_param_orig; 82 submit->cb_param = cb_param_orig;
110 if (cb_fn_orig) 83 if (cb_fn_orig)
111 dma_flags |= DMA_PREP_INTERRUPT; 84 dma_flags |= DMA_PREP_INTERRUPT;
112 } 85 }
113 if (submit->flags & ASYNC_TX_FENCE)
114 dma_flags |= DMA_PREP_FENCE;
115 86
116 /* Since we have clobbered the src_list we are committed 87 /* Drivers force forward progress in case they can not provide
117 * to doing this asynchronously. Drivers force forward 88 * a descriptor
118 * progress in case they can not provide a descriptor
119 */ 89 */
120 for (;;) { 90 for (;;) {
91 dma_dest[0] = unmap->addr[disks - 2];
92 dma_dest[1] = unmap->addr[disks - 1];
121 tx = dma->device_prep_dma_pq(chan, dma_dest, 93 tx = dma->device_prep_dma_pq(chan, dma_dest,
122 &dma_src[src_off], 94 &unmap->addr[src_off],
123 pq_src_cnt, 95 pq_src_cnt,
124 &coefs[src_off], len, 96 &scfs[src_off], unmap->len,
125 dma_flags); 97 dma_flags);
126 if (likely(tx)) 98 if (likely(tx))
127 break; 99 break;
@@ -129,6 +101,7 @@ do_async_gen_syndrome(struct dma_chan *chan, struct page **blocks,
129 dma_async_issue_pending(chan); 101 dma_async_issue_pending(chan);
130 } 102 }
131 103
104 dma_set_unmap(tx, unmap);
132 async_tx_submit(chan, tx, submit); 105 async_tx_submit(chan, tx, submit);
133 submit->depend_tx = tx; 106 submit->depend_tx = tx;
134 107
@@ -188,10 +161,6 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
188 * set to NULL those buffers will be replaced with the raid6_zero_page 161 * set to NULL those buffers will be replaced with the raid6_zero_page
189 * in the synchronous path and omitted in the hardware-asynchronous 162 * in the synchronous path and omitted in the hardware-asynchronous
190 * path. 163 * path.
191 *
192 * 'blocks' note: if submit->scribble is NULL then the contents of
193 * 'blocks' may be overwritten to perform address conversions
194 * (dma_map_page() or page_address()).
195 */ 164 */
196struct dma_async_tx_descriptor * 165struct dma_async_tx_descriptor *
197async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, 166async_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
@@ -202,26 +171,69 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
202 &P(blocks, disks), 2, 171 &P(blocks, disks), 2,
203 blocks, src_cnt, len); 172 blocks, src_cnt, len);
204 struct dma_device *device = chan ? chan->device : NULL; 173 struct dma_device *device = chan ? chan->device : NULL;
205 dma_addr_t *dma_src = NULL; 174 struct dmaengine_unmap_data *unmap = NULL;
206 175
207 BUG_ON(disks > 255 || !(P(blocks, disks) || Q(blocks, disks))); 176 BUG_ON(disks > 255 || !(P(blocks, disks) || Q(blocks, disks)));
208 177
209 if (submit->scribble) 178 if (device)
210 dma_src = submit->scribble; 179 unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOIO);
211 else if (sizeof(dma_addr_t) <= sizeof(struct page *))
212 dma_src = (dma_addr_t *) blocks;
213 180
214 if (dma_src && device && 181 if (unmap &&
215 (src_cnt <= dma_maxpq(device, 0) || 182 (src_cnt <= dma_maxpq(device, 0) ||
216 dma_maxpq(device, DMA_PREP_CONTINUE) > 0) && 183 dma_maxpq(device, DMA_PREP_CONTINUE) > 0) &&
217 is_dma_pq_aligned(device, offset, 0, len)) { 184 is_dma_pq_aligned(device, offset, 0, len)) {
185 struct dma_async_tx_descriptor *tx;
186 enum dma_ctrl_flags dma_flags = 0;
187 unsigned char coefs[src_cnt];
188 int i, j;
189
218 /* run the p+q asynchronously */ 190 /* run the p+q asynchronously */
219 pr_debug("%s: (async) disks: %d len: %zu\n", 191 pr_debug("%s: (async) disks: %d len: %zu\n",
220 __func__, disks, len); 192 __func__, disks, len);
221 return do_async_gen_syndrome(chan, blocks, raid6_gfexp, offset, 193
222 disks, len, dma_src, submit); 194 /* convert source addresses being careful to collapse 'empty'
195 * sources and update the coefficients accordingly
196 */
197 unmap->len = len;
198 for (i = 0, j = 0; i < src_cnt; i++) {
199 if (blocks[i] == NULL)
200 continue;
201 unmap->addr[j] = dma_map_page(device->dev, blocks[i], offset,
202 len, DMA_TO_DEVICE);
203 coefs[j] = raid6_gfexp[i];
204 unmap->to_cnt++;
205 j++;
206 }
207
208 /*
209 * DMAs use destinations as sources,
210 * so use BIDIRECTIONAL mapping
211 */
212 unmap->bidi_cnt++;
213 if (P(blocks, disks))
214 unmap->addr[j++] = dma_map_page(device->dev, P(blocks, disks),
215 offset, len, DMA_BIDIRECTIONAL);
216 else {
217 unmap->addr[j++] = 0;
218 dma_flags |= DMA_PREP_PQ_DISABLE_P;
219 }
220
221 unmap->bidi_cnt++;
222 if (Q(blocks, disks))
223 unmap->addr[j++] = dma_map_page(device->dev, Q(blocks, disks),
224 offset, len, DMA_BIDIRECTIONAL);
225 else {
226 unmap->addr[j++] = 0;
227 dma_flags |= DMA_PREP_PQ_DISABLE_Q;
228 }
229
230 tx = do_async_gen_syndrome(chan, coefs, j, unmap, dma_flags, submit);
231 dmaengine_unmap_put(unmap);
232 return tx;
223 } 233 }
224 234
235 dmaengine_unmap_put(unmap);
236
225 /* run the pq synchronously */ 237 /* run the pq synchronously */
226 pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len); 238 pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len);
227 239
@@ -277,50 +289,60 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks,
277 struct dma_async_tx_descriptor *tx; 289 struct dma_async_tx_descriptor *tx;
278 unsigned char coefs[disks-2]; 290 unsigned char coefs[disks-2];
279 enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0; 291 enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0;
280 dma_addr_t *dma_src = NULL; 292 struct dmaengine_unmap_data *unmap = NULL;
281 int src_cnt = 0;
282 293
283 BUG_ON(disks < 4); 294 BUG_ON(disks < 4);
284 295
285 if (submit->scribble) 296 if (device)
286 dma_src = submit->scribble; 297 unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOIO);
287 else if (sizeof(dma_addr_t) <= sizeof(struct page *))
288 dma_src = (dma_addr_t *) blocks;
289 298
290 if (dma_src && device && disks <= dma_maxpq(device, 0) && 299 if (unmap && disks <= dma_maxpq(device, 0) &&
291 is_dma_pq_aligned(device, offset, 0, len)) { 300 is_dma_pq_aligned(device, offset, 0, len)) {
292 struct device *dev = device->dev; 301 struct device *dev = device->dev;
293 dma_addr_t *pq = &dma_src[disks-2]; 302 dma_addr_t pq[2];
294 int i; 303 int i, j = 0, src_cnt = 0;
295 304
296 pr_debug("%s: (async) disks: %d len: %zu\n", 305 pr_debug("%s: (async) disks: %d len: %zu\n",
297 __func__, disks, len); 306 __func__, disks, len);
298 if (!P(blocks, disks)) 307
308 unmap->len = len;
309 for (i = 0; i < disks-2; i++)
310 if (likely(blocks[i])) {
311 unmap->addr[j] = dma_map_page(dev, blocks[i],
312 offset, len,
313 DMA_TO_DEVICE);
314 coefs[j] = raid6_gfexp[i];
315 unmap->to_cnt++;
316 src_cnt++;
317 j++;
318 }
319
320 if (!P(blocks, disks)) {
321 pq[0] = 0;
299 dma_flags |= DMA_PREP_PQ_DISABLE_P; 322 dma_flags |= DMA_PREP_PQ_DISABLE_P;
300 else 323 } else {
301 pq[0] = dma_map_page(dev, P(blocks, disks), 324 pq[0] = dma_map_page(dev, P(blocks, disks),
302 offset, len, 325 offset, len,
303 DMA_TO_DEVICE); 326 DMA_TO_DEVICE);
304 if (!Q(blocks, disks)) 327 unmap->addr[j++] = pq[0];
328 unmap->to_cnt++;
329 }
330 if (!Q(blocks, disks)) {
331 pq[1] = 0;
305 dma_flags |= DMA_PREP_PQ_DISABLE_Q; 332 dma_flags |= DMA_PREP_PQ_DISABLE_Q;
306 else 333 } else {
307 pq[1] = dma_map_page(dev, Q(blocks, disks), 334 pq[1] = dma_map_page(dev, Q(blocks, disks),
308 offset, len, 335 offset, len,
309 DMA_TO_DEVICE); 336 DMA_TO_DEVICE);
337 unmap->addr[j++] = pq[1];
338 unmap->to_cnt++;
339 }
310 340
311 if (submit->flags & ASYNC_TX_FENCE) 341 if (submit->flags & ASYNC_TX_FENCE)
312 dma_flags |= DMA_PREP_FENCE; 342 dma_flags |= DMA_PREP_FENCE;
313 for (i = 0; i < disks-2; i++)
314 if (likely(blocks[i])) {
315 dma_src[src_cnt] = dma_map_page(dev, blocks[i],
316 offset, len,
317 DMA_TO_DEVICE);
318 coefs[src_cnt] = raid6_gfexp[i];
319 src_cnt++;
320 }
321
322 for (;;) { 343 for (;;) {
323 tx = device->device_prep_dma_pq_val(chan, pq, dma_src, 344 tx = device->device_prep_dma_pq_val(chan, pq,
345 unmap->addr,
324 src_cnt, 346 src_cnt,
325 coefs, 347 coefs,
326 len, pqres, 348 len, pqres,
@@ -330,6 +352,8 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks,
330 async_tx_quiesce(&submit->depend_tx); 352 async_tx_quiesce(&submit->depend_tx);
331 dma_async_issue_pending(chan); 353 dma_async_issue_pending(chan);
332 } 354 }
355
356 dma_set_unmap(tx, unmap);
333 async_tx_submit(chan, tx, submit); 357 async_tx_submit(chan, tx, submit);
334 358
335 return tx; 359 return tx;
diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c
index a9f08a6a582e..934a84981495 100644
--- a/crypto/async_tx/async_raid6_recov.c
+++ b/crypto/async_tx/async_raid6_recov.c
@@ -26,6 +26,7 @@
26#include <linux/dma-mapping.h> 26#include <linux/dma-mapping.h>
27#include <linux/raid/pq.h> 27#include <linux/raid/pq.h>
28#include <linux/async_tx.h> 28#include <linux/async_tx.h>
29#include <linux/dmaengine.h>
29 30
30static struct dma_async_tx_descriptor * 31static struct dma_async_tx_descriptor *
31async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef, 32async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
@@ -34,35 +35,45 @@ async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
34 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, 35 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
35 &dest, 1, srcs, 2, len); 36 &dest, 1, srcs, 2, len);
36 struct dma_device *dma = chan ? chan->device : NULL; 37 struct dma_device *dma = chan ? chan->device : NULL;
38 struct dmaengine_unmap_data *unmap = NULL;
37 const u8 *amul, *bmul; 39 const u8 *amul, *bmul;
38 u8 ax, bx; 40 u8 ax, bx;
39 u8 *a, *b, *c; 41 u8 *a, *b, *c;
40 42
41 if (dma) { 43 if (dma)
42 dma_addr_t dma_dest[2]; 44 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOIO);
43 dma_addr_t dma_src[2]; 45
46 if (unmap) {
44 struct device *dev = dma->dev; 47 struct device *dev = dma->dev;
48 dma_addr_t pq[2];
45 struct dma_async_tx_descriptor *tx; 49 struct dma_async_tx_descriptor *tx;
46 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; 50 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
47 51
48 if (submit->flags & ASYNC_TX_FENCE) 52 if (submit->flags & ASYNC_TX_FENCE)
49 dma_flags |= DMA_PREP_FENCE; 53 dma_flags |= DMA_PREP_FENCE;
50 dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); 54 unmap->addr[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
51 dma_src[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE); 55 unmap->addr[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
52 dma_src[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE); 56 unmap->to_cnt = 2;
53 tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 2, coef, 57
58 unmap->addr[2] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
59 unmap->bidi_cnt = 1;
60 /* engine only looks at Q, but expects it to follow P */
61 pq[1] = unmap->addr[2];
62
63 unmap->len = len;
64 tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
54 len, dma_flags); 65 len, dma_flags);
55 if (tx) { 66 if (tx) {
67 dma_set_unmap(tx, unmap);
56 async_tx_submit(chan, tx, submit); 68 async_tx_submit(chan, tx, submit);
69 dmaengine_unmap_put(unmap);
57 return tx; 70 return tx;
58 } 71 }
59 72
60 /* could not get a descriptor, unmap and fall through to 73 /* could not get a descriptor, unmap and fall through to
61 * the synchronous path 74 * the synchronous path
62 */ 75 */
63 dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL); 76 dmaengine_unmap_put(unmap);
64 dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
65 dma_unmap_page(dev, dma_src[1], len, DMA_TO_DEVICE);
66 } 77 }
67 78
68 /* run the operation synchronously */ 79 /* run the operation synchronously */
@@ -89,23 +100,38 @@ async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
89 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, 100 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
90 &dest, 1, &src, 1, len); 101 &dest, 1, &src, 1, len);
91 struct dma_device *dma = chan ? chan->device : NULL; 102 struct dma_device *dma = chan ? chan->device : NULL;
103 struct dmaengine_unmap_data *unmap = NULL;
92 const u8 *qmul; /* Q multiplier table */ 104 const u8 *qmul; /* Q multiplier table */
93 u8 *d, *s; 105 u8 *d, *s;
94 106
95 if (dma) { 107 if (dma)
108 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOIO);
109
110 if (unmap) {
96 dma_addr_t dma_dest[2]; 111 dma_addr_t dma_dest[2];
97 dma_addr_t dma_src[1];
98 struct device *dev = dma->dev; 112 struct device *dev = dma->dev;
99 struct dma_async_tx_descriptor *tx; 113 struct dma_async_tx_descriptor *tx;
100 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; 114 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
101 115
102 if (submit->flags & ASYNC_TX_FENCE) 116 if (submit->flags & ASYNC_TX_FENCE)
103 dma_flags |= DMA_PREP_FENCE; 117 dma_flags |= DMA_PREP_FENCE;
104 dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); 118 unmap->addr[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
105 dma_src[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE); 119 unmap->to_cnt++;
106 tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 1, &coef, 120 unmap->addr[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
107 len, dma_flags); 121 dma_dest[1] = unmap->addr[1];
122 unmap->bidi_cnt++;
123 unmap->len = len;
124
125 /* this looks funny, but the engine looks for Q at
126 * dma_dest[1] and ignores dma_dest[0] as a dest
127 * due to DMA_PREP_PQ_DISABLE_P
128 */
129 tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
130 1, &coef, len, dma_flags);
131
108 if (tx) { 132 if (tx) {
133 dma_set_unmap(tx, unmap);
134 dmaengine_unmap_put(unmap);
109 async_tx_submit(chan, tx, submit); 135 async_tx_submit(chan, tx, submit);
110 return tx; 136 return tx;
111 } 137 }
@@ -113,8 +139,7 @@ async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
113 /* could not get a descriptor, unmap and fall through to 139 /* could not get a descriptor, unmap and fall through to
114 * the synchronous path 140 * the synchronous path
115 */ 141 */
116 dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL); 142 dmaengine_unmap_put(unmap);
117 dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
118 } 143 }
119 144
120 /* no channel available, or failed to allocate a descriptor, so 145 /* no channel available, or failed to allocate a descriptor, so
diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c
index 7be34248b450..39ea4791a3c9 100644
--- a/crypto/async_tx/async_tx.c
+++ b/crypto/async_tx/async_tx.c
@@ -128,7 +128,7 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
128 } 128 }
129 device->device_issue_pending(chan); 129 device->device_issue_pending(chan);
130 } else { 130 } else {
131 if (dma_wait_for_async_tx(depend_tx) != DMA_SUCCESS) 131 if (dma_wait_for_async_tx(depend_tx) != DMA_COMPLETE)
132 panic("%s: DMA error waiting for depend_tx\n", 132 panic("%s: DMA error waiting for depend_tx\n",
133 __func__); 133 __func__);
134 tx->tx_submit(tx); 134 tx->tx_submit(tx);
@@ -280,7 +280,7 @@ void async_tx_quiesce(struct dma_async_tx_descriptor **tx)
280 * we are referring to the correct operation 280 * we are referring to the correct operation
281 */ 281 */
282 BUG_ON(async_tx_test_ack(*tx)); 282 BUG_ON(async_tx_test_ack(*tx));
283 if (dma_wait_for_async_tx(*tx) != DMA_SUCCESS) 283 if (dma_wait_for_async_tx(*tx) != DMA_COMPLETE)
284 panic("%s: DMA error waiting for transaction\n", 284 panic("%s: DMA error waiting for transaction\n",
285 __func__); 285 __func__);
286 async_tx_ack(*tx); 286 async_tx_ack(*tx);
diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c
index 8ade0a0481c6..3c562f5a60bb 100644
--- a/crypto/async_tx/async_xor.c
+++ b/crypto/async_tx/async_xor.c
@@ -33,48 +33,31 @@
33 33
34/* do_async_xor - dma map the pages and perform the xor with an engine */ 34/* do_async_xor - dma map the pages and perform the xor with an engine */
35static __async_inline struct dma_async_tx_descriptor * 35static __async_inline struct dma_async_tx_descriptor *
36do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, 36do_async_xor(struct dma_chan *chan, struct dmaengine_unmap_data *unmap,
37 unsigned int offset, int src_cnt, size_t len, dma_addr_t *dma_src,
38 struct async_submit_ctl *submit) 37 struct async_submit_ctl *submit)
39{ 38{
40 struct dma_device *dma = chan->device; 39 struct dma_device *dma = chan->device;
41 struct dma_async_tx_descriptor *tx = NULL; 40 struct dma_async_tx_descriptor *tx = NULL;
42 int src_off = 0;
43 int i;
44 dma_async_tx_callback cb_fn_orig = submit->cb_fn; 41 dma_async_tx_callback cb_fn_orig = submit->cb_fn;
45 void *cb_param_orig = submit->cb_param; 42 void *cb_param_orig = submit->cb_param;
46 enum async_tx_flags flags_orig = submit->flags; 43 enum async_tx_flags flags_orig = submit->flags;
47 enum dma_ctrl_flags dma_flags; 44 enum dma_ctrl_flags dma_flags = 0;
48 int xor_src_cnt = 0; 45 int src_cnt = unmap->to_cnt;
49 dma_addr_t dma_dest; 46 int xor_src_cnt;
50 47 dma_addr_t dma_dest = unmap->addr[unmap->to_cnt];
51 /* map the dest bidrectional in case it is re-used as a source */ 48 dma_addr_t *src_list = unmap->addr;
52 dma_dest = dma_map_page(dma->dev, dest, offset, len, DMA_BIDIRECTIONAL);
53 for (i = 0; i < src_cnt; i++) {
54 /* only map the dest once */
55 if (!src_list[i])
56 continue;
57 if (unlikely(src_list[i] == dest)) {
58 dma_src[xor_src_cnt++] = dma_dest;
59 continue;
60 }
61 dma_src[xor_src_cnt++] = dma_map_page(dma->dev, src_list[i], offset,
62 len, DMA_TO_DEVICE);
63 }
64 src_cnt = xor_src_cnt;
65 49
66 while (src_cnt) { 50 while (src_cnt) {
51 dma_addr_t tmp;
52
67 submit->flags = flags_orig; 53 submit->flags = flags_orig;
68 dma_flags = 0;
69 xor_src_cnt = min(src_cnt, (int)dma->max_xor); 54 xor_src_cnt = min(src_cnt, (int)dma->max_xor);
70 /* if we are submitting additional xors, leave the chain open, 55 /* if we are submitting additional xors, leave the chain open
71 * clear the callback parameters, and leave the destination 56 * and clear the callback parameters
72 * buffer mapped
73 */ 57 */
74 if (src_cnt > xor_src_cnt) { 58 if (src_cnt > xor_src_cnt) {
75 submit->flags &= ~ASYNC_TX_ACK; 59 submit->flags &= ~ASYNC_TX_ACK;
76 submit->flags |= ASYNC_TX_FENCE; 60 submit->flags |= ASYNC_TX_FENCE;
77 dma_flags = DMA_COMPL_SKIP_DEST_UNMAP;
78 submit->cb_fn = NULL; 61 submit->cb_fn = NULL;
79 submit->cb_param = NULL; 62 submit->cb_param = NULL;
80 } else { 63 } else {
@@ -85,12 +68,18 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
85 dma_flags |= DMA_PREP_INTERRUPT; 68 dma_flags |= DMA_PREP_INTERRUPT;
86 if (submit->flags & ASYNC_TX_FENCE) 69 if (submit->flags & ASYNC_TX_FENCE)
87 dma_flags |= DMA_PREP_FENCE; 70 dma_flags |= DMA_PREP_FENCE;
88 /* Since we have clobbered the src_list we are committed 71
89 * to doing this asynchronously. Drivers force forward progress 72 /* Drivers force forward progress in case they can not provide a
90 * in case they can not provide a descriptor 73 * descriptor
91 */ 74 */
92 tx = dma->device_prep_dma_xor(chan, dma_dest, &dma_src[src_off], 75 tmp = src_list[0];
93 xor_src_cnt, len, dma_flags); 76 if (src_list > unmap->addr)
77 src_list[0] = dma_dest;
78 tx = dma->device_prep_dma_xor(chan, dma_dest, src_list,
79 xor_src_cnt, unmap->len,
80 dma_flags);
81 src_list[0] = tmp;
82
94 83
95 if (unlikely(!tx)) 84 if (unlikely(!tx))
96 async_tx_quiesce(&submit->depend_tx); 85 async_tx_quiesce(&submit->depend_tx);
@@ -99,22 +88,21 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
99 while (unlikely(!tx)) { 88 while (unlikely(!tx)) {
100 dma_async_issue_pending(chan); 89 dma_async_issue_pending(chan);
101 tx = dma->device_prep_dma_xor(chan, dma_dest, 90 tx = dma->device_prep_dma_xor(chan, dma_dest,
102 &dma_src[src_off], 91 src_list,
103 xor_src_cnt, len, 92 xor_src_cnt, unmap->len,
104 dma_flags); 93 dma_flags);
105 } 94 }
106 95
96 dma_set_unmap(tx, unmap);
107 async_tx_submit(chan, tx, submit); 97 async_tx_submit(chan, tx, submit);
108 submit->depend_tx = tx; 98 submit->depend_tx = tx;
109 99
110 if (src_cnt > xor_src_cnt) { 100 if (src_cnt > xor_src_cnt) {
111 /* drop completed sources */ 101 /* drop completed sources */
112 src_cnt -= xor_src_cnt; 102 src_cnt -= xor_src_cnt;
113 src_off += xor_src_cnt;
114
115 /* use the intermediate result a source */ 103 /* use the intermediate result a source */
116 dma_src[--src_off] = dma_dest;
117 src_cnt++; 104 src_cnt++;
105 src_list += xor_src_cnt - 1;
118 } else 106 } else
119 break; 107 break;
120 } 108 }
@@ -189,22 +177,40 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset,
189 struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR, 177 struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR,
190 &dest, 1, src_list, 178 &dest, 1, src_list,
191 src_cnt, len); 179 src_cnt, len);
192 dma_addr_t *dma_src = NULL; 180 struct dma_device *device = chan ? chan->device : NULL;
181 struct dmaengine_unmap_data *unmap = NULL;
193 182
194 BUG_ON(src_cnt <= 1); 183 BUG_ON(src_cnt <= 1);
195 184
196 if (submit->scribble) 185 if (device)
197 dma_src = submit->scribble; 186 unmap = dmaengine_get_unmap_data(device->dev, src_cnt+1, GFP_NOIO);
198 else if (sizeof(dma_addr_t) <= sizeof(struct page *)) 187
199 dma_src = (dma_addr_t *) src_list; 188 if (unmap && is_dma_xor_aligned(device, offset, 0, len)) {
189 struct dma_async_tx_descriptor *tx;
190 int i, j;
200 191
201 if (dma_src && chan && is_dma_xor_aligned(chan->device, offset, 0, len)) {
202 /* run the xor asynchronously */ 192 /* run the xor asynchronously */
203 pr_debug("%s (async): len: %zu\n", __func__, len); 193 pr_debug("%s (async): len: %zu\n", __func__, len);
204 194
205 return do_async_xor(chan, dest, src_list, offset, src_cnt, len, 195 unmap->len = len;
206 dma_src, submit); 196 for (i = 0, j = 0; i < src_cnt; i++) {
197 if (!src_list[i])
198 continue;
199 unmap->to_cnt++;
200 unmap->addr[j++] = dma_map_page(device->dev, src_list[i],
201 offset, len, DMA_TO_DEVICE);
202 }
203
204 /* map it bidirectional as it may be re-used as a source */
205 unmap->addr[j] = dma_map_page(device->dev, dest, offset, len,
206 DMA_BIDIRECTIONAL);
207 unmap->bidi_cnt = 1;
208
209 tx = do_async_xor(chan, unmap, submit);
210 dmaengine_unmap_put(unmap);
211 return tx;
207 } else { 212 } else {
213 dmaengine_unmap_put(unmap);
208 /* run the xor synchronously */ 214 /* run the xor synchronously */
209 pr_debug("%s (sync): len: %zu\n", __func__, len); 215 pr_debug("%s (sync): len: %zu\n", __func__, len);
210 WARN_ONCE(chan, "%s: no space for dma address conversion\n", 216 WARN_ONCE(chan, "%s: no space for dma address conversion\n",
@@ -268,16 +274,14 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
268 struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len); 274 struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len);
269 struct dma_device *device = chan ? chan->device : NULL; 275 struct dma_device *device = chan ? chan->device : NULL;
270 struct dma_async_tx_descriptor *tx = NULL; 276 struct dma_async_tx_descriptor *tx = NULL;
271 dma_addr_t *dma_src = NULL; 277 struct dmaengine_unmap_data *unmap = NULL;
272 278
273 BUG_ON(src_cnt <= 1); 279 BUG_ON(src_cnt <= 1);
274 280
275 if (submit->scribble) 281 if (device)
276 dma_src = submit->scribble; 282 unmap = dmaengine_get_unmap_data(device->dev, src_cnt, GFP_NOIO);
277 else if (sizeof(dma_addr_t) <= sizeof(struct page *))
278 dma_src = (dma_addr_t *) src_list;
279 283
280 if (dma_src && device && src_cnt <= device->max_xor && 284 if (unmap && src_cnt <= device->max_xor &&
281 is_dma_xor_aligned(device, offset, 0, len)) { 285 is_dma_xor_aligned(device, offset, 0, len)) {
282 unsigned long dma_prep_flags = 0; 286 unsigned long dma_prep_flags = 0;
283 int i; 287 int i;
@@ -288,11 +292,15 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
288 dma_prep_flags |= DMA_PREP_INTERRUPT; 292 dma_prep_flags |= DMA_PREP_INTERRUPT;
289 if (submit->flags & ASYNC_TX_FENCE) 293 if (submit->flags & ASYNC_TX_FENCE)
290 dma_prep_flags |= DMA_PREP_FENCE; 294 dma_prep_flags |= DMA_PREP_FENCE;
291 for (i = 0; i < src_cnt; i++)
292 dma_src[i] = dma_map_page(device->dev, src_list[i],
293 offset, len, DMA_TO_DEVICE);
294 295
295 tx = device->device_prep_dma_xor_val(chan, dma_src, src_cnt, 296 for (i = 0; i < src_cnt; i++) {
297 unmap->addr[i] = dma_map_page(device->dev, src_list[i],
298 offset, len, DMA_TO_DEVICE);
299 unmap->to_cnt++;
300 }
301 unmap->len = len;
302
303 tx = device->device_prep_dma_xor_val(chan, unmap->addr, src_cnt,
296 len, result, 304 len, result,
297 dma_prep_flags); 305 dma_prep_flags);
298 if (unlikely(!tx)) { 306 if (unlikely(!tx)) {
@@ -301,11 +309,11 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
301 while (!tx) { 309 while (!tx) {
302 dma_async_issue_pending(chan); 310 dma_async_issue_pending(chan);
303 tx = device->device_prep_dma_xor_val(chan, 311 tx = device->device_prep_dma_xor_val(chan,
304 dma_src, src_cnt, len, result, 312 unmap->addr, src_cnt, len, result,
305 dma_prep_flags); 313 dma_prep_flags);
306 } 314 }
307 } 315 }
308 316 dma_set_unmap(tx, unmap);
309 async_tx_submit(chan, tx, submit); 317 async_tx_submit(chan, tx, submit);
310 } else { 318 } else {
311 enum async_tx_flags flags_orig = submit->flags; 319 enum async_tx_flags flags_orig = submit->flags;
@@ -327,6 +335,7 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
327 async_tx_sync_epilog(submit); 335 async_tx_sync_epilog(submit);
328 submit->flags = flags_orig; 336 submit->flags = flags_orig;
329 } 337 }
338 dmaengine_unmap_put(unmap);
330 339
331 return tx; 340 return tx;
332} 341}
diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c
index 4a92bac744dc..dad95f45b88f 100644
--- a/crypto/async_tx/raid6test.c
+++ b/crypto/async_tx/raid6test.c
@@ -28,7 +28,7 @@
28#undef pr 28#undef pr
29#define pr(fmt, args...) pr_info("raid6test: " fmt, ##args) 29#define pr(fmt, args...) pr_info("raid6test: " fmt, ##args)
30 30
31#define NDISKS 16 /* Including P and Q */ 31#define NDISKS 64 /* Including P and Q */
32 32
33static struct page *dataptrs[NDISKS]; 33static struct page *dataptrs[NDISKS];
34static addr_conv_t addr_conv[NDISKS]; 34static addr_conv_t addr_conv[NDISKS];
@@ -219,6 +219,14 @@ static int raid6_test(void)
219 err += test(11, &tests); 219 err += test(11, &tests);
220 err += test(12, &tests); 220 err += test(12, &tests);
221 } 221 }
222
223 /* the 24 disk case is special for ioatdma as it is the boudary point
224 * at which it needs to switch from 8-source ops to 16-source
225 * ops for continuation (assumes DMA_HAS_PQ_CONTINUE is not set)
226 */
227 if (NDISKS > 24)
228 err += test(24, &tests);
229
222 err += test(NDISKS, &tests); 230 err += test(NDISKS, &tests);
223 231
224 pr("\n"); 232 pr("\n");
diff --git a/crypto/authenc.c b/crypto/authenc.c
index ffce19de05cf..e1223559d5df 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -52,40 +52,52 @@ static void authenc_request_complete(struct aead_request *req, int err)
52 aead_request_complete(req, err); 52 aead_request_complete(req, err);
53} 53}
54 54
55static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key, 55int crypto_authenc_extractkeys(struct crypto_authenc_keys *keys, const u8 *key,
56 unsigned int keylen) 56 unsigned int keylen)
57{ 57{
58 unsigned int authkeylen; 58 struct rtattr *rta = (struct rtattr *)key;
59 unsigned int enckeylen;
60 struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
61 struct crypto_ahash *auth = ctx->auth;
62 struct crypto_ablkcipher *enc = ctx->enc;
63 struct rtattr *rta = (void *)key;
64 struct crypto_authenc_key_param *param; 59 struct crypto_authenc_key_param *param;
65 int err = -EINVAL;
66 60
67 if (!RTA_OK(rta, keylen)) 61 if (!RTA_OK(rta, keylen))
68 goto badkey; 62 return -EINVAL;
69 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) 63 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
70 goto badkey; 64 return -EINVAL;
71 if (RTA_PAYLOAD(rta) < sizeof(*param)) 65 if (RTA_PAYLOAD(rta) < sizeof(*param))
72 goto badkey; 66 return -EINVAL;
73 67
74 param = RTA_DATA(rta); 68 param = RTA_DATA(rta);
75 enckeylen = be32_to_cpu(param->enckeylen); 69 keys->enckeylen = be32_to_cpu(param->enckeylen);
76 70
77 key += RTA_ALIGN(rta->rta_len); 71 key += RTA_ALIGN(rta->rta_len);
78 keylen -= RTA_ALIGN(rta->rta_len); 72 keylen -= RTA_ALIGN(rta->rta_len);
79 73
80 if (keylen < enckeylen) 74 if (keylen < keys->enckeylen)
81 goto badkey; 75 return -EINVAL;
82 76
83 authkeylen = keylen - enckeylen; 77 keys->authkeylen = keylen - keys->enckeylen;
78 keys->authkey = key;
79 keys->enckey = key + keys->authkeylen;
80
81 return 0;
82}
83EXPORT_SYMBOL_GPL(crypto_authenc_extractkeys);
84
85static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
86 unsigned int keylen)
87{
88 struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
89 struct crypto_ahash *auth = ctx->auth;
90 struct crypto_ablkcipher *enc = ctx->enc;
91 struct crypto_authenc_keys keys;
92 int err = -EINVAL;
93
94 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
95 goto badkey;
84 96
85 crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK); 97 crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK);
86 crypto_ahash_set_flags(auth, crypto_aead_get_flags(authenc) & 98 crypto_ahash_set_flags(auth, crypto_aead_get_flags(authenc) &
87 CRYPTO_TFM_REQ_MASK); 99 CRYPTO_TFM_REQ_MASK);
88 err = crypto_ahash_setkey(auth, key, authkeylen); 100 err = crypto_ahash_setkey(auth, keys.authkey, keys.authkeylen);
89 crypto_aead_set_flags(authenc, crypto_ahash_get_flags(auth) & 101 crypto_aead_set_flags(authenc, crypto_ahash_get_flags(auth) &
90 CRYPTO_TFM_RES_MASK); 102 CRYPTO_TFM_RES_MASK);
91 103
@@ -95,7 +107,7 @@ static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
95 crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK); 107 crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
96 crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc) & 108 crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc) &
97 CRYPTO_TFM_REQ_MASK); 109 CRYPTO_TFM_REQ_MASK);
98 err = crypto_ablkcipher_setkey(enc, key + authkeylen, enckeylen); 110 err = crypto_ablkcipher_setkey(enc, keys.enckey, keys.enckeylen);
99 crypto_aead_set_flags(authenc, crypto_ablkcipher_get_flags(enc) & 111 crypto_aead_set_flags(authenc, crypto_ablkcipher_get_flags(enc) &
100 CRYPTO_TFM_RES_MASK); 112 CRYPTO_TFM_RES_MASK);
101 113
@@ -188,7 +200,7 @@ static void authenc_verify_ahash_update_done(struct crypto_async_request *areq,
188 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, 200 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
189 authsize, 0); 201 authsize, 0);
190 202
191 err = memcmp(ihash, ahreq->result, authsize) ? -EBADMSG : 0; 203 err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0;
192 if (err) 204 if (err)
193 goto out; 205 goto out;
194 206
@@ -227,7 +239,7 @@ static void authenc_verify_ahash_done(struct crypto_async_request *areq,
227 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, 239 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
228 authsize, 0); 240 authsize, 0);
229 241
230 err = memcmp(ihash, ahreq->result, authsize) ? -EBADMSG : 0; 242 err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0;
231 if (err) 243 if (err)
232 goto out; 244 goto out;
233 245
@@ -368,9 +380,10 @@ static void crypto_authenc_encrypt_done(struct crypto_async_request *req,
368 if (!err) { 380 if (!err) {
369 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 381 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
370 struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); 382 struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
371 struct ablkcipher_request *abreq = aead_request_ctx(areq); 383 struct authenc_request_ctx *areq_ctx = aead_request_ctx(areq);
372 u8 *iv = (u8 *)(abreq + 1) + 384 struct ablkcipher_request *abreq = (void *)(areq_ctx->tail
373 crypto_ablkcipher_reqsize(ctx->enc); 385 + ctx->reqoff);
386 u8 *iv = (u8 *)abreq - crypto_ablkcipher_ivsize(ctx->enc);
374 387
375 err = crypto_authenc_genicv(areq, iv, 0); 388 err = crypto_authenc_genicv(areq, iv, 0);
376 } 389 }
@@ -462,7 +475,7 @@ static int crypto_authenc_verify(struct aead_request *req,
462 ihash = ohash + authsize; 475 ihash = ohash + authsize;
463 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, 476 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
464 authsize, 0); 477 authsize, 0);
465 return memcmp(ihash, ohash, authsize) ? -EBADMSG : 0; 478 return crypto_memneq(ihash, ohash, authsize) ? -EBADMSG : 0;
466} 479}
467 480
468static int crypto_authenc_iverify(struct aead_request *req, u8 *iv, 481static int crypto_authenc_iverify(struct aead_request *req, u8 *iv,
diff --git a/crypto/authencesn.c b/crypto/authencesn.c
index ab53762fc309..4be0dd4373a9 100644
--- a/crypto/authencesn.c
+++ b/crypto/authencesn.c
@@ -59,37 +59,19 @@ static void authenc_esn_request_complete(struct aead_request *req, int err)
59static int crypto_authenc_esn_setkey(struct crypto_aead *authenc_esn, const u8 *key, 59static int crypto_authenc_esn_setkey(struct crypto_aead *authenc_esn, const u8 *key,
60 unsigned int keylen) 60 unsigned int keylen)
61{ 61{
62 unsigned int authkeylen;
63 unsigned int enckeylen;
64 struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn); 62 struct crypto_authenc_esn_ctx *ctx = crypto_aead_ctx(authenc_esn);
65 struct crypto_ahash *auth = ctx->auth; 63 struct crypto_ahash *auth = ctx->auth;
66 struct crypto_ablkcipher *enc = ctx->enc; 64 struct crypto_ablkcipher *enc = ctx->enc;
67 struct rtattr *rta = (void *)key; 65 struct crypto_authenc_keys keys;
68 struct crypto_authenc_key_param *param;
69 int err = -EINVAL; 66 int err = -EINVAL;
70 67
71 if (!RTA_OK(rta, keylen)) 68 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
72 goto badkey; 69 goto badkey;
73 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
74 goto badkey;
75 if (RTA_PAYLOAD(rta) < sizeof(*param))
76 goto badkey;
77
78 param = RTA_DATA(rta);
79 enckeylen = be32_to_cpu(param->enckeylen);
80
81 key += RTA_ALIGN(rta->rta_len);
82 keylen -= RTA_ALIGN(rta->rta_len);
83
84 if (keylen < enckeylen)
85 goto badkey;
86
87 authkeylen = keylen - enckeylen;
88 70
89 crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK); 71 crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK);
90 crypto_ahash_set_flags(auth, crypto_aead_get_flags(authenc_esn) & 72 crypto_ahash_set_flags(auth, crypto_aead_get_flags(authenc_esn) &
91 CRYPTO_TFM_REQ_MASK); 73 CRYPTO_TFM_REQ_MASK);
92 err = crypto_ahash_setkey(auth, key, authkeylen); 74 err = crypto_ahash_setkey(auth, keys.authkey, keys.authkeylen);
93 crypto_aead_set_flags(authenc_esn, crypto_ahash_get_flags(auth) & 75 crypto_aead_set_flags(authenc_esn, crypto_ahash_get_flags(auth) &
94 CRYPTO_TFM_RES_MASK); 76 CRYPTO_TFM_RES_MASK);
95 77
@@ -99,7 +81,7 @@ static int crypto_authenc_esn_setkey(struct crypto_aead *authenc_esn, const u8 *
99 crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK); 81 crypto_ablkcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK);
100 crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc_esn) & 82 crypto_ablkcipher_set_flags(enc, crypto_aead_get_flags(authenc_esn) &
101 CRYPTO_TFM_REQ_MASK); 83 CRYPTO_TFM_REQ_MASK);
102 err = crypto_ablkcipher_setkey(enc, key + authkeylen, enckeylen); 84 err = crypto_ablkcipher_setkey(enc, keys.enckey, keys.enckeylen);
103 crypto_aead_set_flags(authenc_esn, crypto_ablkcipher_get_flags(enc) & 85 crypto_aead_set_flags(authenc_esn, crypto_ablkcipher_get_flags(enc) &
104 CRYPTO_TFM_RES_MASK); 86 CRYPTO_TFM_RES_MASK);
105 87
@@ -247,7 +229,7 @@ static void authenc_esn_verify_ahash_update_done(struct crypto_async_request *ar
247 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, 229 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
248 authsize, 0); 230 authsize, 0);
249 231
250 err = memcmp(ihash, ahreq->result, authsize) ? -EBADMSG : 0; 232 err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0;
251 if (err) 233 if (err)
252 goto out; 234 goto out;
253 235
@@ -296,7 +278,7 @@ static void authenc_esn_verify_ahash_update_done2(struct crypto_async_request *a
296 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, 278 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
297 authsize, 0); 279 authsize, 0);
298 280
299 err = memcmp(ihash, ahreq->result, authsize) ? -EBADMSG : 0; 281 err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0;
300 if (err) 282 if (err)
301 goto out; 283 goto out;
302 284
@@ -336,7 +318,7 @@ static void authenc_esn_verify_ahash_done(struct crypto_async_request *areq,
336 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, 318 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
337 authsize, 0); 319 authsize, 0);
338 320
339 err = memcmp(ihash, ahreq->result, authsize) ? -EBADMSG : 0; 321 err = crypto_memneq(ihash, ahreq->result, authsize) ? -EBADMSG : 0;
340 if (err) 322 if (err)
341 goto out; 323 goto out;
342 324
@@ -568,7 +550,7 @@ static int crypto_authenc_esn_verify(struct aead_request *req)
568 ihash = ohash + authsize; 550 ihash = ohash + authsize;
569 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, 551 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
570 authsize, 0); 552 authsize, 0);
571 return memcmp(ihash, ohash, authsize) ? -EBADMSG : 0; 553 return crypto_memneq(ihash, ohash, authsize) ? -EBADMSG : 0;
572} 554}
573 555
574static int crypto_authenc_esn_iverify(struct aead_request *req, u8 *iv, 556static int crypto_authenc_esn_iverify(struct aead_request *req, u8 *iv,
diff --git a/crypto/ccm.c b/crypto/ccm.c
index 499c91717d93..1df84217f7c9 100644
--- a/crypto/ccm.c
+++ b/crypto/ccm.c
@@ -271,7 +271,8 @@ static int crypto_ccm_auth(struct aead_request *req, struct scatterlist *plain,
271 } 271 }
272 272
273 /* compute plaintext into mac */ 273 /* compute plaintext into mac */
274 get_data_to_compute(cipher, pctx, plain, cryptlen); 274 if (cryptlen)
275 get_data_to_compute(cipher, pctx, plain, cryptlen);
275 276
276out: 277out:
277 return err; 278 return err;
@@ -363,7 +364,7 @@ static void crypto_ccm_decrypt_done(struct crypto_async_request *areq,
363 364
364 if (!err) { 365 if (!err) {
365 err = crypto_ccm_auth(req, req->dst, cryptlen); 366 err = crypto_ccm_auth(req, req->dst, cryptlen);
366 if (!err && memcmp(pctx->auth_tag, pctx->odata, authsize)) 367 if (!err && crypto_memneq(pctx->auth_tag, pctx->odata, authsize))
367 err = -EBADMSG; 368 err = -EBADMSG;
368 } 369 }
369 aead_request_complete(req, err); 370 aead_request_complete(req, err);
@@ -422,7 +423,7 @@ static int crypto_ccm_decrypt(struct aead_request *req)
422 return err; 423 return err;
423 424
424 /* verify */ 425 /* verify */
425 if (memcmp(authtag, odata, authsize)) 426 if (crypto_memneq(authtag, odata, authsize))
426 return -EBADMSG; 427 return -EBADMSG;
427 428
428 return err; 429 return err;
diff --git a/crypto/gcm.c b/crypto/gcm.c
index 43e1fb05ea54..b4f017939004 100644
--- a/crypto/gcm.c
+++ b/crypto/gcm.c
@@ -582,7 +582,7 @@ static int crypto_gcm_verify(struct aead_request *req,
582 582
583 crypto_xor(auth_tag, iauth_tag, 16); 583 crypto_xor(auth_tag, iauth_tag, 16);
584 scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0); 584 scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
585 return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; 585 return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
586} 586}
587 587
588static void gcm_decrypt_done(struct crypto_async_request *areq, int err) 588static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
diff --git a/crypto/hash_info.c b/crypto/hash_info.c
new file mode 100644
index 000000000000..3e7ff46f26e8
--- /dev/null
+++ b/crypto/hash_info.c
@@ -0,0 +1,56 @@
1/*
2 * Hash Info: Hash algorithms information
3 *
4 * Copyright (c) 2013 Dmitry Kasatkin <d.kasatkin@samsung.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the Free
8 * Software Foundation; either version 2 of the License, or (at your option)
9 * any later version.
10 *
11 */
12
13#include <linux/export.h>
14#include <crypto/hash_info.h>
15
16const char *const hash_algo_name[HASH_ALGO__LAST] = {
17 [HASH_ALGO_MD4] = "md4",
18 [HASH_ALGO_MD5] = "md5",
19 [HASH_ALGO_SHA1] = "sha1",
20 [HASH_ALGO_RIPE_MD_160] = "rmd160",
21 [HASH_ALGO_SHA256] = "sha256",
22 [HASH_ALGO_SHA384] = "sha384",
23 [HASH_ALGO_SHA512] = "sha512",
24 [HASH_ALGO_SHA224] = "sha224",
25 [HASH_ALGO_RIPE_MD_128] = "rmd128",
26 [HASH_ALGO_RIPE_MD_256] = "rmd256",
27 [HASH_ALGO_RIPE_MD_320] = "rmd320",
28 [HASH_ALGO_WP_256] = "wp256",
29 [HASH_ALGO_WP_384] = "wp384",
30 [HASH_ALGO_WP_512] = "wp512",
31 [HASH_ALGO_TGR_128] = "tgr128",
32 [HASH_ALGO_TGR_160] = "tgr160",
33 [HASH_ALGO_TGR_192] = "tgr192",
34};
35EXPORT_SYMBOL_GPL(hash_algo_name);
36
37const int hash_digest_size[HASH_ALGO__LAST] = {
38 [HASH_ALGO_MD4] = MD5_DIGEST_SIZE,
39 [HASH_ALGO_MD5] = MD5_DIGEST_SIZE,
40 [HASH_ALGO_SHA1] = SHA1_DIGEST_SIZE,
41 [HASH_ALGO_RIPE_MD_160] = RMD160_DIGEST_SIZE,
42 [HASH_ALGO_SHA256] = SHA256_DIGEST_SIZE,
43 [HASH_ALGO_SHA384] = SHA384_DIGEST_SIZE,
44 [HASH_ALGO_SHA512] = SHA512_DIGEST_SIZE,
45 [HASH_ALGO_SHA224] = SHA224_DIGEST_SIZE,
46 [HASH_ALGO_RIPE_MD_128] = RMD128_DIGEST_SIZE,
47 [HASH_ALGO_RIPE_MD_256] = RMD256_DIGEST_SIZE,
48 [HASH_ALGO_RIPE_MD_320] = RMD320_DIGEST_SIZE,
49 [HASH_ALGO_WP_256] = WP256_DIGEST_SIZE,
50 [HASH_ALGO_WP_384] = WP384_DIGEST_SIZE,
51 [HASH_ALGO_WP_512] = WP512_DIGEST_SIZE,
52 [HASH_ALGO_TGR_128] = TGR128_DIGEST_SIZE,
53 [HASH_ALGO_TGR_160] = TGR160_DIGEST_SIZE,
54 [HASH_ALGO_TGR_192] = TGR192_DIGEST_SIZE,
55};
56EXPORT_SYMBOL_GPL(hash_digest_size);
diff --git a/crypto/memneq.c b/crypto/memneq.c
new file mode 100644
index 000000000000..cd0162221c14
--- /dev/null
+++ b/crypto/memneq.c
@@ -0,0 +1,138 @@
1/*
2 * Constant-time equality testing of memory regions.
3 *
4 * Authors:
5 *
6 * James Yonan <james@openvpn.net>
7 * Daniel Borkmann <dborkman@redhat.com>
8 *
9 * This file is provided under a dual BSD/GPLv2 license. When using or
10 * redistributing this file, you may do so under either license.
11 *
12 * GPL LICENSE SUMMARY
13 *
14 * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of version 2 of the GNU General Public License as
18 * published by the Free Software Foundation.
19 *
20 * This program is distributed in the hope that it will be useful, but
21 * WITHOUT ANY WARRANTY; without even the implied warranty of
22 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
23 * General Public License for more details.
24 *
25 * You should have received a copy of the GNU General Public License
26 * along with this program; if not, write to the Free Software
27 * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
28 * The full GNU General Public License is included in this distribution
29 * in the file called LICENSE.GPL.
30 *
31 * BSD LICENSE
32 *
33 * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved.
34 *
35 * Redistribution and use in source and binary forms, with or without
36 * modification, are permitted provided that the following conditions
37 * are met:
38 *
39 * * Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * * Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in
43 * the documentation and/or other materials provided with the
44 * distribution.
45 * * Neither the name of OpenVPN Technologies nor the names of its
46 * contributors may be used to endorse or promote products derived
47 * from this software without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
50 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
51 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
52 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
53 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
55 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
56 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
57 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
58 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
59 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
60 */
61
62#include <crypto/algapi.h>
63
64#ifndef __HAVE_ARCH_CRYPTO_MEMNEQ
65
66/* Generic path for arbitrary size */
67static inline unsigned long
68__crypto_memneq_generic(const void *a, const void *b, size_t size)
69{
70 unsigned long neq = 0;
71
72#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
73 while (size >= sizeof(unsigned long)) {
74 neq |= *(unsigned long *)a ^ *(unsigned long *)b;
75 a += sizeof(unsigned long);
76 b += sizeof(unsigned long);
77 size -= sizeof(unsigned long);
78 }
79#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */
80 while (size > 0) {
81 neq |= *(unsigned char *)a ^ *(unsigned char *)b;
82 a += 1;
83 b += 1;
84 size -= 1;
85 }
86 return neq;
87}
88
89/* Loop-free fast-path for frequently used 16-byte size */
90static inline unsigned long __crypto_memneq_16(const void *a, const void *b)
91{
92#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
93 if (sizeof(unsigned long) == 8)
94 return ((*(unsigned long *)(a) ^ *(unsigned long *)(b))
95 | (*(unsigned long *)(a+8) ^ *(unsigned long *)(b+8)));
96 else if (sizeof(unsigned int) == 4)
97 return ((*(unsigned int *)(a) ^ *(unsigned int *)(b))
98 | (*(unsigned int *)(a+4) ^ *(unsigned int *)(b+4))
99 | (*(unsigned int *)(a+8) ^ *(unsigned int *)(b+8))
100 | (*(unsigned int *)(a+12) ^ *(unsigned int *)(b+12)));
101 else
102#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */
103 return ((*(unsigned char *)(a) ^ *(unsigned char *)(b))
104 | (*(unsigned char *)(a+1) ^ *(unsigned char *)(b+1))
105 | (*(unsigned char *)(a+2) ^ *(unsigned char *)(b+2))
106 | (*(unsigned char *)(a+3) ^ *(unsigned char *)(b+3))
107 | (*(unsigned char *)(a+4) ^ *(unsigned char *)(b+4))
108 | (*(unsigned char *)(a+5) ^ *(unsigned char *)(b+5))
109 | (*(unsigned char *)(a+6) ^ *(unsigned char *)(b+6))
110 | (*(unsigned char *)(a+7) ^ *(unsigned char *)(b+7))
111 | (*(unsigned char *)(a+8) ^ *(unsigned char *)(b+8))
112 | (*(unsigned char *)(a+9) ^ *(unsigned char *)(b+9))
113 | (*(unsigned char *)(a+10) ^ *(unsigned char *)(b+10))
114 | (*(unsigned char *)(a+11) ^ *(unsigned char *)(b+11))
115 | (*(unsigned char *)(a+12) ^ *(unsigned char *)(b+12))
116 | (*(unsigned char *)(a+13) ^ *(unsigned char *)(b+13))
117 | (*(unsigned char *)(a+14) ^ *(unsigned char *)(b+14))
118 | (*(unsigned char *)(a+15) ^ *(unsigned char *)(b+15)));
119}
120
121/* Compare two areas of memory without leaking timing information,
122 * and with special optimizations for common sizes. Users should
123 * not call this function directly, but should instead use
124 * crypto_memneq defined in crypto/algapi.h.
125 */
126noinline unsigned long __crypto_memneq(const void *a, const void *b,
127 size_t size)
128{
129 switch (size) {
130 case 16:
131 return __crypto_memneq_16(a, b);
132 default:
133 return __crypto_memneq_generic(a, b, size);
134 }
135}
136EXPORT_SYMBOL(__crypto_memneq);
137
138#endif /* __HAVE_ARCH_CRYPTO_MEMNEQ */
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index 1ab8258fcf56..001f07cdb828 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -1242,6 +1242,10 @@ static int do_test(int m)
1242 ret += tcrypt_test("cmac(des3_ede)"); 1242 ret += tcrypt_test("cmac(des3_ede)");
1243 break; 1243 break;
1244 1244
1245 case 155:
1246 ret += tcrypt_test("authenc(hmac(sha1),cbc(aes))");
1247 break;
1248
1245 case 200: 1249 case 200:
1246 test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0, 1250 test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
1247 speed_template_16_24_32); 1251 speed_template_16_24_32);
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 432afc03e7c3..77955507f6f1 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -503,16 +503,16 @@ static int __test_aead(struct crypto_aead *tfm, int enc,
503 goto out; 503 goto out;
504 } 504 }
505 505
506 sg_init_one(&sg[0], input,
507 template[i].ilen + (enc ? authsize : 0));
508
509 if (diff_dst) { 506 if (diff_dst) {
510 output = xoutbuf[0]; 507 output = xoutbuf[0];
511 output += align_offset; 508 output += align_offset;
509 sg_init_one(&sg[0], input, template[i].ilen);
512 sg_init_one(&sgout[0], output, 510 sg_init_one(&sgout[0], output,
511 template[i].rlen);
512 } else {
513 sg_init_one(&sg[0], input,
513 template[i].ilen + 514 template[i].ilen +
514 (enc ? authsize : 0)); 515 (enc ? authsize : 0));
515 } else {
516 output = input; 516 output = input;
517 } 517 }
518 518
@@ -612,12 +612,6 @@ static int __test_aead(struct crypto_aead *tfm, int enc,
612 memcpy(q, template[i].input + temp, 612 memcpy(q, template[i].input + temp,
613 template[i].tap[k]); 613 template[i].tap[k]);
614 614
615 n = template[i].tap[k];
616 if (k == template[i].np - 1 && enc)
617 n += authsize;
618 if (offset_in_page(q) + n < PAGE_SIZE)
619 q[n] = 0;
620
621 sg_set_buf(&sg[k], q, template[i].tap[k]); 615 sg_set_buf(&sg[k], q, template[i].tap[k]);
622 616
623 if (diff_dst) { 617 if (diff_dst) {
@@ -625,13 +619,17 @@ static int __test_aead(struct crypto_aead *tfm, int enc,
625 offset_in_page(IDX[k]); 619 offset_in_page(IDX[k]);
626 620
627 memset(q, 0, template[i].tap[k]); 621 memset(q, 0, template[i].tap[k]);
628 if (offset_in_page(q) + n < PAGE_SIZE)
629 q[n] = 0;
630 622
631 sg_set_buf(&sgout[k], q, 623 sg_set_buf(&sgout[k], q,
632 template[i].tap[k]); 624 template[i].tap[k]);
633 } 625 }
634 626
627 n = template[i].tap[k];
628 if (k == template[i].np - 1 && enc)
629 n += authsize;
630 if (offset_in_page(q) + n < PAGE_SIZE)
631 q[n] = 0;
632
635 temp += template[i].tap[k]; 633 temp += template[i].tap[k];
636 } 634 }
637 635
@@ -650,10 +648,10 @@ static int __test_aead(struct crypto_aead *tfm, int enc,
650 goto out; 648 goto out;
651 } 649 }
652 650
653 sg[k - 1].length += authsize;
654
655 if (diff_dst) 651 if (diff_dst)
656 sgout[k - 1].length += authsize; 652 sgout[k - 1].length += authsize;
653 else
654 sg[k - 1].length += authsize;
657 } 655 }
658 656
659 sg_init_table(asg, template[i].anp); 657 sg_init_table(asg, template[i].anp);
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-08 04:01:13 -0400