aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/devicetree/bindings/rng/apm,rng.txt17
-rw-r--r--arch/arm64/boot/dts/apm-storm.dtsi21
-rw-r--r--arch/x86/crypto/Makefile1
-rw-r--r--arch/x86/crypto/aes_ctrby8_avx-x86_64.S20
-rw-r--r--arch/x86/crypto/aesni-intel_glue.c4
-rw-r--r--arch/x86/crypto/sha-mb/Makefile11
-rw-r--r--arch/x86/crypto/sha-mb/sha1_mb.c935
-rw-r--r--arch/x86/crypto/sha-mb/sha1_mb_mgr_datastruct.S287
-rw-r--r--arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S327
-rw-r--r--arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c64
-rw-r--r--arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S228
-rw-r--r--arch/x86/crypto/sha-mb/sha1_x8_avx2.S472
-rw-r--r--arch/x86/crypto/sha-mb/sha_mb_ctx.h136
-rw-r--r--arch/x86/crypto/sha-mb/sha_mb_mgr.h110
-rw-r--r--crypto/Kconfig30
-rw-r--r--crypto/Makefile1
-rw-r--r--crypto/ahash.c12
-rw-r--r--crypto/algif_skcipher.c2
-rw-r--r--crypto/drbg.c130
-rw-r--r--crypto/lz4.c2
-rw-r--r--crypto/lz4hc.c2
-rw-r--r--crypto/mcryptd.c705
-rw-r--r--crypto/sha256_generic.c3
-rw-r--r--crypto/sha512_generic.c3
-rw-r--r--crypto/testmgr.c966
-rw-r--r--crypto/testmgr.h66
-rw-r--r--drivers/char/hw_random/Kconfig13
-rw-r--r--drivers/char/hw_random/Makefile1
-rw-r--r--drivers/char/hw_random/amd-rng.c4
-rw-r--r--drivers/char/hw_random/geode-rng.c4
-rw-r--r--drivers/char/hw_random/intel-rng.c13
-rw-r--r--drivers/char/hw_random/pasemi-rng.c2
-rw-r--r--drivers/char/hw_random/pseries-rng.c2
-rw-r--r--drivers/char/hw_random/via-rng.c8
-rw-r--r--drivers/char/hw_random/xgene-rng.c423
-rw-r--r--drivers/crypto/caam/caamhash.c28
-rw-r--r--drivers/crypto/caam/ctrl.c138
-rw-r--r--drivers/crypto/caam/intern.h9
-rw-r--r--drivers/crypto/caam/regs.h51
-rw-r--r--drivers/crypto/caam/sg_sw_sec4.h54
-rw-r--r--drivers/crypto/mv_cesa.h1
-rw-r--r--drivers/crypto/qat/qat_common/adf_ctl_drv.c2
-rw-r--r--drivers/crypto/qat/qat_common/adf_transport_internal.h2
-rw-r--r--drivers/crypto/qat/qat_common/qat_algs.c66
-rw-r--r--drivers/crypto/qat/qat_dh895xcc/adf_isr.c14
-rw-r--r--include/crypto/drbg.h19
-rw-r--r--include/crypto/internal/hash.h9
-rw-r--r--include/crypto/mcryptd.h112
-rw-r--r--include/linux/sched.h1
-rw-r--r--kernel/sched/core.c12
50 files changed, 4706 insertions, 837 deletions
diff --git a/Documentation/devicetree/bindings/rng/apm,rng.txt b/Documentation/devicetree/bindings/rng/apm,rng.txt
new file mode 100644
index 000000000000..4dde4b06cdd9
--- /dev/null
+++ b/Documentation/devicetree/bindings/rng/apm,rng.txt
@@ -0,0 +1,17 @@
1APM X-Gene SoC random number generator.
2
3Required properties:
4
5- compatible : should be "apm,xgene-rng"
6- reg : specifies base physical address and size of the registers map
7- clocks : phandle to clock-controller plus clock-specifier pair
8- interrupts : specify the fault interrupt for the RNG device
9
10Example:
11
12 rng: rng@10520000 {
13 compatible = "apm,xgene-rng";
14 reg = <0x0 0x10520000 0x0 0x100>;
15 interrupts = <0x0 0x41 0x4>;
16 clocks = <&rngpkaclk 0>;
17 };
diff --git a/arch/arm64/boot/dts/apm-storm.dtsi b/arch/arm64/boot/dts/apm-storm.dtsi
index c0aceef7f5b3..f391972ad135 100644
--- a/arch/arm64/boot/dts/apm-storm.dtsi
+++ b/arch/arm64/boot/dts/apm-storm.dtsi
@@ -269,6 +269,19 @@
269 enable-mask = <0x2>; 269 enable-mask = <0x2>;
270 clock-output-names = "rtcclk"; 270 clock-output-names = "rtcclk";
271 }; 271 };
272
273 rngpkaclk: rngpkaclk@17000000 {
274 compatible = "apm,xgene-device-clock";
275 #clock-cells = <1>;
276 clocks = <&socplldiv2 0>;
277 reg = <0x0 0x17000000 0x0 0x2000>;
278 reg-names = "csr-reg";
279 csr-offset = <0xc>;
280 csr-mask = <0x10>;
281 enable-offset = <0x10>;
282 enable-mask = <0x10>;
283 clock-output-names = "rngpkaclk";
284 };
272 }; 285 };
273 286
274 serial0: serial@1c020000 { 287 serial0: serial@1c020000 {
@@ -421,5 +434,13 @@
421 434
422 }; 435 };
423 }; 436 };
437
438 rng: rng@10520000 {
439 compatible = "apm,xgene-rng";
440 reg = <0x0 0x10520000 0x0 0x100>;
441 interrupts = <0x0 0x41 0x4>;
442 clocks = <&rngpkaclk 0>;
443 };
444
424 }; 445 };
425}; 446};
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index d551165a3159..fd0f848938cc 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -26,6 +26,7 @@ obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
26 26
27obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o 27obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
28obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o 28obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o
29obj-$(CONFIG_CRYPTO_SHA1_MB) += sha-mb/
29obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o 30obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o
30obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o 31obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o
31obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o 32obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o
diff --git a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
index f091f122ed24..2df2a0298f5a 100644
--- a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
+++ b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S
@@ -79,9 +79,6 @@
79#define xcounter %xmm8 79#define xcounter %xmm8
80#define xbyteswap %xmm9 80#define xbyteswap %xmm9
81#define xkey0 %xmm10 81#define xkey0 %xmm10
82#define xkey3 %xmm11
83#define xkey6 %xmm12
84#define xkey9 %xmm13
85#define xkey4 %xmm11 82#define xkey4 %xmm11
86#define xkey8 %xmm12 83#define xkey8 %xmm12
87#define xkey12 %xmm13 84#define xkey12 %xmm13
@@ -108,6 +105,10 @@
108 105
109byteswap_const: 106byteswap_const:
110 .octa 0x000102030405060708090A0B0C0D0E0F 107 .octa 0x000102030405060708090A0B0C0D0E0F
108ddq_low_msk:
109 .octa 0x0000000000000000FFFFFFFFFFFFFFFF
110ddq_high_add_1:
111 .octa 0x00000000000000010000000000000000
111ddq_add_1: 112ddq_add_1:
112 .octa 0x00000000000000000000000000000001 113 .octa 0x00000000000000000000000000000001
113ddq_add_2: 114ddq_add_2:
@@ -169,7 +170,12 @@ ddq_add_8:
169 .rept (by - 1) 170 .rept (by - 1)
170 club DDQ_DATA, i 171 club DDQ_DATA, i
171 club XDATA, i 172 club XDATA, i
172 vpaddd var_ddq_add(%rip), xcounter, var_xdata 173 vpaddq var_ddq_add(%rip), xcounter, var_xdata
174 vptest ddq_low_msk(%rip), var_xdata
175 jnz 1f
176 vpaddq ddq_high_add_1(%rip), var_xdata, var_xdata
177 vpaddq ddq_high_add_1(%rip), xcounter, xcounter
178 1:
173 vpshufb xbyteswap, var_xdata, var_xdata 179 vpshufb xbyteswap, var_xdata, var_xdata
174 .set i, (i +1) 180 .set i, (i +1)
175 .endr 181 .endr
@@ -178,7 +184,11 @@ ddq_add_8:
178 184
179 vpxor xkey0, xdata0, xdata0 185 vpxor xkey0, xdata0, xdata0
180 club DDQ_DATA, by 186 club DDQ_DATA, by
181 vpaddd var_ddq_add(%rip), xcounter, xcounter 187 vpaddq var_ddq_add(%rip), xcounter, xcounter
188 vptest ddq_low_msk(%rip), xcounter
189 jnz 1f
190 vpaddq ddq_high_add_1(%rip), xcounter, xcounter
191 1:
182 192
183 .set i, 1 193 .set i, 1
184 .rept (by - 1) 194 .rept (by - 1)
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index a7ccd57f19e4..888950f29fd9 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -481,7 +481,7 @@ static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
481 crypto_inc(ctrblk, AES_BLOCK_SIZE); 481 crypto_inc(ctrblk, AES_BLOCK_SIZE);
482} 482}
483 483
484#if 0 /* temporary disabled due to failing crypto tests */ 484#ifdef CONFIG_AS_AVX
485static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, 485static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
486 const u8 *in, unsigned int len, u8 *iv) 486 const u8 *in, unsigned int len, u8 *iv)
487{ 487{
@@ -1522,7 +1522,7 @@ static int __init aesni_init(void)
1522 aesni_gcm_dec_tfm = aesni_gcm_dec; 1522 aesni_gcm_dec_tfm = aesni_gcm_dec;
1523 } 1523 }
1524 aesni_ctr_enc_tfm = aesni_ctr_enc; 1524 aesni_ctr_enc_tfm = aesni_ctr_enc;
1525#if 0 /* temporary disabled due to failing crypto tests */ 1525#ifdef CONFIG_AS_AVX
1526 if (cpu_has_avx) { 1526 if (cpu_has_avx) {
1527 /* optimize performance of ctr mode encryption transform */ 1527 /* optimize performance of ctr mode encryption transform */
1528 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm; 1528 aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
diff --git a/arch/x86/crypto/sha-mb/Makefile b/arch/x86/crypto/sha-mb/Makefile
new file mode 100644
index 000000000000..2f8756375df5
--- /dev/null
+++ b/arch/x86/crypto/sha-mb/Makefile
@@ -0,0 +1,11 @@
1#
2# Arch-specific CryptoAPI modules.
3#
4
5avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\
6 $(comma)4)$(comma)%ymm2,yes,no)
7ifeq ($(avx2_supported),yes)
8 obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb.o
9 sha1-mb-y := sha1_mb.o sha1_mb_mgr_flush_avx2.o \
10 sha1_mb_mgr_init_avx2.o sha1_mb_mgr_submit_avx2.o sha1_x8_avx2.o
11endif
diff --git a/arch/x86/crypto/sha-mb/sha1_mb.c b/arch/x86/crypto/sha-mb/sha1_mb.c
new file mode 100644
index 000000000000..99eefd812958
--- /dev/null
+++ b/arch/x86/crypto/sha-mb/sha1_mb.c
@@ -0,0 +1,935 @@
1/*
2 * Multi buffer SHA1 algorithm Glue Code
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * Copyright(c) 2014 Intel Corporation.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * Contact Information:
21 * Tim Chen <tim.c.chen@linux.intel.com>
22 *
23 * BSD LICENSE
24 *
25 * Copyright(c) 2014 Intel Corporation.
26 *
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
29 * are met:
30 *
31 * * Redistributions of source code must retain the above copyright
32 * notice, this list of conditions and the following disclaimer.
33 * * Redistributions in binary form must reproduce the above copyright
34 * notice, this list of conditions and the following disclaimer in
35 * the documentation and/or other materials provided with the
36 * distribution.
37 * * Neither the name of Intel Corporation nor the names of its
38 * contributors may be used to endorse or promote products derived
39 * from this software without specific prior written permission.
40 *
41 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
42 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
43 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
44 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
45 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
46 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
47 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
48 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
49 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
50 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
51 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
52 */
53
54#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
55
56#include <crypto/internal/hash.h>
57#include <linux/init.h>
58#include <linux/module.h>
59#include <linux/mm.h>
60#include <linux/cryptohash.h>
61#include <linux/types.h>
62#include <linux/list.h>
63#include <crypto/scatterwalk.h>
64#include <crypto/sha.h>
65#include <crypto/mcryptd.h>
66#include <crypto/crypto_wq.h>
67#include <asm/byteorder.h>
68#include <asm/i387.h>
69#include <asm/xcr.h>
70#include <asm/xsave.h>
71#include <linux/hardirq.h>
72#include <asm/fpu-internal.h>
73#include "sha_mb_ctx.h"
74
75#define FLUSH_INTERVAL 1000 /* in usec */
76
77static struct mcryptd_alg_state sha1_mb_alg_state;
78
79struct sha1_mb_ctx {
80 struct mcryptd_ahash *mcryptd_tfm;
81};
82
83static inline struct mcryptd_hash_request_ctx *cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx)
84{
85 struct shash_desc *desc;
86
87 desc = container_of((void *) hash_ctx, struct shash_desc, __ctx);
88 return container_of(desc, struct mcryptd_hash_request_ctx, desc);
89}
90
91static inline struct ahash_request *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx)
92{
93 return container_of((void *) ctx, struct ahash_request, __ctx);
94}
95
96static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx,
97 struct shash_desc *desc)
98{
99 rctx->flag = HASH_UPDATE;
100}
101
102static asmlinkage void (*sha1_job_mgr_init)(struct sha1_mb_mgr *state);
103static asmlinkage struct job_sha1* (*sha1_job_mgr_submit)(struct sha1_mb_mgr *state,
104 struct job_sha1 *job);
105static asmlinkage struct job_sha1* (*sha1_job_mgr_flush)(struct sha1_mb_mgr *state);
106static asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job)(struct sha1_mb_mgr *state);
107
108inline void sha1_init_digest(uint32_t *digest)
109{
110 static const uint32_t initial_digest[SHA1_DIGEST_LENGTH] = {SHA1_H0,
111 SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 };
112 memcpy(digest, initial_digest, sizeof(initial_digest));
113}
114
115inline uint32_t sha1_pad(uint8_t padblock[SHA1_BLOCK_SIZE * 2],
116 uint32_t total_len)
117{
118 uint32_t i = total_len & (SHA1_BLOCK_SIZE - 1);
119
120 memset(&padblock[i], 0, SHA1_BLOCK_SIZE);
121 padblock[i] = 0x80;
122
123 i += ((SHA1_BLOCK_SIZE - 1) &
124 (0 - (total_len + SHA1_PADLENGTHFIELD_SIZE + 1)))
125 + 1 + SHA1_PADLENGTHFIELD_SIZE;
126
127#if SHA1_PADLENGTHFIELD_SIZE == 16
128 *((uint64_t *) &padblock[i - 16]) = 0;
129#endif
130
131 *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3);
132
133 /* Number of extra blocks to hash */
134 return i >> SHA1_LOG2_BLOCK_SIZE;
135}
136
137static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr, struct sha1_hash_ctx *ctx)
138{
139 while (ctx) {
140 if (ctx->status & HASH_CTX_STS_COMPLETE) {
141 /* Clear PROCESSING bit */
142 ctx->status = HASH_CTX_STS_COMPLETE;
143 return ctx;
144 }
145
146 /*
147 * If the extra blocks are empty, begin hashing what remains
148 * in the user's buffer.
149 */
150 if (ctx->partial_block_buffer_length == 0 &&
151 ctx->incoming_buffer_length) {
152
153 const void *buffer = ctx->incoming_buffer;
154 uint32_t len = ctx->incoming_buffer_length;
155 uint32_t copy_len;
156
157 /*
158 * Only entire blocks can be hashed.
159 * Copy remainder to extra blocks buffer.
160 */
161 copy_len = len & (SHA1_BLOCK_SIZE-1);
162
163 if (copy_len) {
164 len -= copy_len;
165 memcpy(ctx->partial_block_buffer,
166 ((const char *) buffer + len),
167 copy_len);
168 ctx->partial_block_buffer_length = copy_len;
169 }
170
171 ctx->incoming_buffer_length = 0;
172
173 /* len should be a multiple of the block size now */
174 assert((len % SHA1_BLOCK_SIZE) == 0);
175
176 /* Set len to the number of blocks to be hashed */
177 len >>= SHA1_LOG2_BLOCK_SIZE;
178
179 if (len) {
180
181 ctx->job.buffer = (uint8_t *) buffer;
182 ctx->job.len = len;
183 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr,
184 &ctx->job);
185 continue;
186 }
187 }
188
189 /*
190 * If the extra blocks are not empty, then we are
191 * either on the last block(s) or we need more
192 * user input before continuing.
193 */
194 if (ctx->status & HASH_CTX_STS_LAST) {
195
196 uint8_t *buf = ctx->partial_block_buffer;
197 uint32_t n_extra_blocks = sha1_pad(buf, ctx->total_length);
198
199 ctx->status = (HASH_CTX_STS_PROCESSING |
200 HASH_CTX_STS_COMPLETE);
201 ctx->job.buffer = buf;
202 ctx->job.len = (uint32_t) n_extra_blocks;
203 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
204 continue;
205 }
206
207 if (ctx)
208 ctx->status = HASH_CTX_STS_IDLE;
209 return ctx;
210 }
211
212 return NULL;
213}
214
215static struct sha1_hash_ctx *sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr)
216{
217 /*
218 * If get_comp_job returns NULL, there are no jobs complete.
219 * If get_comp_job returns a job, verify that it is safe to return to the user.
220 * If it is not ready, resubmit the job to finish processing.
221 * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned.
222 * Otherwise, all jobs currently being managed by the hash_ctx_mgr still need processing.
223 */
224 struct sha1_hash_ctx *ctx;
225
226 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_get_comp_job(&mgr->mgr);
227 return sha1_ctx_mgr_resubmit(mgr, ctx);
228}
229
230static void sha1_ctx_mgr_init(struct sha1_ctx_mgr *mgr)
231{
232 sha1_job_mgr_init(&mgr->mgr);
233}
234
235static struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr,
236 struct sha1_hash_ctx *ctx,
237 const void *buffer,
238 uint32_t len,
239 int flags)
240{
241 if (flags & (~HASH_ENTIRE)) {
242 /* User should not pass anything other than FIRST, UPDATE, or LAST */
243 ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
244 return ctx;
245 }
246
247 if (ctx->status & HASH_CTX_STS_PROCESSING) {
248 /* Cannot submit to a currently processing job. */
249 ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING;
250 return ctx;
251 }
252
253 if ((ctx->status & HASH_CTX_STS_COMPLETE) && !(flags & HASH_FIRST)) {
254 /* Cannot update a finished job. */
255 ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED;
256 return ctx;
257 }
258
259
260 if (flags & HASH_FIRST) {
261 /* Init digest */
262 sha1_init_digest(ctx->job.result_digest);
263
264 /* Reset byte counter */
265 ctx->total_length = 0;
266
267 /* Clear extra blocks */
268 ctx->partial_block_buffer_length = 0;
269 }
270
271 /* If we made it here, there were no errors during this call to submit */
272 ctx->error = HASH_CTX_ERROR_NONE;
273
274 /* Store buffer ptr info from user */
275 ctx->incoming_buffer = buffer;
276 ctx->incoming_buffer_length = len;
277
278 /* Store the user's request flags and mark this ctx as currently being processed. */
279 ctx->status = (flags & HASH_LAST) ?
280 (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) :
281 HASH_CTX_STS_PROCESSING;
282
283 /* Advance byte counter */
284 ctx->total_length += len;
285
286 /*
287 * If there is anything currently buffered in the extra blocks,
288 * append to it until it contains a whole block.
289 * Or if the user's buffer contains less than a whole block,
290 * append as much as possible to the extra block.
291 */
292 if ((ctx->partial_block_buffer_length) | (len < SHA1_BLOCK_SIZE)) {
293 /* Compute how many bytes to copy from user buffer into extra block */
294 uint32_t copy_len = SHA1_BLOCK_SIZE - ctx->partial_block_buffer_length;
295 if (len < copy_len)
296 copy_len = len;
297
298 if (copy_len) {
299 /* Copy and update relevant pointers and counters */
300 memcpy(&ctx->partial_block_buffer[ctx->partial_block_buffer_length],
301 buffer, copy_len);
302
303 ctx->partial_block_buffer_length += copy_len;
304 ctx->incoming_buffer = (const void *)((const char *)buffer + copy_len);
305 ctx->incoming_buffer_length = len - copy_len;
306 }
307
308 /* The extra block should never contain more than 1 block here */
309 assert(ctx->partial_block_buffer_length <= SHA1_BLOCK_SIZE);
310
311 /* If the extra block buffer contains exactly 1 block, it can be hashed. */
312 if (ctx->partial_block_buffer_length >= SHA1_BLOCK_SIZE) {
313 ctx->partial_block_buffer_length = 0;
314
315 ctx->job.buffer = ctx->partial_block_buffer;
316 ctx->job.len = 1;
317 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
318 }
319 }
320
321 return sha1_ctx_mgr_resubmit(mgr, ctx);
322}
323
324static struct sha1_hash_ctx *sha1_ctx_mgr_flush(struct sha1_ctx_mgr *mgr)
325{
326 struct sha1_hash_ctx *ctx;
327
328 while (1) {
329 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_flush(&mgr->mgr);
330
331 /* If flush returned 0, there are no more jobs in flight. */
332 if (!ctx)
333 return NULL;
334
335 /*
336 * If flush returned a job, resubmit the job to finish processing.
337 */
338 ctx = sha1_ctx_mgr_resubmit(mgr, ctx);
339
340 /*
341 * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned.
342 * Otherwise, all jobs currently being managed by the sha1_ctx_mgr
343 * still need processing. Loop.
344 */
345 if (ctx)
346 return ctx;
347 }
348}
349
350static int sha1_mb_init(struct shash_desc *desc)
351{
352 struct sha1_hash_ctx *sctx = shash_desc_ctx(desc);
353
354 hash_ctx_init(sctx);
355 sctx->job.result_digest[0] = SHA1_H0;
356 sctx->job.result_digest[1] = SHA1_H1;
357 sctx->job.result_digest[2] = SHA1_H2;
358 sctx->job.result_digest[3] = SHA1_H3;
359 sctx->job.result_digest[4] = SHA1_H4;
360 sctx->total_length = 0;
361 sctx->partial_block_buffer_length = 0;
362 sctx->status = HASH_CTX_STS_IDLE;
363
364 return 0;
365}
366
367static int sha1_mb_set_results(struct mcryptd_hash_request_ctx *rctx)
368{
369 int i;
370 struct sha1_hash_ctx *sctx = shash_desc_ctx(&rctx->desc);
371 __be32 *dst = (__be32 *) rctx->out;
372
373 for (i = 0; i < 5; ++i)
374 dst[i] = cpu_to_be32(sctx->job.result_digest[i]);
375
376 return 0;
377}
378
379static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
380 struct mcryptd_alg_cstate *cstate, bool flush)
381{
382 int flag = HASH_UPDATE;
383 int nbytes, err = 0;
384 struct mcryptd_hash_request_ctx *rctx = *ret_rctx;
385 struct sha1_hash_ctx *sha_ctx;
386
387 /* more work ? */
388 while (!(rctx->flag & HASH_DONE)) {
389 nbytes = crypto_ahash_walk_done(&rctx->walk, 0);
390 if (nbytes < 0) {
391 err = nbytes;
392 goto out;
393 }
394 /* check if the walk is done */
395 if (crypto_ahash_walk_last(&rctx->walk)) {
396 rctx->flag |= HASH_DONE;
397 if (rctx->flag & HASH_FINAL)
398 flag |= HASH_LAST;
399
400 }
401 sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(&rctx->desc);
402 kernel_fpu_begin();
403 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, flag);
404 if (!sha_ctx) {
405 if (flush)
406 sha_ctx = sha1_ctx_mgr_flush(cstate->mgr);
407 }
408 kernel_fpu_end();
409 if (sha_ctx)
410 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
411 else {
412 rctx = NULL;
413 goto out;
414 }
415 }
416
417 /* copy the results */
418 if (rctx->flag & HASH_FINAL)
419 sha1_mb_set_results(rctx);
420
421out:
422 *ret_rctx = rctx;
423 return err;
424}
425
426static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
427 struct mcryptd_alg_cstate *cstate,
428 int err)
429{
430 struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
431 struct sha1_hash_ctx *sha_ctx;
432 struct mcryptd_hash_request_ctx *req_ctx;
433 int ret;
434
435 /* remove from work list */
436 spin_lock(&cstate->work_lock);
437 list_del(&rctx->waiter);
438 spin_unlock(&cstate->work_lock);
439
440 if (irqs_disabled())
441 rctx->complete(&req->base, err);
442 else {
443 local_bh_disable();
444 rctx->complete(&req->base, err);
445 local_bh_enable();
446 }
447
448 /* check to see if there are other jobs that are done */
449 sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr);
450 while (sha_ctx) {
451 req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
452 ret = sha_finish_walk(&req_ctx, cstate, false);
453 if (req_ctx) {
454 spin_lock(&cstate->work_lock);
455 list_del(&req_ctx->waiter);
456 spin_unlock(&cstate->work_lock);
457
458 req = cast_mcryptd_ctx_to_req(req_ctx);
459 if (irqs_disabled())
460 rctx->complete(&req->base, ret);
461 else {
462 local_bh_disable();
463 rctx->complete(&req->base, ret);
464 local_bh_enable();
465 }
466 }
467 sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr);
468 }
469
470 return 0;
471}
472
473static void sha1_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
474 struct mcryptd_alg_cstate *cstate)
475{
476 unsigned long next_flush;
477 unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);
478
479 /* initialize tag */
480 rctx->tag.arrival = jiffies; /* tag the arrival time */
481 rctx->tag.seq_num = cstate->next_seq_num++;
482 next_flush = rctx->tag.arrival + delay;
483 rctx->tag.expire = next_flush;
484
485 spin_lock(&cstate->work_lock);
486 list_add_tail(&rctx->waiter, &cstate->work_list);
487 spin_unlock(&cstate->work_lock);
488
489 mcryptd_arm_flusher(cstate, delay);
490}
491
492static int sha1_mb_update(struct shash_desc *desc, const u8 *data,
493 unsigned int len)
494{
495 struct mcryptd_hash_request_ctx *rctx =
496 container_of(desc, struct mcryptd_hash_request_ctx, desc);
497 struct mcryptd_alg_cstate *cstate =
498 this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
499
500 struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
501 struct sha1_hash_ctx *sha_ctx;
502 int ret = 0, nbytes;
503
504
505 /* sanity check */
506 if (rctx->tag.cpu != smp_processor_id()) {
507 pr_err("mcryptd error: cpu clash\n");
508 goto done;
509 }
510
511 /* need to init context */
512 req_ctx_init(rctx, desc);
513
514 nbytes = crypto_ahash_walk_first(req, &rctx->walk);
515
516 if (nbytes < 0) {
517 ret = nbytes;
518 goto done;
519 }
520
521 if (crypto_ahash_walk_last(&rctx->walk))
522 rctx->flag |= HASH_DONE;
523
524 /* submit */
525 sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc);
526 sha1_mb_add_list(rctx, cstate);
527 kernel_fpu_begin();
528 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, HASH_UPDATE);
529 kernel_fpu_end();
530
531 /* check if anything is returned */
532 if (!sha_ctx)
533 return -EINPROGRESS;
534
535 if (sha_ctx->error) {
536 ret = sha_ctx->error;
537 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
538 goto done;
539 }
540
541 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
542 ret = sha_finish_walk(&rctx, cstate, false);
543
544 if (!rctx)
545 return -EINPROGRESS;
546done:
547 sha_complete_job(rctx, cstate, ret);
548 return ret;
549}
550
551static int sha1_mb_finup(struct shash_desc *desc, const u8 *data,
552 unsigned int len, u8 *out)
553{
554 struct mcryptd_hash_request_ctx *rctx =
555 container_of(desc, struct mcryptd_hash_request_ctx, desc);
556 struct mcryptd_alg_cstate *cstate =
557 this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
558
559 struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
560 struct sha1_hash_ctx *sha_ctx;
561 int ret = 0, flag = HASH_UPDATE, nbytes;
562
563 /* sanity check */
564 if (rctx->tag.cpu != smp_processor_id()) {
565 pr_err("mcryptd error: cpu clash\n");
566 goto done;
567 }
568
569 /* need to init context */
570 req_ctx_init(rctx, desc);
571
572 nbytes = crypto_ahash_walk_first(req, &rctx->walk);
573
574 if (nbytes < 0) {
575 ret = nbytes;
576 goto done;
577 }
578
579 if (crypto_ahash_walk_last(&rctx->walk)) {
580 rctx->flag |= HASH_DONE;
581 flag = HASH_LAST;
582 }
583 rctx->out = out;
584
585 /* submit */
586 rctx->flag |= HASH_FINAL;
587 sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc);
588 sha1_mb_add_list(rctx, cstate);
589
590 kernel_fpu_begin();
591 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, nbytes, flag);
592 kernel_fpu_end();
593
594 /* check if anything is returned */
595 if (!sha_ctx)
596 return -EINPROGRESS;
597
598 if (sha_ctx->error) {
599 ret = sha_ctx->error;
600 goto done;
601 }
602
603 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
604 ret = sha_finish_walk(&rctx, cstate, false);
605 if (!rctx)
606 return -EINPROGRESS;
607done:
608 sha_complete_job(rctx, cstate, ret);
609 return ret;
610}
611
612static int sha1_mb_final(struct shash_desc *desc, u8 *out)
613{
614 struct mcryptd_hash_request_ctx *rctx =
615 container_of(desc, struct mcryptd_hash_request_ctx, desc);
616 struct mcryptd_alg_cstate *cstate =
617 this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
618
619 struct sha1_hash_ctx *sha_ctx;
620 int ret = 0;
621 u8 data;
622
623 /* sanity check */
624 if (rctx->tag.cpu != smp_processor_id()) {
625 pr_err("mcryptd error: cpu clash\n");
626 goto done;
627 }
628
629 /* need to init context */
630 req_ctx_init(rctx, desc);
631
632 rctx->out = out;
633 rctx->flag |= HASH_DONE | HASH_FINAL;
634
635 sha_ctx = (struct sha1_hash_ctx *) shash_desc_ctx(desc);
636 /* flag HASH_FINAL and 0 data size */
637 sha1_mb_add_list(rctx, cstate);
638 kernel_fpu_begin();
639 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, HASH_LAST);
640 kernel_fpu_end();
641
642 /* check if anything is returned */
643 if (!sha_ctx)
644 return -EINPROGRESS;
645
646 if (sha_ctx->error) {
647 ret = sha_ctx->error;
648 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
649 goto done;
650 }
651
652 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
653 ret = sha_finish_walk(&rctx, cstate, false);
654 if (!rctx)
655 return -EINPROGRESS;
656done:
657 sha_complete_job(rctx, cstate, ret);
658 return ret;
659}
660
661static int sha1_mb_export(struct shash_desc *desc, void *out)
662{
663 struct sha1_hash_ctx *sctx = shash_desc_ctx(desc);
664
665 memcpy(out, sctx, sizeof(*sctx));
666
667 return 0;
668}
669
670static int sha1_mb_import(struct shash_desc *desc, const void *in)
671{
672 struct sha1_hash_ctx *sctx = shash_desc_ctx(desc);
673
674 memcpy(sctx, in, sizeof(*sctx));
675
676 return 0;
677}
678
679
680static struct shash_alg sha1_mb_shash_alg = {
681 .digestsize = SHA1_DIGEST_SIZE,
682 .init = sha1_mb_init,
683 .update = sha1_mb_update,
684 .final = sha1_mb_final,
685 .finup = sha1_mb_finup,
686 .export = sha1_mb_export,
687 .import = sha1_mb_import,
688 .descsize = sizeof(struct sha1_hash_ctx),
689 .statesize = sizeof(struct sha1_hash_ctx),
690 .base = {
691 .cra_name = "__sha1-mb",
692 .cra_driver_name = "__intel_sha1-mb",
693 .cra_priority = 100,
694 /*
695 * use ASYNC flag as some buffers in multi-buffer
696 * algo may not have completed before hashing thread sleep
697 */
698 .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_ASYNC,
699 .cra_blocksize = SHA1_BLOCK_SIZE,
700 .cra_module = THIS_MODULE,
701 .cra_list = LIST_HEAD_INIT(sha1_mb_shash_alg.base.cra_list),
702 }
703};
704
705static int sha1_mb_async_init(struct ahash_request *req)
706{
707 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
708 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
709 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
710 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
711
712 memcpy(mcryptd_req, req, sizeof(*req));
713 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
714 return crypto_ahash_init(mcryptd_req);
715}
716
717static int sha1_mb_async_update(struct ahash_request *req)
718{
719 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
720
721 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
722 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
723 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
724
725 memcpy(mcryptd_req, req, sizeof(*req));
726 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
727 return crypto_ahash_update(mcryptd_req);
728}
729
730static int sha1_mb_async_finup(struct ahash_request *req)
731{
732 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
733
734 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
735 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
736 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
737
738 memcpy(mcryptd_req, req, sizeof(*req));
739 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
740 return crypto_ahash_finup(mcryptd_req);
741}
742
743static int sha1_mb_async_final(struct ahash_request *req)
744{
745 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
746
747 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
748 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
749 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
750
751 memcpy(mcryptd_req, req, sizeof(*req));
752 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
753 return crypto_ahash_final(mcryptd_req);
754}
755
756static int sha1_mb_async_digest(struct ahash_request *req)
757{
758 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
759 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
760 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
761 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
762
763 memcpy(mcryptd_req, req, sizeof(*req));
764 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
765 return crypto_ahash_digest(mcryptd_req);
766}
767
768static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm)
769{
770 struct mcryptd_ahash *mcryptd_tfm;
771 struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
772 struct mcryptd_hash_ctx *mctx;
773
774 mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb", 0, 0);
775 if (IS_ERR(mcryptd_tfm))
776 return PTR_ERR(mcryptd_tfm);
777 mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
778 mctx->alg_state = &sha1_mb_alg_state;
779 ctx->mcryptd_tfm = mcryptd_tfm;
780 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
781 sizeof(struct ahash_request) +
782 crypto_ahash_reqsize(&mcryptd_tfm->base));
783
784 return 0;
785}
786
787static void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm)
788{
789 struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
790
791 mcryptd_free_ahash(ctx->mcryptd_tfm);
792}
793
794static struct ahash_alg sha1_mb_async_alg = {
795 .init = sha1_mb_async_init,
796 .update = sha1_mb_async_update,
797 .final = sha1_mb_async_final,
798 .finup = sha1_mb_async_finup,
799 .digest = sha1_mb_async_digest,
800 .halg = {
801 .digestsize = SHA1_DIGEST_SIZE,
802 .base = {
803 .cra_name = "sha1",
804 .cra_driver_name = "sha1_mb",
805 .cra_priority = 200,
806 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
807 .cra_blocksize = SHA1_BLOCK_SIZE,
808 .cra_type = &crypto_ahash_type,
809 .cra_module = THIS_MODULE,
810 .cra_list = LIST_HEAD_INIT(sha1_mb_async_alg.halg.base.cra_list),
811 .cra_init = sha1_mb_async_init_tfm,
812 .cra_exit = sha1_mb_async_exit_tfm,
813 .cra_ctxsize = sizeof(struct sha1_mb_ctx),
814 .cra_alignmask = 0,
815 },
816 },
817};
818
819static unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate)
820{
821 struct mcryptd_hash_request_ctx *rctx;
822 unsigned long cur_time;
823 unsigned long next_flush = 0;
824 struct sha1_hash_ctx *sha_ctx;
825
826
827 cur_time = jiffies;
828
829 while (!list_empty(&cstate->work_list)) {
830 rctx = list_entry(cstate->work_list.next,
831 struct mcryptd_hash_request_ctx, waiter);
832 if time_before(cur_time, rctx->tag.expire)
833 break;
834 kernel_fpu_begin();
835 sha_ctx = (struct sha1_hash_ctx *) sha1_ctx_mgr_flush(cstate->mgr);
836 kernel_fpu_end();
837 if (!sha_ctx) {
838 pr_err("sha1_mb error: nothing got flushed for non-empty list\n");
839 break;
840 }
841 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
842 sha_finish_walk(&rctx, cstate, true);
843 sha_complete_job(rctx, cstate, 0);
844 }
845
846 if (!list_empty(&cstate->work_list)) {
847 rctx = list_entry(cstate->work_list.next,
848 struct mcryptd_hash_request_ctx, waiter);
849 /* get the hash context and then flush time */
850 next_flush = rctx->tag.expire;
851 mcryptd_arm_flusher(cstate, get_delay(next_flush));
852 }
853 return next_flush;
854}
855
856static int __init sha1_mb_mod_init(void)
857{
858
859 int cpu;
860 int err;
861 struct mcryptd_alg_cstate *cpu_state;
862
863 /* check for dependent cpu features */
864 if (!boot_cpu_has(X86_FEATURE_AVX2) ||
865 !boot_cpu_has(X86_FEATURE_BMI2))
866 return -ENODEV;
867
868 /* initialize multibuffer structures */
869 sha1_mb_alg_state.alg_cstate = alloc_percpu(struct mcryptd_alg_cstate);
870
871 sha1_job_mgr_init = sha1_mb_mgr_init_avx2;
872 sha1_job_mgr_submit = sha1_mb_mgr_submit_avx2;
873 sha1_job_mgr_flush = sha1_mb_mgr_flush_avx2;
874 sha1_job_mgr_get_comp_job = sha1_mb_mgr_get_comp_job_avx2;
875
876 if (!sha1_mb_alg_state.alg_cstate)
877 return -ENOMEM;
878 for_each_possible_cpu(cpu) {
879 cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
880 cpu_state->next_flush = 0;
881 cpu_state->next_seq_num = 0;
882 cpu_state->flusher_engaged = false;
883 INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher);
884 cpu_state->cpu = cpu;
885 cpu_state->alg_state = &sha1_mb_alg_state;
886 cpu_state->mgr = (struct sha1_ctx_mgr *) kzalloc(sizeof(struct sha1_ctx_mgr), GFP_KERNEL);
887 if (!cpu_state->mgr)
888 goto err2;
889 sha1_ctx_mgr_init(cpu_state->mgr);
890 INIT_LIST_HEAD(&cpu_state->work_list);
891 spin_lock_init(&cpu_state->work_lock);
892 }
893 sha1_mb_alg_state.flusher = &sha1_mb_flusher;
894
895 err = crypto_register_shash(&sha1_mb_shash_alg);
896 if (err)
897 goto err2;
898 err = crypto_register_ahash(&sha1_mb_async_alg);
899 if (err)
900 goto err1;
901
902
903 return 0;
904err1:
905 crypto_unregister_shash(&sha1_mb_shash_alg);
906err2:
907 for_each_possible_cpu(cpu) {
908 cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
909 kfree(cpu_state->mgr);
910 }
911 free_percpu(sha1_mb_alg_state.alg_cstate);
912 return -ENODEV;
913}
914
915static void __exit sha1_mb_mod_fini(void)
916{
917 int cpu;
918 struct mcryptd_alg_cstate *cpu_state;
919
920 crypto_unregister_ahash(&sha1_mb_async_alg);
921 crypto_unregister_shash(&sha1_mb_shash_alg);
922 for_each_possible_cpu(cpu) {
923 cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
924 kfree(cpu_state->mgr);
925 }
926 free_percpu(sha1_mb_alg_state.alg_cstate);
927}
928
929module_init(sha1_mb_mod_init);
930module_exit(sha1_mb_mod_fini);
931
932MODULE_LICENSE("GPL");
933MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, multi buffer accelerated");
934
935MODULE_ALIAS("sha1");
diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_datastruct.S b/arch/x86/crypto/sha-mb/sha1_mb_mgr_datastruct.S
new file mode 100644
index 000000000000..86688c6e7a25
--- /dev/null
+++ b/arch/x86/crypto/sha-mb/sha1_mb_mgr_datastruct.S
@@ -0,0 +1,287 @@
1/*
2 * Header file for multi buffer SHA1 algorithm data structure
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * Copyright(c) 2014 Intel Corporation.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * Contact Information:
21 * James Guilford <james.guilford@intel.com>
22 * Tim Chen <tim.c.chen@linux.intel.com>
23 *
24 * BSD LICENSE
25 *
26 * Copyright(c) 2014 Intel Corporation.
27 *
28 * Redistribution and use in source and binary forms, with or without
29 * modification, are permitted provided that the following conditions
30 * are met:
31 *
32 * * Redistributions of source code must retain the above copyright
33 * notice, this list of conditions and the following disclaimer.
34 * * Redistributions in binary form must reproduce the above copyright
35 * notice, this list of conditions and the following disclaimer in
36 * the documentation and/or other materials provided with the
37 * distribution.
38 * * Neither the name of Intel Corporation nor the names of its
39 * contributors may be used to endorse or promote products derived
40 * from this software without specific prior written permission.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
43 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
44 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
45 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
46 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
47 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
48 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
49 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
50 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
51 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
52 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
53 */
54
55# Macros for defining data structures
56
57# Usage example
58
59#START_FIELDS # JOB_AES
60### name size align
61#FIELD _plaintext, 8, 8 # pointer to plaintext
62#FIELD _ciphertext, 8, 8 # pointer to ciphertext
63#FIELD _IV, 16, 8 # IV
64#FIELD _keys, 8, 8 # pointer to keys
65#FIELD _len, 4, 4 # length in bytes
66#FIELD _status, 4, 4 # status enumeration
67#FIELD _user_data, 8, 8 # pointer to user data
68#UNION _union, size1, align1, \
69# size2, align2, \
70# size3, align3, \
71# ...
72#END_FIELDS
73#%assign _JOB_AES_size _FIELD_OFFSET
74#%assign _JOB_AES_align _STRUCT_ALIGN
75
76#########################################################################
77
78# Alternate "struc-like" syntax:
79# STRUCT job_aes2
80# RES_Q .plaintext, 1
81# RES_Q .ciphertext, 1
82# RES_DQ .IV, 1
83# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN
84# RES_U .union, size1, align1, \
85# size2, align2, \
86# ...
87# ENDSTRUCT
88# # Following only needed if nesting
89# %assign job_aes2_size _FIELD_OFFSET
90# %assign job_aes2_align _STRUCT_ALIGN
91#
92# RES_* macros take a name, a count and an optional alignment.
93# The count in in terms of the base size of the macro, and the
94# default alignment is the base size.
95# The macros are:
96# Macro Base size
97# RES_B 1
98# RES_W 2
99# RES_D 4
100# RES_Q 8
101# RES_DQ 16
102# RES_Y 32
103# RES_Z 64
104#
105# RES_U defines a union. It's arguments are a name and two or more
106# pairs of "size, alignment"
107#
108# The two assigns are only needed if this structure is being nested
109# within another. Even if the assigns are not done, one can still use
110# STRUCT_NAME_size as the size of the structure.
111#
112# Note that for nesting, you still need to assign to STRUCT_NAME_size.
113#
114# The differences between this and using "struc" directly are that each
115# type is implicitly aligned to its natural length (although this can be
116# over-ridden with an explicit third parameter), and that the structure
117# is padded at the end to its overall alignment.
118#
119
120#########################################################################
121
122#ifndef _SHA1_MB_MGR_DATASTRUCT_ASM_
123#define _SHA1_MB_MGR_DATASTRUCT_ASM_
124
125## START_FIELDS
126.macro START_FIELDS
127 _FIELD_OFFSET = 0
128 _STRUCT_ALIGN = 0
129.endm
130
131## FIELD name size align
132.macro FIELD name size align
133 _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1))
134 \name = _FIELD_OFFSET
135 _FIELD_OFFSET = _FIELD_OFFSET + (\size)
136.if (\align > _STRUCT_ALIGN)
137 _STRUCT_ALIGN = \align
138.endif
139.endm
140
141## END_FIELDS
142.macro END_FIELDS
143 _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1))
144.endm
145
146########################################################################
147
148.macro STRUCT p1
149START_FIELDS
150.struc \p1
151.endm
152
153.macro ENDSTRUCT
154 tmp = _FIELD_OFFSET
155 END_FIELDS
156 tmp = (_FIELD_OFFSET - %%tmp)
157.if (tmp > 0)
158 .lcomm tmp
159.endif
160.endstruc
161.endm
162
163## RES_int name size align
164.macro RES_int p1 p2 p3
165 name = \p1
166 size = \p2
167 align = .\p3
168
169 _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1))
170.align align
171.lcomm name size
172 _FIELD_OFFSET = _FIELD_OFFSET + (size)
173.if (align > _STRUCT_ALIGN)
174 _STRUCT_ALIGN = align
175.endif
176.endm
177
178
179
180# macro RES_B name, size [, align]
181.macro RES_B _name, _size, _align=1
182RES_int _name _size _align
183.endm
184
185# macro RES_W name, size [, align]
186.macro RES_W _name, _size, _align=2
187RES_int _name 2*(_size) _align
188.endm
189
190# macro RES_D name, size [, align]
191.macro RES_D _name, _size, _align=4
192RES_int _name 4*(_size) _align
193.endm
194
195# macro RES_Q name, size [, align]
196.macro RES_Q _name, _size, _align=8
197RES_int _name 8*(_size) _align
198.endm
199
200# macro RES_DQ name, size [, align]
201.macro RES_DQ _name, _size, _align=16
202RES_int _name 16*(_size) _align
203.endm
204
205# macro RES_Y name, size [, align]
206.macro RES_Y _name, _size, _align=32
207RES_int _name 32*(_size) _align
208.endm
209
210# macro RES_Z name, size [, align]
211.macro RES_Z _name, _size, _align=64
212RES_int _name 64*(_size) _align
213.endm
214
215
216#endif
217
218########################################################################
219#### Define constants
220########################################################################
221
222########################################################################
223#### Define SHA1 Out Of Order Data Structures
224########################################################################
225
226START_FIELDS # LANE_DATA
227### name size align
228FIELD _job_in_lane, 8, 8 # pointer to job object
229END_FIELDS
230
231_LANE_DATA_size = _FIELD_OFFSET
232_LANE_DATA_align = _STRUCT_ALIGN
233
234########################################################################
235
236START_FIELDS # SHA1_ARGS_X8
237### name size align
238FIELD _digest, 4*5*8, 16 # transposed digest
239FIELD _data_ptr, 8*8, 8 # array of pointers to data
240END_FIELDS
241
242_SHA1_ARGS_X4_size = _FIELD_OFFSET
243_SHA1_ARGS_X4_align = _STRUCT_ALIGN
244_SHA1_ARGS_X8_size = _FIELD_OFFSET
245_SHA1_ARGS_X8_align = _STRUCT_ALIGN
246
247########################################################################
248
249START_FIELDS # MB_MGR
250### name size align
251FIELD _args, _SHA1_ARGS_X4_size, _SHA1_ARGS_X4_align
252FIELD _lens, 4*8, 8
253FIELD _unused_lanes, 8, 8
254FIELD _ldata, _LANE_DATA_size*8, _LANE_DATA_align
255END_FIELDS
256
257_MB_MGR_size = _FIELD_OFFSET
258_MB_MGR_align = _STRUCT_ALIGN
259
260_args_digest = _args + _digest
261_args_data_ptr = _args + _data_ptr
262
263
264########################################################################
265#### Define constants
266########################################################################
267
268#define STS_UNKNOWN 0
269#define STS_BEING_PROCESSED 1
270#define STS_COMPLETED 2
271
272########################################################################
273#### Define JOB_SHA1 structure
274########################################################################
275
276START_FIELDS # JOB_SHA1
277
278### name size align
279FIELD _buffer, 8, 8 # pointer to buffer
280FIELD _len, 4, 4 # length in bytes
281FIELD _result_digest, 5*4, 32 # Digest (output)
282FIELD _status, 4, 4
283FIELD _user_data, 8, 8
284END_FIELDS
285
286_JOB_SHA1_size = _FIELD_OFFSET
287_JOB_SHA1_align = _STRUCT_ALIGN
diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S
new file mode 100644
index 000000000000..85c4e1cf7172
--- /dev/null
+++ b/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S
@@ -0,0 +1,327 @@
1/*
2 * Flush routine for SHA1 multibuffer
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * Copyright(c) 2014 Intel Corporation.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * Contact Information:
21 * James Guilford <james.guilford@intel.com>
22 * Tim Chen <tim.c.chen@linux.intel.com>
23 *
24 * BSD LICENSE
25 *
26 * Copyright(c) 2014 Intel Corporation.
27 *
28 * Redistribution and use in source and binary forms, with or without
29 * modification, are permitted provided that the following conditions
30 * are met:
31 *
32 * * Redistributions of source code must retain the above copyright
33 * notice, this list of conditions and the following disclaimer.
34 * * Redistributions in binary form must reproduce the above copyright
35 * notice, this list of conditions and the following disclaimer in
36 * the documentation and/or other materials provided with the
37 * distribution.
38 * * Neither the name of Intel Corporation nor the names of its
39 * contributors may be used to endorse or promote products derived
40 * from this software without specific prior written permission.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
43 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
44 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
45 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
46 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
47 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
48 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
49 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
50 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
51 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
52 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
53 */
54#include <linux/linkage.h>
55#include "sha1_mb_mgr_datastruct.S"
56
57
58.extern sha1_x8_avx2
59
60# LINUX register definitions
61#define arg1 %rdi
62#define arg2 %rsi
63
64# Common definitions
65#define state arg1
66#define job arg2
67#define len2 arg2
68
69# idx must be a register not clobbered by sha1_x8_avx2
70#define idx %r8
71#define DWORD_idx %r8d
72
73#define unused_lanes %rbx
74#define lane_data %rbx
75#define tmp2 %rbx
76#define tmp2_w %ebx
77
78#define job_rax %rax
79#define tmp1 %rax
80#define size_offset %rax
81#define tmp %rax
82#define start_offset %rax
83
84#define tmp3 %arg1
85
86#define extra_blocks %arg2
87#define p %arg2
88
89
90# STACK_SPACE needs to be an odd multiple of 8
91_XMM_SAVE_SIZE = 10*16
92_GPR_SAVE_SIZE = 8*8
93_ALIGN_SIZE = 8
94
95_XMM_SAVE = 0
96_GPR_SAVE = _XMM_SAVE + _XMM_SAVE_SIZE
97STACK_SPACE = _GPR_SAVE + _GPR_SAVE_SIZE + _ALIGN_SIZE
98
99.macro LABEL prefix n
100\prefix\n\():
101.endm
102
103.macro JNE_SKIP i
104jne skip_\i
105.endm
106
107.altmacro
108.macro SET_OFFSET _offset
109offset = \_offset
110.endm
111.noaltmacro
112
113# JOB* sha1_mb_mgr_flush_avx2(MB_MGR *state)
114# arg 1 : rcx : state
115ENTRY(sha1_mb_mgr_flush_avx2)
116 mov %rsp, %r10
117 sub $STACK_SPACE, %rsp
118 and $~31, %rsp
119 mov %rbx, _GPR_SAVE(%rsp)
120 mov %r10, _GPR_SAVE+8*1(%rsp) #save rsp
121 mov %rbp, _GPR_SAVE+8*3(%rsp)
122 mov %r12, _GPR_SAVE+8*4(%rsp)
123 mov %r13, _GPR_SAVE+8*5(%rsp)
124 mov %r14, _GPR_SAVE+8*6(%rsp)
125 mov %r15, _GPR_SAVE+8*7(%rsp)
126
127 # If bit (32+3) is set, then all lanes are empty
128 mov _unused_lanes(state), unused_lanes
129 bt $32+3, unused_lanes
130 jc return_null
131
132 # find a lane with a non-null job
133 xor idx, idx
134 offset = (_ldata + 1 * _LANE_DATA_size + _job_in_lane)
135 cmpq $0, offset(state)
136 cmovne one(%rip), idx
137 offset = (_ldata + 2 * _LANE_DATA_size + _job_in_lane)
138 cmpq $0, offset(state)
139 cmovne two(%rip), idx
140 offset = (_ldata + 3 * _LANE_DATA_size + _job_in_lane)
141 cmpq $0, offset(state)
142 cmovne three(%rip), idx
143 offset = (_ldata + 4 * _LANE_DATA_size + _job_in_lane)
144 cmpq $0, offset(state)
145 cmovne four(%rip), idx
146 offset = (_ldata + 5 * _LANE_DATA_size + _job_in_lane)
147 cmpq $0, offset(state)
148 cmovne five(%rip), idx
149 offset = (_ldata + 6 * _LANE_DATA_size + _job_in_lane)
150 cmpq $0, offset(state)
151 cmovne six(%rip), idx
152 offset = (_ldata + 7 * _LANE_DATA_size + _job_in_lane)
153 cmpq $0, offset(state)
154 cmovne seven(%rip), idx
155
156 # copy idx to empty lanes
157copy_lane_data:
158 offset = (_args + _data_ptr)
159 mov offset(state,idx,8), tmp
160
161 I = 0
162.rep 8
163 offset = (_ldata + I * _LANE_DATA_size + _job_in_lane)
164 cmpq $0, offset(state)
165.altmacro
166 JNE_SKIP %I
167 offset = (_args + _data_ptr + 8*I)
168 mov tmp, offset(state)
169 offset = (_lens + 4*I)
170 movl $0xFFFFFFFF, offset(state)
171LABEL skip_ %I
172 I = (I+1)
173.noaltmacro
174.endr
175
176 # Find min length
177 vmovdqa _lens+0*16(state), %xmm0
178 vmovdqa _lens+1*16(state), %xmm1
179
180 vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A}
181 vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C}
182 vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F}
183 vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E}
184 vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min value in low dword
185
186 vmovd %xmm2, DWORD_idx
187 mov idx, len2
188 and $0xF, idx
189 shr $4, len2
190 jz len_is_0
191
192 vpand clear_low_nibble(%rip), %xmm2, %xmm2
193 vpshufd $0, %xmm2, %xmm2
194
195 vpsubd %xmm2, %xmm0, %xmm0
196 vpsubd %xmm2, %xmm1, %xmm1
197
198 vmovdqa %xmm0, _lens+0*16(state)
199 vmovdqa %xmm1, _lens+1*16(state)
200
201 # "state" and "args" are the same address, arg1
202 # len is arg2
203 call sha1_x8_avx2
204 # state and idx are intact
205
206
207len_is_0:
208 # process completed job "idx"
209 imul $_LANE_DATA_size, idx, lane_data
210 lea _ldata(state, lane_data), lane_data
211
212 mov _job_in_lane(lane_data), job_rax
213 movq $0, _job_in_lane(lane_data)
214 movl $STS_COMPLETED, _status(job_rax)
215 mov _unused_lanes(state), unused_lanes
216 shl $4, unused_lanes
217 or idx, unused_lanes
218 mov unused_lanes, _unused_lanes(state)
219
220 movl $0xFFFFFFFF, _lens(state, idx, 4)
221
222 vmovd _args_digest(state , idx, 4) , %xmm0
223 vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0
224 vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0
225 vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0
226 movl _args_digest+4*32(state, idx, 4), tmp2_w
227
228 vmovdqu %xmm0, _result_digest(job_rax)
229 offset = (_result_digest + 1*16)
230 mov tmp2_w, offset(job_rax)
231
232return:
233
234 mov _GPR_SAVE(%rsp), %rbx
235 mov _GPR_SAVE+8*1(%rsp), %r10 #saved rsp
236 mov _GPR_SAVE+8*3(%rsp), %rbp
237 mov _GPR_SAVE+8*4(%rsp), %r12
238 mov _GPR_SAVE+8*5(%rsp), %r13
239 mov _GPR_SAVE+8*6(%rsp), %r14
240 mov _GPR_SAVE+8*7(%rsp), %r15
241 mov %r10, %rsp
242
243 ret
244
245return_null:
246 xor job_rax, job_rax
247 jmp return
248ENDPROC(sha1_mb_mgr_flush_avx2)
249
250
251#################################################################
252
253.align 16
254ENTRY(sha1_mb_mgr_get_comp_job_avx2)
255 push %rbx
256
257 ## if bit 32+3 is set, then all lanes are empty
258 mov _unused_lanes(state), unused_lanes
259 bt $(32+3), unused_lanes
260 jc .return_null
261
262 # Find min length
263 vmovdqa _lens(state), %xmm0
264 vmovdqa _lens+1*16(state), %xmm1
265
266 vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A}
267 vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C}
268 vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F}
269 vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E}
270 vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min value in low dword
271
272 vmovd %xmm2, DWORD_idx
273 test $~0xF, idx
274 jnz .return_null
275
276 # process completed job "idx"
277 imul $_LANE_DATA_size, idx, lane_data
278 lea _ldata(state, lane_data), lane_data
279
280 mov _job_in_lane(lane_data), job_rax
281 movq $0, _job_in_lane(lane_data)
282 movl $STS_COMPLETED, _status(job_rax)
283 mov _unused_lanes(state), unused_lanes
284 shl $4, unused_lanes
285 or idx, unused_lanes
286 mov unused_lanes, _unused_lanes(state)
287
288 movl $0xFFFFFFFF, _lens(state, idx, 4)
289
290 vmovd _args_digest(state, idx, 4), %xmm0
291 vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0
292 vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0
293 vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0
294 movl _args_digest+4*32(state, idx, 4), tmp2_w
295
296 vmovdqu %xmm0, _result_digest(job_rax)
297 movl tmp2_w, _result_digest+1*16(job_rax)
298
299 pop %rbx
300
301 ret
302
303.return_null:
304 xor job_rax, job_rax
305 pop %rbx
306 ret
307ENDPROC(sha1_mb_mgr_get_comp_job_avx2)
308
309.data
310
311.align 16
312clear_low_nibble:
313.octa 0x000000000000000000000000FFFFFFF0
314one:
315.quad 1
316two:
317.quad 2
318three:
319.quad 3
320four:
321.quad 4
322five:
323.quad 5
324six:
325.quad 6
326seven:
327.quad 7
diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c b/arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c
new file mode 100644
index 000000000000..4ca7e166a2aa
--- /dev/null
+++ b/arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c
@@ -0,0 +1,64 @@
1/*
2 * Initialization code for multi buffer SHA1 algorithm for AVX2
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * Copyright(c) 2014 Intel Corporation.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * Contact Information:
21 * Tim Chen <tim.c.chen@linux.intel.com>
22 *
23 * BSD LICENSE
24 *
25 * Copyright(c) 2014 Intel Corporation.
26 *
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
29 * are met:
30 *
31 * * Redistributions of source code must retain the above copyright
32 * notice, this list of conditions and the following disclaimer.
33 * * Redistributions in binary form must reproduce the above copyright
34 * notice, this list of conditions and the following disclaimer in
35 * the documentation and/or other materials provided with the
36 * distribution.
37 * * Neither the name of Intel Corporation nor the names of its
38 * contributors may be used to endorse or promote products derived
39 * from this software without specific prior written permission.
40 *
41 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
42 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
43 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
44 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
45 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
46 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
47 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
48 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
49 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
50 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
51 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
52 */
53
54#include "sha_mb_mgr.h"
55
56void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state)
57{
58 unsigned int j;
59 state->unused_lanes = 0xF76543210;
60 for (j = 0; j < 8; j++) {
61 state->lens[j] = 0xFFFFFFFF;
62 state->ldata[j].job_in_lane = NULL;
63 }
64}
diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S
new file mode 100644
index 000000000000..2ab9560b53c8
--- /dev/null
+++ b/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S
@@ -0,0 +1,228 @@
1/*
2 * Buffer submit code for multi buffer SHA1 algorithm
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * Copyright(c) 2014 Intel Corporation.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * Contact Information:
21 * James Guilford <james.guilford@intel.com>
22 * Tim Chen <tim.c.chen@linux.intel.com>
23 *
24 * BSD LICENSE
25 *
26 * Copyright(c) 2014 Intel Corporation.
27 *
28 * Redistribution and use in source and binary forms, with or without
29 * modification, are permitted provided that the following conditions
30 * are met:
31 *
32 * * Redistributions of source code must retain the above copyright
33 * notice, this list of conditions and the following disclaimer.
34 * * Redistributions in binary form must reproduce the above copyright
35 * notice, this list of conditions and the following disclaimer in
36 * the documentation and/or other materials provided with the
37 * distribution.
38 * * Neither the name of Intel Corporation nor the names of its
39 * contributors may be used to endorse or promote products derived
40 * from this software without specific prior written permission.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
43 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
44 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
45 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
46 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
47 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
48 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
49 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
50 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
51 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
52 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
53 */
54
55#include <linux/linkage.h>
56#include "sha1_mb_mgr_datastruct.S"
57
58
59.extern sha1_x8_avx
60
61# LINUX register definitions
62arg1 = %rdi
63arg2 = %rsi
64size_offset = %rcx
65tmp2 = %rcx
66extra_blocks = %rdx
67
68# Common definitions
69#define state arg1
70#define job %rsi
71#define len2 arg2
72#define p2 arg2
73
74# idx must be a register not clobberred by sha1_x8_avx2
75idx = %r8
76DWORD_idx = %r8d
77last_len = %r8
78
79p = %r11
80start_offset = %r11
81
82unused_lanes = %rbx
83BYTE_unused_lanes = %bl
84
85job_rax = %rax
86len = %rax
87DWORD_len = %eax
88
89lane = %rbp
90tmp3 = %rbp
91
92tmp = %r9
93DWORD_tmp = %r9d
94
95lane_data = %r10
96
97# STACK_SPACE needs to be an odd multiple of 8
98STACK_SPACE = 8*8 + 16*10 + 8
99
100# JOB* submit_mb_mgr_submit_avx2(MB_MGR *state, job_sha1 *job)
101# arg 1 : rcx : state
102# arg 2 : rdx : job
103ENTRY(sha1_mb_mgr_submit_avx2)
104
105 mov %rsp, %r10
106 sub $STACK_SPACE, %rsp
107 and $~31, %rsp
108
109 mov %rbx, (%rsp)
110 mov %r10, 8*2(%rsp) #save old rsp
111 mov %rbp, 8*3(%rsp)
112 mov %r12, 8*4(%rsp)
113 mov %r13, 8*5(%rsp)
114 mov %r14, 8*6(%rsp)
115 mov %r15, 8*7(%rsp)
116
117 mov _unused_lanes(state), unused_lanes
118 mov unused_lanes, lane
119 and $0xF, lane
120 shr $4, unused_lanes
121 imul $_LANE_DATA_size, lane, lane_data
122 movl $STS_BEING_PROCESSED, _status(job)
123 lea _ldata(state, lane_data), lane_data
124 mov unused_lanes, _unused_lanes(state)
125 movl _len(job), DWORD_len
126
127 mov job, _job_in_lane(lane_data)
128 shl $4, len
129 or lane, len
130
131 movl DWORD_len, _lens(state , lane, 4)
132
133 # Load digest words from result_digest
134 vmovdqu _result_digest(job), %xmm0
135 mov _result_digest+1*16(job), DWORD_tmp
136 vmovd %xmm0, _args_digest(state, lane, 4)
137 vpextrd $1, %xmm0, _args_digest+1*32(state , lane, 4)
138 vpextrd $2, %xmm0, _args_digest+2*32(state , lane, 4)
139 vpextrd $3, %xmm0, _args_digest+3*32(state , lane, 4)
140 movl DWORD_tmp, _args_digest+4*32(state , lane, 4)
141
142 mov _buffer(job), p
143 mov p, _args_data_ptr(state, lane, 8)
144
145 cmp $0xF, unused_lanes
146 jne return_null
147
148start_loop:
149 # Find min length
150 vmovdqa _lens(state), %xmm0
151 vmovdqa _lens+1*16(state), %xmm1
152
153 vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A}
154 vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C}
155 vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F}
156 vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E}
157 vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min value in low dword
158
159 vmovd %xmm2, DWORD_idx
160 mov idx, len2
161 and $0xF, idx
162 shr $4, len2
163 jz len_is_0
164
165 vpand clear_low_nibble(%rip), %xmm2, %xmm2
166 vpshufd $0, %xmm2, %xmm2
167
168 vpsubd %xmm2, %xmm0, %xmm0
169 vpsubd %xmm2, %xmm1, %xmm1
170
171 vmovdqa %xmm0, _lens + 0*16(state)
172 vmovdqa %xmm1, _lens + 1*16(state)
173
174
175 # "state" and "args" are the same address, arg1
176 # len is arg2
177 call sha1_x8_avx2
178
179 # state and idx are intact
180
181len_is_0:
182 # process completed job "idx"
183 imul $_LANE_DATA_size, idx, lane_data
184 lea _ldata(state, lane_data), lane_data
185
186 mov _job_in_lane(lane_data), job_rax
187 mov _unused_lanes(state), unused_lanes
188 movq $0, _job_in_lane(lane_data)
189 movl $STS_COMPLETED, _status(job_rax)
190 shl $4, unused_lanes
191 or idx, unused_lanes
192 mov unused_lanes, _unused_lanes(state)
193
194 movl $0xFFFFFFFF, _lens(state, idx, 4)
195
196 vmovd _args_digest(state, idx, 4), %xmm0
197 vpinsrd $1, _args_digest+1*32(state , idx, 4), %xmm0, %xmm0
198 vpinsrd $2, _args_digest+2*32(state , idx, 4), %xmm0, %xmm0
199 vpinsrd $3, _args_digest+3*32(state , idx, 4), %xmm0, %xmm0
200 movl 4*32(state, idx, 4), DWORD_tmp
201
202 vmovdqu %xmm0, _result_digest(job_rax)
203 movl DWORD_tmp, _result_digest+1*16(job_rax)
204
205return:
206
207 mov (%rsp), %rbx
208 mov 8*2(%rsp), %r10 #save old rsp
209 mov 8*3(%rsp), %rbp
210 mov 8*4(%rsp), %r12
211 mov 8*5(%rsp), %r13
212 mov 8*6(%rsp), %r14
213 mov 8*7(%rsp), %r15
214 mov %r10, %rsp
215
216 ret
217
218return_null:
219 xor job_rax, job_rax
220 jmp return
221
222ENDPROC(sha1_mb_mgr_submit_avx2)
223
224.data
225
226.align 16
227clear_low_nibble:
228 .octa 0x000000000000000000000000FFFFFFF0
diff --git a/arch/x86/crypto/sha-mb/sha1_x8_avx2.S b/arch/x86/crypto/sha-mb/sha1_x8_avx2.S
new file mode 100644
index 000000000000..8e1b47792b31
--- /dev/null
+++ b/arch/x86/crypto/sha-mb/sha1_x8_avx2.S
@@ -0,0 +1,472 @@
1/*
2 * Multi-buffer SHA1 algorithm hash compute routine
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * Copyright(c) 2014 Intel Corporation.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * Contact Information:
21 * James Guilford <james.guilford@intel.com>
22 * Tim Chen <tim.c.chen@linux.intel.com>
23 *
24 * BSD LICENSE
25 *
26 * Copyright(c) 2014 Intel Corporation.
27 *
28 * Redistribution and use in source and binary forms, with or without
29 * modification, are permitted provided that the following conditions
30 * are met:
31 *
32 * * Redistributions of source code must retain the above copyright
33 * notice, this list of conditions and the following disclaimer.
34 * * Redistributions in binary form must reproduce the above copyright
35 * notice, this list of conditions and the following disclaimer in
36 * the documentation and/or other materials provided with the
37 * distribution.
38 * * Neither the name of Intel Corporation nor the names of its
39 * contributors may be used to endorse or promote products derived
40 * from this software without specific prior written permission.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
43 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
44 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
45 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
46 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
47 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
48 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
49 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
50 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
51 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
52 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
53 */
54
55#include <linux/linkage.h>
56#include "sha1_mb_mgr_datastruct.S"
57
58## code to compute oct SHA1 using SSE-256
59## outer calling routine takes care of save and restore of XMM registers
60
61## Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15# ymm0-15
62##
63## Linux clobbers: rax rbx rcx rdx rsi r9 r10 r11 r12 r13 r14 r15
64## Linux preserves: rdi rbp r8
65##
66## clobbers ymm0-15
67
68
69# TRANSPOSE8 r0, r1, r2, r3, r4, r5, r6, r7, t0, t1
70# "transpose" data in {r0...r7} using temps {t0...t1}
71# Input looks like: {r0 r1 r2 r3 r4 r5 r6 r7}
72# r0 = {a7 a6 a5 a4 a3 a2 a1 a0}
73# r1 = {b7 b6 b5 b4 b3 b2 b1 b0}
74# r2 = {c7 c6 c5 c4 c3 c2 c1 c0}
75# r3 = {d7 d6 d5 d4 d3 d2 d1 d0}
76# r4 = {e7 e6 e5 e4 e3 e2 e1 e0}
77# r5 = {f7 f6 f5 f4 f3 f2 f1 f0}
78# r6 = {g7 g6 g5 g4 g3 g2 g1 g0}
79# r7 = {h7 h6 h5 h4 h3 h2 h1 h0}
80#
81# Output looks like: {r0 r1 r2 r3 r4 r5 r6 r7}
82# r0 = {h0 g0 f0 e0 d0 c0 b0 a0}
83# r1 = {h1 g1 f1 e1 d1 c1 b1 a1}
84# r2 = {h2 g2 f2 e2 d2 c2 b2 a2}
85# r3 = {h3 g3 f3 e3 d3 c3 b3 a3}
86# r4 = {h4 g4 f4 e4 d4 c4 b4 a4}
87# r5 = {h5 g5 f5 e5 d5 c5 b5 a5}
88# r6 = {h6 g6 f6 e6 d6 c6 b6 a6}
89# r7 = {h7 g7 f7 e7 d7 c7 b7 a7}
90#
91
92.macro TRANSPOSE8 r0 r1 r2 r3 r4 r5 r6 r7 t0 t1
93 # process top half (r0..r3) {a...d}
94 vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4 b1 b0 a1 a0}
95 vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6 b3 b2 a3 a2}
96 vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4 d1 d0 c1 c0}
97 vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6 d3 d2 c3 c2}
98 vshufps $0xDD, \t1, \t0, \r3 # r3 = {d5 c5 b5 a5 d1 c1 b1 a1}
99 vshufps $0x88, \r2, \r0, \r1 # r1 = {d6 c6 b6 a6 d2 c2 b2 a2}
100 vshufps $0xDD, \r2, \r0, \r0 # r0 = {d7 c7 b7 a7 d3 c3 b3 a3}
101 vshufps $0x88, \t1, \t0, \t0 # t0 = {d4 c4 b4 a4 d0 c0 b0 a0}
102
103 # use r2 in place of t0
104 # process bottom half (r4..r7) {e...h}
105 vshufps $0x44, \r5, \r4, \r2 # r2 = {f5 f4 e5 e4 f1 f0 e1 e0}
106 vshufps $0xEE, \r5, \r4, \r4 # r4 = {f7 f6 e7 e6 f3 f2 e3 e2}
107 vshufps $0x44, \r7, \r6, \t1 # t1 = {h5 h4 g5 g4 h1 h0 g1 g0}
108 vshufps $0xEE, \r7, \r6, \r6 # r6 = {h7 h6 g7 g6 h3 h2 g3 g2}
109 vshufps $0xDD, \t1, \r2, \r7 # r7 = {h5 g5 f5 e5 h1 g1 f1 e1}
110 vshufps $0x88, \r6, \r4, \r5 # r5 = {h6 g6 f6 e6 h2 g2 f2 e2}
111 vshufps $0xDD, \r6, \r4, \r4 # r4 = {h7 g7 f7 e7 h3 g3 f3 e3}
112 vshufps $0x88, \t1, \r2, \t1 # t1 = {h4 g4 f4 e4 h0 g0 f0 e0}
113
114 vperm2f128 $0x13, \r1, \r5, \r6 # h6...a6
115 vperm2f128 $0x02, \r1, \r5, \r2 # h2...a2
116 vperm2f128 $0x13, \r3, \r7, \r5 # h5...a5
117 vperm2f128 $0x02, \r3, \r7, \r1 # h1...a1
118 vperm2f128 $0x13, \r0, \r4, \r7 # h7...a7
119 vperm2f128 $0x02, \r0, \r4, \r3 # h3...a3
120 vperm2f128 $0x13, \t0, \t1, \r4 # h4...a4
121 vperm2f128 $0x02, \t0, \t1, \r0 # h0...a0
122
123.endm
124##
125## Magic functions defined in FIPS 180-1
126##
127# macro MAGIC_F0 F,B,C,D,T ## F = (D ^ (B & (C ^ D)))
128.macro MAGIC_F0 regF regB regC regD regT
129 vpxor \regD, \regC, \regF
130 vpand \regB, \regF, \regF
131 vpxor \regD, \regF, \regF
132.endm
133
134# macro MAGIC_F1 F,B,C,D,T ## F = (B ^ C ^ D)
135.macro MAGIC_F1 regF regB regC regD regT
136 vpxor \regC, \regD, \regF
137 vpxor \regB, \regF, \regF
138.endm
139
140# macro MAGIC_F2 F,B,C,D,T ## F = ((B & C) | (B & D) | (C & D))
141.macro MAGIC_F2 regF regB regC regD regT
142 vpor \regC, \regB, \regF
143 vpand \regC, \regB, \regT
144 vpand \regD, \regF, \regF
145 vpor \regT, \regF, \regF
146.endm
147
148# macro MAGIC_F3 F,B,C,D,T ## F = (B ^ C ^ D)
149.macro MAGIC_F3 regF regB regC regD regT
150 MAGIC_F1 \regF,\regB,\regC,\regD,\regT
151.endm
152
153# PROLD reg, imm, tmp
154.macro PROLD reg imm tmp
155 vpsrld $(32-\imm), \reg, \tmp
156 vpslld $\imm, \reg, \reg
157 vpor \tmp, \reg, \reg
158.endm
159
160.macro PROLD_nd reg imm tmp src
161 vpsrld $(32-\imm), \src, \tmp
162 vpslld $\imm, \src, \reg
163 vpor \tmp, \reg, \reg
164.endm
165
166.macro SHA1_STEP_00_15 regA regB regC regD regE regT regF memW immCNT MAGIC
167 vpaddd \immCNT, \regE, \regE
168 vpaddd \memW*32(%rsp), \regE, \regE
169 PROLD_nd \regT, 5, \regF, \regA
170 vpaddd \regT, \regE, \regE
171 \MAGIC \regF, \regB, \regC, \regD, \regT
172 PROLD \regB, 30, \regT
173 vpaddd \regF, \regE, \regE
174.endm
175
176.macro SHA1_STEP_16_79 regA regB regC regD regE regT regF memW immCNT MAGIC
177 vpaddd \immCNT, \regE, \regE
178 offset = ((\memW - 14) & 15) * 32
179 vmovdqu offset(%rsp), W14
180 vpxor W14, W16, W16
181 offset = ((\memW - 8) & 15) * 32
182 vpxor offset(%rsp), W16, W16
183 offset = ((\memW - 3) & 15) * 32
184 vpxor offset(%rsp), W16, W16
185 vpsrld $(32-1), W16, \regF
186 vpslld $1, W16, W16
187 vpor W16, \regF, \regF
188
189 ROTATE_W
190
191 offset = ((\memW - 0) & 15) * 32
192 vmovdqu \regF, offset(%rsp)
193 vpaddd \regF, \regE, \regE
194 PROLD_nd \regT, 5, \regF, \regA
195 vpaddd \regT, \regE, \regE
196 \MAGIC \regF,\regB,\regC,\regD,\regT ## FUN = MAGIC_Fi(B,C,D)
197 PROLD \regB,30, \regT
198 vpaddd \regF, \regE, \regE
199.endm
200
201########################################################################
202########################################################################
203########################################################################
204
205## FRAMESZ plus pushes must be an odd multiple of 8
206YMM_SAVE = (15-15)*32
207FRAMESZ = 32*16 + YMM_SAVE
208_YMM = FRAMESZ - YMM_SAVE
209
210#define VMOVPS vmovups
211
212IDX = %rax
213inp0 = %r9
214inp1 = %r10
215inp2 = %r11
216inp3 = %r12
217inp4 = %r13
218inp5 = %r14
219inp6 = %r15
220inp7 = %rcx
221arg1 = %rdi
222arg2 = %rsi
223RSP_SAVE = %rdx
224
225# ymm0 A
226# ymm1 B
227# ymm2 C
228# ymm3 D
229# ymm4 E
230# ymm5 F AA
231# ymm6 T0 BB
232# ymm7 T1 CC
233# ymm8 T2 DD
234# ymm9 T3 EE
235# ymm10 T4 TMP
236# ymm11 T5 FUN
237# ymm12 T6 K
238# ymm13 T7 W14
239# ymm14 T8 W15
240# ymm15 T9 W16
241
242
243A = %ymm0
244B = %ymm1
245C = %ymm2
246D = %ymm3
247E = %ymm4
248F = %ymm5
249T0 = %ymm6
250T1 = %ymm7
251T2 = %ymm8
252T3 = %ymm9
253T4 = %ymm10
254T5 = %ymm11
255T6 = %ymm12
256T7 = %ymm13
257T8 = %ymm14
258T9 = %ymm15
259
260AA = %ymm5
261BB = %ymm6
262CC = %ymm7
263DD = %ymm8
264EE = %ymm9
265TMP = %ymm10
266FUN = %ymm11
267K = %ymm12
268W14 = %ymm13
269W15 = %ymm14
270W16 = %ymm15
271
272.macro ROTATE_ARGS
273 TMP_ = E
274 E = D
275 D = C
276 C = B
277 B = A
278 A = TMP_
279.endm
280
281.macro ROTATE_W
282TMP_ = W16
283W16 = W15
284W15 = W14
285W14 = TMP_
286.endm
287
288# 8 streams x 5 32bit words per digest x 4 bytes per word
289#define DIGEST_SIZE (8*5*4)
290
291.align 32
292
293# void sha1_x8_avx2(void **input_data, UINT128 *digest, UINT32 size)
294# arg 1 : pointer to array[4] of pointer to input data
295# arg 2 : size (in blocks) ;; assumed to be >= 1
296#
297ENTRY(sha1_x8_avx2)
298
299 push RSP_SAVE
300
301 #save rsp
302 mov %rsp, RSP_SAVE
303 sub $FRAMESZ, %rsp
304
305 #align rsp to 32 Bytes
306 and $~0x1F, %rsp
307
308 ## Initialize digests
309 vmovdqu 0*32(arg1), A
310 vmovdqu 1*32(arg1), B
311 vmovdqu 2*32(arg1), C
312 vmovdqu 3*32(arg1), D
313 vmovdqu 4*32(arg1), E
314
315 ## transpose input onto stack
316 mov _data_ptr+0*8(arg1),inp0
317 mov _data_ptr+1*8(arg1),inp1
318 mov _data_ptr+2*8(arg1),inp2
319 mov _data_ptr+3*8(arg1),inp3
320 mov _data_ptr+4*8(arg1),inp4
321 mov _data_ptr+5*8(arg1),inp5
322 mov _data_ptr+6*8(arg1),inp6
323 mov _data_ptr+7*8(arg1),inp7
324
325 xor IDX, IDX
326lloop:
327 vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), F
328 I=0
329.rep 2
330 VMOVPS (inp0, IDX), T0
331 VMOVPS (inp1, IDX), T1
332 VMOVPS (inp2, IDX), T2
333 VMOVPS (inp3, IDX), T3
334 VMOVPS (inp4, IDX), T4
335 VMOVPS (inp5, IDX), T5
336 VMOVPS (inp6, IDX), T6
337 VMOVPS (inp7, IDX), T7
338
339 TRANSPOSE8 T0, T1, T2, T3, T4, T5, T6, T7, T8, T9
340 vpshufb F, T0, T0
341 vmovdqu T0, (I*8)*32(%rsp)
342 vpshufb F, T1, T1
343 vmovdqu T1, (I*8+1)*32(%rsp)
344 vpshufb F, T2, T2
345 vmovdqu T2, (I*8+2)*32(%rsp)
346 vpshufb F, T3, T3
347 vmovdqu T3, (I*8+3)*32(%rsp)
348 vpshufb F, T4, T4
349 vmovdqu T4, (I*8+4)*32(%rsp)
350 vpshufb F, T5, T5
351 vmovdqu T5, (I*8+5)*32(%rsp)
352 vpshufb F, T6, T6
353 vmovdqu T6, (I*8+6)*32(%rsp)
354 vpshufb F, T7, T7
355 vmovdqu T7, (I*8+7)*32(%rsp)
356 add $32, IDX
357 I = (I+1)
358.endr
359 # save old digests
360 vmovdqu A,AA
361 vmovdqu B,BB
362 vmovdqu C,CC
363 vmovdqu D,DD
364 vmovdqu E,EE
365
366##
367## perform 0-79 steps
368##
369 vmovdqu K00_19(%rip), K
370## do rounds 0...15
371 I = 0
372.rep 16
373 SHA1_STEP_00_15 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F0
374 ROTATE_ARGS
375 I = (I+1)
376.endr
377
378## do rounds 16...19
379 vmovdqu ((16 - 16) & 15) * 32 (%rsp), W16
380 vmovdqu ((16 - 15) & 15) * 32 (%rsp), W15
381.rep 4
382 SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F0
383 ROTATE_ARGS
384 I = (I+1)
385.endr
386
387## do rounds 20...39
388 vmovdqu K20_39(%rip), K
389.rep 20
390 SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F1
391 ROTATE_ARGS
392 I = (I+1)
393.endr
394
395## do rounds 40...59
396 vmovdqu K40_59(%rip), K
397.rep 20
398 SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F2
399 ROTATE_ARGS
400 I = (I+1)
401.endr
402
403## do rounds 60...79
404 vmovdqu K60_79(%rip), K
405.rep 20
406 SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F3
407 ROTATE_ARGS
408 I = (I+1)
409.endr
410
411 vpaddd AA,A,A
412 vpaddd BB,B,B
413 vpaddd CC,C,C
414 vpaddd DD,D,D
415 vpaddd EE,E,E
416
417 sub $1, arg2
418 jne lloop
419
420 # write out digests
421 vmovdqu A, 0*32(arg1)
422 vmovdqu B, 1*32(arg1)
423 vmovdqu C, 2*32(arg1)
424 vmovdqu D, 3*32(arg1)
425 vmovdqu E, 4*32(arg1)
426
427 # update input pointers
428 add IDX, inp0
429 add IDX, inp1
430 add IDX, inp2
431 add IDX, inp3
432 add IDX, inp4
433 add IDX, inp5
434 add IDX, inp6
435 add IDX, inp7
436 mov inp0, _data_ptr (arg1)
437 mov inp1, _data_ptr + 1*8(arg1)
438 mov inp2, _data_ptr + 2*8(arg1)
439 mov inp3, _data_ptr + 3*8(arg1)
440 mov inp4, _data_ptr + 4*8(arg1)
441 mov inp5, _data_ptr + 5*8(arg1)
442 mov inp6, _data_ptr + 6*8(arg1)
443 mov inp7, _data_ptr + 7*8(arg1)
444
445 ################
446 ## Postamble
447
448 mov RSP_SAVE, %rsp
449 pop RSP_SAVE
450
451 ret
452ENDPROC(sha1_x8_avx2)
453
454
455.data
456
457.align 32
458K00_19:
459.octa 0x5A8279995A8279995A8279995A827999
460.octa 0x5A8279995A8279995A8279995A827999
461K20_39:
462.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1
463.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1
464K40_59:
465.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC
466.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC
467K60_79:
468.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6
469.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6
470PSHUFFLE_BYTE_FLIP_MASK:
471.octa 0x0c0d0e0f08090a0b0405060700010203
472.octa 0x0c0d0e0f08090a0b0405060700010203
diff --git a/arch/x86/crypto/sha-mb/sha_mb_ctx.h b/arch/x86/crypto/sha-mb/sha_mb_ctx.h
new file mode 100644
index 000000000000..e36069d0c1bd
--- /dev/null
+++ b/arch/x86/crypto/sha-mb/sha_mb_ctx.h
@@ -0,0 +1,136 @@
1/*
2 * Header file for multi buffer SHA context
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * Copyright(c) 2014 Intel Corporation.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * Contact Information:
21 * Tim Chen <tim.c.chen@linux.intel.com>
22 *
23 * BSD LICENSE
24 *
25 * Copyright(c) 2014 Intel Corporation.
26 *
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
29 * are met:
30 *
31 * * Redistributions of source code must retain the above copyright
32 * notice, this list of conditions and the following disclaimer.
33 * * Redistributions in binary form must reproduce the above copyright
34 * notice, this list of conditions and the following disclaimer in
35 * the documentation and/or other materials provided with the
36 * distribution.
37 * * Neither the name of Intel Corporation nor the names of its
38 * contributors may be used to endorse or promote products derived
39 * from this software without specific prior written permission.
40 *
41 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
42 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
43 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
44 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
45 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
46 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
47 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
48 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
49 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
50 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
51 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
52 */
53
54#ifndef _SHA_MB_CTX_INTERNAL_H
55#define _SHA_MB_CTX_INTERNAL_H
56
57#include "sha_mb_mgr.h"
58
59#define HASH_UPDATE 0x00
60#define HASH_FIRST 0x01
61#define HASH_LAST 0x02
62#define HASH_ENTIRE 0x03
63#define HASH_DONE 0x04
64#define HASH_FINAL 0x08
65
66#define HASH_CTX_STS_IDLE 0x00
67#define HASH_CTX_STS_PROCESSING 0x01
68#define HASH_CTX_STS_LAST 0x02
69#define HASH_CTX_STS_COMPLETE 0x04
70
71enum hash_ctx_error {
72 HASH_CTX_ERROR_NONE = 0,
73 HASH_CTX_ERROR_INVALID_FLAGS = -1,
74 HASH_CTX_ERROR_ALREADY_PROCESSING = -2,
75 HASH_CTX_ERROR_ALREADY_COMPLETED = -3,
76
77#ifdef HASH_CTX_DEBUG
78 HASH_CTX_ERROR_DEBUG_DIGEST_MISMATCH = -4,
79#endif
80};
81
82
83#define hash_ctx_user_data(ctx) ((ctx)->user_data)
84#define hash_ctx_digest(ctx) ((ctx)->job.result_digest)
85#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING)
86#define hash_ctx_complete(ctx) ((ctx)->status == HASH_CTX_STS_COMPLETE)
87#define hash_ctx_status(ctx) ((ctx)->status)
88#define hash_ctx_error(ctx) ((ctx)->error)
89#define hash_ctx_init(ctx) \
90 do { \
91 (ctx)->error = HASH_CTX_ERROR_NONE; \
92 (ctx)->status = HASH_CTX_STS_COMPLETE; \
93 } while (0)
94
95
96/* Hash Constants and Typedefs */
97#define SHA1_DIGEST_LENGTH 5
98#define SHA1_LOG2_BLOCK_SIZE 6
99
100#define SHA1_PADLENGTHFIELD_SIZE 8
101
102#ifdef SHA_MB_DEBUG
103#define assert(expr) \
104do { \
105 if (unlikely(!(expr))) { \
106 printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \
107 #expr, __FILE__, __func__, __LINE__); \
108 } \
109} while (0)
110#else
111#define assert(expr) do {} while (0)
112#endif
113
114struct sha1_ctx_mgr {
115 struct sha1_mb_mgr mgr;
116};
117
118/* typedef struct sha1_ctx_mgr sha1_ctx_mgr; */
119
120struct sha1_hash_ctx {
121 /* Must be at struct offset 0 */
122 struct job_sha1 job;
123 /* status flag */
124 int status;
125 /* error flag */
126 int error;
127
128 uint32_t total_length;
129 const void *incoming_buffer;
130 uint32_t incoming_buffer_length;
131 uint8_t partial_block_buffer[SHA1_BLOCK_SIZE * 2];
132 uint32_t partial_block_buffer_length;
133 void *user_data;
134};
135
136#endif
diff --git a/arch/x86/crypto/sha-mb/sha_mb_mgr.h b/arch/x86/crypto/sha-mb/sha_mb_mgr.h
new file mode 100644
index 000000000000..08ad1a9acfd7
--- /dev/null
+++ b/arch/x86/crypto/sha-mb/sha_mb_mgr.h
@@ -0,0 +1,110 @@
1/*
2 * Header file for multi buffer SHA1 algorithm manager
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * Copyright(c) 2014 Intel Corporation.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * Contact Information:
21 * James Guilford <james.guilford@intel.com>
22 * Tim Chen <tim.c.chen@linux.intel.com>
23 *
24 * BSD LICENSE
25 *
26 * Copyright(c) 2014 Intel Corporation.
27 *
28 * Redistribution and use in source and binary forms, with or without
29 * modification, are permitted provided that the following conditions
30 * are met:
31 *
32 * * Redistributions of source code must retain the above copyright
33 * notice, this list of conditions and the following disclaimer.
34 * * Redistributions in binary form must reproduce the above copyright
35 * notice, this list of conditions and the following disclaimer in
36 * the documentation and/or other materials provided with the
37 * distribution.
38 * * Neither the name of Intel Corporation nor the names of its
39 * contributors may be used to endorse or promote products derived
40 * from this software without specific prior written permission.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
43 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
44 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
45 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
46 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
47 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
48 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
49 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
50 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
51 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
52 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
53 */
54#ifndef __SHA_MB_MGR_H
55#define __SHA_MB_MGR_H
56
57
58#include <linux/types.h>
59
60#define NUM_SHA1_DIGEST_WORDS 5
61
62enum job_sts { STS_UNKNOWN = 0,
63 STS_BEING_PROCESSED = 1,
64 STS_COMPLETED = 2,
65 STS_INTERNAL_ERROR = 3,
66 STS_ERROR = 4
67};
68
69struct job_sha1 {
70 u8 *buffer;
71 u32 len;
72 u32 result_digest[NUM_SHA1_DIGEST_WORDS] __aligned(32);
73 enum job_sts status;
74 void *user_data;
75};
76
77/* SHA1 out-of-order scheduler */
78
79/* typedef uint32_t sha1_digest_array[5][8]; */
80
81struct sha1_args_x8 {
82 uint32_t digest[5][8];
83 uint8_t *data_ptr[8];
84};
85
86struct sha1_lane_data {
87 struct job_sha1 *job_in_lane;
88};
89
90struct sha1_mb_mgr {
91 struct sha1_args_x8 args;
92
93 uint32_t lens[8];
94
95 /* each byte is index (0...7) of unused lanes */
96 uint64_t unused_lanes;
97 /* byte 4 is set to FF as a flag */
98 struct sha1_lane_data ldata[8];
99};
100
101
102#define SHA1_MB_MGR_NUM_LANES_AVX2 8
103
104void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state);
105struct job_sha1 *sha1_mb_mgr_submit_avx2(struct sha1_mb_mgr *state,
106 struct job_sha1 *job);
107struct job_sha1 *sha1_mb_mgr_flush_avx2(struct sha1_mb_mgr *state);
108struct job_sha1 *sha1_mb_mgr_get_comp_job_avx2(struct sha1_mb_mgr *state);
109
110#endif
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 77daef031db5..87bbc9c1e681 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -158,6 +158,20 @@ config CRYPTO_CRYPTD
158 converts an arbitrary synchronous software crypto algorithm 158 converts an arbitrary synchronous software crypto algorithm
159 into an asynchronous algorithm that executes in a kernel thread. 159 into an asynchronous algorithm that executes in a kernel thread.
160 160
161config CRYPTO_MCRYPTD
162 tristate "Software async multi-buffer crypto daemon"
163 select CRYPTO_BLKCIPHER
164 select CRYPTO_HASH
165 select CRYPTO_MANAGER
166 select CRYPTO_WORKQUEUE
167 help
168 This is a generic software asynchronous crypto daemon that
169 provides the kernel thread to assist multi-buffer crypto
170 algorithms for submitting jobs and flushing jobs in multi-buffer
171 crypto algorithms. Multi-buffer crypto algorithms are executed
172 in the context of this kernel thread and drivers can post
173 their crypto request asynchronously to be processed by this daemon.
174
161config CRYPTO_AUTHENC 175config CRYPTO_AUTHENC
162 tristate "Authenc support" 176 tristate "Authenc support"
163 select CRYPTO_AEAD 177 select CRYPTO_AEAD
@@ -559,6 +573,22 @@ config CRYPTO_SHA1_PPC
559 This is the powerpc hardware accelerated implementation of the 573 This is the powerpc hardware accelerated implementation of the
560 SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). 574 SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2).
561 575
576config CRYPTO_SHA1_MB
577 tristate "SHA1 digest algorithm (x86_64 Multi-Buffer, Experimental)"
578 depends on X86 && 64BIT
579 select CRYPTO_SHA1
580 select CRYPTO_HASH
581 select CRYPTO_MCRYPTD
582 help
583 SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
584 using multi-buffer technique. This algorithm computes on
585 multiple data lanes concurrently with SIMD instructions for
586 better throughput. It should not be enabled by default but
587 used when there is significant amount of work to keep the keep
588 the data lanes filled to get performance benefit. If the data
589 lanes remain unfilled, a flush operation will be initiated to
590 process the crypto jobs, adding a slight latency.
591
562config CRYPTO_SHA256 592config CRYPTO_SHA256
563 tristate "SHA224 and SHA256 digest algorithm" 593 tristate "SHA224 and SHA256 digest algorithm"
564 select CRYPTO_HASH 594 select CRYPTO_HASH
diff --git a/crypto/Makefile b/crypto/Makefile
index cfa57b3f5a4d..1445b9100c05 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -60,6 +60,7 @@ obj-$(CONFIG_CRYPTO_GCM) += gcm.o
60obj-$(CONFIG_CRYPTO_CCM) += ccm.o 60obj-$(CONFIG_CRYPTO_CCM) += ccm.o
61obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o 61obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o
62obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o 62obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o
63obj-$(CONFIG_CRYPTO_MCRYPTD) += mcryptd.o
63obj-$(CONFIG_CRYPTO_DES) += des_generic.o 64obj-$(CONFIG_CRYPTO_DES) += des_generic.o
64obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o 65obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o
65obj-$(CONFIG_CRYPTO_BLOWFISH) += blowfish_generic.o 66obj-$(CONFIG_CRYPTO_BLOWFISH) += blowfish_generic.o
diff --git a/crypto/ahash.c b/crypto/ahash.c
index f2a5d8f656ff..f6a36a52d738 100644
--- a/crypto/ahash.c
+++ b/crypto/ahash.c
@@ -131,8 +131,10 @@ int crypto_hash_walk_first(struct ahash_request *req,
131{ 131{
132 walk->total = req->nbytes; 132 walk->total = req->nbytes;
133 133
134 if (!walk->total) 134 if (!walk->total) {
135 walk->entrylen = 0;
135 return 0; 136 return 0;
137 }
136 138
137 walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req)); 139 walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req));
138 walk->sg = req->src; 140 walk->sg = req->src;
@@ -147,8 +149,10 @@ int crypto_ahash_walk_first(struct ahash_request *req,
147{ 149{
148 walk->total = req->nbytes; 150 walk->total = req->nbytes;
149 151
150 if (!walk->total) 152 if (!walk->total) {
153 walk->entrylen = 0;
151 return 0; 154 return 0;
155 }
152 156
153 walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req)); 157 walk->alignmask = crypto_ahash_alignmask(crypto_ahash_reqtfm(req));
154 walk->sg = req->src; 158 walk->sg = req->src;
@@ -167,8 +171,10 @@ int crypto_hash_walk_first_compat(struct hash_desc *hdesc,
167{ 171{
168 walk->total = len; 172 walk->total = len;
169 173
170 if (!walk->total) 174 if (!walk->total) {
175 walk->entrylen = 0;
171 return 0; 176 return 0;
177 }
172 178
173 walk->alignmask = crypto_hash_alignmask(hdesc->tfm); 179 walk->alignmask = crypto_hash_alignmask(hdesc->tfm);
174 walk->sg = sg; 180 walk->sg = sg;
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
index a19c027b29bd..83187f497c7c 100644
--- a/crypto/algif_skcipher.c
+++ b/crypto/algif_skcipher.c
@@ -49,7 +49,7 @@ struct skcipher_ctx {
49 struct ablkcipher_request req; 49 struct ablkcipher_request req;
50}; 50};
51 51
52#define MAX_SGL_ENTS ((PAGE_SIZE - sizeof(struct skcipher_sg_list)) / \ 52#define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_sg_list)) / \
53 sizeof(struct scatterlist) - 1) 53 sizeof(struct scatterlist) - 1)
54 54
55static inline int skcipher_sndbuf(struct sock *sk) 55static inline int skcipher_sndbuf(struct sock *sk)
diff --git a/crypto/drbg.c b/crypto/drbg.c
index a53ee099e281..54cfd4820abc 100644
--- a/crypto/drbg.c
+++ b/crypto/drbg.c
@@ -117,27 +117,18 @@ static const struct drbg_core drbg_cores[] = {
117 { 117 {
118 .flags = DRBG_CTR | DRBG_STRENGTH128, 118 .flags = DRBG_CTR | DRBG_STRENGTH128,
119 .statelen = 32, /* 256 bits as defined in 10.2.1 */ 119 .statelen = 32, /* 256 bits as defined in 10.2.1 */
120 .max_addtllen = 35,
121 .max_bits = 19,
122 .max_req = 48,
123 .blocklen_bytes = 16, 120 .blocklen_bytes = 16,
124 .cra_name = "ctr_aes128", 121 .cra_name = "ctr_aes128",
125 .backend_cra_name = "ecb(aes)", 122 .backend_cra_name = "ecb(aes)",
126 }, { 123 }, {
127 .flags = DRBG_CTR | DRBG_STRENGTH192, 124 .flags = DRBG_CTR | DRBG_STRENGTH192,
128 .statelen = 40, /* 320 bits as defined in 10.2.1 */ 125 .statelen = 40, /* 320 bits as defined in 10.2.1 */
129 .max_addtllen = 35,
130 .max_bits = 19,
131 .max_req = 48,
132 .blocklen_bytes = 16, 126 .blocklen_bytes = 16,
133 .cra_name = "ctr_aes192", 127 .cra_name = "ctr_aes192",
134 .backend_cra_name = "ecb(aes)", 128 .backend_cra_name = "ecb(aes)",
135 }, { 129 }, {
136 .flags = DRBG_CTR | DRBG_STRENGTH256, 130 .flags = DRBG_CTR | DRBG_STRENGTH256,
137 .statelen = 48, /* 384 bits as defined in 10.2.1 */ 131 .statelen = 48, /* 384 bits as defined in 10.2.1 */
138 .max_addtllen = 35,
139 .max_bits = 19,
140 .max_req = 48,
141 .blocklen_bytes = 16, 132 .blocklen_bytes = 16,
142 .cra_name = "ctr_aes256", 133 .cra_name = "ctr_aes256",
143 .backend_cra_name = "ecb(aes)", 134 .backend_cra_name = "ecb(aes)",
@@ -147,36 +138,24 @@ static const struct drbg_core drbg_cores[] = {
147 { 138 {
148 .flags = DRBG_HASH | DRBG_STRENGTH128, 139 .flags = DRBG_HASH | DRBG_STRENGTH128,
149 .statelen = 55, /* 440 bits */ 140 .statelen = 55, /* 440 bits */
150 .max_addtllen = 35,
151 .max_bits = 19,
152 .max_req = 48,
153 .blocklen_bytes = 20, 141 .blocklen_bytes = 20,
154 .cra_name = "sha1", 142 .cra_name = "sha1",
155 .backend_cra_name = "sha1", 143 .backend_cra_name = "sha1",
156 }, { 144 }, {
157 .flags = DRBG_HASH | DRBG_STRENGTH256, 145 .flags = DRBG_HASH | DRBG_STRENGTH256,
158 .statelen = 111, /* 888 bits */ 146 .statelen = 111, /* 888 bits */
159 .max_addtllen = 35,
160 .max_bits = 19,
161 .max_req = 48,
162 .blocklen_bytes = 48, 147 .blocklen_bytes = 48,
163 .cra_name = "sha384", 148 .cra_name = "sha384",
164 .backend_cra_name = "sha384", 149 .backend_cra_name = "sha384",
165 }, { 150 }, {
166 .flags = DRBG_HASH | DRBG_STRENGTH256, 151 .flags = DRBG_HASH | DRBG_STRENGTH256,
167 .statelen = 111, /* 888 bits */ 152 .statelen = 111, /* 888 bits */
168 .max_addtllen = 35,
169 .max_bits = 19,
170 .max_req = 48,
171 .blocklen_bytes = 64, 153 .blocklen_bytes = 64,
172 .cra_name = "sha512", 154 .cra_name = "sha512",
173 .backend_cra_name = "sha512", 155 .backend_cra_name = "sha512",
174 }, { 156 }, {
175 .flags = DRBG_HASH | DRBG_STRENGTH256, 157 .flags = DRBG_HASH | DRBG_STRENGTH256,
176 .statelen = 55, /* 440 bits */ 158 .statelen = 55, /* 440 bits */
177 .max_addtllen = 35,
178 .max_bits = 19,
179 .max_req = 48,
180 .blocklen_bytes = 32, 159 .blocklen_bytes = 32,
181 .cra_name = "sha256", 160 .cra_name = "sha256",
182 .backend_cra_name = "sha256", 161 .backend_cra_name = "sha256",
@@ -186,36 +165,24 @@ static const struct drbg_core drbg_cores[] = {
186 { 165 {
187 .flags = DRBG_HMAC | DRBG_STRENGTH128, 166 .flags = DRBG_HMAC | DRBG_STRENGTH128,
188 .statelen = 20, /* block length of cipher */ 167 .statelen = 20, /* block length of cipher */
189 .max_addtllen = 35,
190 .max_bits = 19,
191 .max_req = 48,
192 .blocklen_bytes = 20, 168 .blocklen_bytes = 20,
193 .cra_name = "hmac_sha1", 169 .cra_name = "hmac_sha1",
194 .backend_cra_name = "hmac(sha1)", 170 .backend_cra_name = "hmac(sha1)",
195 }, { 171 }, {
196 .flags = DRBG_HMAC | DRBG_STRENGTH256, 172 .flags = DRBG_HMAC | DRBG_STRENGTH256,
197 .statelen = 48, /* block length of cipher */ 173 .statelen = 48, /* block length of cipher */
198 .max_addtllen = 35,
199 .max_bits = 19,
200 .max_req = 48,
201 .blocklen_bytes = 48, 174 .blocklen_bytes = 48,
202 .cra_name = "hmac_sha384", 175 .cra_name = "hmac_sha384",
203 .backend_cra_name = "hmac(sha384)", 176 .backend_cra_name = "hmac(sha384)",
204 }, { 177 }, {
205 .flags = DRBG_HMAC | DRBG_STRENGTH256, 178 .flags = DRBG_HMAC | DRBG_STRENGTH256,
206 .statelen = 64, /* block length of cipher */ 179 .statelen = 64, /* block length of cipher */
207 .max_addtllen = 35,
208 .max_bits = 19,
209 .max_req = 48,
210 .blocklen_bytes = 64, 180 .blocklen_bytes = 64,
211 .cra_name = "hmac_sha512", 181 .cra_name = "hmac_sha512",
212 .backend_cra_name = "hmac(sha512)", 182 .backend_cra_name = "hmac(sha512)",
213 }, { 183 }, {
214 .flags = DRBG_HMAC | DRBG_STRENGTH256, 184 .flags = DRBG_HMAC | DRBG_STRENGTH256,
215 .statelen = 32, /* block length of cipher */ 185 .statelen = 32, /* block length of cipher */
216 .max_addtllen = 35,
217 .max_bits = 19,
218 .max_req = 48,
219 .blocklen_bytes = 32, 186 .blocklen_bytes = 32,
220 .cra_name = "hmac_sha256", 187 .cra_name = "hmac_sha256",
221 .backend_cra_name = "hmac(sha256)", 188 .backend_cra_name = "hmac(sha256)",
@@ -302,20 +269,19 @@ static bool drbg_fips_continuous_test(struct drbg_state *drbg,
302 * Convert an integer into a byte representation of this integer. 269 * Convert an integer into a byte representation of this integer.
303 * The byte representation is big-endian 270 * The byte representation is big-endian
304 * 271 *
305 * @buf buffer holding the converted integer
306 * @val value to be converted 272 * @val value to be converted
307 * @buflen length of buffer 273 * @buf buffer holding the converted integer -- caller must ensure that
274 * buffer size is at least 32 bit
308 */ 275 */
309#if (defined(CONFIG_CRYPTO_DRBG_HASH) || defined(CONFIG_CRYPTO_DRBG_CTR)) 276#if (defined(CONFIG_CRYPTO_DRBG_HASH) || defined(CONFIG_CRYPTO_DRBG_CTR))
310static inline void drbg_int2byte(unsigned char *buf, uint64_t val, 277static inline void drbg_cpu_to_be32(__u32 val, unsigned char *buf)
311 size_t buflen)
312{ 278{
313 unsigned char *byte; 279 struct s {
314 uint64_t i; 280 __be32 conv;
281 };
282 struct s *conversion = (struct s *) buf;
315 283
316 byte = buf + (buflen - 1); 284 conversion->conv = cpu_to_be32(val);
317 for (i = 0; i < buflen; i++)
318 *(byte--) = val >> (i * 8) & 0xff;
319} 285}
320 286
321/* 287/*
@@ -483,10 +449,10 @@ static int drbg_ctr_df(struct drbg_state *drbg,
483 /* 10.4.2 step 2 -- calculate the entire length of all input data */ 449 /* 10.4.2 step 2 -- calculate the entire length of all input data */
484 list_for_each_entry(seed, seedlist, list) 450 list_for_each_entry(seed, seedlist, list)
485 inputlen += seed->len; 451 inputlen += seed->len;
486 drbg_int2byte(&L_N[0], inputlen, 4); 452 drbg_cpu_to_be32(inputlen, &L_N[0]);
487 453
488 /* 10.4.2 step 3 */ 454 /* 10.4.2 step 3 */
489 drbg_int2byte(&L_N[4], bytes_to_return, 4); 455 drbg_cpu_to_be32(bytes_to_return, &L_N[4]);
490 456
491 /* 10.4.2 step 5: length is L_N, input_string, one byte, padding */ 457 /* 10.4.2 step 5: length is L_N, input_string, one byte, padding */
492 padlen = (inputlen + sizeof(L_N) + 1) % (drbg_blocklen(drbg)); 458 padlen = (inputlen + sizeof(L_N) + 1) % (drbg_blocklen(drbg));
@@ -517,7 +483,7 @@ static int drbg_ctr_df(struct drbg_state *drbg,
517 * holds zeros after allocation -- even the increment of i 483 * holds zeros after allocation -- even the increment of i
518 * is irrelevant as the increment remains within length of i 484 * is irrelevant as the increment remains within length of i
519 */ 485 */
520 drbg_int2byte(iv, i, 4); 486 drbg_cpu_to_be32(i, iv);
521 /* 10.4.2 step 9.2 -- BCC and concatenation with temp */ 487 /* 10.4.2 step 9.2 -- BCC and concatenation with temp */
522 ret = drbg_ctr_bcc(drbg, temp + templen, K, &bcc_list); 488 ret = drbg_ctr_bcc(drbg, temp + templen, K, &bcc_list);
523 if (ret) 489 if (ret)
@@ -729,11 +695,9 @@ static int drbg_hmac_update(struct drbg_state *drbg, struct list_head *seed,
729 LIST_HEAD(seedlist); 695 LIST_HEAD(seedlist);
730 LIST_HEAD(vdatalist); 696 LIST_HEAD(vdatalist);
731 697
732 if (!reseed) { 698 if (!reseed)
733 /* 10.1.2.3 step 2 */ 699 /* 10.1.2.3 step 2 -- memset(0) of C is implicit with kzalloc */
734 memset(drbg->C, 0, drbg_statelen(drbg));
735 memset(drbg->V, 1, drbg_statelen(drbg)); 700 memset(drbg->V, 1, drbg_statelen(drbg));
736 }
737 701
738 drbg_string_fill(&seed1, drbg->V, drbg_statelen(drbg)); 702 drbg_string_fill(&seed1, drbg->V, drbg_statelen(drbg));
739 list_add_tail(&seed1.list, &seedlist); 703 list_add_tail(&seed1.list, &seedlist);
@@ -862,7 +826,7 @@ static int drbg_hash_df(struct drbg_state *drbg,
862 826
863 /* 10.4.1 step 3 */ 827 /* 10.4.1 step 3 */
864 input[0] = 1; 828 input[0] = 1;
865 drbg_int2byte(&input[1], (outlen * 8), 4); 829 drbg_cpu_to_be32((outlen * 8), &input[1]);
866 830
867 /* 10.4.1 step 4.1 -- concatenation of data for input into hash */ 831 /* 10.4.1 step 4.1 -- concatenation of data for input into hash */
868 drbg_string_fill(&data, input, 5); 832 drbg_string_fill(&data, input, 5);
@@ -1023,7 +987,10 @@ static int drbg_hash_generate(struct drbg_state *drbg,
1023{ 987{
1024 int len = 0; 988 int len = 0;
1025 int ret = 0; 989 int ret = 0;
1026 unsigned char req[8]; 990 union {
991 unsigned char req[8];
992 __be64 req_int;
993 } u;
1027 unsigned char prefix = DRBG_PREFIX3; 994 unsigned char prefix = DRBG_PREFIX3;
1028 struct drbg_string data1, data2; 995 struct drbg_string data1, data2;
1029 LIST_HEAD(datalist); 996 LIST_HEAD(datalist);
@@ -1053,8 +1020,8 @@ static int drbg_hash_generate(struct drbg_state *drbg,
1053 drbg->scratchpad, drbg_blocklen(drbg)); 1020 drbg->scratchpad, drbg_blocklen(drbg));
1054 drbg_add_buf(drbg->V, drbg_statelen(drbg), 1021 drbg_add_buf(drbg->V, drbg_statelen(drbg),
1055 drbg->C, drbg_statelen(drbg)); 1022 drbg->C, drbg_statelen(drbg));
1056 drbg_int2byte(req, drbg->reseed_ctr, sizeof(req)); 1023 u.req_int = cpu_to_be64(drbg->reseed_ctr);
1057 drbg_add_buf(drbg->V, drbg_statelen(drbg), req, 8); 1024 drbg_add_buf(drbg->V, drbg_statelen(drbg), u.req, 8);
1058 1025
1059out: 1026out:
1060 memset(drbg->scratchpad, 0, drbg_blocklen(drbg)); 1027 memset(drbg->scratchpad, 0, drbg_blocklen(drbg));
@@ -1142,6 +1109,11 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
1142 pr_devel("DRBG: using personalization string\n"); 1109 pr_devel("DRBG: using personalization string\n");
1143 } 1110 }
1144 1111
1112 if (!reseed) {
1113 memset(drbg->V, 0, drbg_statelen(drbg));
1114 memset(drbg->C, 0, drbg_statelen(drbg));
1115 }
1116
1145 ret = drbg->d_ops->update(drbg, &seedlist, reseed); 1117 ret = drbg->d_ops->update(drbg, &seedlist, reseed);
1146 if (ret) 1118 if (ret)
1147 goto out; 1119 goto out;
@@ -1151,8 +1123,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
1151 drbg->reseed_ctr = 1; 1123 drbg->reseed_ctr = 1;
1152 1124
1153out: 1125out:
1154 if (entropy) 1126 kzfree(entropy);
1155 kzfree(entropy);
1156 return ret; 1127 return ret;
1157} 1128}
1158 1129
@@ -1161,19 +1132,15 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg)
1161{ 1132{
1162 if (!drbg) 1133 if (!drbg)
1163 return; 1134 return;
1164 if (drbg->V) 1135 kzfree(drbg->V);
1165 kzfree(drbg->V);
1166 drbg->V = NULL; 1136 drbg->V = NULL;
1167 if (drbg->C) 1137 kzfree(drbg->C);
1168 kzfree(drbg->C);
1169 drbg->C = NULL; 1138 drbg->C = NULL;
1170 if (drbg->scratchpad) 1139 kzfree(drbg->scratchpad);
1171 kzfree(drbg->scratchpad);
1172 drbg->scratchpad = NULL; 1140 drbg->scratchpad = NULL;
1173 drbg->reseed_ctr = 0; 1141 drbg->reseed_ctr = 0;
1174#ifdef CONFIG_CRYPTO_FIPS 1142#ifdef CONFIG_CRYPTO_FIPS
1175 if (drbg->prev) 1143 kzfree(drbg->prev);
1176 kzfree(drbg->prev);
1177 drbg->prev = NULL; 1144 drbg->prev = NULL;
1178 drbg->fips_primed = false; 1145 drbg->fips_primed = false;
1179#endif 1146#endif
@@ -1188,17 +1155,14 @@ static inline int drbg_alloc_state(struct drbg_state *drbg)
1188 int ret = -ENOMEM; 1155 int ret = -ENOMEM;
1189 unsigned int sb_size = 0; 1156 unsigned int sb_size = 0;
1190 1157
1191 if (!drbg) 1158 drbg->V = kmalloc(drbg_statelen(drbg), GFP_KERNEL);
1192 return -EINVAL;
1193
1194 drbg->V = kzalloc(drbg_statelen(drbg), GFP_KERNEL);
1195 if (!drbg->V) 1159 if (!drbg->V)
1196 goto err; 1160 goto err;
1197 drbg->C = kzalloc(drbg_statelen(drbg), GFP_KERNEL); 1161 drbg->C = kmalloc(drbg_statelen(drbg), GFP_KERNEL);
1198 if (!drbg->C) 1162 if (!drbg->C)
1199 goto err; 1163 goto err;
1200#ifdef CONFIG_CRYPTO_FIPS 1164#ifdef CONFIG_CRYPTO_FIPS
1201 drbg->prev = kzalloc(drbg_blocklen(drbg), GFP_KERNEL); 1165 drbg->prev = kmalloc(drbg_blocklen(drbg), GFP_KERNEL);
1202 if (!drbg->prev) 1166 if (!drbg->prev)
1203 goto err; 1167 goto err;
1204 drbg->fips_primed = false; 1168 drbg->fips_primed = false;
@@ -1263,15 +1227,6 @@ static int drbg_make_shadow(struct drbg_state *drbg, struct drbg_state **shadow)
1263 int ret = -ENOMEM; 1227 int ret = -ENOMEM;
1264 struct drbg_state *tmp = NULL; 1228 struct drbg_state *tmp = NULL;
1265 1229
1266 if (!drbg || !drbg->core || !drbg->V || !drbg->C) {
1267 pr_devel("DRBG: attempt to generate shadow copy for "
1268 "uninitialized DRBG state rejected\n");
1269 return -EINVAL;
1270 }
1271 /* HMAC does not have a scratchpad */
1272 if (!(drbg->core->flags & DRBG_HMAC) && NULL == drbg->scratchpad)
1273 return -EINVAL;
1274
1275 tmp = kzalloc(sizeof(struct drbg_state), GFP_KERNEL); 1230 tmp = kzalloc(sizeof(struct drbg_state), GFP_KERNEL);
1276 if (!tmp) 1231 if (!tmp)
1277 return -ENOMEM; 1232 return -ENOMEM;
@@ -1293,8 +1248,7 @@ static int drbg_make_shadow(struct drbg_state *drbg, struct drbg_state **shadow)
1293 return 0; 1248 return 0;
1294 1249
1295err: 1250err:
1296 if (tmp) 1251 kzfree(tmp);
1297 kzfree(tmp);
1298 return ret; 1252 return ret;
1299} 1253}
1300 1254
@@ -1385,11 +1339,9 @@ static int drbg_generate(struct drbg_state *drbg,
1385 shadow->seeded = false; 1339 shadow->seeded = false;
1386 1340
1387 /* allocate cipher handle */ 1341 /* allocate cipher handle */
1388 if (shadow->d_ops->crypto_init) { 1342 len = shadow->d_ops->crypto_init(shadow);
1389 len = shadow->d_ops->crypto_init(shadow); 1343 if (len)
1390 if (len) 1344 goto err;
1391 goto err;
1392 }
1393 1345
1394 if (shadow->pr || !shadow->seeded) { 1346 if (shadow->pr || !shadow->seeded) {
1395 pr_devel("DRBG: reseeding before generation (prediction " 1347 pr_devel("DRBG: reseeding before generation (prediction "
@@ -1471,8 +1423,7 @@ static int drbg_generate(struct drbg_state *drbg,
1471#endif 1423#endif
1472 1424
1473err: 1425err:
1474 if (shadow->d_ops->crypto_fini) 1426 shadow->d_ops->crypto_fini(shadow);
1475 shadow->d_ops->crypto_fini(shadow);
1476 drbg_restore_shadow(drbg, &shadow); 1427 drbg_restore_shadow(drbg, &shadow);
1477 return len; 1428 return len;
1478} 1429}
@@ -1566,11 +1517,10 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers,
1566 return ret; 1517 return ret;
1567 1518
1568 ret = -EFAULT; 1519 ret = -EFAULT;
1569 if (drbg->d_ops->crypto_init && drbg->d_ops->crypto_init(drbg)) 1520 if (drbg->d_ops->crypto_init(drbg))
1570 goto err; 1521 goto err;
1571 ret = drbg_seed(drbg, pers, false); 1522 ret = drbg_seed(drbg, pers, false);
1572 if (drbg->d_ops->crypto_fini) 1523 drbg->d_ops->crypto_fini(drbg);
1573 drbg->d_ops->crypto_fini(drbg);
1574 if (ret) 1524 if (ret)
1575 goto err; 1525 goto err;
1576 1526
diff --git a/crypto/lz4.c b/crypto/lz4.c
index 4586dd15b0d8..34d072b72a73 100644
--- a/crypto/lz4.c
+++ b/crypto/lz4.c
@@ -68,7 +68,7 @@ static int lz4_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
68 size_t tmp_len = *dlen; 68 size_t tmp_len = *dlen;
69 size_t __slen = slen; 69 size_t __slen = slen;
70 70
71 err = lz4_decompress(src, &__slen, dst, tmp_len); 71 err = lz4_decompress_unknownoutputsize(src, __slen, dst, &tmp_len);
72 if (err < 0) 72 if (err < 0)
73 return -EINVAL; 73 return -EINVAL;
74 74
diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c
index 151ba31d34e3..9218b3fed5e3 100644
--- a/crypto/lz4hc.c
+++ b/crypto/lz4hc.c
@@ -68,7 +68,7 @@ static int lz4hc_decompress_crypto(struct crypto_tfm *tfm, const u8 *src,
68 size_t tmp_len = *dlen; 68 size_t tmp_len = *dlen;
69 size_t __slen = slen; 69 size_t __slen = slen;
70 70
71 err = lz4_decompress(src, &__slen, dst, tmp_len); 71 err = lz4_decompress_unknownoutputsize(src, __slen, dst, &tmp_len);
72 if (err < 0) 72 if (err < 0)
73 return -EINVAL; 73 return -EINVAL;
74 74
diff --git a/crypto/mcryptd.c b/crypto/mcryptd.c
new file mode 100644
index 000000000000..b39fbd530102
--- /dev/null
+++ b/crypto/mcryptd.c
@@ -0,0 +1,705 @@
1/*
2 * Software multibuffer async crypto daemon.
3 *
4 * Copyright (c) 2014 Tim Chen <tim.c.chen@linux.intel.com>
5 *
6 * Adapted from crypto daemon.
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 */
14
15#include <crypto/algapi.h>
16#include <crypto/internal/hash.h>
17#include <crypto/internal/aead.h>
18#include <crypto/mcryptd.h>
19#include <crypto/crypto_wq.h>
20#include <linux/err.h>
21#include <linux/init.h>
22#include <linux/kernel.h>
23#include <linux/list.h>
24#include <linux/module.h>
25#include <linux/scatterlist.h>
26#include <linux/sched.h>
27#include <linux/slab.h>
28#include <linux/hardirq.h>
29
30#define MCRYPTD_MAX_CPU_QLEN 100
31#define MCRYPTD_BATCH 9
32
33static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
34 unsigned int tail);
35
36struct mcryptd_flush_list {
37 struct list_head list;
38 struct mutex lock;
39};
40
41static struct mcryptd_flush_list __percpu *mcryptd_flist;
42
43struct hashd_instance_ctx {
44 struct crypto_shash_spawn spawn;
45 struct mcryptd_queue *queue;
46};
47
48static void mcryptd_queue_worker(struct work_struct *work);
49
50void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay)
51{
52 struct mcryptd_flush_list *flist;
53
54 if (!cstate->flusher_engaged) {
55 /* put the flusher on the flush list */
56 flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
57 mutex_lock(&flist->lock);
58 list_add_tail(&cstate->flush_list, &flist->list);
59 cstate->flusher_engaged = true;
60 cstate->next_flush = jiffies + delay;
61 queue_delayed_work_on(smp_processor_id(), kcrypto_wq,
62 &cstate->flush, delay);
63 mutex_unlock(&flist->lock);
64 }
65}
66EXPORT_SYMBOL(mcryptd_arm_flusher);
67
68static int mcryptd_init_queue(struct mcryptd_queue *queue,
69 unsigned int max_cpu_qlen)
70{
71 int cpu;
72 struct mcryptd_cpu_queue *cpu_queue;
73
74 queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue);
75 pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue);
76 if (!queue->cpu_queue)
77 return -ENOMEM;
78 for_each_possible_cpu(cpu) {
79 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
80 pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue);
81 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
82 INIT_WORK(&cpu_queue->work, mcryptd_queue_worker);
83 }
84 return 0;
85}
86
87static void mcryptd_fini_queue(struct mcryptd_queue *queue)
88{
89 int cpu;
90 struct mcryptd_cpu_queue *cpu_queue;
91
92 for_each_possible_cpu(cpu) {
93 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
94 BUG_ON(cpu_queue->queue.qlen);
95 }
96 free_percpu(queue->cpu_queue);
97}
98
99static int mcryptd_enqueue_request(struct mcryptd_queue *queue,
100 struct crypto_async_request *request,
101 struct mcryptd_hash_request_ctx *rctx)
102{
103 int cpu, err;
104 struct mcryptd_cpu_queue *cpu_queue;
105
106 cpu = get_cpu();
107 cpu_queue = this_cpu_ptr(queue->cpu_queue);
108 rctx->tag.cpu = cpu;
109
110 err = crypto_enqueue_request(&cpu_queue->queue, request);
111 pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n",
112 cpu, cpu_queue, request);
113 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
114 put_cpu();
115
116 return err;
117}
118
119/*
120 * Try to opportunisticlly flush the partially completed jobs if
121 * crypto daemon is the only task running.
122 */
123static void mcryptd_opportunistic_flush(void)
124{
125 struct mcryptd_flush_list *flist;
126 struct mcryptd_alg_cstate *cstate;
127
128 flist = per_cpu_ptr(mcryptd_flist, smp_processor_id());
129 while (single_task_running()) {
130 mutex_lock(&flist->lock);
131 if (list_empty(&flist->list)) {
132 mutex_unlock(&flist->lock);
133 return;
134 }
135 cstate = list_entry(flist->list.next,
136 struct mcryptd_alg_cstate, flush_list);
137 if (!cstate->flusher_engaged) {
138 mutex_unlock(&flist->lock);
139 return;
140 }
141 list_del(&cstate->flush_list);
142 cstate->flusher_engaged = false;
143 mutex_unlock(&flist->lock);
144 cstate->alg_state->flusher(cstate);
145 }
146}
147
148/*
149 * Called in workqueue context, do one real cryption work (via
150 * req->complete) and reschedule itself if there are more work to
151 * do.
152 */
153static void mcryptd_queue_worker(struct work_struct *work)
154{
155 struct mcryptd_cpu_queue *cpu_queue;
156 struct crypto_async_request *req, *backlog;
157 int i;
158
159 /*
160 * Need to loop through more than once for multi-buffer to
161 * be effective.
162 */
163
164 cpu_queue = container_of(work, struct mcryptd_cpu_queue, work);
165 i = 0;
166 while (i < MCRYPTD_BATCH || single_task_running()) {
167 /*
168 * preempt_disable/enable is used to prevent
169 * being preempted by mcryptd_enqueue_request()
170 */
171 local_bh_disable();
172 preempt_disable();
173 backlog = crypto_get_backlog(&cpu_queue->queue);
174 req = crypto_dequeue_request(&cpu_queue->queue);
175 preempt_enable();
176 local_bh_enable();
177
178 if (!req) {
179 mcryptd_opportunistic_flush();
180 return;
181 }
182
183 if (backlog)
184 backlog->complete(backlog, -EINPROGRESS);
185 req->complete(req, 0);
186 if (!cpu_queue->queue.qlen)
187 return;
188 ++i;
189 }
190 if (cpu_queue->queue.qlen)
191 queue_work(kcrypto_wq, &cpu_queue->work);
192}
193
194void mcryptd_flusher(struct work_struct *__work)
195{
196 struct mcryptd_alg_cstate *alg_cpu_state;
197 struct mcryptd_alg_state *alg_state;
198 struct mcryptd_flush_list *flist;
199 int cpu;
200
201 cpu = smp_processor_id();
202 alg_cpu_state = container_of(to_delayed_work(__work),
203 struct mcryptd_alg_cstate, flush);
204 alg_state = alg_cpu_state->alg_state;
205 if (alg_cpu_state->cpu != cpu)
206 pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n",
207 cpu, alg_cpu_state->cpu);
208
209 if (alg_cpu_state->flusher_engaged) {
210 flist = per_cpu_ptr(mcryptd_flist, cpu);
211 mutex_lock(&flist->lock);
212 list_del(&alg_cpu_state->flush_list);
213 alg_cpu_state->flusher_engaged = false;
214 mutex_unlock(&flist->lock);
215 alg_state->flusher(alg_cpu_state);
216 }
217}
218EXPORT_SYMBOL_GPL(mcryptd_flusher);
219
220static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm)
221{
222 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
223 struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
224
225 return ictx->queue;
226}
227
228static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
229 unsigned int tail)
230{
231 char *p;
232 struct crypto_instance *inst;
233 int err;
234
235 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
236 if (!p)
237 return ERR_PTR(-ENOMEM);
238
239 inst = (void *)(p + head);
240
241 err = -ENAMETOOLONG;
242 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
243 "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
244 goto out_free_inst;
245
246 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
247
248 inst->alg.cra_priority = alg->cra_priority + 50;
249 inst->alg.cra_blocksize = alg->cra_blocksize;
250 inst->alg.cra_alignmask = alg->cra_alignmask;
251
252out:
253 return p;
254
255out_free_inst:
256 kfree(p);
257 p = ERR_PTR(err);
258 goto out;
259}
260
261static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm)
262{
263 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
264 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
265 struct crypto_shash_spawn *spawn = &ictx->spawn;
266 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
267 struct crypto_shash *hash;
268
269 hash = crypto_spawn_shash(spawn);
270 if (IS_ERR(hash))
271 return PTR_ERR(hash);
272
273 ctx->child = hash;
274 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
275 sizeof(struct mcryptd_hash_request_ctx) +
276 crypto_shash_descsize(hash));
277 return 0;
278}
279
280static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm)
281{
282 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
283
284 crypto_free_shash(ctx->child);
285}
286
287static int mcryptd_hash_setkey(struct crypto_ahash *parent,
288 const u8 *key, unsigned int keylen)
289{
290 struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
291 struct crypto_shash *child = ctx->child;
292 int err;
293
294 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
295 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
296 CRYPTO_TFM_REQ_MASK);
297 err = crypto_shash_setkey(child, key, keylen);
298 crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
299 CRYPTO_TFM_RES_MASK);
300 return err;
301}
302
303static int mcryptd_hash_enqueue(struct ahash_request *req,
304 crypto_completion_t complete)
305{
306 int ret;
307
308 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
309 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
310 struct mcryptd_queue *queue =
311 mcryptd_get_queue(crypto_ahash_tfm(tfm));
312
313 rctx->complete = req->base.complete;
314 req->base.complete = complete;
315
316 ret = mcryptd_enqueue_request(queue, &req->base, rctx);
317
318 return ret;
319}
320
321static void mcryptd_hash_init(struct crypto_async_request *req_async, int err)
322{
323 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
324 struct crypto_shash *child = ctx->child;
325 struct ahash_request *req = ahash_request_cast(req_async);
326 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
327 struct shash_desc *desc = &rctx->desc;
328
329 if (unlikely(err == -EINPROGRESS))
330 goto out;
331
332 desc->tfm = child;
333 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
334
335 err = crypto_shash_init(desc);
336
337 req->base.complete = rctx->complete;
338
339out:
340 local_bh_disable();
341 rctx->complete(&req->base, err);
342 local_bh_enable();
343}
344
345static int mcryptd_hash_init_enqueue(struct ahash_request *req)
346{
347 return mcryptd_hash_enqueue(req, mcryptd_hash_init);
348}
349
350static void mcryptd_hash_update(struct crypto_async_request *req_async, int err)
351{
352 struct ahash_request *req = ahash_request_cast(req_async);
353 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
354
355 if (unlikely(err == -EINPROGRESS))
356 goto out;
357
358 err = shash_ahash_mcryptd_update(req, &rctx->desc);
359 if (err) {
360 req->base.complete = rctx->complete;
361 goto out;
362 }
363
364 return;
365out:
366 local_bh_disable();
367 rctx->complete(&req->base, err);
368 local_bh_enable();
369}
370
371static int mcryptd_hash_update_enqueue(struct ahash_request *req)
372{
373 return mcryptd_hash_enqueue(req, mcryptd_hash_update);
374}
375
376static void mcryptd_hash_final(struct crypto_async_request *req_async, int err)
377{
378 struct ahash_request *req = ahash_request_cast(req_async);
379 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
380
381 if (unlikely(err == -EINPROGRESS))
382 goto out;
383
384 err = shash_ahash_mcryptd_final(req, &rctx->desc);
385 if (err) {
386 req->base.complete = rctx->complete;
387 goto out;
388 }
389
390 return;
391out:
392 local_bh_disable();
393 rctx->complete(&req->base, err);
394 local_bh_enable();
395}
396
397static int mcryptd_hash_final_enqueue(struct ahash_request *req)
398{
399 return mcryptd_hash_enqueue(req, mcryptd_hash_final);
400}
401
402static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err)
403{
404 struct ahash_request *req = ahash_request_cast(req_async);
405 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
406
407 if (unlikely(err == -EINPROGRESS))
408 goto out;
409
410 err = shash_ahash_mcryptd_finup(req, &rctx->desc);
411
412 if (err) {
413 req->base.complete = rctx->complete;
414 goto out;
415 }
416
417 return;
418out:
419 local_bh_disable();
420 rctx->complete(&req->base, err);
421 local_bh_enable();
422}
423
424static int mcryptd_hash_finup_enqueue(struct ahash_request *req)
425{
426 return mcryptd_hash_enqueue(req, mcryptd_hash_finup);
427}
428
429static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err)
430{
431 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
432 struct crypto_shash *child = ctx->child;
433 struct ahash_request *req = ahash_request_cast(req_async);
434 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
435 struct shash_desc *desc = &rctx->desc;
436
437 if (unlikely(err == -EINPROGRESS))
438 goto out;
439
440 desc->tfm = child;
441 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; /* check this again */
442
443 err = shash_ahash_mcryptd_digest(req, desc);
444
445 if (err) {
446 req->base.complete = rctx->complete;
447 goto out;
448 }
449
450 return;
451out:
452 local_bh_disable();
453 rctx->complete(&req->base, err);
454 local_bh_enable();
455}
456
457static int mcryptd_hash_digest_enqueue(struct ahash_request *req)
458{
459 return mcryptd_hash_enqueue(req, mcryptd_hash_digest);
460}
461
462static int mcryptd_hash_export(struct ahash_request *req, void *out)
463{
464 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
465
466 return crypto_shash_export(&rctx->desc, out);
467}
468
469static int mcryptd_hash_import(struct ahash_request *req, const void *in)
470{
471 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
472
473 return crypto_shash_import(&rctx->desc, in);
474}
475
476static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
477 struct mcryptd_queue *queue)
478{
479 struct hashd_instance_ctx *ctx;
480 struct ahash_instance *inst;
481 struct shash_alg *salg;
482 struct crypto_alg *alg;
483 int err;
484
485 salg = shash_attr_alg(tb[1], 0, 0);
486 if (IS_ERR(salg))
487 return PTR_ERR(salg);
488
489 alg = &salg->base;
490 pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name);
491 inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(),
492 sizeof(*ctx));
493 err = PTR_ERR(inst);
494 if (IS_ERR(inst))
495 goto out_put_alg;
496
497 ctx = ahash_instance_ctx(inst);
498 ctx->queue = queue;
499
500 err = crypto_init_shash_spawn(&ctx->spawn, salg,
501 ahash_crypto_instance(inst));
502 if (err)
503 goto out_free_inst;
504
505 inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC;
506
507 inst->alg.halg.digestsize = salg->digestsize;
508 inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx);
509
510 inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm;
511 inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm;
512
513 inst->alg.init = mcryptd_hash_init_enqueue;
514 inst->alg.update = mcryptd_hash_update_enqueue;
515 inst->alg.final = mcryptd_hash_final_enqueue;
516 inst->alg.finup = mcryptd_hash_finup_enqueue;
517 inst->alg.export = mcryptd_hash_export;
518 inst->alg.import = mcryptd_hash_import;
519 inst->alg.setkey = mcryptd_hash_setkey;
520 inst->alg.digest = mcryptd_hash_digest_enqueue;
521
522 err = ahash_register_instance(tmpl, inst);
523 if (err) {
524 crypto_drop_shash(&ctx->spawn);
525out_free_inst:
526 kfree(inst);
527 }
528
529out_put_alg:
530 crypto_mod_put(alg);
531 return err;
532}
533
534static struct mcryptd_queue mqueue;
535
536static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
537{
538 struct crypto_attr_type *algt;
539
540 algt = crypto_get_attr_type(tb);
541 if (IS_ERR(algt))
542 return PTR_ERR(algt);
543
544 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
545 case CRYPTO_ALG_TYPE_DIGEST:
546 return mcryptd_create_hash(tmpl, tb, &mqueue);
547 break;
548 }
549
550 return -EINVAL;
551}
552
553static void mcryptd_free(struct crypto_instance *inst)
554{
555 struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
556 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
557
558 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
559 case CRYPTO_ALG_TYPE_AHASH:
560 crypto_drop_shash(&hctx->spawn);
561 kfree(ahash_instance(inst));
562 return;
563 default:
564 crypto_drop_spawn(&ctx->spawn);
565 kfree(inst);
566 }
567}
568
569static struct crypto_template mcryptd_tmpl = {
570 .name = "mcryptd",
571 .create = mcryptd_create,
572 .free = mcryptd_free,
573 .module = THIS_MODULE,
574};
575
576struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name,
577 u32 type, u32 mask)
578{
579 char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME];
580 struct crypto_ahash *tfm;
581
582 if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME,
583 "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
584 return ERR_PTR(-EINVAL);
585 tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask);
586 if (IS_ERR(tfm))
587 return ERR_CAST(tfm);
588 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
589 crypto_free_ahash(tfm);
590 return ERR_PTR(-EINVAL);
591 }
592
593 return __mcryptd_ahash_cast(tfm);
594}
595EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash);
596
597int shash_ahash_mcryptd_digest(struct ahash_request *req,
598 struct shash_desc *desc)
599{
600 int err;
601
602 err = crypto_shash_init(desc) ?:
603 shash_ahash_mcryptd_finup(req, desc);
604
605 return err;
606}
607EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_digest);
608
609int shash_ahash_mcryptd_update(struct ahash_request *req,
610 struct shash_desc *desc)
611{
612 struct crypto_shash *tfm = desc->tfm;
613 struct shash_alg *shash = crypto_shash_alg(tfm);
614
615 /* alignment is to be done by multi-buffer crypto algorithm if needed */
616
617 return shash->update(desc, NULL, 0);
618}
619EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_update);
620
621int shash_ahash_mcryptd_finup(struct ahash_request *req,
622 struct shash_desc *desc)
623{
624 struct crypto_shash *tfm = desc->tfm;
625 struct shash_alg *shash = crypto_shash_alg(tfm);
626
627 /* alignment is to be done by multi-buffer crypto algorithm if needed */
628
629 return shash->finup(desc, NULL, 0, req->result);
630}
631EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_finup);
632
633int shash_ahash_mcryptd_final(struct ahash_request *req,
634 struct shash_desc *desc)
635{
636 struct crypto_shash *tfm = desc->tfm;
637 struct shash_alg *shash = crypto_shash_alg(tfm);
638
639 /* alignment is to be done by multi-buffer crypto algorithm if needed */
640
641 return shash->final(desc, req->result);
642}
643EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_final);
644
645struct crypto_shash *mcryptd_ahash_child(struct mcryptd_ahash *tfm)
646{
647 struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
648
649 return ctx->child;
650}
651EXPORT_SYMBOL_GPL(mcryptd_ahash_child);
652
653struct shash_desc *mcryptd_shash_desc(struct ahash_request *req)
654{
655 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
656 return &rctx->desc;
657}
658EXPORT_SYMBOL_GPL(mcryptd_shash_desc);
659
660void mcryptd_free_ahash(struct mcryptd_ahash *tfm)
661{
662 crypto_free_ahash(&tfm->base);
663}
664EXPORT_SYMBOL_GPL(mcryptd_free_ahash);
665
666
667static int __init mcryptd_init(void)
668{
669 int err, cpu;
670 struct mcryptd_flush_list *flist;
671
672 mcryptd_flist = alloc_percpu(struct mcryptd_flush_list);
673 for_each_possible_cpu(cpu) {
674 flist = per_cpu_ptr(mcryptd_flist, cpu);
675 INIT_LIST_HEAD(&flist->list);
676 mutex_init(&flist->lock);
677 }
678
679 err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN);
680 if (err) {
681 free_percpu(mcryptd_flist);
682 return err;
683 }
684
685 err = crypto_register_template(&mcryptd_tmpl);
686 if (err) {
687 mcryptd_fini_queue(&mqueue);
688 free_percpu(mcryptd_flist);
689 }
690
691 return err;
692}
693
694static void __exit mcryptd_exit(void)
695{
696 mcryptd_fini_queue(&mqueue);
697 crypto_unregister_template(&mcryptd_tmpl);
698 free_percpu(mcryptd_flist);
699}
700
701subsys_initcall(mcryptd_init);
702module_exit(mcryptd_exit);
703
704MODULE_LICENSE("GPL");
705MODULE_DESCRIPTION("Software async multibuffer crypto daemon");
diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c
index 543366779524..0bb558344699 100644
--- a/crypto/sha256_generic.c
+++ b/crypto/sha256_generic.c
@@ -24,6 +24,7 @@
24#include <linux/types.h> 24#include <linux/types.h>
25#include <crypto/sha.h> 25#include <crypto/sha.h>
26#include <asm/byteorder.h> 26#include <asm/byteorder.h>
27#include <asm/unaligned.h>
27 28
28static inline u32 Ch(u32 x, u32 y, u32 z) 29static inline u32 Ch(u32 x, u32 y, u32 z)
29{ 30{
@@ -42,7 +43,7 @@ static inline u32 Maj(u32 x, u32 y, u32 z)
42 43
43static inline void LOAD_OP(int I, u32 *W, const u8 *input) 44static inline void LOAD_OP(int I, u32 *W, const u8 *input)
44{ 45{
45 W[I] = __be32_to_cpu( ((__be32*)(input))[I] ); 46 W[I] = get_unaligned_be32((__u32 *)input + I);
46} 47}
47 48
48static inline void BLEND_OP(int I, u32 *W) 49static inline void BLEND_OP(int I, u32 *W)
diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c
index 6ed124f3ea0f..6dde57dc511b 100644
--- a/crypto/sha512_generic.c
+++ b/crypto/sha512_generic.c
@@ -20,6 +20,7 @@
20#include <crypto/sha.h> 20#include <crypto/sha.h>
21#include <linux/percpu.h> 21#include <linux/percpu.h>
22#include <asm/byteorder.h> 22#include <asm/byteorder.h>
23#include <asm/unaligned.h>
23 24
24static inline u64 Ch(u64 x, u64 y, u64 z) 25static inline u64 Ch(u64 x, u64 y, u64 z)
25{ 26{
@@ -68,7 +69,7 @@ static const u64 sha512_K[80] = {
68 69
69static inline void LOAD_OP(int I, u64 *W, const u8 *input) 70static inline void LOAD_OP(int I, u64 *W, const u8 *input)
70{ 71{
71 W[I] = __be64_to_cpu( ((__be64*)(input))[I] ); 72 W[I] = get_unaligned_be64((__u64 *)input + I);
72} 73}
73 74
74static inline void BLEND_OP(int I, u64 *W) 75static inline void BLEND_OP(int I, u64 *W)
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index ac2b63105afc..9459dfd7357f 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -178,9 +178,7 @@ static void testmgr_free_buf(char *buf[XBUFSIZE])
178 free_page((unsigned long)buf[i]); 178 free_page((unsigned long)buf[i]);
179} 179}
180 180
181static int do_one_async_hash_op(struct ahash_request *req, 181static int wait_async_op(struct tcrypt_result *tr, int ret)
182 struct tcrypt_result *tr,
183 int ret)
184{ 182{
185 if (ret == -EINPROGRESS || ret == -EBUSY) { 183 if (ret == -EINPROGRESS || ret == -EBUSY) {
186 ret = wait_for_completion_interruptible(&tr->completion); 184 ret = wait_for_completion_interruptible(&tr->completion);
@@ -264,30 +262,26 @@ static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
264 262
265 ahash_request_set_crypt(req, sg, result, template[i].psize); 263 ahash_request_set_crypt(req, sg, result, template[i].psize);
266 if (use_digest) { 264 if (use_digest) {
267 ret = do_one_async_hash_op(req, &tresult, 265 ret = wait_async_op(&tresult, crypto_ahash_digest(req));
268 crypto_ahash_digest(req));
269 if (ret) { 266 if (ret) {
270 pr_err("alg: hash: digest failed on test %d " 267 pr_err("alg: hash: digest failed on test %d "
271 "for %s: ret=%d\n", j, algo, -ret); 268 "for %s: ret=%d\n", j, algo, -ret);
272 goto out; 269 goto out;
273 } 270 }
274 } else { 271 } else {
275 ret = do_one_async_hash_op(req, &tresult, 272 ret = wait_async_op(&tresult, crypto_ahash_init(req));
276 crypto_ahash_init(req));
277 if (ret) { 273 if (ret) {
278 pr_err("alt: hash: init failed on test %d " 274 pr_err("alt: hash: init failed on test %d "
279 "for %s: ret=%d\n", j, algo, -ret); 275 "for %s: ret=%d\n", j, algo, -ret);
280 goto out; 276 goto out;
281 } 277 }
282 ret = do_one_async_hash_op(req, &tresult, 278 ret = wait_async_op(&tresult, crypto_ahash_update(req));
283 crypto_ahash_update(req));
284 if (ret) { 279 if (ret) {
285 pr_err("alt: hash: update failed on test %d " 280 pr_err("alt: hash: update failed on test %d "
286 "for %s: ret=%d\n", j, algo, -ret); 281 "for %s: ret=%d\n", j, algo, -ret);
287 goto out; 282 goto out;
288 } 283 }
289 ret = do_one_async_hash_op(req, &tresult, 284 ret = wait_async_op(&tresult, crypto_ahash_final(req));
290 crypto_ahash_final(req));
291 if (ret) { 285 if (ret) {
292 pr_err("alt: hash: final failed on test %d " 286 pr_err("alt: hash: final failed on test %d "
293 "for %s: ret=%d\n", j, algo, -ret); 287 "for %s: ret=%d\n", j, algo, -ret);
@@ -311,78 +305,75 @@ static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
311 if (align_offset != 0) 305 if (align_offset != 0)
312 break; 306 break;
313 307
314 if (template[i].np) { 308 if (!template[i].np)
315 j++; 309 continue;
316 memset(result, 0, MAX_DIGEST_SIZE);
317 310
318 temp = 0; 311 j++;
319 sg_init_table(sg, template[i].np); 312 memset(result, 0, MAX_DIGEST_SIZE);
320 ret = -EINVAL; 313
321 for (k = 0; k < template[i].np; k++) { 314 temp = 0;
322 if (WARN_ON(offset_in_page(IDX[k]) + 315 sg_init_table(sg, template[i].np);
323 template[i].tap[k] > PAGE_SIZE)) 316 ret = -EINVAL;
324 goto out; 317 for (k = 0; k < template[i].np; k++) {
325 sg_set_buf(&sg[k], 318 if (WARN_ON(offset_in_page(IDX[k]) +
326 memcpy(xbuf[IDX[k] >> PAGE_SHIFT] + 319 template[i].tap[k] > PAGE_SIZE))
327 offset_in_page(IDX[k]),
328 template[i].plaintext + temp,
329 template[i].tap[k]),
330 template[i].tap[k]);
331 temp += template[i].tap[k];
332 }
333
334 if (template[i].ksize) {
335 if (template[i].ksize > MAX_KEYLEN) {
336 pr_err("alg: hash: setkey failed on test %d for %s: key size %d > %d\n",
337 j, algo, template[i].ksize,
338 MAX_KEYLEN);
339 ret = -EINVAL;
340 goto out;
341 }
342 crypto_ahash_clear_flags(tfm, ~0);
343 memcpy(key, template[i].key, template[i].ksize);
344 ret = crypto_ahash_setkey(tfm, key,
345 template[i].ksize);
346
347 if (ret) {
348 printk(KERN_ERR "alg: hash: setkey "
349 "failed on chunking test %d "
350 "for %s: ret=%d\n", j, algo,
351 -ret);
352 goto out;
353 }
354 }
355
356 ahash_request_set_crypt(req, sg, result,
357 template[i].psize);
358 ret = crypto_ahash_digest(req);
359 switch (ret) {
360 case 0:
361 break;
362 case -EINPROGRESS:
363 case -EBUSY:
364 ret = wait_for_completion_interruptible(
365 &tresult.completion);
366 if (!ret && !(ret = tresult.err)) {
367 reinit_completion(&tresult.completion);
368 break;
369 }
370 /* fall through */
371 default:
372 printk(KERN_ERR "alg: hash: digest failed "
373 "on chunking test %d for %s: "
374 "ret=%d\n", j, algo, -ret);
375 goto out; 320 goto out;
376 } 321 sg_set_buf(&sg[k],
322 memcpy(xbuf[IDX[k] >> PAGE_SHIFT] +
323 offset_in_page(IDX[k]),
324 template[i].plaintext + temp,
325 template[i].tap[k]),
326 template[i].tap[k]);
327 temp += template[i].tap[k];
328 }
377 329
378 if (memcmp(result, template[i].digest, 330 if (template[i].ksize) {
379 crypto_ahash_digestsize(tfm))) { 331 if (template[i].ksize > MAX_KEYLEN) {
380 printk(KERN_ERR "alg: hash: Chunking test %d " 332 pr_err("alg: hash: setkey failed on test %d for %s: key size %d > %d\n",
381 "failed for %s\n", j, algo); 333 j, algo, template[i].ksize, MAX_KEYLEN);
382 hexdump(result, crypto_ahash_digestsize(tfm));
383 ret = -EINVAL; 334 ret = -EINVAL;
384 goto out; 335 goto out;
385 } 336 }
337 crypto_ahash_clear_flags(tfm, ~0);
338 memcpy(key, template[i].key, template[i].ksize);
339 ret = crypto_ahash_setkey(tfm, key, template[i].ksize);
340
341 if (ret) {
342 printk(KERN_ERR "alg: hash: setkey "
343 "failed on chunking test %d "
344 "for %s: ret=%d\n", j, algo, -ret);
345 goto out;
346 }
347 }
348
349 ahash_request_set_crypt(req, sg, result, template[i].psize);
350 ret = crypto_ahash_digest(req);
351 switch (ret) {
352 case 0:
353 break;
354 case -EINPROGRESS:
355 case -EBUSY:
356 ret = wait_for_completion_interruptible(
357 &tresult.completion);
358 if (!ret && !(ret = tresult.err)) {
359 reinit_completion(&tresult.completion);
360 break;
361 }
362 /* fall through */
363 default:
364 printk(KERN_ERR "alg: hash: digest failed "
365 "on chunking test %d for %s: "
366 "ret=%d\n", j, algo, -ret);
367 goto out;
368 }
369
370 if (memcmp(result, template[i].digest,
371 crypto_ahash_digestsize(tfm))) {
372 printk(KERN_ERR "alg: hash: Chunking test %d "
373 "failed for %s\n", j, algo);
374 hexdump(result, crypto_ahash_digestsize(tfm));
375 ret = -EINVAL;
376 goto out;
386 } 377 }
387 } 378 }
388 379
@@ -492,121 +483,116 @@ static int __test_aead(struct crypto_aead *tfm, int enc,
492 tcrypt_complete, &result); 483 tcrypt_complete, &result);
493 484
494 for (i = 0, j = 0; i < tcount; i++) { 485 for (i = 0, j = 0; i < tcount; i++) {
495 if (!template[i].np) { 486 if (template[i].np)
496 j++; 487 continue;
497 488
498 /* some templates have no input data but they will 489 j++;
499 * touch input
500 */
501 input = xbuf[0];
502 input += align_offset;
503 assoc = axbuf[0];
504 490
505 ret = -EINVAL; 491 /* some templates have no input data but they will
506 if (WARN_ON(align_offset + template[i].ilen > 492 * touch input
507 PAGE_SIZE || template[i].alen > PAGE_SIZE)) 493 */
508 goto out; 494 input = xbuf[0];
495 input += align_offset;
496 assoc = axbuf[0];
509 497
510 memcpy(input, template[i].input, template[i].ilen); 498 ret = -EINVAL;
511 memcpy(assoc, template[i].assoc, template[i].alen); 499 if (WARN_ON(align_offset + template[i].ilen >
512 if (template[i].iv) 500 PAGE_SIZE || template[i].alen > PAGE_SIZE))
513 memcpy(iv, template[i].iv, MAX_IVLEN); 501 goto out;
514 else
515 memset(iv, 0, MAX_IVLEN);
516 502
517 crypto_aead_clear_flags(tfm, ~0); 503 memcpy(input, template[i].input, template[i].ilen);
518 if (template[i].wk) 504 memcpy(assoc, template[i].assoc, template[i].alen);
519 crypto_aead_set_flags( 505 if (template[i].iv)
520 tfm, CRYPTO_TFM_REQ_WEAK_KEY); 506 memcpy(iv, template[i].iv, MAX_IVLEN);
507 else
508 memset(iv, 0, MAX_IVLEN);
521 509
522 if (template[i].klen > MAX_KEYLEN) { 510 crypto_aead_clear_flags(tfm, ~0);
523 pr_err("alg: aead%s: setkey failed on test %d for %s: key size %d > %d\n", 511 if (template[i].wk)
524 d, j, algo, template[i].klen, 512 crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY);
525 MAX_KEYLEN);
526 ret = -EINVAL;
527 goto out;
528 }
529 memcpy(key, template[i].key, template[i].klen);
530 513
531 ret = crypto_aead_setkey(tfm, key, 514 if (template[i].klen > MAX_KEYLEN) {
532 template[i].klen); 515 pr_err("alg: aead%s: setkey failed on test %d for %s: key size %d > %d\n",
533 if (!ret == template[i].fail) { 516 d, j, algo, template[i].klen,
534 pr_err("alg: aead%s: setkey failed on test %d for %s: flags=%x\n", 517 MAX_KEYLEN);
535 d, j, algo, crypto_aead_get_flags(tfm)); 518 ret = -EINVAL;
536 goto out; 519 goto out;
537 } else if (ret) 520 }
538 continue; 521 memcpy(key, template[i].key, template[i].klen);
539 522
540 authsize = abs(template[i].rlen - template[i].ilen); 523 ret = crypto_aead_setkey(tfm, key, template[i].klen);
541 ret = crypto_aead_setauthsize(tfm, authsize); 524 if (!ret == template[i].fail) {
542 if (ret) { 525 pr_err("alg: aead%s: setkey failed on test %d for %s: flags=%x\n",
543 pr_err("alg: aead%s: Failed to set authsize to %u on test %d for %s\n", 526 d, j, algo, crypto_aead_get_flags(tfm));
544 d, authsize, j, algo); 527 goto out;
545 goto out; 528 } else if (ret)
546 } 529 continue;
547 530
548 if (diff_dst) { 531 authsize = abs(template[i].rlen - template[i].ilen);
549 output = xoutbuf[0]; 532 ret = crypto_aead_setauthsize(tfm, authsize);
550 output += align_offset; 533 if (ret) {
551 sg_init_one(&sg[0], input, template[i].ilen); 534 pr_err("alg: aead%s: Failed to set authsize to %u on test %d for %s\n",
552 sg_init_one(&sgout[0], output, 535 d, authsize, j, algo);
553 template[i].rlen); 536 goto out;
554 } else { 537 }
555 sg_init_one(&sg[0], input,
556 template[i].ilen +
557 (enc ? authsize : 0));
558 output = input;
559 }
560 538
561 sg_init_one(&asg[0], assoc, template[i].alen); 539 if (diff_dst) {
540 output = xoutbuf[0];
541 output += align_offset;
542 sg_init_one(&sg[0], input, template[i].ilen);
543 sg_init_one(&sgout[0], output, template[i].rlen);
544 } else {
545 sg_init_one(&sg[0], input,
546 template[i].ilen + (enc ? authsize : 0));
547 output = input;
548 }
562 549
563 aead_request_set_crypt(req, sg, (diff_dst) ? sgout : sg, 550 sg_init_one(&asg[0], assoc, template[i].alen);
564 template[i].ilen, iv);
565 551
566 aead_request_set_assoc(req, asg, template[i].alen); 552 aead_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
553 template[i].ilen, iv);
567 554
568 ret = enc ? 555 aead_request_set_assoc(req, asg, template[i].alen);
569 crypto_aead_encrypt(req) :
570 crypto_aead_decrypt(req);
571 556
572 switch (ret) { 557 ret = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
573 case 0:
574 if (template[i].novrfy) {
575 /* verification was supposed to fail */
576 pr_err("alg: aead%s: %s failed on test %d for %s: ret was 0, expected -EBADMSG\n",
577 d, e, j, algo);
578 /* so really, we got a bad message */
579 ret = -EBADMSG;
580 goto out;
581 }
582 break;
583 case -EINPROGRESS:
584 case -EBUSY:
585 ret = wait_for_completion_interruptible(
586 &result.completion);
587 if (!ret && !(ret = result.err)) {
588 reinit_completion(&result.completion);
589 break;
590 }
591 case -EBADMSG:
592 if (template[i].novrfy)
593 /* verification failure was expected */
594 continue;
595 /* fall through */
596 default:
597 pr_err("alg: aead%s: %s failed on test %d for %s: ret=%d\n",
598 d, e, j, algo, -ret);
599 goto out;
600 }
601 558
602 q = output; 559 switch (ret) {
603 if (memcmp(q, template[i].result, template[i].rlen)) { 560 case 0:
604 pr_err("alg: aead%s: Test %d failed on %s for %s\n", 561 if (template[i].novrfy) {
605 d, j, e, algo); 562 /* verification was supposed to fail */
606 hexdump(q, template[i].rlen); 563 pr_err("alg: aead%s: %s failed on test %d for %s: ret was 0, expected -EBADMSG\n",
607 ret = -EINVAL; 564 d, e, j, algo);
565 /* so really, we got a bad message */
566 ret = -EBADMSG;
608 goto out; 567 goto out;
609 } 568 }
569 break;
570 case -EINPROGRESS:
571 case -EBUSY:
572 ret = wait_for_completion_interruptible(
573 &result.completion);
574 if (!ret && !(ret = result.err)) {
575 reinit_completion(&result.completion);
576 break;
577 }
578 case -EBADMSG:
579 if (template[i].novrfy)
580 /* verification failure was expected */
581 continue;
582 /* fall through */
583 default:
584 pr_err("alg: aead%s: %s failed on test %d for %s: ret=%d\n",
585 d, e, j, algo, -ret);
586 goto out;
587 }
588
589 q = output;
590 if (memcmp(q, template[i].result, template[i].rlen)) {
591 pr_err("alg: aead%s: Test %d failed on %s for %s\n",
592 d, j, e, algo);
593 hexdump(q, template[i].rlen);
594 ret = -EINVAL;
595 goto out;
610 } 596 }
611 } 597 }
612 598
@@ -615,191 +601,182 @@ static int __test_aead(struct crypto_aead *tfm, int enc,
615 if (align_offset != 0) 601 if (align_offset != 0)
616 break; 602 break;
617 603
618 if (template[i].np) { 604 if (!template[i].np)
619 j++; 605 continue;
620
621 if (template[i].iv)
622 memcpy(iv, template[i].iv, MAX_IVLEN);
623 else
624 memset(iv, 0, MAX_IVLEN);
625
626 crypto_aead_clear_flags(tfm, ~0);
627 if (template[i].wk)
628 crypto_aead_set_flags(
629 tfm, CRYPTO_TFM_REQ_WEAK_KEY);
630 if (template[i].klen > MAX_KEYLEN) {
631 pr_err("alg: aead%s: setkey failed on test %d for %s: key size %d > %d\n",
632 d, j, algo, template[i].klen,
633 MAX_KEYLEN);
634 ret = -EINVAL;
635 goto out;
636 }
637 memcpy(key, template[i].key, template[i].klen);
638 606
639 ret = crypto_aead_setkey(tfm, key, template[i].klen); 607 j++;
640 if (!ret == template[i].fail) {
641 pr_err("alg: aead%s: setkey failed on chunk test %d for %s: flags=%x\n",
642 d, j, algo, crypto_aead_get_flags(tfm));
643 goto out;
644 } else if (ret)
645 continue;
646 608
647 authsize = abs(template[i].rlen - template[i].ilen); 609 if (template[i].iv)
610 memcpy(iv, template[i].iv, MAX_IVLEN);
611 else
612 memset(iv, 0, MAX_IVLEN);
648 613
614 crypto_aead_clear_flags(tfm, ~0);
615 if (template[i].wk)
616 crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY);
617 if (template[i].klen > MAX_KEYLEN) {
618 pr_err("alg: aead%s: setkey failed on test %d for %s: key size %d > %d\n",
619 d, j, algo, template[i].klen, MAX_KEYLEN);
649 ret = -EINVAL; 620 ret = -EINVAL;
650 sg_init_table(sg, template[i].np); 621 goto out;
651 if (diff_dst) 622 }
652 sg_init_table(sgout, template[i].np); 623 memcpy(key, template[i].key, template[i].klen);
653 for (k = 0, temp = 0; k < template[i].np; k++) {
654 if (WARN_ON(offset_in_page(IDX[k]) +
655 template[i].tap[k] > PAGE_SIZE))
656 goto out;
657
658 q = xbuf[IDX[k] >> PAGE_SHIFT] +
659 offset_in_page(IDX[k]);
660 624
661 memcpy(q, template[i].input + temp, 625 ret = crypto_aead_setkey(tfm, key, template[i].klen);
662 template[i].tap[k]); 626 if (!ret == template[i].fail) {
627 pr_err("alg: aead%s: setkey failed on chunk test %d for %s: flags=%x\n",
628 d, j, algo, crypto_aead_get_flags(tfm));
629 goto out;
630 } else if (ret)
631 continue;
663 632
664 sg_set_buf(&sg[k], q, template[i].tap[k]); 633 authsize = abs(template[i].rlen - template[i].ilen);
665 634
666 if (diff_dst) { 635 ret = -EINVAL;
667 q = xoutbuf[IDX[k] >> PAGE_SHIFT] + 636 sg_init_table(sg, template[i].np);
668 offset_in_page(IDX[k]); 637 if (diff_dst)
638 sg_init_table(sgout, template[i].np);
639 for (k = 0, temp = 0; k < template[i].np; k++) {
640 if (WARN_ON(offset_in_page(IDX[k]) +
641 template[i].tap[k] > PAGE_SIZE))
642 goto out;
669 643
670 memset(q, 0, template[i].tap[k]); 644 q = xbuf[IDX[k] >> PAGE_SHIFT] + offset_in_page(IDX[k]);
645 memcpy(q, template[i].input + temp, template[i].tap[k]);
646 sg_set_buf(&sg[k], q, template[i].tap[k]);
671 647
672 sg_set_buf(&sgout[k], q, 648 if (diff_dst) {
673 template[i].tap[k]); 649 q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
674 } 650 offset_in_page(IDX[k]);
675 651
676 n = template[i].tap[k]; 652 memset(q, 0, template[i].tap[k]);
677 if (k == template[i].np - 1 && enc)
678 n += authsize;
679 if (offset_in_page(q) + n < PAGE_SIZE)
680 q[n] = 0;
681 653
682 temp += template[i].tap[k]; 654 sg_set_buf(&sgout[k], q, template[i].tap[k]);
683 } 655 }
684 656
685 ret = crypto_aead_setauthsize(tfm, authsize); 657 n = template[i].tap[k];
686 if (ret) { 658 if (k == template[i].np - 1 && enc)
687 pr_err("alg: aead%s: Failed to set authsize to %u on chunk test %d for %s\n", 659 n += authsize;
688 d, authsize, j, algo); 660 if (offset_in_page(q) + n < PAGE_SIZE)
661 q[n] = 0;
662
663 temp += template[i].tap[k];
664 }
665
666 ret = crypto_aead_setauthsize(tfm, authsize);
667 if (ret) {
668 pr_err("alg: aead%s: Failed to set authsize to %u on chunk test %d for %s\n",
669 d, authsize, j, algo);
670 goto out;
671 }
672
673 if (enc) {
674 if (WARN_ON(sg[k - 1].offset +
675 sg[k - 1].length + authsize >
676 PAGE_SIZE)) {
677 ret = -EINVAL;
689 goto out; 678 goto out;
690 } 679 }
691 680
692 if (enc) { 681 if (diff_dst)
693 if (WARN_ON(sg[k - 1].offset + 682 sgout[k - 1].length += authsize;
694 sg[k - 1].length + authsize > 683 else
695 PAGE_SIZE)) { 684 sg[k - 1].length += authsize;
696 ret = -EINVAL; 685 }
697 goto out;
698 }
699 686
700 if (diff_dst) 687 sg_init_table(asg, template[i].anp);
701 sgout[k - 1].length += authsize; 688 ret = -EINVAL;
702 else 689 for (k = 0, temp = 0; k < template[i].anp; k++) {
703 sg[k - 1].length += authsize; 690 if (WARN_ON(offset_in_page(IDX[k]) +
691 template[i].atap[k] > PAGE_SIZE))
692 goto out;
693 sg_set_buf(&asg[k],
694 memcpy(axbuf[IDX[k] >> PAGE_SHIFT] +
695 offset_in_page(IDX[k]),
696 template[i].assoc + temp,
697 template[i].atap[k]),
698 template[i].atap[k]);
699 temp += template[i].atap[k];
700 }
701
702 aead_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
703 template[i].ilen,
704 iv);
705
706 aead_request_set_assoc(req, asg, template[i].alen);
707
708 ret = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
709
710 switch (ret) {
711 case 0:
712 if (template[i].novrfy) {
713 /* verification was supposed to fail */
714 pr_err("alg: aead%s: %s failed on chunk test %d for %s: ret was 0, expected -EBADMSG\n",
715 d, e, j, algo);
716 /* so really, we got a bad message */
717 ret = -EBADMSG;
718 goto out;
704 } 719 }
705 720 break;
706 sg_init_table(asg, template[i].anp); 721 case -EINPROGRESS:
707 ret = -EINVAL; 722 case -EBUSY:
708 for (k = 0, temp = 0; k < template[i].anp; k++) { 723 ret = wait_for_completion_interruptible(
709 if (WARN_ON(offset_in_page(IDX[k]) + 724 &result.completion);
710 template[i].atap[k] > PAGE_SIZE)) 725 if (!ret && !(ret = result.err)) {
711 goto out; 726 reinit_completion(&result.completion);
712 sg_set_buf(&asg[k],
713 memcpy(axbuf[IDX[k] >> PAGE_SHIFT] +
714 offset_in_page(IDX[k]),
715 template[i].assoc + temp,
716 template[i].atap[k]),
717 template[i].atap[k]);
718 temp += template[i].atap[k];
719 }
720
721 aead_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
722 template[i].ilen,
723 iv);
724
725 aead_request_set_assoc(req, asg, template[i].alen);
726
727 ret = enc ?
728 crypto_aead_encrypt(req) :
729 crypto_aead_decrypt(req);
730
731 switch (ret) {
732 case 0:
733 if (template[i].novrfy) {
734 /* verification was supposed to fail */
735 pr_err("alg: aead%s: %s failed on chunk test %d for %s: ret was 0, expected -EBADMSG\n",
736 d, e, j, algo);
737 /* so really, we got a bad message */
738 ret = -EBADMSG;
739 goto out;
740 }
741 break; 727 break;
742 case -EINPROGRESS:
743 case -EBUSY:
744 ret = wait_for_completion_interruptible(
745 &result.completion);
746 if (!ret && !(ret = result.err)) {
747 reinit_completion(&result.completion);
748 break;
749 }
750 case -EBADMSG:
751 if (template[i].novrfy)
752 /* verification failure was expected */
753 continue;
754 /* fall through */
755 default:
756 pr_err("alg: aead%s: %s failed on chunk test %d for %s: ret=%d\n",
757 d, e, j, algo, -ret);
758 goto out;
759 } 728 }
729 case -EBADMSG:
730 if (template[i].novrfy)
731 /* verification failure was expected */
732 continue;
733 /* fall through */
734 default:
735 pr_err("alg: aead%s: %s failed on chunk test %d for %s: ret=%d\n",
736 d, e, j, algo, -ret);
737 goto out;
738 }
760 739
761 ret = -EINVAL; 740 ret = -EINVAL;
762 for (k = 0, temp = 0; k < template[i].np; k++) { 741 for (k = 0, temp = 0; k < template[i].np; k++) {
763 if (diff_dst) 742 if (diff_dst)
764 q = xoutbuf[IDX[k] >> PAGE_SHIFT] + 743 q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
765 offset_in_page(IDX[k]); 744 offset_in_page(IDX[k]);
766 else 745 else
767 q = xbuf[IDX[k] >> PAGE_SHIFT] + 746 q = xbuf[IDX[k] >> PAGE_SHIFT] +
768 offset_in_page(IDX[k]); 747 offset_in_page(IDX[k]);
769
770 n = template[i].tap[k];
771 if (k == template[i].np - 1)
772 n += enc ? authsize : -authsize;
773 748
774 if (memcmp(q, template[i].result + temp, n)) { 749 n = template[i].tap[k];
775 pr_err("alg: aead%s: Chunk test %d failed on %s at page %u for %s\n", 750 if (k == template[i].np - 1)
776 d, j, e, k, algo); 751 n += enc ? authsize : -authsize;
777 hexdump(q, n);
778 goto out;
779 }
780 752
781 q += n; 753 if (memcmp(q, template[i].result + temp, n)) {
782 if (k == template[i].np - 1 && !enc) { 754 pr_err("alg: aead%s: Chunk test %d failed on %s at page %u for %s\n",
783 if (!diff_dst && 755 d, j, e, k, algo);
784 memcmp(q, template[i].input + 756 hexdump(q, n);
785 temp + n, authsize)) 757 goto out;
786 n = authsize; 758 }
787 else
788 n = 0;
789 } else {
790 for (n = 0; offset_in_page(q + n) &&
791 q[n]; n++)
792 ;
793 }
794 if (n) {
795 pr_err("alg: aead%s: Result buffer corruption in chunk test %d on %s at page %u for %s: %u bytes:\n",
796 d, j, e, k, algo, n);
797 hexdump(q, n);
798 goto out;
799 }
800 759
801 temp += template[i].tap[k]; 760 q += n;
761 if (k == template[i].np - 1 && !enc) {
762 if (!diff_dst &&
763 memcmp(q, template[i].input +
764 temp + n, authsize))
765 n = authsize;
766 else
767 n = 0;
768 } else {
769 for (n = 0; offset_in_page(q + n) && q[n]; n++)
770 ;
771 }
772 if (n) {
773 pr_err("alg: aead%s: Result buffer corruption in chunk test %d on %s at page %u for %s: %u bytes:\n",
774 d, j, e, k, algo, n);
775 hexdump(q, n);
776 goto out;
802 } 777 }
778
779 temp += template[i].tap[k];
803 } 780 }
804 } 781 }
805 782
@@ -978,78 +955,73 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc,
978 955
979 j = 0; 956 j = 0;
980 for (i = 0; i < tcount; i++) { 957 for (i = 0; i < tcount; i++) {
958 if (template[i].np && !template[i].also_non_np)
959 continue;
960
981 if (template[i].iv) 961 if (template[i].iv)
982 memcpy(iv, template[i].iv, MAX_IVLEN); 962 memcpy(iv, template[i].iv, MAX_IVLEN);
983 else 963 else
984 memset(iv, 0, MAX_IVLEN); 964 memset(iv, 0, MAX_IVLEN);
985 965
986 if (!(template[i].np) || (template[i].also_non_np)) { 966 j++;
987 j++; 967 ret = -EINVAL;
968 if (WARN_ON(align_offset + template[i].ilen > PAGE_SIZE))
969 goto out;
988 970
989 ret = -EINVAL; 971 data = xbuf[0];
990 if (WARN_ON(align_offset + template[i].ilen > 972 data += align_offset;
991 PAGE_SIZE)) 973 memcpy(data, template[i].input, template[i].ilen);
992 goto out;
993 974
994 data = xbuf[0]; 975 crypto_ablkcipher_clear_flags(tfm, ~0);
995 data += align_offset; 976 if (template[i].wk)
996 memcpy(data, template[i].input, template[i].ilen); 977 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY);
997
998 crypto_ablkcipher_clear_flags(tfm, ~0);
999 if (template[i].wk)
1000 crypto_ablkcipher_set_flags(
1001 tfm, CRYPTO_TFM_REQ_WEAK_KEY);
1002
1003 ret = crypto_ablkcipher_setkey(tfm, template[i].key,
1004 template[i].klen);
1005 if (!ret == template[i].fail) {
1006 pr_err("alg: skcipher%s: setkey failed on test %d for %s: flags=%x\n",
1007 d, j, algo,
1008 crypto_ablkcipher_get_flags(tfm));
1009 goto out;
1010 } else if (ret)
1011 continue;
1012 978
1013 sg_init_one(&sg[0], data, template[i].ilen); 979 ret = crypto_ablkcipher_setkey(tfm, template[i].key,
1014 if (diff_dst) { 980 template[i].klen);
1015 data = xoutbuf[0]; 981 if (!ret == template[i].fail) {
1016 data += align_offset; 982 pr_err("alg: skcipher%s: setkey failed on test %d for %s: flags=%x\n",
1017 sg_init_one(&sgout[0], data, template[i].ilen); 983 d, j, algo, crypto_ablkcipher_get_flags(tfm));
1018 } 984 goto out;
985 } else if (ret)
986 continue;
987
988 sg_init_one(&sg[0], data, template[i].ilen);
989 if (diff_dst) {
990 data = xoutbuf[0];
991 data += align_offset;
992 sg_init_one(&sgout[0], data, template[i].ilen);
993 }
1019 994
1020 ablkcipher_request_set_crypt(req, sg, 995 ablkcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
1021 (diff_dst) ? sgout : sg, 996 template[i].ilen, iv);
1022 template[i].ilen, iv); 997 ret = enc ? crypto_ablkcipher_encrypt(req) :
1023 ret = enc ? 998 crypto_ablkcipher_decrypt(req);
1024 crypto_ablkcipher_encrypt(req) :
1025 crypto_ablkcipher_decrypt(req);
1026 999
1027 switch (ret) { 1000 switch (ret) {
1028 case 0: 1001 case 0:
1002 break;
1003 case -EINPROGRESS:
1004 case -EBUSY:
1005 ret = wait_for_completion_interruptible(
1006 &result.completion);
1007 if (!ret && !((ret = result.err))) {
1008 reinit_completion(&result.completion);
1029 break; 1009 break;
1030 case -EINPROGRESS:
1031 case -EBUSY:
1032 ret = wait_for_completion_interruptible(
1033 &result.completion);
1034 if (!ret && !((ret = result.err))) {
1035 reinit_completion(&result.completion);
1036 break;
1037 }
1038 /* fall through */
1039 default:
1040 pr_err("alg: skcipher%s: %s failed on test %d for %s: ret=%d\n",
1041 d, e, j, algo, -ret);
1042 goto out;
1043 } 1010 }
1011 /* fall through */
1012 default:
1013 pr_err("alg: skcipher%s: %s failed on test %d for %s: ret=%d\n",
1014 d, e, j, algo, -ret);
1015 goto out;
1016 }
1044 1017
1045 q = data; 1018 q = data;
1046 if (memcmp(q, template[i].result, template[i].rlen)) { 1019 if (memcmp(q, template[i].result, template[i].rlen)) {
1047 pr_err("alg: skcipher%s: Test %d failed on %s for %s\n", 1020 pr_err("alg: skcipher%s: Test %d failed on %s for %s\n",
1048 d, j, e, algo); 1021 d, j, e, algo);
1049 hexdump(q, template[i].rlen); 1022 hexdump(q, template[i].rlen);
1050 ret = -EINVAL; 1023 ret = -EINVAL;
1051 goto out; 1024 goto out;
1052 }
1053 } 1025 }
1054 } 1026 }
1055 1027
@@ -1059,121 +1031,113 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc,
1059 if (align_offset != 0) 1031 if (align_offset != 0)
1060 break; 1032 break;
1061 1033
1034 if (!template[i].np)
1035 continue;
1036
1062 if (template[i].iv) 1037 if (template[i].iv)
1063 memcpy(iv, template[i].iv, MAX_IVLEN); 1038 memcpy(iv, template[i].iv, MAX_IVLEN);
1064 else 1039 else
1065 memset(iv, 0, MAX_IVLEN); 1040 memset(iv, 0, MAX_IVLEN);
1066 1041
1067 if (template[i].np) { 1042 j++;
1068 j++; 1043 crypto_ablkcipher_clear_flags(tfm, ~0);
1044 if (template[i].wk)
1045 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_REQ_WEAK_KEY);
1069 1046
1070 crypto_ablkcipher_clear_flags(tfm, ~0); 1047 ret = crypto_ablkcipher_setkey(tfm, template[i].key,
1071 if (template[i].wk) 1048 template[i].klen);
1072 crypto_ablkcipher_set_flags( 1049 if (!ret == template[i].fail) {
1073 tfm, CRYPTO_TFM_REQ_WEAK_KEY); 1050 pr_err("alg: skcipher%s: setkey failed on chunk test %d for %s: flags=%x\n",
1051 d, j, algo, crypto_ablkcipher_get_flags(tfm));
1052 goto out;
1053 } else if (ret)
1054 continue;
1074 1055
1075 ret = crypto_ablkcipher_setkey(tfm, template[i].key, 1056 temp = 0;
1076 template[i].klen); 1057 ret = -EINVAL;
1077 if (!ret == template[i].fail) { 1058 sg_init_table(sg, template[i].np);
1078 pr_err("alg: skcipher%s: setkey failed on chunk test %d for %s: flags=%x\n", 1059 if (diff_dst)
1079 d, j, algo, 1060 sg_init_table(sgout, template[i].np);
1080 crypto_ablkcipher_get_flags(tfm)); 1061 for (k = 0; k < template[i].np; k++) {
1062 if (WARN_ON(offset_in_page(IDX[k]) +
1063 template[i].tap[k] > PAGE_SIZE))
1081 goto out; 1064 goto out;
1082 } else if (ret)
1083 continue;
1084 1065
1085 temp = 0; 1066 q = xbuf[IDX[k] >> PAGE_SHIFT] + offset_in_page(IDX[k]);
1086 ret = -EINVAL;
1087 sg_init_table(sg, template[i].np);
1088 if (diff_dst)
1089 sg_init_table(sgout, template[i].np);
1090 for (k = 0; k < template[i].np; k++) {
1091 if (WARN_ON(offset_in_page(IDX[k]) +
1092 template[i].tap[k] > PAGE_SIZE))
1093 goto out;
1094 1067
1095 q = xbuf[IDX[k] >> PAGE_SHIFT] + 1068 memcpy(q, template[i].input + temp, template[i].tap[k]);
1069
1070 if (offset_in_page(q) + template[i].tap[k] < PAGE_SIZE)
1071 q[template[i].tap[k]] = 0;
1072
1073 sg_set_buf(&sg[k], q, template[i].tap[k]);
1074 if (diff_dst) {
1075 q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
1096 offset_in_page(IDX[k]); 1076 offset_in_page(IDX[k]);
1097 1077
1098 memcpy(q, template[i].input + temp, 1078 sg_set_buf(&sgout[k], q, template[i].tap[k]);
1099 template[i].tap[k]);
1100 1079
1101 if (offset_in_page(q) + template[i].tap[k] < 1080 memset(q, 0, template[i].tap[k]);
1102 PAGE_SIZE) 1081 if (offset_in_page(q) +
1082 template[i].tap[k] < PAGE_SIZE)
1103 q[template[i].tap[k]] = 0; 1083 q[template[i].tap[k]] = 0;
1084 }
1104 1085
1105 sg_set_buf(&sg[k], q, template[i].tap[k]); 1086 temp += template[i].tap[k];
1106 if (diff_dst) { 1087 }
1107 q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
1108 offset_in_page(IDX[k]);
1109 1088
1110 sg_set_buf(&sgout[k], q, 1089 ablkcipher_request_set_crypt(req, sg, (diff_dst) ? sgout : sg,
1111 template[i].tap[k]); 1090 template[i].ilen, iv);
1112 1091
1113 memset(q, 0, template[i].tap[k]); 1092 ret = enc ? crypto_ablkcipher_encrypt(req) :
1114 if (offset_in_page(q) + 1093 crypto_ablkcipher_decrypt(req);
1115 template[i].tap[k] < PAGE_SIZE)
1116 q[template[i].tap[k]] = 0;
1117 }
1118 1094
1119 temp += template[i].tap[k]; 1095 switch (ret) {
1096 case 0:
1097 break;
1098 case -EINPROGRESS:
1099 case -EBUSY:
1100 ret = wait_for_completion_interruptible(
1101 &result.completion);
1102 if (!ret && !((ret = result.err))) {
1103 reinit_completion(&result.completion);
1104 break;
1120 } 1105 }
1106 /* fall through */
1107 default:
1108 pr_err("alg: skcipher%s: %s failed on chunk test %d for %s: ret=%d\n",
1109 d, e, j, algo, -ret);
1110 goto out;
1111 }
1121 1112
1122 ablkcipher_request_set_crypt(req, sg, 1113 temp = 0;
1123 (diff_dst) ? sgout : sg, 1114 ret = -EINVAL;
1124 template[i].ilen, iv); 1115 for (k = 0; k < template[i].np; k++) {
1125 1116 if (diff_dst)
1126 ret = enc ? 1117 q = xoutbuf[IDX[k] >> PAGE_SHIFT] +
1127 crypto_ablkcipher_encrypt(req) : 1118 offset_in_page(IDX[k]);
1128 crypto_ablkcipher_decrypt(req); 1119 else
1120 q = xbuf[IDX[k] >> PAGE_SHIFT] +
1121 offset_in_page(IDX[k]);
1129 1122
1130 switch (ret) { 1123 if (memcmp(q, template[i].result + temp,
1131 case 0: 1124 template[i].tap[k])) {
1132 break; 1125 pr_err("alg: skcipher%s: Chunk test %d failed on %s at page %u for %s\n",
1133 case -EINPROGRESS: 1126 d, j, e, k, algo);
1134 case -EBUSY: 1127 hexdump(q, template[i].tap[k]);
1135 ret = wait_for_completion_interruptible(
1136 &result.completion);
1137 if (!ret && !((ret = result.err))) {
1138 reinit_completion(&result.completion);
1139 break;
1140 }
1141 /* fall through */
1142 default:
1143 pr_err("alg: skcipher%s: %s failed on chunk test %d for %s: ret=%d\n",
1144 d, e, j, algo, -ret);
1145 goto out; 1128 goto out;
1146 } 1129 }
1147 1130
1148 temp = 0; 1131 q += template[i].tap[k];
1149 ret = -EINVAL; 1132 for (n = 0; offset_in_page(q + n) && q[n]; n++)
1150 for (k = 0; k < template[i].np; k++) { 1133 ;
1151 if (diff_dst) 1134 if (n) {
1152 q = xoutbuf[IDX[k] >> PAGE_SHIFT] + 1135 pr_err("alg: skcipher%s: Result buffer corruption in chunk test %d on %s at page %u for %s: %u bytes:\n",
1153 offset_in_page(IDX[k]); 1136 d, j, e, k, algo, n);
1154 else 1137 hexdump(q, n);
1155 q = xbuf[IDX[k] >> PAGE_SHIFT] + 1138 goto out;
1156 offset_in_page(IDX[k]);
1157
1158 if (memcmp(q, template[i].result + temp,
1159 template[i].tap[k])) {
1160 pr_err("alg: skcipher%s: Chunk test %d failed on %s at page %u for %s\n",
1161 d, j, e, k, algo);
1162 hexdump(q, template[i].tap[k]);
1163 goto out;
1164 }
1165
1166 q += template[i].tap[k];
1167 for (n = 0; offset_in_page(q + n) && q[n]; n++)
1168 ;
1169 if (n) {
1170 pr_err("alg: skcipher%s: Result buffer corruption in chunk test %d on %s at page %u for %s: %u bytes:\n",
1171 d, j, e, k, algo, n);
1172 hexdump(q, n);
1173 goto out;
1174 }
1175 temp += template[i].tap[k];
1176 } 1139 }
1140 temp += template[i].tap[k];
1177 } 1141 }
1178 } 1142 }
1179 1143
@@ -3213,6 +3177,38 @@ static const struct alg_test_desc alg_test_descs[] = {
3213 } 3177 }
3214 } 3178 }
3215 }, { 3179 }, {
3180 .alg = "lz4",
3181 .test = alg_test_comp,
3182 .fips_allowed = 1,
3183 .suite = {
3184 .comp = {
3185 .comp = {
3186 .vecs = lz4_comp_tv_template,
3187 .count = LZ4_COMP_TEST_VECTORS
3188 },
3189 .decomp = {
3190 .vecs = lz4_decomp_tv_template,
3191 .count = LZ4_DECOMP_TEST_VECTORS
3192 }
3193 }
3194 }
3195 }, {
3196 .alg = "lz4hc",
3197 .test = alg_test_comp,
3198 .fips_allowed = 1,
3199 .suite = {
3200 .comp = {
3201 .comp = {
3202 .vecs = lz4hc_comp_tv_template,
3203 .count = LZ4HC_COMP_TEST_VECTORS
3204 },
3205 .decomp = {
3206 .vecs = lz4hc_decomp_tv_template,
3207 .count = LZ4HC_DECOMP_TEST_VECTORS
3208 }
3209 }
3210 }
3211 }, {
3216 .alg = "lzo", 3212 .alg = "lzo",
3217 .test = alg_test_comp, 3213 .test = alg_test_comp,
3218 .fips_allowed = 1, 3214 .fips_allowed = 1,
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 6597203eccfa..62e2485bb428 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -29473,4 +29473,70 @@ static struct hash_testvec bfin_crc_tv_template[] = {
29473 29473
29474}; 29474};
29475 29475
29476#define LZ4_COMP_TEST_VECTORS 1
29477#define LZ4_DECOMP_TEST_VECTORS 1
29478
29479static struct comp_testvec lz4_comp_tv_template[] = {
29480 {
29481 .inlen = 70,
29482 .outlen = 45,
29483 .input = "Join us now and share the software "
29484 "Join us now and share the software ",
29485 .output = "\xf0\x10\x4a\x6f\x69\x6e\x20\x75"
29486 "\x73\x20\x6e\x6f\x77\x20\x61\x6e"
29487 "\x64\x20\x73\x68\x61\x72\x65\x20"
29488 "\x74\x68\x65\x20\x73\x6f\x66\x74"
29489 "\x77\x0d\x00\x0f\x23\x00\x0b\x50"
29490 "\x77\x61\x72\x65\x20",
29491 },
29492};
29493
29494static struct comp_testvec lz4_decomp_tv_template[] = {
29495 {
29496 .inlen = 45,
29497 .outlen = 70,
29498 .input = "\xf0\x10\x4a\x6f\x69\x6e\x20\x75"
29499 "\x73\x20\x6e\x6f\x77\x20\x61\x6e"
29500 "\x64\x20\x73\x68\x61\x72\x65\x20"
29501 "\x74\x68\x65\x20\x73\x6f\x66\x74"
29502 "\x77\x0d\x00\x0f\x23\x00\x0b\x50"
29503 "\x77\x61\x72\x65\x20",
29504 .output = "Join us now and share the software "
29505 "Join us now and share the software ",
29506 },
29507};
29508
29509#define LZ4HC_COMP_TEST_VECTORS 1
29510#define LZ4HC_DECOMP_TEST_VECTORS 1
29511
29512static struct comp_testvec lz4hc_comp_tv_template[] = {
29513 {
29514 .inlen = 70,
29515 .outlen = 45,
29516 .input = "Join us now and share the software "
29517 "Join us now and share the software ",
29518 .output = "\xf0\x10\x4a\x6f\x69\x6e\x20\x75"
29519 "\x73\x20\x6e\x6f\x77\x20\x61\x6e"
29520 "\x64\x20\x73\x68\x61\x72\x65\x20"
29521 "\x74\x68\x65\x20\x73\x6f\x66\x74"
29522 "\x77\x0d\x00\x0f\x23\x00\x0b\x50"
29523 "\x77\x61\x72\x65\x20",
29524 },
29525};
29526
29527static struct comp_testvec lz4hc_decomp_tv_template[] = {
29528 {
29529 .inlen = 45,
29530 .outlen = 70,
29531 .input = "\xf0\x10\x4a\x6f\x69\x6e\x20\x75"
29532 "\x73\x20\x6e\x6f\x77\x20\x61\x6e"
29533 "\x64\x20\x73\x68\x61\x72\x65\x20"
29534 "\x74\x68\x65\x20\x73\x6f\x66\x74"
29535 "\x77\x0d\x00\x0f\x23\x00\x0b\x50"
29536 "\x77\x61\x72\x65\x20",
29537 .output = "Join us now and share the software "
29538 "Join us now and share the software ",
29539 },
29540};
29541
29476#endif /* _CRYPTO_TESTMGR_H */ 29542#endif /* _CRYPTO_TESTMGR_H */
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index 836b061ced35..91a04ae8003c 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -333,6 +333,19 @@ config HW_RANDOM_MSM
333 333
334 If unsure, say Y. 334 If unsure, say Y.
335 335
336config HW_RANDOM_XGENE
337 tristate "APM X-Gene True Random Number Generator (TRNG) support"
338 depends on HW_RANDOM && ARCH_XGENE
339 default HW_RANDOM
340 ---help---
341 This driver provides kernel-side support for the Random Number
342 Generator hardware found on APM X-Gene SoC.
343
344 To compile this driver as a module, choose M here: the
345 module will be called xgene_rng.
346
347 If unsure, say Y.
348
336endif # HW_RANDOM 349endif # HW_RANDOM
337 350
338config UML_RANDOM 351config UML_RANDOM
diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile
index 199ed283e149..0b4cd57f4e24 100644
--- a/drivers/char/hw_random/Makefile
+++ b/drivers/char/hw_random/Makefile
@@ -29,3 +29,4 @@ obj-$(CONFIG_HW_RANDOM_EXYNOS) += exynos-rng.o
29obj-$(CONFIG_HW_RANDOM_TPM) += tpm-rng.o 29obj-$(CONFIG_HW_RANDOM_TPM) += tpm-rng.o
30obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o 30obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o
31obj-$(CONFIG_HW_RANDOM_MSM) += msm-rng.o 31obj-$(CONFIG_HW_RANDOM_MSM) += msm-rng.o
32obj-$(CONFIG_HW_RANDOM_XGENE) += xgene-rng.o
diff --git a/drivers/char/hw_random/amd-rng.c b/drivers/char/hw_random/amd-rng.c
index c6af038682f1..48f6a83cdd61 100644
--- a/drivers/char/hw_random/amd-rng.c
+++ b/drivers/char/hw_random/amd-rng.c
@@ -142,10 +142,10 @@ found:
142 amd_rng.priv = (unsigned long)pmbase; 142 amd_rng.priv = (unsigned long)pmbase;
143 amd_pdev = pdev; 143 amd_pdev = pdev;
144 144
145 printk(KERN_INFO "AMD768 RNG detected\n"); 145 pr_info("AMD768 RNG detected\n");
146 err = hwrng_register(&amd_rng); 146 err = hwrng_register(&amd_rng);
147 if (err) { 147 if (err) {
148 printk(KERN_ERR PFX "RNG registering failed (%d)\n", 148 pr_err(PFX "RNG registering failed (%d)\n",
149 err); 149 err);
150 release_region(pmbase + 0xF0, 8); 150 release_region(pmbase + 0xF0, 8);
151 goto out; 151 goto out;
diff --git a/drivers/char/hw_random/geode-rng.c b/drivers/char/hw_random/geode-rng.c
index 4c4d4e140f98..0d0579fe465e 100644
--- a/drivers/char/hw_random/geode-rng.c
+++ b/drivers/char/hw_random/geode-rng.c
@@ -109,10 +109,10 @@ found:
109 goto out; 109 goto out;
110 geode_rng.priv = (unsigned long)mem; 110 geode_rng.priv = (unsigned long)mem;
111 111
112 printk(KERN_INFO "AMD Geode RNG detected\n"); 112 pr_info("AMD Geode RNG detected\n");
113 err = hwrng_register(&geode_rng); 113 err = hwrng_register(&geode_rng);
114 if (err) { 114 if (err) {
115 printk(KERN_ERR PFX "RNG registering failed (%d)\n", 115 pr_err(PFX "RNG registering failed (%d)\n",
116 err); 116 err);
117 goto err_unmap; 117 goto err_unmap;
118 } 118 }
diff --git a/drivers/char/hw_random/intel-rng.c b/drivers/char/hw_random/intel-rng.c
index 86fe45c19968..290c880266bf 100644
--- a/drivers/char/hw_random/intel-rng.c
+++ b/drivers/char/hw_random/intel-rng.c
@@ -199,7 +199,7 @@ static int intel_rng_init(struct hwrng *rng)
199 if ((hw_status & INTEL_RNG_ENABLED) == 0) 199 if ((hw_status & INTEL_RNG_ENABLED) == 0)
200 hw_status = hwstatus_set(mem, hw_status | INTEL_RNG_ENABLED); 200 hw_status = hwstatus_set(mem, hw_status | INTEL_RNG_ENABLED);
201 if ((hw_status & INTEL_RNG_ENABLED) == 0) { 201 if ((hw_status & INTEL_RNG_ENABLED) == 0) {
202 printk(KERN_ERR PFX "cannot enable RNG, aborting\n"); 202 pr_err(PFX "cannot enable RNG, aborting\n");
203 goto out; 203 goto out;
204 } 204 }
205 err = 0; 205 err = 0;
@@ -216,7 +216,7 @@ static void intel_rng_cleanup(struct hwrng *rng)
216 if (hw_status & INTEL_RNG_ENABLED) 216 if (hw_status & INTEL_RNG_ENABLED)
217 hwstatus_set(mem, hw_status & ~INTEL_RNG_ENABLED); 217 hwstatus_set(mem, hw_status & ~INTEL_RNG_ENABLED);
218 else 218 else
219 printk(KERN_WARNING PFX "unusual: RNG already disabled\n"); 219 pr_warn(PFX "unusual: RNG already disabled\n");
220} 220}
221 221
222 222
@@ -274,7 +274,7 @@ static int __init intel_rng_hw_init(void *_intel_rng_hw)
274 if (mfc != INTEL_FWH_MANUFACTURER_CODE || 274 if (mfc != INTEL_FWH_MANUFACTURER_CODE ||
275 (dvc != INTEL_FWH_DEVICE_CODE_8M && 275 (dvc != INTEL_FWH_DEVICE_CODE_8M &&
276 dvc != INTEL_FWH_DEVICE_CODE_4M)) { 276 dvc != INTEL_FWH_DEVICE_CODE_4M)) {
277 printk(KERN_NOTICE PFX "FWH not detected\n"); 277 pr_notice(PFX "FWH not detected\n");
278 return -ENODEV; 278 return -ENODEV;
279 } 279 }
280 280
@@ -306,7 +306,6 @@ static int __init intel_init_hw_struct(struct intel_rng_hw *intel_rng_hw,
306 (BIOS_CNTL_LOCK_ENABLE_MASK|BIOS_CNTL_WRITE_ENABLE_MASK)) 306 (BIOS_CNTL_LOCK_ENABLE_MASK|BIOS_CNTL_WRITE_ENABLE_MASK))
307 == BIOS_CNTL_LOCK_ENABLE_MASK) { 307 == BIOS_CNTL_LOCK_ENABLE_MASK) {
308 static __initdata /*const*/ char warning[] = 308 static __initdata /*const*/ char warning[] =
309 KERN_WARNING
310PFX "Firmware space is locked read-only. If you can't or\n" 309PFX "Firmware space is locked read-only. If you can't or\n"
311PFX "don't want to disable this in firmware setup, and if\n" 310PFX "don't want to disable this in firmware setup, and if\n"
312PFX "you are certain that your system has a functional\n" 311PFX "you are certain that your system has a functional\n"
@@ -314,7 +313,7 @@ PFX "RNG, try using the 'no_fwh_detect' option.\n";
314 313
315 if (no_fwh_detect) 314 if (no_fwh_detect)
316 return -ENODEV; 315 return -ENODEV;
317 printk(warning); 316 pr_warn("%s", warning);
318 return -EBUSY; 317 return -EBUSY;
319 } 318 }
320 319
@@ -392,10 +391,10 @@ fwh_done:
392 goto out; 391 goto out;
393 } 392 }
394 393
395 printk(KERN_INFO "Intel 82802 RNG detected\n"); 394 pr_info("Intel 82802 RNG detected\n");
396 err = hwrng_register(&intel_rng); 395 err = hwrng_register(&intel_rng);
397 if (err) { 396 if (err) {
398 printk(KERN_ERR PFX "RNG registering failed (%d)\n", 397 pr_err(PFX "RNG registering failed (%d)\n",
399 err); 398 err);
400 iounmap(mem); 399 iounmap(mem);
401 } 400 }
diff --git a/drivers/char/hw_random/pasemi-rng.c b/drivers/char/hw_random/pasemi-rng.c
index c66279bb6ef3..c0347d1dded0 100644
--- a/drivers/char/hw_random/pasemi-rng.c
+++ b/drivers/char/hw_random/pasemi-rng.c
@@ -113,7 +113,7 @@ static int rng_probe(struct platform_device *ofdev)
113 113
114 pasemi_rng.priv = (unsigned long)rng_regs; 114 pasemi_rng.priv = (unsigned long)rng_regs;
115 115
116 printk(KERN_INFO "Registering PA Semi RNG\n"); 116 pr_info("Registering PA Semi RNG\n");
117 117
118 err = hwrng_register(&pasemi_rng); 118 err = hwrng_register(&pasemi_rng);
119 119
diff --git a/drivers/char/hw_random/pseries-rng.c b/drivers/char/hw_random/pseries-rng.c
index ab7ffdec0ec3..6226aa08c36a 100644
--- a/drivers/char/hw_random/pseries-rng.c
+++ b/drivers/char/hw_random/pseries-rng.c
@@ -86,7 +86,7 @@ static struct vio_driver pseries_rng_driver = {
86 86
87static int __init rng_init(void) 87static int __init rng_init(void)
88{ 88{
89 printk(KERN_INFO "Registering IBM pSeries RNG driver\n"); 89 pr_info("Registering IBM pSeries RNG driver\n");
90 return vio_register_driver(&pseries_rng_driver); 90 return vio_register_driver(&pseries_rng_driver);
91} 91}
92 92
diff --git a/drivers/char/hw_random/via-rng.c b/drivers/char/hw_random/via-rng.c
index de5a6dcfb3e2..a3bebef255ad 100644
--- a/drivers/char/hw_random/via-rng.c
+++ b/drivers/char/hw_random/via-rng.c
@@ -141,7 +141,7 @@ static int via_rng_init(struct hwrng *rng)
141 * register */ 141 * register */
142 if ((c->x86 == 6) && (c->x86_model >= 0x0f)) { 142 if ((c->x86 == 6) && (c->x86_model >= 0x0f)) {
143 if (!cpu_has_xstore_enabled) { 143 if (!cpu_has_xstore_enabled) {
144 printk(KERN_ERR PFX "can't enable hardware RNG " 144 pr_err(PFX "can't enable hardware RNG "
145 "if XSTORE is not enabled\n"); 145 "if XSTORE is not enabled\n");
146 return -ENODEV; 146 return -ENODEV;
147 } 147 }
@@ -180,7 +180,7 @@ static int via_rng_init(struct hwrng *rng)
180 unneeded */ 180 unneeded */
181 rdmsr(MSR_VIA_RNG, lo, hi); 181 rdmsr(MSR_VIA_RNG, lo, hi);
182 if ((lo & VIA_RNG_ENABLE) == 0) { 182 if ((lo & VIA_RNG_ENABLE) == 0) {
183 printk(KERN_ERR PFX "cannot enable VIA C3 RNG, aborting\n"); 183 pr_err(PFX "cannot enable VIA C3 RNG, aborting\n");
184 return -ENODEV; 184 return -ENODEV;
185 } 185 }
186 186
@@ -202,10 +202,10 @@ static int __init mod_init(void)
202 202
203 if (!cpu_has_xstore) 203 if (!cpu_has_xstore)
204 return -ENODEV; 204 return -ENODEV;
205 printk(KERN_INFO "VIA RNG detected\n"); 205 pr_info("VIA RNG detected\n");
206 err = hwrng_register(&via_rng); 206 err = hwrng_register(&via_rng);
207 if (err) { 207 if (err) {
208 printk(KERN_ERR PFX "RNG registering failed (%d)\n", 208 pr_err(PFX "RNG registering failed (%d)\n",
209 err); 209 err);
210 goto out; 210 goto out;
211 } 211 }
diff --git a/drivers/char/hw_random/xgene-rng.c b/drivers/char/hw_random/xgene-rng.c
new file mode 100644
index 000000000000..23caa05380a8
--- /dev/null
+++ b/drivers/char/hw_random/xgene-rng.c
@@ -0,0 +1,423 @@
1/*
2 * APM X-Gene SoC RNG Driver
3 *
4 * Copyright (c) 2014, Applied Micro Circuits Corporation
5 * Author: Rameshwar Prasad Sahu <rsahu@apm.com>
6 * Shamal Winchurkar <swinchurkar@apm.com>
7 * Feng Kan <fkan@apm.com>
8 *
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU General Public License as published by the
11 * Free Software Foundation; either version 2 of the License, or (at your
12 * option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program. If not, see <http://www.gnu.org/licenses/>.
21 *
22 */
23
24#include <linux/clk.h>
25#include <linux/delay.h>
26#include <linux/hw_random.h>
27#include <linux/init.h>
28#include <linux/interrupt.h>
29#include <linux/module.h>
30#include <linux/of_platform.h>
31#include <linux/of_irq.h>
32#include <linux/of_address.h>
33#include <linux/timer.h>
34
35#define RNG_MAX_DATUM 4
36#define MAX_TRY 100
37#define XGENE_RNG_RETRY_COUNT 20
38#define XGENE_RNG_RETRY_INTERVAL 10
39
40/* RNG Registers */
41#define RNG_INOUT_0 0x00
42#define RNG_INTR_STS_ACK 0x10
43#define RNG_CONTROL 0x14
44#define RNG_CONFIG 0x18
45#define RNG_ALARMCNT 0x1c
46#define RNG_FROENABLE 0x20
47#define RNG_FRODETUNE 0x24
48#define RNG_ALARMMASK 0x28
49#define RNG_ALARMSTOP 0x2c
50#define RNG_OPTIONS 0x78
51#define RNG_EIP_REV 0x7c
52
53#define MONOBIT_FAIL_MASK BIT(7)
54#define POKER_FAIL_MASK BIT(6)
55#define LONG_RUN_FAIL_MASK BIT(5)
56#define RUN_FAIL_MASK BIT(4)
57#define NOISE_FAIL_MASK BIT(3)
58#define STUCK_OUT_MASK BIT(2)
59#define SHUTDOWN_OFLO_MASK BIT(1)
60#define READY_MASK BIT(0)
61
62#define MAJOR_HW_REV_RD(src) (((src) & 0x0f000000) >> 24)
63#define MINOR_HW_REV_RD(src) (((src) & 0x00f00000) >> 20)
64#define HW_PATCH_LEVEL_RD(src) (((src) & 0x000f0000) >> 16)
65#define MAX_REFILL_CYCLES_SET(dst, src) \
66 ((dst & ~0xffff0000) | (((u32)src << 16) & 0xffff0000))
67#define MIN_REFILL_CYCLES_SET(dst, src) \
68 ((dst & ~0x000000ff) | (((u32)src) & 0x000000ff))
69#define ALARM_THRESHOLD_SET(dst, src) \
70 ((dst & ~0x000000ff) | (((u32)src) & 0x000000ff))
71#define ENABLE_RNG_SET(dst, src) \
72 ((dst & ~BIT(10)) | (((u32)src << 10) & BIT(10)))
73#define REGSPEC_TEST_MODE_SET(dst, src) \
74 ((dst & ~BIT(8)) | (((u32)src << 8) & BIT(8)))
75#define MONOBIT_FAIL_MASK_SET(dst, src) \
76 ((dst & ~BIT(7)) | (((u32)src << 7) & BIT(7)))
77#define POKER_FAIL_MASK_SET(dst, src) \
78 ((dst & ~BIT(6)) | (((u32)src << 6) & BIT(6)))
79#define LONG_RUN_FAIL_MASK_SET(dst, src) \
80 ((dst & ~BIT(5)) | (((u32)src << 5) & BIT(5)))
81#define RUN_FAIL_MASK_SET(dst, src) \
82 ((dst & ~BIT(4)) | (((u32)src << 4) & BIT(4)))
83#define NOISE_FAIL_MASK_SET(dst, src) \
84 ((dst & ~BIT(3)) | (((u32)src << 3) & BIT(3)))
85#define STUCK_OUT_MASK_SET(dst, src) \
86 ((dst & ~BIT(2)) | (((u32)src << 2) & BIT(2)))
87#define SHUTDOWN_OFLO_MASK_SET(dst, src) \
88 ((dst & ~BIT(1)) | (((u32)src << 1) & BIT(1)))
89
90struct xgene_rng_dev {
91 u32 irq;
92 void __iomem *csr_base;
93 u32 revision;
94 u32 datum_size;
95 u32 failure_cnt; /* Failure count last minute */
96 unsigned long failure_ts;/* First failure timestamp */
97 struct timer_list failure_timer;
98 struct device *dev;
99 struct clk *clk;
100};
101
102static void xgene_rng_expired_timer(unsigned long arg)
103{
104 struct xgene_rng_dev *ctx = (struct xgene_rng_dev *) arg;
105
106 /* Clear failure counter as timer expired */
107 disable_irq(ctx->irq);
108 ctx->failure_cnt = 0;
109 del_timer(&ctx->failure_timer);
110 enable_irq(ctx->irq);
111}
112
113static void xgene_rng_start_timer(struct xgene_rng_dev *ctx)
114{
115 ctx->failure_timer.data = (unsigned long) ctx;
116 ctx->failure_timer.function = xgene_rng_expired_timer;
117 ctx->failure_timer.expires = jiffies + 120 * HZ;
118 add_timer(&ctx->failure_timer);
119}
120
121/*
122 * Initialize or reinit free running oscillators (FROs)
123 */
124static void xgene_rng_init_fro(struct xgene_rng_dev *ctx, u32 fro_val)
125{
126 writel(fro_val, ctx->csr_base + RNG_FRODETUNE);
127 writel(0x00000000, ctx->csr_base + RNG_ALARMMASK);
128 writel(0x00000000, ctx->csr_base + RNG_ALARMSTOP);
129 writel(0xFFFFFFFF, ctx->csr_base + RNG_FROENABLE);
130}
131
132static void xgene_rng_chk_overflow(struct xgene_rng_dev *ctx)
133{
134 u32 val;
135
136 val = readl(ctx->csr_base + RNG_INTR_STS_ACK);
137 if (val & MONOBIT_FAIL_MASK)
138 /*
139 * LFSR detected an out-of-bounds number of 1s after
140 * checking 20,000 bits (test T1 as specified in the
141 * AIS-31 standard)
142 */
143 dev_err(ctx->dev, "test monobit failure error 0x%08X\n", val);
144 if (val & POKER_FAIL_MASK)
145 /*
146 * LFSR detected an out-of-bounds value in at least one
147 * of the 16 poker_count_X counters or an out of bounds sum
148 * of squares value after checking 20,000 bits (test T2 as
149 * specified in the AIS-31 standard)
150 */
151 dev_err(ctx->dev, "test poker failure error 0x%08X\n", val);
152 if (val & LONG_RUN_FAIL_MASK)
153 /*
154 * LFSR detected a sequence of 34 identical bits
155 * (test T4 as specified in the AIS-31 standard)
156 */
157 dev_err(ctx->dev, "test long run failure error 0x%08X\n", val);
158 if (val & RUN_FAIL_MASK)
159 /*
160 * LFSR detected an outof-bounds value for at least one
161 * of the running counters after checking 20,000 bits
162 * (test T3 as specified in the AIS-31 standard)
163 */
164 dev_err(ctx->dev, "test run failure error 0x%08X\n", val);
165 if (val & NOISE_FAIL_MASK)
166 /* LFSR detected a sequence of 48 identical bits */
167 dev_err(ctx->dev, "noise failure error 0x%08X\n", val);
168 if (val & STUCK_OUT_MASK)
169 /*
170 * Detected output data registers generated same value twice
171 * in a row
172 */
173 dev_err(ctx->dev, "stuck out failure error 0x%08X\n", val);
174
175 if (val & SHUTDOWN_OFLO_MASK) {
176 u32 frostopped;
177
178 /* FROs shut down after a second error event. Try recover. */
179 if (++ctx->failure_cnt == 1) {
180 /* 1st time, just recover */
181 ctx->failure_ts = jiffies;
182 frostopped = readl(ctx->csr_base + RNG_ALARMSTOP);
183 xgene_rng_init_fro(ctx, frostopped);
184
185 /*
186 * We must start a timer to clear out this error
187 * in case the system timer wrap around
188 */
189 xgene_rng_start_timer(ctx);
190 } else {
191 /* 2nd time failure in lesser than 1 minute? */
192 if (time_after(ctx->failure_ts + 60 * HZ, jiffies)) {
193 dev_err(ctx->dev,
194 "FRO shutdown failure error 0x%08X\n",
195 val);
196 } else {
197 /* 2nd time failure after 1 minutes, recover */
198 ctx->failure_ts = jiffies;
199 ctx->failure_cnt = 1;
200 /*
201 * We must start a timer to clear out this
202 * error in case the system timer wrap
203 * around
204 */
205 xgene_rng_start_timer(ctx);
206 }
207 frostopped = readl(ctx->csr_base + RNG_ALARMSTOP);
208 xgene_rng_init_fro(ctx, frostopped);
209 }
210 }
211 /* Clear them all */
212 writel(val, ctx->csr_base + RNG_INTR_STS_ACK);
213}
214
215static irqreturn_t xgene_rng_irq_handler(int irq, void *id)
216{
217 struct xgene_rng_dev *ctx = (struct xgene_rng_dev *) id;
218
219 /* RNG Alarm Counter overflow */
220 xgene_rng_chk_overflow(ctx);
221
222 return IRQ_HANDLED;
223}
224
225static int xgene_rng_data_present(struct hwrng *rng, int wait)
226{
227 struct xgene_rng_dev *ctx = (struct xgene_rng_dev *) rng->priv;
228 u32 i, val = 0;
229
230 for (i = 0; i < XGENE_RNG_RETRY_COUNT; i++) {
231 val = readl(ctx->csr_base + RNG_INTR_STS_ACK);
232 if ((val & READY_MASK) || !wait)
233 break;
234 udelay(XGENE_RNG_RETRY_INTERVAL);
235 }
236
237 return (val & READY_MASK);
238}
239
240static int xgene_rng_data_read(struct hwrng *rng, u32 *data)
241{
242 struct xgene_rng_dev *ctx = (struct xgene_rng_dev *) rng->priv;
243 int i;
244
245 for (i = 0; i < ctx->datum_size; i++)
246 data[i] = readl(ctx->csr_base + RNG_INOUT_0 + i * 4);
247
248 /* Clear ready bit to start next transaction */
249 writel(READY_MASK, ctx->csr_base + RNG_INTR_STS_ACK);
250
251 return ctx->datum_size << 2;
252}
253
254static void xgene_rng_init_internal(struct xgene_rng_dev *ctx)
255{
256 u32 val;
257
258 writel(0x00000000, ctx->csr_base + RNG_CONTROL);
259
260 val = MAX_REFILL_CYCLES_SET(0, 10);
261 val = MIN_REFILL_CYCLES_SET(val, 10);
262 writel(val, ctx->csr_base + RNG_CONFIG);
263
264 val = ALARM_THRESHOLD_SET(0, 0xFF);
265 writel(val, ctx->csr_base + RNG_ALARMCNT);
266
267 xgene_rng_init_fro(ctx, 0);
268
269 writel(MONOBIT_FAIL_MASK |
270 POKER_FAIL_MASK |
271 LONG_RUN_FAIL_MASK |
272 RUN_FAIL_MASK |
273 NOISE_FAIL_MASK |
274 STUCK_OUT_MASK |
275 SHUTDOWN_OFLO_MASK |
276 READY_MASK, ctx->csr_base + RNG_INTR_STS_ACK);
277
278 val = ENABLE_RNG_SET(0, 1);
279 val = MONOBIT_FAIL_MASK_SET(val, 1);
280 val = POKER_FAIL_MASK_SET(val, 1);
281 val = LONG_RUN_FAIL_MASK_SET(val, 1);
282 val = RUN_FAIL_MASK_SET(val, 1);
283 val = NOISE_FAIL_MASK_SET(val, 1);
284 val = STUCK_OUT_MASK_SET(val, 1);
285 val = SHUTDOWN_OFLO_MASK_SET(val, 1);
286 writel(val, ctx->csr_base + RNG_CONTROL);
287}
288
289static int xgene_rng_init(struct hwrng *rng)
290{
291 struct xgene_rng_dev *ctx = (struct xgene_rng_dev *) rng->priv;
292
293 ctx->failure_cnt = 0;
294 init_timer(&ctx->failure_timer);
295
296 ctx->revision = readl(ctx->csr_base + RNG_EIP_REV);
297
298 dev_dbg(ctx->dev, "Rev %d.%d.%d\n",
299 MAJOR_HW_REV_RD(ctx->revision),
300 MINOR_HW_REV_RD(ctx->revision),
301 HW_PATCH_LEVEL_RD(ctx->revision));
302
303 dev_dbg(ctx->dev, "Options 0x%08X",
304 readl(ctx->csr_base + RNG_OPTIONS));
305
306 xgene_rng_init_internal(ctx);
307
308 ctx->datum_size = RNG_MAX_DATUM;
309
310 return 0;
311}
312
313static struct hwrng xgene_rng_func = {
314 .name = "xgene-rng",
315 .init = xgene_rng_init,
316 .data_present = xgene_rng_data_present,
317 .data_read = xgene_rng_data_read,
318};
319
320static int xgene_rng_probe(struct platform_device *pdev)
321{
322 struct resource *res;
323 struct xgene_rng_dev *ctx;
324 int rc = 0;
325
326 ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL);
327 if (!ctx)
328 return -ENOMEM;
329
330 ctx->dev = &pdev->dev;
331 platform_set_drvdata(pdev, ctx);
332
333 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
334 ctx->csr_base = devm_ioremap_resource(&pdev->dev, res);
335 if (IS_ERR(ctx->csr_base))
336 return PTR_ERR(ctx->csr_base);
337
338 ctx->irq = platform_get_irq(pdev, 0);
339 if (ctx->irq < 0) {
340 dev_err(&pdev->dev, "No IRQ resource\n");
341 return ctx->irq;
342 }
343
344 dev_dbg(&pdev->dev, "APM X-Gene RNG BASE %p ALARM IRQ %d",
345 ctx->csr_base, ctx->irq);
346
347 rc = devm_request_irq(&pdev->dev, ctx->irq, xgene_rng_irq_handler, 0,
348 dev_name(&pdev->dev), ctx);
349 if (rc) {
350 dev_err(&pdev->dev, "Could not request RNG alarm IRQ\n");
351 return rc;
352 }
353
354 /* Enable IP clock */
355 ctx->clk = devm_clk_get(&pdev->dev, NULL);
356 if (IS_ERR(ctx->clk)) {
357 dev_warn(&pdev->dev, "Couldn't get the clock for RNG\n");
358 } else {
359 rc = clk_prepare_enable(ctx->clk);
360 if (rc) {
361 dev_warn(&pdev->dev,
362 "clock prepare enable failed for RNG");
363 return rc;
364 }
365 }
366
367 xgene_rng_func.priv = (unsigned long) ctx;
368
369 rc = hwrng_register(&xgene_rng_func);
370 if (rc) {
371 dev_err(&pdev->dev, "RNG registering failed error %d\n", rc);
372 if (!IS_ERR(ctx->clk))
373 clk_disable_unprepare(ctx->clk);
374 return rc;
375 }
376
377 rc = device_init_wakeup(&pdev->dev, 1);
378 if (rc) {
379 dev_err(&pdev->dev, "RNG device_init_wakeup failed error %d\n",
380 rc);
381 if (!IS_ERR(ctx->clk))
382 clk_disable_unprepare(ctx->clk);
383 hwrng_unregister(&xgene_rng_func);
384 return rc;
385 }
386
387 return 0;
388}
389
390static int xgene_rng_remove(struct platform_device *pdev)
391{
392 struct xgene_rng_dev *ctx = platform_get_drvdata(pdev);
393 int rc;
394
395 rc = device_init_wakeup(&pdev->dev, 0);
396 if (rc)
397 dev_err(&pdev->dev, "RNG init wakeup failed error %d\n", rc);
398 if (!IS_ERR(ctx->clk))
399 clk_disable_unprepare(ctx->clk);
400 hwrng_unregister(&xgene_rng_func);
401
402 return rc;
403}
404
405static const struct of_device_id xgene_rng_of_match[] = {
406 { .compatible = "apm,xgene-rng" },
407 { }
408};
409
410MODULE_DEVICE_TABLE(of, xgene_rng_of_match);
411
412static struct platform_driver xgene_rng_driver = {
413 .probe = xgene_rng_probe,
414 .remove = xgene_rng_remove,
415 .driver = {
416 .name = "xgene-rng",
417 .of_match_table = xgene_rng_of_match,
418 },
419};
420
421module_platform_driver(xgene_rng_driver);
422MODULE_DESCRIPTION("APM X-Gene RNG driver");
423MODULE_LICENSE("GPL");
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index b464d03ebf40..f347ab7eea95 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -836,8 +836,9 @@ static int ahash_update_ctx(struct ahash_request *req)
836 edesc->sec4_sg + sec4_sg_src_index, 836 edesc->sec4_sg + sec4_sg_src_index,
837 chained); 837 chained);
838 if (*next_buflen) { 838 if (*next_buflen) {
839 sg_copy_part(next_buf, req->src, to_hash - 839 scatterwalk_map_and_copy(next_buf, req->src,
840 *buflen, req->nbytes); 840 to_hash - *buflen,
841 *next_buflen, 0);
841 state->current_buf = !state->current_buf; 842 state->current_buf = !state->current_buf;
842 } 843 }
843 } else { 844 } else {
@@ -878,7 +879,8 @@ static int ahash_update_ctx(struct ahash_request *req)
878 kfree(edesc); 879 kfree(edesc);
879 } 880 }
880 } else if (*next_buflen) { 881 } else if (*next_buflen) {
881 sg_copy(buf + *buflen, req->src, req->nbytes); 882 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
883 req->nbytes, 0);
882 *buflen = *next_buflen; 884 *buflen = *next_buflen;
883 *next_buflen = last_buflen; 885 *next_buflen = last_buflen;
884 } 886 }
@@ -1262,8 +1264,9 @@ static int ahash_update_no_ctx(struct ahash_request *req)
1262 src_map_to_sec4_sg(jrdev, req->src, src_nents, 1264 src_map_to_sec4_sg(jrdev, req->src, src_nents,
1263 edesc->sec4_sg + 1, chained); 1265 edesc->sec4_sg + 1, chained);
1264 if (*next_buflen) { 1266 if (*next_buflen) {
1265 sg_copy_part(next_buf, req->src, to_hash - *buflen, 1267 scatterwalk_map_and_copy(next_buf, req->src,
1266 req->nbytes); 1268 to_hash - *buflen,
1269 *next_buflen, 0);
1267 state->current_buf = !state->current_buf; 1270 state->current_buf = !state->current_buf;
1268 } 1271 }
1269 1272
@@ -1304,7 +1307,8 @@ static int ahash_update_no_ctx(struct ahash_request *req)
1304 kfree(edesc); 1307 kfree(edesc);
1305 } 1308 }
1306 } else if (*next_buflen) { 1309 } else if (*next_buflen) {
1307 sg_copy(buf + *buflen, req->src, req->nbytes); 1310 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
1311 req->nbytes, 0);
1308 *buflen = *next_buflen; 1312 *buflen = *next_buflen;
1309 *next_buflen = 0; 1313 *next_buflen = 0;
1310 } 1314 }
@@ -1413,9 +1417,9 @@ static int ahash_update_first(struct ahash_request *req)
1413 struct device *jrdev = ctx->jrdev; 1417 struct device *jrdev = ctx->jrdev;
1414 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | 1418 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1415 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; 1419 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1416 u8 *next_buf = state->buf_0 + state->current_buf * 1420 u8 *next_buf = state->current_buf ? state->buf_1 : state->buf_0;
1417 CAAM_MAX_HASH_BLOCK_SIZE; 1421 int *next_buflen = state->current_buf ?
1418 int *next_buflen = &state->buflen_0 + state->current_buf; 1422 &state->buflen_1 : &state->buflen_0;
1419 int to_hash; 1423 int to_hash;
1420 u32 *sh_desc = ctx->sh_desc_update_first, *desc; 1424 u32 *sh_desc = ctx->sh_desc_update_first, *desc;
1421 dma_addr_t ptr = ctx->sh_desc_update_first_dma; 1425 dma_addr_t ptr = ctx->sh_desc_update_first_dma;
@@ -1476,7 +1480,8 @@ static int ahash_update_first(struct ahash_request *req)
1476 } 1480 }
1477 1481
1478 if (*next_buflen) 1482 if (*next_buflen)
1479 sg_copy_part(next_buf, req->src, to_hash, req->nbytes); 1483 scatterwalk_map_and_copy(next_buf, req->src, to_hash,
1484 *next_buflen, 0);
1480 1485
1481 sh_len = desc_len(sh_desc); 1486 sh_len = desc_len(sh_desc);
1482 desc = edesc->hw_desc; 1487 desc = edesc->hw_desc;
@@ -1511,7 +1516,8 @@ static int ahash_update_first(struct ahash_request *req)
1511 state->update = ahash_update_no_ctx; 1516 state->update = ahash_update_no_ctx;
1512 state->finup = ahash_finup_no_ctx; 1517 state->finup = ahash_finup_no_ctx;
1513 state->final = ahash_final_no_ctx; 1518 state->final = ahash_final_no_ctx;
1514 sg_copy(next_buf, req->src, req->nbytes); 1519 scatterwalk_map_and_copy(next_buf, req->src, 0,
1520 req->nbytes, 0);
1515 } 1521 }
1516#ifdef DEBUG 1522#ifdef DEBUG
1517 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", 1523 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index 3cade79ea41e..31000c8c4a90 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -1,5 +1,4 @@
1/* 1/* * CAAM control-plane driver backend
2 * CAAM control-plane driver backend
3 * Controller-level driver, kernel property detection, initialization 2 * Controller-level driver, kernel property detection, initialization
4 * 3 *
5 * Copyright 2008-2012 Freescale Semiconductor, Inc. 4 * Copyright 2008-2012 Freescale Semiconductor, Inc.
@@ -81,38 +80,37 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc,
81 u32 *status) 80 u32 *status)
82{ 81{
83 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); 82 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
84 struct caam_full __iomem *topregs; 83 struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl;
84 struct caam_deco __iomem *deco = ctrlpriv->deco;
85 unsigned int timeout = 100000; 85 unsigned int timeout = 100000;
86 u32 deco_dbg_reg, flags; 86 u32 deco_dbg_reg, flags;
87 int i; 87 int i;
88 88
89 /* Set the bit to request direct access to DECO0 */
90 topregs = (struct caam_full __iomem *)ctrlpriv->ctrl;
91 89
92 if (ctrlpriv->virt_en == 1) { 90 if (ctrlpriv->virt_en == 1) {
93 setbits32(&topregs->ctrl.deco_rsr, DECORSR_JR0); 91 setbits32(&ctrl->deco_rsr, DECORSR_JR0);
94 92
95 while (!(rd_reg32(&topregs->ctrl.deco_rsr) & DECORSR_VALID) && 93 while (!(rd_reg32(&ctrl->deco_rsr) & DECORSR_VALID) &&
96 --timeout) 94 --timeout)
97 cpu_relax(); 95 cpu_relax();
98 96
99 timeout = 100000; 97 timeout = 100000;
100 } 98 }
101 99
102 setbits32(&topregs->ctrl.deco_rq, DECORR_RQD0ENABLE); 100 setbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE);
103 101
104 while (!(rd_reg32(&topregs->ctrl.deco_rq) & DECORR_DEN0) && 102 while (!(rd_reg32(&ctrl->deco_rq) & DECORR_DEN0) &&
105 --timeout) 103 --timeout)
106 cpu_relax(); 104 cpu_relax();
107 105
108 if (!timeout) { 106 if (!timeout) {
109 dev_err(ctrldev, "failed to acquire DECO 0\n"); 107 dev_err(ctrldev, "failed to acquire DECO 0\n");
110 clrbits32(&topregs->ctrl.deco_rq, DECORR_RQD0ENABLE); 108 clrbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE);
111 return -ENODEV; 109 return -ENODEV;
112 } 110 }
113 111
114 for (i = 0; i < desc_len(desc); i++) 112 for (i = 0; i < desc_len(desc); i++)
115 wr_reg32(&topregs->deco.descbuf[i], *(desc + i)); 113 wr_reg32(&deco->descbuf[i], *(desc + i));
116 114
117 flags = DECO_JQCR_WHL; 115 flags = DECO_JQCR_WHL;
118 /* 116 /*
@@ -123,11 +121,11 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc,
123 flags |= DECO_JQCR_FOUR; 121 flags |= DECO_JQCR_FOUR;
124 122
125 /* Instruct the DECO to execute it */ 123 /* Instruct the DECO to execute it */
126 wr_reg32(&topregs->deco.jr_ctl_hi, flags); 124 wr_reg32(&deco->jr_ctl_hi, flags);
127 125
128 timeout = 10000000; 126 timeout = 10000000;
129 do { 127 do {
130 deco_dbg_reg = rd_reg32(&topregs->deco.desc_dbg); 128 deco_dbg_reg = rd_reg32(&deco->desc_dbg);
131 /* 129 /*
132 * If an error occured in the descriptor, then 130 * If an error occured in the descriptor, then
133 * the DECO status field will be set to 0x0D 131 * the DECO status field will be set to 0x0D
@@ -138,14 +136,14 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc,
138 cpu_relax(); 136 cpu_relax();
139 } while ((deco_dbg_reg & DESC_DBG_DECO_STAT_VALID) && --timeout); 137 } while ((deco_dbg_reg & DESC_DBG_DECO_STAT_VALID) && --timeout);
140 138
141 *status = rd_reg32(&topregs->deco.op_status_hi) & 139 *status = rd_reg32(&deco->op_status_hi) &
142 DECO_OP_STATUS_HI_ERR_MASK; 140 DECO_OP_STATUS_HI_ERR_MASK;
143 141
144 if (ctrlpriv->virt_en == 1) 142 if (ctrlpriv->virt_en == 1)
145 clrbits32(&topregs->ctrl.deco_rsr, DECORSR_JR0); 143 clrbits32(&ctrl->deco_rsr, DECORSR_JR0);
146 144
147 /* Mark the DECO as free */ 145 /* Mark the DECO as free */
148 clrbits32(&topregs->ctrl.deco_rq, DECORR_RQD0ENABLE); 146 clrbits32(&ctrl->deco_rq, DECORR_RQD0ENABLE);
149 147
150 if (!timeout) 148 if (!timeout)
151 return -EAGAIN; 149 return -EAGAIN;
@@ -176,13 +174,13 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask,
176 int gen_sk) 174 int gen_sk)
177{ 175{
178 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); 176 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
179 struct caam_full __iomem *topregs; 177 struct caam_ctrl __iomem *ctrl;
180 struct rng4tst __iomem *r4tst; 178 struct rng4tst __iomem *r4tst;
181 u32 *desc, status, rdsta_val; 179 u32 *desc, status, rdsta_val;
182 int ret = 0, sh_idx; 180 int ret = 0, sh_idx;
183 181
184 topregs = (struct caam_full __iomem *)ctrlpriv->ctrl; 182 ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl;
185 r4tst = &topregs->ctrl.r4tst[0]; 183 r4tst = &ctrl->r4tst[0];
186 184
187 desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL); 185 desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL);
188 if (!desc) 186 if (!desc)
@@ -212,12 +210,11 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask,
212 * CAAM eras), then try again. 210 * CAAM eras), then try again.
213 */ 211 */
214 rdsta_val = 212 rdsta_val =
215 rd_reg32(&topregs->ctrl.r4tst[0].rdsta) & RDSTA_IFMASK; 213 rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK;
216 if (status || !(rdsta_val & (1 << sh_idx))) 214 if (status || !(rdsta_val & (1 << sh_idx)))
217 ret = -EAGAIN; 215 ret = -EAGAIN;
218 if (ret) 216 if (ret)
219 break; 217 break;
220
221 dev_info(ctrldev, "Instantiated RNG4 SH%d\n", sh_idx); 218 dev_info(ctrldev, "Instantiated RNG4 SH%d\n", sh_idx);
222 /* Clear the contents before recreating the descriptor */ 219 /* Clear the contents before recreating the descriptor */
223 memset(desc, 0x00, CAAM_CMD_SZ * 7); 220 memset(desc, 0x00, CAAM_CMD_SZ * 7);
@@ -285,12 +282,12 @@ static int caam_remove(struct platform_device *pdev)
285{ 282{
286 struct device *ctrldev; 283 struct device *ctrldev;
287 struct caam_drv_private *ctrlpriv; 284 struct caam_drv_private *ctrlpriv;
288 struct caam_full __iomem *topregs; 285 struct caam_ctrl __iomem *ctrl;
289 int ring, ret = 0; 286 int ring, ret = 0;
290 287
291 ctrldev = &pdev->dev; 288 ctrldev = &pdev->dev;
292 ctrlpriv = dev_get_drvdata(ctrldev); 289 ctrlpriv = dev_get_drvdata(ctrldev);
293 topregs = (struct caam_full __iomem *)ctrlpriv->ctrl; 290 ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl;
294 291
295 /* Remove platform devices for JobRs */ 292 /* Remove platform devices for JobRs */
296 for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) { 293 for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) {
@@ -308,7 +305,7 @@ static int caam_remove(struct platform_device *pdev)
308#endif 305#endif
309 306
310 /* Unmap controller region */ 307 /* Unmap controller region */
311 iounmap(&topregs->ctrl); 308 iounmap(&ctrl);
312 309
313 return ret; 310 return ret;
314} 311}
@@ -323,12 +320,12 @@ static void kick_trng(struct platform_device *pdev, int ent_delay)
323{ 320{
324 struct device *ctrldev = &pdev->dev; 321 struct device *ctrldev = &pdev->dev;
325 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); 322 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
326 struct caam_full __iomem *topregs; 323 struct caam_ctrl __iomem *ctrl;
327 struct rng4tst __iomem *r4tst; 324 struct rng4tst __iomem *r4tst;
328 u32 val; 325 u32 val;
329 326
330 topregs = (struct caam_full __iomem *)ctrlpriv->ctrl; 327 ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl;
331 r4tst = &topregs->ctrl.r4tst[0]; 328 r4tst = &ctrl->r4tst[0];
332 329
333 /* put RNG4 into program mode */ 330 /* put RNG4 into program mode */
334 setbits32(&r4tst->rtmctl, RTMCTL_PRGM); 331 setbits32(&r4tst->rtmctl, RTMCTL_PRGM);
@@ -355,10 +352,19 @@ static void kick_trng(struct platform_device *pdev, int ent_delay)
355 wr_reg32(&r4tst->rtsdctl, val); 352 wr_reg32(&r4tst->rtsdctl, val);
356 /* min. freq. count, equal to 1/4 of the entropy sample length */ 353 /* min. freq. count, equal to 1/4 of the entropy sample length */
357 wr_reg32(&r4tst->rtfrqmin, ent_delay >> 2); 354 wr_reg32(&r4tst->rtfrqmin, ent_delay >> 2);
358 /* max. freq. count, equal to 8 times the entropy sample length */ 355 /* disable maximum frequency count */
359 wr_reg32(&r4tst->rtfrqmax, ent_delay << 3); 356 wr_reg32(&r4tst->rtfrqmax, RTFRQMAX_DISABLE);
357 /* read the control register */
358 val = rd_reg32(&r4tst->rtmctl);
359 /*
360 * select raw sampling in both entropy shifter
361 * and statistical checker
362 */
363 setbits32(&val, RTMCTL_SAMP_MODE_RAW_ES_SC);
360 /* put RNG4 into run mode */ 364 /* put RNG4 into run mode */
361 clrbits32(&r4tst->rtmctl, RTMCTL_PRGM); 365 clrbits32(&val, RTMCTL_PRGM);
366 /* write back the control register */
367 wr_reg32(&r4tst->rtmctl, val);
362} 368}
363 369
364/** 370/**
@@ -387,13 +393,14 @@ static int caam_probe(struct platform_device *pdev)
387 struct device *dev; 393 struct device *dev;
388 struct device_node *nprop, *np; 394 struct device_node *nprop, *np;
389 struct caam_ctrl __iomem *ctrl; 395 struct caam_ctrl __iomem *ctrl;
390 struct caam_full __iomem *topregs;
391 struct caam_drv_private *ctrlpriv; 396 struct caam_drv_private *ctrlpriv;
392#ifdef CONFIG_DEBUG_FS 397#ifdef CONFIG_DEBUG_FS
393 struct caam_perfmon *perfmon; 398 struct caam_perfmon *perfmon;
394#endif 399#endif
395 u32 scfgr, comp_params; 400 u32 scfgr, comp_params;
396 u32 cha_vid_ls; 401 u32 cha_vid_ls;
402 int pg_size;
403 int BLOCK_OFFSET = 0;
397 404
398 ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(struct caam_drv_private), 405 ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(struct caam_drv_private),
399 GFP_KERNEL); 406 GFP_KERNEL);
@@ -412,10 +419,27 @@ static int caam_probe(struct platform_device *pdev)
412 dev_err(dev, "caam: of_iomap() failed\n"); 419 dev_err(dev, "caam: of_iomap() failed\n");
413 return -ENOMEM; 420 return -ENOMEM;
414 } 421 }
415 ctrlpriv->ctrl = (struct caam_ctrl __force *)ctrl; 422 /* Finding the page size for using the CTPR_MS register */
423 comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms);
424 pg_size = (comp_params & CTPR_MS_PG_SZ_MASK) >> CTPR_MS_PG_SZ_SHIFT;
416 425
417 /* topregs used to derive pointers to CAAM sub-blocks only */ 426 /* Allocating the BLOCK_OFFSET based on the supported page size on
418 topregs = (struct caam_full __iomem *)ctrl; 427 * the platform
428 */
429 if (pg_size == 0)
430 BLOCK_OFFSET = PG_SIZE_4K;
431 else
432 BLOCK_OFFSET = PG_SIZE_64K;
433
434 ctrlpriv->ctrl = (struct caam_ctrl __force *)ctrl;
435 ctrlpriv->assure = (struct caam_assurance __force *)
436 ((uint8_t *)ctrl +
437 BLOCK_OFFSET * ASSURE_BLOCK_NUMBER
438 );
439 ctrlpriv->deco = (struct caam_deco __force *)
440 ((uint8_t *)ctrl +
441 BLOCK_OFFSET * DECO_BLOCK_NUMBER
442 );
419 443
420 /* Get the IRQ of the controller (for security violations only) */ 444 /* Get the IRQ of the controller (for security violations only) */
421 ctrlpriv->secvio_irq = irq_of_parse_and_map(nprop, 0); 445 ctrlpriv->secvio_irq = irq_of_parse_and_map(nprop, 0);
@@ -424,15 +448,14 @@ static int caam_probe(struct platform_device *pdev)
424 * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel, 448 * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel,
425 * long pointers in master configuration register 449 * long pointers in master configuration register
426 */ 450 */
427 setbits32(&topregs->ctrl.mcr, MCFGR_WDENABLE | 451 setbits32(&ctrl->mcr, MCFGR_WDENABLE |
428 (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); 452 (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0));
429 453
430 /* 454 /*
431 * Read the Compile Time paramters and SCFGR to determine 455 * Read the Compile Time paramters and SCFGR to determine
432 * if Virtualization is enabled for this platform 456 * if Virtualization is enabled for this platform
433 */ 457 */
434 comp_params = rd_reg32(&topregs->ctrl.perfmon.comp_parms_ms); 458 scfgr = rd_reg32(&ctrl->scfgr);
435 scfgr = rd_reg32(&topregs->ctrl.scfgr);
436 459
437 ctrlpriv->virt_en = 0; 460 ctrlpriv->virt_en = 0;
438 if (comp_params & CTPR_MS_VIRT_EN_INCL) { 461 if (comp_params & CTPR_MS_VIRT_EN_INCL) {
@@ -450,7 +473,7 @@ static int caam_probe(struct platform_device *pdev)
450 } 473 }
451 474
452 if (ctrlpriv->virt_en == 1) 475 if (ctrlpriv->virt_en == 1)
453 setbits32(&topregs->ctrl.jrstart, JRSTART_JR0_START | 476 setbits32(&ctrl->jrstart, JRSTART_JR0_START |
454 JRSTART_JR1_START | JRSTART_JR2_START | 477 JRSTART_JR1_START | JRSTART_JR2_START |
455 JRSTART_JR3_START); 478 JRSTART_JR3_START);
456 479
@@ -477,7 +500,7 @@ static int caam_probe(struct platform_device *pdev)
477 sizeof(struct platform_device *) * rspec, 500 sizeof(struct platform_device *) * rspec,
478 GFP_KERNEL); 501 GFP_KERNEL);
479 if (ctrlpriv->jrpdev == NULL) { 502 if (ctrlpriv->jrpdev == NULL) {
480 iounmap(&topregs->ctrl); 503 iounmap(&ctrl);
481 return -ENOMEM; 504 return -ENOMEM;
482 } 505 }
483 506
@@ -493,18 +516,26 @@ static int caam_probe(struct platform_device *pdev)
493 ring); 516 ring);
494 continue; 517 continue;
495 } 518 }
519 ctrlpriv->jr[ring] = (struct caam_job_ring __force *)
520 ((uint8_t *)ctrl +
521 (ring + JR_BLOCK_NUMBER) *
522 BLOCK_OFFSET
523 );
496 ctrlpriv->total_jobrs++; 524 ctrlpriv->total_jobrs++;
497 ring++; 525 ring++;
498 } 526 }
499 527
500 /* Check to see if QI present. If so, enable */ 528 /* Check to see if QI present. If so, enable */
501 ctrlpriv->qi_present = 529 ctrlpriv->qi_present =
502 !!(rd_reg32(&topregs->ctrl.perfmon.comp_parms_ms) & 530 !!(rd_reg32(&ctrl->perfmon.comp_parms_ms) &
503 CTPR_MS_QI_MASK); 531 CTPR_MS_QI_MASK);
504 if (ctrlpriv->qi_present) { 532 if (ctrlpriv->qi_present) {
505 ctrlpriv->qi = (struct caam_queue_if __force *)&topregs->qi; 533 ctrlpriv->qi = (struct caam_queue_if __force *)
534 ((uint8_t *)ctrl +
535 BLOCK_OFFSET * QI_BLOCK_NUMBER
536 );
506 /* This is all that's required to physically enable QI */ 537 /* This is all that's required to physically enable QI */
507 wr_reg32(&topregs->qi.qi_control_lo, QICTL_DQEN); 538 wr_reg32(&ctrlpriv->qi->qi_control_lo, QICTL_DQEN);
508 } 539 }
509 540
510 /* If no QI and no rings specified, quit and go home */ 541 /* If no QI and no rings specified, quit and go home */
@@ -514,7 +545,7 @@ static int caam_probe(struct platform_device *pdev)
514 return -ENOMEM; 545 return -ENOMEM;
515 } 546 }
516 547
517 cha_vid_ls = rd_reg32(&topregs->ctrl.perfmon.cha_id_ls); 548 cha_vid_ls = rd_reg32(&ctrl->perfmon.cha_id_ls);
518 549
519 /* 550 /*
520 * If SEC has RNG version >= 4 and RNG state handle has not been 551 * If SEC has RNG version >= 4 and RNG state handle has not been
@@ -522,7 +553,7 @@ static int caam_probe(struct platform_device *pdev)
522 */ 553 */
523 if ((cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) { 554 if ((cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) {
524 ctrlpriv->rng4_sh_init = 555 ctrlpriv->rng4_sh_init =
525 rd_reg32(&topregs->ctrl.r4tst[0].rdsta); 556 rd_reg32(&ctrl->r4tst[0].rdsta);
526 /* 557 /*
527 * If the secure keys (TDKEK, JDKEK, TDSK), were already 558 * If the secure keys (TDKEK, JDKEK, TDSK), were already
528 * generated, signal this to the function that is instantiating 559 * generated, signal this to the function that is instantiating
@@ -533,7 +564,7 @@ static int caam_probe(struct platform_device *pdev)
533 ctrlpriv->rng4_sh_init &= RDSTA_IFMASK; 564 ctrlpriv->rng4_sh_init &= RDSTA_IFMASK;
534 do { 565 do {
535 int inst_handles = 566 int inst_handles =
536 rd_reg32(&topregs->ctrl.r4tst[0].rdsta) & 567 rd_reg32(&ctrl->r4tst[0].rdsta) &
537 RDSTA_IFMASK; 568 RDSTA_IFMASK;
538 /* 569 /*
539 * If either SH were instantiated by somebody else 570 * If either SH were instantiated by somebody else
@@ -544,6 +575,9 @@ static int caam_probe(struct platform_device *pdev)
544 * the TRNG parameters. 575 * the TRNG parameters.
545 */ 576 */
546 if (!(ctrlpriv->rng4_sh_init || inst_handles)) { 577 if (!(ctrlpriv->rng4_sh_init || inst_handles)) {
578 dev_info(dev,
579 "Entropy delay = %u\n",
580 ent_delay);
547 kick_trng(pdev, ent_delay); 581 kick_trng(pdev, ent_delay);
548 ent_delay += 400; 582 ent_delay += 400;
549 } 583 }
@@ -556,6 +590,12 @@ static int caam_probe(struct platform_device *pdev)
556 */ 590 */
557 ret = instantiate_rng(dev, inst_handles, 591 ret = instantiate_rng(dev, inst_handles,
558 gen_sk); 592 gen_sk);
593 if (ret == -EAGAIN)
594 /*
595 * if here, the loop will rerun,
596 * so don't hog the CPU
597 */
598 cpu_relax();
559 } while ((ret == -EAGAIN) && (ent_delay < RTSDCTL_ENT_DLY_MAX)); 599 } while ((ret == -EAGAIN) && (ent_delay < RTSDCTL_ENT_DLY_MAX));
560 if (ret) { 600 if (ret) {
561 dev_err(dev, "failed to instantiate RNG"); 601 dev_err(dev, "failed to instantiate RNG");
@@ -569,13 +609,13 @@ static int caam_probe(struct platform_device *pdev)
569 ctrlpriv->rng4_sh_init = ~ctrlpriv->rng4_sh_init & RDSTA_IFMASK; 609 ctrlpriv->rng4_sh_init = ~ctrlpriv->rng4_sh_init & RDSTA_IFMASK;
570 610
571 /* Enable RDB bit so that RNG works faster */ 611 /* Enable RDB bit so that RNG works faster */
572 setbits32(&topregs->ctrl.scfgr, SCFGR_RDBENABLE); 612 setbits32(&ctrl->scfgr, SCFGR_RDBENABLE);
573 } 613 }
574 614
575 /* NOTE: RTIC detection ought to go here, around Si time */ 615 /* NOTE: RTIC detection ought to go here, around Si time */
576 616
577 caam_id = (u64)rd_reg32(&topregs->ctrl.perfmon.caam_id_ms) << 32 | 617 caam_id = (u64)rd_reg32(&ctrl->perfmon.caam_id_ms) << 32 |
578 (u64)rd_reg32(&topregs->ctrl.perfmon.caam_id_ls); 618 (u64)rd_reg32(&ctrl->perfmon.caam_id_ls);
579 619
580 /* Report "alive" for developer to see */ 620 /* Report "alive" for developer to see */
581 dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id, 621 dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id,
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index 97363db4e56e..89b94cc9e7a2 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -70,10 +70,11 @@ struct caam_drv_private {
70 struct platform_device *pdev; 70 struct platform_device *pdev;
71 71
72 /* Physical-presence section */ 72 /* Physical-presence section */
73 struct caam_ctrl *ctrl; /* controller region */ 73 struct caam_ctrl __iomem *ctrl; /* controller region */
74 struct caam_deco **deco; /* DECO/CCB views */ 74 struct caam_deco __iomem *deco; /* DECO/CCB views */
75 struct caam_assurance *ac; 75 struct caam_assurance __iomem *assure;
76 struct caam_queue_if *qi; /* QI control region */ 76 struct caam_queue_if __iomem *qi; /* QI control region */
77 struct caam_job_ring __iomem *jr[4]; /* JobR's register space */
77 78
78 /* 79 /*
79 * Detected geometry block. Filled in from device tree if powerpc, 80 * Detected geometry block. Filled in from device tree if powerpc,
diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h
index f48e344ffc39..378ddc17f60e 100644
--- a/drivers/crypto/caam/regs.h
+++ b/drivers/crypto/caam/regs.h
@@ -194,6 +194,8 @@ struct caam_perfmon {
194#define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT) 194#define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT)
195#define CTPR_MS_VIRT_EN_INCL 0x00000001 195#define CTPR_MS_VIRT_EN_INCL 0x00000001
196#define CTPR_MS_VIRT_EN_POR 0x00000002 196#define CTPR_MS_VIRT_EN_POR 0x00000002
197#define CTPR_MS_PG_SZ_MASK 0x10
198#define CTPR_MS_PG_SZ_SHIFT 4
197 u32 comp_parms_ms; /* CTPR - Compile Parameters Register */ 199 u32 comp_parms_ms; /* CTPR - Compile Parameters Register */
198 u32 comp_parms_ls; /* CTPR - Compile Parameters Register */ 200 u32 comp_parms_ls; /* CTPR - Compile Parameters Register */
199 u64 rsvd1[2]; 201 u64 rsvd1[2];
@@ -269,6 +271,16 @@ struct rngtst {
269/* RNG4 TRNG test registers */ 271/* RNG4 TRNG test registers */
270struct rng4tst { 272struct rng4tst {
271#define RTMCTL_PRGM 0x00010000 /* 1 -> program mode, 0 -> run mode */ 273#define RTMCTL_PRGM 0x00010000 /* 1 -> program mode, 0 -> run mode */
274#define RTMCTL_SAMP_MODE_VON_NEUMANN_ES_SC 0 /* use von Neumann data in
275 both entropy shifter and
276 statistical checker */
277#define RTMCTL_SAMP_MODE_RAW_ES_SC 1 /* use raw data in both
278 entropy shifter and
279 statistical checker */
280#define RTMCTL_SAMP_MODE_VON_NEUMANN_ES_RAW_SC 2 /* use von Neumann data in
281 entropy shifter, raw data
282 in statistical checker */
283#define RTMCTL_SAMP_MODE_INVALID 3 /* invalid combination */
272 u32 rtmctl; /* misc. control register */ 284 u32 rtmctl; /* misc. control register */
273 u32 rtscmisc; /* statistical check misc. register */ 285 u32 rtscmisc; /* statistical check misc. register */
274 u32 rtpkrrng; /* poker range register */ 286 u32 rtpkrrng; /* poker range register */
@@ -278,7 +290,7 @@ struct rng4tst {
278 }; 290 };
279#define RTSDCTL_ENT_DLY_SHIFT 16 291#define RTSDCTL_ENT_DLY_SHIFT 16
280#define RTSDCTL_ENT_DLY_MASK (0xffff << RTSDCTL_ENT_DLY_SHIFT) 292#define RTSDCTL_ENT_DLY_MASK (0xffff << RTSDCTL_ENT_DLY_SHIFT)
281#define RTSDCTL_ENT_DLY_MIN 1200 293#define RTSDCTL_ENT_DLY_MIN 3200
282#define RTSDCTL_ENT_DLY_MAX 12800 294#define RTSDCTL_ENT_DLY_MAX 12800
283 u32 rtsdctl; /* seed control register */ 295 u32 rtsdctl; /* seed control register */
284 union { 296 union {
@@ -286,6 +298,7 @@ struct rng4tst {
286 u32 rttotsam; /* PRGM=0: total samples register */ 298 u32 rttotsam; /* PRGM=0: total samples register */
287 }; 299 };
288 u32 rtfrqmin; /* frequency count min. limit register */ 300 u32 rtfrqmin; /* frequency count min. limit register */
301#define RTFRQMAX_DISABLE (1 << 20)
289 union { 302 union {
290 u32 rtfrqmax; /* PRGM=1: freq. count max. limit register */ 303 u32 rtfrqmax; /* PRGM=1: freq. count max. limit register */
291 u32 rtfrqcnt; /* PRGM=0: freq. count register */ 304 u32 rtfrqcnt; /* PRGM=0: freq. count register */
@@ -758,34 +771,10 @@ struct caam_deco {
758#define DECO_JQCR_WHL 0x20000000 771#define DECO_JQCR_WHL 0x20000000
759#define DECO_JQCR_FOUR 0x10000000 772#define DECO_JQCR_FOUR 0x10000000
760 773
761/* 774#define JR_BLOCK_NUMBER 1
762 * Current top-level view of memory map is: 775#define ASSURE_BLOCK_NUMBER 6
763 * 776#define QI_BLOCK_NUMBER 7
764 * 0x0000 - 0x0fff - CAAM Top-Level Control 777#define DECO_BLOCK_NUMBER 8
765 * 0x1000 - 0x1fff - Job Ring 0 778#define PG_SIZE_4K 0x1000
766 * 0x2000 - 0x2fff - Job Ring 1 779#define PG_SIZE_64K 0x10000
767 * 0x3000 - 0x3fff - Job Ring 2
768 * 0x4000 - 0x4fff - Job Ring 3
769 * 0x5000 - 0x5fff - (unused)
770 * 0x6000 - 0x6fff - Assurance Controller
771 * 0x7000 - 0x7fff - Queue Interface
772 * 0x8000 - 0x8fff - DECO-CCB 0
773 * 0x9000 - 0x9fff - DECO-CCB 1
774 * 0xa000 - 0xafff - DECO-CCB 2
775 * 0xb000 - 0xbfff - DECO-CCB 3
776 * 0xc000 - 0xcfff - DECO-CCB 4
777 *
778 * caam_full describes the full register view of CAAM if useful,
779 * although many configurations may choose to implement parts of
780 * the register map separately, in differing privilege regions
781 */
782struct caam_full {
783 struct caam_ctrl __iomem ctrl;
784 struct caam_job_ring jr[4];
785 u64 rsvd[512];
786 struct caam_assurance assure;
787 struct caam_queue_if qi;
788 struct caam_deco deco;
789};
790
791#endif /* REGS_H */ 780#endif /* REGS_H */
diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h
index b12ff85f4241..ce28a563effc 100644
--- a/drivers/crypto/caam/sg_sw_sec4.h
+++ b/drivers/crypto/caam/sg_sw_sec4.h
@@ -116,57 +116,3 @@ static int dma_unmap_sg_chained(struct device *dev, struct scatterlist *sg,
116 } 116 }
117 return nents; 117 return nents;
118} 118}
119
120/* Map SG page in kernel virtual address space and copy */
121static inline void sg_map_copy(u8 *dest, struct scatterlist *sg,
122 int len, int offset)
123{
124 u8 *mapped_addr;
125
126 /*
127 * Page here can be user-space pinned using get_user_pages
128 * Same must be kmapped before use and kunmapped subsequently
129 */
130 mapped_addr = kmap_atomic(sg_page(sg));
131 memcpy(dest, mapped_addr + offset, len);
132 kunmap_atomic(mapped_addr);
133}
134
135/* Copy from len bytes of sg to dest, starting from beginning */
136static inline void sg_copy(u8 *dest, struct scatterlist *sg, unsigned int len)
137{
138 struct scatterlist *current_sg = sg;
139 int cpy_index = 0, next_cpy_index = current_sg->length;
140
141 while (next_cpy_index < len) {
142 sg_map_copy(dest + cpy_index, current_sg, current_sg->length,
143 current_sg->offset);
144 current_sg = scatterwalk_sg_next(current_sg);
145 cpy_index = next_cpy_index;
146 next_cpy_index += current_sg->length;
147 }
148 if (cpy_index < len)
149 sg_map_copy(dest + cpy_index, current_sg, len-cpy_index,
150 current_sg->offset);
151}
152
153/* Copy sg data, from to_skip to end, to dest */
154static inline void sg_copy_part(u8 *dest, struct scatterlist *sg,
155 int to_skip, unsigned int end)
156{
157 struct scatterlist *current_sg = sg;
158 int sg_index, cpy_index, offset;
159
160 sg_index = current_sg->length;
161 while (sg_index <= to_skip) {
162 current_sg = scatterwalk_sg_next(current_sg);
163 sg_index += current_sg->length;
164 }
165 cpy_index = sg_index - to_skip;
166 offset = current_sg->offset + current_sg->length - cpy_index;
167 sg_map_copy(dest, current_sg, cpy_index, offset);
168 if (end - sg_index) {
169 current_sg = scatterwalk_sg_next(current_sg);
170 sg_copy(dest + cpy_index, current_sg, end - sg_index);
171 }
172}
diff --git a/drivers/crypto/mv_cesa.h b/drivers/crypto/mv_cesa.h
index 08fcb1116d90..9249d3ed184b 100644
--- a/drivers/crypto/mv_cesa.h
+++ b/drivers/crypto/mv_cesa.h
@@ -1,4 +1,5 @@
1#ifndef __MV_CRYPTO_H__ 1#ifndef __MV_CRYPTO_H__
2#define __MV_CRYPTO_H__
2 3
3#define DIGEST_INITIAL_VAL_A 0xdd00 4#define DIGEST_INITIAL_VAL_A 0xdd00
4#define DIGEST_INITIAL_VAL_B 0xdd04 5#define DIGEST_INITIAL_VAL_B 0xdd04
diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c
index 6a92284a86b2..244d73378f0e 100644
--- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c
+++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c
@@ -111,7 +111,7 @@ static int adf_chr_drv_create(void)
111 drv_device = device_create(adt_ctl_drv.drv_class, NULL, 111 drv_device = device_create(adt_ctl_drv.drv_class, NULL,
112 MKDEV(adt_ctl_drv.major, 0), 112 MKDEV(adt_ctl_drv.major, 0),
113 NULL, DEVICE_NAME); 113 NULL, DEVICE_NAME);
114 if (!drv_device) { 114 if (IS_ERR(drv_device)) {
115 pr_err("QAT: failed to create device\n"); 115 pr_err("QAT: failed to create device\n");
116 goto err_cdev_del; 116 goto err_cdev_del;
117 } 117 }
diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h
index f854bac276b0..c40546079981 100644
--- a/drivers/crypto/qat/qat_common/adf_transport_internal.h
+++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h
@@ -75,7 +75,7 @@ struct adf_etr_ring_data {
75 75
76struct adf_etr_bank_data { 76struct adf_etr_bank_data {
77 struct adf_etr_ring_data rings[ADF_ETR_MAX_RINGS_PER_BANK]; 77 struct adf_etr_ring_data rings[ADF_ETR_MAX_RINGS_PER_BANK];
78 struct tasklet_struct resp_hanlder; 78 struct tasklet_struct resp_handler;
79 void __iomem *csr_addr; 79 void __iomem *csr_addr;
80 struct adf_accel_dev *accel_dev; 80 struct adf_accel_dev *accel_dev;
81 uint32_t irq_coalesc_timer; 81 uint32_t irq_coalesc_timer;
diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c
index 59df48872955..3e26fa2b293f 100644
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_algs.c
@@ -105,7 +105,7 @@ struct qat_alg_cd {
105#define MAX_AUTH_STATE_SIZE sizeof(struct icp_qat_hw_auth_algo_blk) 105#define MAX_AUTH_STATE_SIZE sizeof(struct icp_qat_hw_auth_algo_blk)
106 106
107struct qat_auth_state { 107struct qat_auth_state {
108 uint8_t data[MAX_AUTH_STATE_SIZE]; 108 uint8_t data[MAX_AUTH_STATE_SIZE + 64];
109} __aligned(64); 109} __aligned(64);
110 110
111struct qat_alg_session_ctx { 111struct qat_alg_session_ctx {
@@ -113,10 +113,6 @@ struct qat_alg_session_ctx {
113 dma_addr_t enc_cd_paddr; 113 dma_addr_t enc_cd_paddr;
114 struct qat_alg_cd *dec_cd; 114 struct qat_alg_cd *dec_cd;
115 dma_addr_t dec_cd_paddr; 115 dma_addr_t dec_cd_paddr;
116 struct qat_auth_state *auth_hw_state_enc;
117 dma_addr_t auth_state_enc_paddr;
118 struct qat_auth_state *auth_hw_state_dec;
119 dma_addr_t auth_state_dec_paddr;
120 struct icp_qat_fw_la_bulk_req enc_fw_req_tmpl; 116 struct icp_qat_fw_la_bulk_req enc_fw_req_tmpl;
121 struct icp_qat_fw_la_bulk_req dec_fw_req_tmpl; 117 struct icp_qat_fw_la_bulk_req dec_fw_req_tmpl;
122 struct qat_crypto_instance *inst; 118 struct qat_crypto_instance *inst;
@@ -150,8 +146,9 @@ static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
150static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, 146static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
151 struct qat_alg_session_ctx *ctx, 147 struct qat_alg_session_ctx *ctx,
152 const uint8_t *auth_key, 148 const uint8_t *auth_key,
153 unsigned int auth_keylen, uint8_t *auth_state) 149 unsigned int auth_keylen)
154{ 150{
151 struct qat_auth_state auth_state;
155 struct { 152 struct {
156 struct shash_desc shash; 153 struct shash_desc shash;
157 char ctx[crypto_shash_descsize(ctx->hash_tfm)]; 154 char ctx[crypto_shash_descsize(ctx->hash_tfm)];
@@ -161,12 +158,13 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
161 struct sha512_state sha512; 158 struct sha512_state sha512;
162 int block_size = crypto_shash_blocksize(ctx->hash_tfm); 159 int block_size = crypto_shash_blocksize(ctx->hash_tfm);
163 int digest_size = crypto_shash_digestsize(ctx->hash_tfm); 160 int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
164 uint8_t *ipad = auth_state; 161 uint8_t *ipad = auth_state.data;
165 uint8_t *opad = ipad + block_size; 162 uint8_t *opad = ipad + block_size;
166 __be32 *hash_state_out; 163 __be32 *hash_state_out;
167 __be64 *hash512_state_out; 164 __be64 *hash512_state_out;
168 int i, offset; 165 int i, offset;
169 166
167 memset(auth_state.data, '\0', MAX_AUTH_STATE_SIZE + 64);
170 desc.shash.tfm = ctx->hash_tfm; 168 desc.shash.tfm = ctx->hash_tfm;
171 desc.shash.flags = 0x0; 169 desc.shash.flags = 0x0;
172 170
@@ -298,10 +296,6 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx,
298 void *ptr = &req_tmpl->cd_ctrl; 296 void *ptr = &req_tmpl->cd_ctrl;
299 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; 297 struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
300 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; 298 struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
301 struct icp_qat_fw_la_auth_req_params *auth_param =
302 (struct icp_qat_fw_la_auth_req_params *)
303 ((char *)&req_tmpl->serv_specif_rqpars +
304 sizeof(struct icp_qat_fw_la_cipher_req_params));
305 299
306 /* CD setup */ 300 /* CD setup */
307 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg); 301 cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg);
@@ -312,8 +306,7 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx,
312 hash->sha.inner_setup.auth_counter.counter = 306 hash->sha.inner_setup.auth_counter.counter =
313 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); 307 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
314 308
315 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen, 309 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
316 (uint8_t *)ctx->auth_hw_state_enc))
317 return -EFAULT; 310 return -EFAULT;
318 311
319 /* Request setup */ 312 /* Request setup */
@@ -359,9 +352,6 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx,
359 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + 352 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
360 ((sizeof(struct icp_qat_hw_auth_setup) + 353 ((sizeof(struct icp_qat_hw_auth_setup) +
361 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); 354 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
362 auth_param->u1.auth_partial_st_prefix = ctx->auth_state_enc_paddr +
363 sizeof(struct icp_qat_hw_auth_counter) +
364 round_up(hash_cd_ctrl->inner_state1_sz, 8);
365 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); 355 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
366 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); 356 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
367 return 0; 357 return 0;
@@ -399,8 +389,7 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx,
399 hash->sha.inner_setup.auth_counter.counter = 389 hash->sha.inner_setup.auth_counter.counter =
400 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); 390 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
401 391
402 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen, 392 if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
403 (uint8_t *)ctx->auth_hw_state_dec))
404 return -EFAULT; 393 return -EFAULT;
405 394
406 /* Request setup */ 395 /* Request setup */
@@ -450,9 +439,6 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx,
450 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + 439 hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
451 ((sizeof(struct icp_qat_hw_auth_setup) + 440 ((sizeof(struct icp_qat_hw_auth_setup) +
452 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); 441 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
453 auth_param->u1.auth_partial_st_prefix = ctx->auth_state_enc_paddr +
454 sizeof(struct icp_qat_hw_auth_counter) +
455 round_up(hash_cd_ctrl->inner_state1_sz, 8);
456 auth_param->auth_res_sz = digestsize; 442 auth_param->auth_res_sz = digestsize;
457 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); 443 ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
458 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); 444 ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
@@ -512,10 +498,6 @@ static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key,
512 dev = &GET_DEV(ctx->inst->accel_dev); 498 dev = &GET_DEV(ctx->inst->accel_dev);
513 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); 499 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
514 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); 500 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
515 memset(ctx->auth_hw_state_enc, 0,
516 sizeof(struct qat_auth_state));
517 memset(ctx->auth_hw_state_dec, 0,
518 sizeof(struct qat_auth_state));
519 memset(&ctx->enc_fw_req_tmpl, 0, 501 memset(&ctx->enc_fw_req_tmpl, 0,
520 sizeof(struct icp_qat_fw_la_bulk_req)); 502 sizeof(struct icp_qat_fw_la_bulk_req));
521 memset(&ctx->dec_fw_req_tmpl, 0, 503 memset(&ctx->dec_fw_req_tmpl, 0,
@@ -548,22 +530,6 @@ static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key,
548 spin_unlock(&ctx->lock); 530 spin_unlock(&ctx->lock);
549 goto out_free_enc; 531 goto out_free_enc;
550 } 532 }
551 ctx->auth_hw_state_enc =
552 dma_zalloc_coherent(dev, sizeof(struct qat_auth_state),
553 &ctx->auth_state_enc_paddr,
554 GFP_ATOMIC);
555 if (!ctx->auth_hw_state_enc) {
556 spin_unlock(&ctx->lock);
557 goto out_free_dec;
558 }
559 ctx->auth_hw_state_dec =
560 dma_zalloc_coherent(dev, sizeof(struct qat_auth_state),
561 &ctx->auth_state_dec_paddr,
562 GFP_ATOMIC);
563 if (!ctx->auth_hw_state_dec) {
564 spin_unlock(&ctx->lock);
565 goto out_free_auth_enc;
566 }
567 } 533 }
568 spin_unlock(&ctx->lock); 534 spin_unlock(&ctx->lock);
569 if (qat_alg_init_sessions(ctx, key, keylen)) 535 if (qat_alg_init_sessions(ctx, key, keylen))
@@ -572,14 +538,6 @@ static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key,
572 return 0; 538 return 0;
573 539
574out_free_all: 540out_free_all:
575 dma_free_coherent(dev, sizeof(struct qat_auth_state),
576 ctx->auth_hw_state_dec, ctx->auth_state_dec_paddr);
577 ctx->auth_hw_state_dec = NULL;
578out_free_auth_enc:
579 dma_free_coherent(dev, sizeof(struct qat_auth_state),
580 ctx->auth_hw_state_enc, ctx->auth_state_enc_paddr);
581 ctx->auth_hw_state_enc = NULL;
582out_free_dec:
583 dma_free_coherent(dev, sizeof(struct qat_alg_cd), 541 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
584 ctx->dec_cd, ctx->dec_cd_paddr); 542 ctx->dec_cd, ctx->dec_cd_paddr);
585 ctx->dec_cd = NULL; 543 ctx->dec_cd = NULL;
@@ -924,16 +882,6 @@ static void qat_alg_exit(struct crypto_tfm *tfm)
924 if (ctx->dec_cd) 882 if (ctx->dec_cd)
925 dma_free_coherent(dev, sizeof(struct qat_alg_cd), 883 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
926 ctx->dec_cd, ctx->dec_cd_paddr); 884 ctx->dec_cd, ctx->dec_cd_paddr);
927 if (ctx->auth_hw_state_enc)
928 dma_free_coherent(dev, sizeof(struct qat_auth_state),
929 ctx->auth_hw_state_enc,
930 ctx->auth_state_enc_paddr);
931
932 if (ctx->auth_hw_state_dec)
933 dma_free_coherent(dev, sizeof(struct qat_auth_state),
934 ctx->auth_hw_state_dec,
935 ctx->auth_state_dec_paddr);
936
937 qat_crypto_put_instance(inst); 885 qat_crypto_put_instance(inst);
938} 886}
939 887
diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_isr.c b/drivers/crypto/qat/qat_dh895xcc/adf_isr.c
index d4172dedf775..67ec61e51185 100644
--- a/drivers/crypto/qat/qat_dh895xcc/adf_isr.c
+++ b/drivers/crypto/qat/qat_dh895xcc/adf_isr.c
@@ -70,9 +70,9 @@ static int adf_enable_msix(struct adf_accel_dev *accel_dev)
70 for (i = 0; i < msix_num_entries; i++) 70 for (i = 0; i < msix_num_entries; i++)
71 pci_dev_info->msix_entries.entries[i].entry = i; 71 pci_dev_info->msix_entries.entries[i].entry = i;
72 72
73 if (pci_enable_msix(pci_dev_info->pci_dev, 73 if (pci_enable_msix_exact(pci_dev_info->pci_dev,
74 pci_dev_info->msix_entries.entries, 74 pci_dev_info->msix_entries.entries,
75 msix_num_entries)) { 75 msix_num_entries)) {
76 pr_err("QAT: Failed to enable MSIX IRQ\n"); 76 pr_err("QAT: Failed to enable MSIX IRQ\n");
77 return -EFAULT; 77 return -EFAULT;
78 } 78 }
@@ -89,7 +89,7 @@ static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)
89 struct adf_etr_bank_data *bank = bank_ptr; 89 struct adf_etr_bank_data *bank = bank_ptr;
90 90
91 WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0); 91 WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0);
92 tasklet_hi_schedule(&bank->resp_hanlder); 92 tasklet_hi_schedule(&bank->resp_handler);
93 return IRQ_HANDLED; 93 return IRQ_HANDLED;
94} 94}
95 95
@@ -217,7 +217,7 @@ static int adf_setup_bh(struct adf_accel_dev *accel_dev)
217 int i; 217 int i;
218 218
219 for (i = 0; i < hw_data->num_banks; i++) 219 for (i = 0; i < hw_data->num_banks; i++)
220 tasklet_init(&priv_data->banks[i].resp_hanlder, 220 tasklet_init(&priv_data->banks[i].resp_handler,
221 adf_response_handler, 221 adf_response_handler,
222 (unsigned long)&priv_data->banks[i]); 222 (unsigned long)&priv_data->banks[i]);
223 return 0; 223 return 0;
@@ -230,8 +230,8 @@ static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)
230 int i; 230 int i;
231 231
232 for (i = 0; i < hw_data->num_banks; i++) { 232 for (i = 0; i < hw_data->num_banks; i++) {
233 tasklet_disable(&priv_data->banks[i].resp_hanlder); 233 tasklet_disable(&priv_data->banks[i].resp_handler);
234 tasklet_kill(&priv_data->banks[i].resp_hanlder); 234 tasklet_kill(&priv_data->banks[i].resp_handler);
235 } 235 }
236} 236}
237 237
diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h
index 882675e7c055..5186f750c713 100644
--- a/include/crypto/drbg.h
+++ b/include/crypto/drbg.h
@@ -82,15 +82,6 @@ typedef uint32_t drbg_flag_t;
82struct drbg_core { 82struct drbg_core {
83 drbg_flag_t flags; /* flags for the cipher */ 83 drbg_flag_t flags; /* flags for the cipher */
84 __u8 statelen; /* maximum state length */ 84 __u8 statelen; /* maximum state length */
85 /*
86 * maximum length of personalization string or additional input
87 * string -- exponent for base 2
88 */
89 __u8 max_addtllen;
90 /* maximum bits per RNG request -- exponent for base 2*/
91 __u8 max_bits;
92 /* maximum number of requests -- exponent for base 2 */
93 __u8 max_req;
94 __u8 blocklen_bytes; /* block size of output in bytes */ 85 __u8 blocklen_bytes; /* block size of output in bytes */
95 char cra_name[CRYPTO_MAX_ALG_NAME]; /* mapping to kernel crypto API */ 86 char cra_name[CRYPTO_MAX_ALG_NAME]; /* mapping to kernel crypto API */
96 /* kernel crypto API backend cipher name */ 87 /* kernel crypto API backend cipher name */
@@ -156,12 +147,13 @@ static inline __u8 drbg_keylen(struct drbg_state *drbg)
156 147
157static inline size_t drbg_max_request_bytes(struct drbg_state *drbg) 148static inline size_t drbg_max_request_bytes(struct drbg_state *drbg)
158{ 149{
159 /* max_bits is in bits, but buflen is in bytes */ 150 /* SP800-90A requires the limit 2**19 bits, but we return bytes */
160 return (1 << (drbg->core->max_bits - 3)); 151 return (1 << 16);
161} 152}
162 153
163static inline size_t drbg_max_addtl(struct drbg_state *drbg) 154static inline size_t drbg_max_addtl(struct drbg_state *drbg)
164{ 155{
156 /* SP800-90A requires 2**35 bytes additional info str / pers str */
165#if (__BITS_PER_LONG == 32) 157#if (__BITS_PER_LONG == 32)
166 /* 158 /*
167 * SP800-90A allows smaller maximum numbers to be returned -- we 159 * SP800-90A allows smaller maximum numbers to be returned -- we
@@ -170,16 +162,17 @@ static inline size_t drbg_max_addtl(struct drbg_state *drbg)
170 */ 162 */
171 return (SIZE_MAX - 1); 163 return (SIZE_MAX - 1);
172#else 164#else
173 return (1UL<<(drbg->core->max_addtllen)); 165 return (1UL<<35);
174#endif 166#endif
175} 167}
176 168
177static inline size_t drbg_max_requests(struct drbg_state *drbg) 169static inline size_t drbg_max_requests(struct drbg_state *drbg)
178{ 170{
171 /* SP800-90A requires 2**48 maximum requests before reseeding */
179#if (__BITS_PER_LONG == 32) 172#if (__BITS_PER_LONG == 32)
180 return SIZE_MAX; 173 return SIZE_MAX;
181#else 174#else
182 return (1UL<<(drbg->core->max_req)); 175 return (1UL<<48);
183#endif 176#endif
184} 177}
185 178
diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h
index 9b6f32a6cad1..3b4af1d7c7e9 100644
--- a/include/crypto/internal/hash.h
+++ b/include/crypto/internal/hash.h
@@ -117,6 +117,15 @@ int shash_ahash_update(struct ahash_request *req, struct shash_desc *desc);
117int shash_ahash_finup(struct ahash_request *req, struct shash_desc *desc); 117int shash_ahash_finup(struct ahash_request *req, struct shash_desc *desc);
118int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc); 118int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc);
119 119
120int shash_ahash_mcryptd_update(struct ahash_request *req,
121 struct shash_desc *desc);
122int shash_ahash_mcryptd_final(struct ahash_request *req,
123 struct shash_desc *desc);
124int shash_ahash_mcryptd_finup(struct ahash_request *req,
125 struct shash_desc *desc);
126int shash_ahash_mcryptd_digest(struct ahash_request *req,
127 struct shash_desc *desc);
128
120int crypto_init_shash_ops_async(struct crypto_tfm *tfm); 129int crypto_init_shash_ops_async(struct crypto_tfm *tfm);
121 130
122static inline void *crypto_ahash_ctx(struct crypto_ahash *tfm) 131static inline void *crypto_ahash_ctx(struct crypto_ahash *tfm)
diff --git a/include/crypto/mcryptd.h b/include/crypto/mcryptd.h
new file mode 100644
index 000000000000..c23ee1f7ee80
--- /dev/null
+++ b/include/crypto/mcryptd.h
@@ -0,0 +1,112 @@
1/*
2 * Software async multibuffer crypto daemon headers
3 *
4 * Author:
5 * Tim Chen <tim.c.chen@linux.intel.com>
6 *
7 * Copyright (c) 2014, Intel Corporation.
8 */
9
10#ifndef _CRYPTO_MCRYPT_H
11#define _CRYPTO_MCRYPT_H
12
13#include <linux/crypto.h>
14#include <linux/kernel.h>
15#include <crypto/hash.h>
16
17struct mcryptd_ahash {
18 struct crypto_ahash base;
19};
20
21static inline struct mcryptd_ahash *__mcryptd_ahash_cast(
22 struct crypto_ahash *tfm)
23{
24 return (struct mcryptd_ahash *)tfm;
25}
26
27struct mcryptd_cpu_queue {
28 struct crypto_queue queue;
29 struct work_struct work;
30};
31
32struct mcryptd_queue {
33 struct mcryptd_cpu_queue __percpu *cpu_queue;
34};
35
36struct mcryptd_instance_ctx {
37 struct crypto_spawn spawn;
38 struct mcryptd_queue *queue;
39};
40
41struct mcryptd_hash_ctx {
42 struct crypto_shash *child;
43 struct mcryptd_alg_state *alg_state;
44};
45
46struct mcryptd_tag {
47 /* seq number of request */
48 unsigned seq_num;
49 /* arrival time of request */
50 unsigned long arrival;
51 unsigned long expire;
52 int cpu;
53};
54
55struct mcryptd_hash_request_ctx {
56 struct list_head waiter;
57 crypto_completion_t complete;
58 struct mcryptd_tag tag;
59 struct crypto_hash_walk walk;
60 u8 *out;
61 int flag;
62 struct shash_desc desc;
63};
64
65struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name,
66 u32 type, u32 mask);
67struct crypto_shash *mcryptd_ahash_child(struct mcryptd_ahash *tfm);
68struct shash_desc *mcryptd_shash_desc(struct ahash_request *req);
69void mcryptd_free_ahash(struct mcryptd_ahash *tfm);
70void mcryptd_flusher(struct work_struct *work);
71
72enum mcryptd_req_type {
73 MCRYPTD_NONE,
74 MCRYPTD_UPDATE,
75 MCRYPTD_FINUP,
76 MCRYPTD_DIGEST,
77 MCRYPTD_FINAL
78};
79
80struct mcryptd_alg_cstate {
81 unsigned long next_flush;
82 unsigned next_seq_num;
83 bool flusher_engaged;
84 struct delayed_work flush;
85 int cpu;
86 struct mcryptd_alg_state *alg_state;
87 void *mgr;
88 spinlock_t work_lock;
89 struct list_head work_list;
90 struct list_head flush_list;
91};
92
93struct mcryptd_alg_state {
94 struct mcryptd_alg_cstate __percpu *alg_cstate;
95 unsigned long (*flusher)(struct mcryptd_alg_cstate *cstate);
96};
97
98/* return delay in jiffies from current time */
99static inline unsigned long get_delay(unsigned long t)
100{
101 long delay;
102
103 delay = (long) t - (long) jiffies;
104 if (delay <= 0)
105 return 0;
106 else
107 return (unsigned long) delay;
108}
109
110void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay);
111
112#endif
diff --git a/include/linux/sched.h b/include/linux/sched.h
index b867a4dab38a..9c6353d9e63a 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -167,6 +167,7 @@ extern int nr_threads;
167DECLARE_PER_CPU(unsigned long, process_counts); 167DECLARE_PER_CPU(unsigned long, process_counts);
168extern int nr_processes(void); 168extern int nr_processes(void);
169extern unsigned long nr_running(void); 169extern unsigned long nr_running(void);
170extern bool single_task_running(void);
170extern unsigned long nr_iowait(void); 171extern unsigned long nr_iowait(void);
171extern unsigned long nr_iowait_cpu(int cpu); 172extern unsigned long nr_iowait_cpu(int cpu);
172extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load); 173extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index ec1a286684a5..59965ec0b7de 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2366,6 +2366,18 @@ unsigned long nr_running(void)
2366 return sum; 2366 return sum;
2367} 2367}
2368 2368
2369/*
2370 * Check if only the current task is running on the cpu.
2371 */
2372bool single_task_running(void)
2373{
2374 if (cpu_rq(smp_processor_id())->nr_running == 1)
2375 return true;
2376 else
2377 return false;
2378}
2379EXPORT_SYMBOL(single_task_running);
2380
2369unsigned long long nr_context_switches(void) 2381unsigned long long nr_context_switches(void)
2370{ 2382{
2371 int i; 2383 int i;
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-17 23:22:30 -0500