aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHerbert Xu <herbert@gondor.apana.org.au>2005-07-06 16:51:31 -0400
committerDavid S. Miller <davem@davemloft.net>2005-07-06 16:51:31 -0400
commitc774e93e2152d0be2612739418689e6e6400f4eb (patch)
treeabe25ec0577bd95128adb3f38609a09f0a3e2469
parent8279dd748f9704b811e528b31304e2fab026abc5 (diff)
[CRYPTO] Add plumbing for multi-block operations
The VIA Padlock device is able to perform much better when multiple blocks are fed to it at once. As this device offers an exceptional throughput rate it is worthwhile to optimise the infrastructure specifically for it. We shift the existing page-sized fast path down to the CBC/ECB functions. We can then replace the CBC/ECB functions with functions provided by the underlying algorithm that performs the multi-block operations. As a side-effect this improves the performance of large cipher operations for all existing algorithm implementations. I've measured the gain to be around 5% for 3DES and 15% for AES. Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r--crypto/cipher.c246
-rw-r--r--crypto/scatterwalk.c4
-rw-r--r--crypto/scatterwalk.h6
3 files changed, 161 insertions, 95 deletions
diff --git a/crypto/cipher.c b/crypto/cipher.c
index 69264497b48c..c4243345b154 100644
--- a/crypto/cipher.c
+++ b/crypto/cipher.c
@@ -4,6 +4,7 @@
4 * Cipher operations. 4 * Cipher operations.
5 * 5 *
6 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> 6 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
7 * Copyright (c) 2005 Herbert Xu <herbert@gondor.apana.org.au>
7 * 8 *
8 * This program is free software; you can redistribute it and/or modify it 9 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free 10 * under the terms of the GNU General Public License as published by the Free
@@ -22,9 +23,13 @@
22#include "internal.h" 23#include "internal.h"
23#include "scatterwalk.h" 24#include "scatterwalk.h"
24 25
25typedef void (cryptfn_t)(void *, u8 *, const u8 *); 26struct cipher_desc {
26typedef void (procfn_t)(struct crypto_tfm *, u8 *, 27 struct crypto_tfm *tfm;
27 u8*, cryptfn_t, void *); 28 void (*crfn)(void *ctx, u8 *dst, const u8 *src);
29 unsigned int (*prfn)(const struct cipher_desc *desc, u8 *dst,
30 const u8 *src, unsigned int nbytes);
31 void *info;
32};
28 33
29static inline void xor_64(u8 *a, const u8 *b) 34static inline void xor_64(u8 *a, const u8 *b)
30{ 35{
@@ -39,63 +44,57 @@ static inline void xor_128(u8 *a, const u8 *b)
39 ((u32 *)a)[2] ^= ((u32 *)b)[2]; 44 ((u32 *)a)[2] ^= ((u32 *)b)[2];
40 ((u32 *)a)[3] ^= ((u32 *)b)[3]; 45 ((u32 *)a)[3] ^= ((u32 *)b)[3];
41} 46}
42 47
43static inline void *prepare_src(struct scatter_walk *walk, int bsize, 48static unsigned int crypt_slow(const struct cipher_desc *desc,
44 void *tmp, int in_place) 49 struct scatter_walk *in,
50 struct scatter_walk *out, unsigned int bsize)
45{ 51{
46 void *src = walk->data; 52 u8 src[bsize];
47 int n = bsize; 53 u8 dst[bsize];
54 unsigned int n;
48 55
49 if (unlikely(scatterwalk_across_pages(walk, bsize))) { 56 n = scatterwalk_copychunks(src, in, bsize, 0);
50 src = tmp; 57 scatterwalk_advance(in, n);
51 n = scatterwalk_copychunks(src, walk, bsize, 0);
52 }
53 scatterwalk_advance(walk, n);
54 return src;
55}
56 58
57static inline void *prepare_dst(struct scatter_walk *walk, int bsize, 59 desc->prfn(desc, dst, src, bsize);
58 void *tmp, int in_place)
59{
60 void *dst = walk->data;
61 60
62 if (unlikely(scatterwalk_across_pages(walk, bsize)) || in_place) 61 n = scatterwalk_copychunks(dst, out, bsize, 1);
63 dst = tmp; 62 scatterwalk_advance(out, n);
64 return dst;
65}
66 63
67static inline void complete_src(struct scatter_walk *walk, int bsize, 64 return bsize;
68 void *src, int in_place)
69{
70} 65}
71 66
72static inline void complete_dst(struct scatter_walk *walk, int bsize, 67static inline unsigned int crypt_fast(const struct cipher_desc *desc,
73 void *dst, int in_place) 68 struct scatter_walk *in,
69 struct scatter_walk *out,
70 unsigned int nbytes)
74{ 71{
75 int n = bsize; 72 u8 *src, *dst;
73
74 src = in->data;
75 dst = scatterwalk_samebuf(in, out) ? src : out->data;
76
77 nbytes = desc->prfn(desc, dst, src, nbytes);
78
79 scatterwalk_advance(in, nbytes);
80 scatterwalk_advance(out, nbytes);
76 81
77 if (unlikely(scatterwalk_across_pages(walk, bsize))) 82 return nbytes;
78 n = scatterwalk_copychunks(dst, walk, bsize, 1);
79 else if (in_place)
80 memcpy(walk->data, dst, bsize);
81 scatterwalk_advance(walk, n);
82} 83}
83 84
84/* 85/*
85 * Generic encrypt/decrypt wrapper for ciphers, handles operations across 86 * Generic encrypt/decrypt wrapper for ciphers, handles operations across
86 * multiple page boundaries by using temporary blocks. In user context, 87 * multiple page boundaries by using temporary blocks. In user context,
87 * the kernel is given a chance to schedule us once per block. 88 * the kernel is given a chance to schedule us once per page.
88 */ 89 */
89static int crypt(struct crypto_tfm *tfm, 90static int crypt(const struct cipher_desc *desc,
90 struct scatterlist *dst, 91 struct scatterlist *dst,
91 struct scatterlist *src, 92 struct scatterlist *src,
92 unsigned int nbytes, cryptfn_t crfn, 93 unsigned int nbytes)
93 procfn_t prfn, void *info)
94{ 94{
95 struct scatter_walk walk_in, walk_out; 95 struct scatter_walk walk_in, walk_out;
96 struct crypto_tfm *tfm = desc->tfm;
96 const unsigned int bsize = crypto_tfm_alg_blocksize(tfm); 97 const unsigned int bsize = crypto_tfm_alg_blocksize(tfm);
97 u8 tmp_src[bsize];
98 u8 tmp_dst[bsize];
99 98
100 if (!nbytes) 99 if (!nbytes)
101 return 0; 100 return 0;
@@ -109,29 +108,20 @@ static int crypt(struct crypto_tfm *tfm,
109 scatterwalk_start(&walk_out, dst); 108 scatterwalk_start(&walk_out, dst);
110 109
111 for(;;) { 110 for(;;) {
112 u8 *src_p, *dst_p; 111 unsigned int n;
113 int in_place;
114 112
115 scatterwalk_map(&walk_in, 0); 113 scatterwalk_map(&walk_in, 0);
116 scatterwalk_map(&walk_out, 1); 114 scatterwalk_map(&walk_out, 1);
117 115
118 in_place = scatterwalk_samebuf(&walk_in, &walk_out); 116 n = scatterwalk_clamp(&walk_in, nbytes);
117 n = scatterwalk_clamp(&walk_out, n);
119 118
120 do { 119 if (likely(n >= bsize))
121 src_p = prepare_src(&walk_in, bsize, tmp_src, 120 n = crypt_fast(desc, &walk_in, &walk_out, n);
122 in_place); 121 else
123 dst_p = prepare_dst(&walk_out, bsize, tmp_dst, 122 n = crypt_slow(desc, &walk_in, &walk_out, bsize);
124 in_place);
125 123
126 prfn(tfm, dst_p, src_p, crfn, info); 124 nbytes -= n;
127
128 complete_src(&walk_in, bsize, src_p, in_place);
129 complete_dst(&walk_out, bsize, dst_p, in_place);
130
131 nbytes -= bsize;
132 } while (nbytes &&
133 !scatterwalk_across_pages(&walk_in, bsize) &&
134 !scatterwalk_across_pages(&walk_out, bsize));
135 125
136 scatterwalk_done(&walk_in, 0, nbytes); 126 scatterwalk_done(&walk_in, 0, nbytes);
137 scatterwalk_done(&walk_out, 1, nbytes); 127 scatterwalk_done(&walk_out, 1, nbytes);
@@ -143,30 +133,78 @@ static int crypt(struct crypto_tfm *tfm,
143 } 133 }
144} 134}
145 135
146static void cbc_process_encrypt(struct crypto_tfm *tfm, u8 *dst, u8 *src, 136static unsigned int cbc_process_encrypt(const struct cipher_desc *desc,
147 cryptfn_t fn, void *info) 137 u8 *dst, const u8 *src,
138 unsigned int nbytes)
148{ 139{
149 u8 *iv = info; 140 struct crypto_tfm *tfm = desc->tfm;
141 void (*xor)(u8 *, const u8 *) = tfm->crt_u.cipher.cit_xor_block;
142 int bsize = crypto_tfm_alg_blocksize(tfm);
143
144 void (*fn)(void *, u8 *, const u8 *) = desc->crfn;
145 u8 *iv = desc->info;
146 unsigned int done = 0;
147
148 do {
149 xor(iv, src);
150 fn(crypto_tfm_ctx(tfm), dst, iv);
151 memcpy(iv, dst, bsize);
150 152
151 tfm->crt_u.cipher.cit_xor_block(iv, src); 153 src += bsize;
152 fn(crypto_tfm_ctx(tfm), dst, iv); 154 dst += bsize;
153 memcpy(iv, dst, crypto_tfm_alg_blocksize(tfm)); 155 } while ((done += bsize) < nbytes);
156
157 return done;
154} 158}
155 159
156static void cbc_process_decrypt(struct crypto_tfm *tfm, u8 *dst, u8 *src, 160static unsigned int cbc_process_decrypt(const struct cipher_desc *desc,
157 cryptfn_t fn, void *info) 161 u8 *dst, const u8 *src,
162 unsigned int nbytes)
158{ 163{
159 u8 *iv = info; 164 struct crypto_tfm *tfm = desc->tfm;
165 void (*xor)(u8 *, const u8 *) = tfm->crt_u.cipher.cit_xor_block;
166 int bsize = crypto_tfm_alg_blocksize(tfm);
167
168 u8 stack[src == dst ? bsize : 0];
169 u8 *buf = stack;
170 u8 **dst_p = src == dst ? &buf : &dst;
171
172 void (*fn)(void *, u8 *, const u8 *) = desc->crfn;
173 u8 *iv = desc->info;
174 unsigned int done = 0;
175
176 do {
177 u8 *tmp_dst = *dst_p;
160 178
161 fn(crypto_tfm_ctx(tfm), dst, src); 179 fn(crypto_tfm_ctx(tfm), tmp_dst, src);
162 tfm->crt_u.cipher.cit_xor_block(dst, iv); 180 xor(tmp_dst, iv);
163 memcpy(iv, src, crypto_tfm_alg_blocksize(tfm)); 181 memcpy(iv, src, bsize);
182 if (tmp_dst != dst)
183 memcpy(dst, tmp_dst, bsize);
184
185 src += bsize;
186 dst += bsize;
187 } while ((done += bsize) < nbytes);
188
189 return done;
164} 190}
165 191
166static void ecb_process(struct crypto_tfm *tfm, u8 *dst, u8 *src, 192static unsigned int ecb_process(const struct cipher_desc *desc, u8 *dst,
167 cryptfn_t fn, void *info) 193 const u8 *src, unsigned int nbytes)
168{ 194{
169 fn(crypto_tfm_ctx(tfm), dst, src); 195 struct crypto_tfm *tfm = desc->tfm;
196 int bsize = crypto_tfm_alg_blocksize(tfm);
197 void (*fn)(void *, u8 *, const u8 *) = desc->crfn;
198 unsigned int done = 0;
199
200 do {
201 fn(crypto_tfm_ctx(tfm), dst, src);
202
203 src += bsize;
204 dst += bsize;
205 } while ((done += bsize) < nbytes);
206
207 return done;
170} 208}
171 209
172static int setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) 210static int setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen)
@@ -185,9 +223,13 @@ static int ecb_encrypt(struct crypto_tfm *tfm,
185 struct scatterlist *dst, 223 struct scatterlist *dst,
186 struct scatterlist *src, unsigned int nbytes) 224 struct scatterlist *src, unsigned int nbytes)
187{ 225{
188 return crypt(tfm, dst, src, nbytes, 226 struct cipher_desc desc;
189 tfm->__crt_alg->cra_cipher.cia_encrypt, 227
190 ecb_process, NULL); 228 desc.tfm = tfm;
229 desc.crfn = tfm->__crt_alg->cra_cipher.cia_encrypt;
230 desc.prfn = ecb_process;
231
232 return crypt(&desc, dst, src, nbytes);
191} 233}
192 234
193static int ecb_decrypt(struct crypto_tfm *tfm, 235static int ecb_decrypt(struct crypto_tfm *tfm,
@@ -195,9 +237,13 @@ static int ecb_decrypt(struct crypto_tfm *tfm,
195 struct scatterlist *src, 237 struct scatterlist *src,
196 unsigned int nbytes) 238 unsigned int nbytes)
197{ 239{
198 return crypt(tfm, dst, src, nbytes, 240 struct cipher_desc desc;
199 tfm->__crt_alg->cra_cipher.cia_decrypt, 241
200 ecb_process, NULL); 242 desc.tfm = tfm;
243 desc.crfn = tfm->__crt_alg->cra_cipher.cia_decrypt;
244 desc.prfn = ecb_process;
245
246 return crypt(&desc, dst, src, nbytes);
201} 247}
202 248
203static int cbc_encrypt(struct crypto_tfm *tfm, 249static int cbc_encrypt(struct crypto_tfm *tfm,
@@ -205,9 +251,14 @@ static int cbc_encrypt(struct crypto_tfm *tfm,
205 struct scatterlist *src, 251 struct scatterlist *src,
206 unsigned int nbytes) 252 unsigned int nbytes)
207{ 253{
208 return crypt(tfm, dst, src, nbytes, 254 struct cipher_desc desc;
209 tfm->__crt_alg->cra_cipher.cia_encrypt, 255
210 cbc_process_encrypt, tfm->crt_cipher.cit_iv); 256 desc.tfm = tfm;
257 desc.crfn = tfm->__crt_alg->cra_cipher.cia_encrypt;
258 desc.prfn = cbc_process_encrypt;
259 desc.info = tfm->crt_cipher.cit_iv;
260
261 return crypt(&desc, dst, src, nbytes);
211} 262}
212 263
213static int cbc_encrypt_iv(struct crypto_tfm *tfm, 264static int cbc_encrypt_iv(struct crypto_tfm *tfm,
@@ -215,9 +266,14 @@ static int cbc_encrypt_iv(struct crypto_tfm *tfm,
215 struct scatterlist *src, 266 struct scatterlist *src,
216 unsigned int nbytes, u8 *iv) 267 unsigned int nbytes, u8 *iv)
217{ 268{
218 return crypt(tfm, dst, src, nbytes, 269 struct cipher_desc desc;
219 tfm->__crt_alg->cra_cipher.cia_encrypt, 270
220 cbc_process_encrypt, iv); 271 desc.tfm = tfm;
272 desc.crfn = tfm->__crt_alg->cra_cipher.cia_encrypt;
273 desc.prfn = cbc_process_encrypt;
274 desc.info = iv;
275
276 return crypt(&desc, dst, src, nbytes);
221} 277}
222 278
223static int cbc_decrypt(struct crypto_tfm *tfm, 279static int cbc_decrypt(struct crypto_tfm *tfm,
@@ -225,9 +281,14 @@ static int cbc_decrypt(struct crypto_tfm *tfm,
225 struct scatterlist *src, 281 struct scatterlist *src,
226 unsigned int nbytes) 282 unsigned int nbytes)
227{ 283{
228 return crypt(tfm, dst, src, nbytes, 284 struct cipher_desc desc;
229 tfm->__crt_alg->cra_cipher.cia_decrypt, 285
230 cbc_process_decrypt, tfm->crt_cipher.cit_iv); 286 desc.tfm = tfm;
287 desc.crfn = tfm->__crt_alg->cra_cipher.cia_decrypt;
288 desc.prfn = cbc_process_decrypt;
289 desc.info = tfm->crt_cipher.cit_iv;
290
291 return crypt(&desc, dst, src, nbytes);
231} 292}
232 293
233static int cbc_decrypt_iv(struct crypto_tfm *tfm, 294static int cbc_decrypt_iv(struct crypto_tfm *tfm,
@@ -235,9 +296,14 @@ static int cbc_decrypt_iv(struct crypto_tfm *tfm,
235 struct scatterlist *src, 296 struct scatterlist *src,
236 unsigned int nbytes, u8 *iv) 297 unsigned int nbytes, u8 *iv)
237{ 298{
238 return crypt(tfm, dst, src, nbytes, 299 struct cipher_desc desc;
239 tfm->__crt_alg->cra_cipher.cia_decrypt, 300
240 cbc_process_decrypt, iv); 301 desc.tfm = tfm;
302 desc.crfn = tfm->__crt_alg->cra_cipher.cia_decrypt;
303 desc.prfn = cbc_process_decrypt;
304 desc.info = iv;
305
306 return crypt(&desc, dst, src, nbytes);
241} 307}
242 308
243static int nocrypt(struct crypto_tfm *tfm, 309static int nocrypt(struct crypto_tfm *tfm,
diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c
index 50c9461e8cc6..47ac90e615f4 100644
--- a/crypto/scatterwalk.c
+++ b/crypto/scatterwalk.c
@@ -100,7 +100,7 @@ void scatterwalk_done(struct scatter_walk *walk, int out, int more)
100int scatterwalk_copychunks(void *buf, struct scatter_walk *walk, 100int scatterwalk_copychunks(void *buf, struct scatter_walk *walk,
101 size_t nbytes, int out) 101 size_t nbytes, int out)
102{ 102{
103 do { 103 while (nbytes > walk->len_this_page) {
104 memcpy_dir(buf, walk->data, walk->len_this_page, out); 104 memcpy_dir(buf, walk->data, walk->len_this_page, out);
105 buf += walk->len_this_page; 105 buf += walk->len_this_page;
106 nbytes -= walk->len_this_page; 106 nbytes -= walk->len_this_page;
@@ -108,7 +108,7 @@ int scatterwalk_copychunks(void *buf, struct scatter_walk *walk,
108 scatterwalk_unmap(walk, out); 108 scatterwalk_unmap(walk, out);
109 scatterwalk_pagedone(walk, out, 1); 109 scatterwalk_pagedone(walk, out, 1);
110 scatterwalk_map(walk, out); 110 scatterwalk_map(walk, out);
111 } while (nbytes > walk->len_this_page); 111 }
112 112
113 memcpy_dir(buf, walk->data, nbytes, out); 113 memcpy_dir(buf, walk->data, nbytes, out);
114 return nbytes; 114 return nbytes;
diff --git a/crypto/scatterwalk.h b/crypto/scatterwalk.h
index 02aa56c649b4..5495bb970816 100644
--- a/crypto/scatterwalk.h
+++ b/crypto/scatterwalk.h
@@ -40,10 +40,10 @@ static inline int scatterwalk_samebuf(struct scatter_walk *walk_in,
40 walk_in->offset == walk_out->offset; 40 walk_in->offset == walk_out->offset;
41} 41}
42 42
43static inline int scatterwalk_across_pages(struct scatter_walk *walk, 43static inline unsigned int scatterwalk_clamp(struct scatter_walk *walk,
44 unsigned int nbytes) 44 unsigned int nbytes)
45{ 45{
46 return nbytes > walk->len_this_page; 46 return nbytes > walk->len_this_page ? walk->len_this_page : nbytes;
47} 47}
48 48
49static inline void scatterwalk_advance(struct scatter_walk *walk, 49static inline void scatterwalk_advance(struct scatter_walk *walk,