aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/crypto/Makefile2
-rw-r--r--arch/x86/crypto/blowfish-x86_64-asm_64.S392
-rw-r--r--arch/x86/crypto/blowfish_glue.c487
-rw-r--r--crypto/Kconfig15
4 files changed, 896 insertions, 0 deletions
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 57c7f7b4436d..725addfacf0a 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -7,6 +7,7 @@ obj-$(CONFIG_CRYPTO_TWOFISH_586) += twofish-i586.o
7obj-$(CONFIG_CRYPTO_SALSA20_586) += salsa20-i586.o 7obj-$(CONFIG_CRYPTO_SALSA20_586) += salsa20-i586.o
8 8
9obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o 9obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o
10obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o
10obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o 11obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o
11obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o 12obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o
12obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o 13obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
@@ -20,6 +21,7 @@ twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o
20salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o 21salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o
21 22
22aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o 23aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o
24blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o
23twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o 25twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o
24salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o 26salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o
25 27
diff --git a/arch/x86/crypto/blowfish-x86_64-asm_64.S b/arch/x86/crypto/blowfish-x86_64-asm_64.S
new file mode 100644
index 000000000000..44eb23ab9676
--- /dev/null
+++ b/arch/x86/crypto/blowfish-x86_64-asm_64.S
@@ -0,0 +1,392 @@
1/*
2 * Blowfish Cipher Algorithm (x86_64)
3 *
4 * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
19 * USA
20 *
21 */
22
23.file "blowfish-x86_64-asm.S"
24.text
25
26/* structure of crypto context */
27#define p 0
28#define s0 ((16 + 2) * 4)
29#define s1 ((16 + 2 + (1 * 256)) * 4)
30#define s2 ((16 + 2 + (2 * 256)) * 4)
31#define s3 ((16 + 2 + (3 * 256)) * 4)
32
33/* register macros */
34#define CTX %rdi
35#define RIO %rsi
36
37#define RX0 %rax
38#define RX1 %rbx
39#define RX2 %rcx
40#define RX3 %rdx
41
42#define RX0d %eax
43#define RX1d %ebx
44#define RX2d %ecx
45#define RX3d %edx
46
47#define RX0bl %al
48#define RX1bl %bl
49#define RX2bl %cl
50#define RX3bl %dl
51
52#define RX0bh %ah
53#define RX1bh %bh
54#define RX2bh %ch
55#define RX3bh %dh
56
57#define RT0 %rbp
58#define RT1 %rsi
59
60#define RT0d %ebp
61#define RT1d %esi
62
63#define RK0 %r8
64#define RK1 %r9
65#define RK2 %r10
66#define RK3 %r11
67
68#define RK0d %r8d
69#define RK1d %r9d
70#define RK2d %r10d
71#define RK3d %r11d
72
73#define RKEY %r12
74
75/***********************************************************************
76 * 1-way blowfish
77 ***********************************************************************/
78#define F(x, k) \
79 rorq $16, x; \
80 movzbl x ## bh, RT0d; \
81 movzbl x ## bl, RT1d; \
82 rolq $16, x; \
83 movl s0(CTX,RT0,4), k ## d; \
84 addl s1(CTX,RT1,4), k ## d; \
85 movzbl x ## bh, RT0d; \
86 movzbl x ## bl, RT1d; \
87 rolq $32, x; \
88 xorl s2(CTX,RT0,4), k ## d; \
89 addl s3(CTX,RT1,4), k ## d; \
90 xorq k, x;
91
92#define add_roundkey_enc(n) \
93 xorq p+4*(n)(CTX), RX0;
94
95#define round_enc(n) \
96 add_roundkey_enc(n); \
97 \
98 F(RX0, RK0); \
99 F(RX0, RK0);
100
101#define round_final_enc(n) \
102 xorq p+4*(n)(CTX), RX0;
103
104#define add_roundkey_dec(n) \
105 movq p+4*(n-1)(CTX), RT0; \
106 rorq $32, RT0; \
107 xorq RT0, RX0;
108
109#define round_dec(n) \
110 add_roundkey_dec(n); \
111 \
112 F(RX0, RK0); \
113 F(RX0, RK0); \
114
115#define read_block() \
116 movq (RIO), RX0; \
117 rorq $32, RX0; \
118 bswapq RX0;
119
120#define write_block() \
121 bswapq RX0; \
122 movq RX0, (RIO);
123
124#define xor_block() \
125 bswapq RX0; \
126 xorq RX0, (RIO);
127
128.align 8
129.global __blowfish_enc_blk
130.type __blowfish_enc_blk,@function;
131
132__blowfish_enc_blk:
133 // input:
134 // %rdi: ctx, CTX
135 // %rsi: dst
136 // %rdx: src
137 // %rcx: bool xor
138 pushq %rbp;
139 pushq %rbx;
140
141 pushq %rsi;
142 pushq %rcx;
143 movq %rdx, RIO;
144
145 read_block();
146
147 round_enc(0);
148 round_enc(2);
149 round_enc(4);
150 round_enc(6);
151 round_enc(8);
152 round_enc(10);
153 round_enc(12);
154 round_enc(14);
155 add_roundkey_enc(16);
156
157 popq %rbp;
158 popq RIO;
159
160 test %bpl, %bpl;
161 jnz __enc_xor;
162
163 write_block();
164
165__enc_ret:
166 popq %rbx;
167 popq %rbp;
168
169 ret;
170
171__enc_xor:
172 xor_block();
173
174 jmp __enc_ret;
175
176.align 8
177.global blowfish_dec_blk
178.type blowfish_dec_blk,@function;
179
180blowfish_dec_blk:
181 // input:
182 // %rdi: ctx, CTX
183 // %rsi: dst
184 // %rdx: src
185 pushq %rbp;
186 pushq %rbx;
187
188 pushq %rsi;
189 movq %rdx, RIO;
190
191 read_block();
192
193 round_dec(17);
194 round_dec(15);
195 round_dec(13);
196 round_dec(11);
197 round_dec(9);
198 round_dec(7);
199 round_dec(5);
200 round_dec(3);
201 add_roundkey_dec(1);
202
203 popq RIO;
204 write_block();
205
206 popq %rbx;
207 popq %rbp;
208
209 ret;
210
211/**********************************************************************
212 4-way blowfish, four blocks parallel
213 **********************************************************************/
214#define add_preloaded_roundkey4() \
215 xorq RKEY, RX0; \
216 xorq RKEY, RX1; \
217 xorq RKEY, RX2; \
218 xorq RKEY, RX3;
219
220#define preload_roundkey_enc(n) \
221 movq p+4*(n)(CTX), RKEY;
222
223#define add_roundkey_enc4(n) \
224 add_preloaded_roundkey4(); \
225 preload_roundkey_enc(n + 2);
226
227#define round_enc4(n) \
228 add_roundkey_enc4(n); \
229 \
230 F(RX0, RK0); \
231 F(RX1, RK1); \
232 F(RX2, RK2); \
233 F(RX3, RK3); \
234 \
235 F(RX0, RK0); \
236 F(RX1, RK1); \
237 F(RX2, RK2); \
238 F(RX3, RK3);
239
240#define preload_roundkey_dec(n) \
241 movq p+4*((n)-1)(CTX), RKEY; \
242 rorq $32, RKEY;
243
244#define add_roundkey_dec4(n) \
245 add_preloaded_roundkey4(); \
246 preload_roundkey_dec(n - 2);
247
248#define round_dec4(n) \
249 add_roundkey_dec4(n); \
250 \
251 F(RX0, RK0); \
252 F(RX1, RK1); \
253 F(RX2, RK2); \
254 F(RX3, RK3); \
255 \
256 F(RX0, RK0); \
257 F(RX1, RK1); \
258 F(RX2, RK2); \
259 F(RX3, RK3);
260
261#define read_block4() \
262 movq (RIO), RX0; \
263 rorq $32, RX0; \
264 bswapq RX0; \
265 \
266 movq 8(RIO), RX1; \
267 rorq $32, RX1; \
268 bswapq RX1; \
269 \
270 movq 16(RIO), RX2; \
271 rorq $32, RX2; \
272 bswapq RX2; \
273 \
274 movq 24(RIO), RX3; \
275 rorq $32, RX3; \
276 bswapq RX3;
277
278#define write_block4() \
279 bswapq RX0; \
280 movq RX0, (RIO); \
281 \
282 bswapq RX1; \
283 movq RX1, 8(RIO); \
284 \
285 bswapq RX2; \
286 movq RX2, 16(RIO); \
287 \
288 bswapq RX3; \
289 movq RX3, 24(RIO);
290
291#define xor_block4() \
292 bswapq RX0; \
293 xorq RX0, (RIO); \
294 \
295 bswapq RX1; \
296 xorq RX1, 8(RIO); \
297 \
298 bswapq RX2; \
299 xorq RX2, 16(RIO); \
300 \
301 bswapq RX3; \
302 xorq RX3, 24(RIO);
303
304.align 8
305.global __blowfish_enc_blk_4way
306.type __blowfish_enc_blk_4way,@function;
307
308__blowfish_enc_blk_4way:
309 // input:
310 // %rdi: ctx, CTX
311 // %rsi: dst
312 // %rdx: src
313 // %rcx: bool xor
314 pushq %rbp;
315 pushq %rbx;
316 pushq RKEY;
317 preload_roundkey_enc(0);
318
319 pushq %rsi;
320 pushq %rcx;
321 movq %rdx, RIO;
322
323 read_block4();
324
325 round_enc4(0);
326 round_enc4(2);
327 round_enc4(4);
328 round_enc4(6);
329 round_enc4(8);
330 round_enc4(10);
331 round_enc4(12);
332 round_enc4(14);
333 add_preloaded_roundkey4();
334
335 popq %rbp;
336 popq RIO;
337
338 test %bpl, %bpl;
339 jnz __enc_xor4;
340
341 write_block4();
342
343__enc_ret4:
344 popq RKEY;
345 popq %rbx;
346 popq %rbp;
347
348 ret;
349
350__enc_xor4:
351 xor_block4();
352
353 jmp __enc_ret4;
354
355.align 8
356.global blowfish_dec_blk_4way
357.type blowfish_dec_blk_4way,@function;
358
359blowfish_dec_blk_4way:
360 // input:
361 // %rdi: ctx, CTX
362 // %rsi: dst
363 // %rdx: src
364 pushq %rbp;
365 pushq %rbx;
366 pushq RKEY;
367 preload_roundkey_dec(17);
368
369 pushq %rsi;
370 movq %rdx, RIO;
371
372 read_block4();
373
374 round_dec4(17);
375 round_dec4(15);
376 round_dec4(13);
377 round_dec4(11);
378 round_dec4(9);
379 round_dec4(7);
380 round_dec4(5);
381 round_dec4(3);
382 add_preloaded_roundkey4();
383
384 popq RIO;
385 write_block4();
386
387 popq RKEY;
388 popq %rbx;
389 popq %rbp;
390
391 ret;
392
diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c
new file mode 100644
index 000000000000..40911abb7b11
--- /dev/null
+++ b/arch/x86/crypto/blowfish_glue.c
@@ -0,0 +1,487 @@
1/*
2 * Glue Code for assembler optimized version of Blowfish
3 *
4 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
19 * USA
20 *
21 */
22
23#include <crypto/blowfish.h>
24#include <linux/crypto.h>
25#include <linux/init.h>
26#include <linux/module.h>
27#include <linux/types.h>
28#include <crypto/algapi.h>
29
30/* regular block cipher functions */
31asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src,
32 bool xor);
33asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);
34
35/* 4-way parallel cipher functions */
36asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
37 const u8 *src, bool xor);
38asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,
39 const u8 *src);
40
41static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src)
42{
43 __blowfish_enc_blk(ctx, dst, src, false);
44}
45
46static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst,
47 const u8 *src)
48{
49 __blowfish_enc_blk(ctx, dst, src, true);
50}
51
52static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
53 const u8 *src)
54{
55 __blowfish_enc_blk_4way(ctx, dst, src, false);
56}
57
58static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst,
59 const u8 *src)
60{
61 __blowfish_enc_blk_4way(ctx, dst, src, true);
62}
63
64static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
65{
66 blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src);
67}
68
69static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
70{
71 blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
72}
73
74static struct crypto_alg bf_alg = {
75 .cra_name = "blowfish",
76 .cra_driver_name = "blowfish-asm",
77 .cra_priority = 200,
78 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
79 .cra_blocksize = BF_BLOCK_SIZE,
80 .cra_ctxsize = sizeof(struct bf_ctx),
81 .cra_alignmask = 3,
82 .cra_module = THIS_MODULE,
83 .cra_list = LIST_HEAD_INIT(bf_alg.cra_list),
84 .cra_u = {
85 .cipher = {
86 .cia_min_keysize = BF_MIN_KEY_SIZE,
87 .cia_max_keysize = BF_MAX_KEY_SIZE,
88 .cia_setkey = blowfish_setkey,
89 .cia_encrypt = blowfish_encrypt,
90 .cia_decrypt = blowfish_decrypt,
91 }
92 }
93};
94
95static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
96 void (*fn)(struct bf_ctx *, u8 *, const u8 *),
97 void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *))
98{
99 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
100 unsigned int bsize = BF_BLOCK_SIZE;
101 unsigned int nbytes;
102 int err;
103
104 err = blkcipher_walk_virt(desc, walk);
105
106 while ((nbytes = walk->nbytes)) {
107 u8 *wsrc = walk->src.virt.addr;
108 u8 *wdst = walk->dst.virt.addr;
109
110 /* Process four block batch */
111 if (nbytes >= bsize * 4) {
112 do {
113 fn_4way(ctx, wdst, wsrc);
114
115 wsrc += bsize * 4;
116 wdst += bsize * 4;
117 nbytes -= bsize * 4;
118 } while (nbytes >= bsize * 4);
119
120 if (nbytes < bsize)
121 goto done;
122 }
123
124 /* Handle leftovers */
125 do {
126 fn(ctx, wdst, wsrc);
127
128 wsrc += bsize;
129 wdst += bsize;
130 nbytes -= bsize;
131 } while (nbytes >= bsize);
132
133done:
134 err = blkcipher_walk_done(desc, walk, nbytes);
135 }
136
137 return err;
138}
139
140static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
141 struct scatterlist *src, unsigned int nbytes)
142{
143 struct blkcipher_walk walk;
144
145 blkcipher_walk_init(&walk, dst, src, nbytes);
146 return ecb_crypt(desc, &walk, blowfish_enc_blk, blowfish_enc_blk_4way);
147}
148
149static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
150 struct scatterlist *src, unsigned int nbytes)
151{
152 struct blkcipher_walk walk;
153
154 blkcipher_walk_init(&walk, dst, src, nbytes);
155 return ecb_crypt(desc, &walk, blowfish_dec_blk, blowfish_dec_blk_4way);
156}
157
158static struct crypto_alg blk_ecb_alg = {
159 .cra_name = "ecb(blowfish)",
160 .cra_driver_name = "ecb-blowfish-asm",
161 .cra_priority = 300,
162 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
163 .cra_blocksize = BF_BLOCK_SIZE,
164 .cra_ctxsize = sizeof(struct bf_ctx),
165 .cra_alignmask = 0,
166 .cra_type = &crypto_blkcipher_type,
167 .cra_module = THIS_MODULE,
168 .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
169 .cra_u = {
170 .blkcipher = {
171 .min_keysize = BF_MIN_KEY_SIZE,
172 .max_keysize = BF_MAX_KEY_SIZE,
173 .setkey = blowfish_setkey,
174 .encrypt = ecb_encrypt,
175 .decrypt = ecb_decrypt,
176 },
177 },
178};
179
180static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
181 struct blkcipher_walk *walk)
182{
183 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
184 unsigned int bsize = BF_BLOCK_SIZE;
185 unsigned int nbytes = walk->nbytes;
186 u64 *src = (u64 *)walk->src.virt.addr;
187 u64 *dst = (u64 *)walk->dst.virt.addr;
188 u64 *iv = (u64 *)walk->iv;
189
190 do {
191 *dst = *src ^ *iv;
192 blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
193 iv = dst;
194
195 src += 1;
196 dst += 1;
197 nbytes -= bsize;
198 } while (nbytes >= bsize);
199
200 *(u64 *)walk->iv = *iv;
201 return nbytes;
202}
203
204static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
205 struct scatterlist *src, unsigned int nbytes)
206{
207 struct blkcipher_walk walk;
208 int err;
209
210 blkcipher_walk_init(&walk, dst, src, nbytes);
211 err = blkcipher_walk_virt(desc, &walk);
212
213 while ((nbytes = walk.nbytes)) {
214 nbytes = __cbc_encrypt(desc, &walk);
215 err = blkcipher_walk_done(desc, &walk, nbytes);
216 }
217
218 return err;
219}
220
221static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
222 struct blkcipher_walk *walk)
223{
224 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
225 unsigned int bsize = BF_BLOCK_SIZE;
226 unsigned int nbytes = walk->nbytes;
227 u64 *src = (u64 *)walk->src.virt.addr;
228 u64 *dst = (u64 *)walk->dst.virt.addr;
229 u64 ivs[4 - 1];
230 u64 last_iv;
231
232 /* Start of the last block. */
233 src += nbytes / bsize - 1;
234 dst += nbytes / bsize - 1;
235
236 last_iv = *src;
237
238 /* Process four block batch */
239 if (nbytes >= bsize * 4) {
240 do {
241 nbytes -= bsize * 4 - bsize;
242 src -= 4 - 1;
243 dst -= 4 - 1;
244
245 ivs[0] = src[0];
246 ivs[1] = src[1];
247 ivs[2] = src[2];
248
249 blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src);
250
251 dst[1] ^= ivs[0];
252 dst[2] ^= ivs[1];
253 dst[3] ^= ivs[2];
254
255 nbytes -= bsize;
256 if (nbytes < bsize)
257 goto done;
258
259 *dst ^= *(src - 1);
260 src -= 1;
261 dst -= 1;
262 } while (nbytes >= bsize * 4);
263
264 if (nbytes < bsize)
265 goto done;
266 }
267
268 /* Handle leftovers */
269 for (;;) {
270 blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
271
272 nbytes -= bsize;
273 if (nbytes < bsize)
274 break;
275
276 *dst ^= *(src - 1);
277 src -= 1;
278 dst -= 1;
279 }
280
281done:
282 *dst ^= *(u64 *)walk->iv;
283 *(u64 *)walk->iv = last_iv;
284
285 return nbytes;
286}
287
288static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
289 struct scatterlist *src, unsigned int nbytes)
290{
291 struct blkcipher_walk walk;
292 int err;
293
294 blkcipher_walk_init(&walk, dst, src, nbytes);
295 err = blkcipher_walk_virt(desc, &walk);
296
297 while ((nbytes = walk.nbytes)) {
298 nbytes = __cbc_decrypt(desc, &walk);
299 err = blkcipher_walk_done(desc, &walk, nbytes);
300 }
301
302 return err;
303}
304
305static struct crypto_alg blk_cbc_alg = {
306 .cra_name = "cbc(blowfish)",
307 .cra_driver_name = "cbc-blowfish-asm",
308 .cra_priority = 300,
309 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
310 .cra_blocksize = BF_BLOCK_SIZE,
311 .cra_ctxsize = sizeof(struct bf_ctx),
312 .cra_alignmask = 0,
313 .cra_type = &crypto_blkcipher_type,
314 .cra_module = THIS_MODULE,
315 .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
316 .cra_u = {
317 .blkcipher = {
318 .min_keysize = BF_MIN_KEY_SIZE,
319 .max_keysize = BF_MAX_KEY_SIZE,
320 .ivsize = BF_BLOCK_SIZE,
321 .setkey = blowfish_setkey,
322 .encrypt = cbc_encrypt,
323 .decrypt = cbc_decrypt,
324 },
325 },
326};
327
328static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk)
329{
330 u8 *ctrblk = walk->iv;
331 u8 keystream[BF_BLOCK_SIZE];
332 u8 *src = walk->src.virt.addr;
333 u8 *dst = walk->dst.virt.addr;
334 unsigned int nbytes = walk->nbytes;
335
336 blowfish_enc_blk(ctx, keystream, ctrblk);
337 crypto_xor(keystream, src, nbytes);
338 memcpy(dst, keystream, nbytes);
339
340 crypto_inc(ctrblk, BF_BLOCK_SIZE);
341}
342
343static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
344 struct blkcipher_walk *walk)
345{
346 struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
347 unsigned int bsize = BF_BLOCK_SIZE;
348 unsigned int nbytes = walk->nbytes;
349 u64 *src = (u64 *)walk->src.virt.addr;
350 u64 *dst = (u64 *)walk->dst.virt.addr;
351 u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
352 __be64 ctrblocks[4];
353
354 /* Process four block batch */
355 if (nbytes >= bsize * 4) {
356 do {
357 if (dst != src) {
358 dst[0] = src[0];
359 dst[1] = src[1];
360 dst[2] = src[2];
361 dst[3] = src[3];
362 }
363
364 /* create ctrblks for parallel encrypt */
365 ctrblocks[0] = cpu_to_be64(ctrblk++);
366 ctrblocks[1] = cpu_to_be64(ctrblk++);
367 ctrblocks[2] = cpu_to_be64(ctrblk++);
368 ctrblocks[3] = cpu_to_be64(ctrblk++);
369
370 blowfish_enc_blk_xor_4way(ctx, (u8 *)dst,
371 (u8 *)ctrblocks);
372
373 src += 4;
374 dst += 4;
375 } while ((nbytes -= bsize * 4) >= bsize * 4);
376
377 if (nbytes < bsize)
378 goto done;
379 }
380
381 /* Handle leftovers */
382 do {
383 if (dst != src)
384 *dst = *src;
385
386 ctrblocks[0] = cpu_to_be64(ctrblk++);
387
388 blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks);
389
390 src += 1;
391 dst += 1;
392 } while ((nbytes -= bsize) >= bsize);
393
394done:
395 *(__be64 *)walk->iv = cpu_to_be64(ctrblk);
396 return nbytes;
397}
398
399static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
400 struct scatterlist *src, unsigned int nbytes)
401{
402 struct blkcipher_walk walk;
403 int err;
404
405 blkcipher_walk_init(&walk, dst, src, nbytes);
406 err = blkcipher_walk_virt_block(desc, &walk, BF_BLOCK_SIZE);
407
408 while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) {
409 nbytes = __ctr_crypt(desc, &walk);
410 err = blkcipher_walk_done(desc, &walk, nbytes);
411 }
412
413 if (walk.nbytes) {
414 ctr_crypt_final(crypto_blkcipher_ctx(desc->tfm), &walk);
415 err = blkcipher_walk_done(desc, &walk, 0);
416 }
417
418 return err;
419}
420
421static struct crypto_alg blk_ctr_alg = {
422 .cra_name = "ctr(blowfish)",
423 .cra_driver_name = "ctr-blowfish-asm",
424 .cra_priority = 300,
425 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
426 .cra_blocksize = BF_BLOCK_SIZE,
427 .cra_ctxsize = sizeof(struct bf_ctx),
428 .cra_alignmask = 0,
429 .cra_type = &crypto_blkcipher_type,
430 .cra_module = THIS_MODULE,
431 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
432 .cra_u = {
433 .blkcipher = {
434 .min_keysize = BF_MIN_KEY_SIZE,
435 .max_keysize = BF_MAX_KEY_SIZE,
436 .ivsize = BF_BLOCK_SIZE,
437 .setkey = blowfish_setkey,
438 .encrypt = ctr_crypt,
439 .decrypt = ctr_crypt,
440 },
441 },
442};
443
444static int __init init(void)
445{
446 int err;
447
448 err = crypto_register_alg(&bf_alg);
449 if (err)
450 goto bf_err;
451 err = crypto_register_alg(&blk_ecb_alg);
452 if (err)
453 goto ecb_err;
454 err = crypto_register_alg(&blk_cbc_alg);
455 if (err)
456 goto cbc_err;
457 err = crypto_register_alg(&blk_ctr_alg);
458 if (err)
459 goto ctr_err;
460
461 return 0;
462
463ctr_err:
464 crypto_unregister_alg(&blk_cbc_alg);
465cbc_err:
466 crypto_unregister_alg(&blk_ecb_alg);
467ecb_err:
468 crypto_unregister_alg(&bf_alg);
469bf_err:
470 return err;
471}
472
473static void __exit fini(void)
474{
475 crypto_unregister_alg(&blk_ctr_alg);
476 crypto_unregister_alg(&blk_cbc_alg);
477 crypto_unregister_alg(&blk_ecb_alg);
478 crypto_unregister_alg(&bf_alg);
479}
480
481module_init(init);
482module_exit(fini);
483
484MODULE_LICENSE("GPL");
485MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized");
486MODULE_ALIAS("blowfish");
487MODULE_ALIAS("blowfish-asm");
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 108cb98e2170..07637745e234 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -620,6 +620,21 @@ config CRYPTO_BLOWFISH_COMMON
620 See also: 620 See also:
621 <http://www.schneier.com/blowfish.html> 621 <http://www.schneier.com/blowfish.html>
622 622
623config CRYPTO_BLOWFISH_X86_64
624 tristate "Blowfish cipher algorithm (x86_64)"
625 depends on (X86 || UML_X86) && 64BIT
626 select CRYPTO_ALGAPI
627 select CRYPTO_BLOWFISH_COMMON
628 help
629 Blowfish cipher algorithm (x86_64), by Bruce Schneier.
630
631 This is a variable key length cipher which can use keys from 32
632 bits to 448 bits in length. It's fast, simple and specifically
633 designed for use on "large microprocessors".
634
635 See also:
636 <http://www.schneier.com/blowfish.html>
637
623config CRYPTO_CAMELLIA 638config CRYPTO_CAMELLIA
624 tristate "Camellia cipher algorithms" 639 tristate "Camellia cipher algorithms"
625 depends on CRYPTO 640 depends on CRYPTO
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-18 19:44:10 -0500