aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2011-01-13 13:25:58 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2011-01-13 13:25:58 -0500
commit27d189c02ba25851973c8582e419c0bded9f7e5b (patch)
treebe142d664bc4e3cec7ab2878a243343f46e897ee
parenta1703154200c390ab03c10224c586e815d3e31e8 (diff)
parent55db8387a5e8d07407f0b7c6b2526417a2bc6243 (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
* git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (46 commits) hwrng: via_rng - Fix memory scribbling on some CPUs crypto: padlock - Move padlock.h into include/crypto hwrng: via_rng - Fix asm constraints crypto: n2 - use __devexit not __exit in n2_unregister_algs crypto: mark crypto workqueues CPU_INTENSIVE crypto: mv_cesa - dont return PTR_ERR() of wrong pointer crypto: ripemd - Set module author and update email address crypto: omap-sham - backlog handling fix crypto: gf128mul - Remove experimental tag crypto: af_alg - fix af_alg memory_allocated data type crypto: aesni-intel - Fixed build with binutils 2.16 crypto: af_alg - Make sure sk_security is initialized on accept()ed sockets net: Add missing lockdep class names for af_alg include: Install linux/if_alg.h for user-space crypto API crypto: omap-aes - checkpatch --file warning fixes crypto: omap-aes - initialize aes module once per request crypto: omap-aes - unnecessary code removed crypto: omap-aes - error handling implementation improved crypto: omap-aes - redundant locking is removed crypto: omap-aes - DMA initialization fixes for OMAP off mode ...
Diffstat
-rw-r--r--arch/x86/crypto/aesni-intel_asm.S1832
-rw-r--r--arch/x86/crypto/aesni-intel_glue.c540
-rw-r--r--crypto/Kconfig34
-rw-r--r--crypto/Makefile17
-rw-r--r--crypto/af_alg.c483
-rw-r--r--crypto/algif_hash.c319
-rw-r--r--crypto/algif_skcipher.c632
-rw-r--r--crypto/authenc.c22
-rw-r--r--crypto/cast5.c74
-rw-r--r--crypto/crypto_wq.c3
-rw-r--r--crypto/deflate.c3
-rw-r--r--crypto/eseqiv.c18
-rw-r--r--crypto/gcm.c19
-rw-r--r--crypto/pcrypt.c3
-rw-r--r--crypto/rmd128.c3
-rw-r--r--crypto/rmd160.c3
-rw-r--r--crypto/rmd256.c3
-rw-r--r--crypto/rmd320.c3
-rw-r--r--crypto/shash.c8
-rw-r--r--crypto/tcrypt.c11
-rw-r--r--crypto/testmgr.c24
-rw-r--r--crypto/testmgr.h361
-rw-r--r--crypto/zlib.c3
-rw-r--r--drivers/char/hw_random/via-rng.c10
-rw-r--r--drivers/crypto/mv_cesa.c2
-rw-r--r--drivers/crypto/n2_core.c2
-rw-r--r--drivers/crypto/omap-aes.c260
-rw-r--r--drivers/crypto/omap-sham.c374
-rw-r--r--drivers/crypto/padlock-aes.c2
-rw-r--r--drivers/crypto/padlock-sha.c8
-rw-r--r--include/crypto/if_alg.h92
-rw-r--r--include/crypto/padlock.h (renamed from drivers/crypto/padlock.h)8
-rw-r--r--include/crypto/scatterwalk.h15
-rw-r--r--include/linux/Kbuild1
-rw-r--r--include/linux/if_alg.h40
-rw-r--r--include/linux/socket.h5
-rw-r--r--net/core/sock.c6
37 files changed, 4779 insertions, 464 deletions
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index ff16756a51c1..8fe2a4966b7a 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -9,6 +9,20 @@
9 * Vinodh Gopal <vinodh.gopal@intel.com> 9 * Vinodh Gopal <vinodh.gopal@intel.com>
10 * Kahraman Akdemir 10 * Kahraman Akdemir
11 * 11 *
12 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
13 * interface for 64-bit kernels.
14 * Authors: Erdinc Ozturk (erdinc.ozturk@intel.com)
15 * Aidan O'Mahony (aidan.o.mahony@intel.com)
16 * Adrian Hoban <adrian.hoban@intel.com>
17 * James Guilford (james.guilford@intel.com)
18 * Gabriele Paoloni <gabriele.paoloni@intel.com>
19 * Tadeusz Struk (tadeusz.struk@intel.com)
20 * Wajdi Feghali (wajdi.k.feghali@intel.com)
21 * Copyright (c) 2010, Intel Corporation.
22 *
23 * Ported x86_64 version to x86:
24 * Author: Mathias Krause <minipli@googlemail.com>
25 *
12 * This program is free software; you can redistribute it and/or modify 26 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by 27 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or 28 * the Free Software Foundation; either version 2 of the License, or
@@ -18,8 +32,62 @@
18#include <linux/linkage.h> 32#include <linux/linkage.h>
19#include <asm/inst.h> 33#include <asm/inst.h>
20 34
35#ifdef __x86_64__
36.data
37POLY: .octa 0xC2000000000000000000000000000001
38TWOONE: .octa 0x00000001000000000000000000000001
39
40# order of these constants should not change.
41# more specifically, ALL_F should follow SHIFT_MASK,
42# and ZERO should follow ALL_F
43
44SHUF_MASK: .octa 0x000102030405060708090A0B0C0D0E0F
45MASK1: .octa 0x0000000000000000ffffffffffffffff
46MASK2: .octa 0xffffffffffffffff0000000000000000
47SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100
48ALL_F: .octa 0xffffffffffffffffffffffffffffffff
49ZERO: .octa 0x00000000000000000000000000000000
50ONE: .octa 0x00000000000000000000000000000001
51F_MIN_MASK: .octa 0xf1f2f3f4f5f6f7f8f9fafbfcfdfeff0
52dec: .octa 0x1
53enc: .octa 0x2
54
55
21.text 56.text
22 57
58
59#define STACK_OFFSET 8*3
60#define HashKey 16*0 // store HashKey <<1 mod poly here
61#define HashKey_2 16*1 // store HashKey^2 <<1 mod poly here
62#define HashKey_3 16*2 // store HashKey^3 <<1 mod poly here
63#define HashKey_4 16*3 // store HashKey^4 <<1 mod poly here
64#define HashKey_k 16*4 // store XOR of High 64 bits and Low 64
65 // bits of HashKey <<1 mod poly here
66 //(for Karatsuba purposes)
67#define HashKey_2_k 16*5 // store XOR of High 64 bits and Low 64
68 // bits of HashKey^2 <<1 mod poly here
69 // (for Karatsuba purposes)
70#define HashKey_3_k 16*6 // store XOR of High 64 bits and Low 64
71 // bits of HashKey^3 <<1 mod poly here
72 // (for Karatsuba purposes)
73#define HashKey_4_k 16*7 // store XOR of High 64 bits and Low 64
74 // bits of HashKey^4 <<1 mod poly here
75 // (for Karatsuba purposes)
76#define VARIABLE_OFFSET 16*8
77
78#define arg1 rdi
79#define arg2 rsi
80#define arg3 rdx
81#define arg4 rcx
82#define arg5 r8
83#define arg6 r9
84#define arg7 STACK_OFFSET+8(%r14)
85#define arg8 STACK_OFFSET+16(%r14)
86#define arg9 STACK_OFFSET+24(%r14)
87#define arg10 STACK_OFFSET+32(%r14)
88#endif
89
90
23#define STATE1 %xmm0 91#define STATE1 %xmm0
24#define STATE2 %xmm4 92#define STATE2 %xmm4
25#define STATE3 %xmm5 93#define STATE3 %xmm5
@@ -32,12 +100,16 @@
32#define IN IN1 100#define IN IN1
33#define KEY %xmm2 101#define KEY %xmm2
34#define IV %xmm3 102#define IV %xmm3
103
35#define BSWAP_MASK %xmm10 104#define BSWAP_MASK %xmm10
36#define CTR %xmm11 105#define CTR %xmm11
37#define INC %xmm12 106#define INC %xmm12
38 107
108#ifdef __x86_64__
109#define AREG %rax
39#define KEYP %rdi 110#define KEYP %rdi
40#define OUTP %rsi 111#define OUTP %rsi
112#define UKEYP OUTP
41#define INP %rdx 113#define INP %rdx
42#define LEN %rcx 114#define LEN %rcx
43#define IVP %r8 115#define IVP %r8
@@ -46,6 +118,1588 @@
46#define TKEYP T1 118#define TKEYP T1
47#define T2 %r11 119#define T2 %r11
48#define TCTR_LOW T2 120#define TCTR_LOW T2
121#else
122#define AREG %eax
123#define KEYP %edi
124#define OUTP AREG
125#define UKEYP OUTP
126#define INP %edx
127#define LEN %esi
128#define IVP %ebp
129#define KLEN %ebx
130#define T1 %ecx
131#define TKEYP T1
132#endif
133
134
135#ifdef __x86_64__
136/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
137*
138*
139* Input: A and B (128-bits each, bit-reflected)
140* Output: C = A*B*x mod poly, (i.e. >>1 )
141* To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
142* GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
143*
144*/
145.macro GHASH_MUL GH HK TMP1 TMP2 TMP3 TMP4 TMP5
146 movdqa \GH, \TMP1
147 pshufd $78, \GH, \TMP2
148 pshufd $78, \HK, \TMP3
149 pxor \GH, \TMP2 # TMP2 = a1+a0
150 pxor \HK, \TMP3 # TMP3 = b1+b0
151 PCLMULQDQ 0x11, \HK, \TMP1 # TMP1 = a1*b1
152 PCLMULQDQ 0x00, \HK, \GH # GH = a0*b0
153 PCLMULQDQ 0x00, \TMP3, \TMP2 # TMP2 = (a0+a1)*(b1+b0)
154 pxor \GH, \TMP2
155 pxor \TMP1, \TMP2 # TMP2 = (a0*b0)+(a1*b0)
156 movdqa \TMP2, \TMP3
157 pslldq $8, \TMP3 # left shift TMP3 2 DWs
158 psrldq $8, \TMP2 # right shift TMP2 2 DWs
159 pxor \TMP3, \GH
160 pxor \TMP2, \TMP1 # TMP2:GH holds the result of GH*HK
161
162 # first phase of the reduction
163
164 movdqa \GH, \TMP2
165 movdqa \GH, \TMP3
166 movdqa \GH, \TMP4 # copy GH into TMP2,TMP3 and TMP4
167 # in in order to perform
168 # independent shifts
169 pslld $31, \TMP2 # packed right shift <<31
170 pslld $30, \TMP3 # packed right shift <<30
171 pslld $25, \TMP4 # packed right shift <<25
172 pxor \TMP3, \TMP2 # xor the shifted versions
173 pxor \TMP4, \TMP2
174 movdqa \TMP2, \TMP5
175 psrldq $4, \TMP5 # right shift TMP5 1 DW
176 pslldq $12, \TMP2 # left shift TMP2 3 DWs
177 pxor \TMP2, \GH
178
179 # second phase of the reduction
180
181 movdqa \GH,\TMP2 # copy GH into TMP2,TMP3 and TMP4
182 # in in order to perform
183 # independent shifts
184 movdqa \GH,\TMP3
185 movdqa \GH,\TMP4
186 psrld $1,\TMP2 # packed left shift >>1
187 psrld $2,\TMP3 # packed left shift >>2
188 psrld $7,\TMP4 # packed left shift >>7
189 pxor \TMP3,\TMP2 # xor the shifted versions
190 pxor \TMP4,\TMP2
191 pxor \TMP5, \TMP2
192 pxor \TMP2, \GH
193 pxor \TMP1, \GH # result is in TMP1
194.endm
195
196/*
197* if a = number of total plaintext bytes
198* b = floor(a/16)
199* num_initial_blocks = b mod 4
200* encrypt the initial num_initial_blocks blocks and apply ghash on
201* the ciphertext
202* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
203* are clobbered
204* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified
205*/
206
207
208.macro INITIAL_BLOCKS_DEC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
209XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
210 mov arg7, %r10 # %r10 = AAD
211 mov arg8, %r12 # %r12 = aadLen
212 mov %r12, %r11
213 pxor %xmm\i, %xmm\i
214_get_AAD_loop\num_initial_blocks\operation:
215 movd (%r10), \TMP1
216 pslldq $12, \TMP1
217 psrldq $4, %xmm\i
218 pxor \TMP1, %xmm\i
219 add $4, %r10
220 sub $4, %r12
221 jne _get_AAD_loop\num_initial_blocks\operation
222 cmp $16, %r11
223 je _get_AAD_loop2_done\num_initial_blocks\operation
224 mov $16, %r12
225_get_AAD_loop2\num_initial_blocks\operation:
226 psrldq $4, %xmm\i
227 sub $4, %r12
228 cmp %r11, %r12
229 jne _get_AAD_loop2\num_initial_blocks\operation
230_get_AAD_loop2_done\num_initial_blocks\operation:
231 movdqa SHUF_MASK(%rip), %xmm14
232 PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data
233
234 xor %r11, %r11 # initialise the data pointer offset as zero
235
236 # start AES for num_initial_blocks blocks
237
238 mov %arg5, %rax # %rax = *Y0
239 movdqu (%rax), \XMM0 # XMM0 = Y0
240 movdqa SHUF_MASK(%rip), %xmm14
241 PSHUFB_XMM %xmm14, \XMM0
242
243.if (\i == 5) || (\i == 6) || (\i == 7)
244.irpc index, \i_seq
245 paddd ONE(%rip), \XMM0 # INCR Y0
246 movdqa \XMM0, %xmm\index
247 movdqa SHUF_MASK(%rip), %xmm14
248 PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap
249
250.endr
251.irpc index, \i_seq
252 pxor 16*0(%arg1), %xmm\index
253.endr
254.irpc index, \i_seq
255 movaps 0x10(%rdi), \TMP1
256 AESENC \TMP1, %xmm\index # Round 1
257.endr
258.irpc index, \i_seq
259 movaps 0x20(%arg1), \TMP1
260 AESENC \TMP1, %xmm\index # Round 2
261.endr
262.irpc index, \i_seq
263 movaps 0x30(%arg1), \TMP1
264 AESENC \TMP1, %xmm\index # Round 2
265.endr
266.irpc index, \i_seq
267 movaps 0x40(%arg1), \TMP1
268 AESENC \TMP1, %xmm\index # Round 2
269.endr
270.irpc index, \i_seq
271 movaps 0x50(%arg1), \TMP1
272 AESENC \TMP1, %xmm\index # Round 2
273.endr
274.irpc index, \i_seq
275 movaps 0x60(%arg1), \TMP1
276 AESENC \TMP1, %xmm\index # Round 2
277.endr
278.irpc index, \i_seq
279 movaps 0x70(%arg1), \TMP1
280 AESENC \TMP1, %xmm\index # Round 2
281.endr
282.irpc index, \i_seq
283 movaps 0x80(%arg1), \TMP1
284 AESENC \TMP1, %xmm\index # Round 2
285.endr
286.irpc index, \i_seq
287 movaps 0x90(%arg1), \TMP1
288 AESENC \TMP1, %xmm\index # Round 2
289.endr
290.irpc index, \i_seq
291 movaps 0xa0(%arg1), \TMP1
292 AESENCLAST \TMP1, %xmm\index # Round 10
293.endr
294.irpc index, \i_seq
295 movdqu (%arg3 , %r11, 1), \TMP1
296 pxor \TMP1, %xmm\index
297 movdqu %xmm\index, (%arg2 , %r11, 1)
298 # write back plaintext/ciphertext for num_initial_blocks
299 add $16, %r11
300
301 movdqa \TMP1, %xmm\index
302 movdqa SHUF_MASK(%rip), %xmm14
303 PSHUFB_XMM %xmm14, %xmm\index
304
305 # prepare plaintext/ciphertext for GHASH computation
306.endr
307.endif
308 GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
309 # apply GHASH on num_initial_blocks blocks
310
311.if \i == 5
312 pxor %xmm5, %xmm6
313 GHASH_MUL %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
314 pxor %xmm6, %xmm7
315 GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
316 pxor %xmm7, %xmm8
317 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
318.elseif \i == 6
319 pxor %xmm6, %xmm7
320 GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
321 pxor %xmm7, %xmm8
322 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
323.elseif \i == 7
324 pxor %xmm7, %xmm8
325 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
326.endif
327 cmp $64, %r13
328 jl _initial_blocks_done\num_initial_blocks\operation
329 # no need for precomputed values
330/*
331*
332* Precomputations for HashKey parallel with encryption of first 4 blocks.
333* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
334*/
335 paddd ONE(%rip), \XMM0 # INCR Y0
336 movdqa \XMM0, \XMM1
337 movdqa SHUF_MASK(%rip), %xmm14
338 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
339
340 paddd ONE(%rip), \XMM0 # INCR Y0
341 movdqa \XMM0, \XMM2
342 movdqa SHUF_MASK(%rip), %xmm14
343 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
344
345 paddd ONE(%rip), \XMM0 # INCR Y0
346 movdqa \XMM0, \XMM3
347 movdqa SHUF_MASK(%rip), %xmm14
348 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
349
350 paddd ONE(%rip), \XMM0 # INCR Y0
351 movdqa \XMM0, \XMM4
352 movdqa SHUF_MASK(%rip), %xmm14
353 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
354
355 pxor 16*0(%arg1), \XMM1
356 pxor 16*0(%arg1), \XMM2
357 pxor 16*0(%arg1), \XMM3
358 pxor 16*0(%arg1), \XMM4
359 movdqa \TMP3, \TMP5
360 pshufd $78, \TMP3, \TMP1
361 pxor \TMP3, \TMP1
362 movdqa \TMP1, HashKey_k(%rsp)
363 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
364# TMP5 = HashKey^2<<1 (mod poly)
365 movdqa \TMP5, HashKey_2(%rsp)
366# HashKey_2 = HashKey^2<<1 (mod poly)
367 pshufd $78, \TMP5, \TMP1
368 pxor \TMP5, \TMP1
369 movdqa \TMP1, HashKey_2_k(%rsp)
370.irpc index, 1234 # do 4 rounds
371 movaps 0x10*\index(%arg1), \TMP1
372 AESENC \TMP1, \XMM1
373 AESENC \TMP1, \XMM2
374 AESENC \TMP1, \XMM3
375 AESENC \TMP1, \XMM4
376.endr
377 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
378# TMP5 = HashKey^3<<1 (mod poly)
379 movdqa \TMP5, HashKey_3(%rsp)
380 pshufd $78, \TMP5, \TMP1
381 pxor \TMP5, \TMP1
382 movdqa \TMP1, HashKey_3_k(%rsp)
383.irpc index, 56789 # do next 5 rounds
384 movaps 0x10*\index(%arg1), \TMP1
385 AESENC \TMP1, \XMM1
386 AESENC \TMP1, \XMM2
387 AESENC \TMP1, \XMM3
388 AESENC \TMP1, \XMM4
389.endr
390 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
391# TMP5 = HashKey^3<<1 (mod poly)
392 movdqa \TMP5, HashKey_4(%rsp)
393 pshufd $78, \TMP5, \TMP1
394 pxor \TMP5, \TMP1
395 movdqa \TMP1, HashKey_4_k(%rsp)
396 movaps 0xa0(%arg1), \TMP2
397 AESENCLAST \TMP2, \XMM1
398 AESENCLAST \TMP2, \XMM2
399 AESENCLAST \TMP2, \XMM3
400 AESENCLAST \TMP2, \XMM4
401 movdqu 16*0(%arg3 , %r11 , 1), \TMP1
402 pxor \TMP1, \XMM1
403 movdqu \XMM1, 16*0(%arg2 , %r11 , 1)
404 movdqa \TMP1, \XMM1
405 movdqu 16*1(%arg3 , %r11 , 1), \TMP1
406 pxor \TMP1, \XMM2
407 movdqu \XMM2, 16*1(%arg2 , %r11 , 1)
408 movdqa \TMP1, \XMM2
409 movdqu 16*2(%arg3 , %r11 , 1), \TMP1
410 pxor \TMP1, \XMM3
411 movdqu \XMM3, 16*2(%arg2 , %r11 , 1)
412 movdqa \TMP1, \XMM3
413 movdqu 16*3(%arg3 , %r11 , 1), \TMP1
414 pxor \TMP1, \XMM4
415 movdqu \XMM4, 16*3(%arg2 , %r11 , 1)
416 movdqa \TMP1, \XMM4
417 add $64, %r11
418 movdqa SHUF_MASK(%rip), %xmm14
419 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
420 pxor \XMMDst, \XMM1
421# combine GHASHed value with the corresponding ciphertext
422 movdqa SHUF_MASK(%rip), %xmm14
423 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
424 movdqa SHUF_MASK(%rip), %xmm14
425 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
426 movdqa SHUF_MASK(%rip), %xmm14
427 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
428
429_initial_blocks_done\num_initial_blocks\operation:
430
431.endm
432
433
434/*
435* if a = number of total plaintext bytes
436* b = floor(a/16)
437* num_initial_blocks = b mod 4
438* encrypt the initial num_initial_blocks blocks and apply ghash on
439* the ciphertext
440* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
441* are clobbered
442* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified
443*/
444
445
446.macro INITIAL_BLOCKS_ENC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
447XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
448 mov arg7, %r10 # %r10 = AAD
449 mov arg8, %r12 # %r12 = aadLen
450 mov %r12, %r11
451 pxor %xmm\i, %xmm\i
452_get_AAD_loop\num_initial_blocks\operation:
453 movd (%r10), \TMP1
454 pslldq $12, \TMP1
455 psrldq $4, %xmm\i
456 pxor \TMP1, %xmm\i
457 add $4, %r10
458 sub $4, %r12
459 jne _get_AAD_loop\num_initial_blocks\operation
460 cmp $16, %r11
461 je _get_AAD_loop2_done\num_initial_blocks\operation
462 mov $16, %r12
463_get_AAD_loop2\num_initial_blocks\operation:
464 psrldq $4, %xmm\i
465 sub $4, %r12
466 cmp %r11, %r12
467 jne _get_AAD_loop2\num_initial_blocks\operation
468_get_AAD_loop2_done\num_initial_blocks\operation:
469 movdqa SHUF_MASK(%rip), %xmm14
470 PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data
471
472 xor %r11, %r11 # initialise the data pointer offset as zero
473
474 # start AES for num_initial_blocks blocks
475
476 mov %arg5, %rax # %rax = *Y0
477 movdqu (%rax), \XMM0 # XMM0 = Y0
478 movdqa SHUF_MASK(%rip), %xmm14
479 PSHUFB_XMM %xmm14, \XMM0
480
481.if (\i == 5) || (\i == 6) || (\i == 7)
482.irpc index, \i_seq
483 paddd ONE(%rip), \XMM0 # INCR Y0
484 movdqa \XMM0, %xmm\index
485 movdqa SHUF_MASK(%rip), %xmm14
486 PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap
487
488.endr
489.irpc index, \i_seq
490 pxor 16*0(%arg1), %xmm\index
491.endr
492.irpc index, \i_seq
493 movaps 0x10(%rdi), \TMP1
494 AESENC \TMP1, %xmm\index # Round 1
495.endr
496.irpc index, \i_seq
497 movaps 0x20(%arg1), \TMP1
498 AESENC \TMP1, %xmm\index # Round 2
499.endr
500.irpc index, \i_seq
501 movaps 0x30(%arg1), \TMP1
502 AESENC \TMP1, %xmm\index # Round 2
503.endr
504.irpc index, \i_seq
505 movaps 0x40(%arg1), \TMP1
506 AESENC \TMP1, %xmm\index # Round 2
507.endr
508.irpc index, \i_seq
509 movaps 0x50(%arg1), \TMP1
510 AESENC \TMP1, %xmm\index # Round 2
511.endr
512.irpc index, \i_seq
513 movaps 0x60(%arg1), \TMP1
514 AESENC \TMP1, %xmm\index # Round 2
515.endr
516.irpc index, \i_seq
517 movaps 0x70(%arg1), \TMP1
518 AESENC \TMP1, %xmm\index # Round 2
519.endr
520.irpc index, \i_seq
521 movaps 0x80(%arg1), \TMP1
522 AESENC \TMP1, %xmm\index # Round 2
523.endr
524.irpc index, \i_seq
525 movaps 0x90(%arg1), \TMP1
526 AESENC \TMP1, %xmm\index # Round 2
527.endr
528.irpc index, \i_seq
529 movaps 0xa0(%arg1), \TMP1
530 AESENCLAST \TMP1, %xmm\index # Round 10
531.endr
532.irpc index, \i_seq
533 movdqu (%arg3 , %r11, 1), \TMP1
534 pxor \TMP1, %xmm\index
535 movdqu %xmm\index, (%arg2 , %r11, 1)
536 # write back plaintext/ciphertext for num_initial_blocks
537 add $16, %r11
538
539 movdqa SHUF_MASK(%rip), %xmm14
540 PSHUFB_XMM %xmm14, %xmm\index
541
542 # prepare plaintext/ciphertext for GHASH computation
543.endr
544.endif
545 GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
546 # apply GHASH on num_initial_blocks blocks
547
548.if \i == 5
549 pxor %xmm5, %xmm6
550 GHASH_MUL %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
551 pxor %xmm6, %xmm7
552 GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
553 pxor %xmm7, %xmm8
554 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
555.elseif \i == 6
556 pxor %xmm6, %xmm7
557 GHASH_MUL %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
558 pxor %xmm7, %xmm8
559 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
560.elseif \i == 7
561 pxor %xmm7, %xmm8
562 GHASH_MUL %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
563.endif
564 cmp $64, %r13
565 jl _initial_blocks_done\num_initial_blocks\operation
566 # no need for precomputed values
567/*
568*
569* Precomputations for HashKey parallel with encryption of first 4 blocks.
570* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
571*/
572 paddd ONE(%rip), \XMM0 # INCR Y0
573 movdqa \XMM0, \XMM1
574 movdqa SHUF_MASK(%rip), %xmm14
575 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
576
577 paddd ONE(%rip), \XMM0 # INCR Y0
578 movdqa \XMM0, \XMM2
579 movdqa SHUF_MASK(%rip), %xmm14
580 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
581
582 paddd ONE(%rip), \XMM0 # INCR Y0
583 movdqa \XMM0, \XMM3
584 movdqa SHUF_MASK(%rip), %xmm14
585 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
586
587 paddd ONE(%rip), \XMM0 # INCR Y0
588 movdqa \XMM0, \XMM4
589 movdqa SHUF_MASK(%rip), %xmm14
590 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
591
592 pxor 16*0(%arg1), \XMM1
593 pxor 16*0(%arg1), \XMM2
594 pxor 16*0(%arg1), \XMM3
595 pxor 16*0(%arg1), \XMM4
596 movdqa \TMP3, \TMP5
597 pshufd $78, \TMP3, \TMP1
598 pxor \TMP3, \TMP1
599 movdqa \TMP1, HashKey_k(%rsp)
600 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
601# TMP5 = HashKey^2<<1 (mod poly)
602 movdqa \TMP5, HashKey_2(%rsp)
603# HashKey_2 = HashKey^2<<1 (mod poly)
604 pshufd $78, \TMP5, \TMP1
605 pxor \TMP5, \TMP1
606 movdqa \TMP1, HashKey_2_k(%rsp)
607.irpc index, 1234 # do 4 rounds
608 movaps 0x10*\index(%arg1), \TMP1
609 AESENC \TMP1, \XMM1
610 AESENC \TMP1, \XMM2
611 AESENC \TMP1, \XMM3
612 AESENC \TMP1, \XMM4
613.endr
614 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
615# TMP5 = HashKey^3<<1 (mod poly)
616 movdqa \TMP5, HashKey_3(%rsp)
617 pshufd $78, \TMP5, \TMP1
618 pxor \TMP5, \TMP1
619 movdqa \TMP1, HashKey_3_k(%rsp)
620.irpc index, 56789 # do next 5 rounds
621 movaps 0x10*\index(%arg1), \TMP1
622 AESENC \TMP1, \XMM1
623 AESENC \TMP1, \XMM2
624 AESENC \TMP1, \XMM3
625 AESENC \TMP1, \XMM4
626.endr
627 GHASH_MUL \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
628# TMP5 = HashKey^3<<1 (mod poly)
629 movdqa \TMP5, HashKey_4(%rsp)
630 pshufd $78, \TMP5, \TMP1
631 pxor \TMP5, \TMP1
632 movdqa \TMP1, HashKey_4_k(%rsp)
633 movaps 0xa0(%arg1), \TMP2
634 AESENCLAST \TMP2, \XMM1
635 AESENCLAST \TMP2, \XMM2
636 AESENCLAST \TMP2, \XMM3
637 AESENCLAST \TMP2, \XMM4
638 movdqu 16*0(%arg3 , %r11 , 1), \TMP1
639 pxor \TMP1, \XMM1
640 movdqu 16*1(%arg3 , %r11 , 1), \TMP1
641 pxor \TMP1, \XMM2
642 movdqu 16*2(%arg3 , %r11 , 1), \TMP1
643 pxor \TMP1, \XMM3
644 movdqu 16*3(%arg3 , %r11 , 1), \TMP1
645 pxor \TMP1, \XMM4
646 movdqu \XMM1, 16*0(%arg2 , %r11 , 1)
647 movdqu \XMM2, 16*1(%arg2 , %r11 , 1)
648 movdqu \XMM3, 16*2(%arg2 , %r11 , 1)
649 movdqu \XMM4, 16*3(%arg2 , %r11 , 1)
650
651 add $64, %r11
652 movdqa SHUF_MASK(%rip), %xmm14
653 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
654 pxor \XMMDst, \XMM1
655# combine GHASHed value with the corresponding ciphertext
656 movdqa SHUF_MASK(%rip), %xmm14
657 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
658 movdqa SHUF_MASK(%rip), %xmm14
659 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
660 movdqa SHUF_MASK(%rip), %xmm14
661 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
662
663_initial_blocks_done\num_initial_blocks\operation:
664
665.endm
666
667/*
668* encrypt 4 blocks at a time
669* ghash the 4 previously encrypted ciphertext blocks
670* arg1, %arg2, %arg3 are used as pointers only, not modified
671* %r11 is the data offset value
672*/
673.macro GHASH_4_ENCRYPT_4_PARALLEL_ENC TMP1 TMP2 TMP3 TMP4 TMP5 \
674TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
675
676 movdqa \XMM1, \XMM5
677 movdqa \XMM2, \XMM6
678 movdqa \XMM3, \XMM7
679 movdqa \XMM4, \XMM8
680
681 movdqa SHUF_MASK(%rip), %xmm15
682 # multiply TMP5 * HashKey using karatsuba
683
684 movdqa \XMM5, \TMP4
685 pshufd $78, \XMM5, \TMP6
686 pxor \XMM5, \TMP6
687 paddd ONE(%rip), \XMM0 # INCR CNT
688 movdqa HashKey_4(%rsp), \TMP5
689 PCLMULQDQ 0x11, \TMP5, \TMP4 # TMP4 = a1*b1
690 movdqa \XMM0, \XMM1
691 paddd ONE(%rip), \XMM0 # INCR CNT
692 movdqa \XMM0, \XMM2
693 paddd ONE(%rip), \XMM0 # INCR CNT
694 movdqa \XMM0, \XMM3
695 paddd ONE(%rip), \XMM0 # INCR CNT
696 movdqa \XMM0, \XMM4
697 PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap
698 PCLMULQDQ 0x00, \TMP5, \XMM5 # XMM5 = a0*b0
699 PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap
700 PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap
701 PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap
702
703 pxor (%arg1), \XMM1
704 pxor (%arg1), \XMM2
705 pxor (%arg1), \XMM3
706 pxor (%arg1), \XMM4
707 movdqa HashKey_4_k(%rsp), \TMP5
708 PCLMULQDQ 0x00, \TMP5, \TMP6 # TMP6 = (a1+a0)*(b1+b0)
709 movaps 0x10(%arg1), \TMP1
710 AESENC \TMP1, \XMM1 # Round 1
711 AESENC \TMP1, \XMM2
712 AESENC \TMP1, \XMM3
713 AESENC \TMP1, \XMM4
714 movaps 0x20(%arg1), \TMP1
715 AESENC \TMP1, \XMM1 # Round 2
716 AESENC \TMP1, \XMM2
717 AESENC \TMP1, \XMM3
718 AESENC \TMP1, \XMM4
719 movdqa \XMM6, \TMP1
720 pshufd $78, \XMM6, \TMP2
721 pxor \XMM6, \TMP2
722 movdqa HashKey_3(%rsp), \TMP5
723 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1 * b1
724 movaps 0x30(%arg1), \TMP3
725 AESENC \TMP3, \XMM1 # Round 3
726 AESENC \TMP3, \XMM2
727 AESENC \TMP3, \XMM3
728 AESENC \TMP3, \XMM4
729 PCLMULQDQ 0x00, \TMP5, \XMM6 # XMM6 = a0*b0
730 movaps 0x40(%arg1), \TMP3
731 AESENC \TMP3, \XMM1 # Round 4
732 AESENC \TMP3, \XMM2
733 AESENC \TMP3, \XMM3
734 AESENC \TMP3, \XMM4
735 movdqa HashKey_3_k(%rsp), \TMP5
736 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
737 movaps 0x50(%arg1), \TMP3
738 AESENC \TMP3, \XMM1 # Round 5
739 AESENC \TMP3, \XMM2
740 AESENC \TMP3, \XMM3
741 AESENC \TMP3, \XMM4
742 pxor \TMP1, \TMP4
743# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
744 pxor \XMM6, \XMM5
745 pxor \TMP2, \TMP6
746 movdqa \XMM7, \TMP1
747 pshufd $78, \XMM7, \TMP2
748 pxor \XMM7, \TMP2
749 movdqa HashKey_2(%rsp ), \TMP5
750
751 # Multiply TMP5 * HashKey using karatsuba
752
753 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1
754 movaps 0x60(%arg1), \TMP3
755 AESENC \TMP3, \XMM1 # Round 6
756 AESENC \TMP3, \XMM2
757 AESENC \TMP3, \XMM3
758 AESENC \TMP3, \XMM4
759 PCLMULQDQ 0x00, \TMP5, \XMM7 # XMM7 = a0*b0
760 movaps 0x70(%arg1), \TMP3
761 AESENC \TMP3, \XMM1 # Round 7
762 AESENC \TMP3, \XMM2
763 AESENC \TMP3, \XMM3
764 AESENC \TMP3, \XMM4
765 movdqa HashKey_2_k(%rsp), \TMP5
766 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
767 movaps 0x80(%arg1), \TMP3
768 AESENC \TMP3, \XMM1 # Round 8
769 AESENC \TMP3, \XMM2
770 AESENC \TMP3, \XMM3
771 AESENC \TMP3, \XMM4
772 pxor \TMP1, \TMP4
773# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
774 pxor \XMM7, \XMM5
775 pxor \TMP2, \TMP6
776
777 # Multiply XMM8 * HashKey
778 # XMM8 and TMP5 hold the values for the two operands
779
780 movdqa \XMM8, \TMP1
781 pshufd $78, \XMM8, \TMP2
782 pxor \XMM8, \TMP2
783 movdqa HashKey(%rsp), \TMP5
784 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1
785 movaps 0x90(%arg1), \TMP3
786 AESENC \TMP3, \XMM1 # Round 9
787 AESENC \TMP3, \XMM2
788 AESENC \TMP3, \XMM3
789 AESENC \TMP3, \XMM4
790 PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0
791 movaps 0xa0(%arg1), \TMP3
792 AESENCLAST \TMP3, \XMM1 # Round 10
793 AESENCLAST \TMP3, \XMM2
794 AESENCLAST \TMP3, \XMM3
795 AESENCLAST \TMP3, \XMM4
796 movdqa HashKey_k(%rsp), \TMP5
797 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
798 movdqu (%arg3,%r11,1), \TMP3
799 pxor \TMP3, \XMM1 # Ciphertext/Plaintext XOR EK
800 movdqu 16(%arg3,%r11,1), \TMP3
801 pxor \TMP3, \XMM2 # Ciphertext/Plaintext XOR EK
802 movdqu 32(%arg3,%r11,1), \TMP3
803 pxor \TMP3, \XMM3 # Ciphertext/Plaintext XOR EK
804 movdqu 48(%arg3,%r11,1), \TMP3
805 pxor \TMP3, \XMM4 # Ciphertext/Plaintext XOR EK
806 movdqu \XMM1, (%arg2,%r11,1) # Write to the ciphertext buffer
807 movdqu \XMM2, 16(%arg2,%r11,1) # Write to the ciphertext buffer
808 movdqu \XMM3, 32(%arg2,%r11,1) # Write to the ciphertext buffer
809 movdqu \XMM4, 48(%arg2,%r11,1) # Write to the ciphertext buffer
810 PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap
811 PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap
812 PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap
813 PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap
814
815 pxor \TMP4, \TMP1
816 pxor \XMM8, \XMM5
817 pxor \TMP6, \TMP2
818 pxor \TMP1, \TMP2
819 pxor \XMM5, \TMP2
820 movdqa \TMP2, \TMP3
821 pslldq $8, \TMP3 # left shift TMP3 2 DWs
822 psrldq $8, \TMP2 # right shift TMP2 2 DWs
823 pxor \TMP3, \XMM5
824 pxor \TMP2, \TMP1 # accumulate the results in TMP1:XMM5
825
826 # first phase of reduction
827
828 movdqa \XMM5, \TMP2
829 movdqa \XMM5, \TMP3
830 movdqa \XMM5, \TMP4
831# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently
832 pslld $31, \TMP2 # packed right shift << 31
833 pslld $30, \TMP3 # packed right shift << 30
834 pslld $25, \TMP4 # packed right shift << 25
835 pxor \TMP3, \TMP2 # xor the shifted versions
836 pxor \TMP4, \TMP2
837 movdqa \TMP2, \TMP5
838 psrldq $4, \TMP5 # right shift T5 1 DW
839 pslldq $12, \TMP2 # left shift T2 3 DWs
840 pxor \TMP2, \XMM5
841
842 # second phase of reduction
843
844 movdqa \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4
845 movdqa \XMM5,\TMP3
846 movdqa \XMM5,\TMP4
847 psrld $1, \TMP2 # packed left shift >>1
848 psrld $2, \TMP3 # packed left shift >>2
849 psrld $7, \TMP4 # packed left shift >>7
850 pxor \TMP3,\TMP2 # xor the shifted versions
851 pxor \TMP4,\TMP2
852 pxor \TMP5, \TMP2
853 pxor \TMP2, \XMM5
854 pxor \TMP1, \XMM5 # result is in TMP1
855
856 pxor \XMM5, \XMM1
857.endm
858
859/*
860* decrypt 4 blocks at a time
861* ghash the 4 previously decrypted ciphertext blocks
862* arg1, %arg2, %arg3 are used as pointers only, not modified
863* %r11 is the data offset value
864*/
865.macro GHASH_4_ENCRYPT_4_PARALLEL_DEC TMP1 TMP2 TMP3 TMP4 TMP5 \
866TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
867
868 movdqa \XMM1, \XMM5
869 movdqa \XMM2, \XMM6
870 movdqa \XMM3, \XMM7
871 movdqa \XMM4, \XMM8
872
873 movdqa SHUF_MASK(%rip), %xmm15
874 # multiply TMP5 * HashKey using karatsuba
875
876 movdqa \XMM5, \TMP4
877 pshufd $78, \XMM5, \TMP6
878 pxor \XMM5, \TMP6
879 paddd ONE(%rip), \XMM0 # INCR CNT
880 movdqa HashKey_4(%rsp), \TMP5
881 PCLMULQDQ 0x11, \TMP5, \TMP4 # TMP4 = a1*b1
882 movdqa \XMM0, \XMM1
883 paddd ONE(%rip), \XMM0 # INCR CNT
884 movdqa \XMM0, \XMM2
885 paddd ONE(%rip), \XMM0 # INCR CNT
886 movdqa \XMM0, \XMM3
887 paddd ONE(%rip), \XMM0 # INCR CNT
888 movdqa \XMM0, \XMM4
889 PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap
890 PCLMULQDQ 0x00, \TMP5, \XMM5 # XMM5 = a0*b0
891 PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap
892 PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap
893 PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap
894
895 pxor (%arg1), \XMM1
896 pxor (%arg1), \XMM2
897 pxor (%arg1), \XMM3
898 pxor (%arg1), \XMM4
899 movdqa HashKey_4_k(%rsp), \TMP5
900 PCLMULQDQ 0x00, \TMP5, \TMP6 # TMP6 = (a1+a0)*(b1+b0)
901 movaps 0x10(%arg1), \TMP1
902 AESENC \TMP1, \XMM1 # Round 1
903 AESENC \TMP1, \XMM2
904 AESENC \TMP1, \XMM3
905 AESENC \TMP1, \XMM4
906 movaps 0x20(%arg1), \TMP1
907 AESENC \TMP1, \XMM1 # Round 2
908 AESENC \TMP1, \XMM2
909 AESENC \TMP1, \XMM3
910 AESENC \TMP1, \XMM4
911 movdqa \XMM6, \TMP1
912 pshufd $78, \XMM6, \TMP2
913 pxor \XMM6, \TMP2
914 movdqa HashKey_3(%rsp), \TMP5
915 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1 * b1
916 movaps 0x30(%arg1), \TMP3
917 AESENC \TMP3, \XMM1 # Round 3
918 AESENC \TMP3, \XMM2
919 AESENC \TMP3, \XMM3
920 AESENC \TMP3, \XMM4
921 PCLMULQDQ 0x00, \TMP5, \XMM6 # XMM6 = a0*b0
922 movaps 0x40(%arg1), \TMP3
923 AESENC \TMP3, \XMM1 # Round 4
924 AESENC \TMP3, \XMM2
925 AESENC \TMP3, \XMM3
926 AESENC \TMP3, \XMM4
927 movdqa HashKey_3_k(%rsp), \TMP5
928 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
929 movaps 0x50(%arg1), \TMP3
930 AESENC \TMP3, \XMM1 # Round 5
931 AESENC \TMP3, \XMM2
932 AESENC \TMP3, \XMM3
933 AESENC \TMP3, \XMM4
934 pxor \TMP1, \TMP4
935# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
936 pxor \XMM6, \XMM5
937 pxor \TMP2, \TMP6
938 movdqa \XMM7, \TMP1
939 pshufd $78, \XMM7, \TMP2
940 pxor \XMM7, \TMP2
941 movdqa HashKey_2(%rsp ), \TMP5
942
943 # Multiply TMP5 * HashKey using karatsuba
944
945 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1
946 movaps 0x60(%arg1), \TMP3
947 AESENC \TMP3, \XMM1 # Round 6
948 AESENC \TMP3, \XMM2
949 AESENC \TMP3, \XMM3
950 AESENC \TMP3, \XMM4
951 PCLMULQDQ 0x00, \TMP5, \XMM7 # XMM7 = a0*b0
952 movaps 0x70(%arg1), \TMP3
953 AESENC \TMP3, \XMM1 # Round 7
954 AESENC \TMP3, \XMM2
955 AESENC \TMP3, \XMM3
956 AESENC \TMP3, \XMM4
957 movdqa HashKey_2_k(%rsp), \TMP5
958 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
959 movaps 0x80(%arg1), \TMP3
960 AESENC \TMP3, \XMM1 # Round 8
961 AESENC \TMP3, \XMM2
962 AESENC \TMP3, \XMM3
963 AESENC \TMP3, \XMM4
964 pxor \TMP1, \TMP4
965# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
966 pxor \XMM7, \XMM5
967 pxor \TMP2, \TMP6
968
969 # Multiply XMM8 * HashKey
970 # XMM8 and TMP5 hold the values for the two operands
971
972 movdqa \XMM8, \TMP1
973 pshufd $78, \XMM8, \TMP2
974 pxor \XMM8, \TMP2
975 movdqa HashKey(%rsp), \TMP5
976 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1
977 movaps 0x90(%arg1), \TMP3
978 AESENC \TMP3, \XMM1 # Round 9
979 AESENC \TMP3, \XMM2
980 AESENC \TMP3, \XMM3
981 AESENC \TMP3, \XMM4
982 PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0
983 movaps 0xa0(%arg1), \TMP3
984 AESENCLAST \TMP3, \XMM1 # Round 10
985 AESENCLAST \TMP3, \XMM2
986 AESENCLAST \TMP3, \XMM3
987 AESENCLAST \TMP3, \XMM4
988 movdqa HashKey_k(%rsp), \TMP5
989 PCLMULQDQ 0x00, \TMP5, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
990 movdqu (%arg3,%r11,1), \TMP3
991 pxor \TMP3, \XMM1 # Ciphertext/Plaintext XOR EK
992 movdqu \XMM1, (%arg2,%r11,1) # Write to plaintext buffer
993 movdqa \TMP3, \XMM1
994 movdqu 16(%arg3,%r11,1), \TMP3
995 pxor \TMP3, \XMM2 # Ciphertext/Plaintext XOR EK
996 movdqu \XMM2, 16(%arg2,%r11,1) # Write to plaintext buffer
997 movdqa \TMP3, \XMM2
998 movdqu 32(%arg3,%r11,1), \TMP3
999 pxor \TMP3, \XMM3 # Ciphertext/Plaintext XOR EK
1000 movdqu \XMM3, 32(%arg2,%r11,1) # Write to plaintext buffer
1001 movdqa \TMP3, \XMM3
1002 movdqu 48(%arg3,%r11,1), \TMP3
1003 pxor \TMP3, \XMM4 # Ciphertext/Plaintext XOR EK
1004 movdqu \XMM4, 48(%arg2,%r11,1) # Write to plaintext buffer
1005 movdqa \TMP3, \XMM4
1006 PSHUFB_XMM %xmm15, \XMM1 # perform a 16 byte swap
1007 PSHUFB_XMM %xmm15, \XMM2 # perform a 16 byte swap
1008 PSHUFB_XMM %xmm15, \XMM3 # perform a 16 byte swap
1009 PSHUFB_XMM %xmm15, \XMM4 # perform a 16 byte swap
1010
1011 pxor \TMP4, \TMP1
1012 pxor \XMM8, \XMM5
1013 pxor \TMP6, \TMP2
1014 pxor \TMP1, \TMP2
1015 pxor \XMM5, \TMP2
1016 movdqa \TMP2, \TMP3
1017 pslldq $8, \TMP3 # left shift TMP3 2 DWs
1018 psrldq $8, \TMP2 # right shift TMP2 2 DWs
1019 pxor \TMP3, \XMM5
1020 pxor \TMP2, \TMP1 # accumulate the results in TMP1:XMM5
1021
1022 # first phase of reduction
1023
1024 movdqa \XMM5, \TMP2
1025 movdqa \XMM5, \TMP3
1026 movdqa \XMM5, \TMP4
1027# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently
1028 pslld $31, \TMP2 # packed right shift << 31
1029 pslld $30, \TMP3 # packed right shift << 30
1030 pslld $25, \TMP4 # packed right shift << 25
1031 pxor \TMP3, \TMP2 # xor the shifted versions
1032 pxor \TMP4, \TMP2
1033 movdqa \TMP2, \TMP5
1034 psrldq $4, \TMP5 # right shift T5 1 DW
1035 pslldq $12, \TMP2 # left shift T2 3 DWs
1036 pxor \TMP2, \XMM5
1037
1038 # second phase of reduction
1039
1040 movdqa \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4
1041 movdqa \XMM5,\TMP3
1042 movdqa \XMM5,\TMP4
1043 psrld $1, \TMP2 # packed left shift >>1
1044 psrld $2, \TMP3 # packed left shift >>2
1045 psrld $7, \TMP4 # packed left shift >>7
1046 pxor \TMP3,\TMP2 # xor the shifted versions
1047 pxor \TMP4,\TMP2
1048 pxor \TMP5, \TMP2
1049 pxor \TMP2, \XMM5
1050 pxor \TMP1, \XMM5 # result is in TMP1
1051
1052 pxor \XMM5, \XMM1
1053.endm
1054
1055/* GHASH the last 4 ciphertext blocks. */
1056.macro GHASH_LAST_4 TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 \
1057TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst
1058
1059 # Multiply TMP6 * HashKey (using Karatsuba)
1060
1061 movdqa \XMM1, \TMP6
1062 pshufd $78, \XMM1, \TMP2
1063 pxor \XMM1, \TMP2
1064 movdqa HashKey_4(%rsp), \TMP5
1065 PCLMULQDQ 0x11, \TMP5, \TMP6 # TMP6 = a1*b1
1066 PCLMULQDQ 0x00, \TMP5, \XMM1 # XMM1 = a0*b0
1067 movdqa HashKey_4_k(%rsp), \TMP4
1068 PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1069 movdqa \XMM1, \XMMDst
1070 movdqa \TMP2, \XMM1 # result in TMP6, XMMDst, XMM1
1071
1072 # Multiply TMP1 * HashKey (using Karatsuba)
1073
1074 movdqa \XMM2, \TMP1
1075 pshufd $78, \XMM2, \TMP2
1076 pxor \XMM2, \TMP2
1077 movdqa HashKey_3(%rsp), \TMP5
1078 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1
1079 PCLMULQDQ 0x00, \TMP5, \XMM2 # XMM2 = a0*b0
1080 movdqa HashKey_3_k(%rsp), \TMP4
1081 PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1082 pxor \TMP1, \TMP6
1083 pxor \XMM2, \XMMDst
1084 pxor \TMP2, \XMM1
1085# results accumulated in TMP6, XMMDst, XMM1
1086
1087 # Multiply TMP1 * HashKey (using Karatsuba)
1088
1089 movdqa \XMM3, \TMP1
1090 pshufd $78, \XMM3, \TMP2
1091 pxor \XMM3, \TMP2
1092 movdqa HashKey_2(%rsp), \TMP5
1093 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1
1094 PCLMULQDQ 0x00, \TMP5, \XMM3 # XMM3 = a0*b0
1095 movdqa HashKey_2_k(%rsp), \TMP4
1096 PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1097 pxor \TMP1, \TMP6
1098 pxor \XMM3, \XMMDst
1099 pxor \TMP2, \XMM1 # results accumulated in TMP6, XMMDst, XMM1
1100
1101 # Multiply TMP1 * HashKey (using Karatsuba)
1102 movdqa \XMM4, \TMP1
1103 pshufd $78, \XMM4, \TMP2
1104 pxor \XMM4, \TMP2
1105 movdqa HashKey(%rsp), \TMP5
1106 PCLMULQDQ 0x11, \TMP5, \TMP1 # TMP1 = a1*b1
1107 PCLMULQDQ 0x00, \TMP5, \XMM4 # XMM4 = a0*b0
1108 movdqa HashKey_k(%rsp), \TMP4
1109 PCLMULQDQ 0x00, \TMP4, \TMP2 # TMP2 = (a1+a0)*(b1+b0)
1110 pxor \TMP1, \TMP6
1111 pxor \XMM4, \XMMDst
1112 pxor \XMM1, \TMP2
1113 pxor \TMP6, \TMP2
1114 pxor \XMMDst, \TMP2
1115 # middle section of the temp results combined as in karatsuba algorithm
1116 movdqa \TMP2, \TMP4
1117 pslldq $8, \TMP4 # left shift TMP4 2 DWs
1118 psrldq $8, \TMP2 # right shift TMP2 2 DWs
1119 pxor \TMP4, \XMMDst
1120 pxor \TMP2, \TMP6
1121# TMP6:XMMDst holds the result of the accumulated carry-less multiplications
1122 # first phase of the reduction
1123 movdqa \XMMDst, \TMP2
1124 movdqa \XMMDst, \TMP3
1125 movdqa \XMMDst, \TMP4
1126# move XMMDst into TMP2, TMP3, TMP4 in order to perform 3 shifts independently
1127 pslld $31, \TMP2 # packed right shifting << 31
1128 pslld $30, \TMP3 # packed right shifting << 30
1129 pslld $25, \TMP4 # packed right shifting << 25
1130 pxor \TMP3, \TMP2 # xor the shifted versions
1131 pxor \TMP4, \TMP2
1132 movdqa \TMP2, \TMP7
1133 psrldq $4, \TMP7 # right shift TMP7 1 DW
1134 pslldq $12, \TMP2 # left shift TMP2 3 DWs
1135 pxor \TMP2, \XMMDst
1136
1137 # second phase of the reduction
1138 movdqa \XMMDst, \TMP2
1139 # make 3 copies of XMMDst for doing 3 shift operations
1140 movdqa \XMMDst, \TMP3
1141 movdqa \XMMDst, \TMP4
1142 psrld $1, \TMP2 # packed left shift >> 1
1143 psrld $2, \TMP3 # packed left shift >> 2
1144 psrld $7, \TMP4 # packed left shift >> 7
1145 pxor \TMP3, \TMP2 # xor the shifted versions
1146 pxor \TMP4, \TMP2
1147 pxor \TMP7, \TMP2
1148 pxor \TMP2, \XMMDst
1149 pxor \TMP6, \XMMDst # reduced result is in XMMDst
1150.endm
1151
1152/* Encryption of a single block done*/
1153.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1
1154
1155 pxor (%arg1), \XMM0
1156 movaps 16(%arg1), \TMP1
1157 AESENC \TMP1, \XMM0
1158 movaps 32(%arg1), \TMP1
1159 AESENC \TMP1, \XMM0
1160 movaps 48(%arg1), \TMP1
1161 AESENC \TMP1, \XMM0
1162 movaps 64(%arg1), \TMP1
1163 AESENC \TMP1, \XMM0
1164 movaps 80(%arg1), \TMP1
1165 AESENC \TMP1, \XMM0
1166 movaps 96(%arg1), \TMP1
1167 AESENC \TMP1, \XMM0
1168 movaps 112(%arg1), \TMP1
1169 AESENC \TMP1, \XMM0
1170 movaps 128(%arg1), \TMP1
1171 AESENC \TMP1, \XMM0
1172 movaps 144(%arg1), \TMP1
1173 AESENC \TMP1, \XMM0
1174 movaps 160(%arg1), \TMP1
1175 AESENCLAST \TMP1, \XMM0
1176.endm
1177
1178
1179/*****************************************************************************
1180* void aesni_gcm_dec(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1181* u8 *out, // Plaintext output. Encrypt in-place is allowed.
1182* const u8 *in, // Ciphertext input
1183* u64 plaintext_len, // Length of data in bytes for decryption.
1184* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association)
1185* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
1186* // concatenated with 0x00000001. 16-byte aligned pointer.
1187* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary.
1188* const u8 *aad, // Additional Authentication Data (AAD)
1189* u64 aad_len, // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes
1190* u8 *auth_tag, // Authenticated Tag output. The driver will compare this to the
1191* // given authentication tag and only return the plaintext if they match.
1192* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16
1193* // (most likely), 12 or 8.
1194*
1195* Assumptions:
1196*
1197* keys:
1198* keys are pre-expanded and aligned to 16 bytes. we are using the first
1199* set of 11 keys in the data structure void *aes_ctx
1200*
1201* iv:
1202* 0 1 2 3
1203* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1204* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1205* | Salt (From the SA) |
1206* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1207* | Initialization Vector |
1208* | (This is the sequence number from IPSec header) |
1209* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1210* | 0x1 |
1211* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1212*
1213*
1214*
1215* AAD:
1216* AAD padded to 128 bits with 0
1217* for example, assume AAD is a u32 vector
1218*
1219* if AAD is 8 bytes:
1220* AAD[3] = {A0, A1};
1221* padded AAD in xmm register = {A1 A0 0 0}
1222*
1223* 0 1 2 3
1224* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1225* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1226* | SPI (A1) |
1227* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1228* | 32-bit Sequence Number (A0) |
1229* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1230* | 0x0 |
1231* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1232*
1233* AAD Format with 32-bit Sequence Number
1234*
1235* if AAD is 12 bytes:
1236* AAD[3] = {A0, A1, A2};
1237* padded AAD in xmm register = {A2 A1 A0 0}
1238*
1239* 0 1 2 3
1240* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1241* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1242* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1243* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1244* | SPI (A2) |
1245* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1246* | 64-bit Extended Sequence Number {A1,A0} |
1247* | |
1248* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1249* | 0x0 |
1250* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1251*
1252* AAD Format with 64-bit Extended Sequence Number
1253*
1254* aadLen:
1255* from the definition of the spec, aadLen can only be 8 or 12 bytes.
1256* The code supports 16 too but for other sizes, the code will fail.
1257*
1258* TLen:
1259* from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
1260* For other sizes, the code will fail.
1261*
1262* poly = x^128 + x^127 + x^126 + x^121 + 1
1263*
1264*****************************************************************************/
1265
1266ENTRY(aesni_gcm_dec)
1267 push %r12
1268 push %r13
1269 push %r14
1270 mov %rsp, %r14
1271/*
1272* states of %xmm registers %xmm6:%xmm15 not saved
1273* all %xmm registers are clobbered
1274*/
1275 sub $VARIABLE_OFFSET, %rsp
1276 and $~63, %rsp # align rsp to 64 bytes
1277 mov %arg6, %r12
1278 movdqu (%r12), %xmm13 # %xmm13 = HashKey
1279 movdqa SHUF_MASK(%rip), %xmm2
1280 PSHUFB_XMM %xmm2, %xmm13
1281
1282
1283# Precompute HashKey<<1 (mod poly) from the hash key (required for GHASH)
1284
1285 movdqa %xmm13, %xmm2
1286 psllq $1, %xmm13
1287 psrlq $63, %xmm2
1288 movdqa %xmm2, %xmm1
1289 pslldq $8, %xmm2
1290 psrldq $8, %xmm1
1291 por %xmm2, %xmm13
1292
1293 # Reduction
1294
1295 pshufd $0x24, %xmm1, %xmm2
1296 pcmpeqd TWOONE(%rip), %xmm2
1297 pand POLY(%rip), %xmm2
1298 pxor %xmm2, %xmm13 # %xmm13 holds the HashKey<<1 (mod poly)
1299
1300
1301 # Decrypt first few blocks
1302
1303 movdqa %xmm13, HashKey(%rsp) # store HashKey<<1 (mod poly)
1304 mov %arg4, %r13 # save the number of bytes of plaintext/ciphertext
1305 and $-16, %r13 # %r13 = %r13 - (%r13 mod 16)
1306 mov %r13, %r12
1307 and $(3<<4), %r12
1308 jz _initial_num_blocks_is_0_decrypt
1309 cmp $(2<<4), %r12
1310 jb _initial_num_blocks_is_1_decrypt
1311 je _initial_num_blocks_is_2_decrypt
1312_initial_num_blocks_is_3_decrypt:
1313 INITIAL_BLOCKS_DEC 3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1314%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, dec
1315 sub $48, %r13
1316 jmp _initial_blocks_decrypted
1317_initial_num_blocks_is_2_decrypt:
1318 INITIAL_BLOCKS_DEC 2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1319%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, dec
1320 sub $32, %r13
1321 jmp _initial_blocks_decrypted
1322_initial_num_blocks_is_1_decrypt:
1323 INITIAL_BLOCKS_DEC 1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1324%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, dec
1325 sub $16, %r13
1326 jmp _initial_blocks_decrypted
1327_initial_num_blocks_is_0_decrypt:
1328 INITIAL_BLOCKS_DEC 0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1329%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, dec
1330_initial_blocks_decrypted:
1331 cmp $0, %r13
1332 je _zero_cipher_left_decrypt
1333 sub $64, %r13
1334 je _four_cipher_left_decrypt
1335_decrypt_by_4:
1336 GHASH_4_ENCRYPT_4_PARALLEL_DEC %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \
1337%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, dec
1338 add $64, %r11
1339 sub $64, %r13
1340 jne _decrypt_by_4
1341_four_cipher_left_decrypt:
1342 GHASH_LAST_4 %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
1343%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
1344_zero_cipher_left_decrypt:
1345 mov %arg4, %r13
1346 and $15, %r13 # %r13 = arg4 (mod 16)
1347 je _multiple_of_16_bytes_decrypt
1348
1349 # Handle the last <16 byte block seperately
1350
1351 paddd ONE(%rip), %xmm0 # increment CNT to get Yn
1352 movdqa SHUF_MASK(%rip), %xmm10
1353 PSHUFB_XMM %xmm10, %xmm0
1354
1355 ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Yn)
1356 sub $16, %r11
1357 add %r13, %r11
1358 movdqu (%arg3,%r11,1), %xmm1 # recieve the last <16 byte block
1359 lea SHIFT_MASK+16(%rip), %r12
1360 sub %r13, %r12
1361# adjust the shuffle mask pointer to be able to shift 16-%r13 bytes
1362# (%r13 is the number of bytes in plaintext mod 16)
1363 movdqu (%r12), %xmm2 # get the appropriate shuffle mask
1364 PSHUFB_XMM %xmm2, %xmm1 # right shift 16-%r13 butes
1365
1366 movdqa %xmm1, %xmm2
1367 pxor %xmm1, %xmm0 # Ciphertext XOR E(K, Yn)
1368 movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
1369 # get the appropriate mask to mask out top 16-%r13 bytes of %xmm0
1370 pand %xmm1, %xmm0 # mask out top 16-%r13 bytes of %xmm0
1371 pand %xmm1, %xmm2
1372 movdqa SHUF_MASK(%rip), %xmm10
1373 PSHUFB_XMM %xmm10 ,%xmm2
1374
1375 pxor %xmm2, %xmm8
1376 GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
1377 # GHASH computation for the last <16 byte block
1378 sub %r13, %r11
1379 add $16, %r11
1380
1381 # output %r13 bytes
1382 MOVQ_R64_XMM %xmm0, %rax
1383 cmp $8, %r13
1384 jle _less_than_8_bytes_left_decrypt
1385 mov %rax, (%arg2 , %r11, 1)
1386 add $8, %r11
1387 psrldq $8, %xmm0
1388 MOVQ_R64_XMM %xmm0, %rax
1389 sub $8, %r13
1390_less_than_8_bytes_left_decrypt:
1391 mov %al, (%arg2, %r11, 1)
1392 add $1, %r11
1393 shr $8, %rax
1394 sub $1, %r13
1395 jne _less_than_8_bytes_left_decrypt
1396_multiple_of_16_bytes_decrypt:
1397 mov arg8, %r12 # %r13 = aadLen (number of bytes)
1398 shl $3, %r12 # convert into number of bits
1399 movd %r12d, %xmm15 # len(A) in %xmm15
1400 shl $3, %arg4 # len(C) in bits (*128)
1401 MOVQ_R64_XMM %arg4, %xmm1
1402 pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
1403 pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C)
1404 pxor %xmm15, %xmm8
1405 GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
1406 # final GHASH computation
1407 movdqa SHUF_MASK(%rip), %xmm10
1408 PSHUFB_XMM %xmm10, %xmm8
1409
1410 mov %arg5, %rax # %rax = *Y0
1411 movdqu (%rax), %xmm0 # %xmm0 = Y0
1412 ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Y0)
1413 pxor %xmm8, %xmm0
1414_return_T_decrypt:
1415 mov arg9, %r10 # %r10 = authTag
1416 mov arg10, %r11 # %r11 = auth_tag_len
1417 cmp $16, %r11
1418 je _T_16_decrypt
1419 cmp $12, %r11
1420 je _T_12_decrypt
1421_T_8_decrypt:
1422 MOVQ_R64_XMM %xmm0, %rax
1423 mov %rax, (%r10)
1424 jmp _return_T_done_decrypt
1425_T_12_decrypt:
1426 MOVQ_R64_XMM %xmm0, %rax
1427 mov %rax, (%r10)
1428 psrldq $8, %xmm0
1429 movd %xmm0, %eax
1430 mov %eax, 8(%r10)
1431 jmp _return_T_done_decrypt
1432_T_16_decrypt:
1433 movdqu %xmm0, (%r10)
1434_return_T_done_decrypt:
1435 mov %r14, %rsp
1436 pop %r14
1437 pop %r13
1438 pop %r12
1439 ret
1440
1441
1442/*****************************************************************************
1443* void aesni_gcm_enc(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1444* u8 *out, // Ciphertext output. Encrypt in-place is allowed.
1445* const u8 *in, // Plaintext input
1446* u64 plaintext_len, // Length of data in bytes for encryption.
1447* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association)
1448* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
1449* // concatenated with 0x00000001. 16-byte aligned pointer.
1450* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary.
1451* const u8 *aad, // Additional Authentication Data (AAD)
1452* u64 aad_len, // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes
1453* u8 *auth_tag, // Authenticated Tag output.
1454* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16 (most likely),
1455* // 12 or 8.
1456*
1457* Assumptions:
1458*
1459* keys:
1460* keys are pre-expanded and aligned to 16 bytes. we are using the
1461* first set of 11 keys in the data structure void *aes_ctx
1462*
1463*
1464* iv:
1465* 0 1 2 3
1466* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1467* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1468* | Salt (From the SA) |
1469* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1470* | Initialization Vector |
1471* | (This is the sequence number from IPSec header) |
1472* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1473* | 0x1 |
1474* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1475*
1476*
1477*
1478* AAD:
1479* AAD padded to 128 bits with 0
1480* for example, assume AAD is a u32 vector
1481*
1482* if AAD is 8 bytes:
1483* AAD[3] = {A0, A1};
1484* padded AAD in xmm register = {A1 A0 0 0}
1485*
1486* 0 1 2 3
1487* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1488* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1489* | SPI (A1) |
1490* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1491* | 32-bit Sequence Number (A0) |
1492* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1493* | 0x0 |
1494* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1495*
1496* AAD Format with 32-bit Sequence Number
1497*
1498* if AAD is 12 bytes:
1499* AAD[3] = {A0, A1, A2};
1500* padded AAD in xmm register = {A2 A1 A0 0}
1501*
1502* 0 1 2 3
1503* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1504* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1505* | SPI (A2) |
1506* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1507* | 64-bit Extended Sequence Number {A1,A0} |
1508* | |
1509* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1510* | 0x0 |
1511* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1512*
1513* AAD Format with 64-bit Extended Sequence Number
1514*
1515* aadLen:
1516* from the definition of the spec, aadLen can only be 8 or 12 bytes.
1517* The code supports 16 too but for other sizes, the code will fail.
1518*
1519* TLen:
1520* from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
1521* For other sizes, the code will fail.
1522*
1523* poly = x^128 + x^127 + x^126 + x^121 + 1
1524***************************************************************************/
1525ENTRY(aesni_gcm_enc)
1526 push %r12
1527 push %r13
1528 push %r14
1529 mov %rsp, %r14
1530#
1531# states of %xmm registers %xmm6:%xmm15 not saved
1532# all %xmm registers are clobbered
1533#
1534 sub $VARIABLE_OFFSET, %rsp
1535 and $~63, %rsp
1536 mov %arg6, %r12
1537 movdqu (%r12), %xmm13
1538 movdqa SHUF_MASK(%rip), %xmm2
1539 PSHUFB_XMM %xmm2, %xmm13
1540
1541
1542# precompute HashKey<<1 mod poly from the HashKey (required for GHASH)
1543
1544 movdqa %xmm13, %xmm2
1545 psllq $1, %xmm13
1546 psrlq $63, %xmm2
1547 movdqa %xmm2, %xmm1
1548 pslldq $8, %xmm2
1549 psrldq $8, %xmm1
1550 por %xmm2, %xmm13
1551
1552 # reduce HashKey<<1
1553
1554 pshufd $0x24, %xmm1, %xmm2
1555 pcmpeqd TWOONE(%rip), %xmm2
1556 pand POLY(%rip), %xmm2
1557 pxor %xmm2, %xmm13
1558 movdqa %xmm13, HashKey(%rsp)
1559 mov %arg4, %r13 # %xmm13 holds HashKey<<1 (mod poly)
1560 and $-16, %r13
1561 mov %r13, %r12
1562
1563 # Encrypt first few blocks
1564
1565 and $(3<<4), %r12
1566 jz _initial_num_blocks_is_0_encrypt
1567 cmp $(2<<4), %r12
1568 jb _initial_num_blocks_is_1_encrypt
1569 je _initial_num_blocks_is_2_encrypt
1570_initial_num_blocks_is_3_encrypt:
1571 INITIAL_BLOCKS_ENC 3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1572%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, enc
1573 sub $48, %r13
1574 jmp _initial_blocks_encrypted
1575_initial_num_blocks_is_2_encrypt:
1576 INITIAL_BLOCKS_ENC 2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1577%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, enc
1578 sub $32, %r13
1579 jmp _initial_blocks_encrypted
1580_initial_num_blocks_is_1_encrypt:
1581 INITIAL_BLOCKS_ENC 1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1582%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, enc
1583 sub $16, %r13
1584 jmp _initial_blocks_encrypted
1585_initial_num_blocks_is_0_encrypt:
1586 INITIAL_BLOCKS_ENC 0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1587%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, enc
1588_initial_blocks_encrypted:
1589
1590 # Main loop - Encrypt remaining blocks
1591
1592 cmp $0, %r13
1593 je _zero_cipher_left_encrypt
1594 sub $64, %r13
1595 je _four_cipher_left_encrypt
1596_encrypt_by_4_encrypt:
1597 GHASH_4_ENCRYPT_4_PARALLEL_ENC %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \
1598%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, enc
1599 add $64, %r11
1600 sub $64, %r13
1601 jne _encrypt_by_4_encrypt
1602_four_cipher_left_encrypt:
1603 GHASH_LAST_4 %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
1604%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
1605_zero_cipher_left_encrypt:
1606 mov %arg4, %r13
1607 and $15, %r13 # %r13 = arg4 (mod 16)
1608 je _multiple_of_16_bytes_encrypt
1609
1610 # Handle the last <16 Byte block seperately
1611 paddd ONE(%rip), %xmm0 # INCR CNT to get Yn
1612 movdqa SHUF_MASK(%rip), %xmm10
1613 PSHUFB_XMM %xmm10, %xmm0
1614
1615 ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # Encrypt(K, Yn)
1616 sub $16, %r11
1617 add %r13, %r11
1618 movdqu (%arg3,%r11,1), %xmm1 # receive the last <16 byte blocks
1619 lea SHIFT_MASK+16(%rip), %r12
1620 sub %r13, %r12
1621 # adjust the shuffle mask pointer to be able to shift 16-r13 bytes
1622 # (%r13 is the number of bytes in plaintext mod 16)
1623 movdqu (%r12), %xmm2 # get the appropriate shuffle mask
1624 PSHUFB_XMM %xmm2, %xmm1 # shift right 16-r13 byte
1625 pxor %xmm1, %xmm0 # Plaintext XOR Encrypt(K, Yn)
1626 movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
1627 # get the appropriate mask to mask out top 16-r13 bytes of xmm0
1628 pand %xmm1, %xmm0 # mask out top 16-r13 bytes of xmm0
1629 movdqa SHUF_MASK(%rip), %xmm10
1630 PSHUFB_XMM %xmm10,%xmm0
1631
1632 pxor %xmm0, %xmm8
1633 GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
1634 # GHASH computation for the last <16 byte block
1635 sub %r13, %r11
1636 add $16, %r11
1637 PSHUFB_XMM %xmm10, %xmm1
1638
1639 # shuffle xmm0 back to output as ciphertext
1640
1641 # Output %r13 bytes
1642 MOVQ_R64_XMM %xmm0, %rax
1643 cmp $8, %r13
1644 jle _less_than_8_bytes_left_encrypt
1645 mov %rax, (%arg2 , %r11, 1)
1646 add $8, %r11
1647 psrldq $8, %xmm0
1648 MOVQ_R64_XMM %xmm0, %rax
1649 sub $8, %r13
1650_less_than_8_bytes_left_encrypt:
1651 mov %al, (%arg2, %r11, 1)
1652 add $1, %r11
1653 shr $8, %rax
1654 sub $1, %r13
1655 jne _less_than_8_bytes_left_encrypt
1656_multiple_of_16_bytes_encrypt:
1657 mov arg8, %r12 # %r12 = addLen (number of bytes)
1658 shl $3, %r12
1659 movd %r12d, %xmm15 # len(A) in %xmm15
1660 shl $3, %arg4 # len(C) in bits (*128)
1661 MOVQ_R64_XMM %arg4, %xmm1
1662 pslldq $8, %xmm15 # %xmm15 = len(A)||0x0000000000000000
1663 pxor %xmm1, %xmm15 # %xmm15 = len(A)||len(C)
1664 pxor %xmm15, %xmm8
1665 GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
1666 # final GHASH computation
1667 movdqa SHUF_MASK(%rip), %xmm10
1668 PSHUFB_XMM %xmm10, %xmm8 # perform a 16 byte swap
1669
1670 mov %arg5, %rax # %rax = *Y0
1671 movdqu (%rax), %xmm0 # %xmm0 = Y0
1672 ENCRYPT_SINGLE_BLOCK %xmm0, %xmm15 # Encrypt(K, Y0)
1673 pxor %xmm8, %xmm0
1674_return_T_encrypt:
1675 mov arg9, %r10 # %r10 = authTag
1676 mov arg10, %r11 # %r11 = auth_tag_len
1677 cmp $16, %r11
1678 je _T_16_encrypt
1679 cmp $12, %r11
1680 je _T_12_encrypt
1681_T_8_encrypt:
1682 MOVQ_R64_XMM %xmm0, %rax
1683 mov %rax, (%r10)
1684 jmp _return_T_done_encrypt
1685_T_12_encrypt:
1686 MOVQ_R64_XMM %xmm0, %rax
1687 mov %rax, (%r10)
1688 psrldq $8, %xmm0
1689 movd %xmm0, %eax
1690 mov %eax, 8(%r10)
1691 jmp _return_T_done_encrypt
1692_T_16_encrypt:
1693 movdqu %xmm0, (%r10)
1694_return_T_done_encrypt:
1695 mov %r14, %rsp
1696 pop %r14
1697 pop %r13
1698 pop %r12
1699 ret
1700
1701#endif
1702
49 1703
50_key_expansion_128: 1704_key_expansion_128:
51_key_expansion_256a: 1705_key_expansion_256a:
@@ -55,10 +1709,11 @@ _key_expansion_256a:
55 shufps $0b10001100, %xmm0, %xmm4 1709 shufps $0b10001100, %xmm0, %xmm4
56 pxor %xmm4, %xmm0 1710 pxor %xmm4, %xmm0
57 pxor %xmm1, %xmm0 1711 pxor %xmm1, %xmm0
58 movaps %xmm0, (%rcx) 1712 movaps %xmm0, (TKEYP)
59 add $0x10, %rcx 1713 add $0x10, TKEYP
60 ret 1714 ret
61 1715
1716.align 4
62_key_expansion_192a: 1717_key_expansion_192a:
63 pshufd $0b01010101, %xmm1, %xmm1 1718 pshufd $0b01010101, %xmm1, %xmm1
64 shufps $0b00010000, %xmm0, %xmm4 1719 shufps $0b00010000, %xmm0, %xmm4
@@ -76,12 +1731,13 @@ _key_expansion_192a:
76 1731
77 movaps %xmm0, %xmm1 1732 movaps %xmm0, %xmm1
78 shufps $0b01000100, %xmm0, %xmm6 1733 shufps $0b01000100, %xmm0, %xmm6
79 movaps %xmm6, (%rcx) 1734 movaps %xmm6, (TKEYP)
80 shufps $0b01001110, %xmm2, %xmm1 1735 shufps $0b01001110, %xmm2, %xmm1
81 movaps %xmm1, 16(%rcx) 1736 movaps %xmm1, 0x10(TKEYP)
82 add $0x20, %rcx 1737 add $0x20, TKEYP
83 ret 1738 ret
84 1739
1740.align 4
85_key_expansion_192b: 1741_key_expansion_192b:
86 pshufd $0b01010101, %xmm1, %xmm1 1742 pshufd $0b01010101, %xmm1, %xmm1
87 shufps $0b00010000, %xmm0, %xmm4 1743 shufps $0b00010000, %xmm0, %xmm4
@@ -96,10 +1752,11 @@ _key_expansion_192b:
96 pxor %xmm3, %xmm2 1752 pxor %xmm3, %xmm2
97 pxor %xmm5, %xmm2 1753 pxor %xmm5, %xmm2
98 1754
99 movaps %xmm0, (%rcx) 1755 movaps %xmm0, (TKEYP)
100 add $0x10, %rcx 1756 add $0x10, TKEYP
101 ret 1757 ret
102 1758
1759.align 4
103_key_expansion_256b: 1760_key_expansion_256b:
104 pshufd $0b10101010, %xmm1, %xmm1 1761 pshufd $0b10101010, %xmm1, %xmm1
105 shufps $0b00010000, %xmm2, %xmm4 1762 shufps $0b00010000, %xmm2, %xmm4
@@ -107,8 +1764,8 @@ _key_expansion_256b:
107 shufps $0b10001100, %xmm2, %xmm4 1764 shufps $0b10001100, %xmm2, %xmm4
108 pxor %xmm4, %xmm2 1765 pxor %xmm4, %xmm2
109 pxor %xmm1, %xmm2 1766 pxor %xmm1, %xmm2
110 movaps %xmm2, (%rcx) 1767 movaps %xmm2, (TKEYP)
111 add $0x10, %rcx 1768 add $0x10, TKEYP
112 ret 1769 ret
113 1770
114/* 1771/*
@@ -116,17 +1773,23 @@ _key_expansion_256b:
116 * unsigned int key_len) 1773 * unsigned int key_len)
117 */ 1774 */
118ENTRY(aesni_set_key) 1775ENTRY(aesni_set_key)
119 movups (%rsi), %xmm0 # user key (first 16 bytes) 1776#ifndef __x86_64__
120 movaps %xmm0, (%rdi) 1777 pushl KEYP
121 lea 0x10(%rdi), %rcx # key addr 1778 movl 8(%esp), KEYP # ctx
122 movl %edx, 480(%rdi) 1779 movl 12(%esp), UKEYP # in_key
1780 movl 16(%esp), %edx # key_len
1781#endif
1782 movups (UKEYP), %xmm0 # user key (first 16 bytes)
1783 movaps %xmm0, (KEYP)
1784 lea 0x10(KEYP), TKEYP # key addr
1785 movl %edx, 480(KEYP)
123 pxor %xmm4, %xmm4 # xmm4 is assumed 0 in _key_expansion_x 1786 pxor %xmm4, %xmm4 # xmm4 is assumed 0 in _key_expansion_x
124 cmp $24, %dl 1787 cmp $24, %dl
125 jb .Lenc_key128 1788 jb .Lenc_key128
126 je .Lenc_key192 1789 je .Lenc_key192
127 movups 0x10(%rsi), %xmm2 # other user key 1790 movups 0x10(UKEYP), %xmm2 # other user key
128 movaps %xmm2, (%rcx) 1791 movaps %xmm2, (TKEYP)
129 add $0x10, %rcx 1792 add $0x10, TKEYP
130 AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1 1793 AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1
131 call _key_expansion_256a 1794 call _key_expansion_256a
132 AESKEYGENASSIST 0x1 %xmm0 %xmm1 1795 AESKEYGENASSIST 0x1 %xmm0 %xmm1
@@ -155,7 +1818,7 @@ ENTRY(aesni_set_key)
155 call _key_expansion_256a 1818 call _key_expansion_256a
156 jmp .Ldec_key 1819 jmp .Ldec_key
157.Lenc_key192: 1820.Lenc_key192:
158 movq 0x10(%rsi), %xmm2 # other user key 1821 movq 0x10(UKEYP), %xmm2 # other user key
159 AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1 1822 AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1
160 call _key_expansion_192a 1823 call _key_expansion_192a
161 AESKEYGENASSIST 0x2 %xmm2 %xmm1 # round 2 1824 AESKEYGENASSIST 0x2 %xmm2 %xmm1 # round 2
@@ -195,33 +1858,47 @@ ENTRY(aesni_set_key)
195 AESKEYGENASSIST 0x36 %xmm0 %xmm1 # round 10 1858 AESKEYGENASSIST 0x36 %xmm0 %xmm1 # round 10
196 call _key_expansion_128 1859 call _key_expansion_128
197.Ldec_key: 1860.Ldec_key:
198 sub $0x10, %rcx 1861 sub $0x10, TKEYP
199 movaps (%rdi), %xmm0 1862 movaps (KEYP), %xmm0
200 movaps (%rcx), %xmm1 1863 movaps (TKEYP), %xmm1
201 movaps %xmm0, 240(%rcx) 1864 movaps %xmm0, 240(TKEYP)
202 movaps %xmm1, 240(%rdi) 1865 movaps %xmm1, 240(KEYP)
203 add $0x10, %rdi 1866 add $0x10, KEYP
204 lea 240-16(%rcx), %rsi 1867 lea 240-16(TKEYP), UKEYP
205.align 4 1868.align 4
206.Ldec_key_loop: 1869.Ldec_key_loop:
207 movaps (%rdi), %xmm0 1870 movaps (KEYP), %xmm0
208 AESIMC %xmm0 %xmm1 1871 AESIMC %xmm0 %xmm1
209 movaps %xmm1, (%rsi) 1872 movaps %xmm1, (UKEYP)
210 add $0x10, %rdi 1873 add $0x10, KEYP
211 sub $0x10, %rsi 1874 sub $0x10, UKEYP
212 cmp %rcx, %rdi 1875 cmp TKEYP, KEYP
213 jb .Ldec_key_loop 1876 jb .Ldec_key_loop
214 xor %rax, %rax 1877 xor AREG, AREG
1878#ifndef __x86_64__
1879 popl KEYP
1880#endif
215 ret 1881 ret
216 1882
217/* 1883/*
218 * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) 1884 * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
219 */ 1885 */
220ENTRY(aesni_enc) 1886ENTRY(aesni_enc)
1887#ifndef __x86_64__
1888 pushl KEYP
1889 pushl KLEN
1890 movl 12(%esp), KEYP
1891 movl 16(%esp), OUTP
1892 movl 20(%esp), INP
1893#endif
221 movl 480(KEYP), KLEN # key length 1894 movl 480(KEYP), KLEN # key length
222 movups (INP), STATE # input 1895 movups (INP), STATE # input
223 call _aesni_enc1 1896 call _aesni_enc1
224 movups STATE, (OUTP) # output 1897 movups STATE, (OUTP) # output
1898#ifndef __x86_64__
1899 popl KLEN
1900 popl KEYP
1901#endif
225 ret 1902 ret
226 1903
227/* 1904/*
@@ -236,6 +1913,7 @@ ENTRY(aesni_enc)
236 * KEY 1913 * KEY
237 * TKEYP (T1) 1914 * TKEYP (T1)
238 */ 1915 */
1916.align 4
239_aesni_enc1: 1917_aesni_enc1:
240 movaps (KEYP), KEY # key 1918 movaps (KEYP), KEY # key
241 mov KEYP, TKEYP 1919 mov KEYP, TKEYP
@@ -298,6 +1976,7 @@ _aesni_enc1:
298 * KEY 1976 * KEY
299 * TKEYP (T1) 1977 * TKEYP (T1)
300 */ 1978 */
1979.align 4
301_aesni_enc4: 1980_aesni_enc4:
302 movaps (KEYP), KEY # key 1981 movaps (KEYP), KEY # key
303 mov KEYP, TKEYP 1982 mov KEYP, TKEYP
@@ -391,11 +2070,22 @@ _aesni_enc4:
391 * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) 2070 * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
392 */ 2071 */
393ENTRY(aesni_dec) 2072ENTRY(aesni_dec)
2073#ifndef __x86_64__
2074 pushl KEYP
2075 pushl KLEN
2076 movl 12(%esp), KEYP
2077 movl 16(%esp), OUTP
2078 movl 20(%esp), INP
2079#endif
394 mov 480(KEYP), KLEN # key length 2080 mov 480(KEYP), KLEN # key length
395 add $240, KEYP 2081 add $240, KEYP
396 movups (INP), STATE # input 2082 movups (INP), STATE # input
397 call _aesni_dec1 2083 call _aesni_dec1
398 movups STATE, (OUTP) #output 2084 movups STATE, (OUTP) #output
2085#ifndef __x86_64__
2086 popl KLEN
2087 popl KEYP
2088#endif
399 ret 2089 ret
400 2090
401/* 2091/*
@@ -410,6 +2100,7 @@ ENTRY(aesni_dec)
410 * KEY 2100 * KEY
411 * TKEYP (T1) 2101 * TKEYP (T1)
412 */ 2102 */
2103.align 4
413_aesni_dec1: 2104_aesni_dec1:
414 movaps (KEYP), KEY # key 2105 movaps (KEYP), KEY # key
415 mov KEYP, TKEYP 2106 mov KEYP, TKEYP
@@ -472,6 +2163,7 @@ _aesni_dec1:
472 * KEY 2163 * KEY
473 * TKEYP (T1) 2164 * TKEYP (T1)
474 */ 2165 */
2166.align 4
475_aesni_dec4: 2167_aesni_dec4:
476 movaps (KEYP), KEY # key 2168 movaps (KEYP), KEY # key
477 mov KEYP, TKEYP 2169 mov KEYP, TKEYP
@@ -566,6 +2258,15 @@ _aesni_dec4:
566 * size_t len) 2258 * size_t len)
567 */ 2259 */
568ENTRY(aesni_ecb_enc) 2260ENTRY(aesni_ecb_enc)
2261#ifndef __x86_64__
2262 pushl LEN
2263 pushl KEYP
2264 pushl KLEN
2265 movl 16(%esp), KEYP
2266 movl 20(%esp), OUTP
2267 movl 24(%esp), INP
2268 movl 28(%esp), LEN
2269#endif
569 test LEN, LEN # check length 2270 test LEN, LEN # check length
570 jz .Lecb_enc_ret 2271 jz .Lecb_enc_ret
571 mov 480(KEYP), KLEN 2272 mov 480(KEYP), KLEN
@@ -602,6 +2303,11 @@ ENTRY(aesni_ecb_enc)
602 cmp $16, LEN 2303 cmp $16, LEN
603 jge .Lecb_enc_loop1 2304 jge .Lecb_enc_loop1
604.Lecb_enc_ret: 2305.Lecb_enc_ret:
2306#ifndef __x86_64__
2307 popl KLEN
2308 popl KEYP
2309 popl LEN
2310#endif
605 ret 2311 ret
606 2312
607/* 2313/*
@@ -609,6 +2315,15 @@ ENTRY(aesni_ecb_enc)
609 * size_t len); 2315 * size_t len);
610 */ 2316 */
611ENTRY(aesni_ecb_dec) 2317ENTRY(aesni_ecb_dec)
2318#ifndef __x86_64__
2319 pushl LEN
2320 pushl KEYP
2321 pushl KLEN
2322 movl 16(%esp), KEYP
2323 movl 20(%esp), OUTP
2324 movl 24(%esp), INP
2325 movl 28(%esp), LEN
2326#endif
612 test LEN, LEN 2327 test LEN, LEN
613 jz .Lecb_dec_ret 2328 jz .Lecb_dec_ret
614 mov 480(KEYP), KLEN 2329 mov 480(KEYP), KLEN
@@ -646,6 +2361,11 @@ ENTRY(aesni_ecb_dec)
646 cmp $16, LEN 2361 cmp $16, LEN
647 jge .Lecb_dec_loop1 2362 jge .Lecb_dec_loop1
648.Lecb_dec_ret: 2363.Lecb_dec_ret:
2364#ifndef __x86_64__
2365 popl KLEN
2366 popl KEYP
2367 popl LEN
2368#endif
649 ret 2369 ret
650 2370
651/* 2371/*
@@ -653,6 +2373,17 @@ ENTRY(aesni_ecb_dec)
653 * size_t len, u8 *iv) 2373 * size_t len, u8 *iv)
654 */ 2374 */
655ENTRY(aesni_cbc_enc) 2375ENTRY(aesni_cbc_enc)
2376#ifndef __x86_64__
2377 pushl IVP
2378 pushl LEN
2379 pushl KEYP
2380 pushl KLEN
2381 movl 20(%esp), KEYP
2382 movl 24(%esp), OUTP
2383 movl 28(%esp), INP
2384 movl 32(%esp), LEN
2385 movl 36(%esp), IVP
2386#endif
656 cmp $16, LEN 2387 cmp $16, LEN
657 jb .Lcbc_enc_ret 2388 jb .Lcbc_enc_ret
658 mov 480(KEYP), KLEN 2389 mov 480(KEYP), KLEN
@@ -670,6 +2401,12 @@ ENTRY(aesni_cbc_enc)
670 jge .Lcbc_enc_loop 2401 jge .Lcbc_enc_loop
671 movups STATE, (IVP) 2402 movups STATE, (IVP)
672.Lcbc_enc_ret: 2403.Lcbc_enc_ret:
2404#ifndef __x86_64__
2405 popl KLEN
2406 popl KEYP
2407 popl LEN
2408 popl IVP
2409#endif
673 ret 2410 ret
674 2411
675/* 2412/*
@@ -677,6 +2414,17 @@ ENTRY(aesni_cbc_enc)
677 * size_t len, u8 *iv) 2414 * size_t len, u8 *iv)
678 */ 2415 */
679ENTRY(aesni_cbc_dec) 2416ENTRY(aesni_cbc_dec)
2417#ifndef __x86_64__
2418 pushl IVP
2419 pushl LEN
2420 pushl KEYP
2421 pushl KLEN
2422 movl 20(%esp), KEYP
2423 movl 24(%esp), OUTP
2424 movl 28(%esp), INP
2425 movl 32(%esp), LEN
2426 movl 36(%esp), IVP
2427#endif
680 cmp $16, LEN 2428 cmp $16, LEN
681 jb .Lcbc_dec_just_ret 2429 jb .Lcbc_dec_just_ret
682 mov 480(KEYP), KLEN 2430 mov 480(KEYP), KLEN
@@ -690,16 +2438,30 @@ ENTRY(aesni_cbc_dec)
690 movaps IN1, STATE1 2438 movaps IN1, STATE1
691 movups 0x10(INP), IN2 2439 movups 0x10(INP), IN2
692 movaps IN2, STATE2 2440 movaps IN2, STATE2
2441#ifdef __x86_64__
693 movups 0x20(INP), IN3 2442 movups 0x20(INP), IN3
694 movaps IN3, STATE3 2443 movaps IN3, STATE3
695 movups 0x30(INP), IN4 2444 movups 0x30(INP), IN4
696 movaps IN4, STATE4 2445 movaps IN4, STATE4
2446#else
2447 movups 0x20(INP), IN1
2448 movaps IN1, STATE3
2449 movups 0x30(INP), IN2
2450 movaps IN2, STATE4
2451#endif
697 call _aesni_dec4 2452 call _aesni_dec4
698 pxor IV, STATE1 2453 pxor IV, STATE1
2454#ifdef __x86_64__
699 pxor IN1, STATE2 2455 pxor IN1, STATE2
700 pxor IN2, STATE3 2456 pxor IN2, STATE3
701 pxor IN3, STATE4 2457 pxor IN3, STATE4
702 movaps IN4, IV 2458 movaps IN4, IV
2459#else
2460 pxor (INP), STATE2
2461 pxor 0x10(INP), STATE3
2462 pxor IN1, STATE4
2463 movaps IN2, IV
2464#endif
703 movups STATE1, (OUTP) 2465 movups STATE1, (OUTP)
704 movups STATE2, 0x10(OUTP) 2466 movups STATE2, 0x10(OUTP)
705 movups STATE3, 0x20(OUTP) 2467 movups STATE3, 0x20(OUTP)
@@ -727,8 +2489,15 @@ ENTRY(aesni_cbc_dec)
727.Lcbc_dec_ret: 2489.Lcbc_dec_ret:
728 movups IV, (IVP) 2490 movups IV, (IVP)
729.Lcbc_dec_just_ret: 2491.Lcbc_dec_just_ret:
2492#ifndef __x86_64__
2493 popl KLEN
2494 popl KEYP
2495 popl LEN
2496 popl IVP
2497#endif
730 ret 2498 ret
731 2499
2500#ifdef __x86_64__
732.align 16 2501.align 16
733.Lbswap_mask: 2502.Lbswap_mask:
734 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 2503 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
@@ -744,6 +2513,7 @@ ENTRY(aesni_cbc_dec)
744 * INC: == 1, in little endian 2513 * INC: == 1, in little endian
745 * BSWAP_MASK == endian swapping mask 2514 * BSWAP_MASK == endian swapping mask
746 */ 2515 */
2516.align 4
747_aesni_inc_init: 2517_aesni_inc_init:
748 movaps .Lbswap_mask, BSWAP_MASK 2518 movaps .Lbswap_mask, BSWAP_MASK
749 movaps IV, CTR 2519 movaps IV, CTR
@@ -768,6 +2538,7 @@ _aesni_inc_init:
768 * CTR: == output IV, in little endian 2538 * CTR: == output IV, in little endian
769 * TCTR_LOW: == lower qword of CTR 2539 * TCTR_LOW: == lower qword of CTR
770 */ 2540 */
2541.align 4
771_aesni_inc: 2542_aesni_inc:
772 paddq INC, CTR 2543 paddq INC, CTR
773 add $1, TCTR_LOW 2544 add $1, TCTR_LOW
@@ -839,3 +2610,4 @@ ENTRY(aesni_ctr_enc)
839 movups IV, (IVP) 2610 movups IV, (IVP)
840.Lctr_enc_just_ret: 2611.Lctr_enc_just_ret:
841 ret 2612 ret
2613#endif
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 2cb3dcc4490a..e1e60c7d5813 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -5,6 +5,14 @@
5 * Copyright (C) 2008, Intel Corp. 5 * Copyright (C) 2008, Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com> 6 * Author: Huang Ying <ying.huang@intel.com>
7 * 7 *
8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9 * interface for 64-bit kernels.
10 * Authors: Adrian Hoban <adrian.hoban@intel.com>
11 * Gabriele Paoloni <gabriele.paoloni@intel.com>
12 * Tadeusz Struk (tadeusz.struk@intel.com)
13 * Aidan O'Mahony (aidan.o.mahony@intel.com)
14 * Copyright (c) 2010, Intel Corporation.
15 *
8 * This program is free software; you can redistribute it and/or modify 16 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by 17 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or 18 * the Free Software Foundation; either version 2 of the License, or
@@ -21,6 +29,10 @@
21#include <crypto/ctr.h> 29#include <crypto/ctr.h>
22#include <asm/i387.h> 30#include <asm/i387.h>
23#include <asm/aes.h> 31#include <asm/aes.h>
32#include <crypto/scatterwalk.h>
33#include <crypto/internal/aead.h>
34#include <linux/workqueue.h>
35#include <linux/spinlock.h>
24 36
25#if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE) 37#if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
26#define HAS_CTR 38#define HAS_CTR
@@ -42,8 +54,31 @@ struct async_aes_ctx {
42 struct cryptd_ablkcipher *cryptd_tfm; 54 struct cryptd_ablkcipher *cryptd_tfm;
43}; 55};
44 56
45#define AESNI_ALIGN 16 57/* This data is stored at the end of the crypto_tfm struct.
58 * It's a type of per "session" data storage location.
59 * This needs to be 16 byte aligned.
60 */
61struct aesni_rfc4106_gcm_ctx {
62 u8 hash_subkey[16];
63 struct crypto_aes_ctx aes_key_expanded;
64 u8 nonce[4];
65 struct cryptd_aead *cryptd_tfm;
66};
67
68struct aesni_gcm_set_hash_subkey_result {
69 int err;
70 struct completion completion;
71};
72
73struct aesni_hash_subkey_req_data {
74 u8 iv[16];
75 struct aesni_gcm_set_hash_subkey_result result;
76 struct scatterlist sg;
77};
78
79#define AESNI_ALIGN (16)
46#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 80#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1))
81#define RFC4106_HASH_SUBKEY_SIZE 16
47 82
48asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 83asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
49 unsigned int key_len); 84 unsigned int key_len);
@@ -59,9 +94,62 @@ asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
59 const u8 *in, unsigned int len, u8 *iv); 94 const u8 *in, unsigned int len, u8 *iv);
60asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 95asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
61 const u8 *in, unsigned int len, u8 *iv); 96 const u8 *in, unsigned int len, u8 *iv);
97#ifdef CONFIG_X86_64
62asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, 98asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
63 const u8 *in, unsigned int len, u8 *iv); 99 const u8 *in, unsigned int len, u8 *iv);
64 100
101/* asmlinkage void aesni_gcm_enc()
102 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
103 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
104 * const u8 *in, Plaintext input
105 * unsigned long plaintext_len, Length of data in bytes for encryption.
106 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
107 * concatenated with 8 byte Initialisation Vector (from IPSec ESP
108 * Payload) concatenated with 0x00000001. 16-byte aligned pointer.
109 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
110 * const u8 *aad, Additional Authentication Data (AAD)
111 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
112 * is going to be 8 or 12 bytes
113 * u8 *auth_tag, Authenticated Tag output.
114 * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
115 * Valid values are 16 (most likely), 12 or 8.
116 */
117asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
118 const u8 *in, unsigned long plaintext_len, u8 *iv,
119 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
120 u8 *auth_tag, unsigned long auth_tag_len);
121
122/* asmlinkage void aesni_gcm_dec()
123 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
124 * u8 *out, Plaintext output. Decrypt in-place is allowed.
125 * const u8 *in, Ciphertext input
126 * unsigned long ciphertext_len, Length of data in bytes for decryption.
127 * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
128 * concatenated with 8 byte Initialisation Vector (from IPSec ESP
129 * Payload) concatenated with 0x00000001. 16-byte aligned pointer.
130 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
131 * const u8 *aad, Additional Authentication Data (AAD)
132 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
133 * to be 8 or 12 bytes
134 * u8 *auth_tag, Authenticated Tag output.
135 * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
136 * Valid values are 16 (most likely), 12 or 8.
137 */
138asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
139 const u8 *in, unsigned long ciphertext_len, u8 *iv,
140 u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
141 u8 *auth_tag, unsigned long auth_tag_len);
142
143static inline struct
144aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
145{
146 return
147 (struct aesni_rfc4106_gcm_ctx *)
148 PTR_ALIGN((u8 *)
149 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
150}
151#endif
152
65static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 153static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
66{ 154{
67 unsigned long addr = (unsigned long)raw_ctx; 155 unsigned long addr = (unsigned long)raw_ctx;
@@ -324,6 +412,7 @@ static struct crypto_alg blk_cbc_alg = {
324 }, 412 },
325}; 413};
326 414
415#ifdef CONFIG_X86_64
327static void ctr_crypt_final(struct crypto_aes_ctx *ctx, 416static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
328 struct blkcipher_walk *walk) 417 struct blkcipher_walk *walk)
329{ 418{
@@ -389,6 +478,7 @@ static struct crypto_alg blk_ctr_alg = {
389 }, 478 },
390 }, 479 },
391}; 480};
481#endif
392 482
393static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, 483static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
394 unsigned int key_len) 484 unsigned int key_len)
@@ -536,6 +626,7 @@ static struct crypto_alg ablk_cbc_alg = {
536 }, 626 },
537}; 627};
538 628
629#ifdef CONFIG_X86_64
539static int ablk_ctr_init(struct crypto_tfm *tfm) 630static int ablk_ctr_init(struct crypto_tfm *tfm)
540{ 631{
541 struct cryptd_ablkcipher *cryptd_tfm; 632 struct cryptd_ablkcipher *cryptd_tfm;
@@ -612,6 +703,7 @@ static struct crypto_alg ablk_rfc3686_ctr_alg = {
612 }, 703 },
613}; 704};
614#endif 705#endif
706#endif
615 707
616#ifdef HAS_LRW 708#ifdef HAS_LRW
617static int ablk_lrw_init(struct crypto_tfm *tfm) 709static int ablk_lrw_init(struct crypto_tfm *tfm)
@@ -730,6 +822,424 @@ static struct crypto_alg ablk_xts_alg = {
730}; 822};
731#endif 823#endif
732 824
825#ifdef CONFIG_X86_64
826static int rfc4106_init(struct crypto_tfm *tfm)
827{
828 struct cryptd_aead *cryptd_tfm;
829 struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
830 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
831 cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
832 if (IS_ERR(cryptd_tfm))
833 return PTR_ERR(cryptd_tfm);
834 ctx->cryptd_tfm = cryptd_tfm;
835 tfm->crt_aead.reqsize = sizeof(struct aead_request)
836 + crypto_aead_reqsize(&cryptd_tfm->base);
837 return 0;
838}
839
840static void rfc4106_exit(struct crypto_tfm *tfm)
841{
842 struct aesni_rfc4106_gcm_ctx *ctx =
843 (struct aesni_rfc4106_gcm_ctx *)
844 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
845 if (!IS_ERR(ctx->cryptd_tfm))
846 cryptd_free_aead(ctx->cryptd_tfm);
847 return;
848}
849
850static void
851rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
852{
853 struct aesni_gcm_set_hash_subkey_result *result = req->data;
854
855 if (err == -EINPROGRESS)
856 return;
857 result->err = err;
858 complete(&result->completion);
859}
860
861static int
862rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
863{
864 struct crypto_ablkcipher *ctr_tfm;
865 struct ablkcipher_request *req;
866 int ret = -EINVAL;
867 struct aesni_hash_subkey_req_data *req_data;
868
869 ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
870 if (IS_ERR(ctr_tfm))
871 return PTR_ERR(ctr_tfm);
872
873 crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
874
875 ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
876 if (ret) {
877 crypto_free_ablkcipher(ctr_tfm);
878 return ret;
879 }
880
881 req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
882 if (!req) {
883 crypto_free_ablkcipher(ctr_tfm);
884 return -EINVAL;
885 }
886
887 req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
888 if (!req_data) {
889 crypto_free_ablkcipher(ctr_tfm);
890 return -ENOMEM;
891 }
892 memset(req_data->iv, 0, sizeof(req_data->iv));
893
894 /* Clear the data in the hash sub key container to zero.*/
895 /* We want to cipher all zeros to create the hash sub key. */
896 memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
897
898 init_completion(&req_data->result.completion);
899 sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
900 ablkcipher_request_set_tfm(req, ctr_tfm);
901 ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
902 CRYPTO_TFM_REQ_MAY_BACKLOG,
903 rfc4106_set_hash_subkey_done,
904 &req_data->result);
905
906 ablkcipher_request_set_crypt(req, &req_data->sg,
907 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
908
909 ret = crypto_ablkcipher_encrypt(req);
910 if (ret == -EINPROGRESS || ret == -EBUSY) {
911 ret = wait_for_completion_interruptible
912 (&req_data->result.completion);
913 if (!ret)
914 ret = req_data->result.err;
915 }
916 ablkcipher_request_free(req);
917 kfree(req_data);
918 crypto_free_ablkcipher(ctr_tfm);
919 return ret;
920}
921
922static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
923 unsigned int key_len)
924{
925 int ret = 0;
926 struct crypto_tfm *tfm = crypto_aead_tfm(parent);
927 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
928 u8 *new_key_mem = NULL;
929
930 if (key_len < 4) {
931 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
932 return -EINVAL;
933 }
934 /*Account for 4 byte nonce at the end.*/
935 key_len -= 4;
936 if (key_len != AES_KEYSIZE_128) {
937 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
938 return -EINVAL;
939 }
940
941 memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
942 /*This must be on a 16 byte boundary!*/
943 if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
944 return -EINVAL;
945
946 if ((unsigned long)key % AESNI_ALIGN) {
947 /*key is not aligned: use an auxuliar aligned pointer*/
948 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
949 if (!new_key_mem)
950 return -ENOMEM;
951
952 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
953 memcpy(new_key_mem, key, key_len);
954 key = new_key_mem;
955 }
956
957 if (!irq_fpu_usable())
958 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
959 key, key_len);
960 else {
961 kernel_fpu_begin();
962 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
963 kernel_fpu_end();
964 }
965 /*This must be on a 16 byte boundary!*/
966 if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
967 ret = -EINVAL;
968 goto exit;
969 }
970 ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
971exit:
972 kfree(new_key_mem);
973 return ret;
974}
975
976/* This is the Integrity Check Value (aka the authentication tag length and can
977 * be 8, 12 or 16 bytes long. */
978static int rfc4106_set_authsize(struct crypto_aead *parent,
979 unsigned int authsize)
980{
981 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
982 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
983
984 switch (authsize) {
985 case 8:
986 case 12:
987 case 16:
988 break;
989 default:
990 return -EINVAL;
991 }
992 crypto_aead_crt(parent)->authsize = authsize;
993 crypto_aead_crt(cryptd_child)->authsize = authsize;
994 return 0;
995}
996
997static int rfc4106_encrypt(struct aead_request *req)
998{
999 int ret;
1000 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1001 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1002 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1003
1004 if (!irq_fpu_usable()) {
1005 struct aead_request *cryptd_req =
1006 (struct aead_request *) aead_request_ctx(req);
1007 memcpy(cryptd_req, req, sizeof(*req));
1008 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1009 return crypto_aead_encrypt(cryptd_req);
1010 } else {
1011 kernel_fpu_begin();
1012 ret = cryptd_child->base.crt_aead.encrypt(req);
1013 kernel_fpu_end();
1014 return ret;
1015 }
1016}
1017
1018static int rfc4106_decrypt(struct aead_request *req)
1019{
1020 int ret;
1021 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1022 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1023 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1024
1025 if (!irq_fpu_usable()) {
1026 struct aead_request *cryptd_req =
1027 (struct aead_request *) aead_request_ctx(req);
1028 memcpy(cryptd_req, req, sizeof(*req));
1029 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1030 return crypto_aead_decrypt(cryptd_req);
1031 } else {
1032 kernel_fpu_begin();
1033 ret = cryptd_child->base.crt_aead.decrypt(req);
1034 kernel_fpu_end();
1035 return ret;
1036 }
1037}
1038
1039static struct crypto_alg rfc4106_alg = {
1040 .cra_name = "rfc4106(gcm(aes))",
1041 .cra_driver_name = "rfc4106-gcm-aesni",
1042 .cra_priority = 400,
1043 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1044 .cra_blocksize = 1,
1045 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1046 .cra_alignmask = 0,
1047 .cra_type = &crypto_nivaead_type,
1048 .cra_module = THIS_MODULE,
1049 .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
1050 .cra_init = rfc4106_init,
1051 .cra_exit = rfc4106_exit,
1052 .cra_u = {
1053 .aead = {
1054 .setkey = rfc4106_set_key,
1055 .setauthsize = rfc4106_set_authsize,
1056 .encrypt = rfc4106_encrypt,
1057 .decrypt = rfc4106_decrypt,
1058 .geniv = "seqiv",
1059 .ivsize = 8,
1060 .maxauthsize = 16,
1061 },
1062 },
1063};
1064
1065static int __driver_rfc4106_encrypt(struct aead_request *req)
1066{
1067 u8 one_entry_in_sg = 0;
1068 u8 *src, *dst, *assoc;
1069 __be32 counter = cpu_to_be32(1);
1070 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1071 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1072 void *aes_ctx = &(ctx->aes_key_expanded);
1073 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1074 u8 iv_tab[16+AESNI_ALIGN];
1075 u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
1076 struct scatter_walk src_sg_walk;
1077 struct scatter_walk assoc_sg_walk;
1078 struct scatter_walk dst_sg_walk;
1079 unsigned int i;
1080
1081 /* Assuming we are supporting rfc4106 64-bit extended */
1082 /* sequence numbers We need to have the AAD length equal */
1083 /* to 8 or 12 bytes */
1084 if (unlikely(req->assoclen != 8 && req->assoclen != 12))
1085 return -EINVAL;
1086 /* IV below built */
1087 for (i = 0; i < 4; i++)
1088 *(iv+i) = ctx->nonce[i];
1089 for (i = 0; i < 8; i++)
1090 *(iv+4+i) = req->iv[i];
1091 *((__be32 *)(iv+12)) = counter;
1092
1093 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1094 one_entry_in_sg = 1;
1095 scatterwalk_start(&src_sg_walk, req->src);
1096 scatterwalk_start(&assoc_sg_walk, req->assoc);
1097 src = scatterwalk_map(&src_sg_walk, 0);
1098 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1099 dst = src;
1100 if (unlikely(req->src != req->dst)) {
1101 scatterwalk_start(&dst_sg_walk, req->dst);
1102 dst = scatterwalk_map(&dst_sg_walk, 0);
1103 }
1104
1105 } else {
1106 /* Allocate memory for src, dst, assoc */
1107 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
1108 GFP_ATOMIC);
1109 if (unlikely(!src))
1110 return -ENOMEM;
1111 assoc = (src + req->cryptlen + auth_tag_len);
1112 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1113 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1114 req->assoclen, 0);
1115 dst = src;
1116 }
1117
1118 aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
1119 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
1120 + ((unsigned long)req->cryptlen), auth_tag_len);
1121
1122 /* The authTag (aka the Integrity Check Value) needs to be written
1123 * back to the packet. */
1124 if (one_entry_in_sg) {
1125 if (unlikely(req->src != req->dst)) {
1126 scatterwalk_unmap(dst, 0);
1127 scatterwalk_done(&dst_sg_walk, 0, 0);
1128 }
1129 scatterwalk_unmap(src, 0);
1130 scatterwalk_unmap(assoc, 0);
1131 scatterwalk_done(&src_sg_walk, 0, 0);
1132 scatterwalk_done(&assoc_sg_walk, 0, 0);
1133 } else {
1134 scatterwalk_map_and_copy(dst, req->dst, 0,
1135 req->cryptlen + auth_tag_len, 1);
1136 kfree(src);
1137 }
1138 return 0;
1139}
1140
1141static int __driver_rfc4106_decrypt(struct aead_request *req)
1142{
1143 u8 one_entry_in_sg = 0;
1144 u8 *src, *dst, *assoc;
1145 unsigned long tempCipherLen = 0;
1146 __be32 counter = cpu_to_be32(1);
1147 int retval = 0;
1148 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1149 struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1150 void *aes_ctx = &(ctx->aes_key_expanded);
1151 unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1152 u8 iv_and_authTag[32+AESNI_ALIGN];
1153 u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
1154 u8 *authTag = iv + 16;
1155 struct scatter_walk src_sg_walk;
1156 struct scatter_walk assoc_sg_walk;
1157 struct scatter_walk dst_sg_walk;
1158 unsigned int i;
1159
1160 if (unlikely((req->cryptlen < auth_tag_len) ||
1161 (req->assoclen != 8 && req->assoclen != 12)))
1162 return -EINVAL;
1163 /* Assuming we are supporting rfc4106 64-bit extended */
1164 /* sequence numbers We need to have the AAD length */
1165 /* equal to 8 or 12 bytes */
1166
1167 tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1168 /* IV below built */
1169 for (i = 0; i < 4; i++)
1170 *(iv+i) = ctx->nonce[i];
1171 for (i = 0; i < 8; i++)
1172 *(iv+4+i) = req->iv[i];
1173 *((__be32 *)(iv+12)) = counter;
1174
1175 if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1176 one_entry_in_sg = 1;
1177 scatterwalk_start(&src_sg_walk, req->src);
1178 scatterwalk_start(&assoc_sg_walk, req->assoc);
1179 src = scatterwalk_map(&src_sg_walk, 0);
1180 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1181 dst = src;
1182 if (unlikely(req->src != req->dst)) {
1183 scatterwalk_start(&dst_sg_walk, req->dst);
1184 dst = scatterwalk_map(&dst_sg_walk, 0);
1185 }
1186
1187 } else {
1188 /* Allocate memory for src, dst, assoc */
1189 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1190 if (!src)
1191 return -ENOMEM;
1192 assoc = (src + req->cryptlen + auth_tag_len);
1193 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1194 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1195 req->assoclen, 0);
1196 dst = src;
1197 }
1198
1199 aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
1200 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
1201 authTag, auth_tag_len);
1202
1203 /* Compare generated tag with passed in tag. */
1204 retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
1205 -EBADMSG : 0;
1206
1207 if (one_entry_in_sg) {
1208 if (unlikely(req->src != req->dst)) {
1209 scatterwalk_unmap(dst, 0);
1210 scatterwalk_done(&dst_sg_walk, 0, 0);
1211 }
1212 scatterwalk_unmap(src, 0);
1213 scatterwalk_unmap(assoc, 0);
1214 scatterwalk_done(&src_sg_walk, 0, 0);
1215 scatterwalk_done(&assoc_sg_walk, 0, 0);
1216 } else {
1217 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
1218 kfree(src);
1219 }
1220 return retval;
1221}
1222
1223static struct crypto_alg __rfc4106_alg = {
1224 .cra_name = "__gcm-aes-aesni",
1225 .cra_driver_name = "__driver-gcm-aes-aesni",
1226 .cra_priority = 0,
1227 .cra_flags = CRYPTO_ALG_TYPE_AEAD,
1228 .cra_blocksize = 1,
1229 .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1230 .cra_alignmask = 0,
1231 .cra_type = &crypto_aead_type,
1232 .cra_module = THIS_MODULE,
1233 .cra_list = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
1234 .cra_u = {
1235 .aead = {
1236 .encrypt = __driver_rfc4106_encrypt,
1237 .decrypt = __driver_rfc4106_decrypt,
1238 },
1239 },
1240};
1241#endif
1242
733static int __init aesni_init(void) 1243static int __init aesni_init(void)
734{ 1244{
735 int err; 1245 int err;
@@ -738,6 +1248,7 @@ static int __init aesni_init(void)
738 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n"); 1248 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n");
739 return -ENODEV; 1249 return -ENODEV;
740 } 1250 }
1251
741 if ((err = crypto_register_alg(&aesni_alg))) 1252 if ((err = crypto_register_alg(&aesni_alg)))
742 goto aes_err; 1253 goto aes_err;
743 if ((err = crypto_register_alg(&__aesni_alg))) 1254 if ((err = crypto_register_alg(&__aesni_alg)))
@@ -746,18 +1257,24 @@ static int __init aesni_init(void)
746 goto blk_ecb_err; 1257 goto blk_ecb_err;
747 if ((err = crypto_register_alg(&blk_cbc_alg))) 1258 if ((err = crypto_register_alg(&blk_cbc_alg)))
748 goto blk_cbc_err; 1259 goto blk_cbc_err;
749 if ((err = crypto_register_alg(&blk_ctr_alg)))
750 goto blk_ctr_err;
751 if ((err = crypto_register_alg(&ablk_ecb_alg))) 1260 if ((err = crypto_register_alg(&ablk_ecb_alg)))
752 goto ablk_ecb_err; 1261 goto ablk_ecb_err;
753 if ((err = crypto_register_alg(&ablk_cbc_alg))) 1262 if ((err = crypto_register_alg(&ablk_cbc_alg)))
754 goto ablk_cbc_err; 1263 goto ablk_cbc_err;
1264#ifdef CONFIG_X86_64
1265 if ((err = crypto_register_alg(&blk_ctr_alg)))
1266 goto blk_ctr_err;
755 if ((err = crypto_register_alg(&ablk_ctr_alg))) 1267 if ((err = crypto_register_alg(&ablk_ctr_alg)))
756 goto ablk_ctr_err; 1268 goto ablk_ctr_err;
1269 if ((err = crypto_register_alg(&__rfc4106_alg)))
1270 goto __aead_gcm_err;
1271 if ((err = crypto_register_alg(&rfc4106_alg)))
1272 goto aead_gcm_err;
757#ifdef HAS_CTR 1273#ifdef HAS_CTR
758 if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg))) 1274 if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
759 goto ablk_rfc3686_ctr_err; 1275 goto ablk_rfc3686_ctr_err;
760#endif 1276#endif
1277#endif
761#ifdef HAS_LRW 1278#ifdef HAS_LRW
762 if ((err = crypto_register_alg(&ablk_lrw_alg))) 1279 if ((err = crypto_register_alg(&ablk_lrw_alg)))
763 goto ablk_lrw_err; 1280 goto ablk_lrw_err;
@@ -770,7 +1287,6 @@ static int __init aesni_init(void)
770 if ((err = crypto_register_alg(&ablk_xts_alg))) 1287 if ((err = crypto_register_alg(&ablk_xts_alg)))
771 goto ablk_xts_err; 1288 goto ablk_xts_err;
772#endif 1289#endif
773
774 return err; 1290 return err;
775 1291
776#ifdef HAS_XTS 1292#ifdef HAS_XTS
@@ -784,18 +1300,24 @@ ablk_pcbc_err:
784 crypto_unregister_alg(&ablk_lrw_alg); 1300 crypto_unregister_alg(&ablk_lrw_alg);
785ablk_lrw_err: 1301ablk_lrw_err:
786#endif 1302#endif
1303#ifdef CONFIG_X86_64
787#ifdef HAS_CTR 1304#ifdef HAS_CTR
788 crypto_unregister_alg(&ablk_rfc3686_ctr_alg); 1305 crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
789ablk_rfc3686_ctr_err: 1306ablk_rfc3686_ctr_err:
790#endif 1307#endif
1308 crypto_unregister_alg(&rfc4106_alg);
1309aead_gcm_err:
1310 crypto_unregister_alg(&__rfc4106_alg);
1311__aead_gcm_err:
791 crypto_unregister_alg(&ablk_ctr_alg); 1312 crypto_unregister_alg(&ablk_ctr_alg);
792ablk_ctr_err: 1313ablk_ctr_err:
1314 crypto_unregister_alg(&blk_ctr_alg);
1315blk_ctr_err:
1316#endif
793 crypto_unregister_alg(&ablk_cbc_alg); 1317 crypto_unregister_alg(&ablk_cbc_alg);
794ablk_cbc_err: 1318ablk_cbc_err:
795 crypto_unregister_alg(&ablk_ecb_alg); 1319 crypto_unregister_alg(&ablk_ecb_alg);
796ablk_ecb_err: 1320ablk_ecb_err:
797 crypto_unregister_alg(&blk_ctr_alg);
798blk_ctr_err:
799 crypto_unregister_alg(&blk_cbc_alg); 1321 crypto_unregister_alg(&blk_cbc_alg);
800blk_cbc_err: 1322blk_cbc_err:
801 crypto_unregister_alg(&blk_ecb_alg); 1323 crypto_unregister_alg(&blk_ecb_alg);
@@ -818,13 +1340,17 @@ static void __exit aesni_exit(void)
818#ifdef HAS_LRW 1340#ifdef HAS_LRW
819 crypto_unregister_alg(&ablk_lrw_alg); 1341 crypto_unregister_alg(&ablk_lrw_alg);
820#endif 1342#endif
1343#ifdef CONFIG_X86_64
821#ifdef HAS_CTR 1344#ifdef HAS_CTR
822 crypto_unregister_alg(&ablk_rfc3686_ctr_alg); 1345 crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
823#endif 1346#endif
1347 crypto_unregister_alg(&rfc4106_alg);
1348 crypto_unregister_alg(&__rfc4106_alg);
824 crypto_unregister_alg(&ablk_ctr_alg); 1349 crypto_unregister_alg(&ablk_ctr_alg);
1350 crypto_unregister_alg(&blk_ctr_alg);
1351#endif
825 crypto_unregister_alg(&ablk_cbc_alg); 1352 crypto_unregister_alg(&ablk_cbc_alg);
826 crypto_unregister_alg(&ablk_ecb_alg); 1353 crypto_unregister_alg(&ablk_ecb_alg);
827 crypto_unregister_alg(&blk_ctr_alg);
828 crypto_unregister_alg(&blk_cbc_alg); 1354 crypto_unregister_alg(&blk_cbc_alg);
829 crypto_unregister_alg(&blk_ecb_alg); 1355 crypto_unregister_alg(&blk_ecb_alg);
830 crypto_unregister_alg(&__aesni_alg); 1356 crypto_unregister_alg(&__aesni_alg);
diff --git a/crypto/Kconfig b/crypto/Kconfig
index e4bac29a32e7..4b7cb0e691cd 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -110,7 +110,6 @@ config CRYPTO_MANAGER_DISABLE_TESTS
110 110
111config CRYPTO_GF128MUL 111config CRYPTO_GF128MUL
112 tristate "GF(2^128) multiplication functions (EXPERIMENTAL)" 112 tristate "GF(2^128) multiplication functions (EXPERIMENTAL)"
113 depends on EXPERIMENTAL
114 help 113 help
115 Efficient table driven implementation of multiplications in the 114 Efficient table driven implementation of multiplications in the
116 field GF(2^128). This is needed by some cypher modes. This 115 field GF(2^128). This is needed by some cypher modes. This
@@ -539,8 +538,9 @@ config CRYPTO_AES_X86_64
539 538
540config CRYPTO_AES_NI_INTEL 539config CRYPTO_AES_NI_INTEL
541 tristate "AES cipher algorithms (AES-NI)" 540 tristate "AES cipher algorithms (AES-NI)"
542 depends on (X86 || UML_X86) && 64BIT 541 depends on (X86 || UML_X86)
543 select CRYPTO_AES_X86_64 542 select CRYPTO_AES_X86_64 if 64BIT
543 select CRYPTO_AES_586 if !64BIT
544 select CRYPTO_CRYPTD 544 select CRYPTO_CRYPTD
545 select CRYPTO_ALGAPI 545 select CRYPTO_ALGAPI
546 select CRYPTO_FPU 546 select CRYPTO_FPU
@@ -563,9 +563,10 @@ config CRYPTO_AES_NI_INTEL
563 563
564 See <http://csrc.nist.gov/encryption/aes/> for more information. 564 See <http://csrc.nist.gov/encryption/aes/> for more information.
565 565
566 In addition to AES cipher algorithm support, the 566 In addition to AES cipher algorithm support, the acceleration
567 acceleration for some popular block cipher mode is supported 567 for some popular block cipher mode is supported too, including
568 too, including ECB, CBC, CTR, LRW, PCBC, XTS. 568 ECB, CBC, LRW, PCBC, XTS. The 64 bit version has additional
569 acceleration for CTR.
569 570
570config CRYPTO_ANUBIS 571config CRYPTO_ANUBIS
571 tristate "Anubis cipher algorithm" 572 tristate "Anubis cipher algorithm"
@@ -841,6 +842,27 @@ config CRYPTO_ANSI_CPRNG
841 ANSI X9.31 A.2.4. Note that this option must be enabled if 842 ANSI X9.31 A.2.4. Note that this option must be enabled if
842 CRYPTO_FIPS is selected 843 CRYPTO_FIPS is selected
843 844
845config CRYPTO_USER_API
846 tristate
847
848config CRYPTO_USER_API_HASH
849 tristate "User-space interface for hash algorithms"
850 depends on NET
851 select CRYPTO_HASH
852 select CRYPTO_USER_API
853 help
854 This option enables the user-spaces interface for hash
855 algorithms.
856
857config CRYPTO_USER_API_SKCIPHER
858 tristate "User-space interface for symmetric key cipher algorithms"
859 depends on NET
860 select CRYPTO_BLKCIPHER
861 select CRYPTO_USER_API
862 help
863 This option enables the user-spaces interface for symmetric
864 key cipher algorithms.
865
844source "drivers/crypto/Kconfig" 866source "drivers/crypto/Kconfig"
845 867
846endif # if CRYPTO 868endif # if CRYPTO
diff --git a/crypto/Makefile b/crypto/Makefile
index 423b7de61f93..e9a399ca69db 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -3,32 +3,32 @@
3# 3#
4 4
5obj-$(CONFIG_CRYPTO) += crypto.o 5obj-$(CONFIG_CRYPTO) += crypto.o
6crypto-objs := api.o cipher.o compress.o 6crypto-y := api.o cipher.o compress.o
7 7
8obj-$(CONFIG_CRYPTO_WORKQUEUE) += crypto_wq.o 8obj-$(CONFIG_CRYPTO_WORKQUEUE) += crypto_wq.o
9 9
10obj-$(CONFIG_CRYPTO_FIPS) += fips.o 10obj-$(CONFIG_CRYPTO_FIPS) += fips.o
11 11
12crypto_algapi-$(CONFIG_PROC_FS) += proc.o 12crypto_algapi-$(CONFIG_PROC_FS) += proc.o
13crypto_algapi-objs := algapi.o scatterwalk.o $(crypto_algapi-y) 13crypto_algapi-y := algapi.o scatterwalk.o $(crypto_algapi-y)
14obj-$(CONFIG_CRYPTO_ALGAPI2) += crypto_algapi.o 14obj-$(CONFIG_CRYPTO_ALGAPI2) += crypto_algapi.o
15 15
16obj-$(CONFIG_CRYPTO_AEAD2) += aead.o 16obj-$(CONFIG_CRYPTO_AEAD2) += aead.o
17 17
18crypto_blkcipher-objs := ablkcipher.o 18crypto_blkcipher-y := ablkcipher.o
19crypto_blkcipher-objs += blkcipher.o 19crypto_blkcipher-y += blkcipher.o
20obj-$(CONFIG_CRYPTO_BLKCIPHER2) += crypto_blkcipher.o 20obj-$(CONFIG_CRYPTO_BLKCIPHER2) += crypto_blkcipher.o
21obj-$(CONFIG_CRYPTO_BLKCIPHER2) += chainiv.o 21obj-$(CONFIG_CRYPTO_BLKCIPHER2) += chainiv.o
22obj-$(CONFIG_CRYPTO_BLKCIPHER2) += eseqiv.o 22obj-$(CONFIG_CRYPTO_BLKCIPHER2) += eseqiv.o
23obj-$(CONFIG_CRYPTO_SEQIV) += seqiv.o 23obj-$(CONFIG_CRYPTO_SEQIV) += seqiv.o
24 24
25crypto_hash-objs += ahash.o 25crypto_hash-y += ahash.o
26crypto_hash-objs += shash.o 26crypto_hash-y += shash.o
27obj-$(CONFIG_CRYPTO_HASH2) += crypto_hash.o 27obj-$(CONFIG_CRYPTO_HASH2) += crypto_hash.o
28 28
29obj-$(CONFIG_CRYPTO_PCOMP2) += pcompress.o 29obj-$(CONFIG_CRYPTO_PCOMP2) += pcompress.o
30 30
31cryptomgr-objs := algboss.o testmgr.o 31cryptomgr-y := algboss.o testmgr.o
32 32
33obj-$(CONFIG_CRYPTO_MANAGER2) += cryptomgr.o 33obj-$(CONFIG_CRYPTO_MANAGER2) += cryptomgr.o
34obj-$(CONFIG_CRYPTO_HMAC) += hmac.o 34obj-$(CONFIG_CRYPTO_HMAC) += hmac.o
@@ -85,6 +85,9 @@ obj-$(CONFIG_CRYPTO_RNG2) += krng.o
85obj-$(CONFIG_CRYPTO_ANSI_CPRNG) += ansi_cprng.o 85obj-$(CONFIG_CRYPTO_ANSI_CPRNG) += ansi_cprng.o
86obj-$(CONFIG_CRYPTO_TEST) += tcrypt.o 86obj-$(CONFIG_CRYPTO_TEST) += tcrypt.o
87obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o 87obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o
88obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o
89obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o
90obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o
88 91
89# 92#
90# generic algorithms and the async_tx api 93# generic algorithms and the async_tx api
diff --git a/crypto/af_alg.c b/crypto/af_alg.c
new file mode 100644
index 000000000000..940d70cb5c25
--- /dev/null
+++ b/crypto/af_alg.c
@@ -0,0 +1,483 @@
1/*
2 * af_alg: User-space algorithm interface
3 *
4 * This file provides the user-space API for algorithms.
5 *
6 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 */
14
15#include <asm/atomic.h>
16#include <crypto/if_alg.h>
17#include <linux/crypto.h>
18#include <linux/init.h>
19#include <linux/kernel.h>
20#include <linux/list.h>
21#include <linux/module.h>
22#include <linux/net.h>
23#include <linux/rwsem.h>
24
25struct alg_type_list {
26 const struct af_alg_type *type;
27 struct list_head list;
28};
29
30static atomic_long_t alg_memory_allocated;
31
32static struct proto alg_proto = {
33 .name = "ALG",
34 .owner = THIS_MODULE,
35 .memory_allocated = &alg_memory_allocated,
36 .obj_size = sizeof(struct alg_sock),
37};
38
39static LIST_HEAD(alg_types);
40static DECLARE_RWSEM(alg_types_sem);
41
42static const struct af_alg_type *alg_get_type(const char *name)
43{
44 const struct af_alg_type *type = ERR_PTR(-ENOENT);
45 struct alg_type_list *node;
46
47 down_read(&alg_types_sem);
48 list_for_each_entry(node, &alg_types, list) {
49 if (strcmp(node->type->name, name))
50 continue;
51
52 if (try_module_get(node->type->owner))
53 type = node->type;
54 break;
55 }
56 up_read(&alg_types_sem);
57
58 return type;
59}
60
61int af_alg_register_type(const struct af_alg_type *type)
62{
63 struct alg_type_list *node;
64 int err = -EEXIST;
65
66 down_write(&alg_types_sem);
67 list_for_each_entry(node, &alg_types, list) {
68 if (!strcmp(node->type->name, type->name))
69 goto unlock;
70 }
71
72 node = kmalloc(sizeof(*node), GFP_KERNEL);
73 err = -ENOMEM;
74 if (!node)
75 goto unlock;
76
77 type->ops->owner = THIS_MODULE;
78 node->type = type;
79 list_add(&node->list, &alg_types);
80 err = 0;
81
82unlock:
83 up_write(&alg_types_sem);
84
85 return err;
86}
87EXPORT_SYMBOL_GPL(af_alg_register_type);
88
89int af_alg_unregister_type(const struct af_alg_type *type)
90{
91 struct alg_type_list *node;
92 int err = -ENOENT;
93
94 down_write(&alg_types_sem);
95 list_for_each_entry(node, &alg_types, list) {
96 if (strcmp(node->type->name, type->name))
97 continue;
98
99 list_del(&node->list);
100 kfree(node);
101 err = 0;
102 break;
103 }
104 up_write(&alg_types_sem);
105
106 return err;
107}
108EXPORT_SYMBOL_GPL(af_alg_unregister_type);
109
110static void alg_do_release(const struct af_alg_type *type, void *private)
111{
112 if (!type)
113 return;
114
115 type->release(private);
116 module_put(type->owner);
117}
118
119int af_alg_release(struct socket *sock)
120{
121 if (sock->sk)
122 sock_put(sock->sk);
123 return 0;
124}
125EXPORT_SYMBOL_GPL(af_alg_release);
126
127static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
128{
129 struct sock *sk = sock->sk;
130 struct alg_sock *ask = alg_sk(sk);
131 struct sockaddr_alg *sa = (void *)uaddr;
132 const struct af_alg_type *type;
133 void *private;
134
135 if (sock->state == SS_CONNECTED)
136 return -EINVAL;
137
138 if (addr_len != sizeof(*sa))
139 return -EINVAL;
140
141 sa->salg_type[sizeof(sa->salg_type) - 1] = 0;
142 sa->salg_name[sizeof(sa->salg_name) - 1] = 0;
143
144 type = alg_get_type(sa->salg_type);
145 if (IS_ERR(type) && PTR_ERR(type) == -ENOENT) {
146 request_module("algif-%s", sa->salg_type);
147 type = alg_get_type(sa->salg_type);
148 }
149
150 if (IS_ERR(type))
151 return PTR_ERR(type);
152
153 private = type->bind(sa->salg_name, sa->salg_feat, sa->salg_mask);
154 if (IS_ERR(private)) {
155 module_put(type->owner);
156 return PTR_ERR(private);
157 }
158
159 lock_sock(sk);
160
161 swap(ask->type, type);
162 swap(ask->private, private);
163
164 release_sock(sk);
165
166 alg_do_release(type, private);
167
168 return 0;
169}
170
171static int alg_setkey(struct sock *sk, char __user *ukey,
172 unsigned int keylen)
173{
174 struct alg_sock *ask = alg_sk(sk);
175 const struct af_alg_type *type = ask->type;
176 u8 *key;
177 int err;
178
179 key = sock_kmalloc(sk, keylen, GFP_KERNEL);
180 if (!key)
181 return -ENOMEM;
182
183 err = -EFAULT;
184 if (copy_from_user(key, ukey, keylen))
185 goto out;
186
187 err = type->setkey(ask->private, key, keylen);
188
189out:
190 sock_kfree_s(sk, key, keylen);
191
192 return err;
193}
194
195static int alg_setsockopt(struct socket *sock, int level, int optname,
196 char __user *optval, unsigned int optlen)
197{
198 struct sock *sk = sock->sk;
199 struct alg_sock *ask = alg_sk(sk);
200 const struct af_alg_type *type;
201 int err = -ENOPROTOOPT;
202
203 lock_sock(sk);
204 type = ask->type;
205
206 if (level != SOL_ALG || !type)
207 goto unlock;
208
209 switch (optname) {
210 case ALG_SET_KEY:
211 if (sock->state == SS_CONNECTED)
212 goto unlock;
213 if (!type->setkey)
214 goto unlock;
215
216 err = alg_setkey(sk, optval, optlen);
217 }
218
219unlock:
220 release_sock(sk);
221
222 return err;
223}
224
225int af_alg_accept(struct sock *sk, struct socket *newsock)
226{
227 struct alg_sock *ask = alg_sk(sk);
228 const struct af_alg_type *type;
229 struct sock *sk2;
230 int err;
231
232 lock_sock(sk);
233 type = ask->type;
234
235 err = -EINVAL;
236 if (!type)
237 goto unlock;
238
239 sk2 = sk_alloc(sock_net(sk), PF_ALG, GFP_KERNEL, &alg_proto);
240 err = -ENOMEM;
241 if (!sk2)
242 goto unlock;
243
244 sock_init_data(newsock, sk2);
245 sock_graft(sk2, newsock);
246
247 err = type->accept(ask->private, sk2);
248 if (err) {
249 sk_free(sk2);
250 goto unlock;
251 }
252
253 sk2->sk_family = PF_ALG;
254
255 sock_hold(sk);
256 alg_sk(sk2)->parent = sk;
257 alg_sk(sk2)->type = type;
258
259 newsock->ops = type->ops;
260 newsock->state = SS_CONNECTED;
261
262 err = 0;
263
264unlock:
265 release_sock(sk);
266
267 return err;
268}
269EXPORT_SYMBOL_GPL(af_alg_accept);
270
271static int alg_accept(struct socket *sock, struct socket *newsock, int flags)
272{
273 return af_alg_accept(sock->sk, newsock);
274}
275
276static const struct proto_ops alg_proto_ops = {
277 .family = PF_ALG,
278 .owner = THIS_MODULE,
279
280 .connect = sock_no_connect,
281 .socketpair = sock_no_socketpair,
282 .getname = sock_no_getname,
283 .ioctl = sock_no_ioctl,
284 .listen = sock_no_listen,
285 .shutdown = sock_no_shutdown,
286 .getsockopt = sock_no_getsockopt,
287 .mmap = sock_no_mmap,
288 .sendpage = sock_no_sendpage,
289 .sendmsg = sock_no_sendmsg,
290 .recvmsg = sock_no_recvmsg,
291 .poll = sock_no_poll,
292
293 .bind = alg_bind,
294 .release = af_alg_release,
295 .setsockopt = alg_setsockopt,
296 .accept = alg_accept,
297};
298
299static void alg_sock_destruct(struct sock *sk)
300{
301 struct alg_sock *ask = alg_sk(sk);
302
303 alg_do_release(ask->type, ask->private);
304}
305
306static int alg_create(struct net *net, struct socket *sock, int protocol,
307 int kern)
308{
309 struct sock *sk;
310 int err;
311
312 if (sock->type != SOCK_SEQPACKET)
313 return -ESOCKTNOSUPPORT;
314 if (protocol != 0)
315 return -EPROTONOSUPPORT;
316
317 err = -ENOMEM;
318 sk = sk_alloc(net, PF_ALG, GFP_KERNEL, &alg_proto);
319 if (!sk)
320 goto out;
321
322 sock->ops = &alg_proto_ops;
323 sock_init_data(sock, sk);
324
325 sk->sk_family = PF_ALG;
326 sk->sk_destruct = alg_sock_destruct;
327
328 return 0;
329out:
330 return err;
331}
332
333static const struct net_proto_family alg_family = {
334 .family = PF_ALG,
335 .create = alg_create,
336 .owner = THIS_MODULE,
337};
338
339int af_alg_make_sg(struct af_alg_sgl *sgl, void __user *addr, int len,
340 int write)
341{
342 unsigned long from = (unsigned long)addr;
343 unsigned long npages;
344 unsigned off;
345 int err;
346 int i;
347
348 err = -EFAULT;
349 if (!access_ok(write ? VERIFY_READ : VERIFY_WRITE, addr, len))
350 goto out;
351
352 off = from & ~PAGE_MASK;
353 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
354 if (npages > ALG_MAX_PAGES)
355 npages = ALG_MAX_PAGES;
356
357 err = get_user_pages_fast(from, npages, write, sgl->pages);
358 if (err < 0)
359 goto out;
360
361 npages = err;
362 err = -EINVAL;
363 if (WARN_ON(npages == 0))
364 goto out;
365
366 err = 0;
367
368 sg_init_table(sgl->sg, npages);
369
370 for (i = 0; i < npages; i++) {
371 int plen = min_t(int, len, PAGE_SIZE - off);
372
373 sg_set_page(sgl->sg + i, sgl->pages[i], plen, off);
374
375 off = 0;
376 len -= plen;
377 err += plen;
378 }
379
380out:
381 return err;
382}
383EXPORT_SYMBOL_GPL(af_alg_make_sg);
384
385void af_alg_free_sg(struct af_alg_sgl *sgl)
386{
387 int i;
388
389 i = 0;
390 do {
391 put_page(sgl->pages[i]);
392 } while (!sg_is_last(sgl->sg + (i++)));
393}
394EXPORT_SYMBOL_GPL(af_alg_free_sg);
395
396int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con)
397{
398 struct cmsghdr *cmsg;
399
400 for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
401 if (!CMSG_OK(msg, cmsg))
402 return -EINVAL;
403 if (cmsg->cmsg_level != SOL_ALG)
404 continue;
405
406 switch(cmsg->cmsg_type) {
407 case ALG_SET_IV:
408 if (cmsg->cmsg_len < CMSG_LEN(sizeof(*con->iv)))
409 return -EINVAL;
410 con->iv = (void *)CMSG_DATA(cmsg);
411 if (cmsg->cmsg_len < CMSG_LEN(con->iv->ivlen +
412 sizeof(*con->iv)))
413 return -EINVAL;
414 break;
415
416 case ALG_SET_OP:
417 if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32)))
418 return -EINVAL;
419 con->op = *(u32 *)CMSG_DATA(cmsg);
420 break;
421
422 default:
423 return -EINVAL;
424 }
425 }
426
427 return 0;
428}
429EXPORT_SYMBOL_GPL(af_alg_cmsg_send);
430
431int af_alg_wait_for_completion(int err, struct af_alg_completion *completion)
432{
433 switch (err) {
434 case -EINPROGRESS:
435 case -EBUSY:
436 wait_for_completion(&completion->completion);
437 INIT_COMPLETION(completion->completion);
438 err = completion->err;
439 break;
440 };
441
442 return err;
443}
444EXPORT_SYMBOL_GPL(af_alg_wait_for_completion);
445
446void af_alg_complete(struct crypto_async_request *req, int err)
447{
448 struct af_alg_completion *completion = req->data;
449
450 completion->err = err;
451 complete(&completion->completion);
452}
453EXPORT_SYMBOL_GPL(af_alg_complete);
454
455static int __init af_alg_init(void)
456{
457 int err = proto_register(&alg_proto, 0);
458
459 if (err)
460 goto out;
461
462 err = sock_register(&alg_family);
463 if (err != 0)
464 goto out_unregister_proto;
465
466out:
467 return err;
468
469out_unregister_proto:
470 proto_unregister(&alg_proto);
471 goto out;
472}
473
474static void __exit af_alg_exit(void)
475{
476 sock_unregister(PF_ALG);
477 proto_unregister(&alg_proto);
478}
479
480module_init(af_alg_init);
481module_exit(af_alg_exit);
482MODULE_LICENSE("GPL");
483MODULE_ALIAS_NETPROTO(AF_ALG);
diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c
new file mode 100644
index 000000000000..62122a1a2f7a
--- /dev/null
+++ b/crypto/algif_hash.c
@@ -0,0 +1,319 @@
1/*
2 * algif_hash: User-space interface for hash algorithms
3 *
4 * This file provides the user-space API for hash algorithms.
5 *
6 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 */
14
15#include <crypto/hash.h>
16#include <crypto/if_alg.h>
17#include <linux/init.h>
18#include <linux/kernel.h>
19#include <linux/mm.h>
20#include <linux/module.h>
21#include <linux/net.h>
22#include <net/sock.h>
23
24struct hash_ctx {
25 struct af_alg_sgl sgl;
26
27 u8 *result;
28
29 struct af_alg_completion completion;
30
31 unsigned int len;
32 bool more;
33
34 struct ahash_request req;
35};
36
37static int hash_sendmsg(struct kiocb *unused, struct socket *sock,
38 struct msghdr *msg, size_t ignored)
39{
40 int limit = ALG_MAX_PAGES * PAGE_SIZE;
41 struct sock *sk = sock->sk;
42 struct alg_sock *ask = alg_sk(sk);
43 struct hash_ctx *ctx = ask->private;
44 unsigned long iovlen;
45 struct iovec *iov;
46 long copied = 0;
47 int err;
48
49 if (limit > sk->sk_sndbuf)
50 limit = sk->sk_sndbuf;
51
52 lock_sock(sk);
53 if (!ctx->more) {
54 err = crypto_ahash_init(&ctx->req);
55 if (err)
56 goto unlock;
57 }
58
59 ctx->more = 0;
60
61 for (iov = msg->msg_iov, iovlen = msg->msg_iovlen; iovlen > 0;
62 iovlen--, iov++) {
63 unsigned long seglen = iov->iov_len;
64 char __user *from = iov->iov_base;
65
66 while (seglen) {
67 int len = min_t(unsigned long, seglen, limit);
68 int newlen;
69
70 newlen = af_alg_make_sg(&ctx->sgl, from, len, 0);
71 if (newlen < 0)
72 goto unlock;
73
74 ahash_request_set_crypt(&ctx->req, ctx->sgl.sg, NULL,
75 newlen);
76
77 err = af_alg_wait_for_completion(
78 crypto_ahash_update(&ctx->req),
79 &ctx->completion);
80
81 af_alg_free_sg(&ctx->sgl);
82
83 if (err)
84 goto unlock;
85
86 seglen -= newlen;
87 from += newlen;
88 copied += newlen;
89 }
90 }
91
92 err = 0;
93
94 ctx->more = msg->msg_flags & MSG_MORE;
95 if (!ctx->more) {
96 ahash_request_set_crypt(&ctx->req, NULL, ctx->result, 0);
97 err = af_alg_wait_for_completion(crypto_ahash_final(&ctx->req),
98 &ctx->completion);
99 }
100
101unlock:
102 release_sock(sk);
103
104 return err ?: copied;
105}
106
107static ssize_t hash_sendpage(struct socket *sock, struct page *page,
108 int offset, size_t size, int flags)
109{
110 struct sock *sk = sock->sk;
111 struct alg_sock *ask = alg_sk(sk);
112 struct hash_ctx *ctx = ask->private;
113 int err;
114
115 lock_sock(sk);
116 sg_init_table(ctx->sgl.sg, 1);
117 sg_set_page(ctx->sgl.sg, page, size, offset);
118
119 ahash_request_set_crypt(&ctx->req, ctx->sgl.sg, ctx->result, size);
120
121 if (!(flags & MSG_MORE)) {
122 if (ctx->more)
123 err = crypto_ahash_finup(&ctx->req);
124 else
125 err = crypto_ahash_digest(&ctx->req);
126 } else {
127 if (!ctx->more) {
128 err = crypto_ahash_init(&ctx->req);
129 if (err)
130 goto unlock;
131 }
132
133 err = crypto_ahash_update(&ctx->req);
134 }
135
136 err = af_alg_wait_for_completion(err, &ctx->completion);
137 if (err)
138 goto unlock;
139
140 ctx->more = flags & MSG_MORE;
141
142unlock:
143 release_sock(sk);
144
145 return err ?: size;
146}
147
148static int hash_recvmsg(struct kiocb *unused, struct socket *sock,
149 struct msghdr *msg, size_t len, int flags)
150{
151 struct sock *sk = sock->sk;
152 struct alg_sock *ask = alg_sk(sk);
153 struct hash_ctx *ctx = ask->private;
154 unsigned ds = crypto_ahash_digestsize(crypto_ahash_reqtfm(&ctx->req));
155 int err;
156
157 if (len > ds)
158 len = ds;
159 else if (len < ds)
160 msg->msg_flags |= MSG_TRUNC;
161
162 lock_sock(sk);
163 if (ctx->more) {
164 ctx->more = 0;
165 ahash_request_set_crypt(&ctx->req, NULL, ctx->result, 0);
166 err = af_alg_wait_for_completion(crypto_ahash_final(&ctx->req),
167 &ctx->completion);
168 if (err)
169 goto unlock;
170 }
171
172 err = memcpy_toiovec(msg->msg_iov, ctx->result, len);
173
174unlock:
175 release_sock(sk);
176
177 return err ?: len;
178}
179
180static int hash_accept(struct socket *sock, struct socket *newsock, int flags)
181{
182 struct sock *sk = sock->sk;
183 struct alg_sock *ask = alg_sk(sk);
184 struct hash_ctx *ctx = ask->private;
185 struct ahash_request *req = &ctx->req;
186 char state[crypto_ahash_statesize(crypto_ahash_reqtfm(req))];
187 struct sock *sk2;
188 struct alg_sock *ask2;
189 struct hash_ctx *ctx2;
190 int err;
191
192 err = crypto_ahash_export(req, state);
193 if (err)
194 return err;
195
196 err = af_alg_accept(ask->parent, newsock);
197 if (err)
198 return err;
199
200 sk2 = newsock->sk;
201 ask2 = alg_sk(sk2);
202 ctx2 = ask2->private;
203 ctx2->more = 1;
204
205 err = crypto_ahash_import(&ctx2->req, state);
206 if (err) {
207 sock_orphan(sk2);
208 sock_put(sk2);
209 }
210
211 return err;
212}
213
214static struct proto_ops algif_hash_ops = {
215 .family = PF_ALG,
216
217 .connect = sock_no_connect,
218 .socketpair = sock_no_socketpair,
219 .getname = sock_no_getname,
220 .ioctl = sock_no_ioctl,
221 .listen = sock_no_listen,
222 .shutdown = sock_no_shutdown,
223 .getsockopt = sock_no_getsockopt,
224 .mmap = sock_no_mmap,
225 .bind = sock_no_bind,
226 .setsockopt = sock_no_setsockopt,
227 .poll = sock_no_poll,
228
229 .release = af_alg_release,
230 .sendmsg = hash_sendmsg,
231 .sendpage = hash_sendpage,
232 .recvmsg = hash_recvmsg,
233 .accept = hash_accept,
234};
235
236static void *hash_bind(const char *name, u32 type, u32 mask)
237{
238 return crypto_alloc_ahash(name, type, mask);
239}
240
241static void hash_release(void *private)
242{
243 crypto_free_ahash(private);
244}
245
246static int hash_setkey(void *private, const u8 *key, unsigned int keylen)
247{
248 return crypto_ahash_setkey(private, key, keylen);
249}
250
251static void hash_sock_destruct(struct sock *sk)
252{
253 struct alg_sock *ask = alg_sk(sk);
254 struct hash_ctx *ctx = ask->private;
255
256 sock_kfree_s(sk, ctx->result,
257 crypto_ahash_digestsize(crypto_ahash_reqtfm(&ctx->req)));
258 sock_kfree_s(sk, ctx, ctx->len);
259 af_alg_release_parent(sk);
260}
261
262static int hash_accept_parent(void *private, struct sock *sk)
263{
264 struct hash_ctx *ctx;
265 struct alg_sock *ask = alg_sk(sk);
266 unsigned len = sizeof(*ctx) + crypto_ahash_reqsize(private);
267 unsigned ds = crypto_ahash_digestsize(private);
268
269 ctx = sock_kmalloc(sk, len, GFP_KERNEL);
270 if (!ctx)
271 return -ENOMEM;
272
273 ctx->result = sock_kmalloc(sk, ds, GFP_KERNEL);
274 if (!ctx->result) {
275 sock_kfree_s(sk, ctx, len);
276 return -ENOMEM;
277 }
278
279 memset(ctx->result, 0, ds);
280
281 ctx->len = len;
282 ctx->more = 0;
283 af_alg_init_completion(&ctx->completion);
284
285 ask->private = ctx;
286
287 ahash_request_set_tfm(&ctx->req, private);
288 ahash_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_BACKLOG,
289 af_alg_complete, &ctx->completion);
290
291 sk->sk_destruct = hash_sock_destruct;
292
293 return 0;
294}
295
296static const struct af_alg_type algif_type_hash = {
297 .bind = hash_bind,
298 .release = hash_release,
299 .setkey = hash_setkey,
300 .accept = hash_accept_parent,
301 .ops = &algif_hash_ops,
302 .name = "hash",
303 .owner = THIS_MODULE
304};
305
306static int __init algif_hash_init(void)
307{
308 return af_alg_register_type(&algif_type_hash);
309}
310
311static void __exit algif_hash_exit(void)
312{
313 int err = af_alg_unregister_type(&algif_type_hash);
314 BUG_ON(err);
315}
316
317module_init(algif_hash_init);
318module_exit(algif_hash_exit);
319MODULE_LICENSE("GPL");
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
new file mode 100644
index 000000000000..6a6dfc062d2a
--- /dev/null
+++ b/crypto/algif_skcipher.c
@@ -0,0 +1,632 @@
1/*
2 * algif_skcipher: User-space interface for skcipher algorithms
3 *
4 * This file provides the user-space API for symmetric key ciphers.
5 *
6 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 */
14
15#include <crypto/scatterwalk.h>
16#include <crypto/skcipher.h>
17#include <crypto/if_alg.h>
18#include <linux/init.h>
19#include <linux/list.h>
20#include <linux/kernel.h>
21#include <linux/mm.h>
22#include <linux/module.h>
23#include <linux/net.h>
24#include <net/sock.h>
25
26struct skcipher_sg_list {
27 struct list_head list;
28
29 int cur;
30
31 struct scatterlist sg[0];
32};
33
34struct skcipher_ctx {
35 struct list_head tsgl;
36 struct af_alg_sgl rsgl;
37
38 void *iv;
39
40 struct af_alg_completion completion;
41
42 unsigned used;
43
44 unsigned int len;
45 bool more;
46 bool merge;
47 bool enc;
48
49 struct ablkcipher_request req;
50};
51
52#define MAX_SGL_ENTS ((PAGE_SIZE - sizeof(struct skcipher_sg_list)) / \
53 sizeof(struct scatterlist) - 1)
54
55static inline int skcipher_sndbuf(struct sock *sk)
56{
57 struct alg_sock *ask = alg_sk(sk);
58 struct skcipher_ctx *ctx = ask->private;
59
60 return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) -
61 ctx->used, 0);
62}
63
64static inline bool skcipher_writable(struct sock *sk)
65{
66 return PAGE_SIZE <= skcipher_sndbuf(sk);
67}
68
69static int skcipher_alloc_sgl(struct sock *sk)
70{
71 struct alg_sock *ask = alg_sk(sk);
72 struct skcipher_ctx *ctx = ask->private;
73 struct skcipher_sg_list *sgl;
74 struct scatterlist *sg = NULL;
75
76 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
77 if (!list_empty(&ctx->tsgl))
78 sg = sgl->sg;
79
80 if (!sg || sgl->cur >= MAX_SGL_ENTS) {
81 sgl = sock_kmalloc(sk, sizeof(*sgl) +
82 sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1),
83 GFP_KERNEL);
84 if (!sgl)
85 return -ENOMEM;
86
87 sg_init_table(sgl->sg, MAX_SGL_ENTS + 1);
88 sgl->cur = 0;
89
90 if (sg)
91 scatterwalk_sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
92
93 list_add_tail(&sgl->list, &ctx->tsgl);
94 }
95
96 return 0;
97}
98
99static void skcipher_pull_sgl(struct sock *sk, int used)
100{
101 struct alg_sock *ask = alg_sk(sk);
102 struct skcipher_ctx *ctx = ask->private;
103 struct skcipher_sg_list *sgl;
104 struct scatterlist *sg;
105 int i;
106
107 while (!list_empty(&ctx->tsgl)) {
108 sgl = list_first_entry(&ctx->tsgl, struct skcipher_sg_list,
109 list);
110 sg = sgl->sg;
111
112 for (i = 0; i < sgl->cur; i++) {
113 int plen = min_t(int, used, sg[i].length);
114
115 if (!sg_page(sg + i))
116 continue;
117
118 sg[i].length -= plen;
119 sg[i].offset += plen;
120
121 used -= plen;
122 ctx->used -= plen;
123
124 if (sg[i].length)
125 return;
126
127 put_page(sg_page(sg + i));
128 sg_assign_page(sg + i, NULL);
129 }
130
131 list_del(&sgl->list);
132 sock_kfree_s(sk, sgl,
133 sizeof(*sgl) + sizeof(sgl->sg[0]) *
134 (MAX_SGL_ENTS + 1));
135 }
136
137 if (!ctx->used)
138 ctx->merge = 0;
139}
140
141static void skcipher_free_sgl(struct sock *sk)
142{
143 struct alg_sock *ask = alg_sk(sk);
144 struct skcipher_ctx *ctx = ask->private;
145
146 skcipher_pull_sgl(sk, ctx->used);
147}
148
149static int skcipher_wait_for_wmem(struct sock *sk, unsigned flags)
150{
151 long timeout;
152 DEFINE_WAIT(wait);
153 int err = -ERESTARTSYS;
154
155 if (flags & MSG_DONTWAIT)
156 return -EAGAIN;
157
158 set_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
159
160 for (;;) {
161 if (signal_pending(current))
162 break;
163 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
164 timeout = MAX_SCHEDULE_TIMEOUT;
165 if (sk_wait_event(sk, &timeout, skcipher_writable(sk))) {
166 err = 0;
167 break;
168 }
169 }
170 finish_wait(sk_sleep(sk), &wait);
171
172 return err;
173}
174
175static void skcipher_wmem_wakeup(struct sock *sk)
176{
177 struct socket_wq *wq;
178
179 if (!skcipher_writable(sk))
180 return;
181
182 rcu_read_lock();
183 wq = rcu_dereference(sk->sk_wq);
184 if (wq_has_sleeper(wq))
185 wake_up_interruptible_sync_poll(&wq->wait, POLLIN |
186 POLLRDNORM |
187 POLLRDBAND);
188 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
189 rcu_read_unlock();
190}
191
192static int skcipher_wait_for_data(struct sock *sk, unsigned flags)
193{
194 struct alg_sock *ask = alg_sk(sk);
195 struct skcipher_ctx *ctx = ask->private;
196 long timeout;
197 DEFINE_WAIT(wait);
198 int err = -ERESTARTSYS;
199
200 if (flags & MSG_DONTWAIT) {
201 return -EAGAIN;
202 }
203
204 set_bit(SOCK_ASYNC_WAITDATA, &sk->sk_socket->flags);
205
206 for (;;) {
207 if (signal_pending(current))
208 break;
209 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
210 timeout = MAX_SCHEDULE_TIMEOUT;
211 if (sk_wait_event(sk, &timeout, ctx->used)) {
212 err = 0;
213 break;
214 }
215 }
216 finish_wait(sk_sleep(sk), &wait);
217
218 clear_bit(SOCK_ASYNC_WAITDATA, &sk->sk_socket->flags);
219
220 return err;
221}
222
223static void skcipher_data_wakeup(struct sock *sk)
224{
225 struct alg_sock *ask = alg_sk(sk);
226 struct skcipher_ctx *ctx = ask->private;
227 struct socket_wq *wq;
228
229 if (!ctx->used)
230 return;
231
232 rcu_read_lock();
233 wq = rcu_dereference(sk->sk_wq);
234 if (wq_has_sleeper(wq))
235 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
236 POLLRDNORM |
237 POLLRDBAND);
238 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
239 rcu_read_unlock();
240}
241
242static int skcipher_sendmsg(struct kiocb *unused, struct socket *sock,
243 struct msghdr *msg, size_t size)
244{
245 struct sock *sk = sock->sk;
246 struct alg_sock *ask = alg_sk(sk);
247 struct skcipher_ctx *ctx = ask->private;
248 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(&ctx->req);
249 unsigned ivsize = crypto_ablkcipher_ivsize(tfm);
250 struct skcipher_sg_list *sgl;
251 struct af_alg_control con = {};
252 long copied = 0;
253 bool enc = 0;
254 int err;
255 int i;
256
257 if (msg->msg_controllen) {
258 err = af_alg_cmsg_send(msg, &con);
259 if (err)
260 return err;
261
262 switch (con.op) {
263 case ALG_OP_ENCRYPT:
264 enc = 1;
265 break;
266 case ALG_OP_DECRYPT:
267 enc = 0;
268 break;
269 default:
270 return -EINVAL;
271 }
272
273 if (con.iv && con.iv->ivlen != ivsize)
274 return -EINVAL;
275 }
276
277 err = -EINVAL;
278
279 lock_sock(sk);
280 if (!ctx->more && ctx->used)
281 goto unlock;
282
283 if (!ctx->used) {
284 ctx->enc = enc;
285 if (con.iv)
286 memcpy(ctx->iv, con.iv->iv, ivsize);
287 }
288
289 while (size) {
290 struct scatterlist *sg;
291 unsigned long len = size;
292 int plen;
293
294 if (ctx->merge) {
295 sgl = list_entry(ctx->tsgl.prev,
296 struct skcipher_sg_list, list);
297 sg = sgl->sg + sgl->cur - 1;
298 len = min_t(unsigned long, len,
299 PAGE_SIZE - sg->offset - sg->length);
300
301 err = memcpy_fromiovec(page_address(sg_page(sg)) +
302 sg->offset + sg->length,
303 msg->msg_iov, len);
304 if (err)
305 goto unlock;
306
307 sg->length += len;
308 ctx->merge = (sg->offset + sg->length) &
309 (PAGE_SIZE - 1);
310
311 ctx->used += len;
312 copied += len;
313 size -= len;
314 continue;
315 }
316
317 if (!skcipher_writable(sk)) {
318 err = skcipher_wait_for_wmem(sk, msg->msg_flags);
319 if (err)
320 goto unlock;
321 }
322
323 len = min_t(unsigned long, len, skcipher_sndbuf(sk));
324
325 err = skcipher_alloc_sgl(sk);
326 if (err)
327 goto unlock;
328
329 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
330 sg = sgl->sg;
331 do {
332 i = sgl->cur;
333 plen = min_t(int, len, PAGE_SIZE);
334
335 sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
336 err = -ENOMEM;
337 if (!sg_page(sg + i))
338 goto unlock;
339
340 err = memcpy_fromiovec(page_address(sg_page(sg + i)),
341 msg->msg_iov, plen);
342 if (err) {
343 __free_page(sg_page(sg + i));
344 sg_assign_page(sg + i, NULL);
345 goto unlock;
346 }
347
348 sg[i].length = plen;
349 len -= plen;
350 ctx->used += plen;
351 copied += plen;
352 size -= plen;
353 sgl->cur++;
354 } while (len && sgl->cur < MAX_SGL_ENTS);
355
356 ctx->merge = plen & (PAGE_SIZE - 1);
357 }
358
359 err = 0;
360
361 ctx->more = msg->msg_flags & MSG_MORE;
362 if (!ctx->more && !list_empty(&ctx->tsgl))
363 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
364
365unlock:
366 skcipher_data_wakeup(sk);
367 release_sock(sk);
368
369 return copied ?: err;
370}
371
372static ssize_t skcipher_sendpage(struct socket *sock, struct page *page,
373 int offset, size_t size, int flags)
374{
375 struct sock *sk = sock->sk;
376 struct alg_sock *ask = alg_sk(sk);
377 struct skcipher_ctx *ctx = ask->private;
378 struct skcipher_sg_list *sgl;
379 int err = -EINVAL;
380
381 lock_sock(sk);
382 if (!ctx->more && ctx->used)
383 goto unlock;
384
385 if (!size)
386 goto done;
387
388 if (!skcipher_writable(sk)) {
389 err = skcipher_wait_for_wmem(sk, flags);
390 if (err)
391 goto unlock;
392 }
393
394 err = skcipher_alloc_sgl(sk);
395 if (err)
396 goto unlock;
397
398 ctx->merge = 0;
399 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
400
401 get_page(page);
402 sg_set_page(sgl->sg + sgl->cur, page, size, offset);
403 sgl->cur++;
404 ctx->used += size;
405
406done:
407 ctx->more = flags & MSG_MORE;
408 if (!ctx->more && !list_empty(&ctx->tsgl))
409 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
410
411unlock:
412 skcipher_data_wakeup(sk);
413 release_sock(sk);
414
415 return err ?: size;
416}
417
418static int skcipher_recvmsg(struct kiocb *unused, struct socket *sock,
419 struct msghdr *msg, size_t ignored, int flags)
420{
421 struct sock *sk = sock->sk;
422 struct alg_sock *ask = alg_sk(sk);
423 struct skcipher_ctx *ctx = ask->private;
424 unsigned bs = crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(
425 &ctx->req));
426 struct skcipher_sg_list *sgl;
427 struct scatterlist *sg;
428 unsigned long iovlen;
429 struct iovec *iov;
430 int err = -EAGAIN;
431 int used;
432 long copied = 0;
433
434 lock_sock(sk);
435 for (iov = msg->msg_iov, iovlen = msg->msg_iovlen; iovlen > 0;
436 iovlen--, iov++) {
437 unsigned long seglen = iov->iov_len;
438 char __user *from = iov->iov_base;
439
440 while (seglen) {
441 sgl = list_first_entry(&ctx->tsgl,
442 struct skcipher_sg_list, list);
443 sg = sgl->sg;
444
445 while (!sg->length)
446 sg++;
447
448 used = ctx->used;
449 if (!used) {
450 err = skcipher_wait_for_data(sk, flags);
451 if (err)
452 goto unlock;
453 }
454
455 used = min_t(unsigned long, used, seglen);
456
457 used = af_alg_make_sg(&ctx->rsgl, from, used, 1);
458 err = used;
459 if (err < 0)
460 goto unlock;
461
462 if (ctx->more || used < ctx->used)
463 used -= used % bs;
464
465 err = -EINVAL;
466 if (!used)
467 goto free;
468
469 ablkcipher_request_set_crypt(&ctx->req, sg,
470 ctx->rsgl.sg, used,
471 ctx->iv);
472
473 err = af_alg_wait_for_completion(
474 ctx->enc ?
475 crypto_ablkcipher_encrypt(&ctx->req) :
476 crypto_ablkcipher_decrypt(&ctx->req),
477 &ctx->completion);
478
479free:
480 af_alg_free_sg(&ctx->rsgl);
481
482 if (err)
483 goto unlock;
484
485 copied += used;
486 from += used;
487 seglen -= used;
488 skcipher_pull_sgl(sk, used);
489 }
490 }
491
492 err = 0;
493
494unlock:
495 skcipher_wmem_wakeup(sk);
496 release_sock(sk);
497
498 return copied ?: err;
499}
500
501
502static unsigned int skcipher_poll(struct file *file, struct socket *sock,
503 poll_table *wait)
504{
505 struct sock *sk = sock->sk;
506 struct alg_sock *ask = alg_sk(sk);
507 struct skcipher_ctx *ctx = ask->private;
508 unsigned int mask;
509
510 sock_poll_wait(file, sk_sleep(sk), wait);
511 mask = 0;
512
513 if (ctx->used)
514 mask |= POLLIN | POLLRDNORM;
515
516 if (skcipher_writable(sk))
517 mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
518
519 return mask;
520}
521
522static struct proto_ops algif_skcipher_ops = {
523 .family = PF_ALG,
524
525 .connect = sock_no_connect,
526 .socketpair = sock_no_socketpair,
527 .getname = sock_no_getname,
528 .ioctl = sock_no_ioctl,
529 .listen = sock_no_listen,
530 .shutdown = sock_no_shutdown,
531 .getsockopt = sock_no_getsockopt,
532 .mmap = sock_no_mmap,
533 .bind = sock_no_bind,
534 .accept = sock_no_accept,
535 .setsockopt = sock_no_setsockopt,
536
537 .release = af_alg_release,
538 .sendmsg = skcipher_sendmsg,
539 .sendpage = skcipher_sendpage,
540 .recvmsg = skcipher_recvmsg,
541 .poll = skcipher_poll,
542};
543
544static void *skcipher_bind(const char *name, u32 type, u32 mask)
545{
546 return crypto_alloc_ablkcipher(name, type, mask);
547}
548
549static void skcipher_release(void *private)
550{
551 crypto_free_ablkcipher(private);
552}
553
554static int skcipher_setkey(void *private, const u8 *key, unsigned int keylen)
555{
556 return crypto_ablkcipher_setkey(private, key, keylen);
557}
558
559static void skcipher_sock_destruct(struct sock *sk)
560{
561 struct alg_sock *ask = alg_sk(sk);
562 struct skcipher_ctx *ctx = ask->private;
563 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(&ctx->req);
564
565 skcipher_free_sgl(sk);
566 sock_kfree_s(sk, ctx->iv, crypto_ablkcipher_ivsize(tfm));
567 sock_kfree_s(sk, ctx, ctx->len);
568 af_alg_release_parent(sk);
569}
570
571static int skcipher_accept_parent(void *private, struct sock *sk)
572{
573 struct skcipher_ctx *ctx;
574 struct alg_sock *ask = alg_sk(sk);
575 unsigned int len = sizeof(*ctx) + crypto_ablkcipher_reqsize(private);
576
577 ctx = sock_kmalloc(sk, len, GFP_KERNEL);
578 if (!ctx)
579 return -ENOMEM;
580
581 ctx->iv = sock_kmalloc(sk, crypto_ablkcipher_ivsize(private),
582 GFP_KERNEL);
583 if (!ctx->iv) {
584 sock_kfree_s(sk, ctx, len);
585 return -ENOMEM;
586 }
587
588 memset(ctx->iv, 0, crypto_ablkcipher_ivsize(private));
589
590 INIT_LIST_HEAD(&ctx->tsgl);
591 ctx->len = len;
592 ctx->used = 0;
593 ctx->more = 0;
594 ctx->merge = 0;
595 ctx->enc = 0;
596 af_alg_init_completion(&ctx->completion);
597
598 ask->private = ctx;
599
600 ablkcipher_request_set_tfm(&ctx->req, private);
601 ablkcipher_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_BACKLOG,
602 af_alg_complete, &ctx->completion);
603
604 sk->sk_destruct = skcipher_sock_destruct;
605
606 return 0;
607}
608
609static const struct af_alg_type algif_type_skcipher = {
610 .bind = skcipher_bind,
611 .release = skcipher_release,
612 .setkey = skcipher_setkey,
613 .accept = skcipher_accept_parent,
614 .ops = &algif_skcipher_ops,
615 .name = "skcipher",
616 .owner = THIS_MODULE
617};
618
619static int __init algif_skcipher_init(void)
620{
621 return af_alg_register_type(&algif_type_skcipher);
622}
623
624static void __exit algif_skcipher_exit(void)
625{
626 int err = af_alg_unregister_type(&algif_type_skcipher);
627 BUG_ON(err);
628}
629
630module_init(algif_skcipher_init);
631module_exit(algif_skcipher_exit);
632MODULE_LICENSE("GPL");
diff --git a/crypto/authenc.c b/crypto/authenc.c
index a5a22cfcd07b..5ef7ba6b6a76 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -107,20 +107,6 @@ badkey:
107 goto out; 107 goto out;
108} 108}
109 109
110static void authenc_chain(struct scatterlist *head, struct scatterlist *sg,
111 int chain)
112{
113 if (chain) {
114 head->length += sg->length;
115 sg = scatterwalk_sg_next(sg);
116 }
117
118 if (sg)
119 scatterwalk_sg_chain(head, 2, sg);
120 else
121 sg_mark_end(head);
122}
123
124static void authenc_geniv_ahash_update_done(struct crypto_async_request *areq, 110static void authenc_geniv_ahash_update_done(struct crypto_async_request *areq,
125 int err) 111 int err)
126{ 112{
@@ -345,7 +331,7 @@ static int crypto_authenc_genicv(struct aead_request *req, u8 *iv,
345 if (ivsize) { 331 if (ivsize) {
346 sg_init_table(cipher, 2); 332 sg_init_table(cipher, 2);
347 sg_set_buf(cipher, iv, ivsize); 333 sg_set_buf(cipher, iv, ivsize);
348 authenc_chain(cipher, dst, vdst == iv + ivsize); 334 scatterwalk_crypto_chain(cipher, dst, vdst == iv + ivsize, 2);
349 dst = cipher; 335 dst = cipher;
350 cryptlen += ivsize; 336 cryptlen += ivsize;
351 } 337 }
@@ -354,7 +340,7 @@ static int crypto_authenc_genicv(struct aead_request *req, u8 *iv,
354 authenc_ahash_fn = crypto_authenc_ahash; 340 authenc_ahash_fn = crypto_authenc_ahash;
355 sg_init_table(asg, 2); 341 sg_init_table(asg, 2);
356 sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset); 342 sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset);
357 authenc_chain(asg, dst, 0); 343 scatterwalk_crypto_chain(asg, dst, 0, 2);
358 dst = asg; 344 dst = asg;
359 cryptlen += req->assoclen; 345 cryptlen += req->assoclen;
360 } 346 }
@@ -499,7 +485,7 @@ static int crypto_authenc_iverify(struct aead_request *req, u8 *iv,
499 if (ivsize) { 485 if (ivsize) {
500 sg_init_table(cipher, 2); 486 sg_init_table(cipher, 2);
501 sg_set_buf(cipher, iv, ivsize); 487 sg_set_buf(cipher, iv, ivsize);
502 authenc_chain(cipher, src, vsrc == iv + ivsize); 488 scatterwalk_crypto_chain(cipher, src, vsrc == iv + ivsize, 2);
503 src = cipher; 489 src = cipher;
504 cryptlen += ivsize; 490 cryptlen += ivsize;
505 } 491 }
@@ -508,7 +494,7 @@ static int crypto_authenc_iverify(struct aead_request *req, u8 *iv,
508 authenc_ahash_fn = crypto_authenc_ahash; 494 authenc_ahash_fn = crypto_authenc_ahash;
509 sg_init_table(asg, 2); 495 sg_init_table(asg, 2);
510 sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset); 496 sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset);
511 authenc_chain(asg, src, 0); 497 scatterwalk_crypto_chain(asg, src, 0, 2);
512 src = asg; 498 src = asg;
513 cryptlen += req->assoclen; 499 cryptlen += req->assoclen;
514 } 500 }
diff --git a/crypto/cast5.c b/crypto/cast5.c
index a1d2294b50ad..4a230ddec877 100644
--- a/crypto/cast5.c
+++ b/crypto/cast5.c
@@ -604,36 +604,23 @@ static void cast5_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
604 * Rounds 3, 6, 9, 12, and 15 use f function Type 3. 604 * Rounds 3, 6, 9, 12, and 15 use f function Type 3.
605 */ 605 */
606 606
607 t = l; l = r; r = t ^ F1(r, Km[0], Kr[0]);
608 t = l; l = r; r = t ^ F2(r, Km[1], Kr[1]);
609 t = l; l = r; r = t ^ F3(r, Km[2], Kr[2]);
610 t = l; l = r; r = t ^ F1(r, Km[3], Kr[3]);
611 t = l; l = r; r = t ^ F2(r, Km[4], Kr[4]);
612 t = l; l = r; r = t ^ F3(r, Km[5], Kr[5]);
613 t = l; l = r; r = t ^ F1(r, Km[6], Kr[6]);
614 t = l; l = r; r = t ^ F2(r, Km[7], Kr[7]);
615 t = l; l = r; r = t ^ F3(r, Km[8], Kr[8]);
616 t = l; l = r; r = t ^ F1(r, Km[9], Kr[9]);
617 t = l; l = r; r = t ^ F2(r, Km[10], Kr[10]);
618 t = l; l = r; r = t ^ F3(r, Km[11], Kr[11]);
607 if (!(c->rr)) { 619 if (!(c->rr)) {
608 t = l; l = r; r = t ^ F1(r, Km[0], Kr[0]);
609 t = l; l = r; r = t ^ F2(r, Km[1], Kr[1]);
610 t = l; l = r; r = t ^ F3(r, Km[2], Kr[2]);
611 t = l; l = r; r = t ^ F1(r, Km[3], Kr[3]);
612 t = l; l = r; r = t ^ F2(r, Km[4], Kr[4]);
613 t = l; l = r; r = t ^ F3(r, Km[5], Kr[5]);
614 t = l; l = r; r = t ^ F1(r, Km[6], Kr[6]);
615 t = l; l = r; r = t ^ F2(r, Km[7], Kr[7]);
616 t = l; l = r; r = t ^ F3(r, Km[8], Kr[8]);
617 t = l; l = r; r = t ^ F1(r, Km[9], Kr[9]);
618 t = l; l = r; r = t ^ F2(r, Km[10], Kr[10]);
619 t = l; l = r; r = t ^ F3(r, Km[11], Kr[11]);
620 t = l; l = r; r = t ^ F1(r, Km[12], Kr[12]); 620 t = l; l = r; r = t ^ F1(r, Km[12], Kr[12]);
621 t = l; l = r; r = t ^ F2(r, Km[13], Kr[13]); 621 t = l; l = r; r = t ^ F2(r, Km[13], Kr[13]);
622 t = l; l = r; r = t ^ F3(r, Km[14], Kr[14]); 622 t = l; l = r; r = t ^ F3(r, Km[14], Kr[14]);
623 t = l; l = r; r = t ^ F1(r, Km[15], Kr[15]); 623 t = l; l = r; r = t ^ F1(r, Km[15], Kr[15]);
624 } else {
625 t = l; l = r; r = t ^ F1(r, Km[0], Kr[0]);
626 t = l; l = r; r = t ^ F2(r, Km[1], Kr[1]);
627 t = l; l = r; r = t ^ F3(r, Km[2], Kr[2]);
628 t = l; l = r; r = t ^ F1(r, Km[3], Kr[3]);
629 t = l; l = r; r = t ^ F2(r, Km[4], Kr[4]);
630 t = l; l = r; r = t ^ F3(r, Km[5], Kr[5]);
631 t = l; l = r; r = t ^ F1(r, Km[6], Kr[6]);
632 t = l; l = r; r = t ^ F2(r, Km[7], Kr[7]);
633 t = l; l = r; r = t ^ F3(r, Km[8], Kr[8]);
634 t = l; l = r; r = t ^ F1(r, Km[9], Kr[9]);
635 t = l; l = r; r = t ^ F2(r, Km[10], Kr[10]);
636 t = l; l = r; r = t ^ F3(r, Km[11], Kr[11]);
637 } 624 }
638 625
639 /* c1...c64 <-- (R16,L16). (Exchange final blocks L16, R16 and 626 /* c1...c64 <-- (R16,L16). (Exchange final blocks L16, R16 and
@@ -663,32 +650,19 @@ static void cast5_decrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf)
663 t = l; l = r; r = t ^ F3(r, Km[14], Kr[14]); 650 t = l; l = r; r = t ^ F3(r, Km[14], Kr[14]);
664 t = l; l = r; r = t ^ F2(r, Km[13], Kr[13]); 651 t = l; l = r; r = t ^ F2(r, Km[13], Kr[13]);
665 t = l; l = r; r = t ^ F1(r, Km[12], Kr[12]); 652 t = l; l = r; r = t ^ F1(r, Km[12], Kr[12]);
666 t = l; l = r; r = t ^ F3(r, Km[11], Kr[11]);
667 t = l; l = r; r = t ^ F2(r, Km[10], Kr[10]);
668 t = l; l = r; r = t ^ F1(r, Km[9], Kr[9]);
669 t = l; l = r; r = t ^ F3(r, Km[8], Kr[8]);
670 t = l; l = r; r = t ^ F2(r, Km[7], Kr[7]);
671 t = l; l = r; r = t ^ F1(r, Km[6], Kr[6]);
672 t = l; l = r; r = t ^ F3(r, Km[5], Kr[5]);
673 t = l; l = r; r = t ^ F2(r, Km[4], Kr[4]);
674 t = l; l = r; r = t ^ F1(r, Km[3], Kr[3]);
675 t = l; l = r; r = t ^ F3(r, Km[2], Kr[2]);
676 t = l; l = r; r = t ^ F2(r, Km[1], Kr[1]);
677 t = l; l = r; r = t ^ F1(r, Km[0], Kr[0]);
678 } else {
679 t = l; l = r; r = t ^ F3(r, Km[11], Kr[11]);
680 t = l; l = r; r = t ^ F2(r, Km[10], Kr[10]);
681 t = l; l = r; r = t ^ F1(r, Km[9], Kr[9]);
682 t = l; l = r; r = t ^ F3(r, Km[8], Kr[8]);
683 t = l; l = r; r = t ^ F2(r, Km[7], Kr[7]);
684 t = l; l = r; r = t ^ F1(r, Km[6], Kr[6]);
685 t = l; l = r; r = t ^ F3(r, Km[5], Kr[5]);
686 t = l; l = r; r = t ^ F2(r, Km[4], Kr[4]);
687 t = l; l = r; r = t ^ F1(r, Km[3], Kr[3]);
688 t = l; l = r; r = t ^ F3(r, Km[2], Kr[2]);
689 t = l; l = r; r = t ^ F2(r, Km[1], Kr[1]);
690 t = l; l = r; r = t ^ F1(r, Km[0], Kr[0]);
691 } 653 }
654 t = l; l = r; r = t ^ F3(r, Km[11], Kr[11]);
655 t = l; l = r; r = t ^ F2(r, Km[10], Kr[10]);
656 t = l; l = r; r = t ^ F1(r, Km[9], Kr[9]);
657 t = l; l = r; r = t ^ F3(r, Km[8], Kr[8]);
658 t = l; l = r; r = t ^ F2(r, Km[7], Kr[7]);
659 t = l; l = r; r = t ^ F1(r, Km[6], Kr[6]);
660 t = l; l = r; r = t ^ F3(r, Km[5], Kr[5]);
661 t = l; l = r; r = t ^ F2(r, Km[4], Kr[4]);
662 t = l; l = r; r = t ^ F1(r, Km[3], Kr[3]);
663 t = l; l = r; r = t ^ F3(r, Km[2], Kr[2]);
664 t = l; l = r; r = t ^ F2(r, Km[1], Kr[1]);
665 t = l; l = r; r = t ^ F1(r, Km[0], Kr[0]);
692 666
693 dst[0] = cpu_to_be32(r); 667 dst[0] = cpu_to_be32(r);
694 dst[1] = cpu_to_be32(l); 668 dst[1] = cpu_to_be32(l);
diff --git a/crypto/crypto_wq.c b/crypto/crypto_wq.c
index fdcf6248f152..b980ee1af459 100644
--- a/crypto/crypto_wq.c
+++ b/crypto/crypto_wq.c
@@ -20,7 +20,8 @@ EXPORT_SYMBOL_GPL(kcrypto_wq);
20 20
21static int __init crypto_wq_init(void) 21static int __init crypto_wq_init(void)
22{ 22{
23 kcrypto_wq = create_workqueue("crypto"); 23 kcrypto_wq = alloc_workqueue("crypto",
24 WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1);
24 if (unlikely(!kcrypto_wq)) 25 if (unlikely(!kcrypto_wq))
25 return -ENOMEM; 26 return -ENOMEM;
26 return 0; 27 return 0;
diff --git a/crypto/deflate.c b/crypto/deflate.c
index 463dc859aa05..cbc7a33a9600 100644
--- a/crypto/deflate.c
+++ b/crypto/deflate.c
@@ -48,12 +48,11 @@ static int deflate_comp_init(struct deflate_ctx *ctx)
48 int ret = 0; 48 int ret = 0;
49 struct z_stream_s *stream = &ctx->comp_stream; 49 struct z_stream_s *stream = &ctx->comp_stream;
50 50
51 stream->workspace = vmalloc(zlib_deflate_workspacesize()); 51 stream->workspace = vzalloc(zlib_deflate_workspacesize());
52 if (!stream->workspace) { 52 if (!stream->workspace) {
53 ret = -ENOMEM; 53 ret = -ENOMEM;
54 goto out; 54 goto out;
55 } 55 }
56 memset(stream->workspace, 0, zlib_deflate_workspacesize());
57 ret = zlib_deflateInit2(stream, DEFLATE_DEF_LEVEL, Z_DEFLATED, 56 ret = zlib_deflateInit2(stream, DEFLATE_DEF_LEVEL, Z_DEFLATED,
58 -DEFLATE_DEF_WINBITS, DEFLATE_DEF_MEMLEVEL, 57 -DEFLATE_DEF_WINBITS, DEFLATE_DEF_MEMLEVEL,
59 Z_DEFAULT_STRATEGY); 58 Z_DEFAULT_STRATEGY);
diff --git a/crypto/eseqiv.c b/crypto/eseqiv.c
index 3ca3b669d5d5..42ce9f570aec 100644
--- a/crypto/eseqiv.c
+++ b/crypto/eseqiv.c
@@ -62,20 +62,6 @@ out:
62 skcipher_givcrypt_complete(req, err); 62 skcipher_givcrypt_complete(req, err);
63} 63}
64 64
65static void eseqiv_chain(struct scatterlist *head, struct scatterlist *sg,
66 int chain)
67{
68 if (chain) {
69 head->length += sg->length;
70 sg = scatterwalk_sg_next(sg);
71 }
72
73 if (sg)
74 scatterwalk_sg_chain(head, 2, sg);
75 else
76 sg_mark_end(head);
77}
78
79static int eseqiv_givencrypt(struct skcipher_givcrypt_request *req) 65static int eseqiv_givencrypt(struct skcipher_givcrypt_request *req)
80{ 66{
81 struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req); 67 struct crypto_ablkcipher *geniv = skcipher_givcrypt_reqtfm(req);
@@ -124,13 +110,13 @@ static int eseqiv_givencrypt(struct skcipher_givcrypt_request *req)
124 110
125 sg_init_table(reqctx->src, 2); 111 sg_init_table(reqctx->src, 2);
126 sg_set_buf(reqctx->src, giv, ivsize); 112 sg_set_buf(reqctx->src, giv, ivsize);
127 eseqiv_chain(reqctx->src, osrc, vsrc == giv + ivsize); 113 scatterwalk_crypto_chain(reqctx->src, osrc, vsrc == giv + ivsize, 2);
128 114
129 dst = reqctx->src; 115 dst = reqctx->src;
130 if (osrc != odst) { 116 if (osrc != odst) {
131 sg_init_table(reqctx->dst, 2); 117 sg_init_table(reqctx->dst, 2);
132 sg_set_buf(reqctx->dst, giv, ivsize); 118 sg_set_buf(reqctx->dst, giv, ivsize);
133 eseqiv_chain(reqctx->dst, odst, vdst == giv + ivsize); 119 scatterwalk_crypto_chain(reqctx->dst, odst, vdst == giv + ivsize, 2);
134 120
135 dst = reqctx->dst; 121 dst = reqctx->dst;
136 } 122 }
diff --git a/crypto/gcm.c b/crypto/gcm.c
index 2f5fbba6576c..1a252639ef91 100644
--- a/crypto/gcm.c
+++ b/crypto/gcm.c
@@ -1102,21 +1102,6 @@ static int crypto_rfc4543_setauthsize(struct crypto_aead *parent,
1102 return crypto_aead_setauthsize(ctx->child, authsize); 1102 return crypto_aead_setauthsize(ctx->child, authsize);
1103} 1103}
1104 1104
1105/* this is the same as crypto_authenc_chain */
1106static void crypto_rfc4543_chain(struct scatterlist *head,
1107 struct scatterlist *sg, int chain)
1108{
1109 if (chain) {
1110 head->length += sg->length;
1111 sg = scatterwalk_sg_next(sg);
1112 }
1113
1114 if (sg)
1115 scatterwalk_sg_chain(head, 2, sg);
1116 else
1117 sg_mark_end(head);
1118}
1119
1120static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req, 1105static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req,
1121 int enc) 1106 int enc)
1122{ 1107{
@@ -1154,13 +1139,13 @@ static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req,
1154 1139
1155 sg_init_table(payload, 2); 1140 sg_init_table(payload, 2);
1156 sg_set_buf(payload, req->iv, 8); 1141 sg_set_buf(payload, req->iv, 8);
1157 crypto_rfc4543_chain(payload, dst, vdst == req->iv + 8); 1142 scatterwalk_crypto_chain(payload, dst, vdst == req->iv + 8, 2);
1158 assoclen += 8 + req->cryptlen - (enc ? 0 : authsize); 1143 assoclen += 8 + req->cryptlen - (enc ? 0 : authsize);
1159 1144
1160 sg_init_table(assoc, 2); 1145 sg_init_table(assoc, 2);
1161 sg_set_page(assoc, sg_page(req->assoc), req->assoc->length, 1146 sg_set_page(assoc, sg_page(req->assoc), req->assoc->length,
1162 req->assoc->offset); 1147 req->assoc->offset);
1163 crypto_rfc4543_chain(assoc, payload, 0); 1148 scatterwalk_crypto_chain(assoc, payload, 0, 2);
1164 1149
1165 aead_request_set_tfm(subreq, ctx->child); 1150 aead_request_set_tfm(subreq, ctx->child);
1166 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 1151 aead_request_set_callback(subreq, req->base.flags, req->base.complete,
diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c
index 75586f1f86e7..29a89dad68b6 100644
--- a/crypto/pcrypt.c
+++ b/crypto/pcrypt.c
@@ -455,7 +455,8 @@ static int pcrypt_init_padata(struct padata_pcrypt *pcrypt,
455 455
456 get_online_cpus(); 456 get_online_cpus();
457 457
458 pcrypt->wq = create_workqueue(name); 458 pcrypt->wq = alloc_workqueue(name,
459 WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1);
459 if (!pcrypt->wq) 460 if (!pcrypt->wq)
460 goto err; 461 goto err;
461 462
diff --git a/crypto/rmd128.c b/crypto/rmd128.c
index 1ceb6735aa53..8a0f68b7f257 100644
--- a/crypto/rmd128.c
+++ b/crypto/rmd128.c
@@ -5,7 +5,7 @@
5 * 5 *
6 * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC 6 * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC
7 * 7 *
8 * Copyright (c) 2008 Adrian-Ken Rueegsegger <rueegsegger (at) swiss-it.ch> 8 * Copyright (c) 2008 Adrian-Ken Rueegsegger <ken@codelabs.ch>
9 * 9 *
10 * This program is free software; you can redistribute it and/or modify it 10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU General Public License as published by the Free 11 * under the terms of the GNU General Public License as published by the Free
@@ -325,4 +325,5 @@ module_init(rmd128_mod_init);
325module_exit(rmd128_mod_fini); 325module_exit(rmd128_mod_fini);
326 326
327MODULE_LICENSE("GPL"); 327MODULE_LICENSE("GPL");
328MODULE_AUTHOR("Adrian-Ken Rueegsegger <ken@codelabs.ch>");
328MODULE_DESCRIPTION("RIPEMD-128 Message Digest"); 329MODULE_DESCRIPTION("RIPEMD-128 Message Digest");
diff --git a/crypto/rmd160.c b/crypto/rmd160.c
index 472261fc913f..525d7bb752cf 100644
--- a/crypto/rmd160.c
+++ b/crypto/rmd160.c
@@ -5,7 +5,7 @@
5 * 5 *
6 * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC 6 * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC
7 * 7 *
8 * Copyright (c) 2008 Adrian-Ken Rueegsegger <rueegsegger (at) swiss-it.ch> 8 * Copyright (c) 2008 Adrian-Ken Rueegsegger <ken@codelabs.ch>
9 * 9 *
10 * This program is free software; you can redistribute it and/or modify it 10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU General Public License as published by the Free 11 * under the terms of the GNU General Public License as published by the Free
@@ -369,4 +369,5 @@ module_init(rmd160_mod_init);
369module_exit(rmd160_mod_fini); 369module_exit(rmd160_mod_fini);
370 370
371MODULE_LICENSE("GPL"); 371MODULE_LICENSE("GPL");
372MODULE_AUTHOR("Adrian-Ken Rueegsegger <ken@codelabs.ch>");
372MODULE_DESCRIPTION("RIPEMD-160 Message Digest"); 373MODULE_DESCRIPTION("RIPEMD-160 Message Digest");
diff --git a/crypto/rmd256.c b/crypto/rmd256.c
index 72eafa8d2e7b..69293d9b56e0 100644
--- a/crypto/rmd256.c
+++ b/crypto/rmd256.c
@@ -5,7 +5,7 @@
5 * 5 *
6 * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC 6 * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC
7 * 7 *
8 * Copyright (c) 2008 Adrian-Ken Rueegsegger <rueegsegger (at) swiss-it.ch> 8 * Copyright (c) 2008 Adrian-Ken Rueegsegger <ken@codelabs.ch>
9 * 9 *
10 * This program is free software; you can redistribute it and/or modify it 10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU General Public License as published by the Free 11 * under the terms of the GNU General Public License as published by the Free
@@ -344,4 +344,5 @@ module_init(rmd256_mod_init);
344module_exit(rmd256_mod_fini); 344module_exit(rmd256_mod_fini);
345 345
346MODULE_LICENSE("GPL"); 346MODULE_LICENSE("GPL");
347MODULE_AUTHOR("Adrian-Ken Rueegsegger <ken@codelabs.ch>");
347MODULE_DESCRIPTION("RIPEMD-256 Message Digest"); 348MODULE_DESCRIPTION("RIPEMD-256 Message Digest");
diff --git a/crypto/rmd320.c b/crypto/rmd320.c
index 86becaba2f05..09f97dfdfbba 100644
--- a/crypto/rmd320.c
+++ b/crypto/rmd320.c
@@ -5,7 +5,7 @@
5 * 5 *
6 * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC 6 * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC
7 * 7 *
8 * Copyright (c) 2008 Adrian-Ken Rueegsegger <rueegsegger (at) swiss-it.ch> 8 * Copyright (c) 2008 Adrian-Ken Rueegsegger <ken@codelabs.ch>
9 * 9 *
10 * This program is free software; you can redistribute it and/or modify it 10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of the GNU General Public License as published by the Free 11 * under the terms of the GNU General Public License as published by the Free
@@ -393,4 +393,5 @@ module_init(rmd320_mod_init);
393module_exit(rmd320_mod_fini); 393module_exit(rmd320_mod_fini);
394 394
395MODULE_LICENSE("GPL"); 395MODULE_LICENSE("GPL");
396MODULE_AUTHOR("Adrian-Ken Rueegsegger <ken@codelabs.ch>");
396MODULE_DESCRIPTION("RIPEMD-320 Message Digest"); 397MODULE_DESCRIPTION("RIPEMD-320 Message Digest");
diff --git a/crypto/shash.c b/crypto/shash.c
index 22fd9433141f..76f74b963151 100644
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -310,7 +310,13 @@ static int shash_async_export(struct ahash_request *req, void *out)
310 310
311static int shash_async_import(struct ahash_request *req, const void *in) 311static int shash_async_import(struct ahash_request *req, const void *in)
312{ 312{
313 return crypto_shash_import(ahash_request_ctx(req), in); 313 struct crypto_shash **ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
314 struct shash_desc *desc = ahash_request_ctx(req);
315
316 desc->tfm = *ctx;
317 desc->flags = req->base.flags;
318
319 return crypto_shash_import(desc, in);
314} 320}
315 321
316static void crypto_exit_shash_ops_async(struct crypto_tfm *tfm) 322static void crypto_exit_shash_ops_async(struct crypto_tfm *tfm)
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index 3ca68f9fc14d..9aac5e58be94 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -8,6 +8,13 @@
8 * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org> 8 * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
9 * Copyright (c) 2007 Nokia Siemens Networks 9 * Copyright (c) 2007 Nokia Siemens Networks
10 * 10 *
11 * Updated RFC4106 AES-GCM testing.
12 * Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
13 * Adrian Hoban <adrian.hoban@intel.com>
14 * Gabriele Paoloni <gabriele.paoloni@intel.com>
15 * Tadeusz Struk (tadeusz.struk@intel.com)
16 * Copyright (c) 2010, Intel Corporation.
17 *
11 * This program is free software; you can redistribute it and/or modify it 18 * This program is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU General Public License as published by the Free 19 * under the terms of the GNU General Public License as published by the Free
13 * Software Foundation; either version 2 of the License, or (at your option) 20 * Software Foundation; either version 2 of the License, or (at your option)
@@ -980,6 +987,10 @@ static int do_test(int m)
980 ret += tcrypt_test("ansi_cprng"); 987 ret += tcrypt_test("ansi_cprng");
981 break; 988 break;
982 989
990 case 151:
991 ret += tcrypt_test("rfc4106(gcm(aes))");
992 break;
993
983 case 200: 994 case 200:
984 test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0, 995 test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
985 speed_template_16_24_32); 996 speed_template_16_24_32);
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index fa8c8f78c8d4..27ea9fe9476f 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -6,6 +6,13 @@
6 * Copyright (c) 2007 Nokia Siemens Networks 6 * Copyright (c) 2007 Nokia Siemens Networks
7 * Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au> 7 * Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
8 * 8 *
9 * Updated RFC4106 AES-GCM testing.
10 * Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
11 * Adrian Hoban <adrian.hoban@intel.com>
12 * Gabriele Paoloni <gabriele.paoloni@intel.com>
13 * Tadeusz Struk (tadeusz.struk@intel.com)
14 * Copyright (c) 2010, Intel Corporation.
15 *
9 * This program is free software; you can redistribute it and/or modify it 16 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU General Public License as published by the Free 17 * under the terms of the GNU General Public License as published by the Free
11 * Software Foundation; either version 2 of the License, or (at your option) 18 * Software Foundation; either version 2 of the License, or (at your option)
@@ -2242,6 +2249,23 @@ static const struct alg_test_desc alg_test_descs[] = {
2242 } 2249 }
2243 } 2250 }
2244 }, { 2251 }, {
2252 .alg = "rfc4106(gcm(aes))",
2253 .test = alg_test_aead,
2254 .suite = {
2255 .aead = {
2256 .enc = {
2257 .vecs = aes_gcm_rfc4106_enc_tv_template,
2258 .count = AES_GCM_4106_ENC_TEST_VECTORS
2259 },
2260 .dec = {
2261 .vecs = aes_gcm_rfc4106_dec_tv_template,
2262 .count = AES_GCM_4106_DEC_TEST_VECTORS
2263 }
2264 }
2265 }
2266 }, {
2267
2268
2245 .alg = "rfc4309(ccm(aes))", 2269 .alg = "rfc4309(ccm(aes))",
2246 .test = alg_test_aead, 2270 .test = alg_test_aead,
2247 .fips_allowed = 1, 2271 .fips_allowed = 1,
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 74e35377fd30..834af7f2adee 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -6,6 +6,15 @@
6 * Copyright (c) 2007 Nokia Siemens Networks 6 * Copyright (c) 2007 Nokia Siemens Networks
7 * Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au> 7 * Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>
8 * 8 *
9 * Updated RFC4106 AES-GCM testing. Some test vectors were taken from
10 * http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/
11 * gcm/gcm-test-vectors.tar.gz
12 * Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
13 * Adrian Hoban <adrian.hoban@intel.com>
14 * Gabriele Paoloni <gabriele.paoloni@intel.com>
15 * Tadeusz Struk (tadeusz.struk@intel.com)
16 * Copyright (c) 2010, Intel Corporation.
17 *
9 * This program is free software; you can redistribute it and/or modify it 18 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU General Public License as published by the Free 19 * under the terms of the GNU General Public License as published by the Free
11 * Software Foundation; either version 2 of the License, or (at your option) 20 * Software Foundation; either version 2 of the License, or (at your option)
@@ -2947,6 +2956,8 @@ static struct cipher_testvec cast6_dec_tv_template[] = {
2947#define AES_CTR_3686_DEC_TEST_VECTORS 6 2956#define AES_CTR_3686_DEC_TEST_VECTORS 6
2948#define AES_GCM_ENC_TEST_VECTORS 9 2957#define AES_GCM_ENC_TEST_VECTORS 9
2949#define AES_GCM_DEC_TEST_VECTORS 8 2958#define AES_GCM_DEC_TEST_VECTORS 8
2959#define AES_GCM_4106_ENC_TEST_VECTORS 7
2960#define AES_GCM_4106_DEC_TEST_VECTORS 7
2950#define AES_CCM_ENC_TEST_VECTORS 7 2961#define AES_CCM_ENC_TEST_VECTORS 7
2951#define AES_CCM_DEC_TEST_VECTORS 7 2962#define AES_CCM_DEC_TEST_VECTORS 7
2952#define AES_CCM_4309_ENC_TEST_VECTORS 7 2963#define AES_CCM_4309_ENC_TEST_VECTORS 7
@@ -5829,6 +5840,356 @@ static struct aead_testvec aes_gcm_dec_tv_template[] = {
5829 } 5840 }
5830}; 5841};
5831 5842
5843static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = {
5844 { /* Generated using Crypto++ */
5845 .key = zeroed_string,
5846 .klen = 20,
5847 .iv = zeroed_string,
5848 .input = zeroed_string,
5849 .ilen = 16,
5850 .assoc = zeroed_string,
5851 .alen = 8,
5852 .result = "\x03\x88\xDA\xCE\x60\xB6\xA3\x92"
5853 "\xF3\x28\xC2\xB9\x71\xB2\xFE\x78"
5854 "\x97\xFE\x4C\x23\x37\x42\x01\xE0"
5855 "\x81\x9F\x8D\xC5\xD7\x41\xA0\x1B",
5856 .rlen = 32,
5857 },{
5858 .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
5859 "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
5860 "\x00\x00\x00\x00",
5861 .klen = 20,
5862 .iv = "\x00\x00\x00\x00\x00\x00\x00\x01"
5863 "\x00\x00\x00\x00",
5864 .input = zeroed_string,
5865 .ilen = 16,
5866 .assoc = zeroed_string,
5867 .alen = 8,
5868 .result = "\xC0\x0D\x8B\x42\x0F\x8F\x34\x18"
5869 "\x88\xB1\xC5\xBC\xC5\xB6\xD6\x28"
5870 "\x6A\x9D\xDF\x11\x5E\xFE\x5E\x9D"
5871 "\x2F\x70\x44\x92\xF7\xF2\xE3\xEF",
5872 .rlen = 32,
5873
5874 }, {
5875 .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
5876 "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
5877 "\x00\x00\x00\x00",
5878 .klen = 20,
5879 .iv = zeroed_string,
5880 .input = "\x01\x01\x01\x01\x01\x01\x01\x01"
5881 "\x01\x01\x01\x01\x01\x01\x01\x01",
5882 .ilen = 16,
5883 .assoc = zeroed_string,
5884 .alen = 8,
5885 .result = "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
5886 "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
5887 "\x0B\x8F\x88\x69\x17\xE6\xB4\x3C"
5888 "\xB1\x68\xFD\x14\x52\x64\x61\xB2",
5889 .rlen = 32,
5890 }, {
5891 .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
5892 "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
5893 "\x00\x00\x00\x00",
5894 .klen = 20,
5895 .iv = zeroed_string,
5896 .input = "\x01\x01\x01\x01\x01\x01\x01\x01"
5897 "\x01\x01\x01\x01\x01\x01\x01\x01",
5898 .ilen = 16,
5899 .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01",
5900 .alen = 8,
5901 .result = "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
5902 "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
5903 "\x90\x92\xB7\xE3\x5F\xA3\x9A\x63"
5904 "\x7E\xD7\x1F\xD8\xD3\x7C\x4B\xF5",
5905 .rlen = 32,
5906 }, {
5907 .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
5908 "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
5909 "\x00\x00\x00\x00",
5910 .klen = 20,
5911 .iv = "\x00\x00\x00\x00\x00\x00\x00\x01"
5912 "\x00\x00\x00\x00",
5913 .input = "\x01\x01\x01\x01\x01\x01\x01\x01"
5914 "\x01\x01\x01\x01\x01\x01\x01\x01",
5915 .ilen = 16,
5916 .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01",
5917 .alen = 8,
5918 .result = "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
5919 "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
5920 "\x64\x50\xF9\x32\x13\xFB\x74\x61"
5921 "\xF4\xED\x52\xD3\xC5\x10\x55\x3C",
5922 .rlen = 32,
5923 }, {
5924 .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
5925 "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
5926 "\x00\x00\x00\x00",
5927 .klen = 20,
5928 .iv = "\x00\x00\x00\x00\x00\x00\x00\x01"
5929 "\x00\x00\x00\x00",
5930 .input = "\x01\x01\x01\x01\x01\x01\x01\x01"
5931 "\x01\x01\x01\x01\x01\x01\x01\x01"
5932 "\x01\x01\x01\x01\x01\x01\x01\x01"
5933 "\x01\x01\x01\x01\x01\x01\x01\x01"
5934 "\x01\x01\x01\x01\x01\x01\x01\x01"
5935 "\x01\x01\x01\x01\x01\x01\x01\x01"
5936 "\x01\x01\x01\x01\x01\x01\x01\x01"
5937 "\x01\x01\x01\x01\x01\x01\x01\x01",
5938 .ilen = 64,
5939 .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01",
5940 .alen = 8,
5941 .result = "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
5942 "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
5943 "\x98\x14\xA1\x42\x37\x80\xFD\x90"
5944 "\x68\x12\x01\xA8\x91\x89\xB9\x83"
5945 "\x5B\x11\x77\x12\x9B\xFF\x24\x89"
5946 "\x94\x5F\x18\x12\xBA\x27\x09\x39"
5947 "\x99\x96\x76\x42\x15\x1C\xCD\xCB"
5948 "\xDC\xD3\xDA\x65\x73\xAF\x80\xCD"
5949 "\xD2\xB6\xC2\x4A\x76\xC2\x92\x85"
5950 "\xBD\xCF\x62\x98\x58\x14\xE5\xBD",
5951 .rlen = 80,
5952 }, {
5953 .key = "\x00\x01\x02\x03\x04\x05\x06\x07"
5954 "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
5955 "\x00\x00\x00\x00",
5956 .klen = 20,
5957 .iv = "\x00\x00\x45\x67\x89\xab\xcd\xef"
5958 "\x00\x00\x00\x00",
5959 .input = "\xff\xff\xff\xff\xff\xff\xff\xff"
5960 "\xff\xff\xff\xff\xff\xff\xff\xff"
5961 "\xff\xff\xff\xff\xff\xff\xff\xff"
5962 "\xff\xff\xff\xff\xff\xff\xff\xff"
5963 "\xff\xff\xff\xff\xff\xff\xff\xff"
5964 "\xff\xff\xff\xff\xff\xff\xff\xff"
5965 "\xff\xff\xff\xff\xff\xff\xff\xff"
5966 "\xff\xff\xff\xff\xff\xff\xff\xff"
5967 "\xff\xff\xff\xff\xff\xff\xff\xff"
5968 "\xff\xff\xff\xff\xff\xff\xff\xff"
5969 "\xff\xff\xff\xff\xff\xff\xff\xff"
5970 "\xff\xff\xff\xff\xff\xff\xff\xff"
5971 "\xff\xff\xff\xff\xff\xff\xff\xff"
5972 "\xff\xff\xff\xff\xff\xff\xff\xff"
5973 "\xff\xff\xff\xff\xff\xff\xff\xff"
5974 "\xff\xff\xff\xff\xff\xff\xff\xff"
5975 "\xff\xff\xff\xff\xff\xff\xff\xff"
5976 "\xff\xff\xff\xff\xff\xff\xff\xff"
5977 "\xff\xff\xff\xff\xff\xff\xff\xff"
5978 "\xff\xff\xff\xff\xff\xff\xff\xff"
5979 "\xff\xff\xff\xff\xff\xff\xff\xff"
5980 "\xff\xff\xff\xff\xff\xff\xff\xff"
5981 "\xff\xff\xff\xff\xff\xff\xff\xff"
5982 "\xff\xff\xff\xff\xff\xff\xff\xff",
5983 .ilen = 192,
5984 .assoc = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
5985 "\xaa\xaa\xaa\xaa",
5986 .alen = 12,
5987 .result = "\xC1\x76\x33\x85\xE2\x9B\x5F\xDE"
5988 "\xDE\x89\x3D\x42\xE7\xC9\x69\x8A"
5989 "\x44\x6D\xC3\x88\x46\x2E\xC2\x01"
5990 "\x5E\xF6\x0C\x39\xF0\xC4\xA5\x82"
5991 "\xCD\xE8\x31\xCC\x0A\x4C\xE4\x44"
5992 "\x41\xA9\x82\x6F\x22\xA1\x23\x1A"
5993 "\xA8\xE3\x16\xFD\x31\x5C\x27\x31"
5994 "\xF1\x7F\x01\x63\xA3\xAF\x70\xA1"
5995 "\xCF\x07\x57\x41\x67\xD0\xC4\x42"
5996 "\xDB\x18\xC6\x4C\x4C\xE0\x3D\x9F"
5997 "\x05\x07\xFB\x13\x7D\x4A\xCA\x5B"
5998 "\xF0\xBF\x64\x7E\x05\xB1\x72\xEE"
5999 "\x7C\x3B\xD4\xCD\x14\x03\xB2\x2C"
6000 "\xD3\xA9\xEE\xFA\x17\xFC\x9C\xDF"
6001 "\xC7\x75\x40\xFF\xAE\xAD\x1E\x59"
6002 "\x2F\x30\x24\xFB\xAD\x6B\x10\xFA"
6003 "\x6C\x9F\x5B\xE7\x25\xD5\xD0\x25"
6004 "\xAC\x4A\x4B\xDA\xFC\x7A\x85\x1B"
6005 "\x7E\x13\x06\x82\x08\x17\xA4\x35"
6006 "\xEC\xC5\x8D\x63\x96\x81\x0A\x8F"
6007 "\xA3\x05\x38\x95\x20\x1A\x47\x04"
6008 "\x6F\x6D\xDA\x8F\xEF\xC1\x76\x35"
6009 "\x6B\xC7\x4D\x0F\x94\x12\xCA\x3E"
6010 "\x2E\xD5\x03\x2E\x86\x7E\xAA\x3B"
6011 "\x37\x08\x1C\xCF\xBA\x5D\x71\x46"
6012 "\x80\x72\xB0\x4C\x82\x0D\x60\x3C",
6013 .rlen = 208,
6014 }
6015};
6016
6017static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = {
6018 { /* Generated using Crypto++ */
6019 .key = zeroed_string,
6020 .klen = 20,
6021 .iv = zeroed_string,
6022 .input = "\x03\x88\xDA\xCE\x60\xB6\xA3\x92"
6023 "\xF3\x28\xC2\xB9\x71\xB2\xFE\x78"
6024 "\x97\xFE\x4C\x23\x37\x42\x01\xE0"
6025 "\x81\x9F\x8D\xC5\xD7\x41\xA0\x1B",
6026 .ilen = 32,
6027 .assoc = zeroed_string,
6028 .alen = 8,
6029 .result = zeroed_string,
6030 .rlen = 16,
6031
6032 },{
6033 .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
6034 "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
6035 "\x00\x00\x00\x00",
6036 .klen = 20,
6037 .iv = "\x00\x00\x00\x00\x00\x00\x00\x01"
6038 "\x00\x00\x00\x00",
6039 .input = "\xC0\x0D\x8B\x42\x0F\x8F\x34\x18"
6040 "\x88\xB1\xC5\xBC\xC5\xB6\xD6\x28"
6041 "\x6A\x9D\xDF\x11\x5E\xFE\x5E\x9D"
6042 "\x2F\x70\x44\x92\xF7\xF2\xE3\xEF",
6043 .ilen = 32,
6044 .assoc = zeroed_string,
6045 .alen = 8,
6046 .result = zeroed_string,
6047 .rlen = 16,
6048 }, {
6049 .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
6050 "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
6051 "\x00\x00\x00\x00",
6052 .klen = 20,
6053 .iv = zeroed_string,
6054 .input = "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
6055 "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
6056 "\x0B\x8F\x88\x69\x17\xE6\xB4\x3C"
6057 "\xB1\x68\xFD\x14\x52\x64\x61\xB2",
6058 .ilen = 32,
6059 .assoc = zeroed_string,
6060 .alen = 8,
6061 .result = "\x01\x01\x01\x01\x01\x01\x01\x01"
6062 "\x01\x01\x01\x01\x01\x01\x01\x01",
6063 .rlen = 16,
6064 }, {
6065 .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
6066 "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
6067 "\x00\x00\x00\x00",
6068 .klen = 20,
6069 .iv = zeroed_string,
6070 .input = "\x4B\xB1\xB5\xE3\x25\x71\x70\xDE"
6071 "\x7F\xC9\x9C\xA5\x14\x19\xF2\xAC"
6072 "\x90\x92\xB7\xE3\x5F\xA3\x9A\x63"
6073 "\x7E\xD7\x1F\xD8\xD3\x7C\x4B\xF5",
6074 .ilen = 32,
6075 .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01",
6076 .alen = 8,
6077 .result = "\x01\x01\x01\x01\x01\x01\x01\x01"
6078 "\x01\x01\x01\x01\x01\x01\x01\x01",
6079 .rlen = 16,
6080
6081 }, {
6082 .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
6083 "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
6084 "\x00\x00\x00\x00",
6085 .klen = 20,
6086 .iv = "\x00\x00\x00\x00\x00\x00\x00\x01"
6087 "\x00\x00\x00\x00",
6088 .input = "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
6089 "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
6090 "\x64\x50\xF9\x32\x13\xFB\x74\x61"
6091 "\xF4\xED\x52\xD3\xC5\x10\x55\x3C",
6092 .ilen = 32,
6093 .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01",
6094 .alen = 8,
6095 .result = "\x01\x01\x01\x01\x01\x01\x01\x01"
6096 "\x01\x01\x01\x01\x01\x01\x01\x01",
6097 .rlen = 16,
6098 }, {
6099 .key = "\xfe\xff\xe9\x92\x86\x65\x73\x1c"
6100 "\x6d\x6a\x8f\x94\x67\x30\x83\x08"
6101 "\x00\x00\x00\x00",
6102 .klen = 20,
6103 .iv = "\x00\x00\x00\x00\x00\x00\x00\x01"
6104 "\x00\x00\x00\x00",
6105 .input = "\xC1\x0C\x8A\x43\x0E\x8E\x35\x19"
6106 "\x89\xB0\xC4\xBD\xC4\xB7\xD7\x29"
6107 "\x98\x14\xA1\x42\x37\x80\xFD\x90"
6108 "\x68\x12\x01\xA8\x91\x89\xB9\x83"
6109 "\x5B\x11\x77\x12\x9B\xFF\x24\x89"
6110 "\x94\x5F\x18\x12\xBA\x27\x09\x39"
6111 "\x99\x96\x76\x42\x15\x1C\xCD\xCB"
6112 "\xDC\xD3\xDA\x65\x73\xAF\x80\xCD"
6113 "\xD2\xB6\xC2\x4A\x76\xC2\x92\x85"
6114 "\xBD\xCF\x62\x98\x58\x14\xE5\xBD",
6115 .ilen = 80,
6116 .assoc = "\x01\x01\x01\x01\x01\x01\x01\x01",
6117 .alen = 8,
6118 .result = "\x01\x01\x01\x01\x01\x01\x01\x01"
6119 "\x01\x01\x01\x01\x01\x01\x01\x01"
6120 "\x01\x01\x01\x01\x01\x01\x01\x01"
6121 "\x01\x01\x01\x01\x01\x01\x01\x01"
6122 "\x01\x01\x01\x01\x01\x01\x01\x01"
6123 "\x01\x01\x01\x01\x01\x01\x01\x01"
6124 "\x01\x01\x01\x01\x01\x01\x01\x01"
6125 "\x01\x01\x01\x01\x01\x01\x01\x01",
6126 .rlen = 64,
6127 }, {
6128 .key = "\x00\x01\x02\x03\x04\x05\x06\x07"
6129 "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
6130 "\x00\x00\x00\x00",
6131 .klen = 20,
6132 .iv = "\x00\x00\x45\x67\x89\xab\xcd\xef"
6133 "\x00\x00\x00\x00",
6134 .input = "\xC1\x76\x33\x85\xE2\x9B\x5F\xDE"
6135 "\xDE\x89\x3D\x42\xE7\xC9\x69\x8A"
6136 "\x44\x6D\xC3\x88\x46\x2E\xC2\x01"
6137 "\x5E\xF6\x0C\x39\xF0\xC4\xA5\x82"
6138 "\xCD\xE8\x31\xCC\x0A\x4C\xE4\x44"
6139 "\x41\xA9\x82\x6F\x22\xA1\x23\x1A"
6140 "\xA8\xE3\x16\xFD\x31\x5C\x27\x31"
6141 "\xF1\x7F\x01\x63\xA3\xAF\x70\xA1"
6142 "\xCF\x07\x57\x41\x67\xD0\xC4\x42"
6143 "\xDB\x18\xC6\x4C\x4C\xE0\x3D\x9F"
6144 "\x05\x07\xFB\x13\x7D\x4A\xCA\x5B"
6145 "\xF0\xBF\x64\x7E\x05\xB1\x72\xEE"
6146 "\x7C\x3B\xD4\xCD\x14\x03\xB2\x2C"
6147 "\xD3\xA9\xEE\xFA\x17\xFC\x9C\xDF"
6148 "\xC7\x75\x40\xFF\xAE\xAD\x1E\x59"
6149 "\x2F\x30\x24\xFB\xAD\x6B\x10\xFA"
6150 "\x6C\x9F\x5B\xE7\x25\xD5\xD0\x25"
6151 "\xAC\x4A\x4B\xDA\xFC\x7A\x85\x1B"
6152 "\x7E\x13\x06\x82\x08\x17\xA4\x35"
6153 "\xEC\xC5\x8D\x63\x96\x81\x0A\x8F"
6154 "\xA3\x05\x38\x95\x20\x1A\x47\x04"
6155 "\x6F\x6D\xDA\x8F\xEF\xC1\x76\x35"
6156 "\x6B\xC7\x4D\x0F\x94\x12\xCA\x3E"
6157 "\x2E\xD5\x03\x2E\x86\x7E\xAA\x3B"
6158 "\x37\x08\x1C\xCF\xBA\x5D\x71\x46"
6159 "\x80\x72\xB0\x4C\x82\x0D\x60\x3C",
6160 .ilen = 208,
6161 .assoc = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa"
6162 "\xaa\xaa\xaa\xaa",
6163 .alen = 12,
6164 .result = "\xff\xff\xff\xff\xff\xff\xff\xff"
6165 "\xff\xff\xff\xff\xff\xff\xff\xff"
6166 "\xff\xff\xff\xff\xff\xff\xff\xff"
6167 "\xff\xff\xff\xff\xff\xff\xff\xff"
6168 "\xff\xff\xff\xff\xff\xff\xff\xff"
6169 "\xff\xff\xff\xff\xff\xff\xff\xff"
6170 "\xff\xff\xff\xff\xff\xff\xff\xff"
6171 "\xff\xff\xff\xff\xff\xff\xff\xff"
6172 "\xff\xff\xff\xff\xff\xff\xff\xff"
6173 "\xff\xff\xff\xff\xff\xff\xff\xff"
6174 "\xff\xff\xff\xff\xff\xff\xff\xff"
6175 "\xff\xff\xff\xff\xff\xff\xff\xff"
6176 "\xff\xff\xff\xff\xff\xff\xff\xff"
6177 "\xff\xff\xff\xff\xff\xff\xff\xff"
6178 "\xff\xff\xff\xff\xff\xff\xff\xff"
6179 "\xff\xff\xff\xff\xff\xff\xff\xff"
6180 "\xff\xff\xff\xff\xff\xff\xff\xff"
6181 "\xff\xff\xff\xff\xff\xff\xff\xff"
6182 "\xff\xff\xff\xff\xff\xff\xff\xff"
6183 "\xff\xff\xff\xff\xff\xff\xff\xff"
6184 "\xff\xff\xff\xff\xff\xff\xff\xff"
6185 "\xff\xff\xff\xff\xff\xff\xff\xff"
6186 "\xff\xff\xff\xff\xff\xff\xff\xff"
6187 "\xff\xff\xff\xff\xff\xff\xff\xff",
6188 .rlen = 192,
6189
6190 }
6191};
6192
5832static struct aead_testvec aes_ccm_enc_tv_template[] = { 6193static struct aead_testvec aes_ccm_enc_tv_template[] = {
5833 { /* From RFC 3610 */ 6194 { /* From RFC 3610 */
5834 .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" 6195 .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7"
diff --git a/crypto/zlib.c b/crypto/zlib.c
index c3015733c990..739b8fca4cea 100644
--- a/crypto/zlib.c
+++ b/crypto/zlib.c
@@ -95,11 +95,10 @@ static int zlib_compress_setup(struct crypto_pcomp *tfm, void *params,
95 zlib_comp_exit(ctx); 95 zlib_comp_exit(ctx);
96 96
97 workspacesize = zlib_deflate_workspacesize(); 97 workspacesize = zlib_deflate_workspacesize();
98 stream->workspace = vmalloc(workspacesize); 98 stream->workspace = vzalloc(workspacesize);
99 if (!stream->workspace) 99 if (!stream->workspace)
100 return -ENOMEM; 100 return -ENOMEM;
101 101
102 memset(stream->workspace, 0, workspacesize);
103 ret = zlib_deflateInit2(stream, 102 ret = zlib_deflateInit2(stream,
104 tb[ZLIB_COMP_LEVEL] 103 tb[ZLIB_COMP_LEVEL]
105 ? nla_get_u32(tb[ZLIB_COMP_LEVEL]) 104 ? nla_get_u32(tb[ZLIB_COMP_LEVEL])
diff --git a/drivers/char/hw_random/via-rng.c b/drivers/char/hw_random/via-rng.c
index 794aacb715c1..d0387a84eec1 100644
--- a/drivers/char/hw_random/via-rng.c
+++ b/drivers/char/hw_random/via-rng.c
@@ -24,6 +24,7 @@
24 * warranty of any kind, whether express or implied. 24 * warranty of any kind, whether express or implied.
25 */ 25 */
26 26
27#include <crypto/padlock.h>
27#include <linux/module.h> 28#include <linux/module.h>
28#include <linux/kernel.h> 29#include <linux/kernel.h>
29#include <linux/hw_random.h> 30#include <linux/hw_random.h>
@@ -34,7 +35,6 @@
34#include <asm/i387.h> 35#include <asm/i387.h>
35 36
36 37
37#define PFX KBUILD_MODNAME ": "
38 38
39 39
40enum { 40enum {
@@ -81,8 +81,7 @@ static inline u32 xstore(u32 *addr, u32 edx_in)
81 ts_state = irq_ts_save(); 81 ts_state = irq_ts_save();
82 82
83 asm(".byte 0x0F,0xA7,0xC0 /* xstore %%edi (addr=%0) */" 83 asm(".byte 0x0F,0xA7,0xC0 /* xstore %%edi (addr=%0) */"
84 :"=m"(*addr), "=a"(eax_out) 84 : "=m" (*addr), "=a" (eax_out), "+d" (edx_in), "+D" (addr));
85 :"D"(addr), "d"(edx_in));
86 85
87 irq_ts_restore(ts_state); 86 irq_ts_restore(ts_state);
88 return eax_out; 87 return eax_out;
@@ -90,8 +89,10 @@ static inline u32 xstore(u32 *addr, u32 edx_in)
90 89
91static int via_rng_data_present(struct hwrng *rng, int wait) 90static int via_rng_data_present(struct hwrng *rng, int wait)
92{ 91{
92 char buf[16 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
93 ((aligned(STACK_ALIGN)));
94 u32 *via_rng_datum = (u32 *)PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
93 u32 bytes_out; 95 u32 bytes_out;
94 u32 *via_rng_datum = (u32 *)(&rng->priv);
95 int i; 96 int i;
96 97
97 /* We choose the recommended 1-byte-per-instruction RNG rate, 98 /* We choose the recommended 1-byte-per-instruction RNG rate,
@@ -115,6 +116,7 @@ static int via_rng_data_present(struct hwrng *rng, int wait)
115 break; 116 break;
116 udelay(10); 117 udelay(10);
117 } 118 }
119 rng->priv = *via_rng_datum;
118 return bytes_out ? 1 : 0; 120 return bytes_out ? 1 : 0;
119} 121}
120 122
diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c
index 7d279e578df5..c99305afa58a 100644
--- a/drivers/crypto/mv_cesa.c
+++ b/drivers/crypto/mv_cesa.c
@@ -857,7 +857,7 @@ static int mv_cra_hash_init(struct crypto_tfm *tfm, const char *base_hash_name,
857 printk(KERN_WARNING MV_CESA 857 printk(KERN_WARNING MV_CESA
858 "Base driver '%s' could not be loaded!\n", 858 "Base driver '%s' could not be loaded!\n",
859 base_hash_name); 859 base_hash_name);
860 err = PTR_ERR(fallback_tfm); 860 err = PTR_ERR(base_hash);
861 goto err_bad_base; 861 goto err_bad_base;
862 } 862 }
863 } 863 }
diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c
index 76141262ea1d..80dc094e78c6 100644
--- a/drivers/crypto/n2_core.c
+++ b/drivers/crypto/n2_core.c
@@ -1542,7 +1542,7 @@ out:
1542 return err; 1542 return err;
1543} 1543}
1544 1544
1545static void __exit n2_unregister_algs(void) 1545static void __devexit n2_unregister_algs(void)
1546{ 1546{
1547 mutex_lock(&spu_lock); 1547 mutex_lock(&spu_lock);
1548 if (!--algs_registered) 1548 if (!--algs_registered)
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 799ca517c121..add2a1a72ba4 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -74,11 +74,9 @@
74#define FLAGS_CBC BIT(1) 74#define FLAGS_CBC BIT(1)
75#define FLAGS_GIV BIT(2) 75#define FLAGS_GIV BIT(2)
76 76
77#define FLAGS_NEW_KEY BIT(4) 77#define FLAGS_INIT BIT(4)
78#define FLAGS_NEW_IV BIT(5) 78#define FLAGS_FAST BIT(5)
79#define FLAGS_INIT BIT(6) 79#define FLAGS_BUSY BIT(6)
80#define FLAGS_FAST BIT(7)
81#define FLAGS_BUSY 8
82 80
83struct omap_aes_ctx { 81struct omap_aes_ctx {
84 struct omap_aes_dev *dd; 82 struct omap_aes_dev *dd;
@@ -98,19 +96,18 @@ struct omap_aes_reqctx {
98struct omap_aes_dev { 96struct omap_aes_dev {
99 struct list_head list; 97 struct list_head list;
100 unsigned long phys_base; 98 unsigned long phys_base;
101 void __iomem *io_base; 99 void __iomem *io_base;
102 struct clk *iclk; 100 struct clk *iclk;
103 struct omap_aes_ctx *ctx; 101 struct omap_aes_ctx *ctx;
104 struct device *dev; 102 struct device *dev;
105 unsigned long flags; 103 unsigned long flags;
104 int err;
106 105
107 u32 *iv; 106 spinlock_t lock;
108 u32 ctrl; 107 struct crypto_queue queue;
109 108
110 spinlock_t lock; 109 struct tasklet_struct done_task;
111 struct crypto_queue queue; 110 struct tasklet_struct queue_task;
112
113 struct tasklet_struct task;
114 111
115 struct ablkcipher_request *req; 112 struct ablkcipher_request *req;
116 size_t total; 113 size_t total;
@@ -179,9 +176,13 @@ static int omap_aes_wait(struct omap_aes_dev *dd, u32 offset, u32 bit)
179 176
180static int omap_aes_hw_init(struct omap_aes_dev *dd) 177static int omap_aes_hw_init(struct omap_aes_dev *dd)
181{ 178{
182 int err = 0; 179 /*
183 180 * clocks are enabled when request starts and disabled when finished.
181 * It may be long delays between requests.
182 * Device might go to off mode to save power.
183 */
184 clk_enable(dd->iclk); 184 clk_enable(dd->iclk);
185
185 if (!(dd->flags & FLAGS_INIT)) { 186 if (!(dd->flags & FLAGS_INIT)) {
186 /* is it necessary to reset before every operation? */ 187 /* is it necessary to reset before every operation? */
187 omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_SOFTRESET, 188 omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_SOFTRESET,
@@ -193,39 +194,26 @@ static int omap_aes_hw_init(struct omap_aes_dev *dd)
193 __asm__ __volatile__("nop"); 194 __asm__ __volatile__("nop");
194 __asm__ __volatile__("nop"); 195 __asm__ __volatile__("nop");
195 196
196 err = omap_aes_wait(dd, AES_REG_SYSSTATUS, 197 if (omap_aes_wait(dd, AES_REG_SYSSTATUS,
197 AES_REG_SYSSTATUS_RESETDONE); 198 AES_REG_SYSSTATUS_RESETDONE))
198 if (!err) 199 return -ETIMEDOUT;
199 dd->flags |= FLAGS_INIT;
200 }
201 200
202 return err; 201 dd->flags |= FLAGS_INIT;
203} 202 dd->err = 0;
203 }
204 204
205static void omap_aes_hw_cleanup(struct omap_aes_dev *dd) 205 return 0;
206{
207 clk_disable(dd->iclk);
208} 206}
209 207
210static void omap_aes_write_ctrl(struct omap_aes_dev *dd) 208static int omap_aes_write_ctrl(struct omap_aes_dev *dd)
211{ 209{
212 unsigned int key32; 210 unsigned int key32;
213 int i; 211 int i, err;
214 u32 val, mask; 212 u32 val, mask;
215 213
216 val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3); 214 err = omap_aes_hw_init(dd);
217 if (dd->flags & FLAGS_CBC) 215 if (err)
218 val |= AES_REG_CTRL_CBC; 216 return err;
219 if (dd->flags & FLAGS_ENCRYPT)
220 val |= AES_REG_CTRL_DIRECTION;
221
222 if (dd->ctrl == val && !(dd->flags & FLAGS_NEW_IV) &&
223 !(dd->ctx->flags & FLAGS_NEW_KEY))
224 goto out;
225
226 /* only need to write control registers for new settings */
227
228 dd->ctrl = val;
229 217
230 val = 0; 218 val = 0;
231 if (dd->dma_lch_out >= 0) 219 if (dd->dma_lch_out >= 0)
@@ -237,30 +225,43 @@ static void omap_aes_write_ctrl(struct omap_aes_dev *dd)
237 225
238 omap_aes_write_mask(dd, AES_REG_MASK, val, mask); 226 omap_aes_write_mask(dd, AES_REG_MASK, val, mask);
239 227
240 pr_debug("Set key\n");
241 key32 = dd->ctx->keylen / sizeof(u32); 228 key32 = dd->ctx->keylen / sizeof(u32);
242 /* set a key */ 229
230 /* it seems a key should always be set even if it has not changed */
243 for (i = 0; i < key32; i++) { 231 for (i = 0; i < key32; i++) {
244 omap_aes_write(dd, AES_REG_KEY(i), 232 omap_aes_write(dd, AES_REG_KEY(i),
245 __le32_to_cpu(dd->ctx->key[i])); 233 __le32_to_cpu(dd->ctx->key[i]));
246 } 234 }
247 dd->ctx->flags &= ~FLAGS_NEW_KEY;
248 235
249 if (dd->flags & FLAGS_NEW_IV) { 236 if ((dd->flags & FLAGS_CBC) && dd->req->info)
250 pr_debug("Set IV\n"); 237 omap_aes_write_n(dd, AES_REG_IV(0), dd->req->info, 4);
251 omap_aes_write_n(dd, AES_REG_IV(0), dd->iv, 4); 238
252 dd->flags &= ~FLAGS_NEW_IV; 239 val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
253 } 240 if (dd->flags & FLAGS_CBC)
241 val |= AES_REG_CTRL_CBC;
242 if (dd->flags & FLAGS_ENCRYPT)
243 val |= AES_REG_CTRL_DIRECTION;
254 244
255 mask = AES_REG_CTRL_CBC | AES_REG_CTRL_DIRECTION | 245 mask = AES_REG_CTRL_CBC | AES_REG_CTRL_DIRECTION |
256 AES_REG_CTRL_KEY_SIZE; 246 AES_REG_CTRL_KEY_SIZE;
257 247
258 omap_aes_write_mask(dd, AES_REG_CTRL, dd->ctrl, mask); 248 omap_aes_write_mask(dd, AES_REG_CTRL, val, mask);
259 249
260out: 250 /* IN */
261 /* start DMA or disable idle mode */ 251 omap_set_dma_dest_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_CONSTANT,
262 omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_START, 252 dd->phys_base + AES_REG_DATA, 0, 4);
263 AES_REG_MASK_START); 253
254 omap_set_dma_dest_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
255 omap_set_dma_src_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
256
257 /* OUT */
258 omap_set_dma_src_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_CONSTANT,
259 dd->phys_base + AES_REG_DATA, 0, 4);
260
261 omap_set_dma_src_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
262 omap_set_dma_dest_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
263
264 return 0;
264} 265}
265 266
266static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx) 267static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx)
@@ -288,8 +289,16 @@ static void omap_aes_dma_callback(int lch, u16 ch_status, void *data)
288{ 289{
289 struct omap_aes_dev *dd = data; 290 struct omap_aes_dev *dd = data;
290 291
291 if (lch == dd->dma_lch_out) 292 if (ch_status != OMAP_DMA_BLOCK_IRQ) {
292 tasklet_schedule(&dd->task); 293 pr_err("omap-aes DMA error status: 0x%hx\n", ch_status);
294 dd->err = -EIO;
295 dd->flags &= ~FLAGS_INIT; /* request to re-initialize */
296 } else if (lch == dd->dma_lch_in) {
297 return;
298 }
299
300 /* dma_lch_out - completed */
301 tasklet_schedule(&dd->done_task);
293} 302}
294 303
295static int omap_aes_dma_init(struct omap_aes_dev *dd) 304static int omap_aes_dma_init(struct omap_aes_dev *dd)
@@ -339,18 +348,6 @@ static int omap_aes_dma_init(struct omap_aes_dev *dd)
339 goto err_dma_out; 348 goto err_dma_out;
340 } 349 }
341 350
342 omap_set_dma_dest_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_CONSTANT,
343 dd->phys_base + AES_REG_DATA, 0, 4);
344
345 omap_set_dma_dest_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
346 omap_set_dma_src_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
347
348 omap_set_dma_src_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_CONSTANT,
349 dd->phys_base + AES_REG_DATA, 0, 4);
350
351 omap_set_dma_src_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
352 omap_set_dma_dest_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
353
354 return 0; 351 return 0;
355 352
356err_dma_out: 353err_dma_out:
@@ -406,6 +403,11 @@ static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf,
406 if (!count) 403 if (!count)
407 return off; 404 return off;
408 405
406 /*
407 * buflen and total are AES_BLOCK_SIZE size aligned,
408 * so count should be also aligned
409 */
410
409 sg_copy_buf(buf + off, *sg, *offset, count, out); 411 sg_copy_buf(buf + off, *sg, *offset, count, out);
410 412
411 off += count; 413 off += count;
@@ -461,7 +463,9 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
461 omap_start_dma(dd->dma_lch_in); 463 omap_start_dma(dd->dma_lch_in);
462 omap_start_dma(dd->dma_lch_out); 464 omap_start_dma(dd->dma_lch_out);
463 465
464 omap_aes_write_ctrl(dd); 466 /* start DMA or disable idle mode */
467 omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_START,
468 AES_REG_MASK_START);
465 469
466 return 0; 470 return 0;
467} 471}
@@ -488,8 +492,10 @@ static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
488 count = min(dd->total, sg_dma_len(dd->in_sg)); 492 count = min(dd->total, sg_dma_len(dd->in_sg));
489 count = min(count, sg_dma_len(dd->out_sg)); 493 count = min(count, sg_dma_len(dd->out_sg));
490 494
491 if (count != dd->total) 495 if (count != dd->total) {
496 pr_err("request length != buffer length\n");
492 return -EINVAL; 497 return -EINVAL;
498 }
493 499
494 pr_debug("fast\n"); 500 pr_debug("fast\n");
495 501
@@ -525,23 +531,25 @@ static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
525 531
526 dd->total -= count; 532 dd->total -= count;
527 533
528 err = omap_aes_hw_init(dd);
529
530 err = omap_aes_crypt_dma(tfm, addr_in, addr_out, count); 534 err = omap_aes_crypt_dma(tfm, addr_in, addr_out, count);
535 if (err) {
536 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
537 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);
538 }
531 539
532 return err; 540 return err;
533} 541}
534 542
535static void omap_aes_finish_req(struct omap_aes_dev *dd, int err) 543static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
536{ 544{
537 struct omap_aes_ctx *ctx; 545 struct ablkcipher_request *req = dd->req;
538 546
539 pr_debug("err: %d\n", err); 547 pr_debug("err: %d\n", err);
540 548
541 ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(dd->req)); 549 clk_disable(dd->iclk);
550 dd->flags &= ~FLAGS_BUSY;
542 551
543 if (!dd->total) 552 req->base.complete(&req->base, err);
544 dd->req->base.complete(&dd->req->base, err);
545} 553}
546 554
547static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) 555static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
@@ -553,8 +561,6 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
553 561
554 omap_aes_write_mask(dd, AES_REG_MASK, 0, AES_REG_MASK_START); 562 omap_aes_write_mask(dd, AES_REG_MASK, 0, AES_REG_MASK_START);
555 563
556 omap_aes_hw_cleanup(dd);
557
558 omap_stop_dma(dd->dma_lch_in); 564 omap_stop_dma(dd->dma_lch_in);
559 omap_stop_dma(dd->dma_lch_out); 565 omap_stop_dma(dd->dma_lch_out);
560 566
@@ -574,40 +580,39 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
574 } 580 }
575 } 581 }
576 582
577 if (err || !dd->total)
578 omap_aes_finish_req(dd, err);
579
580 return err; 583 return err;
581} 584}
582 585
583static int omap_aes_handle_req(struct omap_aes_dev *dd) 586static int omap_aes_handle_queue(struct omap_aes_dev *dd,
587 struct ablkcipher_request *req)
584{ 588{
585 struct crypto_async_request *async_req, *backlog; 589 struct crypto_async_request *async_req, *backlog;
586 struct omap_aes_ctx *ctx; 590 struct omap_aes_ctx *ctx;
587 struct omap_aes_reqctx *rctx; 591 struct omap_aes_reqctx *rctx;
588 struct ablkcipher_request *req;
589 unsigned long flags; 592 unsigned long flags;
590 593 int err, ret = 0;
591 if (dd->total)
592 goto start;
593 594
594 spin_lock_irqsave(&dd->lock, flags); 595 spin_lock_irqsave(&dd->lock, flags);
596 if (req)
597 ret = ablkcipher_enqueue_request(&dd->queue, req);
598 if (dd->flags & FLAGS_BUSY) {
599 spin_unlock_irqrestore(&dd->lock, flags);
600 return ret;
601 }
595 backlog = crypto_get_backlog(&dd->queue); 602 backlog = crypto_get_backlog(&dd->queue);
596 async_req = crypto_dequeue_request(&dd->queue); 603 async_req = crypto_dequeue_request(&dd->queue);
597 if (!async_req) 604 if (async_req)
598 clear_bit(FLAGS_BUSY, &dd->flags); 605 dd->flags |= FLAGS_BUSY;
599 spin_unlock_irqrestore(&dd->lock, flags); 606 spin_unlock_irqrestore(&dd->lock, flags);
600 607
601 if (!async_req) 608 if (!async_req)
602 return 0; 609 return ret;
603 610
604 if (backlog) 611 if (backlog)
605 backlog->complete(backlog, -EINPROGRESS); 612 backlog->complete(backlog, -EINPROGRESS);
606 613
607 req = ablkcipher_request_cast(async_req); 614 req = ablkcipher_request_cast(async_req);
608 615
609 pr_debug("get new req\n");
610
611 /* assign new request to device */ 616 /* assign new request to device */
612 dd->req = req; 617 dd->req = req;
613 dd->total = req->nbytes; 618 dd->total = req->nbytes;
@@ -621,27 +626,22 @@ static int omap_aes_handle_req(struct omap_aes_dev *dd)
621 rctx->mode &= FLAGS_MODE_MASK; 626 rctx->mode &= FLAGS_MODE_MASK;
622 dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; 627 dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
623 628
624 dd->iv = req->info; 629 dd->ctx = ctx;
625 if ((dd->flags & FLAGS_CBC) && dd->iv)
626 dd->flags |= FLAGS_NEW_IV;
627 else
628 dd->flags &= ~FLAGS_NEW_IV;
629
630 ctx->dd = dd; 630 ctx->dd = dd;
631 if (dd->ctx != ctx) {
632 /* assign new context to device */
633 dd->ctx = ctx;
634 ctx->flags |= FLAGS_NEW_KEY;
635 }
636 631
637 if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) 632 err = omap_aes_write_ctrl(dd);
638 pr_err("request size is not exact amount of AES blocks\n"); 633 if (!err)
634 err = omap_aes_crypt_dma_start(dd);
635 if (err) {
636 /* aes_task will not finish it, so do it here */
637 omap_aes_finish_req(dd, err);
638 tasklet_schedule(&dd->queue_task);
639 }
639 640
640start: 641 return ret; /* return ret, which is enqueue return value */
641 return omap_aes_crypt_dma_start(dd);
642} 642}
643 643
644static void omap_aes_task(unsigned long data) 644static void omap_aes_done_task(unsigned long data)
645{ 645{
646 struct omap_aes_dev *dd = (struct omap_aes_dev *)data; 646 struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
647 int err; 647 int err;
@@ -650,40 +650,50 @@ static void omap_aes_task(unsigned long data)
650 650
651 err = omap_aes_crypt_dma_stop(dd); 651 err = omap_aes_crypt_dma_stop(dd);
652 652
653 err = omap_aes_handle_req(dd); 653 err = dd->err ? : err;
654
655 if (dd->total && !err) {
656 err = omap_aes_crypt_dma_start(dd);
657 if (!err)
658 return; /* DMA started. Not fininishing. */
659 }
660
661 omap_aes_finish_req(dd, err);
662 omap_aes_handle_queue(dd, NULL);
654 663
655 pr_debug("exit\n"); 664 pr_debug("exit\n");
656} 665}
657 666
667static void omap_aes_queue_task(unsigned long data)
668{
669 struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
670
671 omap_aes_handle_queue(dd, NULL);
672}
673
658static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode) 674static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
659{ 675{
660 struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx( 676 struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
661 crypto_ablkcipher_reqtfm(req)); 677 crypto_ablkcipher_reqtfm(req));
662 struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req); 678 struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
663 struct omap_aes_dev *dd; 679 struct omap_aes_dev *dd;
664 unsigned long flags;
665 int err;
666 680
667 pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes, 681 pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes,
668 !!(mode & FLAGS_ENCRYPT), 682 !!(mode & FLAGS_ENCRYPT),
669 !!(mode & FLAGS_CBC)); 683 !!(mode & FLAGS_CBC));
670 684
685 if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
686 pr_err("request size is not exact amount of AES blocks\n");
687 return -EINVAL;
688 }
689
671 dd = omap_aes_find_dev(ctx); 690 dd = omap_aes_find_dev(ctx);
672 if (!dd) 691 if (!dd)
673 return -ENODEV; 692 return -ENODEV;
674 693
675 rctx->mode = mode; 694 rctx->mode = mode;
676 695
677 spin_lock_irqsave(&dd->lock, flags); 696 return omap_aes_handle_queue(dd, req);
678 err = ablkcipher_enqueue_request(&dd->queue, req);
679 spin_unlock_irqrestore(&dd->lock, flags);
680
681 if (!test_and_set_bit(FLAGS_BUSY, &dd->flags))
682 omap_aes_handle_req(dd);
683
684 pr_debug("exit\n");
685
686 return err;
687} 697}
688 698
689/* ********************** ALG API ************************************ */ 699/* ********************** ALG API ************************************ */
@@ -701,7 +711,6 @@ static int omap_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
701 711
702 memcpy(ctx->key, key, keylen); 712 memcpy(ctx->key, key, keylen);
703 ctx->keylen = keylen; 713 ctx->keylen = keylen;
704 ctx->flags |= FLAGS_NEW_KEY;
705 714
706 return 0; 715 return 0;
707} 716}
@@ -750,7 +759,7 @@ static struct crypto_alg algs[] = {
750 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 759 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
751 .cra_blocksize = AES_BLOCK_SIZE, 760 .cra_blocksize = AES_BLOCK_SIZE,
752 .cra_ctxsize = sizeof(struct omap_aes_ctx), 761 .cra_ctxsize = sizeof(struct omap_aes_ctx),
753 .cra_alignmask = 0, 762 .cra_alignmask = 0,
754 .cra_type = &crypto_ablkcipher_type, 763 .cra_type = &crypto_ablkcipher_type,
755 .cra_module = THIS_MODULE, 764 .cra_module = THIS_MODULE,
756 .cra_init = omap_aes_cra_init, 765 .cra_init = omap_aes_cra_init,
@@ -770,7 +779,7 @@ static struct crypto_alg algs[] = {
770 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 779 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
771 .cra_blocksize = AES_BLOCK_SIZE, 780 .cra_blocksize = AES_BLOCK_SIZE,
772 .cra_ctxsize = sizeof(struct omap_aes_ctx), 781 .cra_ctxsize = sizeof(struct omap_aes_ctx),
773 .cra_alignmask = 0, 782 .cra_alignmask = 0,
774 .cra_type = &crypto_ablkcipher_type, 783 .cra_type = &crypto_ablkcipher_type,
775 .cra_module = THIS_MODULE, 784 .cra_module = THIS_MODULE,
776 .cra_init = omap_aes_cra_init, 785 .cra_init = omap_aes_cra_init,
@@ -849,7 +858,8 @@ static int omap_aes_probe(struct platform_device *pdev)
849 (reg & AES_REG_REV_MAJOR) >> 4, reg & AES_REG_REV_MINOR); 858 (reg & AES_REG_REV_MAJOR) >> 4, reg & AES_REG_REV_MINOR);
850 clk_disable(dd->iclk); 859 clk_disable(dd->iclk);
851 860
852 tasklet_init(&dd->task, omap_aes_task, (unsigned long)dd); 861 tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd);
862 tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd);
853 863
854 err = omap_aes_dma_init(dd); 864 err = omap_aes_dma_init(dd);
855 if (err) 865 if (err)
@@ -876,7 +886,8 @@ err_algs:
876 crypto_unregister_alg(&algs[j]); 886 crypto_unregister_alg(&algs[j]);
877 omap_aes_dma_cleanup(dd); 887 omap_aes_dma_cleanup(dd);
878err_dma: 888err_dma:
879 tasklet_kill(&dd->task); 889 tasklet_kill(&dd->done_task);
890 tasklet_kill(&dd->queue_task);
880 iounmap(dd->io_base); 891 iounmap(dd->io_base);
881err_io: 892err_io:
882 clk_put(dd->iclk); 893 clk_put(dd->iclk);
@@ -903,7 +914,8 @@ static int omap_aes_remove(struct platform_device *pdev)
903 for (i = 0; i < ARRAY_SIZE(algs); i++) 914 for (i = 0; i < ARRAY_SIZE(algs); i++)
904 crypto_unregister_alg(&algs[i]); 915 crypto_unregister_alg(&algs[i]);
905 916
906 tasklet_kill(&dd->task); 917 tasklet_kill(&dd->done_task);
918 tasklet_kill(&dd->queue_task);
907 omap_aes_dma_cleanup(dd); 919 omap_aes_dma_cleanup(dd);
908 iounmap(dd->io_base); 920 iounmap(dd->io_base);
909 clk_put(dd->iclk); 921 clk_put(dd->iclk);
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index a081c7c7d03f..2e71123516e0 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -72,10 +72,9 @@
72 72
73#define DEFAULT_TIMEOUT_INTERVAL HZ 73#define DEFAULT_TIMEOUT_INTERVAL HZ
74 74
75#define FLAGS_FIRST 0x0001
76#define FLAGS_FINUP 0x0002 75#define FLAGS_FINUP 0x0002
77#define FLAGS_FINAL 0x0004 76#define FLAGS_FINAL 0x0004
78#define FLAGS_FAST 0x0008 77#define FLAGS_SG 0x0008
79#define FLAGS_SHA1 0x0010 78#define FLAGS_SHA1 0x0010
80#define FLAGS_DMA_ACTIVE 0x0020 79#define FLAGS_DMA_ACTIVE 0x0020
81#define FLAGS_OUTPUT_READY 0x0040 80#define FLAGS_OUTPUT_READY 0x0040
@@ -83,13 +82,17 @@
83#define FLAGS_INIT 0x0100 82#define FLAGS_INIT 0x0100
84#define FLAGS_CPU 0x0200 83#define FLAGS_CPU 0x0200
85#define FLAGS_HMAC 0x0400 84#define FLAGS_HMAC 0x0400
86 85#define FLAGS_ERROR 0x0800
87/* 3rd byte */ 86#define FLAGS_BUSY 0x1000
88#define FLAGS_BUSY 16
89 87
90#define OP_UPDATE 1 88#define OP_UPDATE 1
91#define OP_FINAL 2 89#define OP_FINAL 2
92 90
91#define OMAP_ALIGN_MASK (sizeof(u32)-1)
92#define OMAP_ALIGNED __attribute__((aligned(sizeof(u32))))
93
94#define BUFLEN PAGE_SIZE
95
93struct omap_sham_dev; 96struct omap_sham_dev;
94 97
95struct omap_sham_reqctx { 98struct omap_sham_reqctx {
@@ -97,8 +100,8 @@ struct omap_sham_reqctx {
97 unsigned long flags; 100 unsigned long flags;
98 unsigned long op; 101 unsigned long op;
99 102
103 u8 digest[SHA1_DIGEST_SIZE] OMAP_ALIGNED;
100 size_t digcnt; 104 size_t digcnt;
101 u8 *buffer;
102 size_t bufcnt; 105 size_t bufcnt;
103 size_t buflen; 106 size_t buflen;
104 dma_addr_t dma_addr; 107 dma_addr_t dma_addr;
@@ -107,6 +110,8 @@ struct omap_sham_reqctx {
107 struct scatterlist *sg; 110 struct scatterlist *sg;
108 unsigned int offset; /* offset in current sg */ 111 unsigned int offset; /* offset in current sg */
109 unsigned int total; /* total request */ 112 unsigned int total; /* total request */
113
114 u8 buffer[0] OMAP_ALIGNED;
110}; 115};
111 116
112struct omap_sham_hmac_ctx { 117struct omap_sham_hmac_ctx {
@@ -136,6 +141,7 @@ struct omap_sham_dev {
136 int irq; 141 int irq;
137 struct clk *iclk; 142 struct clk *iclk;
138 spinlock_t lock; 143 spinlock_t lock;
144 int err;
139 int dma; 145 int dma;
140 int dma_lch; 146 int dma_lch;
141 struct tasklet_struct done_task; 147 struct tasklet_struct done_task;
@@ -194,53 +200,68 @@ static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit)
194static void omap_sham_copy_hash(struct ahash_request *req, int out) 200static void omap_sham_copy_hash(struct ahash_request *req, int out)
195{ 201{
196 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 202 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
203 u32 *hash = (u32 *)ctx->digest;
204 int i;
205
206 /* MD5 is almost unused. So copy sha1 size to reduce code */
207 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) {
208 if (out)
209 hash[i] = omap_sham_read(ctx->dd,
210 SHA_REG_DIGEST(i));
211 else
212 omap_sham_write(ctx->dd,
213 SHA_REG_DIGEST(i), hash[i]);
214 }
215}
216
217static void omap_sham_copy_ready_hash(struct ahash_request *req)
218{
219 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
220 u32 *in = (u32 *)ctx->digest;
197 u32 *hash = (u32 *)req->result; 221 u32 *hash = (u32 *)req->result;
198 int i; 222 int i;
199 223
224 if (!hash)
225 return;
226
200 if (likely(ctx->flags & FLAGS_SHA1)) { 227 if (likely(ctx->flags & FLAGS_SHA1)) {
201 /* SHA1 results are in big endian */ 228 /* SHA1 results are in big endian */
202 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) 229 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++)
203 if (out) 230 hash[i] = be32_to_cpu(in[i]);
204 hash[i] = be32_to_cpu(omap_sham_read(ctx->dd,
205 SHA_REG_DIGEST(i)));
206 else
207 omap_sham_write(ctx->dd, SHA_REG_DIGEST(i),
208 cpu_to_be32(hash[i]));
209 } else { 231 } else {
210 /* MD5 results are in little endian */ 232 /* MD5 results are in little endian */
211 for (i = 0; i < MD5_DIGEST_SIZE / sizeof(u32); i++) 233 for (i = 0; i < MD5_DIGEST_SIZE / sizeof(u32); i++)
212 if (out) 234 hash[i] = le32_to_cpu(in[i]);
213 hash[i] = le32_to_cpu(omap_sham_read(ctx->dd,
214 SHA_REG_DIGEST(i)));
215 else
216 omap_sham_write(ctx->dd, SHA_REG_DIGEST(i),
217 cpu_to_le32(hash[i]));
218 } 235 }
219} 236}
220 237
221static int omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length, 238static int omap_sham_hw_init(struct omap_sham_dev *dd)
222 int final, int dma)
223{ 239{
224 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 240 clk_enable(dd->iclk);
225 u32 val = length << 5, mask;
226 241
227 if (unlikely(!ctx->digcnt)) { 242 if (!(dd->flags & FLAGS_INIT)) {
243 omap_sham_write_mask(dd, SHA_REG_MASK,
244 SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET);
228 245
229 clk_enable(dd->iclk); 246 if (omap_sham_wait(dd, SHA_REG_SYSSTATUS,
247 SHA_REG_SYSSTATUS_RESETDONE))
248 return -ETIMEDOUT;
230 249
231 if (!(dd->flags & FLAGS_INIT)) { 250 dd->flags |= FLAGS_INIT;
232 omap_sham_write_mask(dd, SHA_REG_MASK, 251 dd->err = 0;
233 SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET); 252 }
234 253
235 if (omap_sham_wait(dd, SHA_REG_SYSSTATUS, 254 return 0;
236 SHA_REG_SYSSTATUS_RESETDONE)) 255}
237 return -ETIMEDOUT;
238 256
239 dd->flags |= FLAGS_INIT; 257static void omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length,
240 } 258 int final, int dma)
241 } else { 259{
260 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
261 u32 val = length << 5, mask;
262
263 if (likely(ctx->digcnt))
242 omap_sham_write(dd, SHA_REG_DIGCNT, ctx->digcnt); 264 omap_sham_write(dd, SHA_REG_DIGCNT, ctx->digcnt);
243 }
244 265
245 omap_sham_write_mask(dd, SHA_REG_MASK, 266 omap_sham_write_mask(dd, SHA_REG_MASK,
246 SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0), 267 SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0),
@@ -260,29 +281,26 @@ static int omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length,
260 SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH; 281 SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH;
261 282
262 omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask); 283 omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask);
263
264 return 0;
265} 284}
266 285
267static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf, 286static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
268 size_t length, int final) 287 size_t length, int final)
269{ 288{
270 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 289 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
271 int err, count, len32; 290 int count, len32;
272 const u32 *buffer = (const u32 *)buf; 291 const u32 *buffer = (const u32 *)buf;
273 292
274 dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n", 293 dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
275 ctx->digcnt, length, final); 294 ctx->digcnt, length, final);
276 295
277 err = omap_sham_write_ctrl(dd, length, final, 0); 296 omap_sham_write_ctrl(dd, length, final, 0);
278 if (err) 297
279 return err; 298 /* should be non-zero before next lines to disable clocks later */
299 ctx->digcnt += length;
280 300
281 if (omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY)) 301 if (omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY))
282 return -ETIMEDOUT; 302 return -ETIMEDOUT;
283 303
284 ctx->digcnt += length;
285
286 if (final) 304 if (final)
287 ctx->flags |= FLAGS_FINAL; /* catch last interrupt */ 305 ctx->flags |= FLAGS_FINAL; /* catch last interrupt */
288 306
@@ -298,16 +316,11 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
298 size_t length, int final) 316 size_t length, int final)
299{ 317{
300 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 318 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
301 int err, len32; 319 int len32;
302 320
303 dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n", 321 dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
304 ctx->digcnt, length, final); 322 ctx->digcnt, length, final);
305 323
306 /* flush cache entries related to our page */
307 if (dma_addr == ctx->dma_addr)
308 dma_sync_single_for_device(dd->dev, dma_addr, length,
309 DMA_TO_DEVICE);
310
311 len32 = DIV_ROUND_UP(length, sizeof(u32)); 324 len32 = DIV_ROUND_UP(length, sizeof(u32));
312 325
313 omap_set_dma_transfer_params(dd->dma_lch, OMAP_DMA_DATA_TYPE_S32, len32, 326 omap_set_dma_transfer_params(dd->dma_lch, OMAP_DMA_DATA_TYPE_S32, len32,
@@ -317,9 +330,7 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
317 omap_set_dma_src_params(dd->dma_lch, 0, OMAP_DMA_AMODE_POST_INC, 330 omap_set_dma_src_params(dd->dma_lch, 0, OMAP_DMA_AMODE_POST_INC,
318 dma_addr, 0, 0); 331 dma_addr, 0, 0);
319 332
320 err = omap_sham_write_ctrl(dd, length, final, 1); 333 omap_sham_write_ctrl(dd, length, final, 1);
321 if (err)
322 return err;
323 334
324 ctx->digcnt += length; 335 ctx->digcnt += length;
325 336
@@ -371,15 +382,29 @@ static size_t omap_sham_append_sg(struct omap_sham_reqctx *ctx)
371 return 0; 382 return 0;
372} 383}
373 384
385static int omap_sham_xmit_dma_map(struct omap_sham_dev *dd,
386 struct omap_sham_reqctx *ctx,
387 size_t length, int final)
388{
389 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,
390 DMA_TO_DEVICE);
391 if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
392 dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen);
393 return -EINVAL;
394 }
395
396 ctx->flags &= ~FLAGS_SG;
397
398 /* next call does not fail... so no unmap in the case of error */
399 return omap_sham_xmit_dma(dd, ctx->dma_addr, length, final);
400}
401
374static int omap_sham_update_dma_slow(struct omap_sham_dev *dd) 402static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
375{ 403{
376 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 404 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
377 unsigned int final; 405 unsigned int final;
378 size_t count; 406 size_t count;
379 407
380 if (!ctx->total)
381 return 0;
382
383 omap_sham_append_sg(ctx); 408 omap_sham_append_sg(ctx);
384 409
385 final = (ctx->flags & FLAGS_FINUP) && !ctx->total; 410 final = (ctx->flags & FLAGS_FINUP) && !ctx->total;
@@ -390,30 +415,68 @@ static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
390 if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) { 415 if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {
391 count = ctx->bufcnt; 416 count = ctx->bufcnt;
392 ctx->bufcnt = 0; 417 ctx->bufcnt = 0;
393 return omap_sham_xmit_dma(dd, ctx->dma_addr, count, final); 418 return omap_sham_xmit_dma_map(dd, ctx, count, final);
394 } 419 }
395 420
396 return 0; 421 return 0;
397} 422}
398 423
399static int omap_sham_update_dma_fast(struct omap_sham_dev *dd) 424/* Start address alignment */
425#define SG_AA(sg) (IS_ALIGNED(sg->offset, sizeof(u32)))
426/* SHA1 block size alignment */
427#define SG_SA(sg) (IS_ALIGNED(sg->length, SHA1_MD5_BLOCK_SIZE))
428
429static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
400{ 430{
401 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 431 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
402 unsigned int length; 432 unsigned int length, final, tail;
433 struct scatterlist *sg;
403 434
404 ctx->flags |= FLAGS_FAST; 435 if (!ctx->total)
436 return 0;
437
438 if (ctx->bufcnt || ctx->offset)
439 return omap_sham_update_dma_slow(dd);
440
441 dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
442 ctx->digcnt, ctx->bufcnt, ctx->total);
443
444 sg = ctx->sg;
405 445
406 length = min(ctx->total, sg_dma_len(ctx->sg)); 446 if (!SG_AA(sg))
407 ctx->total = length; 447 return omap_sham_update_dma_slow(dd);
448
449 if (!sg_is_last(sg) && !SG_SA(sg))
450 /* size is not SHA1_BLOCK_SIZE aligned */
451 return omap_sham_update_dma_slow(dd);
452
453 length = min(ctx->total, sg->length);
454
455 if (sg_is_last(sg)) {
456 if (!(ctx->flags & FLAGS_FINUP)) {
457 /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */
458 tail = length & (SHA1_MD5_BLOCK_SIZE - 1);
459 /* without finup() we need one block to close hash */
460 if (!tail)
461 tail = SHA1_MD5_BLOCK_SIZE;
462 length -= tail;
463 }
464 }
408 465
409 if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) { 466 if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
410 dev_err(dd->dev, "dma_map_sg error\n"); 467 dev_err(dd->dev, "dma_map_sg error\n");
411 return -EINVAL; 468 return -EINVAL;
412 } 469 }
413 470
471 ctx->flags |= FLAGS_SG;
472
414 ctx->total -= length; 473 ctx->total -= length;
474 ctx->offset = length; /* offset where to start slow */
415 475
416 return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, 1); 476 final = (ctx->flags & FLAGS_FINUP) && !ctx->total;
477
478 /* next call does not fail... so no unmap in the case of error */
479 return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final);
417} 480}
418 481
419static int omap_sham_update_cpu(struct omap_sham_dev *dd) 482static int omap_sham_update_cpu(struct omap_sham_dev *dd)
@@ -433,8 +496,17 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
433 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 496 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
434 497
435 omap_stop_dma(dd->dma_lch); 498 omap_stop_dma(dd->dma_lch);
436 if (ctx->flags & FLAGS_FAST) 499 if (ctx->flags & FLAGS_SG) {
437 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); 500 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
501 if (ctx->sg->length == ctx->offset) {
502 ctx->sg = sg_next(ctx->sg);
503 if (ctx->sg)
504 ctx->offset = 0;
505 }
506 } else {
507 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
508 DMA_TO_DEVICE);
509 }
438 510
439 return 0; 511 return 0;
440} 512}
@@ -454,14 +526,7 @@ static void omap_sham_cleanup(struct ahash_request *req)
454 spin_unlock_irqrestore(&dd->lock, flags); 526 spin_unlock_irqrestore(&dd->lock, flags);
455 527
456 if (ctx->digcnt) 528 if (ctx->digcnt)
457 clk_disable(dd->iclk); 529 omap_sham_copy_ready_hash(req);
458
459 if (ctx->dma_addr)
460 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
461 DMA_TO_DEVICE);
462
463 if (ctx->buffer)
464 free_page((unsigned long)ctx->buffer);
465 530
466 dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt); 531 dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt);
467} 532}
@@ -489,8 +554,6 @@ static int omap_sham_init(struct ahash_request *req)
489 554
490 ctx->flags = 0; 555 ctx->flags = 0;
491 556
492 ctx->flags |= FLAGS_FIRST;
493
494 dev_dbg(dd->dev, "init: digest size: %d\n", 557 dev_dbg(dd->dev, "init: digest size: %d\n",
495 crypto_ahash_digestsize(tfm)); 558 crypto_ahash_digestsize(tfm));
496 559
@@ -499,21 +562,7 @@ static int omap_sham_init(struct ahash_request *req)
499 562
500 ctx->bufcnt = 0; 563 ctx->bufcnt = 0;
501 ctx->digcnt = 0; 564 ctx->digcnt = 0;
502 565 ctx->buflen = BUFLEN;
503 ctx->buflen = PAGE_SIZE;
504 ctx->buffer = (void *)__get_free_page(
505 (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
506 GFP_KERNEL : GFP_ATOMIC);
507 if (!ctx->buffer)
508 return -ENOMEM;
509
510 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,
511 DMA_TO_DEVICE);
512 if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
513 dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen);
514 free_page((unsigned long)ctx->buffer);
515 return -EINVAL;
516 }
517 566
518 if (tctx->flags & FLAGS_HMAC) { 567 if (tctx->flags & FLAGS_HMAC) {
519 struct omap_sham_hmac_ctx *bctx = tctx->base; 568 struct omap_sham_hmac_ctx *bctx = tctx->base;
@@ -538,10 +587,8 @@ static int omap_sham_update_req(struct omap_sham_dev *dd)
538 587
539 if (ctx->flags & FLAGS_CPU) 588 if (ctx->flags & FLAGS_CPU)
540 err = omap_sham_update_cpu(dd); 589 err = omap_sham_update_cpu(dd);
541 else if (ctx->flags & FLAGS_FAST)
542 err = omap_sham_update_dma_fast(dd);
543 else 590 else
544 err = omap_sham_update_dma_slow(dd); 591 err = omap_sham_update_dma_start(dd);
545 592
546 /* wait for dma completion before can take more data */ 593 /* wait for dma completion before can take more data */
547 dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt); 594 dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt);
@@ -560,15 +607,12 @@ static int omap_sham_final_req(struct omap_sham_dev *dd)
560 use_dma = 0; 607 use_dma = 0;
561 608
562 if (use_dma) 609 if (use_dma)
563 err = omap_sham_xmit_dma(dd, ctx->dma_addr, ctx->bufcnt, 1); 610 err = omap_sham_xmit_dma_map(dd, ctx, ctx->bufcnt, 1);
564 else 611 else
565 err = omap_sham_xmit_cpu(dd, ctx->buffer, ctx->bufcnt, 1); 612 err = omap_sham_xmit_cpu(dd, ctx->buffer, ctx->bufcnt, 1);
566 613
567 ctx->bufcnt = 0; 614 ctx->bufcnt = 0;
568 615
569 if (err != -EINPROGRESS)
570 omap_sham_cleanup(req);
571
572 dev_dbg(dd->dev, "final_req: err: %d\n", err); 616 dev_dbg(dd->dev, "final_req: err: %d\n", err);
573 617
574 return err; 618 return err;
@@ -576,6 +620,7 @@ static int omap_sham_final_req(struct omap_sham_dev *dd)
576 620
577static int omap_sham_finish_req_hmac(struct ahash_request *req) 621static int omap_sham_finish_req_hmac(struct ahash_request *req)
578{ 622{
623 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
579 struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 624 struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
580 struct omap_sham_hmac_ctx *bctx = tctx->base; 625 struct omap_sham_hmac_ctx *bctx = tctx->base;
581 int bs = crypto_shash_blocksize(bctx->shash); 626 int bs = crypto_shash_blocksize(bctx->shash);
@@ -590,48 +635,56 @@ static int omap_sham_finish_req_hmac(struct ahash_request *req)
590 635
591 return crypto_shash_init(&desc.shash) ?: 636 return crypto_shash_init(&desc.shash) ?:
592 crypto_shash_update(&desc.shash, bctx->opad, bs) ?: 637 crypto_shash_update(&desc.shash, bctx->opad, bs) ?:
593 crypto_shash_finup(&desc.shash, req->result, ds, req->result); 638 crypto_shash_finup(&desc.shash, ctx->digest, ds, ctx->digest);
594} 639}
595 640
596static void omap_sham_finish_req(struct ahash_request *req, int err) 641static void omap_sham_finish_req(struct ahash_request *req, int err)
597{ 642{
598 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 643 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
644 struct omap_sham_dev *dd = ctx->dd;
599 645
600 if (!err) { 646 if (!err) {
601 omap_sham_copy_hash(ctx->dd->req, 1); 647 omap_sham_copy_hash(ctx->dd->req, 1);
602 if (ctx->flags & FLAGS_HMAC) 648 if (ctx->flags & FLAGS_HMAC)
603 err = omap_sham_finish_req_hmac(req); 649 err = omap_sham_finish_req_hmac(req);
650 } else {
651 ctx->flags |= FLAGS_ERROR;
604 } 652 }
605 653
606 if (ctx->flags & FLAGS_FINAL) 654 if ((ctx->flags & FLAGS_FINAL) || err)
607 omap_sham_cleanup(req); 655 omap_sham_cleanup(req);
608 656
609 clear_bit(FLAGS_BUSY, &ctx->dd->flags); 657 clk_disable(dd->iclk);
658 dd->flags &= ~FLAGS_BUSY;
610 659
611 if (req->base.complete) 660 if (req->base.complete)
612 req->base.complete(&req->base, err); 661 req->base.complete(&req->base, err);
613} 662}
614 663
615static int omap_sham_handle_queue(struct omap_sham_dev *dd) 664static int omap_sham_handle_queue(struct omap_sham_dev *dd,
665 struct ahash_request *req)
616{ 666{
617 struct crypto_async_request *async_req, *backlog; 667 struct crypto_async_request *async_req, *backlog;
618 struct omap_sham_reqctx *ctx; 668 struct omap_sham_reqctx *ctx;
619 struct ahash_request *req, *prev_req; 669 struct ahash_request *prev_req;
620 unsigned long flags; 670 unsigned long flags;
621 int err = 0; 671 int err = 0, ret = 0;
622
623 if (test_and_set_bit(FLAGS_BUSY, &dd->flags))
624 return 0;
625 672
626 spin_lock_irqsave(&dd->lock, flags); 673 spin_lock_irqsave(&dd->lock, flags);
674 if (req)
675 ret = ahash_enqueue_request(&dd->queue, req);
676 if (dd->flags & FLAGS_BUSY) {
677 spin_unlock_irqrestore(&dd->lock, flags);
678 return ret;
679 }
627 backlog = crypto_get_backlog(&dd->queue); 680 backlog = crypto_get_backlog(&dd->queue);
628 async_req = crypto_dequeue_request(&dd->queue); 681 async_req = crypto_dequeue_request(&dd->queue);
629 if (!async_req) 682 if (async_req)
630 clear_bit(FLAGS_BUSY, &dd->flags); 683 dd->flags |= FLAGS_BUSY;
631 spin_unlock_irqrestore(&dd->lock, flags); 684 spin_unlock_irqrestore(&dd->lock, flags);
632 685
633 if (!async_req) 686 if (!async_req)
634 return 0; 687 return ret;
635 688
636 if (backlog) 689 if (backlog)
637 backlog->complete(backlog, -EINPROGRESS); 690 backlog->complete(backlog, -EINPROGRESS);
@@ -646,7 +699,22 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd)
646 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", 699 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
647 ctx->op, req->nbytes); 700 ctx->op, req->nbytes);
648 701
649 if (req != prev_req && ctx->digcnt) 702
703 err = omap_sham_hw_init(dd);
704 if (err)
705 goto err1;
706
707 omap_set_dma_dest_params(dd->dma_lch, 0,
708 OMAP_DMA_AMODE_CONSTANT,
709 dd->phys_base + SHA_REG_DIN(0), 0, 16);
710
711 omap_set_dma_dest_burst_mode(dd->dma_lch,
712 OMAP_DMA_DATA_BURST_16);
713
714 omap_set_dma_src_burst_mode(dd->dma_lch,
715 OMAP_DMA_DATA_BURST_4);
716
717 if (ctx->digcnt)
650 /* request has changed - restore hash */ 718 /* request has changed - restore hash */
651 omap_sham_copy_hash(req, 0); 719 omap_sham_copy_hash(req, 0);
652 720
@@ -658,7 +726,7 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd)
658 } else if (ctx->op == OP_FINAL) { 726 } else if (ctx->op == OP_FINAL) {
659 err = omap_sham_final_req(dd); 727 err = omap_sham_final_req(dd);
660 } 728 }
661 729err1:
662 if (err != -EINPROGRESS) { 730 if (err != -EINPROGRESS) {
663 /* done_task will not finish it, so do it here */ 731 /* done_task will not finish it, so do it here */
664 omap_sham_finish_req(req, err); 732 omap_sham_finish_req(req, err);
@@ -667,7 +735,7 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd)
667 735
668 dev_dbg(dd->dev, "exit, err: %d\n", err); 736 dev_dbg(dd->dev, "exit, err: %d\n", err);
669 737
670 return err; 738 return ret;
671} 739}
672 740
673static int omap_sham_enqueue(struct ahash_request *req, unsigned int op) 741static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
@@ -675,18 +743,10 @@ static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
675 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 743 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
676 struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); 744 struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
677 struct omap_sham_dev *dd = tctx->dd; 745 struct omap_sham_dev *dd = tctx->dd;
678 unsigned long flags;
679 int err;
680 746
681 ctx->op = op; 747 ctx->op = op;
682 748
683 spin_lock_irqsave(&dd->lock, flags); 749 return omap_sham_handle_queue(dd, req);
684 err = ahash_enqueue_request(&dd->queue, req);
685 spin_unlock_irqrestore(&dd->lock, flags);
686
687 omap_sham_handle_queue(dd);
688
689 return err;
690} 750}
691 751
692static int omap_sham_update(struct ahash_request *req) 752static int omap_sham_update(struct ahash_request *req)
@@ -709,21 +769,13 @@ static int omap_sham_update(struct ahash_request *req)
709 */ 769 */
710 omap_sham_append_sg(ctx); 770 omap_sham_append_sg(ctx);
711 return 0; 771 return 0;
712 } else if (ctx->bufcnt + ctx->total <= 64) { 772 } else if (ctx->bufcnt + ctx->total <= SHA1_MD5_BLOCK_SIZE) {
773 /*
774 * faster to use CPU for short transfers
775 */
713 ctx->flags |= FLAGS_CPU; 776 ctx->flags |= FLAGS_CPU;
714 } else if (!ctx->bufcnt && sg_is_last(ctx->sg)) {
715 /* may be can use faster functions */
716 int aligned = IS_ALIGNED((u32)ctx->sg->offset,
717 sizeof(u32));
718
719 if (aligned && (ctx->flags & FLAGS_FIRST))
720 /* digest: first and final */
721 ctx->flags |= FLAGS_FAST;
722
723 ctx->flags &= ~FLAGS_FIRST;
724 } 777 }
725 } else if (ctx->bufcnt + ctx->total <= ctx->buflen) { 778 } else if (ctx->bufcnt + ctx->total < ctx->buflen) {
726 /* if not finaup -> not fast */
727 omap_sham_append_sg(ctx); 779 omap_sham_append_sg(ctx);
728 return 0; 780 return 0;
729 } 781 }
@@ -761,12 +813,14 @@ static int omap_sham_final(struct ahash_request *req)
761 813
762 ctx->flags |= FLAGS_FINUP; 814 ctx->flags |= FLAGS_FINUP;
763 815
764 /* OMAP HW accel works only with buffers >= 9 */ 816 if (!(ctx->flags & FLAGS_ERROR)) {
765 /* HMAC is always >= 9 because of ipad */ 817 /* OMAP HW accel works only with buffers >= 9 */
766 if ((ctx->digcnt + ctx->bufcnt) < 9) 818 /* HMAC is always >= 9 because of ipad */
767 err = omap_sham_final_shash(req); 819 if ((ctx->digcnt + ctx->bufcnt) < 9)
768 else if (ctx->bufcnt) 820 err = omap_sham_final_shash(req);
769 return omap_sham_enqueue(req, OP_FINAL); 821 else if (ctx->bufcnt)
822 return omap_sham_enqueue(req, OP_FINAL);
823 }
770 824
771 omap_sham_cleanup(req); 825 omap_sham_cleanup(req);
772 826
@@ -836,6 +890,8 @@ static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
836 struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm); 890 struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
837 const char *alg_name = crypto_tfm_alg_name(tfm); 891 const char *alg_name = crypto_tfm_alg_name(tfm);
838 892
893 pr_info("enter\n");
894
839 /* Allocate a fallback and abort if it failed. */ 895 /* Allocate a fallback and abort if it failed. */
840 tctx->fallback = crypto_alloc_shash(alg_name, 0, 896 tctx->fallback = crypto_alloc_shash(alg_name, 0,
841 CRYPTO_ALG_NEED_FALLBACK); 897 CRYPTO_ALG_NEED_FALLBACK);
@@ -846,7 +902,7 @@ static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
846 } 902 }
847 903
848 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 904 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
849 sizeof(struct omap_sham_reqctx)); 905 sizeof(struct omap_sham_reqctx) + BUFLEN);
850 906
851 if (alg_base) { 907 if (alg_base) {
852 struct omap_sham_hmac_ctx *bctx = tctx->base; 908 struct omap_sham_hmac_ctx *bctx = tctx->base;
@@ -932,7 +988,7 @@ static struct ahash_alg algs[] = {
932 CRYPTO_ALG_NEED_FALLBACK, 988 CRYPTO_ALG_NEED_FALLBACK,
933 .cra_blocksize = SHA1_BLOCK_SIZE, 989 .cra_blocksize = SHA1_BLOCK_SIZE,
934 .cra_ctxsize = sizeof(struct omap_sham_ctx), 990 .cra_ctxsize = sizeof(struct omap_sham_ctx),
935 .cra_alignmask = 0, 991 .cra_alignmask = OMAP_ALIGN_MASK,
936 .cra_module = THIS_MODULE, 992 .cra_module = THIS_MODULE,
937 .cra_init = omap_sham_cra_init, 993 .cra_init = omap_sham_cra_init,
938 .cra_exit = omap_sham_cra_exit, 994 .cra_exit = omap_sham_cra_exit,
@@ -956,7 +1012,7 @@ static struct ahash_alg algs[] = {
956 .cra_blocksize = SHA1_BLOCK_SIZE, 1012 .cra_blocksize = SHA1_BLOCK_SIZE,
957 .cra_ctxsize = sizeof(struct omap_sham_ctx) + 1013 .cra_ctxsize = sizeof(struct omap_sham_ctx) +
958 sizeof(struct omap_sham_hmac_ctx), 1014 sizeof(struct omap_sham_hmac_ctx),
959 .cra_alignmask = 0, 1015 .cra_alignmask = OMAP_ALIGN_MASK,
960 .cra_module = THIS_MODULE, 1016 .cra_module = THIS_MODULE,
961 .cra_init = omap_sham_cra_sha1_init, 1017 .cra_init = omap_sham_cra_sha1_init,
962 .cra_exit = omap_sham_cra_exit, 1018 .cra_exit = omap_sham_cra_exit,
@@ -980,7 +1036,7 @@ static struct ahash_alg algs[] = {
980 .cra_blocksize = SHA1_BLOCK_SIZE, 1036 .cra_blocksize = SHA1_BLOCK_SIZE,
981 .cra_ctxsize = sizeof(struct omap_sham_ctx) + 1037 .cra_ctxsize = sizeof(struct omap_sham_ctx) +
982 sizeof(struct omap_sham_hmac_ctx), 1038 sizeof(struct omap_sham_hmac_ctx),
983 .cra_alignmask = 0, 1039 .cra_alignmask = OMAP_ALIGN_MASK,
984 .cra_module = THIS_MODULE, 1040 .cra_module = THIS_MODULE,
985 .cra_init = omap_sham_cra_md5_init, 1041 .cra_init = omap_sham_cra_md5_init,
986 .cra_exit = omap_sham_cra_exit, 1042 .cra_exit = omap_sham_cra_exit,
@@ -993,7 +1049,7 @@ static void omap_sham_done_task(unsigned long data)
993 struct omap_sham_dev *dd = (struct omap_sham_dev *)data; 1049 struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
994 struct ahash_request *req = dd->req; 1050 struct ahash_request *req = dd->req;
995 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 1051 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
996 int ready = 1; 1052 int ready = 0, err = 0;
997 1053
998 if (ctx->flags & FLAGS_OUTPUT_READY) { 1054 if (ctx->flags & FLAGS_OUTPUT_READY) {
999 ctx->flags &= ~FLAGS_OUTPUT_READY; 1055 ctx->flags &= ~FLAGS_OUTPUT_READY;
@@ -1003,15 +1059,18 @@ static void omap_sham_done_task(unsigned long data)
1003 if (dd->flags & FLAGS_DMA_ACTIVE) { 1059 if (dd->flags & FLAGS_DMA_ACTIVE) {
1004 dd->flags &= ~FLAGS_DMA_ACTIVE; 1060 dd->flags &= ~FLAGS_DMA_ACTIVE;
1005 omap_sham_update_dma_stop(dd); 1061 omap_sham_update_dma_stop(dd);
1006 omap_sham_update_dma_slow(dd); 1062 if (!dd->err)
1063 err = omap_sham_update_dma_start(dd);
1007 } 1064 }
1008 1065
1009 if (ready && !(dd->flags & FLAGS_DMA_ACTIVE)) { 1066 err = dd->err ? : err;
1010 dev_dbg(dd->dev, "update done\n"); 1067
1068 if (err != -EINPROGRESS && (ready || err)) {
1069 dev_dbg(dd->dev, "update done: err: %d\n", err);
1011 /* finish curent request */ 1070 /* finish curent request */
1012 omap_sham_finish_req(req, 0); 1071 omap_sham_finish_req(req, err);
1013 /* start new request */ 1072 /* start new request */
1014 omap_sham_handle_queue(dd); 1073 omap_sham_handle_queue(dd, NULL);
1015 } 1074 }
1016} 1075}
1017 1076
@@ -1019,7 +1078,7 @@ static void omap_sham_queue_task(unsigned long data)
1019{ 1078{
1020 struct omap_sham_dev *dd = (struct omap_sham_dev *)data; 1079 struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
1021 1080
1022 omap_sham_handle_queue(dd); 1081 omap_sham_handle_queue(dd, NULL);
1023} 1082}
1024 1083
1025static irqreturn_t omap_sham_irq(int irq, void *dev_id) 1084static irqreturn_t omap_sham_irq(int irq, void *dev_id)
@@ -1041,6 +1100,7 @@ static irqreturn_t omap_sham_irq(int irq, void *dev_id)
1041 omap_sham_read(dd, SHA_REG_CTRL); 1100 omap_sham_read(dd, SHA_REG_CTRL);
1042 1101
1043 ctx->flags |= FLAGS_OUTPUT_READY; 1102 ctx->flags |= FLAGS_OUTPUT_READY;
1103 dd->err = 0;
1044 tasklet_schedule(&dd->done_task); 1104 tasklet_schedule(&dd->done_task);
1045 1105
1046 return IRQ_HANDLED; 1106 return IRQ_HANDLED;
@@ -1050,8 +1110,13 @@ static void omap_sham_dma_callback(int lch, u16 ch_status, void *data)
1050{ 1110{
1051 struct omap_sham_dev *dd = data; 1111 struct omap_sham_dev *dd = data;
1052 1112
1053 if (likely(lch == dd->dma_lch)) 1113 if (ch_status != OMAP_DMA_BLOCK_IRQ) {
1054 tasklet_schedule(&dd->done_task); 1114 pr_err("omap-sham DMA error status: 0x%hx\n", ch_status);
1115 dd->err = -EIO;
1116 dd->flags &= ~FLAGS_INIT; /* request to re-initialize */
1117 }
1118
1119 tasklet_schedule(&dd->done_task);
1055} 1120}
1056 1121
1057static int omap_sham_dma_init(struct omap_sham_dev *dd) 1122static int omap_sham_dma_init(struct omap_sham_dev *dd)
@@ -1066,15 +1131,6 @@ static int omap_sham_dma_init(struct omap_sham_dev *dd)
1066 dev_err(dd->dev, "Unable to request DMA channel\n"); 1131 dev_err(dd->dev, "Unable to request DMA channel\n");
1067 return err; 1132 return err;
1068 } 1133 }
1069 omap_set_dma_dest_params(dd->dma_lch, 0,
1070 OMAP_DMA_AMODE_CONSTANT,
1071 dd->phys_base + SHA_REG_DIN(0), 0, 16);
1072
1073 omap_set_dma_dest_burst_mode(dd->dma_lch,
1074 OMAP_DMA_DATA_BURST_16);
1075
1076 omap_set_dma_src_burst_mode(dd->dma_lch,
1077 OMAP_DMA_DATA_BURST_4);
1078 1134
1079 return 0; 1135 return 0;
1080} 1136}
diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c
index 8a515baa38f7..db33d300aa23 100644
--- a/drivers/crypto/padlock-aes.c
+++ b/drivers/crypto/padlock-aes.c
@@ -9,6 +9,7 @@
9 9
10#include <crypto/algapi.h> 10#include <crypto/algapi.h>
11#include <crypto/aes.h> 11#include <crypto/aes.h>
12#include <crypto/padlock.h>
12#include <linux/module.h> 13#include <linux/module.h>
13#include <linux/init.h> 14#include <linux/init.h>
14#include <linux/types.h> 15#include <linux/types.h>
@@ -21,7 +22,6 @@
21#include <asm/byteorder.h> 22#include <asm/byteorder.h>
22#include <asm/processor.h> 23#include <asm/processor.h>
23#include <asm/i387.h> 24#include <asm/i387.h>
24#include "padlock.h"
25 25
26/* 26/*
27 * Number of data blocks actually fetched for each xcrypt insn. 27 * Number of data blocks actually fetched for each xcrypt insn.
diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c
index d3a27e0119bc..adf075b6b9a8 100644
--- a/drivers/crypto/padlock-sha.c
+++ b/drivers/crypto/padlock-sha.c
@@ -13,6 +13,7 @@
13 */ 13 */
14 14
15#include <crypto/internal/hash.h> 15#include <crypto/internal/hash.h>
16#include <crypto/padlock.h>
16#include <crypto/sha.h> 17#include <crypto/sha.h>
17#include <linux/err.h> 18#include <linux/err.h>
18#include <linux/module.h> 19#include <linux/module.h>
@@ -22,13 +23,6 @@
22#include <linux/kernel.h> 23#include <linux/kernel.h>
23#include <linux/scatterlist.h> 24#include <linux/scatterlist.h>
24#include <asm/i387.h> 25#include <asm/i387.h>
25#include "padlock.h"
26
27#ifdef CONFIG_64BIT
28#define STACK_ALIGN 16
29#else
30#define STACK_ALIGN 4
31#endif
32 26
33struct padlock_sha_desc { 27struct padlock_sha_desc {
34 struct shash_desc fallback; 28 struct shash_desc fallback;
diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h
new file mode 100644
index 000000000000..c5813c87de06
--- /dev/null
+++ b/include/crypto/if_alg.h
@@ -0,0 +1,92 @@
1/*
2 * if_alg: User-space algorithm interface
3 *
4 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the Free
8 * Software Foundation; either version 2 of the License, or (at your option)
9 * any later version.
10 *
11 */
12
13#ifndef _CRYPTO_IF_ALG_H
14#define _CRYPTO_IF_ALG_H
15
16#include <linux/compiler.h>
17#include <linux/completion.h>
18#include <linux/if_alg.h>
19#include <linux/types.h>
20#include <net/sock.h>
21
22#define ALG_MAX_PAGES 16
23
24struct crypto_async_request;
25
26struct alg_sock {
27 /* struct sock must be the first member of struct alg_sock */
28 struct sock sk;
29
30 struct sock *parent;
31
32 const struct af_alg_type *type;
33 void *private;
34};
35
36struct af_alg_completion {
37 struct completion completion;
38 int err;
39};
40
41struct af_alg_control {
42 struct af_alg_iv *iv;
43 int op;
44};
45
46struct af_alg_type {
47 void *(*bind)(const char *name, u32 type, u32 mask);
48 void (*release)(void *private);
49 int (*setkey)(void *private, const u8 *key, unsigned int keylen);
50 int (*accept)(void *private, struct sock *sk);
51
52 struct proto_ops *ops;
53 struct module *owner;
54 char name[14];
55};
56
57struct af_alg_sgl {
58 struct scatterlist sg[ALG_MAX_PAGES];
59 struct page *pages[ALG_MAX_PAGES];
60};
61
62int af_alg_register_type(const struct af_alg_type *type);
63int af_alg_unregister_type(const struct af_alg_type *type);
64
65int af_alg_release(struct socket *sock);
66int af_alg_accept(struct sock *sk, struct socket *newsock);
67
68int af_alg_make_sg(struct af_alg_sgl *sgl, void __user *addr, int len,
69 int write);
70void af_alg_free_sg(struct af_alg_sgl *sgl);
71
72int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con);
73
74int af_alg_wait_for_completion(int err, struct af_alg_completion *completion);
75void af_alg_complete(struct crypto_async_request *req, int err);
76
77static inline struct alg_sock *alg_sk(struct sock *sk)
78{
79 return (struct alg_sock *)sk;
80}
81
82static inline void af_alg_release_parent(struct sock *sk)
83{
84 sock_put(alg_sk(sk)->parent);
85}
86
87static inline void af_alg_init_completion(struct af_alg_completion *completion)
88{
89 init_completion(&completion->completion);
90}
91
92#endif /* _CRYPTO_IF_ALG_H */
diff --git a/drivers/crypto/padlock.h b/include/crypto/padlock.h
index b728e4518bd1..d2cfa2ef49e8 100644
--- a/drivers/crypto/padlock.h
+++ b/include/crypto/padlock.h
@@ -15,9 +15,15 @@
15 15
16#define PADLOCK_ALIGNMENT 16 16#define PADLOCK_ALIGNMENT 16
17 17
18#define PFX "padlock: " 18#define PFX KBUILD_MODNAME ": "
19 19
20#define PADLOCK_CRA_PRIORITY 300 20#define PADLOCK_CRA_PRIORITY 300
21#define PADLOCK_COMPOSITE_PRIORITY 400 21#define PADLOCK_COMPOSITE_PRIORITY 400
22 22
23#ifdef CONFIG_64BIT
24#define STACK_ALIGN 16
25#else
26#define STACK_ALIGN 4
27#endif
28
23#endif /* _CRYPTO_PADLOCK_H */ 29#endif /* _CRYPTO_PADLOCK_H */
diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h
index 833d208c25d6..4fd95a323beb 100644
--- a/include/crypto/scatterwalk.h
+++ b/include/crypto/scatterwalk.h
@@ -68,6 +68,21 @@ static inline struct scatterlist *scatterwalk_sg_next(struct scatterlist *sg)
68 return (++sg)->length ? sg : (void *)sg_page(sg); 68 return (++sg)->length ? sg : (void *)sg_page(sg);
69} 69}
70 70
71static inline void scatterwalk_crypto_chain(struct scatterlist *head,
72 struct scatterlist *sg,
73 int chain, int num)
74{
75 if (chain) {
76 head->length += sg->length;
77 sg = scatterwalk_sg_next(sg);
78 }
79
80 if (sg)
81 scatterwalk_sg_chain(head, num, sg);
82 else
83 sg_mark_end(head);
84}
85
71static inline unsigned long scatterwalk_samebuf(struct scatter_walk *walk_in, 86static inline unsigned long scatterwalk_samebuf(struct scatter_walk *walk_in,
72 struct scatter_walk *walk_out) 87 struct scatter_walk *walk_out)
73{ 88{
diff --git a/include/linux/Kbuild b/include/linux/Kbuild
index d1580c17cab3..2296d8b1931f 100644
--- a/include/linux/Kbuild
+++ b/include/linux/Kbuild
@@ -158,6 +158,7 @@ header-y += icmpv6.h
158header-y += if.h 158header-y += if.h
159header-y += if_addr.h 159header-y += if_addr.h
160header-y += if_addrlabel.h 160header-y += if_addrlabel.h
161header-y += if_alg.h
161header-y += if_arcnet.h 162header-y += if_arcnet.h
162header-y += if_arp.h 163header-y += if_arp.h
163header-y += if_bonding.h 164header-y += if_bonding.h
diff --git a/include/linux/if_alg.h b/include/linux/if_alg.h
new file mode 100644
index 000000000000..0f9acce5b1ff
--- /dev/null
+++ b/include/linux/if_alg.h
@@ -0,0 +1,40 @@
1/*
2 * if_alg: User-space algorithm interface
3 *
4 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the Free
8 * Software Foundation; either version 2 of the License, or (at your option)
9 * any later version.
10 *
11 */
12
13#ifndef _LINUX_IF_ALG_H
14#define _LINUX_IF_ALG_H
15
16#include <linux/types.h>
17
18struct sockaddr_alg {
19 __u16 salg_family;
20 __u8 salg_type[14];
21 __u32 salg_feat;
22 __u32 salg_mask;
23 __u8 salg_name[64];
24};
25
26struct af_alg_iv {
27 __u32 ivlen;
28 __u8 iv[0];
29};
30
31/* Socket options */
32#define ALG_SET_KEY 1
33#define ALG_SET_IV 2
34#define ALG_SET_OP 3
35
36/* Operations */
37#define ALG_OP_DECRYPT 0
38#define ALG_OP_ENCRYPT 1
39
40#endif /* _LINUX_IF_ALG_H */
diff --git a/include/linux/socket.h b/include/linux/socket.h
index 5f65f14c4f44..edbb1d07ddf4 100644
--- a/include/linux/socket.h
+++ b/include/linux/socket.h
@@ -191,7 +191,8 @@ struct ucred {
191#define AF_PHONET 35 /* Phonet sockets */ 191#define AF_PHONET 35 /* Phonet sockets */
192#define AF_IEEE802154 36 /* IEEE802154 sockets */ 192#define AF_IEEE802154 36 /* IEEE802154 sockets */
193#define AF_CAIF 37 /* CAIF sockets */ 193#define AF_CAIF 37 /* CAIF sockets */
194#define AF_MAX 38 /* For now.. */ 194#define AF_ALG 38 /* Algorithm sockets */
195#define AF_MAX 39 /* For now.. */
195 196
196/* Protocol families, same as address families. */ 197/* Protocol families, same as address families. */
197#define PF_UNSPEC AF_UNSPEC 198#define PF_UNSPEC AF_UNSPEC
@@ -232,6 +233,7 @@ struct ucred {
232#define PF_PHONET AF_PHONET 233#define PF_PHONET AF_PHONET
233#define PF_IEEE802154 AF_IEEE802154 234#define PF_IEEE802154 AF_IEEE802154
234#define PF_CAIF AF_CAIF 235#define PF_CAIF AF_CAIF
236#define PF_ALG AF_ALG
235#define PF_MAX AF_MAX 237#define PF_MAX AF_MAX
236 238
237/* Maximum queue length specifiable by listen. */ 239/* Maximum queue length specifiable by listen. */
@@ -305,6 +307,7 @@ struct ucred {
305#define SOL_RDS 276 307#define SOL_RDS 276
306#define SOL_IUCV 277 308#define SOL_IUCV 277
307#define SOL_CAIF 278 309#define SOL_CAIF 278
310#define SOL_ALG 279
308 311
309/* IPX options */ 312/* IPX options */
310#define IPX_TYPE 1 313#define IPX_TYPE 1
diff --git a/net/core/sock.c b/net/core/sock.c
index a658aeb6d554..7dfed792434d 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -157,7 +157,7 @@ static const char *const af_family_key_strings[AF_MAX+1] = {
157 "sk_lock-27" , "sk_lock-28" , "sk_lock-AF_CAN" , 157 "sk_lock-27" , "sk_lock-28" , "sk_lock-AF_CAN" ,
158 "sk_lock-AF_TIPC" , "sk_lock-AF_BLUETOOTH", "sk_lock-IUCV" , 158 "sk_lock-AF_TIPC" , "sk_lock-AF_BLUETOOTH", "sk_lock-IUCV" ,
159 "sk_lock-AF_RXRPC" , "sk_lock-AF_ISDN" , "sk_lock-AF_PHONET" , 159 "sk_lock-AF_RXRPC" , "sk_lock-AF_ISDN" , "sk_lock-AF_PHONET" ,
160 "sk_lock-AF_IEEE802154", "sk_lock-AF_CAIF" , 160 "sk_lock-AF_IEEE802154", "sk_lock-AF_CAIF" , "sk_lock-AF_ALG" ,
161 "sk_lock-AF_MAX" 161 "sk_lock-AF_MAX"
162}; 162};
163static const char *const af_family_slock_key_strings[AF_MAX+1] = { 163static const char *const af_family_slock_key_strings[AF_MAX+1] = {
@@ -173,7 +173,7 @@ static const char *const af_family_slock_key_strings[AF_MAX+1] = {
173 "slock-27" , "slock-28" , "slock-AF_CAN" , 173 "slock-27" , "slock-28" , "slock-AF_CAN" ,
174 "slock-AF_TIPC" , "slock-AF_BLUETOOTH", "slock-AF_IUCV" , 174 "slock-AF_TIPC" , "slock-AF_BLUETOOTH", "slock-AF_IUCV" ,
175 "slock-AF_RXRPC" , "slock-AF_ISDN" , "slock-AF_PHONET" , 175 "slock-AF_RXRPC" , "slock-AF_ISDN" , "slock-AF_PHONET" ,
176 "slock-AF_IEEE802154", "slock-AF_CAIF" , 176 "slock-AF_IEEE802154", "slock-AF_CAIF" , "slock-AF_ALG" ,
177 "slock-AF_MAX" 177 "slock-AF_MAX"
178}; 178};
179static const char *const af_family_clock_key_strings[AF_MAX+1] = { 179static const char *const af_family_clock_key_strings[AF_MAX+1] = {
@@ -189,7 +189,7 @@ static const char *const af_family_clock_key_strings[AF_MAX+1] = {
189 "clock-27" , "clock-28" , "clock-AF_CAN" , 189 "clock-27" , "clock-28" , "clock-AF_CAN" ,
190 "clock-AF_TIPC" , "clock-AF_BLUETOOTH", "clock-AF_IUCV" , 190 "clock-AF_TIPC" , "clock-AF_BLUETOOTH", "clock-AF_IUCV" ,
191 "clock-AF_RXRPC" , "clock-AF_ISDN" , "clock-AF_PHONET" , 191 "clock-AF_RXRPC" , "clock-AF_ISDN" , "clock-AF_PHONET" ,
192 "clock-AF_IEEE802154", "clock-AF_CAIF" , 192 "clock-AF_IEEE802154", "clock-AF_CAIF" , "clock-AF_ALG" ,
193 "clock-AF_MAX" 193 "clock-AF_MAX"
194}; 194};
195 195
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-27 17:34:07 -0500