aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/crypto/Makefile3
-rw-r--r--arch/x86/crypto/aesni-intel_asm.S517
-rw-r--r--arch/x86/crypto/ghash-clmulni-intel_asm.S157
-rw-r--r--arch/x86/crypto/ghash-clmulni-intel_glue.c333
-rw-r--r--arch/x86/include/asm/cpufeature.h1
-rw-r--r--arch/x86/include/asm/i387.h7
-rw-r--r--arch/x86/include/asm/inst.h150
-rw-r--r--crypto/Kconfig9
-rw-r--r--crypto/ansi_cprng.c82
-rw-r--r--crypto/cryptd.c7
-rw-r--r--crypto/digest.c240
-rw-r--r--crypto/hash.c183
-rw-r--r--crypto/proc.c7
-rw-r--r--crypto/testmgr.c11
-rw-r--r--crypto/testmgr.h15
-rw-r--r--drivers/char/hw_random/core.c108
-rw-r--r--drivers/char/hw_random/virtio-rng.c78
-rw-r--r--include/crypto/algapi.h1
-rw-r--r--include/crypto/cryptd.h1
-rw-r--r--include/linux/crypto.h27
-rw-r--r--include/linux/hw_random.h7
21 files changed, 1033 insertions, 911 deletions
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index cfb0010fa940..1a58ad89fdf7 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o
12obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o 12obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o
13obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o 13obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o
14obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o 14obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
15obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
15 16
16obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o 17obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
17 18
@@ -24,3 +25,5 @@ twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o
24salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o 25salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o
25 26
26aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o 27aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o
28
29ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index eb0566e83319..20bb0e1ac681 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -16,6 +16,7 @@
16 */ 16 */
17 17
18#include <linux/linkage.h> 18#include <linux/linkage.h>
19#include <asm/inst.h>
19 20
20.text 21.text
21 22
@@ -122,103 +123,72 @@ ENTRY(aesni_set_key)
122 movups 0x10(%rsi), %xmm2 # other user key 123 movups 0x10(%rsi), %xmm2 # other user key
123 movaps %xmm2, (%rcx) 124 movaps %xmm2, (%rcx)
124 add $0x10, %rcx 125 add $0x10, %rcx
125 # aeskeygenassist $0x1, %xmm2, %xmm1 # round 1 126 AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1
126 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x01
127 call _key_expansion_256a 127 call _key_expansion_256a
128 # aeskeygenassist $0x1, %xmm0, %xmm1 128 AESKEYGENASSIST 0x1 %xmm0 %xmm1
129 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x01
130 call _key_expansion_256b 129 call _key_expansion_256b
131 # aeskeygenassist $0x2, %xmm2, %xmm1 # round 2 130 AESKEYGENASSIST 0x2 %xmm2 %xmm1 # round 2
132 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x02
133 call _key_expansion_256a 131 call _key_expansion_256a
134 # aeskeygenassist $0x2, %xmm0, %xmm1 132 AESKEYGENASSIST 0x2 %xmm0 %xmm1
135 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x02
136 call _key_expansion_256b 133 call _key_expansion_256b
137 # aeskeygenassist $0x4, %xmm2, %xmm1 # round 3 134 AESKEYGENASSIST 0x4 %xmm2 %xmm1 # round 3
138 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x04
139 call _key_expansion_256a 135 call _key_expansion_256a
140 # aeskeygenassist $0x4, %xmm0, %xmm1 136 AESKEYGENASSIST 0x4 %xmm0 %xmm1
141 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x04
142 call _key_expansion_256b 137 call _key_expansion_256b
143 # aeskeygenassist $0x8, %xmm2, %xmm1 # round 4 138 AESKEYGENASSIST 0x8 %xmm2 %xmm1 # round 4
144 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x08
145 call _key_expansion_256a 139 call _key_expansion_256a
146 # aeskeygenassist $0x8, %xmm0, %xmm1 140 AESKEYGENASSIST 0x8 %xmm0 %xmm1
147 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x08
148 call _key_expansion_256b 141 call _key_expansion_256b
149 # aeskeygenassist $0x10, %xmm2, %xmm1 # round 5 142 AESKEYGENASSIST 0x10 %xmm2 %xmm1 # round 5
150 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x10
151 call _key_expansion_256a 143 call _key_expansion_256a
152 # aeskeygenassist $0x10, %xmm0, %xmm1 144 AESKEYGENASSIST 0x10 %xmm0 %xmm1
153 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x10
154 call _key_expansion_256b 145 call _key_expansion_256b
155 # aeskeygenassist $0x20, %xmm2, %xmm1 # round 6 146 AESKEYGENASSIST 0x20 %xmm2 %xmm1 # round 6
156 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x20
157 call _key_expansion_256a 147 call _key_expansion_256a
158 # aeskeygenassist $0x20, %xmm0, %xmm1 148 AESKEYGENASSIST 0x20 %xmm0 %xmm1
159 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x20
160 call _key_expansion_256b 149 call _key_expansion_256b
161 # aeskeygenassist $0x40, %xmm2, %xmm1 # round 7 150 AESKEYGENASSIST 0x40 %xmm2 %xmm1 # round 7
162 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x40
163 call _key_expansion_256a 151 call _key_expansion_256a
164 jmp .Ldec_key 152 jmp .Ldec_key
165.Lenc_key192: 153.Lenc_key192:
166 movq 0x10(%rsi), %xmm2 # other user key 154 movq 0x10(%rsi), %xmm2 # other user key
167 # aeskeygenassist $0x1, %xmm2, %xmm1 # round 1 155 AESKEYGENASSIST 0x1 %xmm2 %xmm1 # round 1
168 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x01
169 call _key_expansion_192a 156 call _key_expansion_192a
170 # aeskeygenassist $0x2, %xmm2, %xmm1 # round 2 157 AESKEYGENASSIST 0x2 %xmm2 %xmm1 # round 2
171 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x02
172 call _key_expansion_192b 158 call _key_expansion_192b
173 # aeskeygenassist $0x4, %xmm2, %xmm1 # round 3 159 AESKEYGENASSIST 0x4 %xmm2 %xmm1 # round 3
174 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x04
175 call _key_expansion_192a 160 call _key_expansion_192a
176 # aeskeygenassist $0x8, %xmm2, %xmm1 # round 4 161 AESKEYGENASSIST 0x8 %xmm2 %xmm1 # round 4
177 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x08
178 call _key_expansion_192b 162 call _key_expansion_192b
179 # aeskeygenassist $0x10, %xmm2, %xmm1 # round 5 163 AESKEYGENASSIST 0x10 %xmm2 %xmm1 # round 5
180 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x10
181 call _key_expansion_192a 164 call _key_expansion_192a
182 # aeskeygenassist $0x20, %xmm2, %xmm1 # round 6 165 AESKEYGENASSIST 0x20 %xmm2 %xmm1 # round 6
183 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x20
184 call _key_expansion_192b 166 call _key_expansion_192b
185 # aeskeygenassist $0x40, %xmm2, %xmm1 # round 7 167 AESKEYGENASSIST 0x40 %xmm2 %xmm1 # round 7
186 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x40
187 call _key_expansion_192a 168 call _key_expansion_192a
188 # aeskeygenassist $0x80, %xmm2, %xmm1 # round 8 169 AESKEYGENASSIST 0x80 %xmm2 %xmm1 # round 8
189 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x80
190 call _key_expansion_192b 170 call _key_expansion_192b
191 jmp .Ldec_key 171 jmp .Ldec_key
192.Lenc_key128: 172.Lenc_key128:
193 # aeskeygenassist $0x1, %xmm0, %xmm1 # round 1 173 AESKEYGENASSIST 0x1 %xmm0 %xmm1 # round 1
194 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x01
195 call _key_expansion_128 174 call _key_expansion_128
196 # aeskeygenassist $0x2, %xmm0, %xmm1 # round 2 175 AESKEYGENASSIST 0x2 %xmm0 %xmm1 # round 2
197 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x02
198 call _key_expansion_128 176 call _key_expansion_128
199 # aeskeygenassist $0x4, %xmm0, %xmm1 # round 3 177 AESKEYGENASSIST 0x4 %xmm0 %xmm1 # round 3
200 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x04
201 call _key_expansion_128 178 call _key_expansion_128
202 # aeskeygenassist $0x8, %xmm0, %xmm1 # round 4 179 AESKEYGENASSIST 0x8 %xmm0 %xmm1 # round 4
203 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x08
204 call _key_expansion_128 180 call _key_expansion_128
205 # aeskeygenassist $0x10, %xmm0, %xmm1 # round 5 181 AESKEYGENASSIST 0x10 %xmm0 %xmm1 # round 5
206 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x10
207 call _key_expansion_128 182 call _key_expansion_128
208 # aeskeygenassist $0x20, %xmm0, %xmm1 # round 6 183 AESKEYGENASSIST 0x20 %xmm0 %xmm1 # round 6
209 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x20
210 call _key_expansion_128 184 call _key_expansion_128
211 # aeskeygenassist $0x40, %xmm0, %xmm1 # round 7 185 AESKEYGENASSIST 0x40 %xmm0 %xmm1 # round 7
212 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x40
213 call _key_expansion_128 186 call _key_expansion_128
214 # aeskeygenassist $0x80, %xmm0, %xmm1 # round 8 187 AESKEYGENASSIST 0x80 %xmm0 %xmm1 # round 8
215 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x80
216 call _key_expansion_128 188 call _key_expansion_128
217 # aeskeygenassist $0x1b, %xmm0, %xmm1 # round 9 189 AESKEYGENASSIST 0x1b %xmm0 %xmm1 # round 9
218 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x1b
219 call _key_expansion_128 190 call _key_expansion_128
220 # aeskeygenassist $0x36, %xmm0, %xmm1 # round 10 191 AESKEYGENASSIST 0x36 %xmm0 %xmm1 # round 10
221 .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x36
222 call _key_expansion_128 192 call _key_expansion_128
223.Ldec_key: 193.Ldec_key:
224 sub $0x10, %rcx 194 sub $0x10, %rcx
@@ -231,8 +201,7 @@ ENTRY(aesni_set_key)
231.align 4 201.align 4
232.Ldec_key_loop: 202.Ldec_key_loop:
233 movaps (%rdi), %xmm0 203 movaps (%rdi), %xmm0
234 # aesimc %xmm0, %xmm1 204 AESIMC %xmm0 %xmm1
235 .byte 0x66, 0x0f, 0x38, 0xdb, 0xc8
236 movaps %xmm1, (%rsi) 205 movaps %xmm1, (%rsi)
237 add $0x10, %rdi 206 add $0x10, %rdi
238 sub $0x10, %rsi 207 sub $0x10, %rsi
@@ -274,51 +243,37 @@ _aesni_enc1:
274 je .Lenc192 243 je .Lenc192
275 add $0x20, TKEYP 244 add $0x20, TKEYP
276 movaps -0x60(TKEYP), KEY 245 movaps -0x60(TKEYP), KEY
277 # aesenc KEY, STATE 246 AESENC KEY STATE
278 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
279 movaps -0x50(TKEYP), KEY 247 movaps -0x50(TKEYP), KEY
280 # aesenc KEY, STATE 248 AESENC KEY STATE
281 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
282.align 4 249.align 4
283.Lenc192: 250.Lenc192:
284 movaps -0x40(TKEYP), KEY 251 movaps -0x40(TKEYP), KEY
285 # aesenc KEY, STATE 252 AESENC KEY STATE
286 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
287 movaps -0x30(TKEYP), KEY 253 movaps -0x30(TKEYP), KEY
288 # aesenc KEY, STATE 254 AESENC KEY STATE
289 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
290.align 4 255.align 4
291.Lenc128: 256.Lenc128:
292 movaps -0x20(TKEYP), KEY 257 movaps -0x20(TKEYP), KEY
293 # aesenc KEY, STATE 258 AESENC KEY STATE
294 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
295 movaps -0x10(TKEYP), KEY 259 movaps -0x10(TKEYP), KEY
296 # aesenc KEY, STATE 260 AESENC KEY STATE
297 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
298 movaps (TKEYP), KEY 261 movaps (TKEYP), KEY
299 # aesenc KEY, STATE 262 AESENC KEY STATE
300 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
301 movaps 0x10(TKEYP), KEY 263 movaps 0x10(TKEYP), KEY
302 # aesenc KEY, STATE 264 AESENC KEY STATE
303 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
304 movaps 0x20(TKEYP), KEY 265 movaps 0x20(TKEYP), KEY
305 # aesenc KEY, STATE 266 AESENC KEY STATE
306 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
307 movaps 0x30(TKEYP), KEY 267 movaps 0x30(TKEYP), KEY
308 # aesenc KEY, STATE 268 AESENC KEY STATE
309 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
310 movaps 0x40(TKEYP), KEY 269 movaps 0x40(TKEYP), KEY
311 # aesenc KEY, STATE 270 AESENC KEY STATE
312 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
313 movaps 0x50(TKEYP), KEY 271 movaps 0x50(TKEYP), KEY
314 # aesenc KEY, STATE 272 AESENC KEY STATE
315 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
316 movaps 0x60(TKEYP), KEY 273 movaps 0x60(TKEYP), KEY
317 # aesenc KEY, STATE 274 AESENC KEY STATE
318 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2
319 movaps 0x70(TKEYP), KEY 275 movaps 0x70(TKEYP), KEY
320 # aesenclast KEY, STATE # last round 276 AESENCLAST KEY STATE
321 .byte 0x66, 0x0f, 0x38, 0xdd, 0xc2
322 ret 277 ret
323 278
324/* 279/*
@@ -353,135 +308,79 @@ _aesni_enc4:
353 je .L4enc192 308 je .L4enc192
354 add $0x20, TKEYP 309 add $0x20, TKEYP
355 movaps -0x60(TKEYP), KEY 310 movaps -0x60(TKEYP), KEY
356 # aesenc KEY, STATE1 311 AESENC KEY STATE1
357 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 312 AESENC KEY STATE2
358 # aesenc KEY, STATE2 313 AESENC KEY STATE3
359 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 314 AESENC KEY STATE4
360 # aesenc KEY, STATE3
361 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
362 # aesenc KEY, STATE4
363 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
364 movaps -0x50(TKEYP), KEY 315 movaps -0x50(TKEYP), KEY
365 # aesenc KEY, STATE1 316 AESENC KEY STATE1
366 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 317 AESENC KEY STATE2
367 # aesenc KEY, STATE2 318 AESENC KEY STATE3
368 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 319 AESENC KEY STATE4
369 # aesenc KEY, STATE3
370 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
371 # aesenc KEY, STATE4
372 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
373#.align 4 320#.align 4
374.L4enc192: 321.L4enc192:
375 movaps -0x40(TKEYP), KEY 322 movaps -0x40(TKEYP), KEY
376 # aesenc KEY, STATE1 323 AESENC KEY STATE1
377 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 324 AESENC KEY STATE2
378 # aesenc KEY, STATE2 325 AESENC KEY STATE3
379 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 326 AESENC KEY STATE4
380 # aesenc KEY, STATE3
381 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
382 # aesenc KEY, STATE4
383 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
384 movaps -0x30(TKEYP), KEY 327 movaps -0x30(TKEYP), KEY
385 # aesenc KEY, STATE1 328 AESENC KEY STATE1
386 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 329 AESENC KEY STATE2
387 # aesenc KEY, STATE2 330 AESENC KEY STATE3
388 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 331 AESENC KEY STATE4
389 # aesenc KEY, STATE3
390 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
391 # aesenc KEY, STATE4
392 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
393#.align 4 332#.align 4
394.L4enc128: 333.L4enc128:
395 movaps -0x20(TKEYP), KEY 334 movaps -0x20(TKEYP), KEY
396 # aesenc KEY, STATE1 335 AESENC KEY STATE1
397 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 336 AESENC KEY STATE2
398 # aesenc KEY, STATE2 337 AESENC KEY STATE3
399 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 338 AESENC KEY STATE4
400 # aesenc KEY, STATE3
401 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
402 # aesenc KEY, STATE4
403 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
404 movaps -0x10(TKEYP), KEY 339 movaps -0x10(TKEYP), KEY
405 # aesenc KEY, STATE1 340 AESENC KEY STATE1
406 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 341 AESENC KEY STATE2
407 # aesenc KEY, STATE2 342 AESENC KEY STATE3
408 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 343 AESENC KEY STATE4
409 # aesenc KEY, STATE3
410 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
411 # aesenc KEY, STATE4
412 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
413 movaps (TKEYP), KEY 344 movaps (TKEYP), KEY
414 # aesenc KEY, STATE1 345 AESENC KEY STATE1
415 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 346 AESENC KEY STATE2
416 # aesenc KEY, STATE2 347 AESENC KEY STATE3
417 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 348 AESENC KEY STATE4
418 # aesenc KEY, STATE3
419 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
420 # aesenc KEY, STATE4
421 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
422 movaps 0x10(TKEYP), KEY 349 movaps 0x10(TKEYP), KEY
423 # aesenc KEY, STATE1 350 AESENC KEY STATE1
424 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 351 AESENC KEY STATE2
425 # aesenc KEY, STATE2 352 AESENC KEY STATE3
426 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 353 AESENC KEY STATE4
427 # aesenc KEY, STATE3
428 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
429 # aesenc KEY, STATE4
430 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
431 movaps 0x20(TKEYP), KEY 354 movaps 0x20(TKEYP), KEY
432 # aesenc KEY, STATE1 355 AESENC KEY STATE1
433 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 356 AESENC KEY STATE2
434 # aesenc KEY, STATE2 357 AESENC KEY STATE3
435 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 358 AESENC KEY STATE4
436 # aesenc KEY, STATE3
437 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
438 # aesenc KEY, STATE4
439 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
440 movaps 0x30(TKEYP), KEY 359 movaps 0x30(TKEYP), KEY
441 # aesenc KEY, STATE1 360 AESENC KEY STATE1
442 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 361 AESENC KEY STATE2
443 # aesenc KEY, STATE2 362 AESENC KEY STATE3
444 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 363 AESENC KEY STATE4
445 # aesenc KEY, STATE3
446 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
447 # aesenc KEY, STATE4
448 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
449 movaps 0x40(TKEYP), KEY 364 movaps 0x40(TKEYP), KEY
450 # aesenc KEY, STATE1 365 AESENC KEY STATE1
451 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 366 AESENC KEY STATE2
452 # aesenc KEY, STATE2 367 AESENC KEY STATE3
453 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 368 AESENC KEY STATE4
454 # aesenc KEY, STATE3
455 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
456 # aesenc KEY, STATE4
457 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
458 movaps 0x50(TKEYP), KEY 369 movaps 0x50(TKEYP), KEY
459 # aesenc KEY, STATE1 370 AESENC KEY STATE1
460 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 371 AESENC KEY STATE2
461 # aesenc KEY, STATE2 372 AESENC KEY STATE3
462 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 373 AESENC KEY STATE4
463 # aesenc KEY, STATE3
464 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
465 # aesenc KEY, STATE4
466 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
467 movaps 0x60(TKEYP), KEY 374 movaps 0x60(TKEYP), KEY
468 # aesenc KEY, STATE1 375 AESENC KEY STATE1
469 .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 376 AESENC KEY STATE2
470 # aesenc KEY, STATE2 377 AESENC KEY STATE3
471 .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 378 AESENC KEY STATE4
472 # aesenc KEY, STATE3
473 .byte 0x66, 0x0f, 0x38, 0xdc, 0xea
474 # aesenc KEY, STATE4
475 .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2
476 movaps 0x70(TKEYP), KEY 379 movaps 0x70(TKEYP), KEY
477 # aesenclast KEY, STATE1 # last round 380 AESENCLAST KEY STATE1 # last round
478 .byte 0x66, 0x0f, 0x38, 0xdd, 0xc2 381 AESENCLAST KEY STATE2
479 # aesenclast KEY, STATE2 382 AESENCLAST KEY STATE3
480 .byte 0x66, 0x0f, 0x38, 0xdd, 0xe2 383 AESENCLAST KEY STATE4
481 # aesenclast KEY, STATE3
482 .byte 0x66, 0x0f, 0x38, 0xdd, 0xea
483 # aesenclast KEY, STATE4
484 .byte 0x66, 0x0f, 0x38, 0xdd, 0xf2
485 ret 384 ret
486 385
487/* 386/*
@@ -518,51 +417,37 @@ _aesni_dec1:
518 je .Ldec192 417 je .Ldec192
519 add $0x20, TKEYP 418 add $0x20, TKEYP
520 movaps -0x60(TKEYP), KEY 419 movaps -0x60(TKEYP), KEY
521 # aesdec KEY, STATE 420 AESDEC KEY STATE
522 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
523 movaps -0x50(TKEYP), KEY 421 movaps -0x50(TKEYP), KEY
524 # aesdec KEY, STATE 422 AESDEC KEY STATE
525 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
526.align 4 423.align 4
527.Ldec192: 424.Ldec192:
528 movaps -0x40(TKEYP), KEY 425 movaps -0x40(TKEYP), KEY
529 # aesdec KEY, STATE 426 AESDEC KEY STATE
530 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
531 movaps -0x30(TKEYP), KEY 427 movaps -0x30(TKEYP), KEY
532 # aesdec KEY, STATE 428 AESDEC KEY STATE
533 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
534.align 4 429.align 4
535.Ldec128: 430.Ldec128:
536 movaps -0x20(TKEYP), KEY 431 movaps -0x20(TKEYP), KEY
537 # aesdec KEY, STATE 432 AESDEC KEY STATE
538 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
539 movaps -0x10(TKEYP), KEY 433 movaps -0x10(TKEYP), KEY
540 # aesdec KEY, STATE 434 AESDEC KEY STATE
541 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
542 movaps (TKEYP), KEY 435 movaps (TKEYP), KEY
543 # aesdec KEY, STATE 436 AESDEC KEY STATE
544 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
545 movaps 0x10(TKEYP), KEY 437 movaps 0x10(TKEYP), KEY
546 # aesdec KEY, STATE 438 AESDEC KEY STATE
547 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
548 movaps 0x20(TKEYP), KEY 439 movaps 0x20(TKEYP), KEY
549 # aesdec KEY, STATE 440 AESDEC KEY STATE
550 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
551 movaps 0x30(TKEYP), KEY 441 movaps 0x30(TKEYP), KEY
552 # aesdec KEY, STATE 442 AESDEC KEY STATE
553 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
554 movaps 0x40(TKEYP), KEY 443 movaps 0x40(TKEYP), KEY
555 # aesdec KEY, STATE 444 AESDEC KEY STATE
556 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
557 movaps 0x50(TKEYP), KEY 445 movaps 0x50(TKEYP), KEY
558 # aesdec KEY, STATE 446 AESDEC KEY STATE
559 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
560 movaps 0x60(TKEYP), KEY 447 movaps 0x60(TKEYP), KEY
561 # aesdec KEY, STATE 448 AESDEC KEY STATE
562 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2
563 movaps 0x70(TKEYP), KEY 449 movaps 0x70(TKEYP), KEY
564 # aesdeclast KEY, STATE # last round 450 AESDECLAST KEY STATE
565 .byte 0x66, 0x0f, 0x38, 0xdf, 0xc2
566 ret 451 ret
567 452
568/* 453/*
@@ -597,135 +482,79 @@ _aesni_dec4:
597 je .L4dec192 482 je .L4dec192
598 add $0x20, TKEYP 483 add $0x20, TKEYP
599 movaps -0x60(TKEYP), KEY 484 movaps -0x60(TKEYP), KEY
600 # aesdec KEY, STATE1 485 AESDEC KEY STATE1
601 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 486 AESDEC KEY STATE2
602 # aesdec KEY, STATE2 487 AESDEC KEY STATE3
603 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 488 AESDEC KEY STATE4
604 # aesdec KEY, STATE3
605 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
606 # aesdec KEY, STATE4
607 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
608 movaps -0x50(TKEYP), KEY 489 movaps -0x50(TKEYP), KEY
609 # aesdec KEY, STATE1 490 AESDEC KEY STATE1
610 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 491 AESDEC KEY STATE2
611 # aesdec KEY, STATE2 492 AESDEC KEY STATE3
612 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 493 AESDEC KEY STATE4
613 # aesdec KEY, STATE3
614 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
615 # aesdec KEY, STATE4
616 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
617.align 4 494.align 4
618.L4dec192: 495.L4dec192:
619 movaps -0x40(TKEYP), KEY 496 movaps -0x40(TKEYP), KEY
620 # aesdec KEY, STATE1 497 AESDEC KEY STATE1
621 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 498 AESDEC KEY STATE2
622 # aesdec KEY, STATE2 499 AESDEC KEY STATE3
623 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 500 AESDEC KEY STATE4
624 # aesdec KEY, STATE3
625 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
626 # aesdec KEY, STATE4
627 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
628 movaps -0x30(TKEYP), KEY 501 movaps -0x30(TKEYP), KEY
629 # aesdec KEY, STATE1 502 AESDEC KEY STATE1
630 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 503 AESDEC KEY STATE2
631 # aesdec KEY, STATE2 504 AESDEC KEY STATE3
632 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 505 AESDEC KEY STATE4
633 # aesdec KEY, STATE3
634 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
635 # aesdec KEY, STATE4
636 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
637.align 4 506.align 4
638.L4dec128: 507.L4dec128:
639 movaps -0x20(TKEYP), KEY 508 movaps -0x20(TKEYP), KEY
640 # aesdec KEY, STATE1 509 AESDEC KEY STATE1
641 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 510 AESDEC KEY STATE2
642 # aesdec KEY, STATE2 511 AESDEC KEY STATE3
643 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 512 AESDEC KEY STATE4
644 # aesdec KEY, STATE3
645 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
646 # aesdec KEY, STATE4
647 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
648 movaps -0x10(TKEYP), KEY 513 movaps -0x10(TKEYP), KEY
649 # aesdec KEY, STATE1 514 AESDEC KEY STATE1
650 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 515 AESDEC KEY STATE2
651 # aesdec KEY, STATE2 516 AESDEC KEY STATE3
652 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 517 AESDEC KEY STATE4
653 # aesdec KEY, STATE3
654 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
655 # aesdec KEY, STATE4
656 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
657 movaps (TKEYP), KEY 518 movaps (TKEYP), KEY
658 # aesdec KEY, STATE1 519 AESDEC KEY STATE1
659 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 520 AESDEC KEY STATE2
660 # aesdec KEY, STATE2 521 AESDEC KEY STATE3
661 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 522 AESDEC KEY STATE4
662 # aesdec KEY, STATE3
663 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
664 # aesdec KEY, STATE4
665 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
666 movaps 0x10(TKEYP), KEY 523 movaps 0x10(TKEYP), KEY
667 # aesdec KEY, STATE1 524 AESDEC KEY STATE1
668 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 525 AESDEC KEY STATE2
669 # aesdec KEY, STATE2 526 AESDEC KEY STATE3
670 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 527 AESDEC KEY STATE4
671 # aesdec KEY, STATE3
672 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
673 # aesdec KEY, STATE4
674 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
675 movaps 0x20(TKEYP), KEY 528 movaps 0x20(TKEYP), KEY
676 # aesdec KEY, STATE1 529 AESDEC KEY STATE1
677 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 530 AESDEC KEY STATE2
678 # aesdec KEY, STATE2 531 AESDEC KEY STATE3
679 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 532 AESDEC KEY STATE4
680 # aesdec KEY, STATE3
681 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
682 # aesdec KEY, STATE4
683 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
684 movaps 0x30(TKEYP), KEY 533 movaps 0x30(TKEYP), KEY
685 # aesdec KEY, STATE1 534 AESDEC KEY STATE1
686 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 535 AESDEC KEY STATE2
687 # aesdec KEY, STATE2 536 AESDEC KEY STATE3
688 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 537 AESDEC KEY STATE4
689 # aesdec KEY, STATE3
690 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
691 # aesdec KEY, STATE4
692 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
693 movaps 0x40(TKEYP), KEY 538 movaps 0x40(TKEYP), KEY
694 # aesdec KEY, STATE1 539 AESDEC KEY STATE1
695 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 540 AESDEC KEY STATE2
696 # aesdec KEY, STATE2 541 AESDEC KEY STATE3
697 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 542 AESDEC KEY STATE4
698 # aesdec KEY, STATE3
699 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
700 # aesdec KEY, STATE4
701 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
702 movaps 0x50(TKEYP), KEY 543 movaps 0x50(TKEYP), KEY
703 # aesdec KEY, STATE1 544 AESDEC KEY STATE1
704 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 545 AESDEC KEY STATE2
705 # aesdec KEY, STATE2 546 AESDEC KEY STATE3
706 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 547 AESDEC KEY STATE4
707 # aesdec KEY, STATE3
708 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
709 # aesdec KEY, STATE4
710 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
711 movaps 0x60(TKEYP), KEY 548 movaps 0x60(TKEYP), KEY
712 # aesdec KEY, STATE1 549 AESDEC KEY STATE1
713 .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 550 AESDEC KEY STATE2
714 # aesdec KEY, STATE2 551 AESDEC KEY STATE3
715 .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 552 AESDEC KEY STATE4
716 # aesdec KEY, STATE3
717 .byte 0x66, 0x0f, 0x38, 0xde, 0xea
718 # aesdec KEY, STATE4
719 .byte 0x66, 0x0f, 0x38, 0xde, 0xf2
720 movaps 0x70(TKEYP), KEY 553 movaps 0x70(TKEYP), KEY
721 # aesdeclast KEY, STATE1 # last round 554 AESDECLAST KEY STATE1 # last round
722 .byte 0x66, 0x0f, 0x38, 0xdf, 0xc2 555 AESDECLAST KEY STATE2
723 # aesdeclast KEY, STATE2 556 AESDECLAST KEY STATE3
724 .byte 0x66, 0x0f, 0x38, 0xdf, 0xe2 557 AESDECLAST KEY STATE4
725 # aesdeclast KEY, STATE3
726 .byte 0x66, 0x0f, 0x38, 0xdf, 0xea
727 # aesdeclast KEY, STATE4
728 .byte 0x66, 0x0f, 0x38, 0xdf, 0xf2
729 ret 558 ret
730 559
731/* 560/*
diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/arch/x86/crypto/ghash-clmulni-intel_asm.S
new file mode 100644
index 000000000000..1eb7f90cb7b9
--- /dev/null
+++ b/arch/x86/crypto/ghash-clmulni-intel_asm.S
@@ -0,0 +1,157 @@
1/*
2 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
3 * instructions. This file contains accelerated part of ghash
4 * implementation. More information about PCLMULQDQ can be found at:
5 *
6 * http://software.intel.com/en-us/articles/carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/
7 *
8 * Copyright (c) 2009 Intel Corp.
9 * Author: Huang Ying <ying.huang@intel.com>
10 * Vinodh Gopal
11 * Erdinc Ozturk
12 * Deniz Karakoyunlu
13 *
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU General Public License version 2 as published
16 * by the Free Software Foundation.
17 */
18
19#include <linux/linkage.h>
20#include <asm/inst.h>
21
22.data
23
24.align 16
25.Lbswap_mask:
26 .octa 0x000102030405060708090a0b0c0d0e0f
27.Lpoly:
28 .octa 0xc2000000000000000000000000000001
29.Ltwo_one:
30 .octa 0x00000001000000000000000000000001
31
32#define DATA %xmm0
33#define SHASH %xmm1
34#define T1 %xmm2
35#define T2 %xmm3
36#define T3 %xmm4
37#define BSWAP %xmm5
38#define IN1 %xmm6
39
40.text
41
42/*
43 * __clmul_gf128mul_ble: internal ABI
44 * input:
45 * DATA: operand1
46 * SHASH: operand2, hash_key << 1 mod poly
47 * output:
48 * DATA: operand1 * operand2 mod poly
49 * changed:
50 * T1
51 * T2
52 * T3
53 */
54__clmul_gf128mul_ble:
55 movaps DATA, T1
56 pshufd $0b01001110, DATA, T2
57 pshufd $0b01001110, SHASH, T3
58 pxor DATA, T2
59 pxor SHASH, T3
60
61 PCLMULQDQ 0x00 SHASH DATA # DATA = a0 * b0
62 PCLMULQDQ 0x11 SHASH T1 # T1 = a1 * b1
63 PCLMULQDQ 0x00 T3 T2 # T2 = (a1 + a0) * (b1 + b0)
64 pxor DATA, T2
65 pxor T1, T2 # T2 = a0 * b1 + a1 * b0
66
67 movaps T2, T3
68 pslldq $8, T3
69 psrldq $8, T2
70 pxor T3, DATA
71 pxor T2, T1 # <T1:DATA> is result of
72 # carry-less multiplication
73
74 # first phase of the reduction
75 movaps DATA, T3
76 psllq $1, T3
77 pxor DATA, T3
78 psllq $5, T3
79 pxor DATA, T3
80 psllq $57, T3
81 movaps T3, T2
82 pslldq $8, T2
83 psrldq $8, T3
84 pxor T2, DATA
85 pxor T3, T1
86
87 # second phase of the reduction
88 movaps DATA, T2
89 psrlq $5, T2
90 pxor DATA, T2
91 psrlq $1, T2
92 pxor DATA, T2
93 psrlq $1, T2
94 pxor T2, T1
95 pxor T1, DATA
96 ret
97
98/* void clmul_ghash_mul(char *dst, const be128 *shash) */
99ENTRY(clmul_ghash_mul)
100 movups (%rdi), DATA
101 movups (%rsi), SHASH
102 movaps .Lbswap_mask, BSWAP
103 PSHUFB_XMM BSWAP DATA
104 call __clmul_gf128mul_ble
105 PSHUFB_XMM BSWAP DATA
106 movups DATA, (%rdi)
107 ret
108
109/*
110 * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
111 * const be128 *shash);
112 */
113ENTRY(clmul_ghash_update)
114 cmp $16, %rdx
115 jb .Lupdate_just_ret # check length
116 movaps .Lbswap_mask, BSWAP
117 movups (%rdi), DATA
118 movups (%rcx), SHASH
119 PSHUFB_XMM BSWAP DATA
120.align 4
121.Lupdate_loop:
122 movups (%rsi), IN1
123 PSHUFB_XMM BSWAP IN1
124 pxor IN1, DATA
125 call __clmul_gf128mul_ble
126 sub $16, %rdx
127 add $16, %rsi
128 cmp $16, %rdx
129 jge .Lupdate_loop
130 PSHUFB_XMM BSWAP DATA
131 movups DATA, (%rdi)
132.Lupdate_just_ret:
133 ret
134
135/*
136 * void clmul_ghash_setkey(be128 *shash, const u8 *key);
137 *
138 * Calculate hash_key << 1 mod poly
139 */
140ENTRY(clmul_ghash_setkey)
141 movaps .Lbswap_mask, BSWAP
142 movups (%rsi), %xmm0
143 PSHUFB_XMM BSWAP %xmm0
144 movaps %xmm0, %xmm1
145 psllq $1, %xmm0
146 psrlq $63, %xmm1
147 movaps %xmm1, %xmm2
148 pslldq $8, %xmm1
149 psrldq $8, %xmm2
150 por %xmm1, %xmm0
151 # reduction
152 pshufd $0b00100100, %xmm2, %xmm1
153 pcmpeqd .Ltwo_one, %xmm1
154 pand .Lpoly, %xmm1
155 pxor %xmm1, %xmm0
156 movups %xmm0, (%rdi)
157 ret
diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c
new file mode 100644
index 000000000000..cbcc8d8ea93a
--- /dev/null
+++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c
@@ -0,0 +1,333 @@
1/*
2 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
3 * instructions. This file contains glue code.
4 *
5 * Copyright (c) 2009 Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License version 2 as published
10 * by the Free Software Foundation.
11 */
12
13#include <linux/module.h>
14#include <linux/init.h>
15#include <linux/kernel.h>
16#include <linux/crypto.h>
17#include <crypto/algapi.h>
18#include <crypto/cryptd.h>
19#include <crypto/gf128mul.h>
20#include <crypto/internal/hash.h>
21#include <asm/i387.h>
22
23#define GHASH_BLOCK_SIZE 16
24#define GHASH_DIGEST_SIZE 16
25
26void clmul_ghash_mul(char *dst, const be128 *shash);
27
28void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
29 const be128 *shash);
30
31void clmul_ghash_setkey(be128 *shash, const u8 *key);
32
33struct ghash_async_ctx {
34 struct cryptd_ahash *cryptd_tfm;
35};
36
37struct ghash_ctx {
38 be128 shash;
39};
40
41struct ghash_desc_ctx {
42 u8 buffer[GHASH_BLOCK_SIZE];
43 u32 bytes;
44};
45
46static int ghash_init(struct shash_desc *desc)
47{
48 struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
49
50 memset(dctx, 0, sizeof(*dctx));
51
52 return 0;
53}
54
55static int ghash_setkey(struct crypto_shash *tfm,
56 const u8 *key, unsigned int keylen)
57{
58 struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
59
60 if (keylen != GHASH_BLOCK_SIZE) {
61 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
62 return -EINVAL;
63 }
64
65 clmul_ghash_setkey(&ctx->shash, key);
66
67 return 0;
68}
69
70static int ghash_update(struct shash_desc *desc,
71 const u8 *src, unsigned int srclen)
72{
73 struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
74 struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
75 u8 *dst = dctx->buffer;
76
77 kernel_fpu_begin();
78 if (dctx->bytes) {
79 int n = min(srclen, dctx->bytes);
80 u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
81
82 dctx->bytes -= n;
83 srclen -= n;
84
85 while (n--)
86 *pos++ ^= *src++;
87
88 if (!dctx->bytes)
89 clmul_ghash_mul(dst, &ctx->shash);
90 }
91
92 clmul_ghash_update(dst, src, srclen, &ctx->shash);
93 kernel_fpu_end();
94
95 if (srclen & 0xf) {
96 src += srclen - (srclen & 0xf);
97 srclen &= 0xf;
98 dctx->bytes = GHASH_BLOCK_SIZE - srclen;
99 while (srclen--)
100 *dst++ ^= *src++;
101 }
102
103 return 0;
104}
105
106static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx)
107{
108 u8 *dst = dctx->buffer;
109
110 if (dctx->bytes) {
111 u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
112
113 while (dctx->bytes--)
114 *tmp++ ^= 0;
115
116 kernel_fpu_begin();
117 clmul_ghash_mul(dst, &ctx->shash);
118 kernel_fpu_end();
119 }
120
121 dctx->bytes = 0;
122}
123
124static int ghash_final(struct shash_desc *desc, u8 *dst)
125{
126 struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
127 struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
128 u8 *buf = dctx->buffer;
129
130 ghash_flush(ctx, dctx);
131 memcpy(dst, buf, GHASH_BLOCK_SIZE);
132
133 return 0;
134}
135
136static struct shash_alg ghash_alg = {
137 .digestsize = GHASH_DIGEST_SIZE,
138 .init = ghash_init,
139 .update = ghash_update,
140 .final = ghash_final,
141 .setkey = ghash_setkey,
142 .descsize = sizeof(struct ghash_desc_ctx),
143 .base = {
144 .cra_name = "__ghash",
145 .cra_driver_name = "__ghash-pclmulqdqni",
146 .cra_priority = 0,
147 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
148 .cra_blocksize = GHASH_BLOCK_SIZE,
149 .cra_ctxsize = sizeof(struct ghash_ctx),
150 .cra_module = THIS_MODULE,
151 .cra_list = LIST_HEAD_INIT(ghash_alg.base.cra_list),
152 },
153};
154
155static int ghash_async_init(struct ahash_request *req)
156{
157 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
158 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
159 struct ahash_request *cryptd_req = ahash_request_ctx(req);
160 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
161
162 if (!irq_fpu_usable()) {
163 memcpy(cryptd_req, req, sizeof(*req));
164 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
165 return crypto_ahash_init(cryptd_req);
166 } else {
167 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
168 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
169
170 desc->tfm = child;
171 desc->flags = req->base.flags;
172 return crypto_shash_init(desc);
173 }
174}
175
176static int ghash_async_update(struct ahash_request *req)
177{
178 struct ahash_request *cryptd_req = ahash_request_ctx(req);
179
180 if (!irq_fpu_usable()) {
181 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
182 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
183 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
184
185 memcpy(cryptd_req, req, sizeof(*req));
186 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
187 return crypto_ahash_update(cryptd_req);
188 } else {
189 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
190 return shash_ahash_update(req, desc);
191 }
192}
193
194static int ghash_async_final(struct ahash_request *req)
195{
196 struct ahash_request *cryptd_req = ahash_request_ctx(req);
197
198 if (!irq_fpu_usable()) {
199 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
200 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
201 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
202
203 memcpy(cryptd_req, req, sizeof(*req));
204 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
205 return crypto_ahash_final(cryptd_req);
206 } else {
207 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
208 return crypto_shash_final(desc, req->result);
209 }
210}
211
212static int ghash_async_digest(struct ahash_request *req)
213{
214 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
215 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
216 struct ahash_request *cryptd_req = ahash_request_ctx(req);
217 struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
218
219 if (!irq_fpu_usable()) {
220 memcpy(cryptd_req, req, sizeof(*req));
221 ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
222 return crypto_ahash_digest(cryptd_req);
223 } else {
224 struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
225 struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
226
227 desc->tfm = child;
228 desc->flags = req->base.flags;
229 return shash_ahash_digest(req, desc);
230 }
231}
232
233static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
234 unsigned int keylen)
235{
236 struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
237 struct crypto_ahash *child = &ctx->cryptd_tfm->base;
238 int err;
239
240 crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
241 crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
242 & CRYPTO_TFM_REQ_MASK);
243 err = crypto_ahash_setkey(child, key, keylen);
244 crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child)
245 & CRYPTO_TFM_RES_MASK);
246
247 return 0;
248}
249
250static int ghash_async_init_tfm(struct crypto_tfm *tfm)
251{
252 struct cryptd_ahash *cryptd_tfm;
253 struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
254
255 cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni", 0, 0);
256 if (IS_ERR(cryptd_tfm))
257 return PTR_ERR(cryptd_tfm);
258 ctx->cryptd_tfm = cryptd_tfm;
259 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
260 sizeof(struct ahash_request) +
261 crypto_ahash_reqsize(&cryptd_tfm->base));
262
263 return 0;
264}
265
266static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
267{
268 struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
269
270 cryptd_free_ahash(ctx->cryptd_tfm);
271}
272
273static struct ahash_alg ghash_async_alg = {
274 .init = ghash_async_init,
275 .update = ghash_async_update,
276 .final = ghash_async_final,
277 .setkey = ghash_async_setkey,
278 .digest = ghash_async_digest,
279 .halg = {
280 .digestsize = GHASH_DIGEST_SIZE,
281 .base = {
282 .cra_name = "ghash",
283 .cra_driver_name = "ghash-clmulni",
284 .cra_priority = 400,
285 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
286 .cra_blocksize = GHASH_BLOCK_SIZE,
287 .cra_type = &crypto_ahash_type,
288 .cra_module = THIS_MODULE,
289 .cra_list = LIST_HEAD_INIT(ghash_async_alg.halg.base.cra_list),
290 .cra_init = ghash_async_init_tfm,
291 .cra_exit = ghash_async_exit_tfm,
292 },
293 },
294};
295
296static int __init ghash_pclmulqdqni_mod_init(void)
297{
298 int err;
299
300 if (!cpu_has_pclmulqdq) {
301 printk(KERN_INFO "Intel PCLMULQDQ-NI instructions are not"
302 " detected.\n");
303 return -ENODEV;
304 }
305
306 err = crypto_register_shash(&ghash_alg);
307 if (err)
308 goto err_out;
309 err = crypto_register_ahash(&ghash_async_alg);
310 if (err)
311 goto err_shash;
312
313 return 0;
314
315err_shash:
316 crypto_unregister_shash(&ghash_alg);
317err_out:
318 return err;
319}
320
321static void __exit ghash_pclmulqdqni_mod_exit(void)
322{
323 crypto_unregister_ahash(&ghash_async_alg);
324 crypto_unregister_shash(&ghash_alg);
325}
326
327module_init(ghash_pclmulqdqni_mod_init);
328module_exit(ghash_pclmulqdqni_mod_exit);
329
330MODULE_LICENSE("GPL");
331MODULE_DESCRIPTION("GHASH Message Digest Algorithm, "
332 "acclerated by PCLMULQDQ-NI");
333MODULE_ALIAS("ghash");
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 9cfc88b97742..613700f27a4a 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -248,6 +248,7 @@ extern const char * const x86_power_flags[32];
248#define cpu_has_x2apic boot_cpu_has(X86_FEATURE_X2APIC) 248#define cpu_has_x2apic boot_cpu_has(X86_FEATURE_X2APIC)
249#define cpu_has_xsave boot_cpu_has(X86_FEATURE_XSAVE) 249#define cpu_has_xsave boot_cpu_has(X86_FEATURE_XSAVE)
250#define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR) 250#define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR)
251#define cpu_has_pclmulqdq boot_cpu_has(X86_FEATURE_PCLMULQDQ)
251 252
252#if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64) 253#if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64)
253# define cpu_has_invlpg 1 254# define cpu_has_invlpg 1
diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
index 0b20bbb758f2..ebfb8a9e11f7 100644
--- a/arch/x86/include/asm/i387.h
+++ b/arch/x86/include/asm/i387.h
@@ -10,6 +10,8 @@
10#ifndef _ASM_X86_I387_H 10#ifndef _ASM_X86_I387_H
11#define _ASM_X86_I387_H 11#define _ASM_X86_I387_H
12 12
13#ifndef __ASSEMBLY__
14
13#include <linux/sched.h> 15#include <linux/sched.h>
14#include <linux/kernel_stat.h> 16#include <linux/kernel_stat.h>
15#include <linux/regset.h> 17#include <linux/regset.h>
@@ -411,4 +413,9 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
411 } 413 }
412} 414}
413 415
416#endif /* __ASSEMBLY__ */
417
418#define PSHUFB_XMM5_XMM0 .byte 0x66, 0x0f, 0x38, 0x00, 0xc5
419#define PSHUFB_XMM5_XMM6 .byte 0x66, 0x0f, 0x38, 0x00, 0xf5
420
414#endif /* _ASM_X86_I387_H */ 421#endif /* _ASM_X86_I387_H */
diff --git a/arch/x86/include/asm/inst.h b/arch/x86/include/asm/inst.h
new file mode 100644
index 000000000000..14cf526091f9
--- /dev/null
+++ b/arch/x86/include/asm/inst.h
@@ -0,0 +1,150 @@
1/*
2 * Generate .byte code for some instructions not supported by old
3 * binutils.
4 */
5#ifndef X86_ASM_INST_H
6#define X86_ASM_INST_H
7
8#ifdef __ASSEMBLY__
9
10 .macro XMM_NUM opd xmm
11 .ifc \xmm,%xmm0
12 \opd = 0
13 .endif
14 .ifc \xmm,%xmm1
15 \opd = 1
16 .endif
17 .ifc \xmm,%xmm2
18 \opd = 2
19 .endif
20 .ifc \xmm,%xmm3
21 \opd = 3
22 .endif
23 .ifc \xmm,%xmm4
24 \opd = 4
25 .endif
26 .ifc \xmm,%xmm5
27 \opd = 5
28 .endif
29 .ifc \xmm,%xmm6
30 \opd = 6
31 .endif
32 .ifc \xmm,%xmm7
33 \opd = 7
34 .endif
35 .ifc \xmm,%xmm8
36 \opd = 8
37 .endif
38 .ifc \xmm,%xmm9
39 \opd = 9
40 .endif
41 .ifc \xmm,%xmm10
42 \opd = 10
43 .endif
44 .ifc \xmm,%xmm11
45 \opd = 11
46 .endif
47 .ifc \xmm,%xmm12
48 \opd = 12
49 .endif
50 .ifc \xmm,%xmm13
51 \opd = 13
52 .endif
53 .ifc \xmm,%xmm14
54 \opd = 14
55 .endif
56 .ifc \xmm,%xmm15
57 \opd = 15
58 .endif
59 .endm
60
61 .macro PFX_OPD_SIZE
62 .byte 0x66
63 .endm
64
65 .macro PFX_REX opd1 opd2
66 .if (\opd1 | \opd2) & 8
67 .byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1)
68 .endif
69 .endm
70
71 .macro MODRM mod opd1 opd2
72 .byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
73 .endm
74
75 .macro PSHUFB_XMM xmm1 xmm2
76 XMM_NUM pshufb_opd1 \xmm1
77 XMM_NUM pshufb_opd2 \xmm2
78 PFX_OPD_SIZE
79 PFX_REX pshufb_opd1 pshufb_opd2
80 .byte 0x0f, 0x38, 0x00
81 MODRM 0xc0 pshufb_opd1 pshufb_opd2
82 .endm
83
84 .macro PCLMULQDQ imm8 xmm1 xmm2
85 XMM_NUM clmul_opd1 \xmm1
86 XMM_NUM clmul_opd2 \xmm2
87 PFX_OPD_SIZE
88 PFX_REX clmul_opd1 clmul_opd2
89 .byte 0x0f, 0x3a, 0x44
90 MODRM 0xc0 clmul_opd1 clmul_opd2
91 .byte \imm8
92 .endm
93
94 .macro AESKEYGENASSIST rcon xmm1 xmm2
95 XMM_NUM aeskeygen_opd1 \xmm1
96 XMM_NUM aeskeygen_opd2 \xmm2
97 PFX_OPD_SIZE
98 PFX_REX aeskeygen_opd1 aeskeygen_opd2
99 .byte 0x0f, 0x3a, 0xdf
100 MODRM 0xc0 aeskeygen_opd1 aeskeygen_opd2
101 .byte \rcon
102 .endm
103
104 .macro AESIMC xmm1 xmm2
105 XMM_NUM aesimc_opd1 \xmm1
106 XMM_NUM aesimc_opd2 \xmm2
107 PFX_OPD_SIZE
108 PFX_REX aesimc_opd1 aesimc_opd2
109 .byte 0x0f, 0x38, 0xdb
110 MODRM 0xc0 aesimc_opd1 aesimc_opd2
111 .endm
112
113 .macro AESENC xmm1 xmm2
114 XMM_NUM aesenc_opd1 \xmm1
115 XMM_NUM aesenc_opd2 \xmm2
116 PFX_OPD_SIZE
117 PFX_REX aesenc_opd1 aesenc_opd2
118 .byte 0x0f, 0x38, 0xdc
119 MODRM 0xc0 aesenc_opd1 aesenc_opd2
120 .endm
121
122 .macro AESENCLAST xmm1 xmm2
123 XMM_NUM aesenclast_opd1 \xmm1
124 XMM_NUM aesenclast_opd2 \xmm2
125 PFX_OPD_SIZE
126 PFX_REX aesenclast_opd1 aesenclast_opd2
127 .byte 0x0f, 0x38, 0xdd
128 MODRM 0xc0 aesenclast_opd1 aesenclast_opd2
129 .endm
130
131 .macro AESDEC xmm1 xmm2
132 XMM_NUM aesdec_opd1 \xmm1
133 XMM_NUM aesdec_opd2 \xmm2
134 PFX_OPD_SIZE
135 PFX_REX aesdec_opd1 aesdec_opd2
136 .byte 0x0f, 0x38, 0xde
137 MODRM 0xc0 aesdec_opd1 aesdec_opd2
138 .endm
139
140 .macro AESDECLAST xmm1 xmm2
141 XMM_NUM aesdeclast_opd1 \xmm1
142 XMM_NUM aesdeclast_opd2 \xmm2
143 PFX_OPD_SIZE
144 PFX_REX aesdeclast_opd1 aesdeclast_opd2
145 .byte 0x0f, 0x38, 0xdf
146 MODRM 0xc0 aesdeclast_opd1 aesdeclast_opd2
147 .endm
148#endif
149
150#endif
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 26b5dd0cb564..81c185a6971f 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -440,6 +440,15 @@ config CRYPTO_WP512
440 See also: 440 See also:
441 <http://planeta.terra.com.br/informatica/paulobarreto/WhirlpoolPage.html> 441 <http://planeta.terra.com.br/informatica/paulobarreto/WhirlpoolPage.html>
442 442
443config CRYPTO_GHASH_CLMUL_NI_INTEL
444 tristate "GHASH digest algorithm (CLMUL-NI accelerated)"
445 depends on (X86 || UML_X86) && 64BIT
446 select CRYPTO_SHASH
447 select CRYPTO_CRYPTD
448 help
449 GHASH is message digest algorithm for GCM (Galois/Counter Mode).
450 The implementation is accelerated by CLMUL-NI of Intel.
451
443comment "Ciphers" 452comment "Ciphers"
444 453
445config CRYPTO_AES 454config CRYPTO_AES
diff --git a/crypto/ansi_cprng.c b/crypto/ansi_cprng.c
index 3aa6e3834bfe..2bc332142849 100644
--- a/crypto/ansi_cprng.c
+++ b/crypto/ansi_cprng.c
@@ -85,7 +85,7 @@ static void xor_vectors(unsigned char *in1, unsigned char *in2,
85 * Returns DEFAULT_BLK_SZ bytes of random data per call 85 * Returns DEFAULT_BLK_SZ bytes of random data per call
86 * returns 0 if generation succeded, <0 if something went wrong 86 * returns 0 if generation succeded, <0 if something went wrong
87 */ 87 */
88static int _get_more_prng_bytes(struct prng_context *ctx) 88static int _get_more_prng_bytes(struct prng_context *ctx, int cont_test)
89{ 89{
90 int i; 90 int i;
91 unsigned char tmp[DEFAULT_BLK_SZ]; 91 unsigned char tmp[DEFAULT_BLK_SZ];
@@ -132,7 +132,7 @@ static int _get_more_prng_bytes(struct prng_context *ctx)
132 */ 132 */
133 if (!memcmp(ctx->rand_data, ctx->last_rand_data, 133 if (!memcmp(ctx->rand_data, ctx->last_rand_data,
134 DEFAULT_BLK_SZ)) { 134 DEFAULT_BLK_SZ)) {
135 if (fips_enabled) { 135 if (cont_test) {
136 panic("cprng %p Failed repetition check!\n", 136 panic("cprng %p Failed repetition check!\n",
137 ctx); 137 ctx);
138 } 138 }
@@ -185,16 +185,14 @@ static int _get_more_prng_bytes(struct prng_context *ctx)
185} 185}
186 186
187/* Our exported functions */ 187/* Our exported functions */
188static int get_prng_bytes(char *buf, size_t nbytes, struct prng_context *ctx) 188static int get_prng_bytes(char *buf, size_t nbytes, struct prng_context *ctx,
189 int do_cont_test)
189{ 190{
190 unsigned char *ptr = buf; 191 unsigned char *ptr = buf;
191 unsigned int byte_count = (unsigned int)nbytes; 192 unsigned int byte_count = (unsigned int)nbytes;
192 int err; 193 int err;
193 194
194 195
195 if (nbytes < 0)
196 return -EINVAL;
197
198 spin_lock_bh(&ctx->prng_lock); 196 spin_lock_bh(&ctx->prng_lock);
199 197
200 err = -EINVAL; 198 err = -EINVAL;
@@ -220,7 +218,7 @@ static int get_prng_bytes(char *buf, size_t nbytes, struct prng_context *ctx)
220 218
221remainder: 219remainder:
222 if (ctx->rand_data_valid == DEFAULT_BLK_SZ) { 220 if (ctx->rand_data_valid == DEFAULT_BLK_SZ) {
223 if (_get_more_prng_bytes(ctx) < 0) { 221 if (_get_more_prng_bytes(ctx, do_cont_test) < 0) {
224 memset(buf, 0, nbytes); 222 memset(buf, 0, nbytes);
225 err = -EINVAL; 223 err = -EINVAL;
226 goto done; 224 goto done;
@@ -247,7 +245,7 @@ empty_rbuf:
247 */ 245 */
248 for (; byte_count >= DEFAULT_BLK_SZ; byte_count -= DEFAULT_BLK_SZ) { 246 for (; byte_count >= DEFAULT_BLK_SZ; byte_count -= DEFAULT_BLK_SZ) {
249 if (ctx->rand_data_valid == DEFAULT_BLK_SZ) { 247 if (ctx->rand_data_valid == DEFAULT_BLK_SZ) {
250 if (_get_more_prng_bytes(ctx) < 0) { 248 if (_get_more_prng_bytes(ctx, do_cont_test) < 0) {
251 memset(buf, 0, nbytes); 249 memset(buf, 0, nbytes);
252 err = -EINVAL; 250 err = -EINVAL;
253 goto done; 251 goto done;
@@ -356,7 +354,7 @@ static int cprng_get_random(struct crypto_rng *tfm, u8 *rdata,
356{ 354{
357 struct prng_context *prng = crypto_rng_ctx(tfm); 355 struct prng_context *prng = crypto_rng_ctx(tfm);
358 356
359 return get_prng_bytes(rdata, dlen, prng); 357 return get_prng_bytes(rdata, dlen, prng, 0);
360} 358}
361 359
362/* 360/*
@@ -404,19 +402,79 @@ static struct crypto_alg rng_alg = {
404 } 402 }
405}; 403};
406 404
405#ifdef CONFIG_CRYPTO_FIPS
406static int fips_cprng_get_random(struct crypto_rng *tfm, u8 *rdata,
407 unsigned int dlen)
408{
409 struct prng_context *prng = crypto_rng_ctx(tfm);
410
411 return get_prng_bytes(rdata, dlen, prng, 1);
412}
413
414static int fips_cprng_reset(struct crypto_rng *tfm, u8 *seed, unsigned int slen)
415{
416 u8 rdata[DEFAULT_BLK_SZ];
417 int rc;
418
419 struct prng_context *prng = crypto_rng_ctx(tfm);
420
421 rc = cprng_reset(tfm, seed, slen);
422
423 if (!rc)
424 goto out;
425
426 /* this primes our continuity test */
427 rc = get_prng_bytes(rdata, DEFAULT_BLK_SZ, prng, 0);
428 prng->rand_data_valid = DEFAULT_BLK_SZ;
429
430out:
431 return rc;
432}
433
434static struct crypto_alg fips_rng_alg = {
435 .cra_name = "fips(ansi_cprng)",
436 .cra_driver_name = "fips_ansi_cprng",
437 .cra_priority = 300,
438 .cra_flags = CRYPTO_ALG_TYPE_RNG,
439 .cra_ctxsize = sizeof(struct prng_context),
440 .cra_type = &crypto_rng_type,
441 .cra_module = THIS_MODULE,
442 .cra_list = LIST_HEAD_INIT(rng_alg.cra_list),
443 .cra_init = cprng_init,
444 .cra_exit = cprng_exit,
445 .cra_u = {
446 .rng = {
447 .rng_make_random = fips_cprng_get_random,
448 .rng_reset = fips_cprng_reset,
449 .seedsize = DEFAULT_PRNG_KSZ + 2*DEFAULT_BLK_SZ,
450 }
451 }
452};
453#endif
407 454
408/* Module initalization */ 455/* Module initalization */
409static int __init prng_mod_init(void) 456static int __init prng_mod_init(void)
410{ 457{
411 if (fips_enabled) 458 int rc = 0;
412 rng_alg.cra_priority += 200;
413 459
414 return crypto_register_alg(&rng_alg); 460 rc = crypto_register_alg(&rng_alg);
461#ifdef CONFIG_CRYPTO_FIPS
462 if (rc)
463 goto out;
464
465 rc = crypto_register_alg(&fips_rng_alg);
466
467out:
468#endif
469 return rc;
415} 470}
416 471
417static void __exit prng_mod_fini(void) 472static void __exit prng_mod_fini(void)
418{ 473{
419 crypto_unregister_alg(&rng_alg); 474 crypto_unregister_alg(&rng_alg);
475#ifdef CONFIG_CRYPTO_FIPS
476 crypto_unregister_alg(&fips_rng_alg);
477#endif
420 return; 478 return;
421} 479}
422 480
diff --git a/crypto/cryptd.c b/crypto/cryptd.c
index 35335825a4ef..f8ae0d94a647 100644
--- a/crypto/cryptd.c
+++ b/crypto/cryptd.c
@@ -711,6 +711,13 @@ struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm)
711} 711}
712EXPORT_SYMBOL_GPL(cryptd_ahash_child); 712EXPORT_SYMBOL_GPL(cryptd_ahash_child);
713 713
714struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
715{
716 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
717 return &rctx->desc;
718}
719EXPORT_SYMBOL_GPL(cryptd_shash_desc);
720
714void cryptd_free_ahash(struct cryptd_ahash *tfm) 721void cryptd_free_ahash(struct cryptd_ahash *tfm)
715{ 722{
716 crypto_free_ahash(&tfm->base); 723 crypto_free_ahash(&tfm->base);
diff --git a/crypto/digest.c b/crypto/digest.c
deleted file mode 100644
index 5d3f1303da98..000000000000
--- a/crypto/digest.c
+++ /dev/null
@@ -1,240 +0,0 @@
1/*
2 * Cryptographic API.
3 *
4 * Digest operations.
5 *
6 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 */
14
15#include <crypto/internal/hash.h>
16#include <crypto/scatterwalk.h>
17#include <linux/mm.h>
18#include <linux/errno.h>
19#include <linux/hardirq.h>
20#include <linux/highmem.h>
21#include <linux/kernel.h>
22#include <linux/module.h>
23#include <linux/scatterlist.h>
24
25#include "internal.h"
26
27static int init(struct hash_desc *desc)
28{
29 struct crypto_tfm *tfm = crypto_hash_tfm(desc->tfm);
30
31 tfm->__crt_alg->cra_digest.dia_init(tfm);
32 return 0;
33}
34
35static int update2(struct hash_desc *desc,
36 struct scatterlist *sg, unsigned int nbytes)
37{
38 struct crypto_tfm *tfm = crypto_hash_tfm(desc->tfm);
39 unsigned int alignmask = crypto_tfm_alg_alignmask(tfm);
40
41 if (!nbytes)
42 return 0;
43
44 for (;;) {
45 struct page *pg = sg_page(sg);
46 unsigned int offset = sg->offset;
47 unsigned int l = sg->length;
48
49 if (unlikely(l > nbytes))
50 l = nbytes;
51 nbytes -= l;
52
53 do {
54 unsigned int bytes_from_page = min(l, ((unsigned int)
55 (PAGE_SIZE)) -
56 offset);
57 char *src = crypto_kmap(pg, 0);
58 char *p = src + offset;
59
60 if (unlikely(offset & alignmask)) {
61 unsigned int bytes =
62 alignmask + 1 - (offset & alignmask);
63 bytes = min(bytes, bytes_from_page);
64 tfm->__crt_alg->cra_digest.dia_update(tfm, p,
65 bytes);
66 p += bytes;
67 bytes_from_page -= bytes;
68 l -= bytes;
69 }
70 tfm->__crt_alg->cra_digest.dia_update(tfm, p,
71 bytes_from_page);
72 crypto_kunmap(src, 0);
73 crypto_yield(desc->flags);
74 offset = 0;
75 pg++;
76 l -= bytes_from_page;
77 } while (l > 0);
78
79 if (!nbytes)
80 break;
81 sg = scatterwalk_sg_next(sg);
82 }
83
84 return 0;
85}
86
87static int update(struct hash_desc *desc,
88 struct scatterlist *sg, unsigned int nbytes)
89{
90 if (WARN_ON_ONCE(in_irq()))
91 return -EDEADLK;
92 return update2(desc, sg, nbytes);
93}
94
95static int final(struct hash_desc *desc, u8 *out)
96{
97 struct crypto_tfm *tfm = crypto_hash_tfm(desc->tfm);
98 unsigned long alignmask = crypto_tfm_alg_alignmask(tfm);
99 struct digest_alg *digest = &tfm->__crt_alg->cra_digest;
100
101 if (unlikely((unsigned long)out & alignmask)) {
102 unsigned long align = alignmask + 1;
103 unsigned long addr = (unsigned long)crypto_tfm_ctx(tfm);
104 u8 *dst = (u8 *)ALIGN(addr, align) +
105 ALIGN(tfm->__crt_alg->cra_ctxsize, align);
106
107 digest->dia_final(tfm, dst);
108 memcpy(out, dst, digest->dia_digestsize);
109 } else
110 digest->dia_final(tfm, out);
111
112 return 0;
113}
114
115static int nosetkey(struct crypto_hash *tfm, const u8 *key, unsigned int keylen)
116{
117 crypto_hash_clear_flags(tfm, CRYPTO_TFM_RES_MASK);
118 return -ENOSYS;
119}
120
121static int setkey(struct crypto_hash *hash, const u8 *key, unsigned int keylen)
122{
123 struct crypto_tfm *tfm = crypto_hash_tfm(hash);
124
125 crypto_hash_clear_flags(hash, CRYPTO_TFM_RES_MASK);
126 return tfm->__crt_alg->cra_digest.dia_setkey(tfm, key, keylen);
127}
128
129static int digest(struct hash_desc *desc,
130 struct scatterlist *sg, unsigned int nbytes, u8 *out)
131{
132 if (WARN_ON_ONCE(in_irq()))
133 return -EDEADLK;
134
135 init(desc);
136 update2(desc, sg, nbytes);
137 return final(desc, out);
138}
139
140int crypto_init_digest_ops(struct crypto_tfm *tfm)
141{
142 struct hash_tfm *ops = &tfm->crt_hash;
143 struct digest_alg *dalg = &tfm->__crt_alg->cra_digest;
144
145 if (dalg->dia_digestsize > PAGE_SIZE / 8)
146 return -EINVAL;
147
148 ops->init = init;
149 ops->update = update;
150 ops->final = final;
151 ops->digest = digest;
152 ops->setkey = dalg->dia_setkey ? setkey : nosetkey;
153 ops->digestsize = dalg->dia_digestsize;
154
155 return 0;
156}
157
158void crypto_exit_digest_ops(struct crypto_tfm *tfm)
159{
160}
161
162static int digest_async_nosetkey(struct crypto_ahash *tfm_async, const u8 *key,
163 unsigned int keylen)
164{
165 crypto_ahash_clear_flags(tfm_async, CRYPTO_TFM_RES_MASK);
166 return -ENOSYS;
167}
168
169static int digest_async_setkey(struct crypto_ahash *tfm_async, const u8 *key,
170 unsigned int keylen)
171{
172 struct crypto_tfm *tfm = crypto_ahash_tfm(tfm_async);
173 struct digest_alg *dalg = &tfm->__crt_alg->cra_digest;
174
175 crypto_ahash_clear_flags(tfm_async, CRYPTO_TFM_RES_MASK);
176 return dalg->dia_setkey(tfm, key, keylen);
177}
178
179static int digest_async_init(struct ahash_request *req)
180{
181 struct crypto_tfm *tfm = req->base.tfm;
182 struct digest_alg *dalg = &tfm->__crt_alg->cra_digest;
183
184 dalg->dia_init(tfm);
185 return 0;
186}
187
188static int digest_async_update(struct ahash_request *req)
189{
190 struct crypto_tfm *tfm = req->base.tfm;
191 struct hash_desc desc = {
192 .tfm = __crypto_hash_cast(tfm),
193 .flags = req->base.flags,
194 };
195
196 update(&desc, req->src, req->nbytes);
197 return 0;
198}
199
200static int digest_async_final(struct ahash_request *req)
201{
202 struct crypto_tfm *tfm = req->base.tfm;
203 struct hash_desc desc = {
204 .tfm = __crypto_hash_cast(tfm),
205 .flags = req->base.flags,
206 };
207
208 final(&desc, req->result);
209 return 0;
210}
211
212static int digest_async_digest(struct ahash_request *req)
213{
214 struct crypto_tfm *tfm = req->base.tfm;
215 struct hash_desc desc = {
216 .tfm = __crypto_hash_cast(tfm),
217 .flags = req->base.flags,
218 };
219
220 return digest(&desc, req->src, req->nbytes, req->result);
221}
222
223int crypto_init_digest_ops_async(struct crypto_tfm *tfm)
224{
225 struct ahash_tfm *crt = &tfm->crt_ahash;
226 struct digest_alg *dalg = &tfm->__crt_alg->cra_digest;
227
228 if (dalg->dia_digestsize > PAGE_SIZE / 8)
229 return -EINVAL;
230
231 crt->init = digest_async_init;
232 crt->update = digest_async_update;
233 crt->final = digest_async_final;
234 crt->digest = digest_async_digest;
235 crt->setkey = dalg->dia_setkey ? digest_async_setkey :
236 digest_async_nosetkey;
237 crt->digestsize = dalg->dia_digestsize;
238
239 return 0;
240}
diff --git a/crypto/hash.c b/crypto/hash.c
deleted file mode 100644
index cb86b19fd105..000000000000
--- a/crypto/hash.c
+++ /dev/null
@@ -1,183 +0,0 @@
1/*
2 * Cryptographic Hash operations.
3 *
4 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the Free
8 * Software Foundation; either version 2 of the License, or (at your option)
9 * any later version.
10 */
11
12#include <crypto/internal/hash.h>
13#include <linux/errno.h>
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/slab.h>
17#include <linux/seq_file.h>
18
19#include "internal.h"
20
21static unsigned int crypto_hash_ctxsize(struct crypto_alg *alg, u32 type,
22 u32 mask)
23{
24 return alg->cra_ctxsize;
25}
26
27static int hash_setkey_unaligned(struct crypto_hash *crt, const u8 *key,
28 unsigned int keylen)
29{
30 struct crypto_tfm *tfm = crypto_hash_tfm(crt);
31 struct hash_alg *alg = &tfm->__crt_alg->cra_hash;
32 unsigned long alignmask = crypto_hash_alignmask(crt);
33 int ret;
34 u8 *buffer, *alignbuffer;
35 unsigned long absize;
36
37 absize = keylen + alignmask;
38 buffer = kmalloc(absize, GFP_ATOMIC);
39 if (!buffer)
40 return -ENOMEM;
41
42 alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1);
43 memcpy(alignbuffer, key, keylen);
44 ret = alg->setkey(crt, alignbuffer, keylen);
45 memset(alignbuffer, 0, keylen);
46 kfree(buffer);
47 return ret;
48}
49
50static int hash_setkey(struct crypto_hash *crt, const u8 *key,
51 unsigned int keylen)
52{
53 struct crypto_tfm *tfm = crypto_hash_tfm(crt);
54 struct hash_alg *alg = &tfm->__crt_alg->cra_hash;
55 unsigned long alignmask = crypto_hash_alignmask(crt);
56
57 if ((unsigned long)key & alignmask)
58 return hash_setkey_unaligned(crt, key, keylen);
59
60 return alg->setkey(crt, key, keylen);
61}
62
63static int hash_async_setkey(struct crypto_ahash *tfm_async, const u8 *key,
64 unsigned int keylen)
65{
66 struct crypto_tfm *tfm = crypto_ahash_tfm(tfm_async);
67 struct crypto_hash *tfm_hash = __crypto_hash_cast(tfm);
68 struct hash_alg *alg = &tfm->__crt_alg->cra_hash;
69
70 return alg->setkey(tfm_hash, key, keylen);
71}
72
73static int hash_async_init(struct ahash_request *req)
74{
75 struct crypto_tfm *tfm = req->base.tfm;
76 struct hash_alg *alg = &tfm->__crt_alg->cra_hash;
77 struct hash_desc desc = {
78 .tfm = __crypto_hash_cast(tfm),
79 .flags = req->base.flags,
80 };
81
82 return alg->init(&desc);
83}
84
85static int hash_async_update(struct ahash_request *req)
86{
87 struct crypto_tfm *tfm = req->base.tfm;
88 struct hash_alg *alg = &tfm->__crt_alg->cra_hash;
89 struct hash_desc desc = {
90 .tfm = __crypto_hash_cast(tfm),
91 .flags = req->base.flags,
92 };
93
94 return alg->update(&desc, req->src, req->nbytes);
95}
96
97static int hash_async_final(struct ahash_request *req)
98{
99 struct crypto_tfm *tfm = req->base.tfm;
100 struct hash_alg *alg = &tfm->__crt_alg->cra_hash;
101 struct hash_desc desc = {
102 .tfm = __crypto_hash_cast(tfm),
103 .flags = req->base.flags,
104 };
105
106 return alg->final(&desc, req->result);
107}
108
109static int hash_async_digest(struct ahash_request *req)
110{
111 struct crypto_tfm *tfm = req->base.tfm;
112 struct hash_alg *alg = &tfm->__crt_alg->cra_hash;
113 struct hash_desc desc = {
114 .tfm = __crypto_hash_cast(tfm),
115 .flags = req->base.flags,
116 };
117
118 return alg->digest(&desc, req->src, req->nbytes, req->result);
119}
120
121static int crypto_init_hash_ops_async(struct crypto_tfm *tfm)
122{
123 struct ahash_tfm *crt = &tfm->crt_ahash;
124 struct hash_alg *alg = &tfm->__crt_alg->cra_hash;
125
126 crt->init = hash_async_init;
127 crt->update = hash_async_update;
128 crt->final = hash_async_final;
129 crt->digest = hash_async_digest;
130 crt->setkey = hash_async_setkey;
131 crt->digestsize = alg->digestsize;
132
133 return 0;
134}
135
136static int crypto_init_hash_ops_sync(struct crypto_tfm *tfm)
137{
138 struct hash_tfm *crt = &tfm->crt_hash;
139 struct hash_alg *alg = &tfm->__crt_alg->cra_hash;
140
141 crt->init = alg->init;
142 crt->update = alg->update;
143 crt->final = alg->final;
144 crt->digest = alg->digest;
145 crt->setkey = hash_setkey;
146 crt->digestsize = alg->digestsize;
147
148 return 0;
149}
150
151static int crypto_init_hash_ops(struct crypto_tfm *tfm, u32 type, u32 mask)
152{
153 struct hash_alg *alg = &tfm->__crt_alg->cra_hash;
154
155 if (alg->digestsize > PAGE_SIZE / 8)
156 return -EINVAL;
157
158 if ((mask & CRYPTO_ALG_TYPE_HASH_MASK) != CRYPTO_ALG_TYPE_HASH_MASK)
159 return crypto_init_hash_ops_async(tfm);
160 else
161 return crypto_init_hash_ops_sync(tfm);
162}
163
164static void crypto_hash_show(struct seq_file *m, struct crypto_alg *alg)
165 __attribute__ ((unused));
166static void crypto_hash_show(struct seq_file *m, struct crypto_alg *alg)
167{
168 seq_printf(m, "type : hash\n");
169 seq_printf(m, "blocksize : %u\n", alg->cra_blocksize);
170 seq_printf(m, "digestsize : %u\n", alg->cra_hash.digestsize);
171}
172
173const struct crypto_type crypto_hash_type = {
174 .ctxsize = crypto_hash_ctxsize,
175 .init = crypto_init_hash_ops,
176#ifdef CONFIG_PROC_FS
177 .show = crypto_hash_show,
178#endif
179};
180EXPORT_SYMBOL_GPL(crypto_hash_type);
181
182MODULE_LICENSE("GPL");
183MODULE_DESCRIPTION("Generic cryptographic hash type");
diff --git a/crypto/proc.c b/crypto/proc.c
index 1c38733c224d..58fef67d4f4d 100644
--- a/crypto/proc.c
+++ b/crypto/proc.c
@@ -109,13 +109,6 @@ static int c_show(struct seq_file *m, void *p)
109 seq_printf(m, "max keysize : %u\n", 109 seq_printf(m, "max keysize : %u\n",
110 alg->cra_cipher.cia_max_keysize); 110 alg->cra_cipher.cia_max_keysize);
111 break; 111 break;
112
113 case CRYPTO_ALG_TYPE_DIGEST:
114 seq_printf(m, "type : digest\n");
115 seq_printf(m, "blocksize : %u\n", alg->cra_blocksize);
116 seq_printf(m, "digestsize : %u\n",
117 alg->cra_digest.dia_digestsize);
118 break;
119 case CRYPTO_ALG_TYPE_COMPRESS: 112 case CRYPTO_ALG_TYPE_COMPRESS:
120 seq_printf(m, "type : compression\n"); 113 seq_printf(m, "type : compression\n");
121 break; 114 break;
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 6d5b746637be..7620bfce92f2 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -1201,7 +1201,7 @@ static int test_cprng(struct crypto_rng *tfm, struct cprng_testvec *template,
1201 unsigned int tcount) 1201 unsigned int tcount)
1202{ 1202{
1203 const char *algo = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm)); 1203 const char *algo = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm));
1204 int err, i, j, seedsize; 1204 int err = 0, i, j, seedsize;
1205 u8 *seed; 1205 u8 *seed;
1206 char result[32]; 1206 char result[32];
1207 1207
@@ -1943,6 +1943,15 @@ static const struct alg_test_desc alg_test_descs[] = {
1943 } 1943 }
1944 } 1944 }
1945 }, { 1945 }, {
1946 .alg = "ghash",
1947 .test = alg_test_hash,
1948 .suite = {
1949 .hash = {
1950 .vecs = ghash_tv_template,
1951 .count = GHASH_TEST_VECTORS
1952 }
1953 }
1954 }, {
1946 .alg = "hmac(md5)", 1955 .alg = "hmac(md5)",
1947 .test = alg_test_hash, 1956 .test = alg_test_hash,
1948 .suite = { 1957 .suite = {
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 9963b18983ab..fb765173d41c 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -1003,6 +1003,21 @@ static struct hash_testvec tgr128_tv_template[] = {
1003 }, 1003 },
1004}; 1004};
1005 1005
1006#define GHASH_TEST_VECTORS 1
1007
1008static struct hash_testvec ghash_tv_template[] =
1009{
1010 {
1011
1012 .key = "\xdf\xa6\xbf\x4d\xed\x81\xdb\x03\xff\xca\xff\x95\xf8\x30\xf0\x61",
1013 .ksize = 16,
1014 .plaintext = "\x95\x2b\x2a\x56\xa5\x60\x04a\xc0\xb3\x2b\x66\x56\xa0\x5b\x40\xb6",
1015 .psize = 16,
1016 .digest = "\xda\x53\xeb\x0a\xd2\xc5\x5b\xb6"
1017 "\x4f\xc4\x80\x2c\xc3\xfe\xda\x60",
1018 },
1019};
1020
1006/* 1021/*
1007 * HMAC-MD5 test vectors from RFC2202 1022 * HMAC-MD5 test vectors from RFC2202
1008 * (These need to be fixed to not use strlen). 1023 * (These need to be fixed to not use strlen).
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index 1573aebd54b5..8b7d56a0fe3a 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -52,7 +52,9 @@
52static struct hwrng *current_rng; 52static struct hwrng *current_rng;
53static LIST_HEAD(rng_list); 53static LIST_HEAD(rng_list);
54static DEFINE_MUTEX(rng_mutex); 54static DEFINE_MUTEX(rng_mutex);
55 55static int data_avail;
56static u8 rng_buffer[SMP_CACHE_BYTES < 32 ? 32 : SMP_CACHE_BYTES]
57 __cacheline_aligned;
56 58
57static inline int hwrng_init(struct hwrng *rng) 59static inline int hwrng_init(struct hwrng *rng)
58{ 60{
@@ -67,19 +69,6 @@ static inline void hwrng_cleanup(struct hwrng *rng)
67 rng->cleanup(rng); 69 rng->cleanup(rng);
68} 70}
69 71
70static inline int hwrng_data_present(struct hwrng *rng, int wait)
71{
72 if (!rng->data_present)
73 return 1;
74 return rng->data_present(rng, wait);
75}
76
77static inline int hwrng_data_read(struct hwrng *rng, u32 *data)
78{
79 return rng->data_read(rng, data);
80}
81
82
83static int rng_dev_open(struct inode *inode, struct file *filp) 72static int rng_dev_open(struct inode *inode, struct file *filp)
84{ 73{
85 /* enforce read-only access to this chrdev */ 74 /* enforce read-only access to this chrdev */
@@ -91,54 +80,87 @@ static int rng_dev_open(struct inode *inode, struct file *filp)
91 return 0; 80 return 0;
92} 81}
93 82
83static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size,
84 int wait) {
85 int present;
86
87 if (rng->read)
88 return rng->read(rng, (void *)buffer, size, wait);
89
90 if (rng->data_present)
91 present = rng->data_present(rng, wait);
92 else
93 present = 1;
94
95 if (present)
96 return rng->data_read(rng, (u32 *)buffer);
97
98 return 0;
99}
100
94static ssize_t rng_dev_read(struct file *filp, char __user *buf, 101static ssize_t rng_dev_read(struct file *filp, char __user *buf,
95 size_t size, loff_t *offp) 102 size_t size, loff_t *offp)
96{ 103{
97 u32 data;
98 ssize_t ret = 0; 104 ssize_t ret = 0;
99 int err = 0; 105 int err = 0;
100 int bytes_read; 106 int bytes_read, len;
101 107
102 while (size) { 108 while (size) {
103 err = -ERESTARTSYS; 109 if (mutex_lock_interruptible(&rng_mutex)) {
104 if (mutex_lock_interruptible(&rng_mutex)) 110 err = -ERESTARTSYS;
105 goto out; 111 goto out;
112 }
113
106 if (!current_rng) { 114 if (!current_rng) {
107 mutex_unlock(&rng_mutex);
108 err = -ENODEV; 115 err = -ENODEV;
109 goto out; 116 goto out_unlock;
110 } 117 }
111 118
112 bytes_read = 0; 119 if (!data_avail) {
113 if (hwrng_data_present(current_rng, 120 bytes_read = rng_get_data(current_rng, rng_buffer,
114 !(filp->f_flags & O_NONBLOCK))) 121 sizeof(rng_buffer),
115 bytes_read = hwrng_data_read(current_rng, &data); 122 !(filp->f_flags & O_NONBLOCK));
116 mutex_unlock(&rng_mutex); 123 if (bytes_read < 0) {
117 124 err = bytes_read;
118 err = -EAGAIN; 125 goto out_unlock;
119 if (!bytes_read && (filp->f_flags & O_NONBLOCK)) 126 }
120 goto out; 127 data_avail = bytes_read;
121 if (bytes_read < 0) {
122 err = bytes_read;
123 goto out;
124 } 128 }
125 129
126 err = -EFAULT; 130 if (!data_avail) {
127 while (bytes_read && size) { 131 if (filp->f_flags & O_NONBLOCK) {
128 if (put_user((u8)data, buf++)) 132 err = -EAGAIN;
129 goto out; 133 goto out_unlock;
130 size--; 134 }
131 ret++; 135 } else {
132 bytes_read--; 136 len = data_avail;
133 data >>= 8; 137 if (len > size)
138 len = size;
139
140 data_avail -= len;
141
142 if (copy_to_user(buf + ret, rng_buffer + data_avail,
143 len)) {
144 err = -EFAULT;
145 goto out_unlock;
146 }
147
148 size -= len;
149 ret += len;
134 } 150 }
135 151
152 mutex_unlock(&rng_mutex);
153
136 if (need_resched()) 154 if (need_resched())
137 schedule_timeout_interruptible(1); 155 schedule_timeout_interruptible(1);
138 err = -ERESTARTSYS; 156
139 if (signal_pending(current)) 157 if (signal_pending(current)) {
158 err = -ERESTARTSYS;
140 goto out; 159 goto out;
160 }
141 } 161 }
162out_unlock:
163 mutex_unlock(&rng_mutex);
142out: 164out:
143 return ret ? : err; 165 return ret ? : err;
144} 166}
@@ -280,7 +302,7 @@ int hwrng_register(struct hwrng *rng)
280 struct hwrng *old_rng, *tmp; 302 struct hwrng *old_rng, *tmp;
281 303
282 if (rng->name == NULL || 304 if (rng->name == NULL ||
283 rng->data_read == NULL) 305 (rng->data_read == NULL && rng->read == NULL))
284 goto out; 306 goto out;
285 307
286 mutex_lock(&rng_mutex); 308 mutex_lock(&rng_mutex);
diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c
index 915157fcff98..bdaef8e94021 100644
--- a/drivers/char/hw_random/virtio-rng.c
+++ b/drivers/char/hw_random/virtio-rng.c
@@ -16,6 +16,7 @@
16 * along with this program; if not, write to the Free Software 16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 17 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18 */ 18 */
19
19#include <linux/err.h> 20#include <linux/err.h>
20#include <linux/hw_random.h> 21#include <linux/hw_random.h>
21#include <linux/scatterlist.h> 22#include <linux/scatterlist.h>
@@ -23,78 +24,64 @@
23#include <linux/virtio.h> 24#include <linux/virtio.h>
24#include <linux/virtio_rng.h> 25#include <linux/virtio_rng.h>
25 26
26/* The host will fill any buffer we give it with sweet, sweet randomness. We
27 * give it 64 bytes at a time, and the hwrng framework takes it 4 bytes at a
28 * time. */
29#define RANDOM_DATA_SIZE 64
30
31static struct virtqueue *vq; 27static struct virtqueue *vq;
32static u32 *random_data; 28static unsigned int data_avail;
33static unsigned int data_left;
34static DECLARE_COMPLETION(have_data); 29static DECLARE_COMPLETION(have_data);
30static bool busy;
35 31
36static void random_recv_done(struct virtqueue *vq) 32static void random_recv_done(struct virtqueue *vq)
37{ 33{
38 unsigned int len;
39
40 /* We can get spurious callbacks, e.g. shared IRQs + virtio_pci. */ 34 /* We can get spurious callbacks, e.g. shared IRQs + virtio_pci. */
41 if (!vq->vq_ops->get_buf(vq, &len)) 35 if (!vq->vq_ops->get_buf(vq, &data_avail))
42 return; 36 return;
43 37
44 data_left += len;
45 complete(&have_data); 38 complete(&have_data);
46} 39}
47 40
48static void register_buffer(void) 41/* The host will fill any buffer we give it with sweet, sweet randomness. */
42static void register_buffer(u8 *buf, size_t size)
49{ 43{
50 struct scatterlist sg; 44 struct scatterlist sg;
51 45
52 sg_init_one(&sg, random_data+data_left, RANDOM_DATA_SIZE-data_left); 46 sg_init_one(&sg, buf, size);
47
53 /* There should always be room for one buffer. */ 48 /* There should always be room for one buffer. */
54 if (vq->vq_ops->add_buf(vq, &sg, 0, 1, random_data) < 0) 49 if (vq->vq_ops->add_buf(vq, &sg, 0, 1, buf) < 0)
55 BUG(); 50 BUG();
51
56 vq->vq_ops->kick(vq); 52 vq->vq_ops->kick(vq);
57} 53}
58 54
59/* At least we don't udelay() in a loop like some other drivers. */ 55static int virtio_read(struct hwrng *rng, void *buf, size_t size, bool wait)
60static int virtio_data_present(struct hwrng *rng, int wait)
61{ 56{
62 if (data_left >= sizeof(u32))
63 return 1;
64 57
65again: 58 if (!busy) {
59 busy = true;
60 init_completion(&have_data);
61 register_buffer(buf, size);
62 }
63
66 if (!wait) 64 if (!wait)
67 return 0; 65 return 0;
68 66
69 wait_for_completion(&have_data); 67 wait_for_completion(&have_data);
70 68
71 /* Not enough? Re-register. */ 69 busy = false;
72 if (unlikely(data_left < sizeof(u32))) {
73 register_buffer();
74 goto again;
75 }
76 70
77 return 1; 71 return data_avail;
78} 72}
79 73
80/* virtio_data_present() must have succeeded before this is called. */ 74static void virtio_cleanup(struct hwrng *rng)
81static int virtio_data_read(struct hwrng *rng, u32 *data)
82{ 75{
83 BUG_ON(data_left < sizeof(u32)); 76 if (busy)
84 data_left -= sizeof(u32); 77 wait_for_completion(&have_data);
85 *data = random_data[data_left / 4];
86
87 if (data_left < sizeof(u32)) {
88 init_completion(&have_data);
89 register_buffer();
90 }
91 return sizeof(*data);
92} 78}
93 79
80
94static struct hwrng virtio_hwrng = { 81static struct hwrng virtio_hwrng = {
95 .name = "virtio", 82 .name = "virtio",
96 .data_present = virtio_data_present, 83 .cleanup = virtio_cleanup,
97 .data_read = virtio_data_read, 84 .read = virtio_read,
98}; 85};
99 86
100static int virtrng_probe(struct virtio_device *vdev) 87static int virtrng_probe(struct virtio_device *vdev)
@@ -112,7 +99,6 @@ static int virtrng_probe(struct virtio_device *vdev)
112 return err; 99 return err;
113 } 100 }
114 101
115 register_buffer();
116 return 0; 102 return 0;
117} 103}
118 104
@@ -138,21 +124,11 @@ static struct virtio_driver virtio_rng = {
138 124
139static int __init init(void) 125static int __init init(void)
140{ 126{
141 int err; 127 return register_virtio_driver(&virtio_rng);
142
143 random_data = kmalloc(RANDOM_DATA_SIZE, GFP_KERNEL);
144 if (!random_data)
145 return -ENOMEM;
146
147 err = register_virtio_driver(&virtio_rng);
148 if (err)
149 kfree(random_data);
150 return err;
151} 128}
152 129
153static void __exit fini(void) 130static void __exit fini(void)
154{ 131{
155 kfree(random_data);
156 unregister_virtio_driver(&virtio_rng); 132 unregister_virtio_driver(&virtio_rng);
157} 133}
158module_init(init); 134module_init(init);
diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h
index 1ffb53f74d37..fc0d575c71e0 100644
--- a/include/crypto/algapi.h
+++ b/include/crypto/algapi.h
@@ -106,7 +106,6 @@ struct blkcipher_walk {
106extern const struct crypto_type crypto_ablkcipher_type; 106extern const struct crypto_type crypto_ablkcipher_type;
107extern const struct crypto_type crypto_aead_type; 107extern const struct crypto_type crypto_aead_type;
108extern const struct crypto_type crypto_blkcipher_type; 108extern const struct crypto_type crypto_blkcipher_type;
109extern const struct crypto_type crypto_hash_type;
110 109
111void crypto_mod_put(struct crypto_alg *alg); 110void crypto_mod_put(struct crypto_alg *alg);
112 111
diff --git a/include/crypto/cryptd.h b/include/crypto/cryptd.h
index 2f65a6e8ea4d..1c96b255017c 100644
--- a/include/crypto/cryptd.h
+++ b/include/crypto/cryptd.h
@@ -39,6 +39,7 @@ static inline struct cryptd_ahash *__cryptd_ahash_cast(
39struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, 39struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
40 u32 type, u32 mask); 40 u32 type, u32 mask);
41struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm); 41struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm);
42struct shash_desc *cryptd_shash_desc(struct ahash_request *req);
42void cryptd_free_ahash(struct cryptd_ahash *tfm); 43void cryptd_free_ahash(struct cryptd_ahash *tfm);
43 44
44#endif 45#endif
diff --git a/include/linux/crypto.h b/include/linux/crypto.h
index fd929889e8dc..24d2e30f1b46 100644
--- a/include/linux/crypto.h
+++ b/include/linux/crypto.h
@@ -250,29 +250,6 @@ struct cipher_alg {
250 void (*cia_decrypt)(struct crypto_tfm *tfm, u8 *dst, const u8 *src); 250 void (*cia_decrypt)(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
251}; 251};
252 252
253struct digest_alg {
254 unsigned int dia_digestsize;
255 void (*dia_init)(struct crypto_tfm *tfm);
256 void (*dia_update)(struct crypto_tfm *tfm, const u8 *data,
257 unsigned int len);
258 void (*dia_final)(struct crypto_tfm *tfm, u8 *out);
259 int (*dia_setkey)(struct crypto_tfm *tfm, const u8 *key,
260 unsigned int keylen);
261};
262
263struct hash_alg {
264 int (*init)(struct hash_desc *desc);
265 int (*update)(struct hash_desc *desc, struct scatterlist *sg,
266 unsigned int nbytes);
267 int (*final)(struct hash_desc *desc, u8 *out);
268 int (*digest)(struct hash_desc *desc, struct scatterlist *sg,
269 unsigned int nbytes, u8 *out);
270 int (*setkey)(struct crypto_hash *tfm, const u8 *key,
271 unsigned int keylen);
272
273 unsigned int digestsize;
274};
275
276struct compress_alg { 253struct compress_alg {
277 int (*coa_compress)(struct crypto_tfm *tfm, const u8 *src, 254 int (*coa_compress)(struct crypto_tfm *tfm, const u8 *src,
278 unsigned int slen, u8 *dst, unsigned int *dlen); 255 unsigned int slen, u8 *dst, unsigned int *dlen);
@@ -293,8 +270,6 @@ struct rng_alg {
293#define cra_aead cra_u.aead 270#define cra_aead cra_u.aead
294#define cra_blkcipher cra_u.blkcipher 271#define cra_blkcipher cra_u.blkcipher
295#define cra_cipher cra_u.cipher 272#define cra_cipher cra_u.cipher
296#define cra_digest cra_u.digest
297#define cra_hash cra_u.hash
298#define cra_compress cra_u.compress 273#define cra_compress cra_u.compress
299#define cra_rng cra_u.rng 274#define cra_rng cra_u.rng
300 275
@@ -320,8 +295,6 @@ struct crypto_alg {
320 struct aead_alg aead; 295 struct aead_alg aead;
321 struct blkcipher_alg blkcipher; 296 struct blkcipher_alg blkcipher;
322 struct cipher_alg cipher; 297 struct cipher_alg cipher;
323 struct digest_alg digest;
324 struct hash_alg hash;
325 struct compress_alg compress; 298 struct compress_alg compress;
326 struct rng_alg rng; 299 struct rng_alg rng;
327 } cra_u; 300 } cra_u;
diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h
index 7244456e7e65..9bede7633f74 100644
--- a/include/linux/hw_random.h
+++ b/include/linux/hw_random.h
@@ -22,10 +22,12 @@
22 * @cleanup: Cleanup callback (can be NULL). 22 * @cleanup: Cleanup callback (can be NULL).
23 * @data_present: Callback to determine if data is available 23 * @data_present: Callback to determine if data is available
24 * on the RNG. If NULL, it is assumed that 24 * on the RNG. If NULL, it is assumed that
25 * there is always data available. 25 * there is always data available. *OBSOLETE*
26 * @data_read: Read data from the RNG device. 26 * @data_read: Read data from the RNG device.
27 * Returns the number of lower random bytes in "data". 27 * Returns the number of lower random bytes in "data".
28 * Must not be NULL. 28 * Must not be NULL. *OSOLETE*
29 * @read: New API. drivers can fill up to max bytes of data
30 * into the buffer. The buffer is aligned for any type.
29 * @priv: Private data, for use by the RNG driver. 31 * @priv: Private data, for use by the RNG driver.
30 */ 32 */
31struct hwrng { 33struct hwrng {
@@ -34,6 +36,7 @@ struct hwrng {
34 void (*cleanup)(struct hwrng *rng); 36 void (*cleanup)(struct hwrng *rng);
35 int (*data_present)(struct hwrng *rng, int wait); 37 int (*data_present)(struct hwrng *rng, int wait);
36 int (*data_read)(struct hwrng *rng, u32 *data); 38 int (*data_read)(struct hwrng *rng, u32 *data);
39 int (*read)(struct hwrng *rng, void *data, size_t max, bool wait);
37 unsigned long priv; 40 unsigned long priv;
38 41
39 /* internal. */ 42 /* internal. */
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-09 23:14:56 -0500