aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm/mach-omap2/clock2420_data.c2
-rw-r--r--arch/arm/mach-omap2/clock2430_data.c2
-rw-r--r--arch/arm/mach-omap2/clock3xxx_data.c2
-rw-r--r--arch/arm/mach-omap2/devices.c58
-rw-r--r--arch/arm/plat-omap/include/plat/omap34xx.h5
-rw-r--r--arch/x86/crypto/aesni-intel_asm.S115
-rw-r--r--arch/x86/crypto/aesni-intel_glue.c130
-rw-r--r--arch/x86/include/asm/inst.h96
-rw-r--r--crypto/ablkcipher.c277
-rw-r--r--crypto/algapi.c2
-rw-r--r--crypto/authenc.c8
-rw-r--r--crypto/internal.h2
-rw-r--r--crypto/pcrypt.c11
-rw-r--r--crypto/scatterwalk.c2
-rw-r--r--crypto/shash.c2
-rw-r--r--crypto/tcrypt.c343
-rw-r--r--crypto/tcrypt.h29
-rw-r--r--crypto/testmgr.c66
-rw-r--r--crypto/testmgr.h64
-rw-r--r--crypto/vmac.c75
-rw-r--r--drivers/char/random.c10
-rw-r--r--drivers/crypto/Kconfig21
-rw-r--r--drivers/crypto/Makefile4
-rw-r--r--drivers/crypto/geode-aes.c36
-rw-r--r--drivers/crypto/hifn_795x.c18
-rw-r--r--drivers/crypto/mv_cesa.c692
-rw-r--r--drivers/crypto/mv_cesa.h40
-rw-r--r--drivers/crypto/n2_asm.S95
-rw-r--r--drivers/crypto/n2_core.c2083
-rw-r--r--drivers/crypto/n2_core.h231
-rw-r--r--drivers/crypto/omap-sham.c1259
-rw-r--r--drivers/crypto/talitos.c699
-rw-r--r--drivers/crypto/talitos.h12
-rw-r--r--include/crypto/algapi.h40
-rw-r--r--include/linux/padata.h55
-rw-r--r--kernel/padata.c185
36 files changed, 6456 insertions, 315 deletions
diff --git a/arch/arm/mach-omap2/clock2420_data.c b/arch/arm/mach-omap2/clock2420_data.c
index 23bc981574f6..37d65d62ed8f 100644
--- a/arch/arm/mach-omap2/clock2420_data.c
+++ b/arch/arm/mach-omap2/clock2420_data.c
@@ -1836,7 +1836,7 @@ static struct omap_clk omap2420_clks[] = {
1836 CLK(NULL, "vlynq_ick", &vlynq_ick, CK_242X), 1836 CLK(NULL, "vlynq_ick", &vlynq_ick, CK_242X),
1837 CLK(NULL, "vlynq_fck", &vlynq_fck, CK_242X), 1837 CLK(NULL, "vlynq_fck", &vlynq_fck, CK_242X),
1838 CLK(NULL, "des_ick", &des_ick, CK_242X), 1838 CLK(NULL, "des_ick", &des_ick, CK_242X),
1839 CLK(NULL, "sha_ick", &sha_ick, CK_242X), 1839 CLK("omap-sham", "ick", &sha_ick, CK_242X),
1840 CLK("omap_rng", "ick", &rng_ick, CK_242X), 1840 CLK("omap_rng", "ick", &rng_ick, CK_242X),
1841 CLK(NULL, "aes_ick", &aes_ick, CK_242X), 1841 CLK(NULL, "aes_ick", &aes_ick, CK_242X),
1842 CLK(NULL, "pka_ick", &pka_ick, CK_242X), 1842 CLK(NULL, "pka_ick", &pka_ick, CK_242X),
diff --git a/arch/arm/mach-omap2/clock2430_data.c b/arch/arm/mach-omap2/clock2430_data.c
index 2df50d97deb2..b33118fb6a87 100644
--- a/arch/arm/mach-omap2/clock2430_data.c
+++ b/arch/arm/mach-omap2/clock2430_data.c
@@ -1924,7 +1924,7 @@ static struct omap_clk omap2430_clks[] = {
1924 CLK(NULL, "sdma_ick", &sdma_ick, CK_243X), 1924 CLK(NULL, "sdma_ick", &sdma_ick, CK_243X),
1925 CLK(NULL, "sdrc_ick", &sdrc_ick, CK_243X), 1925 CLK(NULL, "sdrc_ick", &sdrc_ick, CK_243X),
1926 CLK(NULL, "des_ick", &des_ick, CK_243X), 1926 CLK(NULL, "des_ick", &des_ick, CK_243X),
1927 CLK(NULL, "sha_ick", &sha_ick, CK_243X), 1927 CLK("omap-sham", "ick", &sha_ick, CK_243X),
1928 CLK("omap_rng", "ick", &rng_ick, CK_243X), 1928 CLK("omap_rng", "ick", &rng_ick, CK_243X),
1929 CLK(NULL, "aes_ick", &aes_ick, CK_243X), 1929 CLK(NULL, "aes_ick", &aes_ick, CK_243X),
1930 CLK(NULL, "pka_ick", &pka_ick, CK_243X), 1930 CLK(NULL, "pka_ick", &pka_ick, CK_243X),
diff --git a/arch/arm/mach-omap2/clock3xxx_data.c b/arch/arm/mach-omap2/clock3xxx_data.c
index 833be485c89e..41b155acfca7 100644
--- a/arch/arm/mach-omap2/clock3xxx_data.c
+++ b/arch/arm/mach-omap2/clock3xxx_data.c
@@ -3284,7 +3284,7 @@ static struct omap_clk omap3xxx_clks[] = {
3284 CLK("mmci-omap-hs.2", "ick", &mmchs3_ick, CK_3430ES2 | CK_AM35XX), 3284 CLK("mmci-omap-hs.2", "ick", &mmchs3_ick, CK_3430ES2 | CK_AM35XX),
3285 CLK(NULL, "icr_ick", &icr_ick, CK_343X), 3285 CLK(NULL, "icr_ick", &icr_ick, CK_343X),
3286 CLK(NULL, "aes2_ick", &aes2_ick, CK_343X), 3286 CLK(NULL, "aes2_ick", &aes2_ick, CK_343X),
3287 CLK(NULL, "sha12_ick", &sha12_ick, CK_343X), 3287 CLK("omap-sham", "ick", &sha12_ick, CK_343X),
3288 CLK(NULL, "des2_ick", &des2_ick, CK_343X), 3288 CLK(NULL, "des2_ick", &des2_ick, CK_343X),
3289 CLK("mmci-omap-hs.1", "ick", &mmchs2_ick, CK_3XXX), 3289 CLK("mmci-omap-hs.1", "ick", &mmchs2_ick, CK_3XXX),
3290 CLK("mmci-omap-hs.0", "ick", &mmchs1_ick, CK_3XXX), 3290 CLK("mmci-omap-hs.0", "ick", &mmchs1_ick, CK_3XXX),
diff --git a/arch/arm/mach-omap2/devices.c b/arch/arm/mach-omap2/devices.c
index 705a7a30a87f..03e6c9ed82a4 100644
--- a/arch/arm/mach-omap2/devices.c
+++ b/arch/arm/mach-omap2/devices.c
@@ -28,6 +28,7 @@
28#include <plat/mux.h> 28#include <plat/mux.h>
29#include <mach/gpio.h> 29#include <mach/gpio.h>
30#include <plat/mmc.h> 30#include <plat/mmc.h>
31#include <plat/dma.h>
31 32
32#include "mux.h" 33#include "mux.h"
33 34
@@ -486,8 +487,10 @@ static void omap_init_pmu(void)
486} 487}
487 488
488 489
489#ifdef CONFIG_OMAP_SHA1_MD5 490#if defined(CONFIG_CRYPTO_DEV_OMAP_SHAM) || defined(CONFIG_CRYPTO_DEV_OMAP_SHAM_MODULE)
490static struct resource sha1_md5_resources[] = { 491
492#ifdef CONFIG_ARCH_OMAP2
493static struct resource omap2_sham_resources[] = {
491 { 494 {
492 .start = OMAP24XX_SEC_SHA1MD5_BASE, 495 .start = OMAP24XX_SEC_SHA1MD5_BASE,
493 .end = OMAP24XX_SEC_SHA1MD5_BASE + 0x64, 496 .end = OMAP24XX_SEC_SHA1MD5_BASE + 0x64,
@@ -498,20 +501,55 @@ static struct resource sha1_md5_resources[] = {
498 .flags = IORESOURCE_IRQ, 501 .flags = IORESOURCE_IRQ,
499 } 502 }
500}; 503};
504static int omap2_sham_resources_sz = ARRAY_SIZE(omap2_sham_resources);
505#else
506#define omap2_sham_resources NULL
507#define omap2_sham_resources_sz 0
508#endif
501 509
502static struct platform_device sha1_md5_device = { 510#ifdef CONFIG_ARCH_OMAP3
503 .name = "OMAP SHA1/MD5", 511static struct resource omap3_sham_resources[] = {
512 {
513 .start = OMAP34XX_SEC_SHA1MD5_BASE,
514 .end = OMAP34XX_SEC_SHA1MD5_BASE + 0x64,
515 .flags = IORESOURCE_MEM,
516 },
517 {
518 .start = INT_34XX_SHA1MD52_IRQ,
519 .flags = IORESOURCE_IRQ,
520 },
521 {
522 .start = OMAP34XX_DMA_SHA1MD5_RX,
523 .flags = IORESOURCE_DMA,
524 }
525};
526static int omap3_sham_resources_sz = ARRAY_SIZE(omap3_sham_resources);
527#else
528#define omap3_sham_resources NULL
529#define omap3_sham_resources_sz 0
530#endif
531
532static struct platform_device sham_device = {
533 .name = "omap-sham",
504 .id = -1, 534 .id = -1,
505 .num_resources = ARRAY_SIZE(sha1_md5_resources),
506 .resource = sha1_md5_resources,
507}; 535};
508 536
509static void omap_init_sha1_md5(void) 537static void omap_init_sham(void)
510{ 538{
511 platform_device_register(&sha1_md5_device); 539 if (cpu_is_omap24xx()) {
540 sham_device.resource = omap2_sham_resources;
541 sham_device.num_resources = omap2_sham_resources_sz;
542 } else if (cpu_is_omap34xx()) {
543 sham_device.resource = omap3_sham_resources;
544 sham_device.num_resources = omap3_sham_resources_sz;
545 } else {
546 pr_err("%s: platform not supported\n", __func__);
547 return;
548 }
549 platform_device_register(&sham_device);
512} 550}
513#else 551#else
514static inline void omap_init_sha1_md5(void) { } 552static inline void omap_init_sham(void) { }
515#endif 553#endif
516 554
517/*-------------------------------------------------------------------------*/ 555/*-------------------------------------------------------------------------*/
@@ -869,7 +907,7 @@ static int __init omap2_init_devices(void)
869 omap_init_pmu(); 907 omap_init_pmu();
870 omap_hdq_init(); 908 omap_hdq_init();
871 omap_init_sti(); 909 omap_init_sti();
872 omap_init_sha1_md5(); 910 omap_init_sham();
873 omap_init_vout(); 911 omap_init_vout();
874 912
875 return 0; 913 return 0;
diff --git a/arch/arm/plat-omap/include/plat/omap34xx.h b/arch/arm/plat-omap/include/plat/omap34xx.h
index 2845fdc658b0..98fc8b4a4cc4 100644
--- a/arch/arm/plat-omap/include/plat/omap34xx.h
+++ b/arch/arm/plat-omap/include/plat/omap34xx.h
@@ -82,5 +82,10 @@
82 82
83#define OMAP34XX_MAILBOX_BASE (L4_34XX_BASE + 0x94000) 83#define OMAP34XX_MAILBOX_BASE (L4_34XX_BASE + 0x94000)
84 84
85/* Security */
86#define OMAP34XX_SEC_BASE (L4_34XX_BASE + 0xA0000)
87#define OMAP34XX_SEC_SHA1MD5_BASE (OMAP34XX_SEC_BASE + 0x23000)
88#define OMAP34XX_SEC_AES_BASE (OMAP34XX_SEC_BASE + 0x25000)
89
85#endif /* __ASM_ARCH_OMAP3_H */ 90#endif /* __ASM_ARCH_OMAP3_H */
86 91
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 20bb0e1ac681..ff16756a51c1 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -32,6 +32,9 @@
32#define IN IN1 32#define IN IN1
33#define KEY %xmm2 33#define KEY %xmm2
34#define IV %xmm3 34#define IV %xmm3
35#define BSWAP_MASK %xmm10
36#define CTR %xmm11
37#define INC %xmm12
35 38
36#define KEYP %rdi 39#define KEYP %rdi
37#define OUTP %rsi 40#define OUTP %rsi
@@ -42,6 +45,7 @@
42#define T1 %r10 45#define T1 %r10
43#define TKEYP T1 46#define TKEYP T1
44#define T2 %r11 47#define T2 %r11
48#define TCTR_LOW T2
45 49
46_key_expansion_128: 50_key_expansion_128:
47_key_expansion_256a: 51_key_expansion_256a:
@@ -724,3 +728,114 @@ ENTRY(aesni_cbc_dec)
724 movups IV, (IVP) 728 movups IV, (IVP)
725.Lcbc_dec_just_ret: 729.Lcbc_dec_just_ret:
726 ret 730 ret
731
732.align 16
733.Lbswap_mask:
734 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
735
736/*
737 * _aesni_inc_init: internal ABI
738 * setup registers used by _aesni_inc
739 * input:
740 * IV
741 * output:
742 * CTR: == IV, in little endian
743 * TCTR_LOW: == lower qword of CTR
744 * INC: == 1, in little endian
745 * BSWAP_MASK == endian swapping mask
746 */
747_aesni_inc_init:
748 movaps .Lbswap_mask, BSWAP_MASK
749 movaps IV, CTR
750 PSHUFB_XMM BSWAP_MASK CTR
751 mov $1, TCTR_LOW
752 MOVQ_R64_XMM TCTR_LOW INC
753 MOVQ_R64_XMM CTR TCTR_LOW
754 ret
755
756/*
757 * _aesni_inc: internal ABI
758 * Increase IV by 1, IV is in big endian
759 * input:
760 * IV
761 * CTR: == IV, in little endian
762 * TCTR_LOW: == lower qword of CTR
763 * INC: == 1, in little endian
764 * BSWAP_MASK == endian swapping mask
765 * output:
766 * IV: Increase by 1
767 * changed:
768 * CTR: == output IV, in little endian
769 * TCTR_LOW: == lower qword of CTR
770 */
771_aesni_inc:
772 paddq INC, CTR
773 add $1, TCTR_LOW
774 jnc .Linc_low
775 pslldq $8, INC
776 paddq INC, CTR
777 psrldq $8, INC
778.Linc_low:
779 movaps CTR, IV
780 PSHUFB_XMM BSWAP_MASK IV
781 ret
782
783/*
784 * void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
785 * size_t len, u8 *iv)
786 */
787ENTRY(aesni_ctr_enc)
788 cmp $16, LEN
789 jb .Lctr_enc_just_ret
790 mov 480(KEYP), KLEN
791 movups (IVP), IV
792 call _aesni_inc_init
793 cmp $64, LEN
794 jb .Lctr_enc_loop1
795.align 4
796.Lctr_enc_loop4:
797 movaps IV, STATE1
798 call _aesni_inc
799 movups (INP), IN1
800 movaps IV, STATE2
801 call _aesni_inc
802 movups 0x10(INP), IN2
803 movaps IV, STATE3
804 call _aesni_inc
805 movups 0x20(INP), IN3
806 movaps IV, STATE4
807 call _aesni_inc
808 movups 0x30(INP), IN4
809 call _aesni_enc4
810 pxor IN1, STATE1
811 movups STATE1, (OUTP)
812 pxor IN2, STATE2
813 movups STATE2, 0x10(OUTP)
814 pxor IN3, STATE3
815 movups STATE3, 0x20(OUTP)
816 pxor IN4, STATE4
817 movups STATE4, 0x30(OUTP)
818 sub $64, LEN
819 add $64, INP
820 add $64, OUTP
821 cmp $64, LEN
822 jge .Lctr_enc_loop4
823 cmp $16, LEN
824 jb .Lctr_enc_ret
825.align 4
826.Lctr_enc_loop1:
827 movaps IV, STATE
828 call _aesni_inc
829 movups (INP), IN
830 call _aesni_enc1
831 pxor IN, STATE
832 movups STATE, (OUTP)
833 sub $16, LEN
834 add $16, INP
835 add $16, OUTP
836 cmp $16, LEN
837 jge .Lctr_enc_loop1
838.Lctr_enc_ret:
839 movups IV, (IVP)
840.Lctr_enc_just_ret:
841 ret
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 49c552c060e9..2cb3dcc4490a 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -18,6 +18,7 @@
18#include <crypto/algapi.h> 18#include <crypto/algapi.h>
19#include <crypto/aes.h> 19#include <crypto/aes.h>
20#include <crypto/cryptd.h> 20#include <crypto/cryptd.h>
21#include <crypto/ctr.h>
21#include <asm/i387.h> 22#include <asm/i387.h>
22#include <asm/aes.h> 23#include <asm/aes.h>
23 24
@@ -58,6 +59,8 @@ asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
58 const u8 *in, unsigned int len, u8 *iv); 59 const u8 *in, unsigned int len, u8 *iv);
59asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, 60asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
60 const u8 *in, unsigned int len, u8 *iv); 61 const u8 *in, unsigned int len, u8 *iv);
62asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
63 const u8 *in, unsigned int len, u8 *iv);
61 64
62static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) 65static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
63{ 66{
@@ -321,6 +324,72 @@ static struct crypto_alg blk_cbc_alg = {
321 }, 324 },
322}; 325};
323 326
327static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
328 struct blkcipher_walk *walk)
329{
330 u8 *ctrblk = walk->iv;
331 u8 keystream[AES_BLOCK_SIZE];
332 u8 *src = walk->src.virt.addr;
333 u8 *dst = walk->dst.virt.addr;
334 unsigned int nbytes = walk->nbytes;
335
336 aesni_enc(ctx, keystream, ctrblk);
337 crypto_xor(keystream, src, nbytes);
338 memcpy(dst, keystream, nbytes);
339 crypto_inc(ctrblk, AES_BLOCK_SIZE);
340}
341
342static int ctr_crypt(struct blkcipher_desc *desc,
343 struct scatterlist *dst, struct scatterlist *src,
344 unsigned int nbytes)
345{
346 struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
347 struct blkcipher_walk walk;
348 int err;
349
350 blkcipher_walk_init(&walk, dst, src, nbytes);
351 err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
352 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
353
354 kernel_fpu_begin();
355 while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
356 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
357 nbytes & AES_BLOCK_MASK, walk.iv);
358 nbytes &= AES_BLOCK_SIZE - 1;
359 err = blkcipher_walk_done(desc, &walk, nbytes);
360 }
361 if (walk.nbytes) {
362 ctr_crypt_final(ctx, &walk);
363 err = blkcipher_walk_done(desc, &walk, 0);
364 }
365 kernel_fpu_end();
366
367 return err;
368}
369
370static struct crypto_alg blk_ctr_alg = {
371 .cra_name = "__ctr-aes-aesni",
372 .cra_driver_name = "__driver-ctr-aes-aesni",
373 .cra_priority = 0,
374 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
375 .cra_blocksize = 1,
376 .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
377 .cra_alignmask = 0,
378 .cra_type = &crypto_blkcipher_type,
379 .cra_module = THIS_MODULE,
380 .cra_list = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
381 .cra_u = {
382 .blkcipher = {
383 .min_keysize = AES_MIN_KEY_SIZE,
384 .max_keysize = AES_MAX_KEY_SIZE,
385 .ivsize = AES_BLOCK_SIZE,
386 .setkey = aes_set_key,
387 .encrypt = ctr_crypt,
388 .decrypt = ctr_crypt,
389 },
390 },
391};
392
324static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, 393static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
325 unsigned int key_len) 394 unsigned int key_len)
326{ 395{
@@ -467,13 +536,11 @@ static struct crypto_alg ablk_cbc_alg = {
467 }, 536 },
468}; 537};
469 538
470#ifdef HAS_CTR
471static int ablk_ctr_init(struct crypto_tfm *tfm) 539static int ablk_ctr_init(struct crypto_tfm *tfm)
472{ 540{
473 struct cryptd_ablkcipher *cryptd_tfm; 541 struct cryptd_ablkcipher *cryptd_tfm;
474 542
475 cryptd_tfm = cryptd_alloc_ablkcipher("fpu(ctr(__driver-aes-aesni))", 543 cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0);
476 0, 0);
477 if (IS_ERR(cryptd_tfm)) 544 if (IS_ERR(cryptd_tfm))
478 return PTR_ERR(cryptd_tfm); 545 return PTR_ERR(cryptd_tfm);
479 ablk_init_common(tfm, cryptd_tfm); 546 ablk_init_common(tfm, cryptd_tfm);
@@ -500,11 +567,50 @@ static struct crypto_alg ablk_ctr_alg = {
500 .ivsize = AES_BLOCK_SIZE, 567 .ivsize = AES_BLOCK_SIZE,
501 .setkey = ablk_set_key, 568 .setkey = ablk_set_key,
502 .encrypt = ablk_encrypt, 569 .encrypt = ablk_encrypt,
503 .decrypt = ablk_decrypt, 570 .decrypt = ablk_encrypt,
504 .geniv = "chainiv", 571 .geniv = "chainiv",
505 }, 572 },
506 }, 573 },
507}; 574};
575
576#ifdef HAS_CTR
577static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
578{
579 struct cryptd_ablkcipher *cryptd_tfm;
580
581 cryptd_tfm = cryptd_alloc_ablkcipher(
582 "rfc3686(__driver-ctr-aes-aesni)", 0, 0);
583 if (IS_ERR(cryptd_tfm))
584 return PTR_ERR(cryptd_tfm);
585 ablk_init_common(tfm, cryptd_tfm);
586 return 0;
587}
588
589static struct crypto_alg ablk_rfc3686_ctr_alg = {
590 .cra_name = "rfc3686(ctr(aes))",
591 .cra_driver_name = "rfc3686-ctr-aes-aesni",
592 .cra_priority = 400,
593 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
594 .cra_blocksize = 1,
595 .cra_ctxsize = sizeof(struct async_aes_ctx),
596 .cra_alignmask = 0,
597 .cra_type = &crypto_ablkcipher_type,
598 .cra_module = THIS_MODULE,
599 .cra_list = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list),
600 .cra_init = ablk_rfc3686_ctr_init,
601 .cra_exit = ablk_exit,
602 .cra_u = {
603 .ablkcipher = {
604 .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
605 .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
606 .ivsize = CTR_RFC3686_IV_SIZE,
607 .setkey = ablk_set_key,
608 .encrypt = ablk_encrypt,
609 .decrypt = ablk_decrypt,
610 .geniv = "seqiv",
611 },
612 },
613};
508#endif 614#endif
509 615
510#ifdef HAS_LRW 616#ifdef HAS_LRW
@@ -640,13 +746,17 @@ static int __init aesni_init(void)
640 goto blk_ecb_err; 746 goto blk_ecb_err;
641 if ((err = crypto_register_alg(&blk_cbc_alg))) 747 if ((err = crypto_register_alg(&blk_cbc_alg)))
642 goto blk_cbc_err; 748 goto blk_cbc_err;
749 if ((err = crypto_register_alg(&blk_ctr_alg)))
750 goto blk_ctr_err;
643 if ((err = crypto_register_alg(&ablk_ecb_alg))) 751 if ((err = crypto_register_alg(&ablk_ecb_alg)))
644 goto ablk_ecb_err; 752 goto ablk_ecb_err;
645 if ((err = crypto_register_alg(&ablk_cbc_alg))) 753 if ((err = crypto_register_alg(&ablk_cbc_alg)))
646 goto ablk_cbc_err; 754 goto ablk_cbc_err;
647#ifdef HAS_CTR
648 if ((err = crypto_register_alg(&ablk_ctr_alg))) 755 if ((err = crypto_register_alg(&ablk_ctr_alg)))
649 goto ablk_ctr_err; 756 goto ablk_ctr_err;
757#ifdef HAS_CTR
758 if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
759 goto ablk_rfc3686_ctr_err;
650#endif 760#endif
651#ifdef HAS_LRW 761#ifdef HAS_LRW
652 if ((err = crypto_register_alg(&ablk_lrw_alg))) 762 if ((err = crypto_register_alg(&ablk_lrw_alg)))
@@ -675,13 +785,17 @@ ablk_pcbc_err:
675ablk_lrw_err: 785ablk_lrw_err:
676#endif 786#endif
677#ifdef HAS_CTR 787#ifdef HAS_CTR
788 crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
789ablk_rfc3686_ctr_err:
790#endif
678 crypto_unregister_alg(&ablk_ctr_alg); 791 crypto_unregister_alg(&ablk_ctr_alg);
679ablk_ctr_err: 792ablk_ctr_err:
680#endif
681 crypto_unregister_alg(&ablk_cbc_alg); 793 crypto_unregister_alg(&ablk_cbc_alg);
682ablk_cbc_err: 794ablk_cbc_err:
683 crypto_unregister_alg(&ablk_ecb_alg); 795 crypto_unregister_alg(&ablk_ecb_alg);
684ablk_ecb_err: 796ablk_ecb_err:
797 crypto_unregister_alg(&blk_ctr_alg);
798blk_ctr_err:
685 crypto_unregister_alg(&blk_cbc_alg); 799 crypto_unregister_alg(&blk_cbc_alg);
686blk_cbc_err: 800blk_cbc_err:
687 crypto_unregister_alg(&blk_ecb_alg); 801 crypto_unregister_alg(&blk_ecb_alg);
@@ -705,10 +819,12 @@ static void __exit aesni_exit(void)
705 crypto_unregister_alg(&ablk_lrw_alg); 819 crypto_unregister_alg(&ablk_lrw_alg);
706#endif 820#endif
707#ifdef HAS_CTR 821#ifdef HAS_CTR
708 crypto_unregister_alg(&ablk_ctr_alg); 822 crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
709#endif 823#endif
824 crypto_unregister_alg(&ablk_ctr_alg);
710 crypto_unregister_alg(&ablk_cbc_alg); 825 crypto_unregister_alg(&ablk_cbc_alg);
711 crypto_unregister_alg(&ablk_ecb_alg); 826 crypto_unregister_alg(&ablk_ecb_alg);
827 crypto_unregister_alg(&blk_ctr_alg);
712 crypto_unregister_alg(&blk_cbc_alg); 828 crypto_unregister_alg(&blk_cbc_alg);
713 crypto_unregister_alg(&blk_ecb_alg); 829 crypto_unregister_alg(&blk_ecb_alg);
714 crypto_unregister_alg(&__aesni_alg); 830 crypto_unregister_alg(&__aesni_alg);
diff --git a/arch/x86/include/asm/inst.h b/arch/x86/include/asm/inst.h
index 14cf526091f9..280bf7fb6aba 100644
--- a/arch/x86/include/asm/inst.h
+++ b/arch/x86/include/asm/inst.h
@@ -7,7 +7,66 @@
7 7
8#ifdef __ASSEMBLY__ 8#ifdef __ASSEMBLY__
9 9
10#define REG_NUM_INVALID 100
11
12#define REG_TYPE_R64 0
13#define REG_TYPE_XMM 1
14#define REG_TYPE_INVALID 100
15
16 .macro R64_NUM opd r64
17 \opd = REG_NUM_INVALID
18 .ifc \r64,%rax
19 \opd = 0
20 .endif
21 .ifc \r64,%rcx
22 \opd = 1
23 .endif
24 .ifc \r64,%rdx
25 \opd = 2
26 .endif
27 .ifc \r64,%rbx
28 \opd = 3
29 .endif
30 .ifc \r64,%rsp
31 \opd = 4
32 .endif
33 .ifc \r64,%rbp
34 \opd = 5
35 .endif
36 .ifc \r64,%rsi
37 \opd = 6
38 .endif
39 .ifc \r64,%rdi
40 \opd = 7
41 .endif
42 .ifc \r64,%r8
43 \opd = 8
44 .endif
45 .ifc \r64,%r9
46 \opd = 9
47 .endif
48 .ifc \r64,%r10
49 \opd = 10
50 .endif
51 .ifc \r64,%r11
52 \opd = 11
53 .endif
54 .ifc \r64,%r12
55 \opd = 12
56 .endif
57 .ifc \r64,%r13
58 \opd = 13
59 .endif
60 .ifc \r64,%r14
61 \opd = 14
62 .endif
63 .ifc \r64,%r15
64 \opd = 15
65 .endif
66 .endm
67
10 .macro XMM_NUM opd xmm 68 .macro XMM_NUM opd xmm
69 \opd = REG_NUM_INVALID
11 .ifc \xmm,%xmm0 70 .ifc \xmm,%xmm0
12 \opd = 0 71 \opd = 0
13 .endif 72 .endif
@@ -58,13 +117,25 @@
58 .endif 117 .endif
59 .endm 118 .endm
60 119
120 .macro REG_TYPE type reg
121 R64_NUM reg_type_r64 \reg
122 XMM_NUM reg_type_xmm \reg
123 .if reg_type_r64 <> REG_NUM_INVALID
124 \type = REG_TYPE_R64
125 .elseif reg_type_xmm <> REG_NUM_INVALID
126 \type = REG_TYPE_XMM
127 .else
128 \type = REG_TYPE_INVALID
129 .endif
130 .endm
131
61 .macro PFX_OPD_SIZE 132 .macro PFX_OPD_SIZE
62 .byte 0x66 133 .byte 0x66
63 .endm 134 .endm
64 135
65 .macro PFX_REX opd1 opd2 136 .macro PFX_REX opd1 opd2 W=0
66 .if (\opd1 | \opd2) & 8 137 .if ((\opd1 | \opd2) & 8) || \W
67 .byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) 138 .byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
68 .endif 139 .endif
69 .endm 140 .endm
70 141
@@ -145,6 +216,25 @@
145 .byte 0x0f, 0x38, 0xdf 216 .byte 0x0f, 0x38, 0xdf
146 MODRM 0xc0 aesdeclast_opd1 aesdeclast_opd2 217 MODRM 0xc0 aesdeclast_opd1 aesdeclast_opd2
147 .endm 218 .endm
219
220 .macro MOVQ_R64_XMM opd1 opd2
221 REG_TYPE movq_r64_xmm_opd1_type \opd1
222 .if movq_r64_xmm_opd1_type == REG_TYPE_XMM
223 XMM_NUM movq_r64_xmm_opd1 \opd1
224 R64_NUM movq_r64_xmm_opd2 \opd2
225 .else
226 R64_NUM movq_r64_xmm_opd1 \opd1
227 XMM_NUM movq_r64_xmm_opd2 \opd2
228 .endif
229 PFX_OPD_SIZE
230 PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
231 .if movq_r64_xmm_opd1_type == REG_TYPE_XMM
232 .byte 0x0f, 0x7e
233 .else
234 .byte 0x0f, 0x6e
235 .endif
236 MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
237 .endm
148#endif 238#endif
149 239
150#endif 240#endif
diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c
index fe980dae1727..98a66103f4f2 100644
--- a/crypto/ablkcipher.c
+++ b/crypto/ablkcipher.c
@@ -24,10 +24,287 @@
24#include <linux/slab.h> 24#include <linux/slab.h>
25#include <linux/seq_file.h> 25#include <linux/seq_file.h>
26 26
27#include <crypto/scatterwalk.h>
28
27#include "internal.h" 29#include "internal.h"
28 30
29static const char *skcipher_default_geniv __read_mostly; 31static const char *skcipher_default_geniv __read_mostly;
30 32
33struct ablkcipher_buffer {
34 struct list_head entry;
35 struct scatter_walk dst;
36 unsigned int len;
37 void *data;
38};
39
40enum {
41 ABLKCIPHER_WALK_SLOW = 1 << 0,
42};
43
44static inline void ablkcipher_buffer_write(struct ablkcipher_buffer *p)
45{
46 scatterwalk_copychunks(p->data, &p->dst, p->len, 1);
47}
48
49void __ablkcipher_walk_complete(struct ablkcipher_walk *walk)
50{
51 struct ablkcipher_buffer *p, *tmp;
52
53 list_for_each_entry_safe(p, tmp, &walk->buffers, entry) {
54 ablkcipher_buffer_write(p);
55 list_del(&p->entry);
56 kfree(p);
57 }
58}
59EXPORT_SYMBOL_GPL(__ablkcipher_walk_complete);
60
61static inline void ablkcipher_queue_write(struct ablkcipher_walk *walk,
62 struct ablkcipher_buffer *p)
63{
64 p->dst = walk->out;
65 list_add_tail(&p->entry, &walk->buffers);
66}
67
68/* Get a spot of the specified length that does not straddle a page.
69 * The caller needs to ensure that there is enough space for this operation.
70 */
71static inline u8 *ablkcipher_get_spot(u8 *start, unsigned int len)
72{
73 u8 *end_page = (u8 *)(((unsigned long)(start + len - 1)) & PAGE_MASK);
74 return max(start, end_page);
75}
76
77static inline unsigned int ablkcipher_done_slow(struct ablkcipher_walk *walk,
78 unsigned int bsize)
79{
80 unsigned int n = bsize;
81
82 for (;;) {
83 unsigned int len_this_page = scatterwalk_pagelen(&walk->out);
84
85 if (len_this_page > n)
86 len_this_page = n;
87 scatterwalk_advance(&walk->out, n);
88 if (n == len_this_page)
89 break;
90 n -= len_this_page;
91 scatterwalk_start(&walk->out, scatterwalk_sg_next(walk->out.sg));
92 }
93
94 return bsize;
95}
96
97static inline unsigned int ablkcipher_done_fast(struct ablkcipher_walk *walk,
98 unsigned int n)
99{
100 scatterwalk_advance(&walk->in, n);
101 scatterwalk_advance(&walk->out, n);
102
103 return n;
104}
105
106static int ablkcipher_walk_next(struct ablkcipher_request *req,
107 struct ablkcipher_walk *walk);
108
109int ablkcipher_walk_done(struct ablkcipher_request *req,
110 struct ablkcipher_walk *walk, int err)
111{
112 struct crypto_tfm *tfm = req->base.tfm;
113 unsigned int nbytes = 0;
114
115 if (likely(err >= 0)) {
116 unsigned int n = walk->nbytes - err;
117
118 if (likely(!(walk->flags & ABLKCIPHER_WALK_SLOW)))
119 n = ablkcipher_done_fast(walk, n);
120 else if (WARN_ON(err)) {
121 err = -EINVAL;
122 goto err;
123 } else
124 n = ablkcipher_done_slow(walk, n);
125
126 nbytes = walk->total - n;
127 err = 0;
128 }
129
130 scatterwalk_done(&walk->in, 0, nbytes);
131 scatterwalk_done(&walk->out, 1, nbytes);
132
133err:
134 walk->total = nbytes;
135 walk->nbytes = nbytes;
136
137 if (nbytes) {
138 crypto_yield(req->base.flags);
139 return ablkcipher_walk_next(req, walk);
140 }
141
142 if (walk->iv != req->info)
143 memcpy(req->info, walk->iv, tfm->crt_ablkcipher.ivsize);
144 if (walk->iv_buffer)
145 kfree(walk->iv_buffer);
146
147 return err;
148}
149EXPORT_SYMBOL_GPL(ablkcipher_walk_done);
150
151static inline int ablkcipher_next_slow(struct ablkcipher_request *req,
152 struct ablkcipher_walk *walk,
153 unsigned int bsize,
154 unsigned int alignmask,
155 void **src_p, void **dst_p)
156{
157 unsigned aligned_bsize = ALIGN(bsize, alignmask + 1);
158 struct ablkcipher_buffer *p;
159 void *src, *dst, *base;
160 unsigned int n;
161
162 n = ALIGN(sizeof(struct ablkcipher_buffer), alignmask + 1);
163 n += (aligned_bsize * 3 - (alignmask + 1) +
164 (alignmask & ~(crypto_tfm_ctx_alignment() - 1)));
165
166 p = kmalloc(n, GFP_ATOMIC);
167 if (!p)
168 ablkcipher_walk_done(req, walk, -ENOMEM);
169
170 base = p + 1;
171
172 dst = (u8 *)ALIGN((unsigned long)base, alignmask + 1);
173 src = dst = ablkcipher_get_spot(dst, bsize);
174
175 p->len = bsize;
176 p->data = dst;
177
178 scatterwalk_copychunks(src, &walk->in, bsize, 0);
179
180 ablkcipher_queue_write(walk, p);
181
182 walk->nbytes = bsize;
183 walk->flags |= ABLKCIPHER_WALK_SLOW;
184
185 *src_p = src;
186 *dst_p = dst;
187
188 return 0;
189}
190
191static inline int ablkcipher_copy_iv(struct ablkcipher_walk *walk,
192 struct crypto_tfm *tfm,
193 unsigned int alignmask)
194{
195 unsigned bs = walk->blocksize;
196 unsigned int ivsize = tfm->crt_ablkcipher.ivsize;
197 unsigned aligned_bs = ALIGN(bs, alignmask + 1);
198 unsigned int size = aligned_bs * 2 + ivsize + max(aligned_bs, ivsize) -
199 (alignmask + 1);
200 u8 *iv;
201
202 size += alignmask & ~(crypto_tfm_ctx_alignment() - 1);
203 walk->iv_buffer = kmalloc(size, GFP_ATOMIC);
204 if (!walk->iv_buffer)
205 return -ENOMEM;
206
207 iv = (u8 *)ALIGN((unsigned long)walk->iv_buffer, alignmask + 1);
208 iv = ablkcipher_get_spot(iv, bs) + aligned_bs;
209 iv = ablkcipher_get_spot(iv, bs) + aligned_bs;
210 iv = ablkcipher_get_spot(iv, ivsize);
211
212 walk->iv = memcpy(iv, walk->iv, ivsize);
213 return 0;
214}
215
216static inline int ablkcipher_next_fast(struct ablkcipher_request *req,
217 struct ablkcipher_walk *walk)
218{
219 walk->src.page = scatterwalk_page(&walk->in);
220 walk->src.offset = offset_in_page(walk->in.offset);
221 walk->dst.page = scatterwalk_page(&walk->out);
222 walk->dst.offset = offset_in_page(walk->out.offset);
223
224 return 0;
225}
226
227static int ablkcipher_walk_next(struct ablkcipher_request *req,
228 struct ablkcipher_walk *walk)
229{
230 struct crypto_tfm *tfm = req->base.tfm;
231 unsigned int alignmask, bsize, n;
232 void *src, *dst;
233 int err;
234
235 alignmask = crypto_tfm_alg_alignmask(tfm);
236 n = walk->total;
237 if (unlikely(n < crypto_tfm_alg_blocksize(tfm))) {
238 req->base.flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN;
239 return ablkcipher_walk_done(req, walk, -EINVAL);
240 }
241
242 walk->flags &= ~ABLKCIPHER_WALK_SLOW;
243 src = dst = NULL;
244
245 bsize = min(walk->blocksize, n);
246 n = scatterwalk_clamp(&walk->in, n);
247 n = scatterwalk_clamp(&walk->out, n);
248
249 if (n < bsize ||
250 !scatterwalk_aligned(&walk->in, alignmask) ||
251 !scatterwalk_aligned(&walk->out, alignmask)) {
252 err = ablkcipher_next_slow(req, walk, bsize, alignmask,
253 &src, &dst);
254 goto set_phys_lowmem;
255 }
256
257 walk->nbytes = n;
258
259 return ablkcipher_next_fast(req, walk);
260
261set_phys_lowmem:
262 if (err >= 0) {
263 walk->src.page = virt_to_page(src);
264 walk->dst.page = virt_to_page(dst);
265 walk->src.offset = ((unsigned long)src & (PAGE_SIZE - 1));
266 walk->dst.offset = ((unsigned long)dst & (PAGE_SIZE - 1));
267 }
268
269 return err;
270}
271
272static int ablkcipher_walk_first(struct ablkcipher_request *req,
273 struct ablkcipher_walk *walk)
274{
275 struct crypto_tfm *tfm = req->base.tfm;
276 unsigned int alignmask;
277
278 alignmask = crypto_tfm_alg_alignmask(tfm);
279 if (WARN_ON_ONCE(in_irq()))
280 return -EDEADLK;
281
282 walk->nbytes = walk->total;
283 if (unlikely(!walk->total))
284 return 0;
285
286 walk->iv_buffer = NULL;
287 walk->iv = req->info;
288 if (unlikely(((unsigned long)walk->iv & alignmask))) {
289 int err = ablkcipher_copy_iv(walk, tfm, alignmask);
290 if (err)
291 return err;
292 }
293
294 scatterwalk_start(&walk->in, walk->in.sg);
295 scatterwalk_start(&walk->out, walk->out.sg);
296
297 return ablkcipher_walk_next(req, walk);
298}
299
300int ablkcipher_walk_phys(struct ablkcipher_request *req,
301 struct ablkcipher_walk *walk)
302{
303 walk->blocksize = crypto_tfm_alg_blocksize(req->base.tfm);
304 return ablkcipher_walk_first(req, walk);
305}
306EXPORT_SYMBOL_GPL(ablkcipher_walk_phys);
307
31static int setkey_unaligned(struct crypto_ablkcipher *tfm, const u8 *key, 308static int setkey_unaligned(struct crypto_ablkcipher *tfm, const u8 *key,
32 unsigned int keylen) 309 unsigned int keylen)
33{ 310{
diff --git a/crypto/algapi.c b/crypto/algapi.c
index 76fae27ed01c..c3cf1a69a47a 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -544,7 +544,7 @@ int crypto_init_spawn2(struct crypto_spawn *spawn, struct crypto_alg *alg,
544{ 544{
545 int err = -EINVAL; 545 int err = -EINVAL;
546 546
547 if (frontend && (alg->cra_flags ^ frontend->type) & frontend->maskset) 547 if ((alg->cra_flags ^ frontend->type) & frontend->maskset)
548 goto out; 548 goto out;
549 549
550 spawn->frontend = frontend; 550 spawn->frontend = frontend;
diff --git a/crypto/authenc.c b/crypto/authenc.c
index 05eb32e0d949..b9884ee0adb6 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -181,6 +181,7 @@ static void authenc_verify_ahash_update_done(struct crypto_async_request *areq,
181 struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); 181 struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
182 struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); 182 struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
183 struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); 183 struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
184 unsigned int cryptlen = req->cryptlen;
184 185
185 if (err) 186 if (err)
186 goto out; 187 goto out;
@@ -196,6 +197,7 @@ static void authenc_verify_ahash_update_done(struct crypto_async_request *areq,
196 goto out; 197 goto out;
197 198
198 authsize = crypto_aead_authsize(authenc); 199 authsize = crypto_aead_authsize(authenc);
200 cryptlen -= authsize;
199 ihash = ahreq->result + authsize; 201 ihash = ahreq->result + authsize;
200 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, 202 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
201 authsize, 0); 203 authsize, 0);
@@ -209,7 +211,7 @@ static void authenc_verify_ahash_update_done(struct crypto_async_request *areq,
209 ablkcipher_request_set_callback(abreq, aead_request_flags(req), 211 ablkcipher_request_set_callback(abreq, aead_request_flags(req),
210 req->base.complete, req->base.data); 212 req->base.complete, req->base.data);
211 ablkcipher_request_set_crypt(abreq, req->src, req->dst, 213 ablkcipher_request_set_crypt(abreq, req->src, req->dst,
212 req->cryptlen, req->iv); 214 cryptlen, req->iv);
213 215
214 err = crypto_ablkcipher_decrypt(abreq); 216 err = crypto_ablkcipher_decrypt(abreq);
215 217
@@ -228,11 +230,13 @@ static void authenc_verify_ahash_done(struct crypto_async_request *areq,
228 struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); 230 struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc);
229 struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); 231 struct authenc_request_ctx *areq_ctx = aead_request_ctx(req);
230 struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); 232 struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff);
233 unsigned int cryptlen = req->cryptlen;
231 234
232 if (err) 235 if (err)
233 goto out; 236 goto out;
234 237
235 authsize = crypto_aead_authsize(authenc); 238 authsize = crypto_aead_authsize(authenc);
239 cryptlen -= authsize;
236 ihash = ahreq->result + authsize; 240 ihash = ahreq->result + authsize;
237 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, 241 scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen,
238 authsize, 0); 242 authsize, 0);
@@ -246,7 +250,7 @@ static void authenc_verify_ahash_done(struct crypto_async_request *areq,
246 ablkcipher_request_set_callback(abreq, aead_request_flags(req), 250 ablkcipher_request_set_callback(abreq, aead_request_flags(req),
247 req->base.complete, req->base.data); 251 req->base.complete, req->base.data);
248 ablkcipher_request_set_crypt(abreq, req->src, req->dst, 252 ablkcipher_request_set_crypt(abreq, req->src, req->dst,
249 req->cryptlen, req->iv); 253 cryptlen, req->iv);
250 254
251 err = crypto_ablkcipher_decrypt(abreq); 255 err = crypto_ablkcipher_decrypt(abreq);
252 256
diff --git a/crypto/internal.h b/crypto/internal.h
index 2d226362e594..d4384b08ab29 100644
--- a/crypto/internal.h
+++ b/crypto/internal.h
@@ -6,7 +6,7 @@
6 * 6 *
7 * This program is free software; you can redistribute it and/or modify it 7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the Free 8 * under the terms of the GNU General Public License as published by the Free
9 * Software Foundation; either version 2 of the License, or (at your option) 9 * Software Foundation; either version 2 of the License, or (at your option)
10 * any later version. 10 * any later version.
11 * 11 *
12 */ 12 */
diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c
index 80201241b698..247178cb98ec 100644
--- a/crypto/pcrypt.c
+++ b/crypto/pcrypt.c
@@ -315,16 +315,13 @@ out_free_inst:
315 goto out; 315 goto out;
316} 316}
317 317
318static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb) 318static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb,
319 u32 type, u32 mask)
319{ 320{
320 struct crypto_instance *inst; 321 struct crypto_instance *inst;
321 struct crypto_alg *alg; 322 struct crypto_alg *alg;
322 struct crypto_attr_type *algt;
323
324 algt = crypto_get_attr_type(tb);
325 323
326 alg = crypto_get_attr_alg(tb, algt->type, 324 alg = crypto_get_attr_alg(tb, type, (mask & CRYPTO_ALG_TYPE_MASK));
327 (algt->mask & CRYPTO_ALG_TYPE_MASK));
328 if (IS_ERR(alg)) 325 if (IS_ERR(alg))
329 return ERR_CAST(alg); 326 return ERR_CAST(alg);
330 327
@@ -365,7 +362,7 @@ static struct crypto_instance *pcrypt_alloc(struct rtattr **tb)
365 362
366 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { 363 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
367 case CRYPTO_ALG_TYPE_AEAD: 364 case CRYPTO_ALG_TYPE_AEAD:
368 return pcrypt_alloc_aead(tb); 365 return pcrypt_alloc_aead(tb, algt->type, algt->mask);
369 } 366 }
370 367
371 return ERR_PTR(-EINVAL); 368 return ERR_PTR(-EINVAL);
diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c
index 3de89a424401..41e529af0773 100644
--- a/crypto/scatterwalk.c
+++ b/crypto/scatterwalk.c
@@ -68,7 +68,7 @@ static void scatterwalk_pagedone(struct scatter_walk *walk, int out,
68 68
69void scatterwalk_done(struct scatter_walk *walk, int out, int more) 69void scatterwalk_done(struct scatter_walk *walk, int out, int more)
70{ 70{
71 if (!offset_in_page(walk->offset) || !more) 71 if (!(scatterwalk_pagelen(walk) & (PAGE_SIZE - 1)) || !more)
72 scatterwalk_pagedone(walk, out, more); 72 scatterwalk_pagedone(walk, out, more);
73} 73}
74EXPORT_SYMBOL_GPL(scatterwalk_done); 74EXPORT_SYMBOL_GPL(scatterwalk_done);
diff --git a/crypto/shash.c b/crypto/shash.c
index 91f7b9d83881..22fd9433141f 100644
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -37,7 +37,7 @@ static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key,
37 u8 *buffer, *alignbuffer; 37 u8 *buffer, *alignbuffer;
38 int err; 38 int err;
39 39
40 absize = keylen + (alignmask & ~(CRYPTO_MINALIGN - 1)); 40 absize = keylen + (alignmask & ~(crypto_tfm_ctx_alignment() - 1));
41 buffer = kmalloc(absize, GFP_KERNEL); 41 buffer = kmalloc(absize, GFP_KERNEL);
42 if (!buffer) 42 if (!buffer)
43 return -ENOMEM; 43 return -ENOMEM;
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index a35159947a26..3ca68f9fc14d 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -394,6 +394,17 @@ out:
394 return 0; 394 return 0;
395} 395}
396 396
397static void test_hash_sg_init(struct scatterlist *sg)
398{
399 int i;
400
401 sg_init_table(sg, TVMEMSIZE);
402 for (i = 0; i < TVMEMSIZE; i++) {
403 sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
404 memset(tvmem[i], 0xff, PAGE_SIZE);
405 }
406}
407
397static void test_hash_speed(const char *algo, unsigned int sec, 408static void test_hash_speed(const char *algo, unsigned int sec,
398 struct hash_speed *speed) 409 struct hash_speed *speed)
399{ 410{
@@ -423,12 +434,7 @@ static void test_hash_speed(const char *algo, unsigned int sec,
423 goto out; 434 goto out;
424 } 435 }
425 436
426 sg_init_table(sg, TVMEMSIZE); 437 test_hash_sg_init(sg);
427 for (i = 0; i < TVMEMSIZE; i++) {
428 sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
429 memset(tvmem[i], 0xff, PAGE_SIZE);
430 }
431
432 for (i = 0; speed[i].blen != 0; i++) { 438 for (i = 0; speed[i].blen != 0; i++) {
433 if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) { 439 if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
434 printk(KERN_ERR 440 printk(KERN_ERR
@@ -437,6 +443,9 @@ static void test_hash_speed(const char *algo, unsigned int sec,
437 goto out; 443 goto out;
438 } 444 }
439 445
446 if (speed[i].klen)
447 crypto_hash_setkey(tfm, tvmem[0], speed[i].klen);
448
440 printk(KERN_INFO "test%3u " 449 printk(KERN_INFO "test%3u "
441 "(%5u byte blocks,%5u bytes per update,%4u updates): ", 450 "(%5u byte blocks,%5u bytes per update,%4u updates): ",
442 i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen); 451 i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
@@ -458,6 +467,250 @@ out:
458 crypto_free_hash(tfm); 467 crypto_free_hash(tfm);
459} 468}
460 469
470struct tcrypt_result {
471 struct completion completion;
472 int err;
473};
474
475static void tcrypt_complete(struct crypto_async_request *req, int err)
476{
477 struct tcrypt_result *res = req->data;
478
479 if (err == -EINPROGRESS)
480 return;
481
482 res->err = err;
483 complete(&res->completion);
484}
485
486static inline int do_one_ahash_op(struct ahash_request *req, int ret)
487{
488 if (ret == -EINPROGRESS || ret == -EBUSY) {
489 struct tcrypt_result *tr = req->base.data;
490
491 ret = wait_for_completion_interruptible(&tr->completion);
492 if (!ret)
493 ret = tr->err;
494 INIT_COMPLETION(tr->completion);
495 }
496 return ret;
497}
498
499static int test_ahash_jiffies_digest(struct ahash_request *req, int blen,
500 char *out, int sec)
501{
502 unsigned long start, end;
503 int bcount;
504 int ret;
505
506 for (start = jiffies, end = start + sec * HZ, bcount = 0;
507 time_before(jiffies, end); bcount++) {
508 ret = do_one_ahash_op(req, crypto_ahash_digest(req));
509 if (ret)
510 return ret;
511 }
512
513 printk("%6u opers/sec, %9lu bytes/sec\n",
514 bcount / sec, ((long)bcount * blen) / sec);
515
516 return 0;
517}
518
519static int test_ahash_jiffies(struct ahash_request *req, int blen,
520 int plen, char *out, int sec)
521{
522 unsigned long start, end;
523 int bcount, pcount;
524 int ret;
525
526 if (plen == blen)
527 return test_ahash_jiffies_digest(req, blen, out, sec);
528
529 for (start = jiffies, end = start + sec * HZ, bcount = 0;
530 time_before(jiffies, end); bcount++) {
531 ret = crypto_ahash_init(req);
532 if (ret)
533 return ret;
534 for (pcount = 0; pcount < blen; pcount += plen) {
535 ret = do_one_ahash_op(req, crypto_ahash_update(req));
536 if (ret)
537 return ret;
538 }
539 /* we assume there is enough space in 'out' for the result */
540 ret = do_one_ahash_op(req, crypto_ahash_final(req));
541 if (ret)
542 return ret;
543 }
544
545 pr_cont("%6u opers/sec, %9lu bytes/sec\n",
546 bcount / sec, ((long)bcount * blen) / sec);
547
548 return 0;
549}
550
551static int test_ahash_cycles_digest(struct ahash_request *req, int blen,
552 char *out)
553{
554 unsigned long cycles = 0;
555 int ret, i;
556
557 /* Warm-up run. */
558 for (i = 0; i < 4; i++) {
559 ret = do_one_ahash_op(req, crypto_ahash_digest(req));
560 if (ret)
561 goto out;
562 }
563
564 /* The real thing. */
565 for (i = 0; i < 8; i++) {
566 cycles_t start, end;
567
568 start = get_cycles();
569
570 ret = do_one_ahash_op(req, crypto_ahash_digest(req));
571 if (ret)
572 goto out;
573
574 end = get_cycles();
575
576 cycles += end - start;
577 }
578
579out:
580 if (ret)
581 return ret;
582
583 pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
584 cycles / 8, cycles / (8 * blen));
585
586 return 0;
587}
588
589static int test_ahash_cycles(struct ahash_request *req, int blen,
590 int plen, char *out)
591{
592 unsigned long cycles = 0;
593 int i, pcount, ret;
594
595 if (plen == blen)
596 return test_ahash_cycles_digest(req, blen, out);
597
598 /* Warm-up run. */
599 for (i = 0; i < 4; i++) {
600 ret = crypto_ahash_init(req);
601 if (ret)
602 goto out;
603 for (pcount = 0; pcount < blen; pcount += plen) {
604 ret = do_one_ahash_op(req, crypto_ahash_update(req));
605 if (ret)
606 goto out;
607 }
608 ret = do_one_ahash_op(req, crypto_ahash_final(req));
609 if (ret)
610 goto out;
611 }
612
613 /* The real thing. */
614 for (i = 0; i < 8; i++) {
615 cycles_t start, end;
616
617 start = get_cycles();
618
619 ret = crypto_ahash_init(req);
620 if (ret)
621 goto out;
622 for (pcount = 0; pcount < blen; pcount += plen) {
623 ret = do_one_ahash_op(req, crypto_ahash_update(req));
624 if (ret)
625 goto out;
626 }
627 ret = do_one_ahash_op(req, crypto_ahash_final(req));
628 if (ret)
629 goto out;
630
631 end = get_cycles();
632
633 cycles += end - start;
634 }
635
636out:
637 if (ret)
638 return ret;
639
640 pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
641 cycles / 8, cycles / (8 * blen));
642
643 return 0;
644}
645
646static void test_ahash_speed(const char *algo, unsigned int sec,
647 struct hash_speed *speed)
648{
649 struct scatterlist sg[TVMEMSIZE];
650 struct tcrypt_result tresult;
651 struct ahash_request *req;
652 struct crypto_ahash *tfm;
653 static char output[1024];
654 int i, ret;
655
656 printk(KERN_INFO "\ntesting speed of async %s\n", algo);
657
658 tfm = crypto_alloc_ahash(algo, 0, 0);
659 if (IS_ERR(tfm)) {
660 pr_err("failed to load transform for %s: %ld\n",
661 algo, PTR_ERR(tfm));
662 return;
663 }
664
665 if (crypto_ahash_digestsize(tfm) > sizeof(output)) {
666 pr_err("digestsize(%u) > outputbuffer(%zu)\n",
667 crypto_ahash_digestsize(tfm), sizeof(output));
668 goto out;
669 }
670
671 test_hash_sg_init(sg);
672 req = ahash_request_alloc(tfm, GFP_KERNEL);
673 if (!req) {
674 pr_err("ahash request allocation failure\n");
675 goto out;
676 }
677
678 init_completion(&tresult.completion);
679 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
680 tcrypt_complete, &tresult);
681
682 for (i = 0; speed[i].blen != 0; i++) {
683 if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
684 pr_err("template (%u) too big for tvmem (%lu)\n",
685 speed[i].blen, TVMEMSIZE * PAGE_SIZE);
686 break;
687 }
688
689 pr_info("test%3u "
690 "(%5u byte blocks,%5u bytes per update,%4u updates): ",
691 i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
692
693 ahash_request_set_crypt(req, sg, output, speed[i].plen);
694
695 if (sec)
696 ret = test_ahash_jiffies(req, speed[i].blen,
697 speed[i].plen, output, sec);
698 else
699 ret = test_ahash_cycles(req, speed[i].blen,
700 speed[i].plen, output);
701
702 if (ret) {
703 pr_err("hashing failed ret=%d\n", ret);
704 break;
705 }
706 }
707
708 ahash_request_free(req);
709
710out:
711 crypto_free_ahash(tfm);
712}
713
461static void test_available(void) 714static void test_available(void)
462{ 715{
463 char **name = check; 716 char **name = check;
@@ -881,9 +1134,87 @@ static int do_test(int m)
881 test_hash_speed("rmd320", sec, generic_hash_speed_template); 1134 test_hash_speed("rmd320", sec, generic_hash_speed_template);
882 if (mode > 300 && mode < 400) break; 1135 if (mode > 300 && mode < 400) break;
883 1136
1137 case 318:
1138 test_hash_speed("ghash-generic", sec, hash_speed_template_16);
1139 if (mode > 300 && mode < 400) break;
1140
884 case 399: 1141 case 399:
885 break; 1142 break;
886 1143
1144 case 400:
1145 /* fall through */
1146
1147 case 401:
1148 test_ahash_speed("md4", sec, generic_hash_speed_template);
1149 if (mode > 400 && mode < 500) break;
1150
1151 case 402:
1152 test_ahash_speed("md5", sec, generic_hash_speed_template);
1153 if (mode > 400 && mode < 500) break;
1154
1155 case 403:
1156 test_ahash_speed("sha1", sec, generic_hash_speed_template);
1157 if (mode > 400 && mode < 500) break;
1158
1159 case 404:
1160 test_ahash_speed("sha256", sec, generic_hash_speed_template);
1161 if (mode > 400 && mode < 500) break;
1162
1163 case 405:
1164 test_ahash_speed("sha384", sec, generic_hash_speed_template);
1165 if (mode > 400 && mode < 500) break;
1166
1167 case 406:
1168 test_ahash_speed("sha512", sec, generic_hash_speed_template);
1169 if (mode > 400 && mode < 500) break;
1170
1171 case 407:
1172 test_ahash_speed("wp256", sec, generic_hash_speed_template);
1173 if (mode > 400 && mode < 500) break;
1174
1175 case 408:
1176 test_ahash_speed("wp384", sec, generic_hash_speed_template);
1177 if (mode > 400 && mode < 500) break;
1178
1179 case 409:
1180 test_ahash_speed("wp512", sec, generic_hash_speed_template);
1181 if (mode > 400 && mode < 500) break;
1182
1183 case 410:
1184 test_ahash_speed("tgr128", sec, generic_hash_speed_template);
1185 if (mode > 400 && mode < 500) break;
1186
1187 case 411:
1188 test_ahash_speed("tgr160", sec, generic_hash_speed_template);
1189 if (mode > 400 && mode < 500) break;
1190
1191 case 412:
1192 test_ahash_speed("tgr192", sec, generic_hash_speed_template);
1193 if (mode > 400 && mode < 500) break;
1194
1195 case 413:
1196 test_ahash_speed("sha224", sec, generic_hash_speed_template);
1197 if (mode > 400 && mode < 500) break;
1198
1199 case 414:
1200 test_ahash_speed("rmd128", sec, generic_hash_speed_template);
1201 if (mode > 400 && mode < 500) break;
1202
1203 case 415:
1204 test_ahash_speed("rmd160", sec, generic_hash_speed_template);
1205 if (mode > 400 && mode < 500) break;
1206
1207 case 416:
1208 test_ahash_speed("rmd256", sec, generic_hash_speed_template);
1209 if (mode > 400 && mode < 500) break;
1210
1211 case 417:
1212 test_ahash_speed("rmd320", sec, generic_hash_speed_template);
1213 if (mode > 400 && mode < 500) break;
1214
1215 case 499:
1216 break;
1217
887 case 1000: 1218 case 1000:
888 test_available(); 1219 test_available();
889 break; 1220 break;
diff --git a/crypto/tcrypt.h b/crypto/tcrypt.h
index 966bbfaf95b1..10cb925132c9 100644
--- a/crypto/tcrypt.h
+++ b/crypto/tcrypt.h
@@ -25,6 +25,7 @@ struct cipher_speed_template {
25struct hash_speed { 25struct hash_speed {
26 unsigned int blen; /* buffer length */ 26 unsigned int blen; /* buffer length */
27 unsigned int plen; /* per-update length */ 27 unsigned int plen; /* per-update length */
28 unsigned int klen; /* key length */
28}; 29};
29 30
30/* 31/*
@@ -83,4 +84,32 @@ static struct hash_speed generic_hash_speed_template[] = {
83 { .blen = 0, .plen = 0, } 84 { .blen = 0, .plen = 0, }
84}; 85};
85 86
87static struct hash_speed hash_speed_template_16[] = {
88 { .blen = 16, .plen = 16, .klen = 16, },
89 { .blen = 64, .plen = 16, .klen = 16, },
90 { .blen = 64, .plen = 64, .klen = 16, },
91 { .blen = 256, .plen = 16, .klen = 16, },
92 { .blen = 256, .plen = 64, .klen = 16, },
93 { .blen = 256, .plen = 256, .klen = 16, },
94 { .blen = 1024, .plen = 16, .klen = 16, },
95 { .blen = 1024, .plen = 256, .klen = 16, },
96 { .blen = 1024, .plen = 1024, .klen = 16, },
97 { .blen = 2048, .plen = 16, .klen = 16, },
98 { .blen = 2048, .plen = 256, .klen = 16, },
99 { .blen = 2048, .plen = 1024, .klen = 16, },
100 { .blen = 2048, .plen = 2048, .klen = 16, },
101 { .blen = 4096, .plen = 16, .klen = 16, },
102 { .blen = 4096, .plen = 256, .klen = 16, },
103 { .blen = 4096, .plen = 1024, .klen = 16, },
104 { .blen = 4096, .plen = 4096, .klen = 16, },
105 { .blen = 8192, .plen = 16, .klen = 16, },
106 { .blen = 8192, .plen = 256, .klen = 16, },
107 { .blen = 8192, .plen = 1024, .klen = 16, },
108 { .blen = 8192, .plen = 4096, .klen = 16, },
109 { .blen = 8192, .plen = 8192, .klen = 16, },
110
111 /* End marker */
112 { .blen = 0, .plen = 0, .klen = 0, }
113};
114
86#endif /* _CRYPTO_TCRYPT_H */ 115#endif /* _CRYPTO_TCRYPT_H */
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index c494d7610be1..5c8aaa0cb0b9 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -153,8 +153,21 @@ static void testmgr_free_buf(char *buf[XBUFSIZE])
153 free_page((unsigned long)buf[i]); 153 free_page((unsigned long)buf[i]);
154} 154}
155 155
156static int do_one_async_hash_op(struct ahash_request *req,
157 struct tcrypt_result *tr,
158 int ret)
159{
160 if (ret == -EINPROGRESS || ret == -EBUSY) {
161 ret = wait_for_completion_interruptible(&tr->completion);
162 if (!ret)
163 ret = tr->err;
164 INIT_COMPLETION(tr->completion);
165 }
166 return ret;
167}
168
156static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, 169static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
157 unsigned int tcount) 170 unsigned int tcount, bool use_digest)
158{ 171{
159 const char *algo = crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm)); 172 const char *algo = crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm));
160 unsigned int i, j, k, temp; 173 unsigned int i, j, k, temp;
@@ -206,23 +219,36 @@ static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template,
206 } 219 }
207 220
208 ahash_request_set_crypt(req, sg, result, template[i].psize); 221 ahash_request_set_crypt(req, sg, result, template[i].psize);
209 ret = crypto_ahash_digest(req); 222 if (use_digest) {
210 switch (ret) { 223 ret = do_one_async_hash_op(req, &tresult,
211 case 0: 224 crypto_ahash_digest(req));
212 break; 225 if (ret) {
213 case -EINPROGRESS: 226 pr_err("alg: hash: digest failed on test %d "
214 case -EBUSY: 227 "for %s: ret=%d\n", j, algo, -ret);
215 ret = wait_for_completion_interruptible( 228 goto out;
216 &tresult.completion); 229 }
217 if (!ret && !(ret = tresult.err)) { 230 } else {
218 INIT_COMPLETION(tresult.completion); 231 ret = do_one_async_hash_op(req, &tresult,
219 break; 232 crypto_ahash_init(req));
233 if (ret) {
234 pr_err("alt: hash: init failed on test %d "
235 "for %s: ret=%d\n", j, algo, -ret);
236 goto out;
237 }
238 ret = do_one_async_hash_op(req, &tresult,
239 crypto_ahash_update(req));
240 if (ret) {
241 pr_err("alt: hash: update failed on test %d "
242 "for %s: ret=%d\n", j, algo, -ret);
243 goto out;
244 }
245 ret = do_one_async_hash_op(req, &tresult,
246 crypto_ahash_final(req));
247 if (ret) {
248 pr_err("alt: hash: final failed on test %d "
249 "for %s: ret=%d\n", j, algo, -ret);
250 goto out;
220 } 251 }
221 /* fall through */
222 default:
223 printk(KERN_ERR "alg: hash: digest failed on test %d "
224 "for %s: ret=%d\n", j, algo, -ret);
225 goto out;
226 } 252 }
227 253
228 if (memcmp(result, template[i].digest, 254 if (memcmp(result, template[i].digest,
@@ -1402,7 +1428,11 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,
1402 return PTR_ERR(tfm); 1428 return PTR_ERR(tfm);
1403 } 1429 }
1404 1430
1405 err = test_hash(tfm, desc->suite.hash.vecs, desc->suite.hash.count); 1431 err = test_hash(tfm, desc->suite.hash.vecs,
1432 desc->suite.hash.count, true);
1433 if (!err)
1434 err = test_hash(tfm, desc->suite.hash.vecs,
1435 desc->suite.hash.count, false);
1406 1436
1407 crypto_free_ahash(tfm); 1437 crypto_free_ahash(tfm);
1408 return err; 1438 return err;
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index fb765173d41c..74e35377fd30 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -1669,17 +1669,73 @@ static struct hash_testvec aes_xcbc128_tv_template[] = {
1669 } 1669 }
1670}; 1670};
1671 1671
1672#define VMAC_AES_TEST_VECTORS 1 1672#define VMAC_AES_TEST_VECTORS 8
1673static char vmac_string[128] = {'\x01', '\x01', '\x01', '\x01', 1673static char vmac_string1[128] = {'\x01', '\x01', '\x01', '\x01',
1674 '\x02', '\x03', '\x02', '\x02', 1674 '\x02', '\x03', '\x02', '\x02',
1675 '\x02', '\x04', '\x01', '\x07', 1675 '\x02', '\x04', '\x01', '\x07',
1676 '\x04', '\x01', '\x04', '\x03',}; 1676 '\x04', '\x01', '\x04', '\x03',};
1677static char vmac_string2[128] = {'a', 'b', 'c',};
1678static char vmac_string3[128] = {'a', 'b', 'c', 'a', 'b', 'c',
1679 'a', 'b', 'c', 'a', 'b', 'c',
1680 'a', 'b', 'c', 'a', 'b', 'c',
1681 'a', 'b', 'c', 'a', 'b', 'c',
1682 'a', 'b', 'c', 'a', 'b', 'c',
1683 'a', 'b', 'c', 'a', 'b', 'c',
1684 'a', 'b', 'c', 'a', 'b', 'c',
1685 'a', 'b', 'c', 'a', 'b', 'c',
1686 };
1687
1677static struct hash_testvec aes_vmac128_tv_template[] = { 1688static struct hash_testvec aes_vmac128_tv_template[] = {
1678 { 1689 {
1690 .key = "\x00\x01\x02\x03\x04\x05\x06\x07"
1691 "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
1692 .plaintext = NULL,
1693 .digest = "\x07\x58\x80\x35\x77\xa4\x7b\x54",
1694 .psize = 0,
1695 .ksize = 16,
1696 }, {
1697 .key = "\x00\x01\x02\x03\x04\x05\x06\x07"
1698 "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
1699 .plaintext = vmac_string1,
1700 .digest = "\xce\xf5\x3c\xd3\xae\x68\x8c\xa1",
1701 .psize = 128,
1702 .ksize = 16,
1703 }, {
1704 .key = "\x00\x01\x02\x03\x04\x05\x06\x07"
1705 "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
1706 .plaintext = vmac_string2,
1707 .digest = "\xc9\x27\xb0\x73\x81\xbd\x14\x2d",
1708 .psize = 128,
1709 .ksize = 16,
1710 }, {
1679 .key = "\x00\x01\x02\x03\x04\x05\x06\x07" 1711 .key = "\x00\x01\x02\x03\x04\x05\x06\x07"
1680 "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 1712 "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
1681 .plaintext = vmac_string, 1713 .plaintext = vmac_string3,
1682 .digest = "\xcb\xd7\x8a\xfd\xb7\x33\x79\xe7", 1714 .digest = "\x8d\x1a\x95\x8c\x98\x47\x0b\x19",
1715 .psize = 128,
1716 .ksize = 16,
1717 }, {
1718 .key = "abcdefghijklmnop",
1719 .plaintext = NULL,
1720 .digest = "\x3b\x89\xa1\x26\x9e\x55\x8f\x84",
1721 .psize = 0,
1722 .ksize = 16,
1723 }, {
1724 .key = "abcdefghijklmnop",
1725 .plaintext = vmac_string1,
1726 .digest = "\xab\x5e\xab\xb0\xf6\x8d\x74\xc2",
1727 .psize = 128,
1728 .ksize = 16,
1729 }, {
1730 .key = "abcdefghijklmnop",
1731 .plaintext = vmac_string2,
1732 .digest = "\x11\x15\x68\x42\x3d\x7b\x09\xdf",
1733 .psize = 128,
1734 .ksize = 16,
1735 }, {
1736 .key = "abcdefghijklmnop",
1737 .plaintext = vmac_string3,
1738 .digest = "\x8b\x32\x8f\xe1\xed\x8f\xfa\xd4",
1683 .psize = 128, 1739 .psize = 128,
1684 .ksize = 16, 1740 .ksize = 16,
1685 }, 1741 },
diff --git a/crypto/vmac.c b/crypto/vmac.c
index 0a9468e575de..0999274a27ac 100644
--- a/crypto/vmac.c
+++ b/crypto/vmac.c
@@ -43,6 +43,8 @@ const u64 m63 = UINT64_C(0x7fffffffffffffff); /* 63-bit mask */
43const u64 m64 = UINT64_C(0xffffffffffffffff); /* 64-bit mask */ 43const u64 m64 = UINT64_C(0xffffffffffffffff); /* 64-bit mask */
44const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */ 44const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */
45 45
46#define pe64_to_cpup le64_to_cpup /* Prefer little endian */
47
46#ifdef __LITTLE_ENDIAN 48#ifdef __LITTLE_ENDIAN
47#define INDEX_HIGH 1 49#define INDEX_HIGH 1
48#define INDEX_LOW 0 50#define INDEX_LOW 0
@@ -110,8 +112,8 @@ const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */
110 int i; u64 th, tl; \ 112 int i; u64 th, tl; \
111 rh = rl = 0; \ 113 rh = rl = 0; \
112 for (i = 0; i < nw; i += 2) { \ 114 for (i = 0; i < nw; i += 2) { \
113 MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ 115 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
114 le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ 116 pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
115 ADD128(rh, rl, th, tl); \ 117 ADD128(rh, rl, th, tl); \
116 } \ 118 } \
117 } while (0) 119 } while (0)
@@ -121,11 +123,11 @@ const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */
121 int i; u64 th, tl; \ 123 int i; u64 th, tl; \
122 rh1 = rl1 = rh = rl = 0; \ 124 rh1 = rl1 = rh = rl = 0; \
123 for (i = 0; i < nw; i += 2) { \ 125 for (i = 0; i < nw; i += 2) { \
124 MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ 126 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
125 le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ 127 pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
126 ADD128(rh, rl, th, tl); \ 128 ADD128(rh, rl, th, tl); \
127 MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i+2], \ 129 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \
128 le64_to_cpup((mp)+i+1)+(kp)[i+3]); \ 130 pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \
129 ADD128(rh1, rl1, th, tl); \ 131 ADD128(rh1, rl1, th, tl); \
130 } \ 132 } \
131 } while (0) 133 } while (0)
@@ -136,17 +138,17 @@ const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */
136 int i; u64 th, tl; \ 138 int i; u64 th, tl; \
137 rh = rl = 0; \ 139 rh = rl = 0; \
138 for (i = 0; i < nw; i += 8) { \ 140 for (i = 0; i < nw; i += 8) { \
139 MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ 141 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
140 le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ 142 pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
141 ADD128(rh, rl, th, tl); \ 143 ADD128(rh, rl, th, tl); \
142 MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+2], \ 144 MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
143 le64_to_cpup((mp)+i+3)+(kp)[i+3]); \ 145 pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \
144 ADD128(rh, rl, th, tl); \ 146 ADD128(rh, rl, th, tl); \
145 MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+4], \ 147 MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
146 le64_to_cpup((mp)+i+5)+(kp)[i+5]); \ 148 pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \
147 ADD128(rh, rl, th, tl); \ 149 ADD128(rh, rl, th, tl); \
148 MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+6], \ 150 MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
149 le64_to_cpup((mp)+i+7)+(kp)[i+7]); \ 151 pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \
150 ADD128(rh, rl, th, tl); \ 152 ADD128(rh, rl, th, tl); \
151 } \ 153 } \
152 } while (0) 154 } while (0)
@@ -156,29 +158,29 @@ const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */
156 int i; u64 th, tl; \ 158 int i; u64 th, tl; \
157 rh1 = rl1 = rh = rl = 0; \ 159 rh1 = rl1 = rh = rl = 0; \
158 for (i = 0; i < nw; i += 8) { \ 160 for (i = 0; i < nw; i += 8) { \
159 MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ 161 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \
160 le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ 162 pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \
161 ADD128(rh, rl, th, tl); \ 163 ADD128(rh, rl, th, tl); \
162 MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i+2], \ 164 MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \
163 le64_to_cpup((mp)+i+1)+(kp)[i+3]); \ 165 pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \
164 ADD128(rh1, rl1, th, tl); \ 166 ADD128(rh1, rl1, th, tl); \
165 MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+2], \ 167 MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \
166 le64_to_cpup((mp)+i+3)+(kp)[i+3]); \ 168 pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \
167 ADD128(rh, rl, th, tl); \ 169 ADD128(rh, rl, th, tl); \
168 MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+4], \ 170 MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4], \
169 le64_to_cpup((mp)+i+3)+(kp)[i+5]); \ 171 pe64_to_cpup((mp)+i+3)+(kp)[i+5]); \
170 ADD128(rh1, rl1, th, tl); \ 172 ADD128(rh1, rl1, th, tl); \
171 MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+4], \ 173 MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \
172 le64_to_cpup((mp)+i+5)+(kp)[i+5]); \ 174 pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \
173 ADD128(rh, rl, th, tl); \ 175 ADD128(rh, rl, th, tl); \
174 MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+6], \ 176 MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6], \
175 le64_to_cpup((mp)+i+5)+(kp)[i+7]); \ 177 pe64_to_cpup((mp)+i+5)+(kp)[i+7]); \
176 ADD128(rh1, rl1, th, tl); \ 178 ADD128(rh1, rl1, th, tl); \
177 MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+6], \ 179 MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \
178 le64_to_cpup((mp)+i+7)+(kp)[i+7]); \ 180 pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \
179 ADD128(rh, rl, th, tl); \ 181 ADD128(rh, rl, th, tl); \
180 MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+8], \ 182 MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8], \
181 le64_to_cpup((mp)+i+7)+(kp)[i+9]); \ 183 pe64_to_cpup((mp)+i+7)+(kp)[i+9]); \
182 ADD128(rh1, rl1, th, tl); \ 184 ADD128(rh1, rl1, th, tl); \
183 } \ 185 } \
184 } while (0) 186 } while (0)
@@ -216,8 +218,8 @@ const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */
216 int i; \ 218 int i; \
217 rh = rl = t = 0; \ 219 rh = rl = t = 0; \
218 for (i = 0; i < nw; i += 2) { \ 220 for (i = 0; i < nw; i += 2) { \
219 t1 = le64_to_cpup(mp+i) + kp[i]; \ 221 t1 = pe64_to_cpup(mp+i) + kp[i]; \
220 t2 = le64_to_cpup(mp+i+1) + kp[i+1]; \ 222 t2 = pe64_to_cpup(mp+i+1) + kp[i+1]; \
221 m2 = MUL32(t1 >> 32, t2); \ 223 m2 = MUL32(t1 >> 32, t2); \
222 m1 = MUL32(t1, t2 >> 32); \ 224 m1 = MUL32(t1, t2 >> 32); \
223 ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32), \ 225 ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32), \
@@ -322,8 +324,7 @@ static void vhash_abort(struct vmac_ctx *ctx)
322 ctx->first_block_processed = 0; 324 ctx->first_block_processed = 0;
323} 325}
324 326
325static u64 l3hash(u64 p1, u64 p2, 327static u64 l3hash(u64 p1, u64 p2, u64 k1, u64 k2, u64 len)
326 u64 k1, u64 k2, u64 len)
327{ 328{
328 u64 rh, rl, t, z = 0; 329 u64 rh, rl, t, z = 0;
329 330
@@ -474,7 +475,7 @@ static u64 vmac(unsigned char m[], unsigned int mbytes,
474 } 475 }
475 p = be64_to_cpup(out_p + i); 476 p = be64_to_cpup(out_p + i);
476 h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx); 477 h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx);
477 return p + h; 478 return le64_to_cpu(p + h);
478} 479}
479 480
480static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx) 481static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx)
@@ -549,10 +550,6 @@ static int vmac_setkey(struct crypto_shash *parent,
549 550
550static int vmac_init(struct shash_desc *pdesc) 551static int vmac_init(struct shash_desc *pdesc)
551{ 552{
552 struct crypto_shash *parent = pdesc->tfm;
553 struct vmac_ctx_t *ctx = crypto_shash_ctx(parent);
554
555 memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx));
556 return 0; 553 return 0;
557} 554}
558 555
diff --git a/drivers/char/random.c b/drivers/char/random.c
index 2fd3d39995d5..8d85587b6d4f 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -257,6 +257,7 @@
257#define INPUT_POOL_WORDS 128 257#define INPUT_POOL_WORDS 128
258#define OUTPUT_POOL_WORDS 32 258#define OUTPUT_POOL_WORDS 32
259#define SEC_XFER_SIZE 512 259#define SEC_XFER_SIZE 512
260#define EXTRACT_SIZE 10
260 261
261/* 262/*
262 * The minimum number of bits of entropy before we wake up a read on 263 * The minimum number of bits of entropy before we wake up a read on
@@ -414,7 +415,7 @@ struct entropy_store {
414 unsigned add_ptr; 415 unsigned add_ptr;
415 int entropy_count; 416 int entropy_count;
416 int input_rotate; 417 int input_rotate;
417 __u8 *last_data; 418 __u8 last_data[EXTRACT_SIZE];
418}; 419};
419 420
420static __u32 input_pool_data[INPUT_POOL_WORDS]; 421static __u32 input_pool_data[INPUT_POOL_WORDS];
@@ -714,8 +715,6 @@ void add_disk_randomness(struct gendisk *disk)
714} 715}
715#endif 716#endif
716 717
717#define EXTRACT_SIZE 10
718
719/********************************************************************* 718/*********************************************************************
720 * 719 *
721 * Entropy extraction routines 720 * Entropy extraction routines
@@ -862,7 +861,7 @@ static ssize_t extract_entropy(struct entropy_store *r, void *buf,
862 while (nbytes) { 861 while (nbytes) {
863 extract_buf(r, tmp); 862 extract_buf(r, tmp);
864 863
865 if (r->last_data) { 864 if (fips_enabled) {
866 spin_lock_irqsave(&r->lock, flags); 865 spin_lock_irqsave(&r->lock, flags);
867 if (!memcmp(tmp, r->last_data, EXTRACT_SIZE)) 866 if (!memcmp(tmp, r->last_data, EXTRACT_SIZE))
868 panic("Hardware RNG duplicated output!\n"); 867 panic("Hardware RNG duplicated output!\n");
@@ -951,9 +950,6 @@ static void init_std_data(struct entropy_store *r)
951 now = ktime_get_real(); 950 now = ktime_get_real();
952 mix_pool_bytes(r, &now, sizeof(now)); 951 mix_pool_bytes(r, &now, sizeof(now));
953 mix_pool_bytes(r, utsname(), sizeof(*(utsname()))); 952 mix_pool_bytes(r, utsname(), sizeof(*(utsname())));
954 /* Enable continuous test in fips mode */
955 if (fips_enabled)
956 r->last_data = kmalloc(EXTRACT_SIZE, GFP_KERNEL);
957} 953}
958 954
959static int rand_initialize(void) 955static int rand_initialize(void)
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index b08403d7d1ca..fbf94cf496f0 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -170,6 +170,18 @@ config CRYPTO_DEV_MV_CESA
170 170
171 Currently the driver supports AES in ECB and CBC mode without DMA. 171 Currently the driver supports AES in ECB and CBC mode without DMA.
172 172
173config CRYPTO_DEV_NIAGARA2
174 tristate "Niagara2 Stream Processing Unit driver"
175 select CRYPTO_ALGAPI
176 depends on SPARC64
177 help
178 Each core of a Niagara2 processor contains a Stream
179 Processing Unit, which itself contains several cryptographic
180 sub-units. One set provides the Modular Arithmetic Unit,
181 used for SSL offload. The other set provides the Cipher
182 Group, which can perform encryption, decryption, hashing,
183 checksumming, and raw copies.
184
173config CRYPTO_DEV_HIFN_795X 185config CRYPTO_DEV_HIFN_795X
174 tristate "Driver HIFN 795x crypto accelerator chips" 186 tristate "Driver HIFN 795x crypto accelerator chips"
175 select CRYPTO_DES 187 select CRYPTO_DES
@@ -222,4 +234,13 @@ config CRYPTO_DEV_PPC4XX
222 help 234 help
223 This option allows you to have support for AMCC crypto acceleration. 235 This option allows you to have support for AMCC crypto acceleration.
224 236
237config CRYPTO_DEV_OMAP_SHAM
238 tristate "Support for OMAP SHA1/MD5 hw accelerator"
239 depends on ARCH_OMAP2 || ARCH_OMAP3
240 select CRYPTO_SHA1
241 select CRYPTO_MD5
242 help
243 OMAP processors have SHA1/MD5 hw accelerator. Select this if you
244 want to use the OMAP module for SHA1/MD5 algorithms.
245
225endif # CRYPTO_HW 246endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 6ffcb3f7f942..6dbbe00c4524 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -1,8 +1,12 @@
1obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o 1obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
2obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o 2obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
3obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o 3obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
4obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o
5n2_crypto-objs := n2_core.o n2_asm.o
4obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o 6obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
5obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o 7obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o
6obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o 8obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
7obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o 9obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o
8obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/ 10obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/
11obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o
12
diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c
index c7a5a43ba691..09389dd2f96b 100644
--- a/drivers/crypto/geode-aes.c
+++ b/drivers/crypto/geode-aes.c
@@ -15,14 +15,14 @@
15#include <crypto/algapi.h> 15#include <crypto/algapi.h>
16#include <crypto/aes.h> 16#include <crypto/aes.h>
17 17
18#include <asm/io.h> 18#include <linux/io.h>
19#include <asm/delay.h> 19#include <linux/delay.h>
20 20
21#include "geode-aes.h" 21#include "geode-aes.h"
22 22
23/* Static structures */ 23/* Static structures */
24 24
25static void __iomem * _iobase; 25static void __iomem *_iobase;
26static spinlock_t lock; 26static spinlock_t lock;
27 27
28/* Write a 128 bit field (either a writable key or IV) */ 28/* Write a 128 bit field (either a writable key or IV) */
@@ -30,7 +30,7 @@ static inline void
30_writefield(u32 offset, void *value) 30_writefield(u32 offset, void *value)
31{ 31{
32 int i; 32 int i;
33 for(i = 0; i < 4; i++) 33 for (i = 0; i < 4; i++)
34 iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4)); 34 iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4));
35} 35}
36 36
@@ -39,7 +39,7 @@ static inline void
39_readfield(u32 offset, void *value) 39_readfield(u32 offset, void *value)
40{ 40{
41 int i; 41 int i;
42 for(i = 0; i < 4; i++) 42 for (i = 0; i < 4; i++)
43 ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4)); 43 ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4));
44} 44}
45 45
@@ -59,7 +59,7 @@ do_crypt(void *src, void *dst, int len, u32 flags)
59 do { 59 do {
60 status = ioread32(_iobase + AES_INTR_REG); 60 status = ioread32(_iobase + AES_INTR_REG);
61 cpu_relax(); 61 cpu_relax();
62 } while(!(status & AES_INTRA_PENDING) && --counter); 62 } while (!(status & AES_INTRA_PENDING) && --counter);
63 63
64 /* Clear the event */ 64 /* Clear the event */
65 iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG); 65 iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG);
@@ -317,7 +317,7 @@ geode_cbc_decrypt(struct blkcipher_desc *desc,
317 err = blkcipher_walk_virt(desc, &walk); 317 err = blkcipher_walk_virt(desc, &walk);
318 op->iv = walk.iv; 318 op->iv = walk.iv;
319 319
320 while((nbytes = walk.nbytes)) { 320 while ((nbytes = walk.nbytes)) {
321 op->src = walk.src.virt.addr, 321 op->src = walk.src.virt.addr,
322 op->dst = walk.dst.virt.addr; 322 op->dst = walk.dst.virt.addr;
323 op->mode = AES_MODE_CBC; 323 op->mode = AES_MODE_CBC;
@@ -349,7 +349,7 @@ geode_cbc_encrypt(struct blkcipher_desc *desc,
349 err = blkcipher_walk_virt(desc, &walk); 349 err = blkcipher_walk_virt(desc, &walk);
350 op->iv = walk.iv; 350 op->iv = walk.iv;
351 351
352 while((nbytes = walk.nbytes)) { 352 while ((nbytes = walk.nbytes)) {
353 op->src = walk.src.virt.addr, 353 op->src = walk.src.virt.addr,
354 op->dst = walk.dst.virt.addr; 354 op->dst = walk.dst.virt.addr;
355 op->mode = AES_MODE_CBC; 355 op->mode = AES_MODE_CBC;
@@ -429,7 +429,7 @@ geode_ecb_decrypt(struct blkcipher_desc *desc,
429 blkcipher_walk_init(&walk, dst, src, nbytes); 429 blkcipher_walk_init(&walk, dst, src, nbytes);
430 err = blkcipher_walk_virt(desc, &walk); 430 err = blkcipher_walk_virt(desc, &walk);
431 431
432 while((nbytes = walk.nbytes)) { 432 while ((nbytes = walk.nbytes)) {
433 op->src = walk.src.virt.addr, 433 op->src = walk.src.virt.addr,
434 op->dst = walk.dst.virt.addr; 434 op->dst = walk.dst.virt.addr;
435 op->mode = AES_MODE_ECB; 435 op->mode = AES_MODE_ECB;
@@ -459,7 +459,7 @@ geode_ecb_encrypt(struct blkcipher_desc *desc,
459 blkcipher_walk_init(&walk, dst, src, nbytes); 459 blkcipher_walk_init(&walk, dst, src, nbytes);
460 err = blkcipher_walk_virt(desc, &walk); 460 err = blkcipher_walk_virt(desc, &walk);
461 461
462 while((nbytes = walk.nbytes)) { 462 while ((nbytes = walk.nbytes)) {
463 op->src = walk.src.virt.addr, 463 op->src = walk.src.virt.addr,
464 op->dst = walk.dst.virt.addr; 464 op->dst = walk.dst.virt.addr;
465 op->mode = AES_MODE_ECB; 465 op->mode = AES_MODE_ECB;
@@ -518,11 +518,12 @@ static int __devinit
518geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id) 518geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
519{ 519{
520 int ret; 520 int ret;
521 521 ret = pci_enable_device(dev);
522 if ((ret = pci_enable_device(dev))) 522 if (ret)
523 return ret; 523 return ret;
524 524
525 if ((ret = pci_request_regions(dev, "geode-aes"))) 525 ret = pci_request_regions(dev, "geode-aes");
526 if (ret)
526 goto eenable; 527 goto eenable;
527 528
528 _iobase = pci_iomap(dev, 0, 0); 529 _iobase = pci_iomap(dev, 0, 0);
@@ -537,13 +538,16 @@ geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
537 /* Clear any pending activity */ 538 /* Clear any pending activity */
538 iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG); 539 iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG);
539 540
540 if ((ret = crypto_register_alg(&geode_alg))) 541 ret = crypto_register_alg(&geode_alg);
542 if (ret)
541 goto eiomap; 543 goto eiomap;
542 544
543 if ((ret = crypto_register_alg(&geode_ecb_alg))) 545 ret = crypto_register_alg(&geode_ecb_alg);
546 if (ret)
544 goto ealg; 547 goto ealg;
545 548
546 if ((ret = crypto_register_alg(&geode_cbc_alg))) 549 ret = crypto_register_alg(&geode_cbc_alg);
550 if (ret)
547 goto eecb; 551 goto eecb;
548 552
549 printk(KERN_NOTICE "geode-aes: GEODE AES engine enabled.\n"); 553 printk(KERN_NOTICE "geode-aes: GEODE AES engine enabled.\n");
diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c
index 73e8b1713b54..16fce3aadf4d 100644
--- a/drivers/crypto/hifn_795x.c
+++ b/drivers/crypto/hifn_795x.c
@@ -638,7 +638,7 @@ struct hifn_crypto_alg
638 638
639#define ASYNC_FLAGS_MISALIGNED (1<<0) 639#define ASYNC_FLAGS_MISALIGNED (1<<0)
640 640
641struct ablkcipher_walk 641struct hifn_cipher_walk
642{ 642{
643 struct scatterlist cache[ASYNC_SCATTERLIST_CACHE]; 643 struct scatterlist cache[ASYNC_SCATTERLIST_CACHE];
644 u32 flags; 644 u32 flags;
@@ -657,7 +657,7 @@ struct hifn_request_context
657 u8 *iv; 657 u8 *iv;
658 unsigned int ivsize; 658 unsigned int ivsize;
659 u8 op, type, mode, unused; 659 u8 op, type, mode, unused;
660 struct ablkcipher_walk walk; 660 struct hifn_cipher_walk walk;
661}; 661};
662 662
663#define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg) 663#define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg)
@@ -1417,7 +1417,7 @@ static int hifn_setup_dma(struct hifn_device *dev,
1417 return 0; 1417 return 0;
1418} 1418}
1419 1419
1420static int ablkcipher_walk_init(struct ablkcipher_walk *w, 1420static int hifn_cipher_walk_init(struct hifn_cipher_walk *w,
1421 int num, gfp_t gfp_flags) 1421 int num, gfp_t gfp_flags)
1422{ 1422{
1423 int i; 1423 int i;
@@ -1442,7 +1442,7 @@ static int ablkcipher_walk_init(struct ablkcipher_walk *w,
1442 return i; 1442 return i;
1443} 1443}
1444 1444
1445static void ablkcipher_walk_exit(struct ablkcipher_walk *w) 1445static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w)
1446{ 1446{
1447 int i; 1447 int i;
1448 1448
@@ -1486,8 +1486,8 @@ static int ablkcipher_add(unsigned int *drestp, struct scatterlist *dst,
1486 return idx; 1486 return idx;
1487} 1487}
1488 1488
1489static int ablkcipher_walk(struct ablkcipher_request *req, 1489static int hifn_cipher_walk(struct ablkcipher_request *req,
1490 struct ablkcipher_walk *w) 1490 struct hifn_cipher_walk *w)
1491{ 1491{
1492 struct scatterlist *dst, *t; 1492 struct scatterlist *dst, *t;
1493 unsigned int nbytes = req->nbytes, offset, copy, diff; 1493 unsigned int nbytes = req->nbytes, offset, copy, diff;
@@ -1600,12 +1600,12 @@ static int hifn_setup_session(struct ablkcipher_request *req)
1600 } 1600 }
1601 1601
1602 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { 1602 if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1603 err = ablkcipher_walk_init(&rctx->walk, idx, GFP_ATOMIC); 1603 err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC);
1604 if (err < 0) 1604 if (err < 0)
1605 return err; 1605 return err;
1606 } 1606 }
1607 1607
1608 sg_num = ablkcipher_walk(req, &rctx->walk); 1608 sg_num = hifn_cipher_walk(req, &rctx->walk);
1609 if (sg_num < 0) { 1609 if (sg_num < 0) {
1610 err = sg_num; 1610 err = sg_num;
1611 goto err_out_exit; 1611 goto err_out_exit;
@@ -1806,7 +1806,7 @@ static void hifn_process_ready(struct ablkcipher_request *req, int error)
1806 kunmap_atomic(saddr, KM_SOFTIRQ0); 1806 kunmap_atomic(saddr, KM_SOFTIRQ0);
1807 } 1807 }
1808 1808
1809 ablkcipher_walk_exit(&rctx->walk); 1809 hifn_cipher_walk_exit(&rctx->walk);
1810 } 1810 }
1811 1811
1812 req->base.complete(&req->base, error); 1812 req->base.complete(&req->base, error);
diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c
index 6f29012bcc43..e095422b58dd 100644
--- a/drivers/crypto/mv_cesa.c
+++ b/drivers/crypto/mv_cesa.c
@@ -15,8 +15,14 @@
15#include <linux/platform_device.h> 15#include <linux/platform_device.h>
16#include <linux/scatterlist.h> 16#include <linux/scatterlist.h>
17#include <linux/slab.h> 17#include <linux/slab.h>
18#include <crypto/internal/hash.h>
19#include <crypto/sha.h>
18 20
19#include "mv_cesa.h" 21#include "mv_cesa.h"
22
23#define MV_CESA "MV-CESA:"
24#define MAX_HW_HASH_SIZE 0xFFFF
25
20/* 26/*
21 * STM: 27 * STM:
22 * /---------------------------------------\ 28 * /---------------------------------------\
@@ -39,10 +45,12 @@ enum engine_status {
39 * @dst_sg_it: sg iterator for dst 45 * @dst_sg_it: sg iterator for dst
40 * @sg_src_left: bytes left in src to process (scatter list) 46 * @sg_src_left: bytes left in src to process (scatter list)
41 * @src_start: offset to add to src start position (scatter list) 47 * @src_start: offset to add to src start position (scatter list)
42 * @crypt_len: length of current crypt process 48 * @crypt_len: length of current hw crypt/hash process
49 * @hw_nbytes: total bytes to process in hw for this request
50 * @copy_back: whether to copy data back (crypt) or not (hash)
43 * @sg_dst_left: bytes left dst to process in this scatter list 51 * @sg_dst_left: bytes left dst to process in this scatter list
44 * @dst_start: offset to add to dst start position (scatter list) 52 * @dst_start: offset to add to dst start position (scatter list)
45 * @total_req_bytes: total number of bytes processed (request). 53 * @hw_processed_bytes: number of bytes processed by hw (request).
46 * 54 *
47 * sg helper are used to iterate over the scatterlist. Since the size of the 55 * sg helper are used to iterate over the scatterlist. Since the size of the
48 * SRAM may be less than the scatter size, this struct struct is used to keep 56 * SRAM may be less than the scatter size, this struct struct is used to keep
@@ -51,15 +59,19 @@ enum engine_status {
51struct req_progress { 59struct req_progress {
52 struct sg_mapping_iter src_sg_it; 60 struct sg_mapping_iter src_sg_it;
53 struct sg_mapping_iter dst_sg_it; 61 struct sg_mapping_iter dst_sg_it;
62 void (*complete) (void);
63 void (*process) (int is_first);
54 64
55 /* src mostly */ 65 /* src mostly */
56 int sg_src_left; 66 int sg_src_left;
57 int src_start; 67 int src_start;
58 int crypt_len; 68 int crypt_len;
69 int hw_nbytes;
59 /* dst mostly */ 70 /* dst mostly */
71 int copy_back;
60 int sg_dst_left; 72 int sg_dst_left;
61 int dst_start; 73 int dst_start;
62 int total_req_bytes; 74 int hw_processed_bytes;
63}; 75};
64 76
65struct crypto_priv { 77struct crypto_priv {
@@ -72,10 +84,12 @@ struct crypto_priv {
72 spinlock_t lock; 84 spinlock_t lock;
73 struct crypto_queue queue; 85 struct crypto_queue queue;
74 enum engine_status eng_st; 86 enum engine_status eng_st;
75 struct ablkcipher_request *cur_req; 87 struct crypto_async_request *cur_req;
76 struct req_progress p; 88 struct req_progress p;
77 int max_req_size; 89 int max_req_size;
78 int sram_size; 90 int sram_size;
91 int has_sha1;
92 int has_hmac_sha1;
79}; 93};
80 94
81static struct crypto_priv *cpg; 95static struct crypto_priv *cpg;
@@ -97,6 +111,31 @@ struct mv_req_ctx {
97 int decrypt; 111 int decrypt;
98}; 112};
99 113
114enum hash_op {
115 COP_SHA1,
116 COP_HMAC_SHA1
117};
118
119struct mv_tfm_hash_ctx {
120 struct crypto_shash *fallback;
121 struct crypto_shash *base_hash;
122 u32 ivs[2 * SHA1_DIGEST_SIZE / 4];
123 int count_add;
124 enum hash_op op;
125};
126
127struct mv_req_hash_ctx {
128 u64 count;
129 u32 state[SHA1_DIGEST_SIZE / 4];
130 u8 buffer[SHA1_BLOCK_SIZE];
131 int first_hash; /* marks that we don't have previous state */
132 int last_chunk; /* marks that this is the 'final' request */
133 int extra_bytes; /* unprocessed bytes in buffer */
134 enum hash_op op;
135 int count_add;
136 struct scatterlist dummysg;
137};
138
100static void compute_aes_dec_key(struct mv_ctx *ctx) 139static void compute_aes_dec_key(struct mv_ctx *ctx)
101{ 140{
102 struct crypto_aes_ctx gen_aes_key; 141 struct crypto_aes_ctx gen_aes_key;
@@ -144,32 +183,51 @@ static int mv_setkey_aes(struct crypto_ablkcipher *cipher, const u8 *key,
144 return 0; 183 return 0;
145} 184}
146 185
147static void setup_data_in(struct ablkcipher_request *req) 186static void copy_src_to_buf(struct req_progress *p, char *dbuf, int len)
148{ 187{
149 int ret; 188 int ret;
150 void *buf; 189 void *sbuf;
190 int copied = 0;
151 191
152 if (!cpg->p.sg_src_left) { 192 while (1) {
153 ret = sg_miter_next(&cpg->p.src_sg_it); 193 if (!p->sg_src_left) {
154 BUG_ON(!ret); 194 ret = sg_miter_next(&p->src_sg_it);
155 cpg->p.sg_src_left = cpg->p.src_sg_it.length; 195 BUG_ON(!ret);
156 cpg->p.src_start = 0; 196 p->sg_src_left = p->src_sg_it.length;
157 } 197 p->src_start = 0;
158 198 }
159 cpg->p.crypt_len = min(cpg->p.sg_src_left, cpg->max_req_size);
160
161 buf = cpg->p.src_sg_it.addr;
162 buf += cpg->p.src_start;
163 199
164 memcpy(cpg->sram + SRAM_DATA_IN_START, buf, cpg->p.crypt_len); 200 sbuf = p->src_sg_it.addr + p->src_start;
201
202 if (p->sg_src_left <= len - copied) {
203 memcpy(dbuf + copied, sbuf, p->sg_src_left);
204 copied += p->sg_src_left;
205 p->sg_src_left = 0;
206 if (copied >= len)
207 break;
208 } else {
209 int copy_len = len - copied;
210 memcpy(dbuf + copied, sbuf, copy_len);
211 p->src_start += copy_len;
212 p->sg_src_left -= copy_len;
213 break;
214 }
215 }
216}
165 217
166 cpg->p.sg_src_left -= cpg->p.crypt_len; 218static void setup_data_in(void)
167 cpg->p.src_start += cpg->p.crypt_len; 219{
220 struct req_progress *p = &cpg->p;
221 int data_in_sram =
222 min(p->hw_nbytes - p->hw_processed_bytes, cpg->max_req_size);
223 copy_src_to_buf(p, cpg->sram + SRAM_DATA_IN_START + p->crypt_len,
224 data_in_sram - p->crypt_len);
225 p->crypt_len = data_in_sram;
168} 226}
169 227
170static void mv_process_current_q(int first_block) 228static void mv_process_current_q(int first_block)
171{ 229{
172 struct ablkcipher_request *req = cpg->cur_req; 230 struct ablkcipher_request *req = ablkcipher_request_cast(cpg->cur_req);
173 struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 231 struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
174 struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); 232 struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req);
175 struct sec_accel_config op; 233 struct sec_accel_config op;
@@ -179,6 +237,7 @@ static void mv_process_current_q(int first_block)
179 op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_ECB; 237 op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_ECB;
180 break; 238 break;
181 case COP_AES_CBC: 239 case COP_AES_CBC:
240 default:
182 op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_CBC; 241 op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_CBC;
183 op.enc_iv = ENC_IV_POINT(SRAM_DATA_IV) | 242 op.enc_iv = ENC_IV_POINT(SRAM_DATA_IV) |
184 ENC_IV_BUF_POINT(SRAM_DATA_IV_BUF); 243 ENC_IV_BUF_POINT(SRAM_DATA_IV_BUF);
@@ -211,7 +270,7 @@ static void mv_process_current_q(int first_block)
211 ENC_P_DST(SRAM_DATA_OUT_START); 270 ENC_P_DST(SRAM_DATA_OUT_START);
212 op.enc_key_p = SRAM_DATA_KEY_P; 271 op.enc_key_p = SRAM_DATA_KEY_P;
213 272
214 setup_data_in(req); 273 setup_data_in();
215 op.enc_len = cpg->p.crypt_len; 274 op.enc_len = cpg->p.crypt_len;
216 memcpy(cpg->sram + SRAM_CONFIG, &op, 275 memcpy(cpg->sram + SRAM_CONFIG, &op,
217 sizeof(struct sec_accel_config)); 276 sizeof(struct sec_accel_config));
@@ -228,91 +287,294 @@ static void mv_process_current_q(int first_block)
228 287
229static void mv_crypto_algo_completion(void) 288static void mv_crypto_algo_completion(void)
230{ 289{
231 struct ablkcipher_request *req = cpg->cur_req; 290 struct ablkcipher_request *req = ablkcipher_request_cast(cpg->cur_req);
232 struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); 291 struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req);
233 292
293 sg_miter_stop(&cpg->p.src_sg_it);
294 sg_miter_stop(&cpg->p.dst_sg_it);
295
234 if (req_ctx->op != COP_AES_CBC) 296 if (req_ctx->op != COP_AES_CBC)
235 return ; 297 return ;
236 298
237 memcpy(req->info, cpg->sram + SRAM_DATA_IV_BUF, 16); 299 memcpy(req->info, cpg->sram + SRAM_DATA_IV_BUF, 16);
238} 300}
239 301
302static void mv_process_hash_current(int first_block)
303{
304 struct ahash_request *req = ahash_request_cast(cpg->cur_req);
305 struct mv_req_hash_ctx *req_ctx = ahash_request_ctx(req);
306 struct req_progress *p = &cpg->p;
307 struct sec_accel_config op = { 0 };
308 int is_last;
309
310 switch (req_ctx->op) {
311 case COP_SHA1:
312 default:
313 op.config = CFG_OP_MAC_ONLY | CFG_MACM_SHA1;
314 break;
315 case COP_HMAC_SHA1:
316 op.config = CFG_OP_MAC_ONLY | CFG_MACM_HMAC_SHA1;
317 break;
318 }
319
320 op.mac_src_p =
321 MAC_SRC_DATA_P(SRAM_DATA_IN_START) | MAC_SRC_TOTAL_LEN((u32)
322 req_ctx->
323 count);
324
325 setup_data_in();
326
327 op.mac_digest =
328 MAC_DIGEST_P(SRAM_DIGEST_BUF) | MAC_FRAG_LEN(p->crypt_len);
329 op.mac_iv =
330 MAC_INNER_IV_P(SRAM_HMAC_IV_IN) |
331 MAC_OUTER_IV_P(SRAM_HMAC_IV_OUT);
332
333 is_last = req_ctx->last_chunk
334 && (p->hw_processed_bytes + p->crypt_len >= p->hw_nbytes)
335 && (req_ctx->count <= MAX_HW_HASH_SIZE);
336 if (req_ctx->first_hash) {
337 if (is_last)
338 op.config |= CFG_NOT_FRAG;
339 else
340 op.config |= CFG_FIRST_FRAG;
341
342 req_ctx->first_hash = 0;
343 } else {
344 if (is_last)
345 op.config |= CFG_LAST_FRAG;
346 else
347 op.config |= CFG_MID_FRAG;
348 }
349
350 memcpy(cpg->sram + SRAM_CONFIG, &op, sizeof(struct sec_accel_config));
351
352 writel(SRAM_CONFIG, cpg->reg + SEC_ACCEL_DESC_P0);
353 /* GO */
354 writel(SEC_CMD_EN_SEC_ACCL0, cpg->reg + SEC_ACCEL_CMD);
355
356 /*
357 * XXX: add timer if the interrupt does not occur for some mystery
358 * reason
359 */
360}
361
362static inline int mv_hash_import_sha1_ctx(const struct mv_req_hash_ctx *ctx,
363 struct shash_desc *desc)
364{
365 int i;
366 struct sha1_state shash_state;
367
368 shash_state.count = ctx->count + ctx->count_add;
369 for (i = 0; i < 5; i++)
370 shash_state.state[i] = ctx->state[i];
371 memcpy(shash_state.buffer, ctx->buffer, sizeof(shash_state.buffer));
372 return crypto_shash_import(desc, &shash_state);
373}
374
375static int mv_hash_final_fallback(struct ahash_request *req)
376{
377 const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm);
378 struct mv_req_hash_ctx *req_ctx = ahash_request_ctx(req);
379 struct {
380 struct shash_desc shash;
381 char ctx[crypto_shash_descsize(tfm_ctx->fallback)];
382 } desc;
383 int rc;
384
385 desc.shash.tfm = tfm_ctx->fallback;
386 desc.shash.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
387 if (unlikely(req_ctx->first_hash)) {
388 crypto_shash_init(&desc.shash);
389 crypto_shash_update(&desc.shash, req_ctx->buffer,
390 req_ctx->extra_bytes);
391 } else {
392 /* only SHA1 for now....
393 */
394 rc = mv_hash_import_sha1_ctx(req_ctx, &desc.shash);
395 if (rc)
396 goto out;
397 }
398 rc = crypto_shash_final(&desc.shash, req->result);
399out:
400 return rc;
401}
402
403static void mv_hash_algo_completion(void)
404{
405 struct ahash_request *req = ahash_request_cast(cpg->cur_req);
406 struct mv_req_hash_ctx *ctx = ahash_request_ctx(req);
407
408 if (ctx->extra_bytes)
409 copy_src_to_buf(&cpg->p, ctx->buffer, ctx->extra_bytes);
410 sg_miter_stop(&cpg->p.src_sg_it);
411
412 ctx->state[0] = readl(cpg->reg + DIGEST_INITIAL_VAL_A);
413 ctx->state[1] = readl(cpg->reg + DIGEST_INITIAL_VAL_B);
414 ctx->state[2] = readl(cpg->reg + DIGEST_INITIAL_VAL_C);
415 ctx->state[3] = readl(cpg->reg + DIGEST_INITIAL_VAL_D);
416 ctx->state[4] = readl(cpg->reg + DIGEST_INITIAL_VAL_E);
417
418 if (likely(ctx->last_chunk)) {
419 if (likely(ctx->count <= MAX_HW_HASH_SIZE)) {
420 memcpy(req->result, cpg->sram + SRAM_DIGEST_BUF,
421 crypto_ahash_digestsize(crypto_ahash_reqtfm
422 (req)));
423 } else
424 mv_hash_final_fallback(req);
425 }
426}
427
240static void dequeue_complete_req(void) 428static void dequeue_complete_req(void)
241{ 429{
242 struct ablkcipher_request *req = cpg->cur_req; 430 struct crypto_async_request *req = cpg->cur_req;
243 void *buf; 431 void *buf;
244 int ret; 432 int ret;
433 cpg->p.hw_processed_bytes += cpg->p.crypt_len;
434 if (cpg->p.copy_back) {
435 int need_copy_len = cpg->p.crypt_len;
436 int sram_offset = 0;
437 do {
438 int dst_copy;
439
440 if (!cpg->p.sg_dst_left) {
441 ret = sg_miter_next(&cpg->p.dst_sg_it);
442 BUG_ON(!ret);
443 cpg->p.sg_dst_left = cpg->p.dst_sg_it.length;
444 cpg->p.dst_start = 0;
445 }
245 446
246 cpg->p.total_req_bytes += cpg->p.crypt_len; 447 buf = cpg->p.dst_sg_it.addr;
247 do { 448 buf += cpg->p.dst_start;
248 int dst_copy;
249
250 if (!cpg->p.sg_dst_left) {
251 ret = sg_miter_next(&cpg->p.dst_sg_it);
252 BUG_ON(!ret);
253 cpg->p.sg_dst_left = cpg->p.dst_sg_it.length;
254 cpg->p.dst_start = 0;
255 }
256
257 buf = cpg->p.dst_sg_it.addr;
258 buf += cpg->p.dst_start;
259 449
260 dst_copy = min(cpg->p.crypt_len, cpg->p.sg_dst_left); 450 dst_copy = min(need_copy_len, cpg->p.sg_dst_left);
261 451
262 memcpy(buf, cpg->sram + SRAM_DATA_OUT_START, dst_copy); 452 memcpy(buf,
453 cpg->sram + SRAM_DATA_OUT_START + sram_offset,
454 dst_copy);
455 sram_offset += dst_copy;
456 cpg->p.sg_dst_left -= dst_copy;
457 need_copy_len -= dst_copy;
458 cpg->p.dst_start += dst_copy;
459 } while (need_copy_len > 0);
460 }
263 461
264 cpg->p.sg_dst_left -= dst_copy; 462 cpg->p.crypt_len = 0;
265 cpg->p.crypt_len -= dst_copy;
266 cpg->p.dst_start += dst_copy;
267 } while (cpg->p.crypt_len > 0);
268 463
269 BUG_ON(cpg->eng_st != ENGINE_W_DEQUEUE); 464 BUG_ON(cpg->eng_st != ENGINE_W_DEQUEUE);
270 if (cpg->p.total_req_bytes < req->nbytes) { 465 if (cpg->p.hw_processed_bytes < cpg->p.hw_nbytes) {
271 /* process next scatter list entry */ 466 /* process next scatter list entry */
272 cpg->eng_st = ENGINE_BUSY; 467 cpg->eng_st = ENGINE_BUSY;
273 mv_process_current_q(0); 468 cpg->p.process(0);
274 } else { 469 } else {
275 sg_miter_stop(&cpg->p.src_sg_it); 470 cpg->p.complete();
276 sg_miter_stop(&cpg->p.dst_sg_it);
277 mv_crypto_algo_completion();
278 cpg->eng_st = ENGINE_IDLE; 471 cpg->eng_st = ENGINE_IDLE;
279 req->base.complete(&req->base, 0); 472 local_bh_disable();
473 req->complete(req, 0);
474 local_bh_enable();
280 } 475 }
281} 476}
282 477
283static int count_sgs(struct scatterlist *sl, unsigned int total_bytes) 478static int count_sgs(struct scatterlist *sl, unsigned int total_bytes)
284{ 479{
285 int i = 0; 480 int i = 0;
286 481 size_t cur_len;
287 do { 482
288 total_bytes -= sl[i].length; 483 while (1) {
289 i++; 484 cur_len = sl[i].length;
290 485 ++i;
291 } while (total_bytes > 0); 486 if (total_bytes > cur_len)
487 total_bytes -= cur_len;
488 else
489 break;
490 }
292 491
293 return i; 492 return i;
294} 493}
295 494
296static void mv_enqueue_new_req(struct ablkcipher_request *req) 495static void mv_start_new_crypt_req(struct ablkcipher_request *req)
297{ 496{
497 struct req_progress *p = &cpg->p;
298 int num_sgs; 498 int num_sgs;
299 499
300 cpg->cur_req = req; 500 cpg->cur_req = &req->base;
301 memset(&cpg->p, 0, sizeof(struct req_progress)); 501 memset(p, 0, sizeof(struct req_progress));
502 p->hw_nbytes = req->nbytes;
503 p->complete = mv_crypto_algo_completion;
504 p->process = mv_process_current_q;
505 p->copy_back = 1;
302 506
303 num_sgs = count_sgs(req->src, req->nbytes); 507 num_sgs = count_sgs(req->src, req->nbytes);
304 sg_miter_start(&cpg->p.src_sg_it, req->src, num_sgs, SG_MITER_FROM_SG); 508 sg_miter_start(&p->src_sg_it, req->src, num_sgs, SG_MITER_FROM_SG);
305 509
306 num_sgs = count_sgs(req->dst, req->nbytes); 510 num_sgs = count_sgs(req->dst, req->nbytes);
307 sg_miter_start(&cpg->p.dst_sg_it, req->dst, num_sgs, SG_MITER_TO_SG); 511 sg_miter_start(&p->dst_sg_it, req->dst, num_sgs, SG_MITER_TO_SG);
512
308 mv_process_current_q(1); 513 mv_process_current_q(1);
309} 514}
310 515
516static void mv_start_new_hash_req(struct ahash_request *req)
517{
518 struct req_progress *p = &cpg->p;
519 struct mv_req_hash_ctx *ctx = ahash_request_ctx(req);
520 const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm);
521 int num_sgs, hw_bytes, old_extra_bytes, rc;
522 cpg->cur_req = &req->base;
523 memset(p, 0, sizeof(struct req_progress));
524 hw_bytes = req->nbytes + ctx->extra_bytes;
525 old_extra_bytes = ctx->extra_bytes;
526
527 if (unlikely(ctx->extra_bytes)) {
528 memcpy(cpg->sram + SRAM_DATA_IN_START, ctx->buffer,
529 ctx->extra_bytes);
530 p->crypt_len = ctx->extra_bytes;
531 }
532
533 memcpy(cpg->sram + SRAM_HMAC_IV_IN, tfm_ctx->ivs, sizeof(tfm_ctx->ivs));
534
535 if (unlikely(!ctx->first_hash)) {
536 writel(ctx->state[0], cpg->reg + DIGEST_INITIAL_VAL_A);
537 writel(ctx->state[1], cpg->reg + DIGEST_INITIAL_VAL_B);
538 writel(ctx->state[2], cpg->reg + DIGEST_INITIAL_VAL_C);
539 writel(ctx->state[3], cpg->reg + DIGEST_INITIAL_VAL_D);
540 writel(ctx->state[4], cpg->reg + DIGEST_INITIAL_VAL_E);
541 }
542
543 ctx->extra_bytes = hw_bytes % SHA1_BLOCK_SIZE;
544 if (ctx->extra_bytes != 0
545 && (!ctx->last_chunk || ctx->count > MAX_HW_HASH_SIZE))
546 hw_bytes -= ctx->extra_bytes;
547 else
548 ctx->extra_bytes = 0;
549
550 num_sgs = count_sgs(req->src, req->nbytes);
551 sg_miter_start(&p->src_sg_it, req->src, num_sgs, SG_MITER_FROM_SG);
552
553 if (hw_bytes) {
554 p->hw_nbytes = hw_bytes;
555 p->complete = mv_hash_algo_completion;
556 p->process = mv_process_hash_current;
557
558 mv_process_hash_current(1);
559 } else {
560 copy_src_to_buf(p, ctx->buffer + old_extra_bytes,
561 ctx->extra_bytes - old_extra_bytes);
562 sg_miter_stop(&p->src_sg_it);
563 if (ctx->last_chunk)
564 rc = mv_hash_final_fallback(req);
565 else
566 rc = 0;
567 cpg->eng_st = ENGINE_IDLE;
568 local_bh_disable();
569 req->base.complete(&req->base, rc);
570 local_bh_enable();
571 }
572}
573
311static int queue_manag(void *data) 574static int queue_manag(void *data)
312{ 575{
313 cpg->eng_st = ENGINE_IDLE; 576 cpg->eng_st = ENGINE_IDLE;
314 do { 577 do {
315 struct ablkcipher_request *req;
316 struct crypto_async_request *async_req = NULL; 578 struct crypto_async_request *async_req = NULL;
317 struct crypto_async_request *backlog; 579 struct crypto_async_request *backlog;
318 580
@@ -338,9 +600,18 @@ static int queue_manag(void *data)
338 } 600 }
339 601
340 if (async_req) { 602 if (async_req) {
341 req = container_of(async_req, 603 if (async_req->tfm->__crt_alg->cra_type !=
342 struct ablkcipher_request, base); 604 &crypto_ahash_type) {
343 mv_enqueue_new_req(req); 605 struct ablkcipher_request *req =
606 container_of(async_req,
607 struct ablkcipher_request,
608 base);
609 mv_start_new_crypt_req(req);
610 } else {
611 struct ahash_request *req =
612 ahash_request_cast(async_req);
613 mv_start_new_hash_req(req);
614 }
344 async_req = NULL; 615 async_req = NULL;
345 } 616 }
346 617
@@ -350,13 +621,13 @@ static int queue_manag(void *data)
350 return 0; 621 return 0;
351} 622}
352 623
353static int mv_handle_req(struct ablkcipher_request *req) 624static int mv_handle_req(struct crypto_async_request *req)
354{ 625{
355 unsigned long flags; 626 unsigned long flags;
356 int ret; 627 int ret;
357 628
358 spin_lock_irqsave(&cpg->lock, flags); 629 spin_lock_irqsave(&cpg->lock, flags);
359 ret = ablkcipher_enqueue_request(&cpg->queue, req); 630 ret = crypto_enqueue_request(&cpg->queue, req);
360 spin_unlock_irqrestore(&cpg->lock, flags); 631 spin_unlock_irqrestore(&cpg->lock, flags);
361 wake_up_process(cpg->queue_th); 632 wake_up_process(cpg->queue_th);
362 return ret; 633 return ret;
@@ -369,7 +640,7 @@ static int mv_enc_aes_ecb(struct ablkcipher_request *req)
369 req_ctx->op = COP_AES_ECB; 640 req_ctx->op = COP_AES_ECB;
370 req_ctx->decrypt = 0; 641 req_ctx->decrypt = 0;
371 642
372 return mv_handle_req(req); 643 return mv_handle_req(&req->base);
373} 644}
374 645
375static int mv_dec_aes_ecb(struct ablkcipher_request *req) 646static int mv_dec_aes_ecb(struct ablkcipher_request *req)
@@ -381,7 +652,7 @@ static int mv_dec_aes_ecb(struct ablkcipher_request *req)
381 req_ctx->decrypt = 1; 652 req_ctx->decrypt = 1;
382 653
383 compute_aes_dec_key(ctx); 654 compute_aes_dec_key(ctx);
384 return mv_handle_req(req); 655 return mv_handle_req(&req->base);
385} 656}
386 657
387static int mv_enc_aes_cbc(struct ablkcipher_request *req) 658static int mv_enc_aes_cbc(struct ablkcipher_request *req)
@@ -391,7 +662,7 @@ static int mv_enc_aes_cbc(struct ablkcipher_request *req)
391 req_ctx->op = COP_AES_CBC; 662 req_ctx->op = COP_AES_CBC;
392 req_ctx->decrypt = 0; 663 req_ctx->decrypt = 0;
393 664
394 return mv_handle_req(req); 665 return mv_handle_req(&req->base);
395} 666}
396 667
397static int mv_dec_aes_cbc(struct ablkcipher_request *req) 668static int mv_dec_aes_cbc(struct ablkcipher_request *req)
@@ -403,7 +674,7 @@ static int mv_dec_aes_cbc(struct ablkcipher_request *req)
403 req_ctx->decrypt = 1; 674 req_ctx->decrypt = 1;
404 675
405 compute_aes_dec_key(ctx); 676 compute_aes_dec_key(ctx);
406 return mv_handle_req(req); 677 return mv_handle_req(&req->base);
407} 678}
408 679
409static int mv_cra_init(struct crypto_tfm *tfm) 680static int mv_cra_init(struct crypto_tfm *tfm)
@@ -412,6 +683,215 @@ static int mv_cra_init(struct crypto_tfm *tfm)
412 return 0; 683 return 0;
413} 684}
414 685
686static void mv_init_hash_req_ctx(struct mv_req_hash_ctx *ctx, int op,
687 int is_last, unsigned int req_len,
688 int count_add)
689{
690 memset(ctx, 0, sizeof(*ctx));
691 ctx->op = op;
692 ctx->count = req_len;
693 ctx->first_hash = 1;
694 ctx->last_chunk = is_last;
695 ctx->count_add = count_add;
696}
697
698static void mv_update_hash_req_ctx(struct mv_req_hash_ctx *ctx, int is_last,
699 unsigned req_len)
700{
701 ctx->last_chunk = is_last;
702 ctx->count += req_len;
703}
704
705static int mv_hash_init(struct ahash_request *req)
706{
707 const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm);
708 mv_init_hash_req_ctx(ahash_request_ctx(req), tfm_ctx->op, 0, 0,
709 tfm_ctx->count_add);
710 return 0;
711}
712
713static int mv_hash_update(struct ahash_request *req)
714{
715 if (!req->nbytes)
716 return 0;
717
718 mv_update_hash_req_ctx(ahash_request_ctx(req), 0, req->nbytes);
719 return mv_handle_req(&req->base);
720}
721
722static int mv_hash_final(struct ahash_request *req)
723{
724 struct mv_req_hash_ctx *ctx = ahash_request_ctx(req);
725 /* dummy buffer of 4 bytes */
726 sg_init_one(&ctx->dummysg, ctx->buffer, 4);
727 /* I think I'm allowed to do that... */
728 ahash_request_set_crypt(req, &ctx->dummysg, req->result, 0);
729 mv_update_hash_req_ctx(ctx, 1, 0);
730 return mv_handle_req(&req->base);
731}
732
733static int mv_hash_finup(struct ahash_request *req)
734{
735 if (!req->nbytes)
736 return mv_hash_final(req);
737
738 mv_update_hash_req_ctx(ahash_request_ctx(req), 1, req->nbytes);
739 return mv_handle_req(&req->base);
740}
741
742static int mv_hash_digest(struct ahash_request *req)
743{
744 const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm);
745 mv_init_hash_req_ctx(ahash_request_ctx(req), tfm_ctx->op, 1,
746 req->nbytes, tfm_ctx->count_add);
747 return mv_handle_req(&req->base);
748}
749
750static void mv_hash_init_ivs(struct mv_tfm_hash_ctx *ctx, const void *istate,
751 const void *ostate)
752{
753 const struct sha1_state *isha1_state = istate, *osha1_state = ostate;
754 int i;
755 for (i = 0; i < 5; i++) {
756 ctx->ivs[i] = cpu_to_be32(isha1_state->state[i]);
757 ctx->ivs[i + 5] = cpu_to_be32(osha1_state->state[i]);
758 }
759}
760
761static int mv_hash_setkey(struct crypto_ahash *tfm, const u8 * key,
762 unsigned int keylen)
763{
764 int rc;
765 struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(&tfm->base);
766 int bs, ds, ss;
767
768 if (!ctx->base_hash)
769 return 0;
770
771 rc = crypto_shash_setkey(ctx->fallback, key, keylen);
772 if (rc)
773 return rc;
774
775 /* Can't see a way to extract the ipad/opad from the fallback tfm
776 so I'm basically copying code from the hmac module */
777 bs = crypto_shash_blocksize(ctx->base_hash);
778 ds = crypto_shash_digestsize(ctx->base_hash);
779 ss = crypto_shash_statesize(ctx->base_hash);
780
781 {
782 struct {
783 struct shash_desc shash;
784 char ctx[crypto_shash_descsize(ctx->base_hash)];
785 } desc;
786 unsigned int i;
787 char ipad[ss];
788 char opad[ss];
789
790 desc.shash.tfm = ctx->base_hash;
791 desc.shash.flags = crypto_shash_get_flags(ctx->base_hash) &
792 CRYPTO_TFM_REQ_MAY_SLEEP;
793
794 if (keylen > bs) {
795 int err;
796
797 err =
798 crypto_shash_digest(&desc.shash, key, keylen, ipad);
799 if (err)
800 return err;
801
802 keylen = ds;
803 } else
804 memcpy(ipad, key, keylen);
805
806 memset(ipad + keylen, 0, bs - keylen);
807 memcpy(opad, ipad, bs);
808
809 for (i = 0; i < bs; i++) {
810 ipad[i] ^= 0x36;
811 opad[i] ^= 0x5c;
812 }
813
814 rc = crypto_shash_init(&desc.shash) ? :
815 crypto_shash_update(&desc.shash, ipad, bs) ? :
816 crypto_shash_export(&desc.shash, ipad) ? :
817 crypto_shash_init(&desc.shash) ? :
818 crypto_shash_update(&desc.shash, opad, bs) ? :
819 crypto_shash_export(&desc.shash, opad);
820
821 if (rc == 0)
822 mv_hash_init_ivs(ctx, ipad, opad);
823
824 return rc;
825 }
826}
827
828static int mv_cra_hash_init(struct crypto_tfm *tfm, const char *base_hash_name,
829 enum hash_op op, int count_add)
830{
831 const char *fallback_driver_name = tfm->__crt_alg->cra_name;
832 struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(tfm);
833 struct crypto_shash *fallback_tfm = NULL;
834 struct crypto_shash *base_hash = NULL;
835 int err = -ENOMEM;
836
837 ctx->op = op;
838 ctx->count_add = count_add;
839
840 /* Allocate a fallback and abort if it failed. */
841 fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0,
842 CRYPTO_ALG_NEED_FALLBACK);
843 if (IS_ERR(fallback_tfm)) {
844 printk(KERN_WARNING MV_CESA
845 "Fallback driver '%s' could not be loaded!\n",
846 fallback_driver_name);
847 err = PTR_ERR(fallback_tfm);
848 goto out;
849 }
850 ctx->fallback = fallback_tfm;
851
852 if (base_hash_name) {
853 /* Allocate a hash to compute the ipad/opad of hmac. */
854 base_hash = crypto_alloc_shash(base_hash_name, 0,
855 CRYPTO_ALG_NEED_FALLBACK);
856 if (IS_ERR(base_hash)) {
857 printk(KERN_WARNING MV_CESA
858 "Base driver '%s' could not be loaded!\n",
859 base_hash_name);
860 err = PTR_ERR(fallback_tfm);
861 goto err_bad_base;
862 }
863 }
864 ctx->base_hash = base_hash;
865
866 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
867 sizeof(struct mv_req_hash_ctx) +
868 crypto_shash_descsize(ctx->fallback));
869 return 0;
870err_bad_base:
871 crypto_free_shash(fallback_tfm);
872out:
873 return err;
874}
875
876static void mv_cra_hash_exit(struct crypto_tfm *tfm)
877{
878 struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(tfm);
879
880 crypto_free_shash(ctx->fallback);
881 if (ctx->base_hash)
882 crypto_free_shash(ctx->base_hash);
883}
884
885static int mv_cra_hash_sha1_init(struct crypto_tfm *tfm)
886{
887 return mv_cra_hash_init(tfm, NULL, COP_SHA1, 0);
888}
889
890static int mv_cra_hash_hmac_sha1_init(struct crypto_tfm *tfm)
891{
892 return mv_cra_hash_init(tfm, "sha1", COP_HMAC_SHA1, SHA1_BLOCK_SIZE);
893}
894
415irqreturn_t crypto_int(int irq, void *priv) 895irqreturn_t crypto_int(int irq, void *priv)
416{ 896{
417 u32 val; 897 u32 val;
@@ -474,6 +954,53 @@ struct crypto_alg mv_aes_alg_cbc = {
474 }, 954 },
475}; 955};
476 956
957struct ahash_alg mv_sha1_alg = {
958 .init = mv_hash_init,
959 .update = mv_hash_update,
960 .final = mv_hash_final,
961 .finup = mv_hash_finup,
962 .digest = mv_hash_digest,
963 .halg = {
964 .digestsize = SHA1_DIGEST_SIZE,
965 .base = {
966 .cra_name = "sha1",
967 .cra_driver_name = "mv-sha1",
968 .cra_priority = 300,
969 .cra_flags =
970 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
971 .cra_blocksize = SHA1_BLOCK_SIZE,
972 .cra_ctxsize = sizeof(struct mv_tfm_hash_ctx),
973 .cra_init = mv_cra_hash_sha1_init,
974 .cra_exit = mv_cra_hash_exit,
975 .cra_module = THIS_MODULE,
976 }
977 }
978};
979
980struct ahash_alg mv_hmac_sha1_alg = {
981 .init = mv_hash_init,
982 .update = mv_hash_update,
983 .final = mv_hash_final,
984 .finup = mv_hash_finup,
985 .digest = mv_hash_digest,
986 .setkey = mv_hash_setkey,
987 .halg = {
988 .digestsize = SHA1_DIGEST_SIZE,
989 .base = {
990 .cra_name = "hmac(sha1)",
991 .cra_driver_name = "mv-hmac-sha1",
992 .cra_priority = 300,
993 .cra_flags =
994 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
995 .cra_blocksize = SHA1_BLOCK_SIZE,
996 .cra_ctxsize = sizeof(struct mv_tfm_hash_ctx),
997 .cra_init = mv_cra_hash_hmac_sha1_init,
998 .cra_exit = mv_cra_hash_exit,
999 .cra_module = THIS_MODULE,
1000 }
1001 }
1002};
1003
477static int mv_probe(struct platform_device *pdev) 1004static int mv_probe(struct platform_device *pdev)
478{ 1005{
479 struct crypto_priv *cp; 1006 struct crypto_priv *cp;
@@ -482,7 +1009,7 @@ static int mv_probe(struct platform_device *pdev)
482 int ret; 1009 int ret;
483 1010
484 if (cpg) { 1011 if (cpg) {
485 printk(KERN_ERR "Second crypto dev?\n"); 1012 printk(KERN_ERR MV_CESA "Second crypto dev?\n");
486 return -EEXIST; 1013 return -EEXIST;
487 } 1014 }
488 1015
@@ -496,7 +1023,7 @@ static int mv_probe(struct platform_device *pdev)
496 1023
497 spin_lock_init(&cp->lock); 1024 spin_lock_init(&cp->lock);
498 crypto_init_queue(&cp->queue, 50); 1025 crypto_init_queue(&cp->queue, 50);
499 cp->reg = ioremap(res->start, res->end - res->start + 1); 1026 cp->reg = ioremap(res->start, resource_size(res));
500 if (!cp->reg) { 1027 if (!cp->reg) {
501 ret = -ENOMEM; 1028 ret = -ENOMEM;
502 goto err; 1029 goto err;
@@ -507,7 +1034,7 @@ static int mv_probe(struct platform_device *pdev)
507 ret = -ENXIO; 1034 ret = -ENXIO;
508 goto err_unmap_reg; 1035 goto err_unmap_reg;
509 } 1036 }
510 cp->sram_size = res->end - res->start + 1; 1037 cp->sram_size = resource_size(res);
511 cp->max_req_size = cp->sram_size - SRAM_CFG_SPACE; 1038 cp->max_req_size = cp->sram_size - SRAM_CFG_SPACE;
512 cp->sram = ioremap(res->start, cp->sram_size); 1039 cp->sram = ioremap(res->start, cp->sram_size);
513 if (!cp->sram) { 1040 if (!cp->sram) {
@@ -546,6 +1073,21 @@ static int mv_probe(struct platform_device *pdev)
546 ret = crypto_register_alg(&mv_aes_alg_cbc); 1073 ret = crypto_register_alg(&mv_aes_alg_cbc);
547 if (ret) 1074 if (ret)
548 goto err_unreg_ecb; 1075 goto err_unreg_ecb;
1076
1077 ret = crypto_register_ahash(&mv_sha1_alg);
1078 if (ret == 0)
1079 cpg->has_sha1 = 1;
1080 else
1081 printk(KERN_WARNING MV_CESA "Could not register sha1 driver\n");
1082
1083 ret = crypto_register_ahash(&mv_hmac_sha1_alg);
1084 if (ret == 0) {
1085 cpg->has_hmac_sha1 = 1;
1086 } else {
1087 printk(KERN_WARNING MV_CESA
1088 "Could not register hmac-sha1 driver\n");
1089 }
1090
549 return 0; 1091 return 0;
550err_unreg_ecb: 1092err_unreg_ecb:
551 crypto_unregister_alg(&mv_aes_alg_ecb); 1093 crypto_unregister_alg(&mv_aes_alg_ecb);
@@ -570,6 +1112,10 @@ static int mv_remove(struct platform_device *pdev)
570 1112
571 crypto_unregister_alg(&mv_aes_alg_ecb); 1113 crypto_unregister_alg(&mv_aes_alg_ecb);
572 crypto_unregister_alg(&mv_aes_alg_cbc); 1114 crypto_unregister_alg(&mv_aes_alg_cbc);
1115 if (cp->has_sha1)
1116 crypto_unregister_ahash(&mv_sha1_alg);
1117 if (cp->has_hmac_sha1)
1118 crypto_unregister_ahash(&mv_hmac_sha1_alg);
573 kthread_stop(cp->queue_th); 1119 kthread_stop(cp->queue_th);
574 free_irq(cp->irq, cp); 1120 free_irq(cp->irq, cp);
575 memset(cp->sram, 0, cp->sram_size); 1121 memset(cp->sram, 0, cp->sram_size);
diff --git a/drivers/crypto/mv_cesa.h b/drivers/crypto/mv_cesa.h
index c3e25d3bb171..08fcb1116d90 100644
--- a/drivers/crypto/mv_cesa.h
+++ b/drivers/crypto/mv_cesa.h
@@ -1,6 +1,10 @@
1#ifndef __MV_CRYPTO_H__ 1#ifndef __MV_CRYPTO_H__
2 2
3#define DIGEST_INITIAL_VAL_A 0xdd00 3#define DIGEST_INITIAL_VAL_A 0xdd00
4#define DIGEST_INITIAL_VAL_B 0xdd04
5#define DIGEST_INITIAL_VAL_C 0xdd08
6#define DIGEST_INITIAL_VAL_D 0xdd0c
7#define DIGEST_INITIAL_VAL_E 0xdd10
4#define DES_CMD_REG 0xdd58 8#define DES_CMD_REG 0xdd58
5 9
6#define SEC_ACCEL_CMD 0xde00 10#define SEC_ACCEL_CMD 0xde00
@@ -70,6 +74,10 @@ struct sec_accel_config {
70#define CFG_AES_LEN_128 (0 << 24) 74#define CFG_AES_LEN_128 (0 << 24)
71#define CFG_AES_LEN_192 (1 << 24) 75#define CFG_AES_LEN_192 (1 << 24)
72#define CFG_AES_LEN_256 (2 << 24) 76#define CFG_AES_LEN_256 (2 << 24)
77#define CFG_NOT_FRAG (0 << 30)
78#define CFG_FIRST_FRAG (1 << 30)
79#define CFG_LAST_FRAG (2 << 30)
80#define CFG_MID_FRAG (3 << 30)
73 81
74 u32 enc_p; 82 u32 enc_p;
75#define ENC_P_SRC(x) (x) 83#define ENC_P_SRC(x) (x)
@@ -90,7 +98,11 @@ struct sec_accel_config {
90#define MAC_SRC_TOTAL_LEN(x) ((x) << 16) 98#define MAC_SRC_TOTAL_LEN(x) ((x) << 16)
91 99
92 u32 mac_digest; 100 u32 mac_digest;
101#define MAC_DIGEST_P(x) (x)
102#define MAC_FRAG_LEN(x) ((x) << 16)
93 u32 mac_iv; 103 u32 mac_iv;
104#define MAC_INNER_IV_P(x) (x)
105#define MAC_OUTER_IV_P(x) ((x) << 16)
94}__attribute__ ((packed)); 106}__attribute__ ((packed));
95 /* 107 /*
96 * /-----------\ 0 108 * /-----------\ 0
@@ -101,19 +113,37 @@ struct sec_accel_config {
101 * | IV IN | 4 * 4 113 * | IV IN | 4 * 4
102 * |-----------| 0x40 (inplace) 114 * |-----------| 0x40 (inplace)
103 * | IV BUF | 4 * 4 115 * | IV BUF | 4 * 4
104 * |-----------| 0x50 116 * |-----------| 0x80
105 * | DATA IN | 16 * x (max ->max_req_size) 117 * | DATA IN | 16 * x (max ->max_req_size)
106 * |-----------| 0x50 (inplace operation) 118 * |-----------| 0x80 (inplace operation)
107 * | DATA OUT | 16 * x (max ->max_req_size) 119 * | DATA OUT | 16 * x (max ->max_req_size)
108 * \-----------/ SRAM size 120 * \-----------/ SRAM size
109 */ 121 */
122
123 /* Hashing memory map:
124 * /-----------\ 0
125 * | ACCEL CFG | 4 * 8
126 * |-----------| 0x20
127 * | Inner IV | 5 * 4
128 * |-----------| 0x34
129 * | Outer IV | 5 * 4
130 * |-----------| 0x48
131 * | Output BUF| 5 * 4
132 * |-----------| 0x80
133 * | DATA IN | 64 * x (max ->max_req_size)
134 * \-----------/ SRAM size
135 */
110#define SRAM_CONFIG 0x00 136#define SRAM_CONFIG 0x00
111#define SRAM_DATA_KEY_P 0x20 137#define SRAM_DATA_KEY_P 0x20
112#define SRAM_DATA_IV 0x40 138#define SRAM_DATA_IV 0x40
113#define SRAM_DATA_IV_BUF 0x40 139#define SRAM_DATA_IV_BUF 0x40
114#define SRAM_DATA_IN_START 0x50 140#define SRAM_DATA_IN_START 0x80
115#define SRAM_DATA_OUT_START 0x50 141#define SRAM_DATA_OUT_START 0x80
142
143#define SRAM_HMAC_IV_IN 0x20
144#define SRAM_HMAC_IV_OUT 0x34
145#define SRAM_DIGEST_BUF 0x48
116 146
117#define SRAM_CFG_SPACE 0x50 147#define SRAM_CFG_SPACE 0x80
118 148
119#endif 149#endif
diff --git a/drivers/crypto/n2_asm.S b/drivers/crypto/n2_asm.S
new file mode 100644
index 000000000000..f7c793745a1e
--- /dev/null
+++ b/drivers/crypto/n2_asm.S
@@ -0,0 +1,95 @@
1/* n2_asm.S: Hypervisor calls for NCS support.
2 *
3 * Copyright (C) 2009 David S. Miller <davem@davemloft.net>
4 */
5
6#include <linux/linkage.h>
7#include <asm/hypervisor.h>
8#include "n2_core.h"
9
10 /* o0: queue type
11 * o1: RA of queue
12 * o2: num entries in queue
13 * o3: address of queue handle return
14 */
15ENTRY(sun4v_ncs_qconf)
16 mov HV_FAST_NCS_QCONF, %o5
17 ta HV_FAST_TRAP
18 stx %o1, [%o3]
19 retl
20 nop
21ENDPROC(sun4v_ncs_qconf)
22
23 /* %o0: queue handle
24 * %o1: address of queue type return
25 * %o2: address of queue base address return
26 * %o3: address of queue num entries return
27 */
28ENTRY(sun4v_ncs_qinfo)
29 mov %o1, %g1
30 mov %o2, %g2
31 mov %o3, %g3
32 mov HV_FAST_NCS_QINFO, %o5
33 ta HV_FAST_TRAP
34 stx %o1, [%g1]
35 stx %o2, [%g2]
36 stx %o3, [%g3]
37 retl
38 nop
39ENDPROC(sun4v_ncs_qinfo)
40
41 /* %o0: queue handle
42 * %o1: address of head offset return
43 */
44ENTRY(sun4v_ncs_gethead)
45 mov %o1, %o2
46 mov HV_FAST_NCS_GETHEAD, %o5
47 ta HV_FAST_TRAP
48 stx %o1, [%o2]
49 retl
50 nop
51ENDPROC(sun4v_ncs_gethead)
52
53 /* %o0: queue handle
54 * %o1: address of tail offset return
55 */
56ENTRY(sun4v_ncs_gettail)
57 mov %o1, %o2
58 mov HV_FAST_NCS_GETTAIL, %o5
59 ta HV_FAST_TRAP
60 stx %o1, [%o2]
61 retl
62 nop
63ENDPROC(sun4v_ncs_gettail)
64
65 /* %o0: queue handle
66 * %o1: new tail offset
67 */
68ENTRY(sun4v_ncs_settail)
69 mov HV_FAST_NCS_SETTAIL, %o5
70 ta HV_FAST_TRAP
71 retl
72 nop
73ENDPROC(sun4v_ncs_settail)
74
75 /* %o0: queue handle
76 * %o1: address of devino return
77 */
78ENTRY(sun4v_ncs_qhandle_to_devino)
79 mov %o1, %o2
80 mov HV_FAST_NCS_QHANDLE_TO_DEVINO, %o5
81 ta HV_FAST_TRAP
82 stx %o1, [%o2]
83 retl
84 nop
85ENDPROC(sun4v_ncs_qhandle_to_devino)
86
87 /* %o0: queue handle
88 * %o1: new head offset
89 */
90ENTRY(sun4v_ncs_sethead_marker)
91 mov HV_FAST_NCS_SETHEAD_MARKER, %o5
92 ta HV_FAST_TRAP
93 retl
94 nop
95ENDPROC(sun4v_ncs_sethead_marker)
diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c
new file mode 100644
index 000000000000..8566be832f51
--- /dev/null
+++ b/drivers/crypto/n2_core.c
@@ -0,0 +1,2083 @@
1/* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
2 *
3 * Copyright (C) 2010 David S. Miller <davem@davemloft.net>
4 */
5
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/kernel.h>
9#include <linux/module.h>
10#include <linux/of.h>
11#include <linux/of_device.h>
12#include <linux/cpumask.h>
13#include <linux/slab.h>
14#include <linux/interrupt.h>
15#include <linux/crypto.h>
16#include <crypto/md5.h>
17#include <crypto/sha.h>
18#include <crypto/aes.h>
19#include <crypto/des.h>
20#include <linux/mutex.h>
21#include <linux/delay.h>
22#include <linux/sched.h>
23
24#include <crypto/internal/hash.h>
25#include <crypto/scatterwalk.h>
26#include <crypto/algapi.h>
27
28#include <asm/hypervisor.h>
29#include <asm/mdesc.h>
30
31#include "n2_core.h"
32
33#define DRV_MODULE_NAME "n2_crypto"
34#define DRV_MODULE_VERSION "0.1"
35#define DRV_MODULE_RELDATE "April 29, 2010"
36
37static char version[] __devinitdata =
38 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
39
40MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
41MODULE_DESCRIPTION("Niagara2 Crypto driver");
42MODULE_LICENSE("GPL");
43MODULE_VERSION(DRV_MODULE_VERSION);
44
45#define N2_CRA_PRIORITY 300
46
47static DEFINE_MUTEX(spu_lock);
48
49struct spu_queue {
50 cpumask_t sharing;
51 unsigned long qhandle;
52
53 spinlock_t lock;
54 u8 q_type;
55 void *q;
56 unsigned long head;
57 unsigned long tail;
58 struct list_head jobs;
59
60 unsigned long devino;
61
62 char irq_name[32];
63 unsigned int irq;
64
65 struct list_head list;
66};
67
68static struct spu_queue **cpu_to_cwq;
69static struct spu_queue **cpu_to_mau;
70
71static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)
72{
73 if (q->q_type == HV_NCS_QTYPE_MAU) {
74 off += MAU_ENTRY_SIZE;
75 if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))
76 off = 0;
77 } else {
78 off += CWQ_ENTRY_SIZE;
79 if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))
80 off = 0;
81 }
82 return off;
83}
84
85struct n2_request_common {
86 struct list_head entry;
87 unsigned int offset;
88};
89#define OFFSET_NOT_RUNNING (~(unsigned int)0)
90
91/* An async job request records the final tail value it used in
92 * n2_request_common->offset, test to see if that offset is in
93 * the range old_head, new_head, inclusive.
94 */
95static inline bool job_finished(struct spu_queue *q, unsigned int offset,
96 unsigned long old_head, unsigned long new_head)
97{
98 if (old_head <= new_head) {
99 if (offset > old_head && offset <= new_head)
100 return true;
101 } else {
102 if (offset > old_head || offset <= new_head)
103 return true;
104 }
105 return false;
106}
107
108/* When the HEAD marker is unequal to the actual HEAD, we get
109 * a virtual device INO interrupt. We should process the
110 * completed CWQ entries and adjust the HEAD marker to clear
111 * the IRQ.
112 */
113static irqreturn_t cwq_intr(int irq, void *dev_id)
114{
115 unsigned long off, new_head, hv_ret;
116 struct spu_queue *q = dev_id;
117
118 pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
119 smp_processor_id(), q->qhandle);
120
121 spin_lock(&q->lock);
122
123 hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);
124
125 pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
126 smp_processor_id(), new_head, hv_ret);
127
128 for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {
129 /* XXX ... XXX */
130 }
131
132 hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);
133 if (hv_ret == HV_EOK)
134 q->head = new_head;
135
136 spin_unlock(&q->lock);
137
138 return IRQ_HANDLED;
139}
140
141static irqreturn_t mau_intr(int irq, void *dev_id)
142{
143 struct spu_queue *q = dev_id;
144 unsigned long head, hv_ret;
145
146 spin_lock(&q->lock);
147
148 pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
149 smp_processor_id(), q->qhandle);
150
151 hv_ret = sun4v_ncs_gethead(q->qhandle, &head);
152
153 pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
154 smp_processor_id(), head, hv_ret);
155
156 sun4v_ncs_sethead_marker(q->qhandle, head);
157
158 spin_unlock(&q->lock);
159
160 return IRQ_HANDLED;
161}
162
163static void *spu_queue_next(struct spu_queue *q, void *cur)
164{
165 return q->q + spu_next_offset(q, cur - q->q);
166}
167
168static int spu_queue_num_free(struct spu_queue *q)
169{
170 unsigned long head = q->head;
171 unsigned long tail = q->tail;
172 unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);
173 unsigned long diff;
174
175 if (head > tail)
176 diff = head - tail;
177 else
178 diff = (end - tail) + head;
179
180 return (diff / CWQ_ENTRY_SIZE) - 1;
181}
182
183static void *spu_queue_alloc(struct spu_queue *q, int num_entries)
184{
185 int avail = spu_queue_num_free(q);
186
187 if (avail >= num_entries)
188 return q->q + q->tail;
189
190 return NULL;
191}
192
193static unsigned long spu_queue_submit(struct spu_queue *q, void *last)
194{
195 unsigned long hv_ret, new_tail;
196
197 new_tail = spu_next_offset(q, last - q->q);
198
199 hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);
200 if (hv_ret == HV_EOK)
201 q->tail = new_tail;
202 return hv_ret;
203}
204
205static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,
206 int enc_type, int auth_type,
207 unsigned int hash_len,
208 bool sfas, bool sob, bool eob, bool encrypt,
209 int opcode)
210{
211 u64 word = (len - 1) & CONTROL_LEN;
212
213 word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);
214 word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);
215 word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);
216 if (sfas)
217 word |= CONTROL_STORE_FINAL_AUTH_STATE;
218 if (sob)
219 word |= CONTROL_START_OF_BLOCK;
220 if (eob)
221 word |= CONTROL_END_OF_BLOCK;
222 if (encrypt)
223 word |= CONTROL_ENCRYPT;
224 if (hmac_key_len)
225 word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;
226 if (hash_len)
227 word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;
228
229 return word;
230}
231
232#if 0
233static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)
234{
235 if (this_len >= 64 ||
236 qp->head != qp->tail)
237 return true;
238 return false;
239}
240#endif
241
242struct n2_base_ctx {
243 struct list_head list;
244};
245
246static void n2_base_ctx_init(struct n2_base_ctx *ctx)
247{
248 INIT_LIST_HEAD(&ctx->list);
249}
250
251struct n2_hash_ctx {
252 struct n2_base_ctx base;
253
254 struct crypto_ahash *fallback;
255
256 /* These next three members must match the layout created by
257 * crypto_init_shash_ops_async. This allows us to properly
258 * plumb requests we can't do in hardware down to the fallback
259 * operation, providing all of the data structures and layouts
260 * expected by those paths.
261 */
262 struct ahash_request fallback_req;
263 struct shash_desc fallback_desc;
264 union {
265 struct md5_state md5;
266 struct sha1_state sha1;
267 struct sha256_state sha256;
268 } u;
269
270 unsigned char hash_key[64];
271 unsigned char keyed_zero_hash[32];
272};
273
274static int n2_hash_async_init(struct ahash_request *req)
275{
276 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
277 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
278
279 ctx->fallback_req.base.tfm = crypto_ahash_tfm(ctx->fallback);
280 ctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
281
282 return crypto_ahash_init(&ctx->fallback_req);
283}
284
285static int n2_hash_async_update(struct ahash_request *req)
286{
287 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
288 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
289
290 ctx->fallback_req.base.tfm = crypto_ahash_tfm(ctx->fallback);
291 ctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
292 ctx->fallback_req.nbytes = req->nbytes;
293 ctx->fallback_req.src = req->src;
294
295 return crypto_ahash_update(&ctx->fallback_req);
296}
297
298static int n2_hash_async_final(struct ahash_request *req)
299{
300 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
301 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
302
303 ctx->fallback_req.base.tfm = crypto_ahash_tfm(ctx->fallback);
304 ctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
305 ctx->fallback_req.result = req->result;
306
307 return crypto_ahash_final(&ctx->fallback_req);
308}
309
310static int n2_hash_async_finup(struct ahash_request *req)
311{
312 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
313 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
314
315 ctx->fallback_req.base.tfm = crypto_ahash_tfm(ctx->fallback);
316 ctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
317 ctx->fallback_req.nbytes = req->nbytes;
318 ctx->fallback_req.src = req->src;
319 ctx->fallback_req.result = req->result;
320
321 return crypto_ahash_finup(&ctx->fallback_req);
322}
323
324static int n2_hash_cra_init(struct crypto_tfm *tfm)
325{
326 const char *fallback_driver_name = tfm->__crt_alg->cra_name;
327 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
328 struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
329 struct crypto_ahash *fallback_tfm;
330 int err;
331
332 fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
333 CRYPTO_ALG_NEED_FALLBACK);
334 if (IS_ERR(fallback_tfm)) {
335 pr_warning("Fallback driver '%s' could not be loaded!\n",
336 fallback_driver_name);
337 err = PTR_ERR(fallback_tfm);
338 goto out;
339 }
340
341 ctx->fallback = fallback_tfm;
342 return 0;
343
344out:
345 return err;
346}
347
348static void n2_hash_cra_exit(struct crypto_tfm *tfm)
349{
350 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
351 struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
352
353 crypto_free_ahash(ctx->fallback);
354}
355
356static unsigned long wait_for_tail(struct spu_queue *qp)
357{
358 unsigned long head, hv_ret;
359
360 do {
361 hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);
362 if (hv_ret != HV_EOK) {
363 pr_err("Hypervisor error on gethead\n");
364 break;
365 }
366 if (head == qp->tail) {
367 qp->head = head;
368 break;
369 }
370 } while (1);
371 return hv_ret;
372}
373
374static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,
375 struct cwq_initial_entry *ent)
376{
377 unsigned long hv_ret = spu_queue_submit(qp, ent);
378
379 if (hv_ret == HV_EOK)
380 hv_ret = wait_for_tail(qp);
381
382 return hv_ret;
383}
384
385static int n2_hash_async_digest(struct ahash_request *req,
386 unsigned int auth_type, unsigned int digest_size,
387 unsigned int result_size, void *hash_loc)
388{
389 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
390 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
391 struct cwq_initial_entry *ent;
392 struct crypto_hash_walk walk;
393 struct spu_queue *qp;
394 unsigned long flags;
395 int err = -ENODEV;
396 int nbytes, cpu;
397
398 /* The total effective length of the operation may not
399 * exceed 2^16.
400 */
401 if (unlikely(req->nbytes > (1 << 16))) {
402 ctx->fallback_req.base.tfm = crypto_ahash_tfm(ctx->fallback);
403 ctx->fallback_req.base.flags =
404 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
405 ctx->fallback_req.nbytes = req->nbytes;
406 ctx->fallback_req.src = req->src;
407 ctx->fallback_req.result = req->result;
408
409 return crypto_ahash_digest(&ctx->fallback_req);
410 }
411
412 n2_base_ctx_init(&ctx->base);
413
414 nbytes = crypto_hash_walk_first(req, &walk);
415
416 cpu = get_cpu();
417 qp = cpu_to_cwq[cpu];
418 if (!qp)
419 goto out;
420
421 spin_lock_irqsave(&qp->lock, flags);
422
423 /* XXX can do better, improve this later by doing a by-hand scatterlist
424 * XXX walk, etc.
425 */
426 ent = qp->q + qp->tail;
427
428 ent->control = control_word_base(nbytes, 0, 0,
429 auth_type, digest_size,
430 false, true, false, false,
431 OPCODE_INPLACE_BIT |
432 OPCODE_AUTH_MAC);
433 ent->src_addr = __pa(walk.data);
434 ent->auth_key_addr = 0UL;
435 ent->auth_iv_addr = __pa(hash_loc);
436 ent->final_auth_state_addr = 0UL;
437 ent->enc_key_addr = 0UL;
438 ent->enc_iv_addr = 0UL;
439 ent->dest_addr = __pa(hash_loc);
440
441 nbytes = crypto_hash_walk_done(&walk, 0);
442 while (nbytes > 0) {
443 ent = spu_queue_next(qp, ent);
444
445 ent->control = (nbytes - 1);
446 ent->src_addr = __pa(walk.data);
447 ent->auth_key_addr = 0UL;
448 ent->auth_iv_addr = 0UL;
449 ent->final_auth_state_addr = 0UL;
450 ent->enc_key_addr = 0UL;
451 ent->enc_iv_addr = 0UL;
452 ent->dest_addr = 0UL;
453
454 nbytes = crypto_hash_walk_done(&walk, 0);
455 }
456 ent->control |= CONTROL_END_OF_BLOCK;
457
458 if (submit_and_wait_for_tail(qp, ent) != HV_EOK)
459 err = -EINVAL;
460 else
461 err = 0;
462
463 spin_unlock_irqrestore(&qp->lock, flags);
464
465 if (!err)
466 memcpy(req->result, hash_loc, result_size);
467out:
468 put_cpu();
469
470 return err;
471}
472
473static int n2_md5_async_digest(struct ahash_request *req)
474{
475 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
476 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
477 struct md5_state *m = &ctx->u.md5;
478
479 if (unlikely(req->nbytes == 0)) {
480 static const char md5_zero[MD5_DIGEST_SIZE] = {
481 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
482 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
483 };
484
485 memcpy(req->result, md5_zero, MD5_DIGEST_SIZE);
486 return 0;
487 }
488 m->hash[0] = cpu_to_le32(0x67452301);
489 m->hash[1] = cpu_to_le32(0xefcdab89);
490 m->hash[2] = cpu_to_le32(0x98badcfe);
491 m->hash[3] = cpu_to_le32(0x10325476);
492
493 return n2_hash_async_digest(req, AUTH_TYPE_MD5,
494 MD5_DIGEST_SIZE, MD5_DIGEST_SIZE,
495 m->hash);
496}
497
498static int n2_sha1_async_digest(struct ahash_request *req)
499{
500 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
501 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
502 struct sha1_state *s = &ctx->u.sha1;
503
504 if (unlikely(req->nbytes == 0)) {
505 static const char sha1_zero[SHA1_DIGEST_SIZE] = {
506 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
507 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
508 0x07, 0x09
509 };
510
511 memcpy(req->result, sha1_zero, SHA1_DIGEST_SIZE);
512 return 0;
513 }
514 s->state[0] = SHA1_H0;
515 s->state[1] = SHA1_H1;
516 s->state[2] = SHA1_H2;
517 s->state[3] = SHA1_H3;
518 s->state[4] = SHA1_H4;
519
520 return n2_hash_async_digest(req, AUTH_TYPE_SHA1,
521 SHA1_DIGEST_SIZE, SHA1_DIGEST_SIZE,
522 s->state);
523}
524
525static int n2_sha256_async_digest(struct ahash_request *req)
526{
527 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
528 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
529 struct sha256_state *s = &ctx->u.sha256;
530
531 if (req->nbytes == 0) {
532 static const char sha256_zero[SHA256_DIGEST_SIZE] = {
533 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
534 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
535 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
536 0x1b, 0x78, 0x52, 0xb8, 0x55
537 };
538
539 memcpy(req->result, sha256_zero, SHA256_DIGEST_SIZE);
540 return 0;
541 }
542 s->state[0] = SHA256_H0;
543 s->state[1] = SHA256_H1;
544 s->state[2] = SHA256_H2;
545 s->state[3] = SHA256_H3;
546 s->state[4] = SHA256_H4;
547 s->state[5] = SHA256_H5;
548 s->state[6] = SHA256_H6;
549 s->state[7] = SHA256_H7;
550
551 return n2_hash_async_digest(req, AUTH_TYPE_SHA256,
552 SHA256_DIGEST_SIZE, SHA256_DIGEST_SIZE,
553 s->state);
554}
555
556static int n2_sha224_async_digest(struct ahash_request *req)
557{
558 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
559 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
560 struct sha256_state *s = &ctx->u.sha256;
561
562 if (req->nbytes == 0) {
563 static const char sha224_zero[SHA224_DIGEST_SIZE] = {
564 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
565 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
566 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
567 0x2f
568 };
569
570 memcpy(req->result, sha224_zero, SHA224_DIGEST_SIZE);
571 return 0;
572 }
573 s->state[0] = SHA224_H0;
574 s->state[1] = SHA224_H1;
575 s->state[2] = SHA224_H2;
576 s->state[3] = SHA224_H3;
577 s->state[4] = SHA224_H4;
578 s->state[5] = SHA224_H5;
579 s->state[6] = SHA224_H6;
580 s->state[7] = SHA224_H7;
581
582 return n2_hash_async_digest(req, AUTH_TYPE_SHA256,
583 SHA256_DIGEST_SIZE, SHA224_DIGEST_SIZE,
584 s->state);
585}
586
587struct n2_cipher_context {
588 int key_len;
589 int enc_type;
590 union {
591 u8 aes[AES_MAX_KEY_SIZE];
592 u8 des[DES_KEY_SIZE];
593 u8 des3[3 * DES_KEY_SIZE];
594 u8 arc4[258]; /* S-box, X, Y */
595 } key;
596};
597
598#define N2_CHUNK_ARR_LEN 16
599
600struct n2_crypto_chunk {
601 struct list_head entry;
602 unsigned long iv_paddr : 44;
603 unsigned long arr_len : 20;
604 unsigned long dest_paddr;
605 unsigned long dest_final;
606 struct {
607 unsigned long src_paddr : 44;
608 unsigned long src_len : 20;
609 } arr[N2_CHUNK_ARR_LEN];
610};
611
612struct n2_request_context {
613 struct ablkcipher_walk walk;
614 struct list_head chunk_list;
615 struct n2_crypto_chunk chunk;
616 u8 temp_iv[16];
617};
618
619/* The SPU allows some level of flexibility for partial cipher blocks
620 * being specified in a descriptor.
621 *
622 * It merely requires that every descriptor's length field is at least
623 * as large as the cipher block size. This means that a cipher block
624 * can span at most 2 descriptors. However, this does not allow a
625 * partial block to span into the final descriptor as that would
626 * violate the rule (since every descriptor's length must be at lest
627 * the block size). So, for example, assuming an 8 byte block size:
628 *
629 * 0xe --> 0xa --> 0x8
630 *
631 * is a valid length sequence, whereas:
632 *
633 * 0xe --> 0xb --> 0x7
634 *
635 * is not a valid sequence.
636 */
637
638struct n2_cipher_alg {
639 struct list_head entry;
640 u8 enc_type;
641 struct crypto_alg alg;
642};
643
644static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm)
645{
646 struct crypto_alg *alg = tfm->__crt_alg;
647
648 return container_of(alg, struct n2_cipher_alg, alg);
649}
650
651struct n2_cipher_request_context {
652 struct ablkcipher_walk walk;
653};
654
655static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
656 unsigned int keylen)
657{
658 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
659 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
660 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
661
662 ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);
663
664 switch (keylen) {
665 case AES_KEYSIZE_128:
666 ctx->enc_type |= ENC_TYPE_ALG_AES128;
667 break;
668 case AES_KEYSIZE_192:
669 ctx->enc_type |= ENC_TYPE_ALG_AES192;
670 break;
671 case AES_KEYSIZE_256:
672 ctx->enc_type |= ENC_TYPE_ALG_AES256;
673 break;
674 default:
675 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
676 return -EINVAL;
677 }
678
679 ctx->key_len = keylen;
680 memcpy(ctx->key.aes, key, keylen);
681 return 0;
682}
683
684static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
685 unsigned int keylen)
686{
687 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
688 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
689 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
690 u32 tmp[DES_EXPKEY_WORDS];
691 int err;
692
693 ctx->enc_type = n2alg->enc_type;
694
695 if (keylen != DES_KEY_SIZE) {
696 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
697 return -EINVAL;
698 }
699
700 err = des_ekey(tmp, key);
701 if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
702 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
703 return -EINVAL;
704 }
705
706 ctx->key_len = keylen;
707 memcpy(ctx->key.des, key, keylen);
708 return 0;
709}
710
711static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
712 unsigned int keylen)
713{
714 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
715 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
716 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
717
718 ctx->enc_type = n2alg->enc_type;
719
720 if (keylen != (3 * DES_KEY_SIZE)) {
721 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
722 return -EINVAL;
723 }
724 ctx->key_len = keylen;
725 memcpy(ctx->key.des3, key, keylen);
726 return 0;
727}
728
729static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
730 unsigned int keylen)
731{
732 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
733 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
734 struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
735 u8 *s = ctx->key.arc4;
736 u8 *x = s + 256;
737 u8 *y = x + 1;
738 int i, j, k;
739
740 ctx->enc_type = n2alg->enc_type;
741
742 j = k = 0;
743 *x = 0;
744 *y = 0;
745 for (i = 0; i < 256; i++)
746 s[i] = i;
747 for (i = 0; i < 256; i++) {
748 u8 a = s[i];
749 j = (j + key[k] + a) & 0xff;
750 s[i] = s[j];
751 s[j] = a;
752 if (++k >= keylen)
753 k = 0;
754 }
755
756 return 0;
757}
758
759static inline int cipher_descriptor_len(int nbytes, unsigned int block_size)
760{
761 int this_len = nbytes;
762
763 this_len -= (nbytes & (block_size - 1));
764 return this_len > (1 << 16) ? (1 << 16) : this_len;
765}
766
767static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp,
768 struct spu_queue *qp, bool encrypt)
769{
770 struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
771 struct cwq_initial_entry *ent;
772 bool in_place;
773 int i;
774
775 ent = spu_queue_alloc(qp, cp->arr_len);
776 if (!ent) {
777 pr_info("queue_alloc() of %d fails\n",
778 cp->arr_len);
779 return -EBUSY;
780 }
781
782 in_place = (cp->dest_paddr == cp->arr[0].src_paddr);
783
784 ent->control = control_word_base(cp->arr[0].src_len,
785 0, ctx->enc_type, 0, 0,
786 false, true, false, encrypt,
787 OPCODE_ENCRYPT |
788 (in_place ? OPCODE_INPLACE_BIT : 0));
789 ent->src_addr = cp->arr[0].src_paddr;
790 ent->auth_key_addr = 0UL;
791 ent->auth_iv_addr = 0UL;
792 ent->final_auth_state_addr = 0UL;
793 ent->enc_key_addr = __pa(&ctx->key);
794 ent->enc_iv_addr = cp->iv_paddr;
795 ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);
796
797 for (i = 1; i < cp->arr_len; i++) {
798 ent = spu_queue_next(qp, ent);
799
800 ent->control = cp->arr[i].src_len - 1;
801 ent->src_addr = cp->arr[i].src_paddr;
802 ent->auth_key_addr = 0UL;
803 ent->auth_iv_addr = 0UL;
804 ent->final_auth_state_addr = 0UL;
805 ent->enc_key_addr = 0UL;
806 ent->enc_iv_addr = 0UL;
807 ent->dest_addr = 0UL;
808 }
809 ent->control |= CONTROL_END_OF_BLOCK;
810
811 return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;
812}
813
814static int n2_compute_chunks(struct ablkcipher_request *req)
815{
816 struct n2_request_context *rctx = ablkcipher_request_ctx(req);
817 struct ablkcipher_walk *walk = &rctx->walk;
818 struct n2_crypto_chunk *chunk;
819 unsigned long dest_prev;
820 unsigned int tot_len;
821 bool prev_in_place;
822 int err, nbytes;
823
824 ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes);
825 err = ablkcipher_walk_phys(req, walk);
826 if (err)
827 return err;
828
829 INIT_LIST_HEAD(&rctx->chunk_list);
830
831 chunk = &rctx->chunk;
832 INIT_LIST_HEAD(&chunk->entry);
833
834 chunk->iv_paddr = 0UL;
835 chunk->arr_len = 0;
836 chunk->dest_paddr = 0UL;
837
838 prev_in_place = false;
839 dest_prev = ~0UL;
840 tot_len = 0;
841
842 while ((nbytes = walk->nbytes) != 0) {
843 unsigned long dest_paddr, src_paddr;
844 bool in_place;
845 int this_len;
846
847 src_paddr = (page_to_phys(walk->src.page) +
848 walk->src.offset);
849 dest_paddr = (page_to_phys(walk->dst.page) +
850 walk->dst.offset);
851 in_place = (src_paddr == dest_paddr);
852 this_len = cipher_descriptor_len(nbytes, walk->blocksize);
853
854 if (chunk->arr_len != 0) {
855 if (in_place != prev_in_place ||
856 (!prev_in_place &&
857 dest_paddr != dest_prev) ||
858 chunk->arr_len == N2_CHUNK_ARR_LEN ||
859 tot_len + this_len > (1 << 16)) {
860 chunk->dest_final = dest_prev;
861 list_add_tail(&chunk->entry,
862 &rctx->chunk_list);
863 chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
864 if (!chunk) {
865 err = -ENOMEM;
866 break;
867 }
868 INIT_LIST_HEAD(&chunk->entry);
869 }
870 }
871 if (chunk->arr_len == 0) {
872 chunk->dest_paddr = dest_paddr;
873 tot_len = 0;
874 }
875 chunk->arr[chunk->arr_len].src_paddr = src_paddr;
876 chunk->arr[chunk->arr_len].src_len = this_len;
877 chunk->arr_len++;
878
879 dest_prev = dest_paddr + this_len;
880 prev_in_place = in_place;
881 tot_len += this_len;
882
883 err = ablkcipher_walk_done(req, walk, nbytes - this_len);
884 if (err)
885 break;
886 }
887 if (!err && chunk->arr_len != 0) {
888 chunk->dest_final = dest_prev;
889 list_add_tail(&chunk->entry, &rctx->chunk_list);
890 }
891
892 return err;
893}
894
895static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv)
896{
897 struct n2_request_context *rctx = ablkcipher_request_ctx(req);
898 struct n2_crypto_chunk *c, *tmp;
899
900 if (final_iv)
901 memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);
902
903 ablkcipher_walk_complete(&rctx->walk);
904 list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
905 list_del(&c->entry);
906 if (unlikely(c != &rctx->chunk))
907 kfree(c);
908 }
909
910}
911
912static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)
913{
914 struct n2_request_context *rctx = ablkcipher_request_ctx(req);
915 struct crypto_tfm *tfm = req->base.tfm;
916 int err = n2_compute_chunks(req);
917 struct n2_crypto_chunk *c, *tmp;
918 unsigned long flags, hv_ret;
919 struct spu_queue *qp;
920
921 if (err)
922 return err;
923
924 qp = cpu_to_cwq[get_cpu()];
925 err = -ENODEV;
926 if (!qp)
927 goto out;
928
929 spin_lock_irqsave(&qp->lock, flags);
930
931 list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
932 err = __n2_crypt_chunk(tfm, c, qp, encrypt);
933 if (err)
934 break;
935 list_del(&c->entry);
936 if (unlikely(c != &rctx->chunk))
937 kfree(c);
938 }
939 if (!err) {
940 hv_ret = wait_for_tail(qp);
941 if (hv_ret != HV_EOK)
942 err = -EINVAL;
943 }
944
945 spin_unlock_irqrestore(&qp->lock, flags);
946
947 put_cpu();
948
949out:
950 n2_chunk_complete(req, NULL);
951 return err;
952}
953
954static int n2_encrypt_ecb(struct ablkcipher_request *req)
955{
956 return n2_do_ecb(req, true);
957}
958
959static int n2_decrypt_ecb(struct ablkcipher_request *req)
960{
961 return n2_do_ecb(req, false);
962}
963
964static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)
965{
966 struct n2_request_context *rctx = ablkcipher_request_ctx(req);
967 struct crypto_tfm *tfm = req->base.tfm;
968 unsigned long flags, hv_ret, iv_paddr;
969 int err = n2_compute_chunks(req);
970 struct n2_crypto_chunk *c, *tmp;
971 struct spu_queue *qp;
972 void *final_iv_addr;
973
974 final_iv_addr = NULL;
975
976 if (err)
977 return err;
978
979 qp = cpu_to_cwq[get_cpu()];
980 err = -ENODEV;
981 if (!qp)
982 goto out;
983
984 spin_lock_irqsave(&qp->lock, flags);
985
986 if (encrypt) {
987 iv_paddr = __pa(rctx->walk.iv);
988 list_for_each_entry_safe(c, tmp, &rctx->chunk_list,
989 entry) {
990 c->iv_paddr = iv_paddr;
991 err = __n2_crypt_chunk(tfm, c, qp, true);
992 if (err)
993 break;
994 iv_paddr = c->dest_final - rctx->walk.blocksize;
995 list_del(&c->entry);
996 if (unlikely(c != &rctx->chunk))
997 kfree(c);
998 }
999 final_iv_addr = __va(iv_paddr);
1000 } else {
1001 list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,
1002 entry) {
1003 if (c == &rctx->chunk) {
1004 iv_paddr = __pa(rctx->walk.iv);
1005 } else {
1006 iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +
1007 tmp->arr[tmp->arr_len-1].src_len -
1008 rctx->walk.blocksize);
1009 }
1010 if (!final_iv_addr) {
1011 unsigned long pa;
1012
1013 pa = (c->arr[c->arr_len-1].src_paddr +
1014 c->arr[c->arr_len-1].src_len -
1015 rctx->walk.blocksize);
1016 final_iv_addr = rctx->temp_iv;
1017 memcpy(rctx->temp_iv, __va(pa),
1018 rctx->walk.blocksize);
1019 }
1020 c->iv_paddr = iv_paddr;
1021 err = __n2_crypt_chunk(tfm, c, qp, false);
1022 if (err)
1023 break;
1024 list_del(&c->entry);
1025 if (unlikely(c != &rctx->chunk))
1026 kfree(c);
1027 }
1028 }
1029 if (!err) {
1030 hv_ret = wait_for_tail(qp);
1031 if (hv_ret != HV_EOK)
1032 err = -EINVAL;
1033 }
1034
1035 spin_unlock_irqrestore(&qp->lock, flags);
1036
1037 put_cpu();
1038
1039out:
1040 n2_chunk_complete(req, err ? NULL : final_iv_addr);
1041 return err;
1042}
1043
1044static int n2_encrypt_chaining(struct ablkcipher_request *req)
1045{
1046 return n2_do_chaining(req, true);
1047}
1048
1049static int n2_decrypt_chaining(struct ablkcipher_request *req)
1050{
1051 return n2_do_chaining(req, false);
1052}
1053
1054struct n2_cipher_tmpl {
1055 const char *name;
1056 const char *drv_name;
1057 u8 block_size;
1058 u8 enc_type;
1059 struct ablkcipher_alg ablkcipher;
1060};
1061
1062static const struct n2_cipher_tmpl cipher_tmpls[] = {
1063 /* ARC4: only ECB is supported (chaining bits ignored) */
1064 { .name = "ecb(arc4)",
1065 .drv_name = "ecb-arc4",
1066 .block_size = 1,
1067 .enc_type = (ENC_TYPE_ALG_RC4_STREAM |
1068 ENC_TYPE_CHAINING_ECB),
1069 .ablkcipher = {
1070 .min_keysize = 1,
1071 .max_keysize = 256,
1072 .setkey = n2_arc4_setkey,
1073 .encrypt = n2_encrypt_ecb,
1074 .decrypt = n2_decrypt_ecb,
1075 },
1076 },
1077
1078 /* DES: ECB CBC and CFB are supported */
1079 { .name = "ecb(des)",
1080 .drv_name = "ecb-des",
1081 .block_size = DES_BLOCK_SIZE,
1082 .enc_type = (ENC_TYPE_ALG_DES |
1083 ENC_TYPE_CHAINING_ECB),
1084 .ablkcipher = {
1085 .min_keysize = DES_KEY_SIZE,
1086 .max_keysize = DES_KEY_SIZE,
1087 .setkey = n2_des_setkey,
1088 .encrypt = n2_encrypt_ecb,
1089 .decrypt = n2_decrypt_ecb,
1090 },
1091 },
1092 { .name = "cbc(des)",
1093 .drv_name = "cbc-des",
1094 .block_size = DES_BLOCK_SIZE,
1095 .enc_type = (ENC_TYPE_ALG_DES |
1096 ENC_TYPE_CHAINING_CBC),
1097 .ablkcipher = {
1098 .ivsize = DES_BLOCK_SIZE,
1099 .min_keysize = DES_KEY_SIZE,
1100 .max_keysize = DES_KEY_SIZE,
1101 .setkey = n2_des_setkey,
1102 .encrypt = n2_encrypt_chaining,
1103 .decrypt = n2_decrypt_chaining,
1104 },
1105 },
1106 { .name = "cfb(des)",
1107 .drv_name = "cfb-des",
1108 .block_size = DES_BLOCK_SIZE,
1109 .enc_type = (ENC_TYPE_ALG_DES |
1110 ENC_TYPE_CHAINING_CFB),
1111 .ablkcipher = {
1112 .min_keysize = DES_KEY_SIZE,
1113 .max_keysize = DES_KEY_SIZE,
1114 .setkey = n2_des_setkey,
1115 .encrypt = n2_encrypt_chaining,
1116 .decrypt = n2_decrypt_chaining,
1117 },
1118 },
1119
1120 /* 3DES: ECB CBC and CFB are supported */
1121 { .name = "ecb(des3_ede)",
1122 .drv_name = "ecb-3des",
1123 .block_size = DES_BLOCK_SIZE,
1124 .enc_type = (ENC_TYPE_ALG_3DES |
1125 ENC_TYPE_CHAINING_ECB),
1126 .ablkcipher = {
1127 .min_keysize = 3 * DES_KEY_SIZE,
1128 .max_keysize = 3 * DES_KEY_SIZE,
1129 .setkey = n2_3des_setkey,
1130 .encrypt = n2_encrypt_ecb,
1131 .decrypt = n2_decrypt_ecb,
1132 },
1133 },
1134 { .name = "cbc(des3_ede)",
1135 .drv_name = "cbc-3des",
1136 .block_size = DES_BLOCK_SIZE,
1137 .enc_type = (ENC_TYPE_ALG_3DES |
1138 ENC_TYPE_CHAINING_CBC),
1139 .ablkcipher = {
1140 .ivsize = DES_BLOCK_SIZE,
1141 .min_keysize = 3 * DES_KEY_SIZE,
1142 .max_keysize = 3 * DES_KEY_SIZE,
1143 .setkey = n2_3des_setkey,
1144 .encrypt = n2_encrypt_chaining,
1145 .decrypt = n2_decrypt_chaining,
1146 },
1147 },
1148 { .name = "cfb(des3_ede)",
1149 .drv_name = "cfb-3des",
1150 .block_size = DES_BLOCK_SIZE,
1151 .enc_type = (ENC_TYPE_ALG_3DES |
1152 ENC_TYPE_CHAINING_CFB),
1153 .ablkcipher = {
1154 .min_keysize = 3 * DES_KEY_SIZE,
1155 .max_keysize = 3 * DES_KEY_SIZE,
1156 .setkey = n2_3des_setkey,
1157 .encrypt = n2_encrypt_chaining,
1158 .decrypt = n2_decrypt_chaining,
1159 },
1160 },
1161 /* AES: ECB CBC and CTR are supported */
1162 { .name = "ecb(aes)",
1163 .drv_name = "ecb-aes",
1164 .block_size = AES_BLOCK_SIZE,
1165 .enc_type = (ENC_TYPE_ALG_AES128 |
1166 ENC_TYPE_CHAINING_ECB),
1167 .ablkcipher = {
1168 .min_keysize = AES_MIN_KEY_SIZE,
1169 .max_keysize = AES_MAX_KEY_SIZE,
1170 .setkey = n2_aes_setkey,
1171 .encrypt = n2_encrypt_ecb,
1172 .decrypt = n2_decrypt_ecb,
1173 },
1174 },
1175 { .name = "cbc(aes)",
1176 .drv_name = "cbc-aes",
1177 .block_size = AES_BLOCK_SIZE,
1178 .enc_type = (ENC_TYPE_ALG_AES128 |
1179 ENC_TYPE_CHAINING_CBC),
1180 .ablkcipher = {
1181 .ivsize = AES_BLOCK_SIZE,
1182 .min_keysize = AES_MIN_KEY_SIZE,
1183 .max_keysize = AES_MAX_KEY_SIZE,
1184 .setkey = n2_aes_setkey,
1185 .encrypt = n2_encrypt_chaining,
1186 .decrypt = n2_decrypt_chaining,
1187 },
1188 },
1189 { .name = "ctr(aes)",
1190 .drv_name = "ctr-aes",
1191 .block_size = AES_BLOCK_SIZE,
1192 .enc_type = (ENC_TYPE_ALG_AES128 |
1193 ENC_TYPE_CHAINING_COUNTER),
1194 .ablkcipher = {
1195 .ivsize = AES_BLOCK_SIZE,
1196 .min_keysize = AES_MIN_KEY_SIZE,
1197 .max_keysize = AES_MAX_KEY_SIZE,
1198 .setkey = n2_aes_setkey,
1199 .encrypt = n2_encrypt_chaining,
1200 .decrypt = n2_encrypt_chaining,
1201 },
1202 },
1203
1204};
1205#define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
1206
1207static LIST_HEAD(cipher_algs);
1208
1209struct n2_hash_tmpl {
1210 const char *name;
1211 int (*digest)(struct ahash_request *req);
1212 u8 digest_size;
1213 u8 block_size;
1214};
1215static const struct n2_hash_tmpl hash_tmpls[] = {
1216 { .name = "md5",
1217 .digest = n2_md5_async_digest,
1218 .digest_size = MD5_DIGEST_SIZE,
1219 .block_size = MD5_HMAC_BLOCK_SIZE },
1220 { .name = "sha1",
1221 .digest = n2_sha1_async_digest,
1222 .digest_size = SHA1_DIGEST_SIZE,
1223 .block_size = SHA1_BLOCK_SIZE },
1224 { .name = "sha256",
1225 .digest = n2_sha256_async_digest,
1226 .digest_size = SHA256_DIGEST_SIZE,
1227 .block_size = SHA256_BLOCK_SIZE },
1228 { .name = "sha224",
1229 .digest = n2_sha224_async_digest,
1230 .digest_size = SHA224_DIGEST_SIZE,
1231 .block_size = SHA224_BLOCK_SIZE },
1232};
1233#define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
1234
1235struct n2_ahash_alg {
1236 struct list_head entry;
1237 struct ahash_alg alg;
1238};
1239static LIST_HEAD(ahash_algs);
1240
1241static int algs_registered;
1242
1243static void __n2_unregister_algs(void)
1244{
1245 struct n2_cipher_alg *cipher, *cipher_tmp;
1246 struct n2_ahash_alg *alg, *alg_tmp;
1247
1248 list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) {
1249 crypto_unregister_alg(&cipher->alg);
1250 list_del(&cipher->entry);
1251 kfree(cipher);
1252 }
1253 list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {
1254 crypto_unregister_ahash(&alg->alg);
1255 list_del(&alg->entry);
1256 kfree(alg);
1257 }
1258}
1259
1260static int n2_cipher_cra_init(struct crypto_tfm *tfm)
1261{
1262 tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context);
1263 return 0;
1264}
1265
1266static int __devinit __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl)
1267{
1268 struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1269 struct crypto_alg *alg;
1270 int err;
1271
1272 if (!p)
1273 return -ENOMEM;
1274
1275 alg = &p->alg;
1276
1277 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1278 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);
1279 alg->cra_priority = N2_CRA_PRIORITY;
1280 alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
1281 alg->cra_blocksize = tmpl->block_size;
1282 p->enc_type = tmpl->enc_type;
1283 alg->cra_ctxsize = sizeof(struct n2_cipher_context);
1284 alg->cra_type = &crypto_ablkcipher_type;
1285 alg->cra_u.ablkcipher = tmpl->ablkcipher;
1286 alg->cra_init = n2_cipher_cra_init;
1287 alg->cra_module = THIS_MODULE;
1288
1289 list_add(&p->entry, &cipher_algs);
1290 err = crypto_register_alg(alg);
1291 if (err) {
1292 list_del(&p->entry);
1293 kfree(p);
1294 }
1295 return err;
1296}
1297
1298static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
1299{
1300 struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1301 struct hash_alg_common *halg;
1302 struct crypto_alg *base;
1303 struct ahash_alg *ahash;
1304 int err;
1305
1306 if (!p)
1307 return -ENOMEM;
1308
1309 ahash = &p->alg;
1310 ahash->init = n2_hash_async_init;
1311 ahash->update = n2_hash_async_update;
1312 ahash->final = n2_hash_async_final;
1313 ahash->finup = n2_hash_async_finup;
1314 ahash->digest = tmpl->digest;
1315
1316 halg = &ahash->halg;
1317 halg->digestsize = tmpl->digest_size;
1318
1319 base = &halg->base;
1320 snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1321 snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
1322 base->cra_priority = N2_CRA_PRIORITY;
1323 base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_NEED_FALLBACK;
1324 base->cra_blocksize = tmpl->block_size;
1325 base->cra_ctxsize = sizeof(struct n2_hash_ctx);
1326 base->cra_module = THIS_MODULE;
1327 base->cra_init = n2_hash_cra_init;
1328 base->cra_exit = n2_hash_cra_exit;
1329
1330 list_add(&p->entry, &ahash_algs);
1331 err = crypto_register_ahash(ahash);
1332 if (err) {
1333 list_del(&p->entry);
1334 kfree(p);
1335 }
1336 return err;
1337}
1338
1339static int __devinit n2_register_algs(void)
1340{
1341 int i, err = 0;
1342
1343 mutex_lock(&spu_lock);
1344 if (algs_registered++)
1345 goto out;
1346
1347 for (i = 0; i < NUM_HASH_TMPLS; i++) {
1348 err = __n2_register_one_ahash(&hash_tmpls[i]);
1349 if (err) {
1350 __n2_unregister_algs();
1351 goto out;
1352 }
1353 }
1354 for (i = 0; i < NUM_CIPHER_TMPLS; i++) {
1355 err = __n2_register_one_cipher(&cipher_tmpls[i]);
1356 if (err) {
1357 __n2_unregister_algs();
1358 goto out;
1359 }
1360 }
1361
1362out:
1363 mutex_unlock(&spu_lock);
1364 return err;
1365}
1366
1367static void __exit n2_unregister_algs(void)
1368{
1369 mutex_lock(&spu_lock);
1370 if (!--algs_registered)
1371 __n2_unregister_algs();
1372 mutex_unlock(&spu_lock);
1373}
1374
1375/* To map CWQ queues to interrupt sources, the hypervisor API provides
1376 * a devino. This isn't very useful to us because all of the
1377 * interrupts listed in the of_device node have been translated to
1378 * Linux virtual IRQ cookie numbers.
1379 *
1380 * So we have to back-translate, going through the 'intr' and 'ino'
1381 * property tables of the n2cp MDESC node, matching it with the OF
1382 * 'interrupts' property entries, in order to to figure out which
1383 * devino goes to which already-translated IRQ.
1384 */
1385static int find_devino_index(struct of_device *dev, struct spu_mdesc_info *ip,
1386 unsigned long dev_ino)
1387{
1388 const unsigned int *dev_intrs;
1389 unsigned int intr;
1390 int i;
1391
1392 for (i = 0; i < ip->num_intrs; i++) {
1393 if (ip->ino_table[i].ino == dev_ino)
1394 break;
1395 }
1396 if (i == ip->num_intrs)
1397 return -ENODEV;
1398
1399 intr = ip->ino_table[i].intr;
1400
1401 dev_intrs = of_get_property(dev->node, "interrupts", NULL);
1402 if (!dev_intrs)
1403 return -ENODEV;
1404
1405 for (i = 0; i < dev->num_irqs; i++) {
1406 if (dev_intrs[i] == intr)
1407 return i;
1408 }
1409
1410 return -ENODEV;
1411}
1412
1413static int spu_map_ino(struct of_device *dev, struct spu_mdesc_info *ip,
1414 const char *irq_name, struct spu_queue *p,
1415 irq_handler_t handler)
1416{
1417 unsigned long herr;
1418 int index;
1419
1420 herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);
1421 if (herr)
1422 return -EINVAL;
1423
1424 index = find_devino_index(dev, ip, p->devino);
1425 if (index < 0)
1426 return index;
1427
1428 p->irq = dev->irqs[index];
1429
1430 sprintf(p->irq_name, "%s-%d", irq_name, index);
1431
1432 return request_irq(p->irq, handler, IRQF_SAMPLE_RANDOM,
1433 p->irq_name, p);
1434}
1435
1436static struct kmem_cache *queue_cache[2];
1437
1438static void *new_queue(unsigned long q_type)
1439{
1440 return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);
1441}
1442
1443static void free_queue(void *p, unsigned long q_type)
1444{
1445 return kmem_cache_free(queue_cache[q_type - 1], p);
1446}
1447
1448static int queue_cache_init(void)
1449{
1450 if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1451 queue_cache[HV_NCS_QTYPE_MAU - 1] =
1452 kmem_cache_create("cwq_queue",
1453 (MAU_NUM_ENTRIES *
1454 MAU_ENTRY_SIZE),
1455 MAU_ENTRY_SIZE, 0, NULL);
1456 if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1457 return -ENOMEM;
1458
1459 if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])
1460 queue_cache[HV_NCS_QTYPE_CWQ - 1] =
1461 kmem_cache_create("cwq_queue",
1462 (CWQ_NUM_ENTRIES *
1463 CWQ_ENTRY_SIZE),
1464 CWQ_ENTRY_SIZE, 0, NULL);
1465 if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {
1466 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1467 return -ENOMEM;
1468 }
1469 return 0;
1470}
1471
1472static void queue_cache_destroy(void)
1473{
1474 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1475 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);
1476}
1477
1478static int spu_queue_register(struct spu_queue *p, unsigned long q_type)
1479{
1480 cpumask_var_t old_allowed;
1481 unsigned long hv_ret;
1482
1483 if (cpumask_empty(&p->sharing))
1484 return -EINVAL;
1485
1486 if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))
1487 return -ENOMEM;
1488
1489 cpumask_copy(old_allowed, &current->cpus_allowed);
1490
1491 set_cpus_allowed_ptr(current, &p->sharing);
1492
1493 hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),
1494 CWQ_NUM_ENTRIES, &p->qhandle);
1495 if (!hv_ret)
1496 sun4v_ncs_sethead_marker(p->qhandle, 0);
1497
1498 set_cpus_allowed_ptr(current, old_allowed);
1499
1500 free_cpumask_var(old_allowed);
1501
1502 return (hv_ret ? -EINVAL : 0);
1503}
1504
1505static int spu_queue_setup(struct spu_queue *p)
1506{
1507 int err;
1508
1509 p->q = new_queue(p->q_type);
1510 if (!p->q)
1511 return -ENOMEM;
1512
1513 err = spu_queue_register(p, p->q_type);
1514 if (err) {
1515 free_queue(p->q, p->q_type);
1516 p->q = NULL;
1517 }
1518
1519 return err;
1520}
1521
1522static void spu_queue_destroy(struct spu_queue *p)
1523{
1524 unsigned long hv_ret;
1525
1526 if (!p->q)
1527 return;
1528
1529 hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);
1530
1531 if (!hv_ret)
1532 free_queue(p->q, p->q_type);
1533}
1534
1535static void spu_list_destroy(struct list_head *list)
1536{
1537 struct spu_queue *p, *n;
1538
1539 list_for_each_entry_safe(p, n, list, list) {
1540 int i;
1541
1542 for (i = 0; i < NR_CPUS; i++) {
1543 if (cpu_to_cwq[i] == p)
1544 cpu_to_cwq[i] = NULL;
1545 }
1546
1547 if (p->irq) {
1548 free_irq(p->irq, p);
1549 p->irq = 0;
1550 }
1551 spu_queue_destroy(p);
1552 list_del(&p->list);
1553 kfree(p);
1554 }
1555}
1556
1557/* Walk the backward arcs of a CWQ 'exec-unit' node,
1558 * gathering cpu membership information.
1559 */
1560static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
1561 struct of_device *dev,
1562 u64 node, struct spu_queue *p,
1563 struct spu_queue **table)
1564{
1565 u64 arc;
1566
1567 mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {
1568 u64 tgt = mdesc_arc_target(mdesc, arc);
1569 const char *name = mdesc_node_name(mdesc, tgt);
1570 const u64 *id;
1571
1572 if (strcmp(name, "cpu"))
1573 continue;
1574 id = mdesc_get_property(mdesc, tgt, "id", NULL);
1575 if (table[*id] != NULL) {
1576 dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",
1577 dev->node->full_name);
1578 return -EINVAL;
1579 }
1580 cpu_set(*id, p->sharing);
1581 table[*id] = p;
1582 }
1583 return 0;
1584}
1585
1586/* Process an 'exec-unit' MDESC node of type 'cwq'. */
1587static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
1588 struct of_device *dev, struct mdesc_handle *mdesc,
1589 u64 node, const char *iname, unsigned long q_type,
1590 irq_handler_t handler, struct spu_queue **table)
1591{
1592 struct spu_queue *p;
1593 int err;
1594
1595 p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
1596 if (!p) {
1597 dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",
1598 dev->node->full_name);
1599 return -ENOMEM;
1600 }
1601
1602 cpus_clear(p->sharing);
1603 spin_lock_init(&p->lock);
1604 p->q_type = q_type;
1605 INIT_LIST_HEAD(&p->jobs);
1606 list_add(&p->list, list);
1607
1608 err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);
1609 if (err)
1610 return err;
1611
1612 err = spu_queue_setup(p);
1613 if (err)
1614 return err;
1615
1616 return spu_map_ino(dev, ip, iname, p, handler);
1617}
1618
1619static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct of_device *dev,
1620 struct spu_mdesc_info *ip, struct list_head *list,
1621 const char *exec_name, unsigned long q_type,
1622 irq_handler_t handler, struct spu_queue **table)
1623{
1624 int err = 0;
1625 u64 node;
1626
1627 mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {
1628 const char *type;
1629
1630 type = mdesc_get_property(mdesc, node, "type", NULL);
1631 if (!type || strcmp(type, exec_name))
1632 continue;
1633
1634 err = handle_exec_unit(ip, list, dev, mdesc, node,
1635 exec_name, q_type, handler, table);
1636 if (err) {
1637 spu_list_destroy(list);
1638 break;
1639 }
1640 }
1641
1642 return err;
1643}
1644
1645static int __devinit get_irq_props(struct mdesc_handle *mdesc, u64 node,
1646 struct spu_mdesc_info *ip)
1647{
1648 const u64 *intr, *ino;
1649 int intr_len, ino_len;
1650 int i;
1651
1652 intr = mdesc_get_property(mdesc, node, "intr", &intr_len);
1653 if (!intr)
1654 return -ENODEV;
1655
1656 ino = mdesc_get_property(mdesc, node, "ino", &ino_len);
1657 if (!intr)
1658 return -ENODEV;
1659
1660 if (intr_len != ino_len)
1661 return -EINVAL;
1662
1663 ip->num_intrs = intr_len / sizeof(u64);
1664 ip->ino_table = kzalloc((sizeof(struct ino_blob) *
1665 ip->num_intrs),
1666 GFP_KERNEL);
1667 if (!ip->ino_table)
1668 return -ENOMEM;
1669
1670 for (i = 0; i < ip->num_intrs; i++) {
1671 struct ino_blob *b = &ip->ino_table[i];
1672 b->intr = intr[i];
1673 b->ino = ino[i];
1674 }
1675
1676 return 0;
1677}
1678
1679static int __devinit grab_mdesc_irq_props(struct mdesc_handle *mdesc,
1680 struct of_device *dev,
1681 struct spu_mdesc_info *ip,
1682 const char *node_name)
1683{
1684 const unsigned int *reg;
1685 u64 node;
1686
1687 reg = of_get_property(dev->node, "reg", NULL);
1688 if (!reg)
1689 return -ENODEV;
1690
1691 mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {
1692 const char *name;
1693 const u64 *chdl;
1694
1695 name = mdesc_get_property(mdesc, node, "name", NULL);
1696 if (!name || strcmp(name, node_name))
1697 continue;
1698 chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);
1699 if (!chdl || (*chdl != *reg))
1700 continue;
1701 ip->cfg_handle = *chdl;
1702 return get_irq_props(mdesc, node, ip);
1703 }
1704
1705 return -ENODEV;
1706}
1707
1708static unsigned long n2_spu_hvapi_major;
1709static unsigned long n2_spu_hvapi_minor;
1710
1711static int __devinit n2_spu_hvapi_register(void)
1712{
1713 int err;
1714
1715 n2_spu_hvapi_major = 2;
1716 n2_spu_hvapi_minor = 0;
1717
1718 err = sun4v_hvapi_register(HV_GRP_NCS,
1719 n2_spu_hvapi_major,
1720 &n2_spu_hvapi_minor);
1721
1722 if (!err)
1723 pr_info("Registered NCS HVAPI version %lu.%lu\n",
1724 n2_spu_hvapi_major,
1725 n2_spu_hvapi_minor);
1726
1727 return err;
1728}
1729
1730static void n2_spu_hvapi_unregister(void)
1731{
1732 sun4v_hvapi_unregister(HV_GRP_NCS);
1733}
1734
1735static int global_ref;
1736
1737static int __devinit grab_global_resources(void)
1738{
1739 int err = 0;
1740
1741 mutex_lock(&spu_lock);
1742
1743 if (global_ref++)
1744 goto out;
1745
1746 err = n2_spu_hvapi_register();
1747 if (err)
1748 goto out;
1749
1750 err = queue_cache_init();
1751 if (err)
1752 goto out_hvapi_release;
1753
1754 err = -ENOMEM;
1755 cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1756 GFP_KERNEL);
1757 if (!cpu_to_cwq)
1758 goto out_queue_cache_destroy;
1759
1760 cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1761 GFP_KERNEL);
1762 if (!cpu_to_mau)
1763 goto out_free_cwq_table;
1764
1765 err = 0;
1766
1767out:
1768 if (err)
1769 global_ref--;
1770 mutex_unlock(&spu_lock);
1771 return err;
1772
1773out_free_cwq_table:
1774 kfree(cpu_to_cwq);
1775 cpu_to_cwq = NULL;
1776
1777out_queue_cache_destroy:
1778 queue_cache_destroy();
1779
1780out_hvapi_release:
1781 n2_spu_hvapi_unregister();
1782 goto out;
1783}
1784
1785static void release_global_resources(void)
1786{
1787 mutex_lock(&spu_lock);
1788 if (!--global_ref) {
1789 kfree(cpu_to_cwq);
1790 cpu_to_cwq = NULL;
1791
1792 kfree(cpu_to_mau);
1793 cpu_to_mau = NULL;
1794
1795 queue_cache_destroy();
1796 n2_spu_hvapi_unregister();
1797 }
1798 mutex_unlock(&spu_lock);
1799}
1800
1801static struct n2_crypto * __devinit alloc_n2cp(void)
1802{
1803 struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);
1804
1805 if (np)
1806 INIT_LIST_HEAD(&np->cwq_list);
1807
1808 return np;
1809}
1810
1811static void free_n2cp(struct n2_crypto *np)
1812{
1813 if (np->cwq_info.ino_table) {
1814 kfree(np->cwq_info.ino_table);
1815 np->cwq_info.ino_table = NULL;
1816 }
1817
1818 kfree(np);
1819}
1820
1821static void __devinit n2_spu_driver_version(void)
1822{
1823 static int n2_spu_version_printed;
1824
1825 if (n2_spu_version_printed++ == 0)
1826 pr_info("%s", version);
1827}
1828
1829static int __devinit n2_crypto_probe(struct of_device *dev,
1830 const struct of_device_id *match)
1831{
1832 struct mdesc_handle *mdesc;
1833 const char *full_name;
1834 struct n2_crypto *np;
1835 int err;
1836
1837 n2_spu_driver_version();
1838
1839 full_name = dev->node->full_name;
1840 pr_info("Found N2CP at %s\n", full_name);
1841
1842 np = alloc_n2cp();
1843 if (!np) {
1844 dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",
1845 full_name);
1846 return -ENOMEM;
1847 }
1848
1849 err = grab_global_resources();
1850 if (err) {
1851 dev_err(&dev->dev, "%s: Unable to grab "
1852 "global resources.\n", full_name);
1853 goto out_free_n2cp;
1854 }
1855
1856 mdesc = mdesc_grab();
1857
1858 if (!mdesc) {
1859 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
1860 full_name);
1861 err = -ENODEV;
1862 goto out_free_global;
1863 }
1864 err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
1865 if (err) {
1866 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
1867 full_name);
1868 mdesc_release(mdesc);
1869 goto out_free_global;
1870 }
1871
1872 err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,
1873 "cwq", HV_NCS_QTYPE_CWQ, cwq_intr,
1874 cpu_to_cwq);
1875 mdesc_release(mdesc);
1876
1877 if (err) {
1878 dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",
1879 full_name);
1880 goto out_free_global;
1881 }
1882
1883 err = n2_register_algs();
1884 if (err) {
1885 dev_err(&dev->dev, "%s: Unable to register algorithms.\n",
1886 full_name);
1887 goto out_free_spu_list;
1888 }
1889
1890 dev_set_drvdata(&dev->dev, np);
1891
1892 return 0;
1893
1894out_free_spu_list:
1895 spu_list_destroy(&np->cwq_list);
1896
1897out_free_global:
1898 release_global_resources();
1899
1900out_free_n2cp:
1901 free_n2cp(np);
1902
1903 return err;
1904}
1905
1906static int __devexit n2_crypto_remove(struct of_device *dev)
1907{
1908 struct n2_crypto *np = dev_get_drvdata(&dev->dev);
1909
1910 n2_unregister_algs();
1911
1912 spu_list_destroy(&np->cwq_list);
1913
1914 release_global_resources();
1915
1916 free_n2cp(np);
1917
1918 return 0;
1919}
1920
1921static struct n2_mau * __devinit alloc_ncp(void)
1922{
1923 struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);
1924
1925 if (mp)
1926 INIT_LIST_HEAD(&mp->mau_list);
1927
1928 return mp;
1929}
1930
1931static void free_ncp(struct n2_mau *mp)
1932{
1933 if (mp->mau_info.ino_table) {
1934 kfree(mp->mau_info.ino_table);
1935 mp->mau_info.ino_table = NULL;
1936 }
1937
1938 kfree(mp);
1939}
1940
1941static int __devinit n2_mau_probe(struct of_device *dev,
1942 const struct of_device_id *match)
1943{
1944 struct mdesc_handle *mdesc;
1945 const char *full_name;
1946 struct n2_mau *mp;
1947 int err;
1948
1949 n2_spu_driver_version();
1950
1951 full_name = dev->node->full_name;
1952 pr_info("Found NCP at %s\n", full_name);
1953
1954 mp = alloc_ncp();
1955 if (!mp) {
1956 dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",
1957 full_name);
1958 return -ENOMEM;
1959 }
1960
1961 err = grab_global_resources();
1962 if (err) {
1963 dev_err(&dev->dev, "%s: Unable to grab "
1964 "global resources.\n", full_name);
1965 goto out_free_ncp;
1966 }
1967
1968 mdesc = mdesc_grab();
1969
1970 if (!mdesc) {
1971 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
1972 full_name);
1973 err = -ENODEV;
1974 goto out_free_global;
1975 }
1976
1977 err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
1978 if (err) {
1979 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
1980 full_name);
1981 mdesc_release(mdesc);
1982 goto out_free_global;
1983 }
1984
1985 err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,
1986 "mau", HV_NCS_QTYPE_MAU, mau_intr,
1987 cpu_to_mau);
1988 mdesc_release(mdesc);
1989
1990 if (err) {
1991 dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",
1992 full_name);
1993 goto out_free_global;
1994 }
1995
1996 dev_set_drvdata(&dev->dev, mp);
1997
1998 return 0;
1999
2000out_free_global:
2001 release_global_resources();
2002
2003out_free_ncp:
2004 free_ncp(mp);
2005
2006 return err;
2007}
2008
2009static int __devexit n2_mau_remove(struct of_device *dev)
2010{
2011 struct n2_mau *mp = dev_get_drvdata(&dev->dev);
2012
2013 spu_list_destroy(&mp->mau_list);
2014
2015 release_global_resources();
2016
2017 free_ncp(mp);
2018
2019 return 0;
2020}
2021
2022static struct of_device_id n2_crypto_match[] = {
2023 {
2024 .name = "n2cp",
2025 .compatible = "SUNW,n2-cwq",
2026 },
2027 {
2028 .name = "n2cp",
2029 .compatible = "SUNW,vf-cwq",
2030 },
2031 {},
2032};
2033
2034MODULE_DEVICE_TABLE(of, n2_crypto_match);
2035
2036static struct of_platform_driver n2_crypto_driver = {
2037 .name = "n2cp",
2038 .match_table = n2_crypto_match,
2039 .probe = n2_crypto_probe,
2040 .remove = __devexit_p(n2_crypto_remove),
2041};
2042
2043static struct of_device_id n2_mau_match[] = {
2044 {
2045 .name = "ncp",
2046 .compatible = "SUNW,n2-mau",
2047 },
2048 {
2049 .name = "ncp",
2050 .compatible = "SUNW,vf-mau",
2051 },
2052 {},
2053};
2054
2055MODULE_DEVICE_TABLE(of, n2_mau_match);
2056
2057static struct of_platform_driver n2_mau_driver = {
2058 .name = "ncp",
2059 .match_table = n2_mau_match,
2060 .probe = n2_mau_probe,
2061 .remove = __devexit_p(n2_mau_remove),
2062};
2063
2064static int __init n2_init(void)
2065{
2066 int err = of_register_driver(&n2_crypto_driver, &of_bus_type);
2067
2068 if (!err) {
2069 err = of_register_driver(&n2_mau_driver, &of_bus_type);
2070 if (err)
2071 of_unregister_driver(&n2_crypto_driver);
2072 }
2073 return err;
2074}
2075
2076static void __exit n2_exit(void)
2077{
2078 of_unregister_driver(&n2_mau_driver);
2079 of_unregister_driver(&n2_crypto_driver);
2080}
2081
2082module_init(n2_init);
2083module_exit(n2_exit);
diff --git a/drivers/crypto/n2_core.h b/drivers/crypto/n2_core.h
new file mode 100644
index 000000000000..4bcbbeae98f5
--- /dev/null
+++ b/drivers/crypto/n2_core.h
@@ -0,0 +1,231 @@
1#ifndef _N2_CORE_H
2#define _N2_CORE_H
3
4#ifndef __ASSEMBLY__
5
6struct ino_blob {
7 u64 intr;
8 u64 ino;
9};
10
11struct spu_mdesc_info {
12 u64 cfg_handle;
13 struct ino_blob *ino_table;
14 int num_intrs;
15};
16
17struct n2_crypto {
18 struct spu_mdesc_info cwq_info;
19 struct list_head cwq_list;
20};
21
22struct n2_mau {
23 struct spu_mdesc_info mau_info;
24 struct list_head mau_list;
25};
26
27#define CWQ_ENTRY_SIZE 64
28#define CWQ_NUM_ENTRIES 64
29
30#define MAU_ENTRY_SIZE 64
31#define MAU_NUM_ENTRIES 64
32
33struct cwq_initial_entry {
34 u64 control;
35 u64 src_addr;
36 u64 auth_key_addr;
37 u64 auth_iv_addr;
38 u64 final_auth_state_addr;
39 u64 enc_key_addr;
40 u64 enc_iv_addr;
41 u64 dest_addr;
42};
43
44struct cwq_ext_entry {
45 u64 len;
46 u64 src_addr;
47 u64 resv1;
48 u64 resv2;
49 u64 resv3;
50 u64 resv4;
51 u64 resv5;
52 u64 resv6;
53};
54
55struct cwq_final_entry {
56 u64 control;
57 u64 src_addr;
58 u64 resv1;
59 u64 resv2;
60 u64 resv3;
61 u64 resv4;
62 u64 resv5;
63 u64 resv6;
64};
65
66#define CONTROL_LEN 0x000000000000ffffULL
67#define CONTROL_LEN_SHIFT 0
68#define CONTROL_HMAC_KEY_LEN 0x0000000000ff0000ULL
69#define CONTROL_HMAC_KEY_LEN_SHIFT 16
70#define CONTROL_ENC_TYPE 0x00000000ff000000ULL
71#define CONTROL_ENC_TYPE_SHIFT 24
72#define ENC_TYPE_ALG_RC4_STREAM 0x00ULL
73#define ENC_TYPE_ALG_RC4_NOSTREAM 0x04ULL
74#define ENC_TYPE_ALG_DES 0x08ULL
75#define ENC_TYPE_ALG_3DES 0x0cULL
76#define ENC_TYPE_ALG_AES128 0x10ULL
77#define ENC_TYPE_ALG_AES192 0x14ULL
78#define ENC_TYPE_ALG_AES256 0x18ULL
79#define ENC_TYPE_ALG_RESERVED 0x1cULL
80#define ENC_TYPE_ALG_MASK 0x1cULL
81#define ENC_TYPE_CHAINING_ECB 0x00ULL
82#define ENC_TYPE_CHAINING_CBC 0x01ULL
83#define ENC_TYPE_CHAINING_CFB 0x02ULL
84#define ENC_TYPE_CHAINING_COUNTER 0x03ULL
85#define ENC_TYPE_CHAINING_MASK 0x03ULL
86#define CONTROL_AUTH_TYPE 0x0000001f00000000ULL
87#define CONTROL_AUTH_TYPE_SHIFT 32
88#define AUTH_TYPE_RESERVED 0x00ULL
89#define AUTH_TYPE_MD5 0x01ULL
90#define AUTH_TYPE_SHA1 0x02ULL
91#define AUTH_TYPE_SHA256 0x03ULL
92#define AUTH_TYPE_CRC32 0x04ULL
93#define AUTH_TYPE_HMAC_MD5 0x05ULL
94#define AUTH_TYPE_HMAC_SHA1 0x06ULL
95#define AUTH_TYPE_HMAC_SHA256 0x07ULL
96#define AUTH_TYPE_TCP_CHECKSUM 0x08ULL
97#define AUTH_TYPE_SSL_HMAC_MD5 0x09ULL
98#define AUTH_TYPE_SSL_HMAC_SHA1 0x0aULL
99#define AUTH_TYPE_SSL_HMAC_SHA256 0x0bULL
100#define CONTROL_STRAND 0x000000e000000000ULL
101#define CONTROL_STRAND_SHIFT 37
102#define CONTROL_HASH_LEN 0x0000ff0000000000ULL
103#define CONTROL_HASH_LEN_SHIFT 40
104#define CONTROL_INTERRUPT 0x0001000000000000ULL
105#define CONTROL_STORE_FINAL_AUTH_STATE 0x0002000000000000ULL
106#define CONTROL_RESERVED 0x001c000000000000ULL
107#define CONTROL_HV_DONE 0x0004000000000000ULL
108#define CONTROL_HV_PROTOCOL_ERROR 0x0008000000000000ULL
109#define CONTROL_HV_HARDWARE_ERROR 0x0010000000000000ULL
110#define CONTROL_END_OF_BLOCK 0x0020000000000000ULL
111#define CONTROL_START_OF_BLOCK 0x0040000000000000ULL
112#define CONTROL_ENCRYPT 0x0080000000000000ULL
113#define CONTROL_OPCODE 0xff00000000000000ULL
114#define CONTROL_OPCODE_SHIFT 56
115#define OPCODE_INPLACE_BIT 0x80ULL
116#define OPCODE_SSL_KEYBLOCK 0x10ULL
117#define OPCODE_COPY 0x20ULL
118#define OPCODE_ENCRYPT 0x40ULL
119#define OPCODE_AUTH_MAC 0x41ULL
120
121#endif /* !(__ASSEMBLY__) */
122
123/* NCS v2.0 hypervisor interfaces */
124#define HV_NCS_QTYPE_MAU 0x01
125#define HV_NCS_QTYPE_CWQ 0x02
126
127/* ncs_qconf()
128 * TRAP: HV_FAST_TRAP
129 * FUNCTION: HV_FAST_NCS_QCONF
130 * ARG0: Queue type (HV_NCS_QTYPE_{MAU,CWQ})
131 * ARG1: Real address of queue, or handle for unconfigure
132 * ARG2: Number of entries in queue, zero for unconfigure
133 * RET0: status
134 * RET1: queue handle
135 *
136 * Configure a queue in the stream processing unit.
137 *
138 * The real address given as the base must be 64-byte
139 * aligned.
140 *
141 * The queue size can range from a minimum of 2 to a maximum
142 * of 64. The queue size must be a power of two.
143 *
144 * To unconfigure a queue, specify a length of zero and place
145 * the queue handle into ARG1.
146 *
147 * On configure success the hypervisor will set the FIRST, HEAD,
148 * and TAIL registers to the address of the first entry in the
149 * queue. The LAST register will be set to point to the last
150 * entry in the queue.
151 */
152#define HV_FAST_NCS_QCONF 0x111
153
154/* ncs_qinfo()
155 * TRAP: HV_FAST_TRAP
156 * FUNCTION: HV_FAST_NCS_QINFO
157 * ARG0: Queue handle
158 * RET0: status
159 * RET1: Queue type (HV_NCS_QTYPE_{MAU,CWQ})
160 * RET2: Queue base address
161 * RET3: Number of entries
162 */
163#define HV_FAST_NCS_QINFO 0x112
164
165/* ncs_gethead()
166 * TRAP: HV_FAST_TRAP
167 * FUNCTION: HV_FAST_NCS_GETHEAD
168 * ARG0: Queue handle
169 * RET0: status
170 * RET1: queue head offset
171 */
172#define HV_FAST_NCS_GETHEAD 0x113
173
174/* ncs_gettail()
175 * TRAP: HV_FAST_TRAP
176 * FUNCTION: HV_FAST_NCS_GETTAIL
177 * ARG0: Queue handle
178 * RET0: status
179 * RET1: queue tail offset
180 */
181#define HV_FAST_NCS_GETTAIL 0x114
182
183/* ncs_settail()
184 * TRAP: HV_FAST_TRAP
185 * FUNCTION: HV_FAST_NCS_SETTAIL
186 * ARG0: Queue handle
187 * ARG1: New tail offset
188 * RET0: status
189 */
190#define HV_FAST_NCS_SETTAIL 0x115
191
192/* ncs_qhandle_to_devino()
193 * TRAP: HV_FAST_TRAP
194 * FUNCTION: HV_FAST_NCS_QHANDLE_TO_DEVINO
195 * ARG0: Queue handle
196 * RET0: status
197 * RET1: devino
198 */
199#define HV_FAST_NCS_QHANDLE_TO_DEVINO 0x116
200
201/* ncs_sethead_marker()
202 * TRAP: HV_FAST_TRAP
203 * FUNCTION: HV_FAST_NCS_SETHEAD_MARKER
204 * ARG0: Queue handle
205 * ARG1: New head offset
206 * RET0: status
207 */
208#define HV_FAST_NCS_SETHEAD_MARKER 0x117
209
210#ifndef __ASSEMBLY__
211extern unsigned long sun4v_ncs_qconf(unsigned long queue_type,
212 unsigned long queue_ra,
213 unsigned long num_entries,
214 unsigned long *qhandle);
215extern unsigned long sun4v_ncs_qinfo(unsigned long qhandle,
216 unsigned long *queue_type,
217 unsigned long *queue_ra,
218 unsigned long *num_entries);
219extern unsigned long sun4v_ncs_gethead(unsigned long qhandle,
220 unsigned long *head);
221extern unsigned long sun4v_ncs_gettail(unsigned long qhandle,
222 unsigned long *tail);
223extern unsigned long sun4v_ncs_settail(unsigned long qhandle,
224 unsigned long tail);
225extern unsigned long sun4v_ncs_qhandle_to_devino(unsigned long qhandle,
226 unsigned long *devino);
227extern unsigned long sun4v_ncs_sethead_marker(unsigned long qhandle,
228 unsigned long head);
229#endif /* !(__ASSEMBLY__) */
230
231#endif /* _N2_CORE_H */
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
new file mode 100644
index 000000000000..8b034337793f
--- /dev/null
+++ b/drivers/crypto/omap-sham.c
@@ -0,0 +1,1259 @@
1/*
2 * Cryptographic API.
3 *
4 * Support for OMAP SHA1/MD5 HW acceleration.
5 *
6 * Copyright (c) 2010 Nokia Corporation
7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as published
11 * by the Free Software Foundation.
12 *
13 * Some ideas are from old omap-sha1-md5.c driver.
14 */
15
16#define pr_fmt(fmt) "%s: " fmt, __func__
17
18#include <linux/version.h>
19#include <linux/err.h>
20#include <linux/device.h>
21#include <linux/module.h>
22#include <linux/init.h>
23#include <linux/errno.h>
24#include <linux/interrupt.h>
25#include <linux/kernel.h>
26#include <linux/clk.h>
27#include <linux/irq.h>
28#include <linux/io.h>
29#include <linux/platform_device.h>
30#include <linux/scatterlist.h>
31#include <linux/dma-mapping.h>
32#include <linux/delay.h>
33#include <linux/crypto.h>
34#include <linux/cryptohash.h>
35#include <crypto/scatterwalk.h>
36#include <crypto/algapi.h>
37#include <crypto/sha.h>
38#include <crypto/hash.h>
39#include <crypto/internal/hash.h>
40
41#include <plat/cpu.h>
42#include <plat/dma.h>
43#include <mach/irqs.h>
44
45#define SHA_REG_DIGEST(x) (0x00 + ((x) * 0x04))
46#define SHA_REG_DIN(x) (0x1C + ((x) * 0x04))
47
48#define SHA1_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE
49#define MD5_DIGEST_SIZE 16
50
51#define SHA_REG_DIGCNT 0x14
52
53#define SHA_REG_CTRL 0x18
54#define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5)
55#define SHA_REG_CTRL_CLOSE_HASH (1 << 4)
56#define SHA_REG_CTRL_ALGO_CONST (1 << 3)
57#define SHA_REG_CTRL_ALGO (1 << 2)
58#define SHA_REG_CTRL_INPUT_READY (1 << 1)
59#define SHA_REG_CTRL_OUTPUT_READY (1 << 0)
60
61#define SHA_REG_REV 0x5C
62#define SHA_REG_REV_MAJOR 0xF0
63#define SHA_REG_REV_MINOR 0x0F
64
65#define SHA_REG_MASK 0x60
66#define SHA_REG_MASK_DMA_EN (1 << 3)
67#define SHA_REG_MASK_IT_EN (1 << 2)
68#define SHA_REG_MASK_SOFTRESET (1 << 1)
69#define SHA_REG_AUTOIDLE (1 << 0)
70
71#define SHA_REG_SYSSTATUS 0x64
72#define SHA_REG_SYSSTATUS_RESETDONE (1 << 0)
73
74#define DEFAULT_TIMEOUT_INTERVAL HZ
75
76#define FLAGS_FIRST 0x0001
77#define FLAGS_FINUP 0x0002
78#define FLAGS_FINAL 0x0004
79#define FLAGS_FAST 0x0008
80#define FLAGS_SHA1 0x0010
81#define FLAGS_DMA_ACTIVE 0x0020
82#define FLAGS_OUTPUT_READY 0x0040
83#define FLAGS_CLEAN 0x0080
84#define FLAGS_INIT 0x0100
85#define FLAGS_CPU 0x0200
86#define FLAGS_HMAC 0x0400
87
88/* 3rd byte */
89#define FLAGS_BUSY 16
90
91#define OP_UPDATE 1
92#define OP_FINAL 2
93
94struct omap_sham_dev;
95
96struct omap_sham_reqctx {
97 struct omap_sham_dev *dd;
98 unsigned long flags;
99 unsigned long op;
100
101 size_t digcnt;
102 u8 *buffer;
103 size_t bufcnt;
104 size_t buflen;
105 dma_addr_t dma_addr;
106
107 /* walk state */
108 struct scatterlist *sg;
109 unsigned int offset; /* offset in current sg */
110 unsigned int total; /* total request */
111};
112
113struct omap_sham_hmac_ctx {
114 struct crypto_shash *shash;
115 u8 ipad[SHA1_MD5_BLOCK_SIZE];
116 u8 opad[SHA1_MD5_BLOCK_SIZE];
117};
118
119struct omap_sham_ctx {
120 struct omap_sham_dev *dd;
121
122 unsigned long flags;
123
124 /* fallback stuff */
125 struct crypto_shash *fallback;
126
127 struct omap_sham_hmac_ctx base[0];
128};
129
130#define OMAP_SHAM_QUEUE_LENGTH 1
131
132struct omap_sham_dev {
133 struct list_head list;
134 unsigned long phys_base;
135 struct device *dev;
136 void __iomem *io_base;
137 int irq;
138 struct clk *iclk;
139 spinlock_t lock;
140 int dma;
141 int dma_lch;
142 struct tasklet_struct done_task;
143 struct tasklet_struct queue_task;
144
145 unsigned long flags;
146 struct crypto_queue queue;
147 struct ahash_request *req;
148};
149
150struct omap_sham_drv {
151 struct list_head dev_list;
152 spinlock_t lock;
153 unsigned long flags;
154};
155
156static struct omap_sham_drv sham = {
157 .dev_list = LIST_HEAD_INIT(sham.dev_list),
158 .lock = __SPIN_LOCK_UNLOCKED(sham.lock),
159};
160
161static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset)
162{
163 return __raw_readl(dd->io_base + offset);
164}
165
166static inline void omap_sham_write(struct omap_sham_dev *dd,
167 u32 offset, u32 value)
168{
169 __raw_writel(value, dd->io_base + offset);
170}
171
172static inline void omap_sham_write_mask(struct omap_sham_dev *dd, u32 address,
173 u32 value, u32 mask)
174{
175 u32 val;
176
177 val = omap_sham_read(dd, address);
178 val &= ~mask;
179 val |= value;
180 omap_sham_write(dd, address, val);
181}
182
183static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit)
184{
185 unsigned long timeout = jiffies + DEFAULT_TIMEOUT_INTERVAL;
186
187 while (!(omap_sham_read(dd, offset) & bit)) {
188 if (time_is_before_jiffies(timeout))
189 return -ETIMEDOUT;
190 }
191
192 return 0;
193}
194
195static void omap_sham_copy_hash(struct ahash_request *req, int out)
196{
197 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
198 u32 *hash = (u32 *)req->result;
199 int i;
200
201 if (likely(ctx->flags & FLAGS_SHA1)) {
202 /* SHA1 results are in big endian */
203 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++)
204 if (out)
205 hash[i] = be32_to_cpu(omap_sham_read(ctx->dd,
206 SHA_REG_DIGEST(i)));
207 else
208 omap_sham_write(ctx->dd, SHA_REG_DIGEST(i),
209 cpu_to_be32(hash[i]));
210 } else {
211 /* MD5 results are in little endian */
212 for (i = 0; i < MD5_DIGEST_SIZE / sizeof(u32); i++)
213 if (out)
214 hash[i] = le32_to_cpu(omap_sham_read(ctx->dd,
215 SHA_REG_DIGEST(i)));
216 else
217 omap_sham_write(ctx->dd, SHA_REG_DIGEST(i),
218 cpu_to_le32(hash[i]));
219 }
220}
221
222static int omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length,
223 int final, int dma)
224{
225 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
226 u32 val = length << 5, mask;
227
228 if (unlikely(!ctx->digcnt)) {
229
230 clk_enable(dd->iclk);
231
232 if (!(dd->flags & FLAGS_INIT)) {
233 omap_sham_write_mask(dd, SHA_REG_MASK,
234 SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET);
235
236 if (omap_sham_wait(dd, SHA_REG_SYSSTATUS,
237 SHA_REG_SYSSTATUS_RESETDONE))
238 return -ETIMEDOUT;
239
240 dd->flags |= FLAGS_INIT;
241 }
242 } else {
243 omap_sham_write(dd, SHA_REG_DIGCNT, ctx->digcnt);
244 }
245
246 omap_sham_write_mask(dd, SHA_REG_MASK,
247 SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0),
248 SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
249 /*
250 * Setting ALGO_CONST only for the first iteration
251 * and CLOSE_HASH only for the last one.
252 */
253 if (ctx->flags & FLAGS_SHA1)
254 val |= SHA_REG_CTRL_ALGO;
255 if (!ctx->digcnt)
256 val |= SHA_REG_CTRL_ALGO_CONST;
257 if (final)
258 val |= SHA_REG_CTRL_CLOSE_HASH;
259
260 mask = SHA_REG_CTRL_ALGO_CONST | SHA_REG_CTRL_CLOSE_HASH |
261 SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH;
262
263 omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask);
264
265 return 0;
266}
267
268static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
269 size_t length, int final)
270{
271 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
272 int err, count, len32;
273 const u32 *buffer = (const u32 *)buf;
274
275 dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
276 ctx->digcnt, length, final);
277
278 err = omap_sham_write_ctrl(dd, length, final, 0);
279 if (err)
280 return err;
281
282 if (omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY))
283 return -ETIMEDOUT;
284
285 ctx->digcnt += length;
286
287 if (final)
288 ctx->flags |= FLAGS_FINAL; /* catch last interrupt */
289
290 len32 = DIV_ROUND_UP(length, sizeof(u32));
291
292 for (count = 0; count < len32; count++)
293 omap_sham_write(dd, SHA_REG_DIN(count), buffer[count]);
294
295 return -EINPROGRESS;
296}
297
298static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
299 size_t length, int final)
300{
301 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
302 int err, len32;
303
304 dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
305 ctx->digcnt, length, final);
306
307 /* flush cache entries related to our page */
308 if (dma_addr == ctx->dma_addr)
309 dma_sync_single_for_device(dd->dev, dma_addr, length,
310 DMA_TO_DEVICE);
311
312 len32 = DIV_ROUND_UP(length, sizeof(u32));
313
314 omap_set_dma_transfer_params(dd->dma_lch, OMAP_DMA_DATA_TYPE_S32, len32,
315 1, OMAP_DMA_SYNC_PACKET, dd->dma, OMAP_DMA_DST_SYNC);
316
317 omap_set_dma_src_params(dd->dma_lch, 0, OMAP_DMA_AMODE_POST_INC,
318 dma_addr, 0, 0);
319
320 err = omap_sham_write_ctrl(dd, length, final, 1);
321 if (err)
322 return err;
323
324 ctx->digcnt += length;
325
326 if (final)
327 ctx->flags |= FLAGS_FINAL; /* catch last interrupt */
328
329 dd->flags |= FLAGS_DMA_ACTIVE;
330
331 omap_start_dma(dd->dma_lch);
332
333 return -EINPROGRESS;
334}
335
336static size_t omap_sham_append_buffer(struct omap_sham_reqctx *ctx,
337 const u8 *data, size_t length)
338{
339 size_t count = min(length, ctx->buflen - ctx->bufcnt);
340
341 count = min(count, ctx->total);
342 if (count <= 0)
343 return 0;
344 memcpy(ctx->buffer + ctx->bufcnt, data, count);
345 ctx->bufcnt += count;
346
347 return count;
348}
349
350static size_t omap_sham_append_sg(struct omap_sham_reqctx *ctx)
351{
352 size_t count;
353
354 while (ctx->sg) {
355 count = omap_sham_append_buffer(ctx,
356 sg_virt(ctx->sg) + ctx->offset,
357 ctx->sg->length - ctx->offset);
358 if (!count)
359 break;
360 ctx->offset += count;
361 ctx->total -= count;
362 if (ctx->offset == ctx->sg->length) {
363 ctx->sg = sg_next(ctx->sg);
364 if (ctx->sg)
365 ctx->offset = 0;
366 else
367 ctx->total = 0;
368 }
369 }
370
371 return 0;
372}
373
374static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
375{
376 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
377 unsigned int final;
378 size_t count;
379
380 if (!ctx->total)
381 return 0;
382
383 omap_sham_append_sg(ctx);
384
385 final = (ctx->flags & FLAGS_FINUP) && !ctx->total;
386
387 dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: %d, final: %d\n",
388 ctx->bufcnt, ctx->digcnt, final);
389
390 if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {
391 count = ctx->bufcnt;
392 ctx->bufcnt = 0;
393 return omap_sham_xmit_dma(dd, ctx->dma_addr, count, final);
394 }
395
396 return 0;
397}
398
399static int omap_sham_update_dma_fast(struct omap_sham_dev *dd)
400{
401 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
402 unsigned int length;
403
404 ctx->flags |= FLAGS_FAST;
405
406 length = min(ctx->total, sg_dma_len(ctx->sg));
407 ctx->total = length;
408
409 if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
410 dev_err(dd->dev, "dma_map_sg error\n");
411 return -EINVAL;
412 }
413
414 ctx->total -= length;
415
416 return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, 1);
417}
418
419static int omap_sham_update_cpu(struct omap_sham_dev *dd)
420{
421 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
422 int bufcnt;
423
424 omap_sham_append_sg(ctx);
425 bufcnt = ctx->bufcnt;
426 ctx->bufcnt = 0;
427
428 return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, 1);
429}
430
431static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
432{
433 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
434
435 omap_stop_dma(dd->dma_lch);
436 if (ctx->flags & FLAGS_FAST)
437 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
438
439 return 0;
440}
441
442static void omap_sham_cleanup(struct ahash_request *req)
443{
444 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
445 struct omap_sham_dev *dd = ctx->dd;
446 unsigned long flags;
447
448 spin_lock_irqsave(&dd->lock, flags);
449 if (ctx->flags & FLAGS_CLEAN) {
450 spin_unlock_irqrestore(&dd->lock, flags);
451 return;
452 }
453 ctx->flags |= FLAGS_CLEAN;
454 spin_unlock_irqrestore(&dd->lock, flags);
455
456 if (ctx->digcnt)
457 clk_disable(dd->iclk);
458
459 if (ctx->dma_addr)
460 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
461 DMA_TO_DEVICE);
462
463 if (ctx->buffer)
464 free_page((unsigned long)ctx->buffer);
465
466 dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt);
467}
468
469static int omap_sham_init(struct ahash_request *req)
470{
471 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
472 struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
473 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
474 struct omap_sham_dev *dd = NULL, *tmp;
475
476 spin_lock_bh(&sham.lock);
477 if (!tctx->dd) {
478 list_for_each_entry(tmp, &sham.dev_list, list) {
479 dd = tmp;
480 break;
481 }
482 tctx->dd = dd;
483 } else {
484 dd = tctx->dd;
485 }
486 spin_unlock_bh(&sham.lock);
487
488 ctx->dd = dd;
489
490 ctx->flags = 0;
491
492 ctx->flags |= FLAGS_FIRST;
493
494 dev_dbg(dd->dev, "init: digest size: %d\n",
495 crypto_ahash_digestsize(tfm));
496
497 if (crypto_ahash_digestsize(tfm) == SHA1_DIGEST_SIZE)
498 ctx->flags |= FLAGS_SHA1;
499
500 ctx->bufcnt = 0;
501 ctx->digcnt = 0;
502
503 ctx->buflen = PAGE_SIZE;
504 ctx->buffer = (void *)__get_free_page(
505 (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
506 GFP_KERNEL : GFP_ATOMIC);
507 if (!ctx->buffer)
508 return -ENOMEM;
509
510 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,
511 DMA_TO_DEVICE);
512 if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
513 dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen);
514 free_page((unsigned long)ctx->buffer);
515 return -EINVAL;
516 }
517
518 if (tctx->flags & FLAGS_HMAC) {
519 struct omap_sham_hmac_ctx *bctx = tctx->base;
520
521 memcpy(ctx->buffer, bctx->ipad, SHA1_MD5_BLOCK_SIZE);
522 ctx->bufcnt = SHA1_MD5_BLOCK_SIZE;
523 ctx->flags |= FLAGS_HMAC;
524 }
525
526 return 0;
527
528}
529
530static int omap_sham_update_req(struct omap_sham_dev *dd)
531{
532 struct ahash_request *req = dd->req;
533 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
534 int err;
535
536 dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n",
537 ctx->total, ctx->digcnt, (ctx->flags & FLAGS_FINUP) != 0);
538
539 if (ctx->flags & FLAGS_CPU)
540 err = omap_sham_update_cpu(dd);
541 else if (ctx->flags & FLAGS_FAST)
542 err = omap_sham_update_dma_fast(dd);
543 else
544 err = omap_sham_update_dma_slow(dd);
545
546 /* wait for dma completion before can take more data */
547 dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt);
548
549 return err;
550}
551
552static int omap_sham_final_req(struct omap_sham_dev *dd)
553{
554 struct ahash_request *req = dd->req;
555 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
556 int err = 0, use_dma = 1;
557
558 if (ctx->bufcnt <= 64)
559 /* faster to handle last block with cpu */
560 use_dma = 0;
561
562 if (use_dma)
563 err = omap_sham_xmit_dma(dd, ctx->dma_addr, ctx->bufcnt, 1);
564 else
565 err = omap_sham_xmit_cpu(dd, ctx->buffer, ctx->bufcnt, 1);
566
567 ctx->bufcnt = 0;
568
569 if (err != -EINPROGRESS)
570 omap_sham_cleanup(req);
571
572 dev_dbg(dd->dev, "final_req: err: %d\n", err);
573
574 return err;
575}
576
577static int omap_sham_finish_req_hmac(struct ahash_request *req)
578{
579 struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
580 struct omap_sham_hmac_ctx *bctx = tctx->base;
581 int bs = crypto_shash_blocksize(bctx->shash);
582 int ds = crypto_shash_digestsize(bctx->shash);
583 struct {
584 struct shash_desc shash;
585 char ctx[crypto_shash_descsize(bctx->shash)];
586 } desc;
587
588 desc.shash.tfm = bctx->shash;
589 desc.shash.flags = 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */
590
591 return crypto_shash_init(&desc.shash) ?:
592 crypto_shash_update(&desc.shash, bctx->opad, bs) ?:
593 crypto_shash_finup(&desc.shash, req->result, ds, req->result);
594}
595
596static void omap_sham_finish_req(struct ahash_request *req, int err)
597{
598 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
599
600 if (!err) {
601 omap_sham_copy_hash(ctx->dd->req, 1);
602 if (ctx->flags & FLAGS_HMAC)
603 err = omap_sham_finish_req_hmac(req);
604 }
605
606 if (ctx->flags & FLAGS_FINAL)
607 omap_sham_cleanup(req);
608
609 clear_bit(FLAGS_BUSY, &ctx->dd->flags);
610
611 if (req->base.complete)
612 req->base.complete(&req->base, err);
613}
614
615static int omap_sham_handle_queue(struct omap_sham_dev *dd)
616{
617 struct crypto_async_request *async_req, *backlog;
618 struct omap_sham_reqctx *ctx;
619 struct ahash_request *req, *prev_req;
620 unsigned long flags;
621 int err = 0;
622
623 if (test_and_set_bit(FLAGS_BUSY, &dd->flags))
624 return 0;
625
626 spin_lock_irqsave(&dd->lock, flags);
627 backlog = crypto_get_backlog(&dd->queue);
628 async_req = crypto_dequeue_request(&dd->queue);
629 if (!async_req)
630 clear_bit(FLAGS_BUSY, &dd->flags);
631 spin_unlock_irqrestore(&dd->lock, flags);
632
633 if (!async_req)
634 return 0;
635
636 if (backlog)
637 backlog->complete(backlog, -EINPROGRESS);
638
639 req = ahash_request_cast(async_req);
640
641 prev_req = dd->req;
642 dd->req = req;
643
644 ctx = ahash_request_ctx(req);
645
646 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
647 ctx->op, req->nbytes);
648
649 if (req != prev_req && ctx->digcnt)
650 /* request has changed - restore hash */
651 omap_sham_copy_hash(req, 0);
652
653 if (ctx->op == OP_UPDATE) {
654 err = omap_sham_update_req(dd);
655 if (err != -EINPROGRESS && (ctx->flags & FLAGS_FINUP))
656 /* no final() after finup() */
657 err = omap_sham_final_req(dd);
658 } else if (ctx->op == OP_FINAL) {
659 err = omap_sham_final_req(dd);
660 }
661
662 if (err != -EINPROGRESS) {
663 /* done_task will not finish it, so do it here */
664 omap_sham_finish_req(req, err);
665 tasklet_schedule(&dd->queue_task);
666 }
667
668 dev_dbg(dd->dev, "exit, err: %d\n", err);
669
670 return err;
671}
672
673static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
674{
675 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
676 struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
677 struct omap_sham_dev *dd = tctx->dd;
678 unsigned long flags;
679 int err;
680
681 ctx->op = op;
682
683 spin_lock_irqsave(&dd->lock, flags);
684 err = ahash_enqueue_request(&dd->queue, req);
685 spin_unlock_irqrestore(&dd->lock, flags);
686
687 omap_sham_handle_queue(dd);
688
689 return err;
690}
691
692static int omap_sham_update(struct ahash_request *req)
693{
694 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
695
696 if (!req->nbytes)
697 return 0;
698
699 ctx->total = req->nbytes;
700 ctx->sg = req->src;
701 ctx->offset = 0;
702
703 if (ctx->flags & FLAGS_FINUP) {
704 if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 9) {
705 /*
706 * OMAP HW accel works only with buffers >= 9
707 * will switch to bypass in final()
708 * final has the same request and data
709 */
710 omap_sham_append_sg(ctx);
711 return 0;
712 } else if (ctx->bufcnt + ctx->total <= 64) {
713 ctx->flags |= FLAGS_CPU;
714 } else if (!ctx->bufcnt && sg_is_last(ctx->sg)) {
715 /* may be can use faster functions */
716 int aligned = IS_ALIGNED((u32)ctx->sg->offset,
717 sizeof(u32));
718
719 if (aligned && (ctx->flags & FLAGS_FIRST))
720 /* digest: first and final */
721 ctx->flags |= FLAGS_FAST;
722
723 ctx->flags &= ~FLAGS_FIRST;
724 }
725 } else if (ctx->bufcnt + ctx->total <= ctx->buflen) {
726 /* if not finaup -> not fast */
727 omap_sham_append_sg(ctx);
728 return 0;
729 }
730
731 return omap_sham_enqueue(req, OP_UPDATE);
732}
733
734static int omap_sham_shash_digest(struct crypto_shash *shash, u32 flags,
735 const u8 *data, unsigned int len, u8 *out)
736{
737 struct {
738 struct shash_desc shash;
739 char ctx[crypto_shash_descsize(shash)];
740 } desc;
741
742 desc.shash.tfm = shash;
743 desc.shash.flags = flags & CRYPTO_TFM_REQ_MAY_SLEEP;
744
745 return crypto_shash_digest(&desc.shash, data, len, out);
746}
747
748static int omap_sham_final_shash(struct ahash_request *req)
749{
750 struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
751 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
752
753 return omap_sham_shash_digest(tctx->fallback, req->base.flags,
754 ctx->buffer, ctx->bufcnt, req->result);
755}
756
757static int omap_sham_final(struct ahash_request *req)
758{
759 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
760 int err = 0;
761
762 ctx->flags |= FLAGS_FINUP;
763
764 /* OMAP HW accel works only with buffers >= 9 */
765 /* HMAC is always >= 9 because of ipad */
766 if ((ctx->digcnt + ctx->bufcnt) < 9)
767 err = omap_sham_final_shash(req);
768 else if (ctx->bufcnt)
769 return omap_sham_enqueue(req, OP_FINAL);
770
771 omap_sham_cleanup(req);
772
773 return err;
774}
775
776static int omap_sham_finup(struct ahash_request *req)
777{
778 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
779 int err1, err2;
780
781 ctx->flags |= FLAGS_FINUP;
782
783 err1 = omap_sham_update(req);
784 if (err1 == -EINPROGRESS)
785 return err1;
786 /*
787 * final() has to be always called to cleanup resources
788 * even if udpate() failed, except EINPROGRESS
789 */
790 err2 = omap_sham_final(req);
791
792 return err1 ?: err2;
793}
794
795static int omap_sham_digest(struct ahash_request *req)
796{
797 return omap_sham_init(req) ?: omap_sham_finup(req);
798}
799
800static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
801 unsigned int keylen)
802{
803 struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
804 struct omap_sham_hmac_ctx *bctx = tctx->base;
805 int bs = crypto_shash_blocksize(bctx->shash);
806 int ds = crypto_shash_digestsize(bctx->shash);
807 int err, i;
808 err = crypto_shash_setkey(tctx->fallback, key, keylen);
809 if (err)
810 return err;
811
812 if (keylen > bs) {
813 err = omap_sham_shash_digest(bctx->shash,
814 crypto_shash_get_flags(bctx->shash),
815 key, keylen, bctx->ipad);
816 if (err)
817 return err;
818 keylen = ds;
819 } else {
820 memcpy(bctx->ipad, key, keylen);
821 }
822
823 memset(bctx->ipad + keylen, 0, bs - keylen);
824 memcpy(bctx->opad, bctx->ipad, bs);
825
826 for (i = 0; i < bs; i++) {
827 bctx->ipad[i] ^= 0x36;
828 bctx->opad[i] ^= 0x5c;
829 }
830
831 return err;
832}
833
834static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
835{
836 struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
837 const char *alg_name = crypto_tfm_alg_name(tfm);
838
839 /* Allocate a fallback and abort if it failed. */
840 tctx->fallback = crypto_alloc_shash(alg_name, 0,
841 CRYPTO_ALG_NEED_FALLBACK);
842 if (IS_ERR(tctx->fallback)) {
843 pr_err("omap-sham: fallback driver '%s' "
844 "could not be loaded.\n", alg_name);
845 return PTR_ERR(tctx->fallback);
846 }
847
848 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
849 sizeof(struct omap_sham_reqctx));
850
851 if (alg_base) {
852 struct omap_sham_hmac_ctx *bctx = tctx->base;
853 tctx->flags |= FLAGS_HMAC;
854 bctx->shash = crypto_alloc_shash(alg_base, 0,
855 CRYPTO_ALG_NEED_FALLBACK);
856 if (IS_ERR(bctx->shash)) {
857 pr_err("omap-sham: base driver '%s' "
858 "could not be loaded.\n", alg_base);
859 crypto_free_shash(tctx->fallback);
860 return PTR_ERR(bctx->shash);
861 }
862
863 }
864
865 return 0;
866}
867
868static int omap_sham_cra_init(struct crypto_tfm *tfm)
869{
870 return omap_sham_cra_init_alg(tfm, NULL);
871}
872
873static int omap_sham_cra_sha1_init(struct crypto_tfm *tfm)
874{
875 return omap_sham_cra_init_alg(tfm, "sha1");
876}
877
878static int omap_sham_cra_md5_init(struct crypto_tfm *tfm)
879{
880 return omap_sham_cra_init_alg(tfm, "md5");
881}
882
883static void omap_sham_cra_exit(struct crypto_tfm *tfm)
884{
885 struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
886
887 crypto_free_shash(tctx->fallback);
888 tctx->fallback = NULL;
889
890 if (tctx->flags & FLAGS_HMAC) {
891 struct omap_sham_hmac_ctx *bctx = tctx->base;
892 crypto_free_shash(bctx->shash);
893 }
894}
895
896static struct ahash_alg algs[] = {
897{
898 .init = omap_sham_init,
899 .update = omap_sham_update,
900 .final = omap_sham_final,
901 .finup = omap_sham_finup,
902 .digest = omap_sham_digest,
903 .halg.digestsize = SHA1_DIGEST_SIZE,
904 .halg.base = {
905 .cra_name = "sha1",
906 .cra_driver_name = "omap-sha1",
907 .cra_priority = 100,
908 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
909 CRYPTO_ALG_ASYNC |
910 CRYPTO_ALG_NEED_FALLBACK,
911 .cra_blocksize = SHA1_BLOCK_SIZE,
912 .cra_ctxsize = sizeof(struct omap_sham_ctx),
913 .cra_alignmask = 0,
914 .cra_module = THIS_MODULE,
915 .cra_init = omap_sham_cra_init,
916 .cra_exit = omap_sham_cra_exit,
917 }
918},
919{
920 .init = omap_sham_init,
921 .update = omap_sham_update,
922 .final = omap_sham_final,
923 .finup = omap_sham_finup,
924 .digest = omap_sham_digest,
925 .halg.digestsize = MD5_DIGEST_SIZE,
926 .halg.base = {
927 .cra_name = "md5",
928 .cra_driver_name = "omap-md5",
929 .cra_priority = 100,
930 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
931 CRYPTO_ALG_ASYNC |
932 CRYPTO_ALG_NEED_FALLBACK,
933 .cra_blocksize = SHA1_BLOCK_SIZE,
934 .cra_ctxsize = sizeof(struct omap_sham_ctx),
935 .cra_alignmask = 0,
936 .cra_module = THIS_MODULE,
937 .cra_init = omap_sham_cra_init,
938 .cra_exit = omap_sham_cra_exit,
939 }
940},
941{
942 .init = omap_sham_init,
943 .update = omap_sham_update,
944 .final = omap_sham_final,
945 .finup = omap_sham_finup,
946 .digest = omap_sham_digest,
947 .setkey = omap_sham_setkey,
948 .halg.digestsize = SHA1_DIGEST_SIZE,
949 .halg.base = {
950 .cra_name = "hmac(sha1)",
951 .cra_driver_name = "omap-hmac-sha1",
952 .cra_priority = 100,
953 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
954 CRYPTO_ALG_ASYNC |
955 CRYPTO_ALG_NEED_FALLBACK,
956 .cra_blocksize = SHA1_BLOCK_SIZE,
957 .cra_ctxsize = sizeof(struct omap_sham_ctx) +
958 sizeof(struct omap_sham_hmac_ctx),
959 .cra_alignmask = 0,
960 .cra_module = THIS_MODULE,
961 .cra_init = omap_sham_cra_sha1_init,
962 .cra_exit = omap_sham_cra_exit,
963 }
964},
965{
966 .init = omap_sham_init,
967 .update = omap_sham_update,
968 .final = omap_sham_final,
969 .finup = omap_sham_finup,
970 .digest = omap_sham_digest,
971 .setkey = omap_sham_setkey,
972 .halg.digestsize = MD5_DIGEST_SIZE,
973 .halg.base = {
974 .cra_name = "hmac(md5)",
975 .cra_driver_name = "omap-hmac-md5",
976 .cra_priority = 100,
977 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
978 CRYPTO_ALG_ASYNC |
979 CRYPTO_ALG_NEED_FALLBACK,
980 .cra_blocksize = SHA1_BLOCK_SIZE,
981 .cra_ctxsize = sizeof(struct omap_sham_ctx) +
982 sizeof(struct omap_sham_hmac_ctx),
983 .cra_alignmask = 0,
984 .cra_module = THIS_MODULE,
985 .cra_init = omap_sham_cra_md5_init,
986 .cra_exit = omap_sham_cra_exit,
987 }
988}
989};
990
991static void omap_sham_done_task(unsigned long data)
992{
993 struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
994 struct ahash_request *req = dd->req;
995 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
996 int ready = 1;
997
998 if (ctx->flags & FLAGS_OUTPUT_READY) {
999 ctx->flags &= ~FLAGS_OUTPUT_READY;
1000 ready = 1;
1001 }
1002
1003 if (dd->flags & FLAGS_DMA_ACTIVE) {
1004 dd->flags &= ~FLAGS_DMA_ACTIVE;
1005 omap_sham_update_dma_stop(dd);
1006 omap_sham_update_dma_slow(dd);
1007 }
1008
1009 if (ready && !(dd->flags & FLAGS_DMA_ACTIVE)) {
1010 dev_dbg(dd->dev, "update done\n");
1011 /* finish curent request */
1012 omap_sham_finish_req(req, 0);
1013 /* start new request */
1014 omap_sham_handle_queue(dd);
1015 }
1016}
1017
1018static void omap_sham_queue_task(unsigned long data)
1019{
1020 struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
1021
1022 omap_sham_handle_queue(dd);
1023}
1024
1025static irqreturn_t omap_sham_irq(int irq, void *dev_id)
1026{
1027 struct omap_sham_dev *dd = dev_id;
1028 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
1029
1030 if (!ctx) {
1031 dev_err(dd->dev, "unknown interrupt.\n");
1032 return IRQ_HANDLED;
1033 }
1034
1035 if (unlikely(ctx->flags & FLAGS_FINAL))
1036 /* final -> allow device to go to power-saving mode */
1037 omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH);
1038
1039 omap_sham_write_mask(dd, SHA_REG_CTRL, SHA_REG_CTRL_OUTPUT_READY,
1040 SHA_REG_CTRL_OUTPUT_READY);
1041 omap_sham_read(dd, SHA_REG_CTRL);
1042
1043 ctx->flags |= FLAGS_OUTPUT_READY;
1044 tasklet_schedule(&dd->done_task);
1045
1046 return IRQ_HANDLED;
1047}
1048
1049static void omap_sham_dma_callback(int lch, u16 ch_status, void *data)
1050{
1051 struct omap_sham_dev *dd = data;
1052
1053 if (likely(lch == dd->dma_lch))
1054 tasklet_schedule(&dd->done_task);
1055}
1056
1057static int omap_sham_dma_init(struct omap_sham_dev *dd)
1058{
1059 int err;
1060
1061 dd->dma_lch = -1;
1062
1063 err = omap_request_dma(dd->dma, dev_name(dd->dev),
1064 omap_sham_dma_callback, dd, &dd->dma_lch);
1065 if (err) {
1066 dev_err(dd->dev, "Unable to request DMA channel\n");
1067 return err;
1068 }
1069 omap_set_dma_dest_params(dd->dma_lch, 0,
1070 OMAP_DMA_AMODE_CONSTANT,
1071 dd->phys_base + SHA_REG_DIN(0), 0, 16);
1072
1073 omap_set_dma_dest_burst_mode(dd->dma_lch,
1074 OMAP_DMA_DATA_BURST_16);
1075
1076 return 0;
1077}
1078
1079static void omap_sham_dma_cleanup(struct omap_sham_dev *dd)
1080{
1081 if (dd->dma_lch >= 0) {
1082 omap_free_dma(dd->dma_lch);
1083 dd->dma_lch = -1;
1084 }
1085}
1086
1087static int __devinit omap_sham_probe(struct platform_device *pdev)
1088{
1089 struct omap_sham_dev *dd;
1090 struct device *dev = &pdev->dev;
1091 struct resource *res;
1092 int err, i, j;
1093
1094 dd = kzalloc(sizeof(struct omap_sham_dev), GFP_KERNEL);
1095 if (dd == NULL) {
1096 dev_err(dev, "unable to alloc data struct.\n");
1097 err = -ENOMEM;
1098 goto data_err;
1099 }
1100 dd->dev = dev;
1101 platform_set_drvdata(pdev, dd);
1102
1103 INIT_LIST_HEAD(&dd->list);
1104 spin_lock_init(&dd->lock);
1105 tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd);
1106 tasklet_init(&dd->queue_task, omap_sham_queue_task, (unsigned long)dd);
1107 crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH);
1108
1109 dd->irq = -1;
1110
1111 /* Get the base address */
1112 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1113 if (!res) {
1114 dev_err(dev, "no MEM resource info\n");
1115 err = -ENODEV;
1116 goto res_err;
1117 }
1118 dd->phys_base = res->start;
1119
1120 /* Get the DMA */
1121 res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
1122 if (!res) {
1123 dev_err(dev, "no DMA resource info\n");
1124 err = -ENODEV;
1125 goto res_err;
1126 }
1127 dd->dma = res->start;
1128
1129 /* Get the IRQ */
1130 dd->irq = platform_get_irq(pdev, 0);
1131 if (dd->irq < 0) {
1132 dev_err(dev, "no IRQ resource info\n");
1133 err = dd->irq;
1134 goto res_err;
1135 }
1136
1137 err = request_irq(dd->irq, omap_sham_irq,
1138 IRQF_TRIGGER_LOW, dev_name(dev), dd);
1139 if (err) {
1140 dev_err(dev, "unable to request irq.\n");
1141 goto res_err;
1142 }
1143
1144 err = omap_sham_dma_init(dd);
1145 if (err)
1146 goto dma_err;
1147
1148 /* Initializing the clock */
1149 dd->iclk = clk_get(dev, "ick");
1150 if (!dd->iclk) {
1151 dev_err(dev, "clock intialization failed.\n");
1152 err = -ENODEV;
1153 goto clk_err;
1154 }
1155
1156 dd->io_base = ioremap(dd->phys_base, SZ_4K);
1157 if (!dd->io_base) {
1158 dev_err(dev, "can't ioremap\n");
1159 err = -ENOMEM;
1160 goto io_err;
1161 }
1162
1163 clk_enable(dd->iclk);
1164 dev_info(dev, "hw accel on OMAP rev %u.%u\n",
1165 (omap_sham_read(dd, SHA_REG_REV) & SHA_REG_REV_MAJOR) >> 4,
1166 omap_sham_read(dd, SHA_REG_REV) & SHA_REG_REV_MINOR);
1167 clk_disable(dd->iclk);
1168
1169 spin_lock(&sham.lock);
1170 list_add_tail(&dd->list, &sham.dev_list);
1171 spin_unlock(&sham.lock);
1172
1173 for (i = 0; i < ARRAY_SIZE(algs); i++) {
1174 err = crypto_register_ahash(&algs[i]);
1175 if (err)
1176 goto err_algs;
1177 }
1178
1179 return 0;
1180
1181err_algs:
1182 for (j = 0; j < i; j++)
1183 crypto_unregister_ahash(&algs[j]);
1184 iounmap(dd->io_base);
1185io_err:
1186 clk_put(dd->iclk);
1187clk_err:
1188 omap_sham_dma_cleanup(dd);
1189dma_err:
1190 if (dd->irq >= 0)
1191 free_irq(dd->irq, dd);
1192res_err:
1193 kfree(dd);
1194 dd = NULL;
1195data_err:
1196 dev_err(dev, "initialization failed.\n");
1197
1198 return err;
1199}
1200
1201static int __devexit omap_sham_remove(struct platform_device *pdev)
1202{
1203 static struct omap_sham_dev *dd;
1204 int i;
1205
1206 dd = platform_get_drvdata(pdev);
1207 if (!dd)
1208 return -ENODEV;
1209 spin_lock(&sham.lock);
1210 list_del(&dd->list);
1211 spin_unlock(&sham.lock);
1212 for (i = 0; i < ARRAY_SIZE(algs); i++)
1213 crypto_unregister_ahash(&algs[i]);
1214 tasklet_kill(&dd->done_task);
1215 tasklet_kill(&dd->queue_task);
1216 iounmap(dd->io_base);
1217 clk_put(dd->iclk);
1218 omap_sham_dma_cleanup(dd);
1219 if (dd->irq >= 0)
1220 free_irq(dd->irq, dd);
1221 kfree(dd);
1222 dd = NULL;
1223
1224 return 0;
1225}
1226
1227static struct platform_driver omap_sham_driver = {
1228 .probe = omap_sham_probe,
1229 .remove = omap_sham_remove,
1230 .driver = {
1231 .name = "omap-sham",
1232 .owner = THIS_MODULE,
1233 },
1234};
1235
1236static int __init omap_sham_mod_init(void)
1237{
1238 pr_info("loading %s driver\n", "omap-sham");
1239
1240 if (!cpu_class_is_omap2() ||
1241 omap_type() != OMAP2_DEVICE_TYPE_SEC) {
1242 pr_err("Unsupported cpu\n");
1243 return -ENODEV;
1244 }
1245
1246 return platform_driver_register(&omap_sham_driver);
1247}
1248
1249static void __exit omap_sham_mod_exit(void)
1250{
1251 platform_driver_unregister(&omap_sham_driver);
1252}
1253
1254module_init(omap_sham_mod_init);
1255module_exit(omap_sham_mod_exit);
1256
1257MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support.");
1258MODULE_LICENSE("GPL v2");
1259MODULE_AUTHOR("Dmitry Kasatkin");
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index dc558a097311..6a0f59d1fc5c 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver 2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 * 3 *
4 * Copyright (c) 2008 Freescale Semiconductor, Inc. 4 * Copyright (c) 2008-2010 Freescale Semiconductor, Inc.
5 * 5 *
6 * Scatterlist Crypto API glue code copied from files with the following: 6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> 7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
@@ -43,9 +43,12 @@
43#include <crypto/aes.h> 43#include <crypto/aes.h>
44#include <crypto/des.h> 44#include <crypto/des.h>
45#include <crypto/sha.h> 45#include <crypto/sha.h>
46#include <crypto/md5.h>
46#include <crypto/aead.h> 47#include <crypto/aead.h>
47#include <crypto/authenc.h> 48#include <crypto/authenc.h>
48#include <crypto/skcipher.h> 49#include <crypto/skcipher.h>
50#include <crypto/hash.h>
51#include <crypto/internal/hash.h>
49#include <crypto/scatterwalk.h> 52#include <crypto/scatterwalk.h>
50 53
51#include "talitos.h" 54#include "talitos.h"
@@ -65,6 +68,13 @@ struct talitos_ptr {
65 __be32 ptr; /* address */ 68 __be32 ptr; /* address */
66}; 69};
67 70
71static const struct talitos_ptr zero_entry = {
72 .len = 0,
73 .j_extent = 0,
74 .eptr = 0,
75 .ptr = 0
76};
77
68/* descriptor */ 78/* descriptor */
69struct talitos_desc { 79struct talitos_desc {
70 __be32 hdr; /* header high bits */ 80 __be32 hdr; /* header high bits */
@@ -146,6 +156,7 @@ struct talitos_private {
146/* .features flag */ 156/* .features flag */
147#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001 157#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
148#define TALITOS_FTR_HW_AUTH_CHECK 0x00000002 158#define TALITOS_FTR_HW_AUTH_CHECK 0x00000002
159#define TALITOS_FTR_SHA224_HWINIT 0x00000004
149 160
150static void to_talitos_ptr(struct talitos_ptr *talitos_ptr, dma_addr_t dma_addr) 161static void to_talitos_ptr(struct talitos_ptr *talitos_ptr, dma_addr_t dma_addr)
151{ 162{
@@ -692,7 +703,7 @@ static void talitos_unregister_rng(struct device *dev)
692#define TALITOS_MAX_KEY_SIZE 64 703#define TALITOS_MAX_KEY_SIZE 64
693#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ 704#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
694 705
695#define MD5_DIGEST_SIZE 16 706#define MD5_BLOCK_SIZE 64
696 707
697struct talitos_ctx { 708struct talitos_ctx {
698 struct device *dev; 709 struct device *dev;
@@ -705,6 +716,23 @@ struct talitos_ctx {
705 unsigned int authsize; 716 unsigned int authsize;
706}; 717};
707 718
719#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
720#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
721
722struct talitos_ahash_req_ctx {
723 u64 count;
724 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
725 unsigned int hw_context_size;
726 u8 buf[HASH_MAX_BLOCK_SIZE];
727 u8 bufnext[HASH_MAX_BLOCK_SIZE];
728 unsigned int swinit;
729 unsigned int first;
730 unsigned int last;
731 unsigned int to_hash_later;
732 struct scatterlist bufsl[2];
733 struct scatterlist *psrc;
734};
735
708static int aead_setauthsize(struct crypto_aead *authenc, 736static int aead_setauthsize(struct crypto_aead *authenc,
709 unsigned int authsize) 737 unsigned int authsize)
710{ 738{
@@ -821,10 +849,14 @@ static void talitos_sg_unmap(struct device *dev,
821 else 849 else
822 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); 850 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
823 851
824 if (edesc->dst_is_chained) 852 if (dst) {
825 talitos_unmap_sg_chain(dev, dst, DMA_FROM_DEVICE); 853 if (edesc->dst_is_chained)
826 else 854 talitos_unmap_sg_chain(dev, dst,
827 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); 855 DMA_FROM_DEVICE);
856 else
857 dma_unmap_sg(dev, dst, dst_nents,
858 DMA_FROM_DEVICE);
859 }
828 } else 860 } else
829 if (edesc->src_is_chained) 861 if (edesc->src_is_chained)
830 talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL); 862 talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL);
@@ -1114,12 +1146,67 @@ static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained)
1114 return sg_nents; 1146 return sg_nents;
1115} 1147}
1116 1148
1149/**
1150 * sg_copy_end_to_buffer - Copy end data from SG list to a linear buffer
1151 * @sgl: The SG list
1152 * @nents: Number of SG entries
1153 * @buf: Where to copy to
1154 * @buflen: The number of bytes to copy
1155 * @skip: The number of bytes to skip before copying.
1156 * Note: skip + buflen should equal SG total size.
1157 *
1158 * Returns the number of copied bytes.
1159 *
1160 **/
1161static size_t sg_copy_end_to_buffer(struct scatterlist *sgl, unsigned int nents,
1162 void *buf, size_t buflen, unsigned int skip)
1163{
1164 unsigned int offset = 0;
1165 unsigned int boffset = 0;
1166 struct sg_mapping_iter miter;
1167 unsigned long flags;
1168 unsigned int sg_flags = SG_MITER_ATOMIC;
1169 size_t total_buffer = buflen + skip;
1170
1171 sg_flags |= SG_MITER_FROM_SG;
1172
1173 sg_miter_start(&miter, sgl, nents, sg_flags);
1174
1175 local_irq_save(flags);
1176
1177 while (sg_miter_next(&miter) && offset < total_buffer) {
1178 unsigned int len;
1179 unsigned int ignore;
1180
1181 if ((offset + miter.length) > skip) {
1182 if (offset < skip) {
1183 /* Copy part of this segment */
1184 ignore = skip - offset;
1185 len = miter.length - ignore;
1186 memcpy(buf + boffset, miter.addr + ignore, len);
1187 } else {
1188 /* Copy all of this segment */
1189 len = miter.length;
1190 memcpy(buf + boffset, miter.addr, len);
1191 }
1192 boffset += len;
1193 }
1194 offset += miter.length;
1195 }
1196
1197 sg_miter_stop(&miter);
1198
1199 local_irq_restore(flags);
1200 return boffset;
1201}
1202
1117/* 1203/*
1118 * allocate and map the extended descriptor 1204 * allocate and map the extended descriptor
1119 */ 1205 */
1120static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, 1206static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1121 struct scatterlist *src, 1207 struct scatterlist *src,
1122 struct scatterlist *dst, 1208 struct scatterlist *dst,
1209 int hash_result,
1123 unsigned int cryptlen, 1210 unsigned int cryptlen,
1124 unsigned int authsize, 1211 unsigned int authsize,
1125 int icv_stashing, 1212 int icv_stashing,
@@ -1139,11 +1226,16 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1139 src_nents = sg_count(src, cryptlen + authsize, &src_chained); 1226 src_nents = sg_count(src, cryptlen + authsize, &src_chained);
1140 src_nents = (src_nents == 1) ? 0 : src_nents; 1227 src_nents = (src_nents == 1) ? 0 : src_nents;
1141 1228
1142 if (dst == src) { 1229 if (hash_result) {
1143 dst_nents = src_nents; 1230 dst_nents = 0;
1144 } else { 1231 } else {
1145 dst_nents = sg_count(dst, cryptlen + authsize, &dst_chained); 1232 if (dst == src) {
1146 dst_nents = (dst_nents == 1) ? 0 : dst_nents; 1233 dst_nents = src_nents;
1234 } else {
1235 dst_nents = sg_count(dst, cryptlen + authsize,
1236 &dst_chained);
1237 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1238 }
1147 } 1239 }
1148 1240
1149 /* 1241 /*
@@ -1172,8 +1264,10 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1172 edesc->src_is_chained = src_chained; 1264 edesc->src_is_chained = src_chained;
1173 edesc->dst_is_chained = dst_chained; 1265 edesc->dst_is_chained = dst_chained;
1174 edesc->dma_len = dma_len; 1266 edesc->dma_len = dma_len;
1175 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], 1267 if (dma_len)
1176 edesc->dma_len, DMA_BIDIRECTIONAL); 1268 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1269 edesc->dma_len,
1270 DMA_BIDIRECTIONAL);
1177 1271
1178 return edesc; 1272 return edesc;
1179} 1273}
@@ -1184,7 +1278,7 @@ static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq,
1184 struct crypto_aead *authenc = crypto_aead_reqtfm(areq); 1278 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1185 struct talitos_ctx *ctx = crypto_aead_ctx(authenc); 1279 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1186 1280
1187 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 1281 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
1188 areq->cryptlen, ctx->authsize, icv_stashing, 1282 areq->cryptlen, ctx->authsize, icv_stashing,
1189 areq->base.flags); 1283 areq->base.flags);
1190} 1284}
@@ -1441,8 +1535,8 @@ static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1441 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); 1535 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1442 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 1536 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1443 1537
1444 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, areq->nbytes, 1538 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
1445 0, 0, areq->base.flags); 1539 areq->nbytes, 0, 0, areq->base.flags);
1446} 1540}
1447 1541
1448static int ablkcipher_encrypt(struct ablkcipher_request *areq) 1542static int ablkcipher_encrypt(struct ablkcipher_request *areq)
@@ -1478,15 +1572,329 @@ static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1478 return common_nonsnoop(edesc, areq, NULL, ablkcipher_done); 1572 return common_nonsnoop(edesc, areq, NULL, ablkcipher_done);
1479} 1573}
1480 1574
1575static void common_nonsnoop_hash_unmap(struct device *dev,
1576 struct talitos_edesc *edesc,
1577 struct ahash_request *areq)
1578{
1579 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1580
1581 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1582
1583 /* When using hashctx-in, must unmap it. */
1584 if (edesc->desc.ptr[1].len)
1585 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1586 DMA_TO_DEVICE);
1587
1588 if (edesc->desc.ptr[2].len)
1589 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1590 DMA_TO_DEVICE);
1591
1592 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL);
1593
1594 if (edesc->dma_len)
1595 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1596 DMA_BIDIRECTIONAL);
1597
1598}
1599
1600static void ahash_done(struct device *dev,
1601 struct talitos_desc *desc, void *context,
1602 int err)
1603{
1604 struct ahash_request *areq = context;
1605 struct talitos_edesc *edesc =
1606 container_of(desc, struct talitos_edesc, desc);
1607 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1608
1609 if (!req_ctx->last && req_ctx->to_hash_later) {
1610 /* Position any partial block for next update/final/finup */
1611 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1612 }
1613 common_nonsnoop_hash_unmap(dev, edesc, areq);
1614
1615 kfree(edesc);
1616
1617 areq->base.complete(&areq->base, err);
1618}
1619
1620static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1621 struct ahash_request *areq, unsigned int length,
1622 void (*callback) (struct device *dev,
1623 struct talitos_desc *desc,
1624 void *context, int error))
1625{
1626 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1627 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1628 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1629 struct device *dev = ctx->dev;
1630 struct talitos_desc *desc = &edesc->desc;
1631 int sg_count, ret;
1632
1633 /* first DWORD empty */
1634 desc->ptr[0] = zero_entry;
1635
1636 /* hash context in */
1637 if (!req_ctx->first || req_ctx->swinit) {
1638 map_single_talitos_ptr(dev, &desc->ptr[1],
1639 req_ctx->hw_context_size,
1640 (char *)req_ctx->hw_context, 0,
1641 DMA_TO_DEVICE);
1642 req_ctx->swinit = 0;
1643 } else {
1644 desc->ptr[1] = zero_entry;
1645 /* Indicate next op is not the first. */
1646 req_ctx->first = 0;
1647 }
1648
1649 /* HMAC key */
1650 if (ctx->keylen)
1651 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1652 (char *)&ctx->key, 0, DMA_TO_DEVICE);
1653 else
1654 desc->ptr[2] = zero_entry;
1655
1656 /*
1657 * data in
1658 */
1659 desc->ptr[3].len = cpu_to_be16(length);
1660 desc->ptr[3].j_extent = 0;
1661
1662 sg_count = talitos_map_sg(dev, req_ctx->psrc,
1663 edesc->src_nents ? : 1,
1664 DMA_TO_DEVICE,
1665 edesc->src_is_chained);
1666
1667 if (sg_count == 1) {
1668 to_talitos_ptr(&desc->ptr[3], sg_dma_address(req_ctx->psrc));
1669 } else {
1670 sg_count = sg_to_link_tbl(req_ctx->psrc, sg_count, length,
1671 &edesc->link_tbl[0]);
1672 if (sg_count > 1) {
1673 desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP;
1674 to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl);
1675 dma_sync_single_for_device(ctx->dev,
1676 edesc->dma_link_tbl,
1677 edesc->dma_len,
1678 DMA_BIDIRECTIONAL);
1679 } else {
1680 /* Only one segment now, so no link tbl needed */
1681 to_talitos_ptr(&desc->ptr[3],
1682 sg_dma_address(req_ctx->psrc));
1683 }
1684 }
1685
1686 /* fifth DWORD empty */
1687 desc->ptr[4] = zero_entry;
1688
1689 /* hash/HMAC out -or- hash context out */
1690 if (req_ctx->last)
1691 map_single_talitos_ptr(dev, &desc->ptr[5],
1692 crypto_ahash_digestsize(tfm),
1693 areq->result, 0, DMA_FROM_DEVICE);
1694 else
1695 map_single_talitos_ptr(dev, &desc->ptr[5],
1696 req_ctx->hw_context_size,
1697 req_ctx->hw_context, 0, DMA_FROM_DEVICE);
1698
1699 /* last DWORD empty */
1700 desc->ptr[6] = zero_entry;
1701
1702 ret = talitos_submit(dev, desc, callback, areq);
1703 if (ret != -EINPROGRESS) {
1704 common_nonsnoop_hash_unmap(dev, edesc, areq);
1705 kfree(edesc);
1706 }
1707 return ret;
1708}
1709
1710static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1711 unsigned int nbytes)
1712{
1713 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1714 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1715 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1716
1717 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, 1,
1718 nbytes, 0, 0, areq->base.flags);
1719}
1720
1721static int ahash_init(struct ahash_request *areq)
1722{
1723 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1724 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1725
1726 /* Initialize the context */
1727 req_ctx->count = 0;
1728 req_ctx->first = 1; /* first indicates h/w must init its context */
1729 req_ctx->swinit = 0; /* assume h/w init of context */
1730 req_ctx->hw_context_size =
1731 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1732 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1733 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1734
1735 return 0;
1736}
1737
1738/*
1739 * on h/w without explicit sha224 support, we initialize h/w context
1740 * manually with sha224 constants, and tell it to run sha256.
1741 */
1742static int ahash_init_sha224_swinit(struct ahash_request *areq)
1743{
1744 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1745
1746 ahash_init(areq);
1747 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1748
1749 req_ctx->hw_context[0] = cpu_to_be32(SHA224_H0);
1750 req_ctx->hw_context[1] = cpu_to_be32(SHA224_H1);
1751 req_ctx->hw_context[2] = cpu_to_be32(SHA224_H2);
1752 req_ctx->hw_context[3] = cpu_to_be32(SHA224_H3);
1753 req_ctx->hw_context[4] = cpu_to_be32(SHA224_H4);
1754 req_ctx->hw_context[5] = cpu_to_be32(SHA224_H5);
1755 req_ctx->hw_context[6] = cpu_to_be32(SHA224_H6);
1756 req_ctx->hw_context[7] = cpu_to_be32(SHA224_H7);
1757
1758 /* init 64-bit count */
1759 req_ctx->hw_context[8] = 0;
1760 req_ctx->hw_context[9] = 0;
1761
1762 return 0;
1763}
1764
1765static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1766{
1767 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1768 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1769 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1770 struct talitos_edesc *edesc;
1771 unsigned int blocksize =
1772 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1773 unsigned int nbytes_to_hash;
1774 unsigned int to_hash_later;
1775 unsigned int index;
1776 int chained;
1777
1778 index = req_ctx->count & (blocksize - 1);
1779 req_ctx->count += nbytes;
1780
1781 if (!req_ctx->last && (index + nbytes) < blocksize) {
1782 /* Buffer the partial block */
1783 sg_copy_to_buffer(areq->src,
1784 sg_count(areq->src, nbytes, &chained),
1785 req_ctx->buf + index, nbytes);
1786 return 0;
1787 }
1788
1789 if (index) {
1790 /* partial block from previous update; chain it in. */
1791 sg_init_table(req_ctx->bufsl, (nbytes) ? 2 : 1);
1792 sg_set_buf(req_ctx->bufsl, req_ctx->buf, index);
1793 if (nbytes)
1794 scatterwalk_sg_chain(req_ctx->bufsl, 2,
1795 areq->src);
1796 req_ctx->psrc = req_ctx->bufsl;
1797 } else {
1798 req_ctx->psrc = areq->src;
1799 }
1800 nbytes_to_hash = index + nbytes;
1801 if (!req_ctx->last) {
1802 to_hash_later = (nbytes_to_hash & (blocksize - 1));
1803 if (to_hash_later) {
1804 int nents;
1805 /* Must copy to_hash_later bytes from the end
1806 * to bufnext (a partial block) for later.
1807 */
1808 nents = sg_count(areq->src, nbytes, &chained);
1809 sg_copy_end_to_buffer(areq->src, nents,
1810 req_ctx->bufnext,
1811 to_hash_later,
1812 nbytes - to_hash_later);
1813
1814 /* Adjust count for what will be hashed now */
1815 nbytes_to_hash -= to_hash_later;
1816 }
1817 req_ctx->to_hash_later = to_hash_later;
1818 }
1819
1820 /* allocate extended descriptor */
1821 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1822 if (IS_ERR(edesc))
1823 return PTR_ERR(edesc);
1824
1825 edesc->desc.hdr = ctx->desc_hdr_template;
1826
1827 /* On last one, request SEC to pad; otherwise continue */
1828 if (req_ctx->last)
1829 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1830 else
1831 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1832
1833 /* request SEC to INIT hash. */
1834 if (req_ctx->first && !req_ctx->swinit)
1835 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1836
1837 /* When the tfm context has a keylen, it's an HMAC.
1838 * A first or last (ie. not middle) descriptor must request HMAC.
1839 */
1840 if (ctx->keylen && (req_ctx->first || req_ctx->last))
1841 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1842
1843 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1844 ahash_done);
1845}
1846
1847static int ahash_update(struct ahash_request *areq)
1848{
1849 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1850
1851 req_ctx->last = 0;
1852
1853 return ahash_process_req(areq, areq->nbytes);
1854}
1855
1856static int ahash_final(struct ahash_request *areq)
1857{
1858 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1859
1860 req_ctx->last = 1;
1861
1862 return ahash_process_req(areq, 0);
1863}
1864
1865static int ahash_finup(struct ahash_request *areq)
1866{
1867 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1868
1869 req_ctx->last = 1;
1870
1871 return ahash_process_req(areq, areq->nbytes);
1872}
1873
1874static int ahash_digest(struct ahash_request *areq)
1875{
1876 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1877 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
1878
1879 ahash->init(areq);
1880 req_ctx->last = 1;
1881
1882 return ahash_process_req(areq, areq->nbytes);
1883}
1884
1481struct talitos_alg_template { 1885struct talitos_alg_template {
1482 struct crypto_alg alg; 1886 u32 type;
1887 union {
1888 struct crypto_alg crypto;
1889 struct ahash_alg hash;
1890 } alg;
1483 __be32 desc_hdr_template; 1891 __be32 desc_hdr_template;
1484}; 1892};
1485 1893
1486static struct talitos_alg_template driver_algs[] = { 1894static struct talitos_alg_template driver_algs[] = {
1487 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ 1895 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
1488 { 1896 { .type = CRYPTO_ALG_TYPE_AEAD,
1489 .alg = { 1897 .alg.crypto = {
1490 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1898 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1491 .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos", 1899 .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
1492 .cra_blocksize = AES_BLOCK_SIZE, 1900 .cra_blocksize = AES_BLOCK_SIZE,
@@ -1511,8 +1919,8 @@ static struct talitos_alg_template driver_algs[] = {
1511 DESC_HDR_MODE1_MDEU_PAD | 1919 DESC_HDR_MODE1_MDEU_PAD |
1512 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 1920 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1513 }, 1921 },
1514 { 1922 { .type = CRYPTO_ALG_TYPE_AEAD,
1515 .alg = { 1923 .alg.crypto = {
1516 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", 1924 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
1517 .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos", 1925 .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
1518 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1926 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
@@ -1538,8 +1946,8 @@ static struct talitos_alg_template driver_algs[] = {
1538 DESC_HDR_MODE1_MDEU_PAD | 1946 DESC_HDR_MODE1_MDEU_PAD |
1539 DESC_HDR_MODE1_MDEU_SHA1_HMAC, 1947 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1540 }, 1948 },
1541 { 1949 { .type = CRYPTO_ALG_TYPE_AEAD,
1542 .alg = { 1950 .alg.crypto = {
1543 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1951 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1544 .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos", 1952 .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
1545 .cra_blocksize = AES_BLOCK_SIZE, 1953 .cra_blocksize = AES_BLOCK_SIZE,
@@ -1564,8 +1972,8 @@ static struct talitos_alg_template driver_algs[] = {
1564 DESC_HDR_MODE1_MDEU_PAD | 1972 DESC_HDR_MODE1_MDEU_PAD |
1565 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 1973 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1566 }, 1974 },
1567 { 1975 { .type = CRYPTO_ALG_TYPE_AEAD,
1568 .alg = { 1976 .alg.crypto = {
1569 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", 1977 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
1570 .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos", 1978 .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
1571 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1979 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
@@ -1591,8 +1999,8 @@ static struct talitos_alg_template driver_algs[] = {
1591 DESC_HDR_MODE1_MDEU_PAD | 1999 DESC_HDR_MODE1_MDEU_PAD |
1592 DESC_HDR_MODE1_MDEU_SHA256_HMAC, 2000 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1593 }, 2001 },
1594 { 2002 { .type = CRYPTO_ALG_TYPE_AEAD,
1595 .alg = { 2003 .alg.crypto = {
1596 .cra_name = "authenc(hmac(md5),cbc(aes))", 2004 .cra_name = "authenc(hmac(md5),cbc(aes))",
1597 .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos", 2005 .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos",
1598 .cra_blocksize = AES_BLOCK_SIZE, 2006 .cra_blocksize = AES_BLOCK_SIZE,
@@ -1617,8 +2025,8 @@ static struct talitos_alg_template driver_algs[] = {
1617 DESC_HDR_MODE1_MDEU_PAD | 2025 DESC_HDR_MODE1_MDEU_PAD |
1618 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2026 DESC_HDR_MODE1_MDEU_MD5_HMAC,
1619 }, 2027 },
1620 { 2028 { .type = CRYPTO_ALG_TYPE_AEAD,
1621 .alg = { 2029 .alg.crypto = {
1622 .cra_name = "authenc(hmac(md5),cbc(des3_ede))", 2030 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
1623 .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos", 2031 .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos",
1624 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2032 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
@@ -1645,8 +2053,8 @@ static struct talitos_alg_template driver_algs[] = {
1645 DESC_HDR_MODE1_MDEU_MD5_HMAC, 2053 DESC_HDR_MODE1_MDEU_MD5_HMAC,
1646 }, 2054 },
1647 /* ABLKCIPHER algorithms. */ 2055 /* ABLKCIPHER algorithms. */
1648 { 2056 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
1649 .alg = { 2057 .alg.crypto = {
1650 .cra_name = "cbc(aes)", 2058 .cra_name = "cbc(aes)",
1651 .cra_driver_name = "cbc-aes-talitos", 2059 .cra_driver_name = "cbc-aes-talitos",
1652 .cra_blocksize = AES_BLOCK_SIZE, 2060 .cra_blocksize = AES_BLOCK_SIZE,
@@ -1667,8 +2075,8 @@ static struct talitos_alg_template driver_algs[] = {
1667 DESC_HDR_SEL0_AESU | 2075 DESC_HDR_SEL0_AESU |
1668 DESC_HDR_MODE0_AESU_CBC, 2076 DESC_HDR_MODE0_AESU_CBC,
1669 }, 2077 },
1670 { 2078 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
1671 .alg = { 2079 .alg.crypto = {
1672 .cra_name = "cbc(des3_ede)", 2080 .cra_name = "cbc(des3_ede)",
1673 .cra_driver_name = "cbc-3des-talitos", 2081 .cra_driver_name = "cbc-3des-talitos",
1674 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2082 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
@@ -1689,14 +2097,140 @@ static struct talitos_alg_template driver_algs[] = {
1689 DESC_HDR_SEL0_DEU | 2097 DESC_HDR_SEL0_DEU |
1690 DESC_HDR_MODE0_DEU_CBC | 2098 DESC_HDR_MODE0_DEU_CBC |
1691 DESC_HDR_MODE0_DEU_3DES, 2099 DESC_HDR_MODE0_DEU_3DES,
1692 } 2100 },
2101 /* AHASH algorithms. */
2102 { .type = CRYPTO_ALG_TYPE_AHASH,
2103 .alg.hash = {
2104 .init = ahash_init,
2105 .update = ahash_update,
2106 .final = ahash_final,
2107 .finup = ahash_finup,
2108 .digest = ahash_digest,
2109 .halg.digestsize = MD5_DIGEST_SIZE,
2110 .halg.base = {
2111 .cra_name = "md5",
2112 .cra_driver_name = "md5-talitos",
2113 .cra_blocksize = MD5_BLOCK_SIZE,
2114 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2115 CRYPTO_ALG_ASYNC,
2116 .cra_type = &crypto_ahash_type
2117 }
2118 },
2119 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2120 DESC_HDR_SEL0_MDEUA |
2121 DESC_HDR_MODE0_MDEU_MD5,
2122 },
2123 { .type = CRYPTO_ALG_TYPE_AHASH,
2124 .alg.hash = {
2125 .init = ahash_init,
2126 .update = ahash_update,
2127 .final = ahash_final,
2128 .finup = ahash_finup,
2129 .digest = ahash_digest,
2130 .halg.digestsize = SHA1_DIGEST_SIZE,
2131 .halg.base = {
2132 .cra_name = "sha1",
2133 .cra_driver_name = "sha1-talitos",
2134 .cra_blocksize = SHA1_BLOCK_SIZE,
2135 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2136 CRYPTO_ALG_ASYNC,
2137 .cra_type = &crypto_ahash_type
2138 }
2139 },
2140 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2141 DESC_HDR_SEL0_MDEUA |
2142 DESC_HDR_MODE0_MDEU_SHA1,
2143 },
2144 { .type = CRYPTO_ALG_TYPE_AHASH,
2145 .alg.hash = {
2146 .init = ahash_init,
2147 .update = ahash_update,
2148 .final = ahash_final,
2149 .finup = ahash_finup,
2150 .digest = ahash_digest,
2151 .halg.digestsize = SHA224_DIGEST_SIZE,
2152 .halg.base = {
2153 .cra_name = "sha224",
2154 .cra_driver_name = "sha224-talitos",
2155 .cra_blocksize = SHA224_BLOCK_SIZE,
2156 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2157 CRYPTO_ALG_ASYNC,
2158 .cra_type = &crypto_ahash_type
2159 }
2160 },
2161 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2162 DESC_HDR_SEL0_MDEUA |
2163 DESC_HDR_MODE0_MDEU_SHA224,
2164 },
2165 { .type = CRYPTO_ALG_TYPE_AHASH,
2166 .alg.hash = {
2167 .init = ahash_init,
2168 .update = ahash_update,
2169 .final = ahash_final,
2170 .finup = ahash_finup,
2171 .digest = ahash_digest,
2172 .halg.digestsize = SHA256_DIGEST_SIZE,
2173 .halg.base = {
2174 .cra_name = "sha256",
2175 .cra_driver_name = "sha256-talitos",
2176 .cra_blocksize = SHA256_BLOCK_SIZE,
2177 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2178 CRYPTO_ALG_ASYNC,
2179 .cra_type = &crypto_ahash_type
2180 }
2181 },
2182 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2183 DESC_HDR_SEL0_MDEUA |
2184 DESC_HDR_MODE0_MDEU_SHA256,
2185 },
2186 { .type = CRYPTO_ALG_TYPE_AHASH,
2187 .alg.hash = {
2188 .init = ahash_init,
2189 .update = ahash_update,
2190 .final = ahash_final,
2191 .finup = ahash_finup,
2192 .digest = ahash_digest,
2193 .halg.digestsize = SHA384_DIGEST_SIZE,
2194 .halg.base = {
2195 .cra_name = "sha384",
2196 .cra_driver_name = "sha384-talitos",
2197 .cra_blocksize = SHA384_BLOCK_SIZE,
2198 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2199 CRYPTO_ALG_ASYNC,
2200 .cra_type = &crypto_ahash_type
2201 }
2202 },
2203 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2204 DESC_HDR_SEL0_MDEUB |
2205 DESC_HDR_MODE0_MDEUB_SHA384,
2206 },
2207 { .type = CRYPTO_ALG_TYPE_AHASH,
2208 .alg.hash = {
2209 .init = ahash_init,
2210 .update = ahash_update,
2211 .final = ahash_final,
2212 .finup = ahash_finup,
2213 .digest = ahash_digest,
2214 .halg.digestsize = SHA512_DIGEST_SIZE,
2215 .halg.base = {
2216 .cra_name = "sha512",
2217 .cra_driver_name = "sha512-talitos",
2218 .cra_blocksize = SHA512_BLOCK_SIZE,
2219 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2220 CRYPTO_ALG_ASYNC,
2221 .cra_type = &crypto_ahash_type
2222 }
2223 },
2224 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2225 DESC_HDR_SEL0_MDEUB |
2226 DESC_HDR_MODE0_MDEUB_SHA512,
2227 },
1693}; 2228};
1694 2229
1695struct talitos_crypto_alg { 2230struct talitos_crypto_alg {
1696 struct list_head entry; 2231 struct list_head entry;
1697 struct device *dev; 2232 struct device *dev;
1698 __be32 desc_hdr_template; 2233 struct talitos_alg_template algt;
1699 struct crypto_alg crypto_alg;
1700}; 2234};
1701 2235
1702static int talitos_cra_init(struct crypto_tfm *tfm) 2236static int talitos_cra_init(struct crypto_tfm *tfm)
@@ -1705,13 +2239,28 @@ static int talitos_cra_init(struct crypto_tfm *tfm)
1705 struct talitos_crypto_alg *talitos_alg; 2239 struct talitos_crypto_alg *talitos_alg;
1706 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 2240 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
1707 2241
1708 talitos_alg = container_of(alg, struct talitos_crypto_alg, crypto_alg); 2242 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2243 talitos_alg = container_of(__crypto_ahash_alg(alg),
2244 struct talitos_crypto_alg,
2245 algt.alg.hash);
2246 else
2247 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2248 algt.alg.crypto);
1709 2249
1710 /* update context with ptr to dev */ 2250 /* update context with ptr to dev */
1711 ctx->dev = talitos_alg->dev; 2251 ctx->dev = talitos_alg->dev;
1712 2252
1713 /* copy descriptor header template value */ 2253 /* copy descriptor header template value */
1714 ctx->desc_hdr_template = talitos_alg->desc_hdr_template; 2254 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2255
2256 return 0;
2257}
2258
2259static int talitos_cra_init_aead(struct crypto_tfm *tfm)
2260{
2261 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2262
2263 talitos_cra_init(tfm);
1715 2264
1716 /* random first IV */ 2265 /* random first IV */
1717 get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH); 2266 get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
@@ -1719,6 +2268,19 @@ static int talitos_cra_init(struct crypto_tfm *tfm)
1719 return 0; 2268 return 0;
1720} 2269}
1721 2270
2271static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2272{
2273 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2274
2275 talitos_cra_init(tfm);
2276
2277 ctx->keylen = 0;
2278 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2279 sizeof(struct talitos_ahash_req_ctx));
2280
2281 return 0;
2282}
2283
1722/* 2284/*
1723 * given the alg's descriptor header template, determine whether descriptor 2285 * given the alg's descriptor header template, determine whether descriptor
1724 * type and primary/secondary execution units required match the hw 2286 * type and primary/secondary execution units required match the hw
@@ -1747,7 +2309,15 @@ static int talitos_remove(struct of_device *ofdev)
1747 int i; 2309 int i;
1748 2310
1749 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { 2311 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
1750 crypto_unregister_alg(&t_alg->crypto_alg); 2312 switch (t_alg->algt.type) {
2313 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2314 case CRYPTO_ALG_TYPE_AEAD:
2315 crypto_unregister_alg(&t_alg->algt.alg.crypto);
2316 break;
2317 case CRYPTO_ALG_TYPE_AHASH:
2318 crypto_unregister_ahash(&t_alg->algt.alg.hash);
2319 break;
2320 }
1751 list_del(&t_alg->entry); 2321 list_del(&t_alg->entry);
1752 kfree(t_alg); 2322 kfree(t_alg);
1753 } 2323 }
@@ -1781,6 +2351,7 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
1781 struct talitos_alg_template 2351 struct talitos_alg_template
1782 *template) 2352 *template)
1783{ 2353{
2354 struct talitos_private *priv = dev_get_drvdata(dev);
1784 struct talitos_crypto_alg *t_alg; 2355 struct talitos_crypto_alg *t_alg;
1785 struct crypto_alg *alg; 2356 struct crypto_alg *alg;
1786 2357
@@ -1788,16 +2359,36 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
1788 if (!t_alg) 2359 if (!t_alg)
1789 return ERR_PTR(-ENOMEM); 2360 return ERR_PTR(-ENOMEM);
1790 2361
1791 alg = &t_alg->crypto_alg; 2362 t_alg->algt = *template;
1792 *alg = template->alg; 2363
2364 switch (t_alg->algt.type) {
2365 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2366 alg = &t_alg->algt.alg.crypto;
2367 alg->cra_init = talitos_cra_init;
2368 break;
2369 case CRYPTO_ALG_TYPE_AEAD:
2370 alg = &t_alg->algt.alg.crypto;
2371 alg->cra_init = talitos_cra_init_aead;
2372 break;
2373 case CRYPTO_ALG_TYPE_AHASH:
2374 alg = &t_alg->algt.alg.hash.halg.base;
2375 alg->cra_init = talitos_cra_init_ahash;
2376 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
2377 !strcmp(alg->cra_name, "sha224")) {
2378 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2379 t_alg->algt.desc_hdr_template =
2380 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2381 DESC_HDR_SEL0_MDEUA |
2382 DESC_HDR_MODE0_MDEU_SHA256;
2383 }
2384 break;
2385 }
1793 2386
1794 alg->cra_module = THIS_MODULE; 2387 alg->cra_module = THIS_MODULE;
1795 alg->cra_init = talitos_cra_init;
1796 alg->cra_priority = TALITOS_CRA_PRIORITY; 2388 alg->cra_priority = TALITOS_CRA_PRIORITY;
1797 alg->cra_alignmask = 0; 2389 alg->cra_alignmask = 0;
1798 alg->cra_ctxsize = sizeof(struct talitos_ctx); 2390 alg->cra_ctxsize = sizeof(struct talitos_ctx);
1799 2391
1800 t_alg->desc_hdr_template = template->desc_hdr_template;
1801 t_alg->dev = dev; 2392 t_alg->dev = dev;
1802 2393
1803 return t_alg; 2394 return t_alg;
@@ -1877,7 +2468,8 @@ static int talitos_probe(struct of_device *ofdev,
1877 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; 2468 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
1878 2469
1879 if (of_device_is_compatible(np, "fsl,sec2.1")) 2470 if (of_device_is_compatible(np, "fsl,sec2.1"))
1880 priv->features |= TALITOS_FTR_HW_AUTH_CHECK; 2471 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
2472 TALITOS_FTR_SHA224_HWINIT;
1881 2473
1882 priv->chan = kzalloc(sizeof(struct talitos_channel) * 2474 priv->chan = kzalloc(sizeof(struct talitos_channel) *
1883 priv->num_channels, GFP_KERNEL); 2475 priv->num_channels, GFP_KERNEL);
@@ -1931,6 +2523,7 @@ static int talitos_probe(struct of_device *ofdev,
1931 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { 2523 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
1932 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { 2524 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
1933 struct talitos_crypto_alg *t_alg; 2525 struct talitos_crypto_alg *t_alg;
2526 char *name = NULL;
1934 2527
1935 t_alg = talitos_alg_alloc(dev, &driver_algs[i]); 2528 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
1936 if (IS_ERR(t_alg)) { 2529 if (IS_ERR(t_alg)) {
@@ -1938,15 +2531,27 @@ static int talitos_probe(struct of_device *ofdev,
1938 goto err_out; 2531 goto err_out;
1939 } 2532 }
1940 2533
1941 err = crypto_register_alg(&t_alg->crypto_alg); 2534 switch (t_alg->algt.type) {
2535 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2536 case CRYPTO_ALG_TYPE_AEAD:
2537 err = crypto_register_alg(
2538 &t_alg->algt.alg.crypto);
2539 name = t_alg->algt.alg.crypto.cra_driver_name;
2540 break;
2541 case CRYPTO_ALG_TYPE_AHASH:
2542 err = crypto_register_ahash(
2543 &t_alg->algt.alg.hash);
2544 name =
2545 t_alg->algt.alg.hash.halg.base.cra_driver_name;
2546 break;
2547 }
1942 if (err) { 2548 if (err) {
1943 dev_err(dev, "%s alg registration failed\n", 2549 dev_err(dev, "%s alg registration failed\n",
1944 t_alg->crypto_alg.cra_driver_name); 2550 name);
1945 kfree(t_alg); 2551 kfree(t_alg);
1946 } else { 2552 } else {
1947 list_add_tail(&t_alg->entry, &priv->alg_list); 2553 list_add_tail(&t_alg->entry, &priv->alg_list);
1948 dev_info(dev, "%s\n", 2554 dev_info(dev, "%s\n", name);
1949 t_alg->crypto_alg.cra_driver_name);
1950 } 2555 }
1951 } 2556 }
1952 } 2557 }
diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h
index ff5a1450e145..0b746aca4587 100644
--- a/drivers/crypto/talitos.h
+++ b/drivers/crypto/talitos.h
@@ -1,7 +1,7 @@
1/* 1/*
2 * Freescale SEC (talitos) device register and descriptor header defines 2 * Freescale SEC (talitos) device register and descriptor header defines
3 * 3 *
4 * Copyright (c) 2006-2008 Freescale Semiconductor, Inc. 4 * Copyright (c) 2006-2010 Freescale Semiconductor, Inc.
5 * 5 *
6 * Redistribution and use in source and binary forms, with or without 6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions 7 * modification, are permitted provided that the following conditions
@@ -130,6 +130,9 @@
130#define TALITOS_CRCUISR 0xf030 /* cyclic redundancy check unit*/ 130#define TALITOS_CRCUISR 0xf030 /* cyclic redundancy check unit*/
131#define TALITOS_CRCUISR_LO 0xf034 131#define TALITOS_CRCUISR_LO 0xf034
132 132
133#define TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 0x28
134#define TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 0x48
135
133/* 136/*
134 * talitos descriptor header (hdr) bits 137 * talitos descriptor header (hdr) bits
135 */ 138 */
@@ -157,12 +160,16 @@
157#define DESC_HDR_MODE0_AESU_CBC cpu_to_be32(0x00200000) 160#define DESC_HDR_MODE0_AESU_CBC cpu_to_be32(0x00200000)
158#define DESC_HDR_MODE0_DEU_CBC cpu_to_be32(0x00400000) 161#define DESC_HDR_MODE0_DEU_CBC cpu_to_be32(0x00400000)
159#define DESC_HDR_MODE0_DEU_3DES cpu_to_be32(0x00200000) 162#define DESC_HDR_MODE0_DEU_3DES cpu_to_be32(0x00200000)
163#define DESC_HDR_MODE0_MDEU_CONT cpu_to_be32(0x08000000)
160#define DESC_HDR_MODE0_MDEU_INIT cpu_to_be32(0x01000000) 164#define DESC_HDR_MODE0_MDEU_INIT cpu_to_be32(0x01000000)
161#define DESC_HDR_MODE0_MDEU_HMAC cpu_to_be32(0x00800000) 165#define DESC_HDR_MODE0_MDEU_HMAC cpu_to_be32(0x00800000)
162#define DESC_HDR_MODE0_MDEU_PAD cpu_to_be32(0x00400000) 166#define DESC_HDR_MODE0_MDEU_PAD cpu_to_be32(0x00400000)
167#define DESC_HDR_MODE0_MDEU_SHA224 cpu_to_be32(0x00300000)
163#define DESC_HDR_MODE0_MDEU_MD5 cpu_to_be32(0x00200000) 168#define DESC_HDR_MODE0_MDEU_MD5 cpu_to_be32(0x00200000)
164#define DESC_HDR_MODE0_MDEU_SHA256 cpu_to_be32(0x00100000) 169#define DESC_HDR_MODE0_MDEU_SHA256 cpu_to_be32(0x00100000)
165#define DESC_HDR_MODE0_MDEU_SHA1 cpu_to_be32(0x00000000) 170#define DESC_HDR_MODE0_MDEU_SHA1 cpu_to_be32(0x00000000)
171#define DESC_HDR_MODE0_MDEUB_SHA384 cpu_to_be32(0x00000000)
172#define DESC_HDR_MODE0_MDEUB_SHA512 cpu_to_be32(0x00200000)
166#define DESC_HDR_MODE0_MDEU_MD5_HMAC (DESC_HDR_MODE0_MDEU_MD5 | \ 173#define DESC_HDR_MODE0_MDEU_MD5_HMAC (DESC_HDR_MODE0_MDEU_MD5 | \
167 DESC_HDR_MODE0_MDEU_HMAC) 174 DESC_HDR_MODE0_MDEU_HMAC)
168#define DESC_HDR_MODE0_MDEU_SHA256_HMAC (DESC_HDR_MODE0_MDEU_SHA256 | \ 175#define DESC_HDR_MODE0_MDEU_SHA256_HMAC (DESC_HDR_MODE0_MDEU_SHA256 | \
@@ -181,9 +188,12 @@
181#define DESC_HDR_MODE1_MDEU_INIT cpu_to_be32(0x00001000) 188#define DESC_HDR_MODE1_MDEU_INIT cpu_to_be32(0x00001000)
182#define DESC_HDR_MODE1_MDEU_HMAC cpu_to_be32(0x00000800) 189#define DESC_HDR_MODE1_MDEU_HMAC cpu_to_be32(0x00000800)
183#define DESC_HDR_MODE1_MDEU_PAD cpu_to_be32(0x00000400) 190#define DESC_HDR_MODE1_MDEU_PAD cpu_to_be32(0x00000400)
191#define DESC_HDR_MODE1_MDEU_SHA224 cpu_to_be32(0x00000300)
184#define DESC_HDR_MODE1_MDEU_MD5 cpu_to_be32(0x00000200) 192#define DESC_HDR_MODE1_MDEU_MD5 cpu_to_be32(0x00000200)
185#define DESC_HDR_MODE1_MDEU_SHA256 cpu_to_be32(0x00000100) 193#define DESC_HDR_MODE1_MDEU_SHA256 cpu_to_be32(0x00000100)
186#define DESC_HDR_MODE1_MDEU_SHA1 cpu_to_be32(0x00000000) 194#define DESC_HDR_MODE1_MDEU_SHA1 cpu_to_be32(0x00000000)
195#define DESC_HDR_MODE1_MDEUB_SHA384 cpu_to_be32(0x00000000)
196#define DESC_HDR_MODE1_MDEUB_SHA512 cpu_to_be32(0x00000200)
187#define DESC_HDR_MODE1_MDEU_MD5_HMAC (DESC_HDR_MODE1_MDEU_MD5 | \ 197#define DESC_HDR_MODE1_MDEU_MD5_HMAC (DESC_HDR_MODE1_MDEU_MD5 | \
188 DESC_HDR_MODE1_MDEU_HMAC) 198 DESC_HDR_MODE1_MDEU_HMAC)
189#define DESC_HDR_MODE1_MDEU_SHA256_HMAC (DESC_HDR_MODE1_MDEU_SHA256 | \ 199#define DESC_HDR_MODE1_MDEU_SHA256_HMAC (DESC_HDR_MODE1_MDEU_SHA256 | \
diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h
index fc0d575c71e0..59c3e5bd2c06 100644
--- a/include/crypto/algapi.h
+++ b/include/crypto/algapi.h
@@ -103,6 +103,23 @@ struct blkcipher_walk {
103 unsigned int blocksize; 103 unsigned int blocksize;
104}; 104};
105 105
106struct ablkcipher_walk {
107 struct {
108 struct page *page;
109 unsigned int offset;
110 } src, dst;
111
112 struct scatter_walk in;
113 unsigned int nbytes;
114 struct scatter_walk out;
115 unsigned int total;
116 struct list_head buffers;
117 u8 *iv_buffer;
118 u8 *iv;
119 int flags;
120 unsigned int blocksize;
121};
122
106extern const struct crypto_type crypto_ablkcipher_type; 123extern const struct crypto_type crypto_ablkcipher_type;
107extern const struct crypto_type crypto_aead_type; 124extern const struct crypto_type crypto_aead_type;
108extern const struct crypto_type crypto_blkcipher_type; 125extern const struct crypto_type crypto_blkcipher_type;
@@ -173,6 +190,12 @@ int blkcipher_walk_virt_block(struct blkcipher_desc *desc,
173 struct blkcipher_walk *walk, 190 struct blkcipher_walk *walk,
174 unsigned int blocksize); 191 unsigned int blocksize);
175 192
193int ablkcipher_walk_done(struct ablkcipher_request *req,
194 struct ablkcipher_walk *walk, int err);
195int ablkcipher_walk_phys(struct ablkcipher_request *req,
196 struct ablkcipher_walk *walk);
197void __ablkcipher_walk_complete(struct ablkcipher_walk *walk);
198
176static inline void *crypto_tfm_ctx_aligned(struct crypto_tfm *tfm) 199static inline void *crypto_tfm_ctx_aligned(struct crypto_tfm *tfm)
177{ 200{
178 return PTR_ALIGN(crypto_tfm_ctx(tfm), 201 return PTR_ALIGN(crypto_tfm_ctx(tfm),
@@ -283,6 +306,23 @@ static inline void blkcipher_walk_init(struct blkcipher_walk *walk,
283 walk->total = nbytes; 306 walk->total = nbytes;
284} 307}
285 308
309static inline void ablkcipher_walk_init(struct ablkcipher_walk *walk,
310 struct scatterlist *dst,
311 struct scatterlist *src,
312 unsigned int nbytes)
313{
314 walk->in.sg = src;
315 walk->out.sg = dst;
316 walk->total = nbytes;
317 INIT_LIST_HEAD(&walk->buffers);
318}
319
320static inline void ablkcipher_walk_complete(struct ablkcipher_walk *walk)
321{
322 if (unlikely(!list_empty(&walk->buffers)))
323 __ablkcipher_walk_complete(walk);
324}
325
286static inline struct crypto_async_request *crypto_get_backlog( 326static inline struct crypto_async_request *crypto_get_backlog(
287 struct crypto_queue *queue) 327 struct crypto_queue *queue)
288{ 328{
diff --git a/include/linux/padata.h b/include/linux/padata.h
index 51611da9c498..8d8406246eef 100644
--- a/include/linux/padata.h
+++ b/include/linux/padata.h
@@ -24,7 +24,19 @@
24#include <linux/workqueue.h> 24#include <linux/workqueue.h>
25#include <linux/spinlock.h> 25#include <linux/spinlock.h>
26#include <linux/list.h> 26#include <linux/list.h>
27#include <linux/timer.h>
27 28
29/**
30 * struct padata_priv - Embedded to the users data structure.
31 *
32 * @list: List entry, to attach to the padata lists.
33 * @pd: Pointer to the internal control structure.
34 * @cb_cpu: Callback cpu for serializatioon.
35 * @seq_nr: Sequence number of the parallelized data object.
36 * @info: Used to pass information from the parallel to the serial function.
37 * @parallel: Parallel execution function.
38 * @serial: Serial complete function.
39 */
28struct padata_priv { 40struct padata_priv {
29 struct list_head list; 41 struct list_head list;
30 struct parallel_data *pd; 42 struct parallel_data *pd;
@@ -35,11 +47,29 @@ struct padata_priv {
35 void (*serial)(struct padata_priv *padata); 47 void (*serial)(struct padata_priv *padata);
36}; 48};
37 49
50/**
51 * struct padata_list
52 *
53 * @list: List head.
54 * @lock: List lock.
55 */
38struct padata_list { 56struct padata_list {
39 struct list_head list; 57 struct list_head list;
40 spinlock_t lock; 58 spinlock_t lock;
41}; 59};
42 60
61/**
62 * struct padata_queue - The percpu padata queues.
63 *
64 * @parallel: List to wait for parallelization.
65 * @reorder: List to wait for reordering after parallel processing.
66 * @serial: List to wait for serialization after reordering.
67 * @pwork: work struct for parallelization.
68 * @swork: work struct for serialization.
69 * @pd: Backpointer to the internal control structure.
70 * @num_obj: Number of objects that are processed by this cpu.
71 * @cpu_index: Index of the cpu.
72 */
43struct padata_queue { 73struct padata_queue {
44 struct padata_list parallel; 74 struct padata_list parallel;
45 struct padata_list reorder; 75 struct padata_list reorder;
@@ -51,6 +81,20 @@ struct padata_queue {
51 int cpu_index; 81 int cpu_index;
52}; 82};
53 83
84/**
85 * struct parallel_data - Internal control structure, covers everything
86 * that depends on the cpumask in use.
87 *
88 * @pinst: padata instance.
89 * @queue: percpu padata queues.
90 * @seq_nr: The sequence number that will be attached to the next object.
91 * @reorder_objects: Number of objects waiting in the reorder queues.
92 * @refcnt: Number of objects holding a reference on this parallel_data.
93 * @max_seq_nr: Maximal used sequence number.
94 * @cpumask: cpumask in use.
95 * @lock: Reorder lock.
96 * @timer: Reorder timer.
97 */
54struct parallel_data { 98struct parallel_data {
55 struct padata_instance *pinst; 99 struct padata_instance *pinst;
56 struct padata_queue *queue; 100 struct padata_queue *queue;
@@ -60,8 +104,19 @@ struct parallel_data {
60 unsigned int max_seq_nr; 104 unsigned int max_seq_nr;
61 cpumask_var_t cpumask; 105 cpumask_var_t cpumask;
62 spinlock_t lock; 106 spinlock_t lock;
107 struct timer_list timer;
63}; 108};
64 109
110/**
111 * struct padata_instance - The overall control structure.
112 *
113 * @cpu_notifier: cpu hotplug notifier.
114 * @wq: The workqueue in use.
115 * @pd: The internal control structure.
116 * @cpumask: User supplied cpumask.
117 * @lock: padata instance lock.
118 * @flags: padata flags.
119 */
65struct padata_instance { 120struct padata_instance {
66 struct notifier_block cpu_notifier; 121 struct notifier_block cpu_notifier;
67 struct workqueue_struct *wq; 122 struct workqueue_struct *wq;
diff --git a/kernel/padata.c b/kernel/padata.c
index fd03513c7327..b1c9857f8402 100644
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -29,7 +29,7 @@
29#include <linux/rcupdate.h> 29#include <linux/rcupdate.h>
30 30
31#define MAX_SEQ_NR INT_MAX - NR_CPUS 31#define MAX_SEQ_NR INT_MAX - NR_CPUS
32#define MAX_OBJ_NUM 10000 * NR_CPUS 32#define MAX_OBJ_NUM 1000
33 33
34static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index) 34static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)
35{ 35{
@@ -88,7 +88,7 @@ static void padata_parallel_worker(struct work_struct *work)
88 local_bh_enable(); 88 local_bh_enable();
89} 89}
90 90
91/* 91/**
92 * padata_do_parallel - padata parallelization function 92 * padata_do_parallel - padata parallelization function
93 * 93 *
94 * @pinst: padata instance 94 * @pinst: padata instance
@@ -152,6 +152,23 @@ out:
152} 152}
153EXPORT_SYMBOL(padata_do_parallel); 153EXPORT_SYMBOL(padata_do_parallel);
154 154
155/*
156 * padata_get_next - Get the next object that needs serialization.
157 *
158 * Return values are:
159 *
160 * A pointer to the control struct of the next object that needs
161 * serialization, if present in one of the percpu reorder queues.
162 *
163 * NULL, if all percpu reorder queues are empty.
164 *
165 * -EINPROGRESS, if the next object that needs serialization will
166 * be parallel processed by another cpu and is not yet present in
167 * the cpu's reorder queue.
168 *
169 * -ENODATA, if this cpu has to do the parallel processing for
170 * the next object.
171 */
155static struct padata_priv *padata_get_next(struct parallel_data *pd) 172static struct padata_priv *padata_get_next(struct parallel_data *pd)
156{ 173{
157 int cpu, num_cpus, empty, calc_seq_nr; 174 int cpu, num_cpus, empty, calc_seq_nr;
@@ -173,7 +190,7 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd)
173 190
174 /* 191 /*
175 * Calculate the seq_nr of the object that should be 192 * Calculate the seq_nr of the object that should be
176 * next in this queue. 193 * next in this reorder queue.
177 */ 194 */
178 overrun = 0; 195 overrun = 0;
179 calc_seq_nr = (atomic_read(&queue->num_obj) * num_cpus) 196 calc_seq_nr = (atomic_read(&queue->num_obj) * num_cpus)
@@ -231,7 +248,8 @@ static struct padata_priv *padata_get_next(struct parallel_data *pd)
231 goto out; 248 goto out;
232 } 249 }
233 250
234 if (next_nr % num_cpus == next_queue->cpu_index) { 251 queue = per_cpu_ptr(pd->queue, smp_processor_id());
252 if (queue->cpu_index == next_queue->cpu_index) {
235 padata = ERR_PTR(-ENODATA); 253 padata = ERR_PTR(-ENODATA);
236 goto out; 254 goto out;
237 } 255 }
@@ -247,19 +265,40 @@ static void padata_reorder(struct parallel_data *pd)
247 struct padata_queue *queue; 265 struct padata_queue *queue;
248 struct padata_instance *pinst = pd->pinst; 266 struct padata_instance *pinst = pd->pinst;
249 267
250try_again: 268 /*
269 * We need to ensure that only one cpu can work on dequeueing of
270 * the reorder queue the time. Calculating in which percpu reorder
271 * queue the next object will arrive takes some time. A spinlock
272 * would be highly contended. Also it is not clear in which order
273 * the objects arrive to the reorder queues. So a cpu could wait to
274 * get the lock just to notice that there is nothing to do at the
275 * moment. Therefore we use a trylock and let the holder of the lock
276 * care for all the objects enqueued during the holdtime of the lock.
277 */
251 if (!spin_trylock_bh(&pd->lock)) 278 if (!spin_trylock_bh(&pd->lock))
252 goto out; 279 return;
253 280
254 while (1) { 281 while (1) {
255 padata = padata_get_next(pd); 282 padata = padata_get_next(pd);
256 283
284 /*
285 * All reorder queues are empty, or the next object that needs
286 * serialization is parallel processed by another cpu and is
287 * still on it's way to the cpu's reorder queue, nothing to
288 * do for now.
289 */
257 if (!padata || PTR_ERR(padata) == -EINPROGRESS) 290 if (!padata || PTR_ERR(padata) == -EINPROGRESS)
258 break; 291 break;
259 292
293 /*
294 * This cpu has to do the parallel processing of the next
295 * object. It's waiting in the cpu's parallelization queue,
296 * so exit imediately.
297 */
260 if (PTR_ERR(padata) == -ENODATA) { 298 if (PTR_ERR(padata) == -ENODATA) {
299 del_timer(&pd->timer);
261 spin_unlock_bh(&pd->lock); 300 spin_unlock_bh(&pd->lock);
262 goto out; 301 return;
263 } 302 }
264 303
265 queue = per_cpu_ptr(pd->queue, padata->cb_cpu); 304 queue = per_cpu_ptr(pd->queue, padata->cb_cpu);
@@ -273,13 +312,27 @@ try_again:
273 312
274 spin_unlock_bh(&pd->lock); 313 spin_unlock_bh(&pd->lock);
275 314
276 if (atomic_read(&pd->reorder_objects)) 315 /*
277 goto try_again; 316 * The next object that needs serialization might have arrived to
317 * the reorder queues in the meantime, we will be called again
318 * from the timer function if noone else cares for it.
319 */
320 if (atomic_read(&pd->reorder_objects)
321 && !(pinst->flags & PADATA_RESET))
322 mod_timer(&pd->timer, jiffies + HZ);
323 else
324 del_timer(&pd->timer);
278 325
279out:
280 return; 326 return;
281} 327}
282 328
329static void padata_reorder_timer(unsigned long arg)
330{
331 struct parallel_data *pd = (struct parallel_data *)arg;
332
333 padata_reorder(pd);
334}
335
283static void padata_serial_worker(struct work_struct *work) 336static void padata_serial_worker(struct work_struct *work)
284{ 337{
285 struct padata_queue *queue; 338 struct padata_queue *queue;
@@ -308,7 +361,7 @@ static void padata_serial_worker(struct work_struct *work)
308 local_bh_enable(); 361 local_bh_enable();
309} 362}
310 363
311/* 364/**
312 * padata_do_serial - padata serialization function 365 * padata_do_serial - padata serialization function
313 * 366 *
314 * @padata: object to be serialized. 367 * @padata: object to be serialized.
@@ -338,6 +391,7 @@ void padata_do_serial(struct padata_priv *padata)
338} 391}
339EXPORT_SYMBOL(padata_do_serial); 392EXPORT_SYMBOL(padata_do_serial);
340 393
394/* Allocate and initialize the internal cpumask dependend resources. */
341static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst, 395static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst,
342 const struct cpumask *cpumask) 396 const struct cpumask *cpumask)
343{ 397{
@@ -358,17 +412,15 @@ static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst,
358 if (!alloc_cpumask_var(&pd->cpumask, GFP_KERNEL)) 412 if (!alloc_cpumask_var(&pd->cpumask, GFP_KERNEL))
359 goto err_free_queue; 413 goto err_free_queue;
360 414
361 for_each_possible_cpu(cpu) { 415 cpumask_and(pd->cpumask, cpumask, cpu_active_mask);
416
417 for_each_cpu(cpu, pd->cpumask) {
362 queue = per_cpu_ptr(pd->queue, cpu); 418 queue = per_cpu_ptr(pd->queue, cpu);
363 419
364 queue->pd = pd; 420 queue->pd = pd;
365 421
366 if (cpumask_test_cpu(cpu, cpumask) 422 queue->cpu_index = cpu_index;
367 && cpumask_test_cpu(cpu, cpu_active_mask)) { 423 cpu_index++;
368 queue->cpu_index = cpu_index;
369 cpu_index++;
370 } else
371 queue->cpu_index = -1;
372 424
373 INIT_LIST_HEAD(&queue->reorder.list); 425 INIT_LIST_HEAD(&queue->reorder.list);
374 INIT_LIST_HEAD(&queue->parallel.list); 426 INIT_LIST_HEAD(&queue->parallel.list);
@@ -382,11 +434,10 @@ static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst,
382 atomic_set(&queue->num_obj, 0); 434 atomic_set(&queue->num_obj, 0);
383 } 435 }
384 436
385 cpumask_and(pd->cpumask, cpumask, cpu_active_mask);
386
387 num_cpus = cpumask_weight(pd->cpumask); 437 num_cpus = cpumask_weight(pd->cpumask);
388 pd->max_seq_nr = (MAX_SEQ_NR / num_cpus) * num_cpus - 1; 438 pd->max_seq_nr = (MAX_SEQ_NR / num_cpus) * num_cpus - 1;
389 439
440 setup_timer(&pd->timer, padata_reorder_timer, (unsigned long)pd);
390 atomic_set(&pd->seq_nr, -1); 441 atomic_set(&pd->seq_nr, -1);
391 atomic_set(&pd->reorder_objects, 0); 442 atomic_set(&pd->reorder_objects, 0);
392 atomic_set(&pd->refcnt, 0); 443 atomic_set(&pd->refcnt, 0);
@@ -410,6 +461,31 @@ static void padata_free_pd(struct parallel_data *pd)
410 kfree(pd); 461 kfree(pd);
411} 462}
412 463
464/* Flush all objects out of the padata queues. */
465static void padata_flush_queues(struct parallel_data *pd)
466{
467 int cpu;
468 struct padata_queue *queue;
469
470 for_each_cpu(cpu, pd->cpumask) {
471 queue = per_cpu_ptr(pd->queue, cpu);
472 flush_work(&queue->pwork);
473 }
474
475 del_timer_sync(&pd->timer);
476
477 if (atomic_read(&pd->reorder_objects))
478 padata_reorder(pd);
479
480 for_each_cpu(cpu, pd->cpumask) {
481 queue = per_cpu_ptr(pd->queue, cpu);
482 flush_work(&queue->swork);
483 }
484
485 BUG_ON(atomic_read(&pd->refcnt) != 0);
486}
487
488/* Replace the internal control stucture with a new one. */
413static void padata_replace(struct padata_instance *pinst, 489static void padata_replace(struct padata_instance *pinst,
414 struct parallel_data *pd_new) 490 struct parallel_data *pd_new)
415{ 491{
@@ -421,17 +497,13 @@ static void padata_replace(struct padata_instance *pinst,
421 497
422 synchronize_rcu(); 498 synchronize_rcu();
423 499
424 while (atomic_read(&pd_old->refcnt) != 0) 500 padata_flush_queues(pd_old);
425 yield();
426
427 flush_workqueue(pinst->wq);
428
429 padata_free_pd(pd_old); 501 padata_free_pd(pd_old);
430 502
431 pinst->flags &= ~PADATA_RESET; 503 pinst->flags &= ~PADATA_RESET;
432} 504}
433 505
434/* 506/**
435 * padata_set_cpumask - set the cpumask that padata should use 507 * padata_set_cpumask - set the cpumask that padata should use
436 * 508 *
437 * @pinst: padata instance 509 * @pinst: padata instance
@@ -443,10 +515,10 @@ int padata_set_cpumask(struct padata_instance *pinst,
443 struct parallel_data *pd; 515 struct parallel_data *pd;
444 int err = 0; 516 int err = 0;
445 517
446 might_sleep();
447
448 mutex_lock(&pinst->lock); 518 mutex_lock(&pinst->lock);
449 519
520 get_online_cpus();
521
450 pd = padata_alloc_pd(pinst, cpumask); 522 pd = padata_alloc_pd(pinst, cpumask);
451 if (!pd) { 523 if (!pd) {
452 err = -ENOMEM; 524 err = -ENOMEM;
@@ -458,6 +530,8 @@ int padata_set_cpumask(struct padata_instance *pinst,
458 padata_replace(pinst, pd); 530 padata_replace(pinst, pd);
459 531
460out: 532out:
533 put_online_cpus();
534
461 mutex_unlock(&pinst->lock); 535 mutex_unlock(&pinst->lock);
462 536
463 return err; 537 return err;
@@ -479,7 +553,7 @@ static int __padata_add_cpu(struct padata_instance *pinst, int cpu)
479 return 0; 553 return 0;
480} 554}
481 555
482/* 556/**
483 * padata_add_cpu - add a cpu to the padata cpumask 557 * padata_add_cpu - add a cpu to the padata cpumask
484 * 558 *
485 * @pinst: padata instance 559 * @pinst: padata instance
@@ -489,12 +563,12 @@ int padata_add_cpu(struct padata_instance *pinst, int cpu)
489{ 563{
490 int err; 564 int err;
491 565
492 might_sleep();
493
494 mutex_lock(&pinst->lock); 566 mutex_lock(&pinst->lock);
495 567
568 get_online_cpus();
496 cpumask_set_cpu(cpu, pinst->cpumask); 569 cpumask_set_cpu(cpu, pinst->cpumask);
497 err = __padata_add_cpu(pinst, cpu); 570 err = __padata_add_cpu(pinst, cpu);
571 put_online_cpus();
498 572
499 mutex_unlock(&pinst->lock); 573 mutex_unlock(&pinst->lock);
500 574
@@ -517,7 +591,7 @@ static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)
517 return 0; 591 return 0;
518} 592}
519 593
520/* 594/**
521 * padata_remove_cpu - remove a cpu from the padata cpumask 595 * padata_remove_cpu - remove a cpu from the padata cpumask
522 * 596 *
523 * @pinst: padata instance 597 * @pinst: padata instance
@@ -527,12 +601,12 @@ int padata_remove_cpu(struct padata_instance *pinst, int cpu)
527{ 601{
528 int err; 602 int err;
529 603
530 might_sleep();
531
532 mutex_lock(&pinst->lock); 604 mutex_lock(&pinst->lock);
533 605
606 get_online_cpus();
534 cpumask_clear_cpu(cpu, pinst->cpumask); 607 cpumask_clear_cpu(cpu, pinst->cpumask);
535 err = __padata_remove_cpu(pinst, cpu); 608 err = __padata_remove_cpu(pinst, cpu);
609 put_online_cpus();
536 610
537 mutex_unlock(&pinst->lock); 611 mutex_unlock(&pinst->lock);
538 612
@@ -540,38 +614,35 @@ int padata_remove_cpu(struct padata_instance *pinst, int cpu)
540} 614}
541EXPORT_SYMBOL(padata_remove_cpu); 615EXPORT_SYMBOL(padata_remove_cpu);
542 616
543/* 617/**
544 * padata_start - start the parallel processing 618 * padata_start - start the parallel processing
545 * 619 *
546 * @pinst: padata instance to start 620 * @pinst: padata instance to start
547 */ 621 */
548void padata_start(struct padata_instance *pinst) 622void padata_start(struct padata_instance *pinst)
549{ 623{
550 might_sleep();
551
552 mutex_lock(&pinst->lock); 624 mutex_lock(&pinst->lock);
553 pinst->flags |= PADATA_INIT; 625 pinst->flags |= PADATA_INIT;
554 mutex_unlock(&pinst->lock); 626 mutex_unlock(&pinst->lock);
555} 627}
556EXPORT_SYMBOL(padata_start); 628EXPORT_SYMBOL(padata_start);
557 629
558/* 630/**
559 * padata_stop - stop the parallel processing 631 * padata_stop - stop the parallel processing
560 * 632 *
561 * @pinst: padata instance to stop 633 * @pinst: padata instance to stop
562 */ 634 */
563void padata_stop(struct padata_instance *pinst) 635void padata_stop(struct padata_instance *pinst)
564{ 636{
565 might_sleep();
566
567 mutex_lock(&pinst->lock); 637 mutex_lock(&pinst->lock);
568 pinst->flags &= ~PADATA_INIT; 638 pinst->flags &= ~PADATA_INIT;
569 mutex_unlock(&pinst->lock); 639 mutex_unlock(&pinst->lock);
570} 640}
571EXPORT_SYMBOL(padata_stop); 641EXPORT_SYMBOL(padata_stop);
572 642
573static int __cpuinit padata_cpu_callback(struct notifier_block *nfb, 643#ifdef CONFIG_HOTPLUG_CPU
574 unsigned long action, void *hcpu) 644static int padata_cpu_callback(struct notifier_block *nfb,
645 unsigned long action, void *hcpu)
575{ 646{
576 int err; 647 int err;
577 struct padata_instance *pinst; 648 struct padata_instance *pinst;
@@ -621,8 +692,9 @@ static int __cpuinit padata_cpu_callback(struct notifier_block *nfb,
621 692
622 return NOTIFY_OK; 693 return NOTIFY_OK;
623} 694}
695#endif
624 696
625/* 697/**
626 * padata_alloc - allocate and initialize a padata instance 698 * padata_alloc - allocate and initialize a padata instance
627 * 699 *
628 * @cpumask: cpumask that padata uses for parallelization 700 * @cpumask: cpumask that padata uses for parallelization
@@ -631,7 +703,6 @@ static int __cpuinit padata_cpu_callback(struct notifier_block *nfb,
631struct padata_instance *padata_alloc(const struct cpumask *cpumask, 703struct padata_instance *padata_alloc(const struct cpumask *cpumask,
632 struct workqueue_struct *wq) 704 struct workqueue_struct *wq)
633{ 705{
634 int err;
635 struct padata_instance *pinst; 706 struct padata_instance *pinst;
636 struct parallel_data *pd; 707 struct parallel_data *pd;
637 708
@@ -639,6 +710,8 @@ struct padata_instance *padata_alloc(const struct cpumask *cpumask,
639 if (!pinst) 710 if (!pinst)
640 goto err; 711 goto err;
641 712
713 get_online_cpus();
714
642 pd = padata_alloc_pd(pinst, cpumask); 715 pd = padata_alloc_pd(pinst, cpumask);
643 if (!pd) 716 if (!pd)
644 goto err_free_inst; 717 goto err_free_inst;
@@ -654,31 +727,32 @@ struct padata_instance *padata_alloc(const struct cpumask *cpumask,
654 727
655 pinst->flags = 0; 728 pinst->flags = 0;
656 729
730#ifdef CONFIG_HOTPLUG_CPU
657 pinst->cpu_notifier.notifier_call = padata_cpu_callback; 731 pinst->cpu_notifier.notifier_call = padata_cpu_callback;
658 pinst->cpu_notifier.priority = 0; 732 pinst->cpu_notifier.priority = 0;
659 err = register_hotcpu_notifier(&pinst->cpu_notifier); 733 register_hotcpu_notifier(&pinst->cpu_notifier);
660 if (err) 734#endif
661 goto err_free_cpumask; 735
736 put_online_cpus();
662 737
663 mutex_init(&pinst->lock); 738 mutex_init(&pinst->lock);
664 739
665 return pinst; 740 return pinst;
666 741
667err_free_cpumask:
668 free_cpumask_var(pinst->cpumask);
669err_free_pd: 742err_free_pd:
670 padata_free_pd(pd); 743 padata_free_pd(pd);
671err_free_inst: 744err_free_inst:
672 kfree(pinst); 745 kfree(pinst);
746 put_online_cpus();
673err: 747err:
674 return NULL; 748 return NULL;
675} 749}
676EXPORT_SYMBOL(padata_alloc); 750EXPORT_SYMBOL(padata_alloc);
677 751
678/* 752/**
679 * padata_free - free a padata instance 753 * padata_free - free a padata instance
680 * 754 *
681 * @ padata_inst: padata instance to free 755 * @padata_inst: padata instance to free
682 */ 756 */
683void padata_free(struct padata_instance *pinst) 757void padata_free(struct padata_instance *pinst)
684{ 758{
@@ -686,10 +760,13 @@ void padata_free(struct padata_instance *pinst)
686 760
687 synchronize_rcu(); 761 synchronize_rcu();
688 762
689 while (atomic_read(&pinst->pd->refcnt) != 0) 763#ifdef CONFIG_HOTPLUG_CPU
690 yield();
691
692 unregister_hotcpu_notifier(&pinst->cpu_notifier); 764 unregister_hotcpu_notifier(&pinst->cpu_notifier);
765#endif
766 get_online_cpus();
767 padata_flush_queues(pinst->pd);
768 put_online_cpus();
769
693 padata_free_pd(pinst->pd); 770 padata_free_pd(pinst->pd);
694 free_cpumask_var(pinst->cpumask); 771 free_cpumask_var(pinst->cpumask);
695 kfree(pinst); 772 kfree(pinst);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-14 04:23:16 -0500