aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/devicetree/bindings/crypto/fsl-sec4.txt2
-rw-r--r--arch/x86/crypto/Makefile2
-rw-r--r--arch/x86/crypto/aes-i586-asm_32.S15
-rw-r--r--arch/x86/crypto/aes-x86_64-asm_64.S30
-rw-r--r--arch/x86/crypto/aesni-intel_asm.S23
-rw-r--r--arch/x86/crypto/blowfish-x86_64-asm_64.S39
-rw-r--r--arch/x86/crypto/camellia-aesni-avx-asm_64.S38
-rw-r--r--arch/x86/crypto/camellia-x86_64-asm_64.S50
-rw-r--r--arch/x86/crypto/cast5-avx-x86_64-asm_64.S48
-rw-r--r--arch/x86/crypto/cast6-avx-x86_64-asm_64.S35
-rw-r--r--arch/x86/crypto/crc32-pclmul_asm.S246
-rw-r--r--arch/x86/crypto/crc32-pclmul_glue.c201
-rw-r--r--arch/x86/crypto/crc32c-pcl-intel-asm_64.S8
-rw-r--r--arch/x86/crypto/ghash-clmulni-intel_asm.S4
-rw-r--r--arch/x86/crypto/salsa20-i586-asm_32.S28
-rw-r--r--arch/x86/crypto/salsa20-x86_64-asm_64.S28
-rw-r--r--arch/x86/crypto/salsa20_glue.c5
-rw-r--r--arch/x86/crypto/serpent-avx-x86_64-asm_64.S35
-rw-r--r--arch/x86/crypto/serpent-sse2-i586-asm_32.S20
-rw-r--r--arch/x86/crypto/serpent-sse2-x86_64-asm_64.S20
-rw-r--r--arch/x86/crypto/sha1_ssse3_asm.S10
-rw-r--r--arch/x86/crypto/twofish-avx-x86_64-asm_64.S35
-rw-r--r--arch/x86/crypto/twofish-i586-asm_32.S11
-rw-r--r--arch/x86/crypto/twofish-x86_64-asm_64-3way.S20
-rw-r--r--arch/x86/crypto/twofish-x86_64-asm_64.S11
-rw-r--r--crypto/Kconfig21
-rw-r--r--crypto/Makefile1
-rw-r--r--crypto/ablkcipher.c12
-rw-r--r--crypto/aead.c15
-rw-r--r--crypto/ahash.c2
-rw-r--r--crypto/algapi.c4
-rw-r--r--crypto/authenc.c3
-rw-r--r--crypto/authencesn.c3
-rw-r--r--crypto/blkcipher.c12
-rw-r--r--crypto/ccm.c23
-rw-r--r--crypto/chainiv.c3
-rw-r--r--crypto/crc32.c158
-rw-r--r--crypto/crypto_user.c38
-rw-r--r--crypto/ctr.c6
-rw-r--r--crypto/cts.c3
-rw-r--r--crypto/gcm.c29
-rw-r--r--crypto/pcompress.c3
-rw-r--r--crypto/rng.c2
-rw-r--r--crypto/seqiv.c3
-rw-r--r--crypto/shash.c3
-rw-r--r--crypto/testmgr.c15
-rw-r--r--drivers/crypto/atmel-aes.c2
-rw-r--r--drivers/crypto/bfin_crc.c6
-rw-r--r--drivers/crypto/omap-aes.c658
-rw-r--r--drivers/crypto/omap-sham.c918
-rw-r--r--drivers/crypto/s5p-sss.c4
51 files changed, 2146 insertions, 765 deletions
diff --git a/Documentation/devicetree/bindings/crypto/fsl-sec4.txt b/Documentation/devicetree/bindings/crypto/fsl-sec4.txt
index 6d21c0288e9e..e4022776ac6e 100644
--- a/Documentation/devicetree/bindings/crypto/fsl-sec4.txt
+++ b/Documentation/devicetree/bindings/crypto/fsl-sec4.txt
@@ -113,7 +113,7 @@ PROPERTIES
113EXAMPLE 113EXAMPLE
114 crypto@300000 { 114 crypto@300000 {
115 compatible = "fsl,sec-v4.0"; 115 compatible = "fsl,sec-v4.0";
116 fsl,sec-era = <0x2>; 116 fsl,sec-era = <2>;
117 #address-cells = <1>; 117 #address-cells = <1>;
118 #size-cells = <1>; 118 #size-cells = <1>;
119 reg = <0x300000 0x10000>; 119 reg = <0x300000 0x10000>;
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index e0ca7c9ac383..63947a8f9f0f 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -27,6 +27,7 @@ obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
27 27
28obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o 28obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
29obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o 29obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o
30obj-$(CONFIG_CRYPTO_CRC32_PCLMUL) += crc32-pclmul.o
30 31
31aes-i586-y := aes-i586-asm_32.o aes_glue.o 32aes-i586-y := aes-i586-asm_32.o aes_glue.o
32twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o 33twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o
@@ -52,3 +53,4 @@ ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
52sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o 53sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o
53crc32c-intel-y := crc32c-intel_glue.o 54crc32c-intel-y := crc32c-intel_glue.o
54crc32c-intel-$(CONFIG_CRYPTO_CRC32C_X86_64) += crc32c-pcl-intel-asm_64.o 55crc32c-intel-$(CONFIG_CRYPTO_CRC32C_X86_64) += crc32c-pcl-intel-asm_64.o
56crc32-pclmul-y := crc32-pclmul_asm.o crc32-pclmul_glue.o
diff --git a/arch/x86/crypto/aes-i586-asm_32.S b/arch/x86/crypto/aes-i586-asm_32.S
index b949ec2f9af4..2849dbc59e11 100644
--- a/arch/x86/crypto/aes-i586-asm_32.S
+++ b/arch/x86/crypto/aes-i586-asm_32.S
@@ -36,6 +36,7 @@
36.file "aes-i586-asm.S" 36.file "aes-i586-asm.S"
37.text 37.text
38 38
39#include <linux/linkage.h>
39#include <asm/asm-offsets.h> 40#include <asm/asm-offsets.h>
40 41
41#define tlen 1024 // length of each of 4 'xor' arrays (256 32-bit words) 42#define tlen 1024 // length of each of 4 'xor' arrays (256 32-bit words)
@@ -219,14 +220,10 @@
219// AES (Rijndael) Encryption Subroutine 220// AES (Rijndael) Encryption Subroutine
220/* void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out_blk, const u8 *in_blk) */ 221/* void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out_blk, const u8 *in_blk) */
221 222
222.global aes_enc_blk
223
224.extern crypto_ft_tab 223.extern crypto_ft_tab
225.extern crypto_fl_tab 224.extern crypto_fl_tab
226 225
227.align 4 226ENTRY(aes_enc_blk)
228
229aes_enc_blk:
230 push %ebp 227 push %ebp
231 mov ctx(%esp),%ebp 228 mov ctx(%esp),%ebp
232 229
@@ -290,18 +287,15 @@ aes_enc_blk:
290 mov %r0,(%ebp) 287 mov %r0,(%ebp)
291 pop %ebp 288 pop %ebp
292 ret 289 ret
290ENDPROC(aes_enc_blk)
293 291
294// AES (Rijndael) Decryption Subroutine 292// AES (Rijndael) Decryption Subroutine
295/* void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out_blk, const u8 *in_blk) */ 293/* void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out_blk, const u8 *in_blk) */
296 294
297.global aes_dec_blk
298
299.extern crypto_it_tab 295.extern crypto_it_tab
300.extern crypto_il_tab 296.extern crypto_il_tab
301 297
302.align 4 298ENTRY(aes_dec_blk)
303
304aes_dec_blk:
305 push %ebp 299 push %ebp
306 mov ctx(%esp),%ebp 300 mov ctx(%esp),%ebp
307 301
@@ -365,3 +359,4 @@ aes_dec_blk:
365 mov %r0,(%ebp) 359 mov %r0,(%ebp)
366 pop %ebp 360 pop %ebp
367 ret 361 ret
362ENDPROC(aes_dec_blk)
diff --git a/arch/x86/crypto/aes-x86_64-asm_64.S b/arch/x86/crypto/aes-x86_64-asm_64.S
index 5b577d5a059b..910565547163 100644
--- a/arch/x86/crypto/aes-x86_64-asm_64.S
+++ b/arch/x86/crypto/aes-x86_64-asm_64.S
@@ -15,6 +15,7 @@
15 15
16.text 16.text
17 17
18#include <linux/linkage.h>
18#include <asm/asm-offsets.h> 19#include <asm/asm-offsets.h>
19 20
20#define R1 %rax 21#define R1 %rax
@@ -49,10 +50,8 @@
49#define R11 %r11 50#define R11 %r11
50 51
51#define prologue(FUNC,KEY,B128,B192,r1,r2,r3,r4,r5,r6,r7,r8,r9,r10,r11) \ 52#define prologue(FUNC,KEY,B128,B192,r1,r2,r3,r4,r5,r6,r7,r8,r9,r10,r11) \
52 .global FUNC; \ 53 ENTRY(FUNC); \
53 .type FUNC,@function; \ 54 movq r1,r2; \
54 .align 8; \
55FUNC: movq r1,r2; \
56 movq r3,r4; \ 55 movq r3,r4; \
57 leaq KEY+48(r8),r9; \ 56 leaq KEY+48(r8),r9; \
58 movq r10,r11; \ 57 movq r10,r11; \
@@ -71,14 +70,15 @@ FUNC: movq r1,r2; \
71 je B192; \ 70 je B192; \
72 leaq 32(r9),r9; 71 leaq 32(r9),r9;
73 72
74#define epilogue(r1,r2,r3,r4,r5,r6,r7,r8,r9) \ 73#define epilogue(FUNC,r1,r2,r3,r4,r5,r6,r7,r8,r9) \
75 movq r1,r2; \ 74 movq r1,r2; \
76 movq r3,r4; \ 75 movq r3,r4; \
77 movl r5 ## E,(r9); \ 76 movl r5 ## E,(r9); \
78 movl r6 ## E,4(r9); \ 77 movl r6 ## E,4(r9); \
79 movl r7 ## E,8(r9); \ 78 movl r7 ## E,8(r9); \
80 movl r8 ## E,12(r9); \ 79 movl r8 ## E,12(r9); \
81 ret; 80 ret; \
81 ENDPROC(FUNC);
82 82
83#define round(TAB,OFFSET,r1,r2,r3,r4,r5,r6,r7,r8,ra,rb,rc,rd) \ 83#define round(TAB,OFFSET,r1,r2,r3,r4,r5,r6,r7,r8,ra,rb,rc,rd) \
84 movzbl r2 ## H,r5 ## E; \ 84 movzbl r2 ## H,r5 ## E; \
@@ -133,7 +133,7 @@ FUNC: movq r1,r2; \
133#define entry(FUNC,KEY,B128,B192) \ 133#define entry(FUNC,KEY,B128,B192) \
134 prologue(FUNC,KEY,B128,B192,R2,R8,R7,R9,R1,R3,R4,R6,R10,R5,R11) 134 prologue(FUNC,KEY,B128,B192,R2,R8,R7,R9,R1,R3,R4,R6,R10,R5,R11)
135 135
136#define return epilogue(R8,R2,R9,R7,R5,R6,R3,R4,R11) 136#define return(FUNC) epilogue(FUNC,R8,R2,R9,R7,R5,R6,R3,R4,R11)
137 137
138#define encrypt_round(TAB,OFFSET) \ 138#define encrypt_round(TAB,OFFSET) \
139 round(TAB,OFFSET,R1,R2,R3,R4,R5,R6,R7,R10,R5,R6,R3,R4) \ 139 round(TAB,OFFSET,R1,R2,R3,R4,R5,R6,R7,R10,R5,R6,R3,R4) \
@@ -151,12 +151,12 @@ FUNC: movq r1,r2; \
151 151
152/* void aes_enc_blk(stuct crypto_tfm *tfm, u8 *out, const u8 *in) */ 152/* void aes_enc_blk(stuct crypto_tfm *tfm, u8 *out, const u8 *in) */
153 153
154 entry(aes_enc_blk,0,enc128,enc192) 154 entry(aes_enc_blk,0,.Le128,.Le192)
155 encrypt_round(crypto_ft_tab,-96) 155 encrypt_round(crypto_ft_tab,-96)
156 encrypt_round(crypto_ft_tab,-80) 156 encrypt_round(crypto_ft_tab,-80)
157enc192: encrypt_round(crypto_ft_tab,-64) 157.Le192: encrypt_round(crypto_ft_tab,-64)
158 encrypt_round(crypto_ft_tab,-48) 158 encrypt_round(crypto_ft_tab,-48)
159enc128: encrypt_round(crypto_ft_tab,-32) 159.Le128: encrypt_round(crypto_ft_tab,-32)
160 encrypt_round(crypto_ft_tab,-16) 160 encrypt_round(crypto_ft_tab,-16)
161 encrypt_round(crypto_ft_tab, 0) 161 encrypt_round(crypto_ft_tab, 0)
162 encrypt_round(crypto_ft_tab, 16) 162 encrypt_round(crypto_ft_tab, 16)
@@ -166,16 +166,16 @@ enc128: encrypt_round(crypto_ft_tab,-32)
166 encrypt_round(crypto_ft_tab, 80) 166 encrypt_round(crypto_ft_tab, 80)
167 encrypt_round(crypto_ft_tab, 96) 167 encrypt_round(crypto_ft_tab, 96)
168 encrypt_final(crypto_fl_tab,112) 168 encrypt_final(crypto_fl_tab,112)
169 return 169 return(aes_enc_blk)
170 170
171/* void aes_dec_blk(struct crypto_tfm *tfm, u8 *out, const u8 *in) */ 171/* void aes_dec_blk(struct crypto_tfm *tfm, u8 *out, const u8 *in) */
172 172
173 entry(aes_dec_blk,240,dec128,dec192) 173 entry(aes_dec_blk,240,.Ld128,.Ld192)
174 decrypt_round(crypto_it_tab,-96) 174 decrypt_round(crypto_it_tab,-96)
175 decrypt_round(crypto_it_tab,-80) 175 decrypt_round(crypto_it_tab,-80)
176dec192: decrypt_round(crypto_it_tab,-64) 176.Ld192: decrypt_round(crypto_it_tab,-64)
177 decrypt_round(crypto_it_tab,-48) 177 decrypt_round(crypto_it_tab,-48)
178dec128: decrypt_round(crypto_it_tab,-32) 178.Ld128: decrypt_round(crypto_it_tab,-32)
179 decrypt_round(crypto_it_tab,-16) 179 decrypt_round(crypto_it_tab,-16)
180 decrypt_round(crypto_it_tab, 0) 180 decrypt_round(crypto_it_tab, 0)
181 decrypt_round(crypto_it_tab, 16) 181 decrypt_round(crypto_it_tab, 16)
@@ -185,4 +185,4 @@ dec128: decrypt_round(crypto_it_tab,-32)
185 decrypt_round(crypto_it_tab, 80) 185 decrypt_round(crypto_it_tab, 80)
186 decrypt_round(crypto_it_tab, 96) 186 decrypt_round(crypto_it_tab, 96)
187 decrypt_final(crypto_il_tab,112) 187 decrypt_final(crypto_il_tab,112)
188 return 188 return(aes_dec_blk)
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 3470624d7835..04b797767b9e 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -1262,7 +1262,6 @@ TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst
1262* poly = x^128 + x^127 + x^126 + x^121 + 1 1262* poly = x^128 + x^127 + x^126 + x^121 + 1
1263* 1263*
1264*****************************************************************************/ 1264*****************************************************************************/
1265
1266ENTRY(aesni_gcm_dec) 1265ENTRY(aesni_gcm_dec)
1267 push %r12 1266 push %r12
1268 push %r13 1267 push %r13
@@ -1437,6 +1436,7 @@ _return_T_done_decrypt:
1437 pop %r13 1436 pop %r13
1438 pop %r12 1437 pop %r12
1439 ret 1438 ret
1439ENDPROC(aesni_gcm_dec)
1440 1440
1441 1441
1442/***************************************************************************** 1442/*****************************************************************************
@@ -1700,10 +1700,12 @@ _return_T_done_encrypt:
1700 pop %r13 1700 pop %r13
1701 pop %r12 1701 pop %r12
1702 ret 1702 ret
1703ENDPROC(aesni_gcm_enc)
1703 1704
1704#endif 1705#endif
1705 1706
1706 1707
1708.align 4
1707_key_expansion_128: 1709_key_expansion_128:
1708_key_expansion_256a: 1710_key_expansion_256a:
1709 pshufd $0b11111111, %xmm1, %xmm1 1711 pshufd $0b11111111, %xmm1, %xmm1
@@ -1715,6 +1717,8 @@ _key_expansion_256a:
1715 movaps %xmm0, (TKEYP) 1717 movaps %xmm0, (TKEYP)
1716 add $0x10, TKEYP 1718 add $0x10, TKEYP
1717 ret 1719 ret
1720ENDPROC(_key_expansion_128)
1721ENDPROC(_key_expansion_256a)
1718 1722
1719.align 4 1723.align 4
1720_key_expansion_192a: 1724_key_expansion_192a:
@@ -1739,6 +1743,7 @@ _key_expansion_192a:
1739 movaps %xmm1, 0x10(TKEYP) 1743 movaps %xmm1, 0x10(TKEYP)
1740 add $0x20, TKEYP 1744 add $0x20, TKEYP
1741 ret 1745 ret
1746ENDPROC(_key_expansion_192a)
1742 1747
1743.align 4 1748.align 4
1744_key_expansion_192b: 1749_key_expansion_192b:
@@ -1758,6 +1763,7 @@ _key_expansion_192b:
1758 movaps %xmm0, (TKEYP) 1763 movaps %xmm0, (TKEYP)
1759 add $0x10, TKEYP 1764 add $0x10, TKEYP
1760 ret 1765 ret
1766ENDPROC(_key_expansion_192b)
1761 1767
1762.align 4 1768.align 4
1763_key_expansion_256b: 1769_key_expansion_256b:
@@ -1770,6 +1776,7 @@ _key_expansion_256b:
1770 movaps %xmm2, (TKEYP) 1776 movaps %xmm2, (TKEYP)
1771 add $0x10, TKEYP 1777 add $0x10, TKEYP
1772 ret 1778 ret
1779ENDPROC(_key_expansion_256b)
1773 1780
1774/* 1781/*
1775 * int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 1782 * int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
@@ -1882,6 +1889,7 @@ ENTRY(aesni_set_key)
1882 popl KEYP 1889 popl KEYP
1883#endif 1890#endif
1884 ret 1891 ret
1892ENDPROC(aesni_set_key)
1885 1893
1886/* 1894/*
1887 * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) 1895 * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
@@ -1903,6 +1911,7 @@ ENTRY(aesni_enc)
1903 popl KEYP 1911 popl KEYP
1904#endif 1912#endif
1905 ret 1913 ret
1914ENDPROC(aesni_enc)
1906 1915
1907/* 1916/*
1908 * _aesni_enc1: internal ABI 1917 * _aesni_enc1: internal ABI
@@ -1960,6 +1969,7 @@ _aesni_enc1:
1960 movaps 0x70(TKEYP), KEY 1969 movaps 0x70(TKEYP), KEY
1961 AESENCLAST KEY STATE 1970 AESENCLAST KEY STATE
1962 ret 1971 ret
1972ENDPROC(_aesni_enc1)
1963 1973
1964/* 1974/*
1965 * _aesni_enc4: internal ABI 1975 * _aesni_enc4: internal ABI
@@ -2068,6 +2078,7 @@ _aesni_enc4:
2068 AESENCLAST KEY STATE3 2078 AESENCLAST KEY STATE3
2069 AESENCLAST KEY STATE4 2079 AESENCLAST KEY STATE4
2070 ret 2080 ret
2081ENDPROC(_aesni_enc4)
2071 2082
2072/* 2083/*
2073 * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) 2084 * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
@@ -2090,6 +2101,7 @@ ENTRY(aesni_dec)
2090 popl KEYP 2101 popl KEYP
2091#endif 2102#endif
2092 ret 2103 ret
2104ENDPROC(aesni_dec)
2093 2105
2094/* 2106/*
2095 * _aesni_dec1: internal ABI 2107 * _aesni_dec1: internal ABI
@@ -2147,6 +2159,7 @@ _aesni_dec1:
2147 movaps 0x70(TKEYP), KEY 2159 movaps 0x70(TKEYP), KEY
2148 AESDECLAST KEY STATE 2160 AESDECLAST KEY STATE
2149 ret 2161 ret
2162ENDPROC(_aesni_dec1)
2150 2163
2151/* 2164/*
2152 * _aesni_dec4: internal ABI 2165 * _aesni_dec4: internal ABI
@@ -2255,6 +2268,7 @@ _aesni_dec4:
2255 AESDECLAST KEY STATE3 2268 AESDECLAST KEY STATE3
2256 AESDECLAST KEY STATE4 2269 AESDECLAST KEY STATE4
2257 ret 2270 ret
2271ENDPROC(_aesni_dec4)
2258 2272
2259/* 2273/*
2260 * void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, 2274 * void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2312,6 +2326,7 @@ ENTRY(aesni_ecb_enc)
2312 popl LEN 2326 popl LEN
2313#endif 2327#endif
2314 ret 2328 ret
2329ENDPROC(aesni_ecb_enc)
2315 2330
2316/* 2331/*
2317 * void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, 2332 * void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2370,6 +2385,7 @@ ENTRY(aesni_ecb_dec)
2370 popl LEN 2385 popl LEN
2371#endif 2386#endif
2372 ret 2387 ret
2388ENDPROC(aesni_ecb_dec)
2373 2389
2374/* 2390/*
2375 * void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, 2391 * void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2411,6 +2427,7 @@ ENTRY(aesni_cbc_enc)
2411 popl IVP 2427 popl IVP
2412#endif 2428#endif
2413 ret 2429 ret
2430ENDPROC(aesni_cbc_enc)
2414 2431
2415/* 2432/*
2416 * void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, 2433 * void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2501,6 +2518,7 @@ ENTRY(aesni_cbc_dec)
2501 popl IVP 2518 popl IVP
2502#endif 2519#endif
2503 ret 2520 ret
2521ENDPROC(aesni_cbc_dec)
2504 2522
2505#ifdef __x86_64__ 2523#ifdef __x86_64__
2506.align 16 2524.align 16
@@ -2527,6 +2545,7 @@ _aesni_inc_init:
2527 MOVQ_R64_XMM TCTR_LOW INC 2545 MOVQ_R64_XMM TCTR_LOW INC
2528 MOVQ_R64_XMM CTR TCTR_LOW 2546 MOVQ_R64_XMM CTR TCTR_LOW
2529 ret 2547 ret
2548ENDPROC(_aesni_inc_init)
2530 2549
2531/* 2550/*
2532 * _aesni_inc: internal ABI 2551 * _aesni_inc: internal ABI
@@ -2555,6 +2574,7 @@ _aesni_inc:
2555 movaps CTR, IV 2574 movaps CTR, IV
2556 PSHUFB_XMM BSWAP_MASK IV 2575 PSHUFB_XMM BSWAP_MASK IV
2557 ret 2576 ret
2577ENDPROC(_aesni_inc)
2558 2578
2559/* 2579/*
2560 * void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, 2580 * void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2615,4 +2635,5 @@ ENTRY(aesni_ctr_enc)
2615 movups IV, (IVP) 2635 movups IV, (IVP)
2616.Lctr_enc_just_ret: 2636.Lctr_enc_just_ret:
2617 ret 2637 ret
2638ENDPROC(aesni_ctr_enc)
2618#endif 2639#endif
diff --git a/arch/x86/crypto/blowfish-x86_64-asm_64.S b/arch/x86/crypto/blowfish-x86_64-asm_64.S
index 391d245dc086..246c67006ed0 100644
--- a/arch/x86/crypto/blowfish-x86_64-asm_64.S
+++ b/arch/x86/crypto/blowfish-x86_64-asm_64.S
@@ -20,6 +20,8 @@
20 * 20 *
21 */ 21 */
22 22
23#include <linux/linkage.h>
24
23.file "blowfish-x86_64-asm.S" 25.file "blowfish-x86_64-asm.S"
24.text 26.text
25 27
@@ -116,11 +118,7 @@
116 bswapq RX0; \ 118 bswapq RX0; \
117 xorq RX0, (RIO); 119 xorq RX0, (RIO);
118 120
119.align 8 121ENTRY(__blowfish_enc_blk)
120.global __blowfish_enc_blk
121.type __blowfish_enc_blk,@function;
122
123__blowfish_enc_blk:
124 /* input: 122 /* input:
125 * %rdi: ctx, CTX 123 * %rdi: ctx, CTX
126 * %rsi: dst 124 * %rsi: dst
@@ -148,19 +146,16 @@ __blowfish_enc_blk:
148 146
149 movq %r10, RIO; 147 movq %r10, RIO;
150 test %cl, %cl; 148 test %cl, %cl;
151 jnz __enc_xor; 149 jnz .L__enc_xor;
152 150
153 write_block(); 151 write_block();
154 ret; 152 ret;
155__enc_xor: 153.L__enc_xor:
156 xor_block(); 154 xor_block();
157 ret; 155 ret;
156ENDPROC(__blowfish_enc_blk)
158 157
159.align 8 158ENTRY(blowfish_dec_blk)
160.global blowfish_dec_blk
161.type blowfish_dec_blk,@function;
162
163blowfish_dec_blk:
164 /* input: 159 /* input:
165 * %rdi: ctx, CTX 160 * %rdi: ctx, CTX
166 * %rsi: dst 161 * %rsi: dst
@@ -189,6 +184,7 @@ blowfish_dec_blk:
189 movq %r11, %rbp; 184 movq %r11, %rbp;
190 185
191 ret; 186 ret;
187ENDPROC(blowfish_dec_blk)
192 188
193/********************************************************************** 189/**********************************************************************
194 4-way blowfish, four blocks parallel 190 4-way blowfish, four blocks parallel
@@ -300,11 +296,7 @@ blowfish_dec_blk:
300 bswapq RX3; \ 296 bswapq RX3; \
301 xorq RX3, 24(RIO); 297 xorq RX3, 24(RIO);
302 298
303.align 8 299ENTRY(__blowfish_enc_blk_4way)
304.global __blowfish_enc_blk_4way
305.type __blowfish_enc_blk_4way,@function;
306
307__blowfish_enc_blk_4way:
308 /* input: 300 /* input:
309 * %rdi: ctx, CTX 301 * %rdi: ctx, CTX
310 * %rsi: dst 302 * %rsi: dst
@@ -336,7 +328,7 @@ __blowfish_enc_blk_4way:
336 movq %r11, RIO; 328 movq %r11, RIO;
337 329
338 test %bpl, %bpl; 330 test %bpl, %bpl;
339 jnz __enc_xor4; 331 jnz .L__enc_xor4;
340 332
341 write_block4(); 333 write_block4();
342 334
@@ -344,18 +336,15 @@ __blowfish_enc_blk_4way:
344 popq %rbp; 336 popq %rbp;
345 ret; 337 ret;
346 338
347__enc_xor4: 339.L__enc_xor4:
348 xor_block4(); 340 xor_block4();
349 341
350 popq %rbx; 342 popq %rbx;
351 popq %rbp; 343 popq %rbp;
352 ret; 344 ret;
345ENDPROC(__blowfish_enc_blk_4way)
353 346
354.align 8 347ENTRY(blowfish_dec_blk_4way)
355.global blowfish_dec_blk_4way
356.type blowfish_dec_blk_4way,@function;
357
358blowfish_dec_blk_4way:
359 /* input: 348 /* input:
360 * %rdi: ctx, CTX 349 * %rdi: ctx, CTX
361 * %rsi: dst 350 * %rsi: dst
@@ -387,4 +376,4 @@ blowfish_dec_blk_4way:
387 popq %rbp; 376 popq %rbp;
388 377
389 ret; 378 ret;
390 379ENDPROC(blowfish_dec_blk_4way)
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index 2306d2e4816f..cfc163469c71 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -15,6 +15,8 @@
15 * http://koti.mbnet.fi/axh/crypto/camellia-BSD-1.2.0-aesni1.tar.xz 15 * http://koti.mbnet.fi/axh/crypto/camellia-BSD-1.2.0-aesni1.tar.xz
16 */ 16 */
17 17
18#include <linux/linkage.h>
19
18#define CAMELLIA_TABLE_BYTE_LEN 272 20#define CAMELLIA_TABLE_BYTE_LEN 272
19 21
20/* struct camellia_ctx: */ 22/* struct camellia_ctx: */
@@ -190,6 +192,7 @@ roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd:
190 %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, 192 %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15,
191 %rcx, (%r9)); 193 %rcx, (%r9));
192 ret; 194 ret;
195ENDPROC(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd)
193 196
194.align 8 197.align 8
195roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab: 198roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab:
@@ -197,6 +200,7 @@ roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab:
197 %xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11, 200 %xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11,
198 %rax, (%r9)); 201 %rax, (%r9));
199 ret; 202 ret;
203ENDPROC(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab)
200 204
201/* 205/*
202 * IN/OUT: 206 * IN/OUT:
@@ -709,8 +713,6 @@ roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab:
709.text 713.text
710 714
711.align 8 715.align 8
712.type __camellia_enc_blk16,@function;
713
714__camellia_enc_blk16: 716__camellia_enc_blk16:
715 /* input: 717 /* input:
716 * %rdi: ctx, CTX 718 * %rdi: ctx, CTX
@@ -793,10 +795,9 @@ __camellia_enc_blk16:
793 %xmm15, %rax, %rcx, 24); 795 %xmm15, %rax, %rcx, 24);
794 796
795 jmp .Lenc_done; 797 jmp .Lenc_done;
798ENDPROC(__camellia_enc_blk16)
796 799
797.align 8 800.align 8
798.type __camellia_dec_blk16,@function;
799
800__camellia_dec_blk16: 801__camellia_dec_blk16:
801 /* input: 802 /* input:
802 * %rdi: ctx, CTX 803 * %rdi: ctx, CTX
@@ -877,12 +878,9 @@ __camellia_dec_blk16:
877 ((key_table + (24) * 8) + 4)(CTX)); 878 ((key_table + (24) * 8) + 4)(CTX));
878 879
879 jmp .Ldec_max24; 880 jmp .Ldec_max24;
881ENDPROC(__camellia_dec_blk16)
880 882
881.align 8 883ENTRY(camellia_ecb_enc_16way)
882.global camellia_ecb_enc_16way
883.type camellia_ecb_enc_16way,@function;
884
885camellia_ecb_enc_16way:
886 /* input: 884 /* input:
887 * %rdi: ctx, CTX 885 * %rdi: ctx, CTX
888 * %rsi: dst (16 blocks) 886 * %rsi: dst (16 blocks)
@@ -903,12 +901,9 @@ camellia_ecb_enc_16way:
903 %xmm8, %rsi); 901 %xmm8, %rsi);
904 902
905 ret; 903 ret;
904ENDPROC(camellia_ecb_enc_16way)
906 905
907.align 8 906ENTRY(camellia_ecb_dec_16way)
908.global camellia_ecb_dec_16way
909.type camellia_ecb_dec_16way,@function;
910
911camellia_ecb_dec_16way:
912 /* input: 907 /* input:
913 * %rdi: ctx, CTX 908 * %rdi: ctx, CTX
914 * %rsi: dst (16 blocks) 909 * %rsi: dst (16 blocks)
@@ -934,12 +929,9 @@ camellia_ecb_dec_16way:
934 %xmm8, %rsi); 929 %xmm8, %rsi);
935 930
936 ret; 931 ret;
932ENDPROC(camellia_ecb_dec_16way)
937 933
938.align 8 934ENTRY(camellia_cbc_dec_16way)
939.global camellia_cbc_dec_16way
940.type camellia_cbc_dec_16way,@function;
941
942camellia_cbc_dec_16way:
943 /* input: 935 /* input:
944 * %rdi: ctx, CTX 936 * %rdi: ctx, CTX
945 * %rsi: dst (16 blocks) 937 * %rsi: dst (16 blocks)
@@ -986,6 +978,7 @@ camellia_cbc_dec_16way:
986 %xmm8, %rsi); 978 %xmm8, %rsi);
987 979
988 ret; 980 ret;
981ENDPROC(camellia_cbc_dec_16way)
989 982
990#define inc_le128(x, minus_one, tmp) \ 983#define inc_le128(x, minus_one, tmp) \
991 vpcmpeqq minus_one, x, tmp; \ 984 vpcmpeqq minus_one, x, tmp; \
@@ -993,11 +986,7 @@ camellia_cbc_dec_16way:
993 vpslldq $8, tmp, tmp; \ 986 vpslldq $8, tmp, tmp; \
994 vpsubq tmp, x, x; 987 vpsubq tmp, x, x;
995 988
996.align 8 989ENTRY(camellia_ctr_16way)
997.global camellia_ctr_16way
998.type camellia_ctr_16way,@function;
999
1000camellia_ctr_16way:
1001 /* input: 990 /* input:
1002 * %rdi: ctx, CTX 991 * %rdi: ctx, CTX
1003 * %rsi: dst (16 blocks) 992 * %rsi: dst (16 blocks)
@@ -1100,3 +1089,4 @@ camellia_ctr_16way:
1100 %xmm8, %rsi); 1089 %xmm8, %rsi);
1101 1090
1102 ret; 1091 ret;
1092ENDPROC(camellia_ctr_16way)
diff --git a/arch/x86/crypto/camellia-x86_64-asm_64.S b/arch/x86/crypto/camellia-x86_64-asm_64.S
index 0b3374335fdc..310319c601ed 100644
--- a/arch/x86/crypto/camellia-x86_64-asm_64.S
+++ b/arch/x86/crypto/camellia-x86_64-asm_64.S
@@ -20,6 +20,8 @@
20 * 20 *
21 */ 21 */
22 22
23#include <linux/linkage.h>
24
23.file "camellia-x86_64-asm_64.S" 25.file "camellia-x86_64-asm_64.S"
24.text 26.text
25 27
@@ -188,10 +190,7 @@
188 bswapq RAB0; \ 190 bswapq RAB0; \
189 movq RAB0, 4*2(RIO); 191 movq RAB0, 4*2(RIO);
190 192
191.global __camellia_enc_blk; 193ENTRY(__camellia_enc_blk)
192.type __camellia_enc_blk,@function;
193
194__camellia_enc_blk:
195 /* input: 194 /* input:
196 * %rdi: ctx, CTX 195 * %rdi: ctx, CTX
197 * %rsi: dst 196 * %rsi: dst
@@ -214,33 +213,31 @@ __camellia_enc_blk:
214 movl $24, RT1d; /* max */ 213 movl $24, RT1d; /* max */
215 214
216 cmpb $16, key_length(CTX); 215 cmpb $16, key_length(CTX);
217 je __enc_done; 216 je .L__enc_done;
218 217
219 enc_fls(24); 218 enc_fls(24);
220 enc_rounds(24); 219 enc_rounds(24);
221 movl $32, RT1d; /* max */ 220 movl $32, RT1d; /* max */
222 221
223__enc_done: 222.L__enc_done:
224 testb RXORbl, RXORbl; 223 testb RXORbl, RXORbl;
225 movq RDST, RIO; 224 movq RDST, RIO;
226 225
227 jnz __enc_xor; 226 jnz .L__enc_xor;
228 227
229 enc_outunpack(mov, RT1); 228 enc_outunpack(mov, RT1);
230 229
231 movq RRBP, %rbp; 230 movq RRBP, %rbp;
232 ret; 231 ret;
233 232
234__enc_xor: 233.L__enc_xor:
235 enc_outunpack(xor, RT1); 234 enc_outunpack(xor, RT1);
236 235
237 movq RRBP, %rbp; 236 movq RRBP, %rbp;
238 ret; 237 ret;
238ENDPROC(__camellia_enc_blk)
239 239
240.global camellia_dec_blk; 240ENTRY(camellia_dec_blk)
241.type camellia_dec_blk,@function;
242
243camellia_dec_blk:
244 /* input: 241 /* input:
245 * %rdi: ctx, CTX 242 * %rdi: ctx, CTX
246 * %rsi: dst 243 * %rsi: dst
@@ -258,12 +255,12 @@ camellia_dec_blk:
258 dec_inpack(RT2); 255 dec_inpack(RT2);
259 256
260 cmpb $24, RT2bl; 257 cmpb $24, RT2bl;
261 je __dec_rounds16; 258 je .L__dec_rounds16;
262 259
263 dec_rounds(24); 260 dec_rounds(24);
264 dec_fls(24); 261 dec_fls(24);
265 262
266__dec_rounds16: 263.L__dec_rounds16:
267 dec_rounds(16); 264 dec_rounds(16);
268 dec_fls(16); 265 dec_fls(16);
269 dec_rounds(8); 266 dec_rounds(8);
@@ -276,6 +273,7 @@ __dec_rounds16:
276 273
277 movq RRBP, %rbp; 274 movq RRBP, %rbp;
278 ret; 275 ret;
276ENDPROC(camellia_dec_blk)
279 277
280/********************************************************************** 278/**********************************************************************
281 2-way camellia 279 2-way camellia
@@ -426,10 +424,7 @@ __dec_rounds16:
426 bswapq RAB1; \ 424 bswapq RAB1; \
427 movq RAB1, 12*2(RIO); 425 movq RAB1, 12*2(RIO);
428 426
429.global __camellia_enc_blk_2way; 427ENTRY(__camellia_enc_blk_2way)
430.type __camellia_enc_blk_2way,@function;
431
432__camellia_enc_blk_2way:
433 /* input: 428 /* input:
434 * %rdi: ctx, CTX 429 * %rdi: ctx, CTX
435 * %rsi: dst 430 * %rsi: dst
@@ -453,16 +448,16 @@ __camellia_enc_blk_2way:
453 movl $24, RT2d; /* max */ 448 movl $24, RT2d; /* max */
454 449
455 cmpb $16, key_length(CTX); 450 cmpb $16, key_length(CTX);
456 je __enc2_done; 451 je .L__enc2_done;
457 452
458 enc_fls2(24); 453 enc_fls2(24);
459 enc_rounds2(24); 454 enc_rounds2(24);
460 movl $32, RT2d; /* max */ 455 movl $32, RT2d; /* max */
461 456
462__enc2_done: 457.L__enc2_done:
463 test RXORbl, RXORbl; 458 test RXORbl, RXORbl;
464 movq RDST, RIO; 459 movq RDST, RIO;
465 jnz __enc2_xor; 460 jnz .L__enc2_xor;
466 461
467 enc_outunpack2(mov, RT2); 462 enc_outunpack2(mov, RT2);
468 463
@@ -470,17 +465,15 @@ __enc2_done:
470 popq %rbx; 465 popq %rbx;
471 ret; 466 ret;
472 467
473__enc2_xor: 468.L__enc2_xor:
474 enc_outunpack2(xor, RT2); 469 enc_outunpack2(xor, RT2);
475 470
476 movq RRBP, %rbp; 471 movq RRBP, %rbp;
477 popq %rbx; 472 popq %rbx;
478 ret; 473 ret;
474ENDPROC(__camellia_enc_blk_2way)
479 475
480.global camellia_dec_blk_2way; 476ENTRY(camellia_dec_blk_2way)
481.type camellia_dec_blk_2way,@function;
482
483camellia_dec_blk_2way:
484 /* input: 477 /* input:
485 * %rdi: ctx, CTX 478 * %rdi: ctx, CTX
486 * %rsi: dst 479 * %rsi: dst
@@ -499,12 +492,12 @@ camellia_dec_blk_2way:
499 dec_inpack2(RT2); 492 dec_inpack2(RT2);
500 493
501 cmpb $24, RT2bl; 494 cmpb $24, RT2bl;
502 je __dec2_rounds16; 495 je .L__dec2_rounds16;
503 496
504 dec_rounds2(24); 497 dec_rounds2(24);
505 dec_fls2(24); 498 dec_fls2(24);
506 499
507__dec2_rounds16: 500.L__dec2_rounds16:
508 dec_rounds2(16); 501 dec_rounds2(16);
509 dec_fls2(16); 502 dec_fls2(16);
510 dec_rounds2(8); 503 dec_rounds2(8);
@@ -518,3 +511,4 @@ __dec2_rounds16:
518 movq RRBP, %rbp; 511 movq RRBP, %rbp;
519 movq RXOR, %rbx; 512 movq RXOR, %rbx;
520 ret; 513 ret;
514ENDPROC(camellia_dec_blk_2way)
diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
index 15b00ac7cbd3..c35fd5d6ecd2 100644
--- a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
@@ -23,6 +23,8 @@
23 * 23 *
24 */ 24 */
25 25
26#include <linux/linkage.h>
27
26.file "cast5-avx-x86_64-asm_64.S" 28.file "cast5-avx-x86_64-asm_64.S"
27 29
28.extern cast_s1 30.extern cast_s1
@@ -211,8 +213,6 @@
211.text 213.text
212 214
213.align 16 215.align 16
214.type __cast5_enc_blk16,@function;
215
216__cast5_enc_blk16: 216__cast5_enc_blk16:
217 /* input: 217 /* input:
218 * %rdi: ctx, CTX 218 * %rdi: ctx, CTX
@@ -263,14 +263,14 @@ __cast5_enc_blk16:
263 263
264 movzbl rr(CTX), %eax; 264 movzbl rr(CTX), %eax;
265 testl %eax, %eax; 265 testl %eax, %eax;
266 jnz __skip_enc; 266 jnz .L__skip_enc;
267 267
268 round(RL, RR, 12, 1); 268 round(RL, RR, 12, 1);
269 round(RR, RL, 13, 2); 269 round(RR, RL, 13, 2);
270 round(RL, RR, 14, 3); 270 round(RL, RR, 14, 3);
271 round(RR, RL, 15, 1); 271 round(RR, RL, 15, 1);
272 272
273__skip_enc: 273.L__skip_enc:
274 popq %rbx; 274 popq %rbx;
275 popq %rbp; 275 popq %rbp;
276 276
@@ -282,10 +282,9 @@ __skip_enc:
282 outunpack_blocks(RR4, RL4, RTMP, RX, RKM); 282 outunpack_blocks(RR4, RL4, RTMP, RX, RKM);
283 283
284 ret; 284 ret;
285ENDPROC(__cast5_enc_blk16)
285 286
286.align 16 287.align 16
287.type __cast5_dec_blk16,@function;
288
289__cast5_dec_blk16: 288__cast5_dec_blk16:
290 /* input: 289 /* input:
291 * %rdi: ctx, CTX 290 * %rdi: ctx, CTX
@@ -323,14 +322,14 @@ __cast5_dec_blk16:
323 322
324 movzbl rr(CTX), %eax; 323 movzbl rr(CTX), %eax;
325 testl %eax, %eax; 324 testl %eax, %eax;
326 jnz __skip_dec; 325 jnz .L__skip_dec;
327 326
328 round(RL, RR, 15, 1); 327 round(RL, RR, 15, 1);
329 round(RR, RL, 14, 3); 328 round(RR, RL, 14, 3);
330 round(RL, RR, 13, 2); 329 round(RL, RR, 13, 2);
331 round(RR, RL, 12, 1); 330 round(RR, RL, 12, 1);
332 331
333__dec_tail: 332.L__dec_tail:
334 round(RL, RR, 11, 3); 333 round(RL, RR, 11, 3);
335 round(RR, RL, 10, 2); 334 round(RR, RL, 10, 2);
336 round(RL, RR, 9, 1); 335 round(RL, RR, 9, 1);
@@ -355,15 +354,12 @@ __dec_tail:
355 354
356 ret; 355 ret;
357 356
358__skip_dec: 357.L__skip_dec:
359 vpsrldq $4, RKR, RKR; 358 vpsrldq $4, RKR, RKR;
360 jmp __dec_tail; 359 jmp .L__dec_tail;
360ENDPROC(__cast5_dec_blk16)
361 361
362.align 16 362ENTRY(cast5_ecb_enc_16way)
363.global cast5_ecb_enc_16way
364.type cast5_ecb_enc_16way,@function;
365
366cast5_ecb_enc_16way:
367 /* input: 363 /* input:
368 * %rdi: ctx, CTX 364 * %rdi: ctx, CTX
369 * %rsi: dst 365 * %rsi: dst
@@ -393,12 +389,9 @@ cast5_ecb_enc_16way:
393 vmovdqu RL4, (7*4*4)(%r11); 389 vmovdqu RL4, (7*4*4)(%r11);
394 390
395 ret; 391 ret;
392ENDPROC(cast5_ecb_enc_16way)
396 393
397.align 16 394ENTRY(cast5_ecb_dec_16way)
398.global cast5_ecb_dec_16way
399.type cast5_ecb_dec_16way,@function;
400
401cast5_ecb_dec_16way:
402 /* input: 395 /* input:
403 * %rdi: ctx, CTX 396 * %rdi: ctx, CTX
404 * %rsi: dst 397 * %rsi: dst
@@ -428,12 +421,9 @@ cast5_ecb_dec_16way:
428 vmovdqu RL4, (7*4*4)(%r11); 421 vmovdqu RL4, (7*4*4)(%r11);
429 422
430 ret; 423 ret;
424ENDPROC(cast5_ecb_dec_16way)
431 425
432.align 16 426ENTRY(cast5_cbc_dec_16way)
433.global cast5_cbc_dec_16way
434.type cast5_cbc_dec_16way,@function;
435
436cast5_cbc_dec_16way:
437 /* input: 427 /* input:
438 * %rdi: ctx, CTX 428 * %rdi: ctx, CTX
439 * %rsi: dst 429 * %rsi: dst
@@ -480,12 +470,9 @@ cast5_cbc_dec_16way:
480 popq %r12; 470 popq %r12;
481 471
482 ret; 472 ret;
473ENDPROC(cast5_cbc_dec_16way)
483 474
484.align 16 475ENTRY(cast5_ctr_16way)
485.global cast5_ctr_16way
486.type cast5_ctr_16way,@function;
487
488cast5_ctr_16way:
489 /* input: 476 /* input:
490 * %rdi: ctx, CTX 477 * %rdi: ctx, CTX
491 * %rsi: dst 478 * %rsi: dst
@@ -556,3 +543,4 @@ cast5_ctr_16way:
556 popq %r12; 543 popq %r12;
557 544
558 ret; 545 ret;
546ENDPROC(cast5_ctr_16way)
diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
index 2569d0da841f..f93b6105a0ce 100644
--- a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
@@ -23,6 +23,7 @@
23 * 23 *
24 */ 24 */
25 25
26#include <linux/linkage.h>
26#include "glue_helper-asm-avx.S" 27#include "glue_helper-asm-avx.S"
27 28
28.file "cast6-avx-x86_64-asm_64.S" 29.file "cast6-avx-x86_64-asm_64.S"
@@ -250,8 +251,6 @@
250.text 251.text
251 252
252.align 8 253.align 8
253.type __cast6_enc_blk8,@function;
254
255__cast6_enc_blk8: 254__cast6_enc_blk8:
256 /* input: 255 /* input:
257 * %rdi: ctx, CTX 256 * %rdi: ctx, CTX
@@ -295,10 +294,9 @@ __cast6_enc_blk8:
295 outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); 294 outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
296 295
297 ret; 296 ret;
297ENDPROC(__cast6_enc_blk8)
298 298
299.align 8 299.align 8
300.type __cast6_dec_blk8,@function;
301
302__cast6_dec_blk8: 300__cast6_dec_blk8:
303 /* input: 301 /* input:
304 * %rdi: ctx, CTX 302 * %rdi: ctx, CTX
@@ -341,12 +339,9 @@ __cast6_dec_blk8:
341 outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM); 339 outunpack_blocks(RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
342 340
343 ret; 341 ret;
342ENDPROC(__cast6_dec_blk8)
344 343
345.align 8 344ENTRY(cast6_ecb_enc_8way)
346.global cast6_ecb_enc_8way
347.type cast6_ecb_enc_8way,@function;
348
349cast6_ecb_enc_8way:
350 /* input: 345 /* input:
351 * %rdi: ctx, CTX 346 * %rdi: ctx, CTX
352 * %rsi: dst 347 * %rsi: dst
@@ -362,12 +357,9 @@ cast6_ecb_enc_8way:
362 store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 357 store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
363 358
364 ret; 359 ret;
360ENDPROC(cast6_ecb_enc_8way)
365 361
366.align 8 362ENTRY(cast6_ecb_dec_8way)
367.global cast6_ecb_dec_8way
368.type cast6_ecb_dec_8way,@function;
369
370cast6_ecb_dec_8way:
371 /* input: 363 /* input:
372 * %rdi: ctx, CTX 364 * %rdi: ctx, CTX
373 * %rsi: dst 365 * %rsi: dst
@@ -383,12 +375,9 @@ cast6_ecb_dec_8way:
383 store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 375 store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
384 376
385 ret; 377 ret;
378ENDPROC(cast6_ecb_dec_8way)
386 379
387.align 8 380ENTRY(cast6_cbc_dec_8way)
388.global cast6_cbc_dec_8way
389.type cast6_cbc_dec_8way,@function;
390
391cast6_cbc_dec_8way:
392 /* input: 381 /* input:
393 * %rdi: ctx, CTX 382 * %rdi: ctx, CTX
394 * %rsi: dst 383 * %rsi: dst
@@ -409,12 +398,9 @@ cast6_cbc_dec_8way:
409 popq %r12; 398 popq %r12;
410 399
411 ret; 400 ret;
401ENDPROC(cast6_cbc_dec_8way)
412 402
413.align 8 403ENTRY(cast6_ctr_8way)
414.global cast6_ctr_8way
415.type cast6_ctr_8way,@function;
416
417cast6_ctr_8way:
418 /* input: 404 /* input:
419 * %rdi: ctx, CTX 405 * %rdi: ctx, CTX
420 * %rsi: dst 406 * %rsi: dst
@@ -437,3 +423,4 @@ cast6_ctr_8way:
437 popq %r12; 423 popq %r12;
438 424
439 ret; 425 ret;
426ENDPROC(cast6_ctr_8way)
diff --git a/arch/x86/crypto/crc32-pclmul_asm.S b/arch/x86/crypto/crc32-pclmul_asm.S
new file mode 100644
index 000000000000..c8335014a044
--- /dev/null
+++ b/arch/x86/crypto/crc32-pclmul_asm.S
@@ -0,0 +1,246 @@
1/* GPL HEADER START
2 *
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 only,
7 * as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License version 2 for more details (a copy is included
13 * in the LICENSE file that accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License
16 * version 2 along with this program; If not, see http://www.gnu.org/licenses
17 *
18 * Please visit http://www.xyratex.com/contact if you need additional
19 * information or have any questions.
20 *
21 * GPL HEADER END
22 */
23
24/*
25 * Copyright 2012 Xyratex Technology Limited
26 *
27 * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
28 * calculation.
29 * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
30 * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
31 * at:
32 * http://www.intel.com/products/processor/manuals/
33 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
34 * Volume 2B: Instruction Set Reference, N-Z
35 *
36 * Authors: Gregory Prestas <Gregory_Prestas@us.xyratex.com>
37 * Alexander Boyko <Alexander_Boyko@xyratex.com>
38 */
39
40#include <linux/linkage.h>
41#include <asm/inst.h>
42
43
44.align 16
45/*
46 * [x4*128+32 mod P(x) << 32)]' << 1 = 0x154442bd4
47 * #define CONSTANT_R1 0x154442bd4LL
48 *
49 * [(x4*128-32 mod P(x) << 32)]' << 1 = 0x1c6e41596
50 * #define CONSTANT_R2 0x1c6e41596LL
51 */
52.Lconstant_R2R1:
53 .octa 0x00000001c6e415960000000154442bd4
54/*
55 * [(x128+32 mod P(x) << 32)]' << 1 = 0x1751997d0
56 * #define CONSTANT_R3 0x1751997d0LL
57 *
58 * [(x128-32 mod P(x) << 32)]' << 1 = 0x0ccaa009e
59 * #define CONSTANT_R4 0x0ccaa009eLL
60 */
61.Lconstant_R4R3:
62 .octa 0x00000000ccaa009e00000001751997d0
63/*
64 * [(x64 mod P(x) << 32)]' << 1 = 0x163cd6124
65 * #define CONSTANT_R5 0x163cd6124LL
66 */
67.Lconstant_R5:
68 .octa 0x00000000000000000000000163cd6124
69.Lconstant_mask32:
70 .octa 0x000000000000000000000000FFFFFFFF
71/*
72 * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
73 *
74 * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))` = 0x1F7011641LL
75 * #define CONSTANT_RU 0x1F7011641LL
76 */
77.Lconstant_RUpoly:
78 .octa 0x00000001F701164100000001DB710641
79
80#define CONSTANT %xmm0
81
82#ifdef __x86_64__
83#define BUF %rdi
84#define LEN %rsi
85#define CRC %edx
86#else
87#define BUF %eax
88#define LEN %edx
89#define CRC %ecx
90#endif
91
92
93
94.text
95/**
96 * Calculate crc32
97 * BUF - buffer (16 bytes aligned)
98 * LEN - sizeof buffer (16 bytes aligned), LEN should be grater than 63
99 * CRC - initial crc32
100 * return %eax crc32
101 * uint crc32_pclmul_le_16(unsigned char const *buffer,
102 * size_t len, uint crc32)
103 */
104.globl crc32_pclmul_le_16
105.align 4, 0x90
106crc32_pclmul_le_16:/* buffer and buffer size are 16 bytes aligned */
107 movdqa (BUF), %xmm1
108 movdqa 0x10(BUF), %xmm2
109 movdqa 0x20(BUF), %xmm3
110 movdqa 0x30(BUF), %xmm4
111 movd CRC, CONSTANT
112 pxor CONSTANT, %xmm1
113 sub $0x40, LEN
114 add $0x40, BUF
115#ifndef __x86_64__
116 /* This is for position independent code(-fPIC) support for 32bit */
117 call delta
118delta:
119 pop %ecx
120#endif
121 cmp $0x40, LEN
122 jb less_64
123
124#ifdef __x86_64__
125 movdqa .Lconstant_R2R1(%rip), CONSTANT
126#else
127 movdqa .Lconstant_R2R1 - delta(%ecx), CONSTANT
128#endif
129
130loop_64:/* 64 bytes Full cache line folding */
131 prefetchnta 0x40(BUF)
132 movdqa %xmm1, %xmm5
133 movdqa %xmm2, %xmm6
134 movdqa %xmm3, %xmm7
135#ifdef __x86_64__
136 movdqa %xmm4, %xmm8
137#endif
138 PCLMULQDQ 00, CONSTANT, %xmm1
139 PCLMULQDQ 00, CONSTANT, %xmm2
140 PCLMULQDQ 00, CONSTANT, %xmm3
141#ifdef __x86_64__
142 PCLMULQDQ 00, CONSTANT, %xmm4
143#endif
144 PCLMULQDQ 0x11, CONSTANT, %xmm5
145 PCLMULQDQ 0x11, CONSTANT, %xmm6
146 PCLMULQDQ 0x11, CONSTANT, %xmm7
147#ifdef __x86_64__
148 PCLMULQDQ 0x11, CONSTANT, %xmm8
149#endif
150 pxor %xmm5, %xmm1
151 pxor %xmm6, %xmm2
152 pxor %xmm7, %xmm3
153#ifdef __x86_64__
154 pxor %xmm8, %xmm4
155#else
156 /* xmm8 unsupported for x32 */
157 movdqa %xmm4, %xmm5
158 PCLMULQDQ 00, CONSTANT, %xmm4
159 PCLMULQDQ 0x11, CONSTANT, %xmm5
160 pxor %xmm5, %xmm4
161#endif
162
163 pxor (BUF), %xmm1
164 pxor 0x10(BUF), %xmm2
165 pxor 0x20(BUF), %xmm3
166 pxor 0x30(BUF), %xmm4
167
168 sub $0x40, LEN
169 add $0x40, BUF
170 cmp $0x40, LEN
171 jge loop_64
172less_64:/* Folding cache line into 128bit */
173#ifdef __x86_64__
174 movdqa .Lconstant_R4R3(%rip), CONSTANT
175#else
176 movdqa .Lconstant_R4R3 - delta(%ecx), CONSTANT
177#endif
178 prefetchnta (BUF)
179
180 movdqa %xmm1, %xmm5
181 PCLMULQDQ 0x00, CONSTANT, %xmm1
182 PCLMULQDQ 0x11, CONSTANT, %xmm5
183 pxor %xmm5, %xmm1
184 pxor %xmm2, %xmm1
185
186 movdqa %xmm1, %xmm5
187 PCLMULQDQ 0x00, CONSTANT, %xmm1
188 PCLMULQDQ 0x11, CONSTANT, %xmm5
189 pxor %xmm5, %xmm1
190 pxor %xmm3, %xmm1
191
192 movdqa %xmm1, %xmm5
193 PCLMULQDQ 0x00, CONSTANT, %xmm1
194 PCLMULQDQ 0x11, CONSTANT, %xmm5
195 pxor %xmm5, %xmm1
196 pxor %xmm4, %xmm1
197
198 cmp $0x10, LEN
199 jb fold_64
200loop_16:/* Folding rest buffer into 128bit */
201 movdqa %xmm1, %xmm5
202 PCLMULQDQ 0x00, CONSTANT, %xmm1
203 PCLMULQDQ 0x11, CONSTANT, %xmm5
204 pxor %xmm5, %xmm1
205 pxor (BUF), %xmm1
206 sub $0x10, LEN
207 add $0x10, BUF
208 cmp $0x10, LEN
209 jge loop_16
210
211fold_64:
212 /* perform the last 64 bit fold, also adds 32 zeroes
213 * to the input stream */
214 PCLMULQDQ 0x01, %xmm1, CONSTANT /* R4 * xmm1.low */
215 psrldq $0x08, %xmm1
216 pxor CONSTANT, %xmm1
217
218 /* final 32-bit fold */
219 movdqa %xmm1, %xmm2
220#ifdef __x86_64__
221 movdqa .Lconstant_R5(%rip), CONSTANT
222 movdqa .Lconstant_mask32(%rip), %xmm3
223#else
224 movdqa .Lconstant_R5 - delta(%ecx), CONSTANT
225 movdqa .Lconstant_mask32 - delta(%ecx), %xmm3
226#endif
227 psrldq $0x04, %xmm2
228 pand %xmm3, %xmm1
229 PCLMULQDQ 0x00, CONSTANT, %xmm1
230 pxor %xmm2, %xmm1
231
232 /* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
233#ifdef __x86_64__
234 movdqa .Lconstant_RUpoly(%rip), CONSTANT
235#else
236 movdqa .Lconstant_RUpoly - delta(%ecx), CONSTANT
237#endif
238 movdqa %xmm1, %xmm2
239 pand %xmm3, %xmm1
240 PCLMULQDQ 0x10, CONSTANT, %xmm1
241 pand %xmm3, %xmm1
242 PCLMULQDQ 0x00, CONSTANT, %xmm1
243 pxor %xmm2, %xmm1
244 pextrd $0x01, %xmm1, %eax
245
246 ret
diff --git a/arch/x86/crypto/crc32-pclmul_glue.c b/arch/x86/crypto/crc32-pclmul_glue.c
new file mode 100644
index 000000000000..9d014a74ef96
--- /dev/null
+++ b/arch/x86/crypto/crc32-pclmul_glue.c
@@ -0,0 +1,201 @@
1/* GPL HEADER START
2 *
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 only,
7 * as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License version 2 for more details (a copy is included
13 * in the LICENSE file that accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License
16 * version 2 along with this program; If not, see http://www.gnu.org/licenses
17 *
18 * Please visit http://www.xyratex.com/contact if you need additional
19 * information or have any questions.
20 *
21 * GPL HEADER END
22 */
23
24/*
25 * Copyright 2012 Xyratex Technology Limited
26 *
27 * Wrappers for kernel crypto shash api to pclmulqdq crc32 imlementation.
28 */
29#include <linux/init.h>
30#include <linux/module.h>
31#include <linux/string.h>
32#include <linux/kernel.h>
33#include <linux/crc32.h>
34#include <crypto/internal/hash.h>
35
36#include <asm/cpufeature.h>
37#include <asm/cpu_device_id.h>
38#include <asm/i387.h>
39
40#define CHKSUM_BLOCK_SIZE 1
41#define CHKSUM_DIGEST_SIZE 4
42
43#define PCLMUL_MIN_LEN 64L /* minimum size of buffer
44 * for crc32_pclmul_le_16 */
45#define SCALE_F 16L /* size of xmm register */
46#define SCALE_F_MASK (SCALE_F - 1)
47
48u32 crc32_pclmul_le_16(unsigned char const *buffer, size_t len, u32 crc32);
49
50static u32 __attribute__((pure))
51 crc32_pclmul_le(u32 crc, unsigned char const *p, size_t len)
52{
53 unsigned int iquotient;
54 unsigned int iremainder;
55 unsigned int prealign;
56
57 if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !irq_fpu_usable())
58 return crc32_le(crc, p, len);
59
60 if ((long)p & SCALE_F_MASK) {
61 /* align p to 16 byte */
62 prealign = SCALE_F - ((long)p & SCALE_F_MASK);
63
64 crc = crc32_le(crc, p, prealign);
65 len -= prealign;
66 p = (unsigned char *)(((unsigned long)p + SCALE_F_MASK) &
67 ~SCALE_F_MASK);
68 }
69 iquotient = len & (~SCALE_F_MASK);
70 iremainder = len & SCALE_F_MASK;
71
72 kernel_fpu_begin();
73 crc = crc32_pclmul_le_16(p, iquotient, crc);
74 kernel_fpu_end();
75
76 if (iremainder)
77 crc = crc32_le(crc, p + iquotient, iremainder);
78
79 return crc;
80}
81
82static int crc32_pclmul_cra_init(struct crypto_tfm *tfm)
83{
84 u32 *key = crypto_tfm_ctx(tfm);
85
86 *key = 0;
87
88 return 0;
89}
90
91static int crc32_pclmul_setkey(struct crypto_shash *hash, const u8 *key,
92 unsigned int keylen)
93{
94 u32 *mctx = crypto_shash_ctx(hash);
95
96 if (keylen != sizeof(u32)) {
97 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
98 return -EINVAL;
99 }
100 *mctx = le32_to_cpup((__le32 *)key);
101 return 0;
102}
103
104static int crc32_pclmul_init(struct shash_desc *desc)
105{
106 u32 *mctx = crypto_shash_ctx(desc->tfm);
107 u32 *crcp = shash_desc_ctx(desc);
108
109 *crcp = *mctx;
110
111 return 0;
112}
113
114static int crc32_pclmul_update(struct shash_desc *desc, const u8 *data,
115 unsigned int len)
116{
117 u32 *crcp = shash_desc_ctx(desc);
118
119 *crcp = crc32_pclmul_le(*crcp, data, len);
120 return 0;
121}
122
123/* No final XOR 0xFFFFFFFF, like crc32_le */
124static int __crc32_pclmul_finup(u32 *crcp, const u8 *data, unsigned int len,
125 u8 *out)
126{
127 *(__le32 *)out = cpu_to_le32(crc32_pclmul_le(*crcp, data, len));
128 return 0;
129}
130
131static int crc32_pclmul_finup(struct shash_desc *desc, const u8 *data,
132 unsigned int len, u8 *out)
133{
134 return __crc32_pclmul_finup(shash_desc_ctx(desc), data, len, out);
135}
136
137static int crc32_pclmul_final(struct shash_desc *desc, u8 *out)
138{
139 u32 *crcp = shash_desc_ctx(desc);
140
141 *(__le32 *)out = cpu_to_le32p(crcp);
142 return 0;
143}
144
145static int crc32_pclmul_digest(struct shash_desc *desc, const u8 *data,
146 unsigned int len, u8 *out)
147{
148 return __crc32_pclmul_finup(crypto_shash_ctx(desc->tfm), data, len,
149 out);
150}
151
152static struct shash_alg alg = {
153 .setkey = crc32_pclmul_setkey,
154 .init = crc32_pclmul_init,
155 .update = crc32_pclmul_update,
156 .final = crc32_pclmul_final,
157 .finup = crc32_pclmul_finup,
158 .digest = crc32_pclmul_digest,
159 .descsize = sizeof(u32),
160 .digestsize = CHKSUM_DIGEST_SIZE,
161 .base = {
162 .cra_name = "crc32",
163 .cra_driver_name = "crc32-pclmul",
164 .cra_priority = 200,
165 .cra_blocksize = CHKSUM_BLOCK_SIZE,
166 .cra_ctxsize = sizeof(u32),
167 .cra_module = THIS_MODULE,
168 .cra_init = crc32_pclmul_cra_init,
169 }
170};
171
172static const struct x86_cpu_id crc32pclmul_cpu_id[] = {
173 X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ),
174 {}
175};
176MODULE_DEVICE_TABLE(x86cpu, crc32pclmul_cpu_id);
177
178
179static int __init crc32_pclmul_mod_init(void)
180{
181
182 if (!x86_match_cpu(crc32pclmul_cpu_id)) {
183 pr_info("PCLMULQDQ-NI instructions are not detected.\n");
184 return -ENODEV;
185 }
186 return crypto_register_shash(&alg);
187}
188
189static void __exit crc32_pclmul_mod_fini(void)
190{
191 crypto_unregister_shash(&alg);
192}
193
194module_init(crc32_pclmul_mod_init);
195module_exit(crc32_pclmul_mod_fini);
196
197MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>");
198MODULE_LICENSE("GPL");
199
200MODULE_ALIAS("crc32");
201MODULE_ALIAS("crc32-pclmul");
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 93c6d39237ac..cf1a7ec4cc3a 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -42,6 +42,8 @@
42 * SOFTWARE. 42 * SOFTWARE.
43 */ 43 */
44 44
45#include <linux/linkage.h>
46
45## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction 47## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction
46 48
47.macro LABEL prefix n 49.macro LABEL prefix n
@@ -68,8 +70,7 @@
68 70
69# unsigned int crc_pcl(u8 *buffer, int len, unsigned int crc_init); 71# unsigned int crc_pcl(u8 *buffer, int len, unsigned int crc_init);
70 72
71.global crc_pcl 73ENTRY(crc_pcl)
72crc_pcl:
73#define bufp %rdi 74#define bufp %rdi
74#define bufp_dw %edi 75#define bufp_dw %edi
75#define bufp_w %di 76#define bufp_w %di
@@ -323,6 +324,9 @@ JMPTBL_ENTRY %i
323.noaltmacro 324.noaltmacro
324 i=i+1 325 i=i+1
325.endr 326.endr
327
328ENDPROC(crc_pcl)
329
326 ################################################################ 330 ################################################################
327 ## PCLMULQDQ tables 331 ## PCLMULQDQ tables
328 ## Table is 128 entries x 2 quad words each 332 ## Table is 128 entries x 2 quad words each
diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/arch/x86/crypto/ghash-clmulni-intel_asm.S
index 1eb7f90cb7b9..586f41aac361 100644
--- a/arch/x86/crypto/ghash-clmulni-intel_asm.S
+++ b/arch/x86/crypto/ghash-clmulni-intel_asm.S
@@ -94,6 +94,7 @@ __clmul_gf128mul_ble:
94 pxor T2, T1 94 pxor T2, T1
95 pxor T1, DATA 95 pxor T1, DATA
96 ret 96 ret
97ENDPROC(__clmul_gf128mul_ble)
97 98
98/* void clmul_ghash_mul(char *dst, const be128 *shash) */ 99/* void clmul_ghash_mul(char *dst, const be128 *shash) */
99ENTRY(clmul_ghash_mul) 100ENTRY(clmul_ghash_mul)
@@ -105,6 +106,7 @@ ENTRY(clmul_ghash_mul)
105 PSHUFB_XMM BSWAP DATA 106 PSHUFB_XMM BSWAP DATA
106 movups DATA, (%rdi) 107 movups DATA, (%rdi)
107 ret 108 ret
109ENDPROC(clmul_ghash_mul)
108 110
109/* 111/*
110 * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen, 112 * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
@@ -131,6 +133,7 @@ ENTRY(clmul_ghash_update)
131 movups DATA, (%rdi) 133 movups DATA, (%rdi)
132.Lupdate_just_ret: 134.Lupdate_just_ret:
133 ret 135 ret
136ENDPROC(clmul_ghash_update)
134 137
135/* 138/*
136 * void clmul_ghash_setkey(be128 *shash, const u8 *key); 139 * void clmul_ghash_setkey(be128 *shash, const u8 *key);
@@ -155,3 +158,4 @@ ENTRY(clmul_ghash_setkey)
155 pxor %xmm1, %xmm0 158 pxor %xmm1, %xmm0
156 movups %xmm0, (%rdi) 159 movups %xmm0, (%rdi)
157 ret 160 ret
161ENDPROC(clmul_ghash_setkey)
diff --git a/arch/x86/crypto/salsa20-i586-asm_32.S b/arch/x86/crypto/salsa20-i586-asm_32.S
index 72eb306680b2..329452b8f794 100644
--- a/arch/x86/crypto/salsa20-i586-asm_32.S
+++ b/arch/x86/crypto/salsa20-i586-asm_32.S
@@ -2,11 +2,12 @@
2# D. J. Bernstein 2# D. J. Bernstein
3# Public domain. 3# Public domain.
4 4
5# enter ECRYPT_encrypt_bytes 5#include <linux/linkage.h>
6
6.text 7.text
7.p2align 5 8
8.globl ECRYPT_encrypt_bytes 9# enter salsa20_encrypt_bytes
9ECRYPT_encrypt_bytes: 10ENTRY(salsa20_encrypt_bytes)
10 mov %esp,%eax 11 mov %esp,%eax
11 and $31,%eax 12 and $31,%eax
12 add $256,%eax 13 add $256,%eax
@@ -933,11 +934,10 @@ ECRYPT_encrypt_bytes:
933 add $64,%esi 934 add $64,%esi
934 # goto bytesatleast1 935 # goto bytesatleast1
935 jmp ._bytesatleast1 936 jmp ._bytesatleast1
936# enter ECRYPT_keysetup 937ENDPROC(salsa20_encrypt_bytes)
937.text 938
938.p2align 5 939# enter salsa20_keysetup
939.globl ECRYPT_keysetup 940ENTRY(salsa20_keysetup)
940ECRYPT_keysetup:
941 mov %esp,%eax 941 mov %esp,%eax
942 and $31,%eax 942 and $31,%eax
943 add $256,%eax 943 add $256,%eax
@@ -1060,11 +1060,10 @@ ECRYPT_keysetup:
1060 # leave 1060 # leave
1061 add %eax,%esp 1061 add %eax,%esp
1062 ret 1062 ret
1063# enter ECRYPT_ivsetup 1063ENDPROC(salsa20_keysetup)
1064.text 1064
1065.p2align 5 1065# enter salsa20_ivsetup
1066.globl ECRYPT_ivsetup 1066ENTRY(salsa20_ivsetup)
1067ECRYPT_ivsetup:
1068 mov %esp,%eax 1067 mov %esp,%eax
1069 and $31,%eax 1068 and $31,%eax
1070 add $256,%eax 1069 add $256,%eax
@@ -1112,3 +1111,4 @@ ECRYPT_ivsetup:
1112 # leave 1111 # leave
1113 add %eax,%esp 1112 add %eax,%esp
1114 ret 1113 ret
1114ENDPROC(salsa20_ivsetup)
diff --git a/arch/x86/crypto/salsa20-x86_64-asm_64.S b/arch/x86/crypto/salsa20-x86_64-asm_64.S
index 6214a9b09706..9279e0b2d60e 100644
--- a/arch/x86/crypto/salsa20-x86_64-asm_64.S
+++ b/arch/x86/crypto/salsa20-x86_64-asm_64.S
@@ -1,8 +1,7 @@
1# enter ECRYPT_encrypt_bytes 1#include <linux/linkage.h>
2.text 2
3.p2align 5 3# enter salsa20_encrypt_bytes
4.globl ECRYPT_encrypt_bytes 4ENTRY(salsa20_encrypt_bytes)
5ECRYPT_encrypt_bytes:
6 mov %rsp,%r11 5 mov %rsp,%r11
7 and $31,%r11 6 and $31,%r11
8 add $256,%r11 7 add $256,%r11
@@ -802,11 +801,10 @@ ECRYPT_encrypt_bytes:
802 # comment:fp stack unchanged by jump 801 # comment:fp stack unchanged by jump
803 # goto bytesatleast1 802 # goto bytesatleast1
804 jmp ._bytesatleast1 803 jmp ._bytesatleast1
805# enter ECRYPT_keysetup 804ENDPROC(salsa20_encrypt_bytes)
806.text 805
807.p2align 5 806# enter salsa20_keysetup
808.globl ECRYPT_keysetup 807ENTRY(salsa20_keysetup)
809ECRYPT_keysetup:
810 mov %rsp,%r11 808 mov %rsp,%r11
811 and $31,%r11 809 and $31,%r11
812 add $256,%r11 810 add $256,%r11
@@ -892,11 +890,10 @@ ECRYPT_keysetup:
892 mov %rdi,%rax 890 mov %rdi,%rax
893 mov %rsi,%rdx 891 mov %rsi,%rdx
894 ret 892 ret
895# enter ECRYPT_ivsetup 893ENDPROC(salsa20_keysetup)
896.text 894
897.p2align 5 895# enter salsa20_ivsetup
898.globl ECRYPT_ivsetup 896ENTRY(salsa20_ivsetup)
899ECRYPT_ivsetup:
900 mov %rsp,%r11 897 mov %rsp,%r11
901 and $31,%r11 898 and $31,%r11
902 add $256,%r11 899 add $256,%r11
@@ -918,3 +915,4 @@ ECRYPT_ivsetup:
918 mov %rdi,%rax 915 mov %rdi,%rax
919 mov %rsi,%rdx 916 mov %rsi,%rdx
920 ret 917 ret
918ENDPROC(salsa20_ivsetup)
diff --git a/arch/x86/crypto/salsa20_glue.c b/arch/x86/crypto/salsa20_glue.c
index a3a3c0205c16..5e8e67739bb5 100644
--- a/arch/x86/crypto/salsa20_glue.c
+++ b/arch/x86/crypto/salsa20_glue.c
@@ -26,11 +26,6 @@
26#define SALSA20_MIN_KEY_SIZE 16U 26#define SALSA20_MIN_KEY_SIZE 16U
27#define SALSA20_MAX_KEY_SIZE 32U 27#define SALSA20_MAX_KEY_SIZE 32U
28 28
29// use the ECRYPT_* function names
30#define salsa20_keysetup ECRYPT_keysetup
31#define salsa20_ivsetup ECRYPT_ivsetup
32#define salsa20_encrypt_bytes ECRYPT_encrypt_bytes
33
34struct salsa20_ctx 29struct salsa20_ctx
35{ 30{
36 u32 input[16]; 31 u32 input[16];
diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
index 02b0e9fe997c..43c938612b74 100644
--- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
@@ -24,6 +24,7 @@
24 * 24 *
25 */ 25 */
26 26
27#include <linux/linkage.h>
27#include "glue_helper-asm-avx.S" 28#include "glue_helper-asm-avx.S"
28 29
29.file "serpent-avx-x86_64-asm_64.S" 30.file "serpent-avx-x86_64-asm_64.S"
@@ -566,8 +567,6 @@
566 transpose_4x4(x0, x1, x2, x3, t0, t1, t2) 567 transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
567 568
568.align 8 569.align 8
569.type __serpent_enc_blk8_avx,@function;
570
571__serpent_enc_blk8_avx: 570__serpent_enc_blk8_avx:
572 /* input: 571 /* input:
573 * %rdi: ctx, CTX 572 * %rdi: ctx, CTX
@@ -619,10 +618,9 @@ __serpent_enc_blk8_avx:
619 write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2); 618 write_blocks(RA2, RB2, RC2, RD2, RK0, RK1, RK2);
620 619
621 ret; 620 ret;
621ENDPROC(__serpent_enc_blk8_avx)
622 622
623.align 8 623.align 8
624.type __serpent_dec_blk8_avx,@function;
625
626__serpent_dec_blk8_avx: 624__serpent_dec_blk8_avx:
627 /* input: 625 /* input:
628 * %rdi: ctx, CTX 626 * %rdi: ctx, CTX
@@ -674,12 +672,9 @@ __serpent_dec_blk8_avx:
674 write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2); 672 write_blocks(RC2, RD2, RB2, RE2, RK0, RK1, RK2);
675 673
676 ret; 674 ret;
675ENDPROC(__serpent_dec_blk8_avx)
677 676
678.align 8 677ENTRY(serpent_ecb_enc_8way_avx)
679.global serpent_ecb_enc_8way_avx
680.type serpent_ecb_enc_8way_avx,@function;
681
682serpent_ecb_enc_8way_avx:
683 /* input: 678 /* input:
684 * %rdi: ctx, CTX 679 * %rdi: ctx, CTX
685 * %rsi: dst 680 * %rsi: dst
@@ -693,12 +688,9 @@ serpent_ecb_enc_8way_avx:
693 store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 688 store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
694 689
695 ret; 690 ret;
691ENDPROC(serpent_ecb_enc_8way_avx)
696 692
697.align 8 693ENTRY(serpent_ecb_dec_8way_avx)
698.global serpent_ecb_dec_8way_avx
699.type serpent_ecb_dec_8way_avx,@function;
700
701serpent_ecb_dec_8way_avx:
702 /* input: 694 /* input:
703 * %rdi: ctx, CTX 695 * %rdi: ctx, CTX
704 * %rsi: dst 696 * %rsi: dst
@@ -712,12 +704,9 @@ serpent_ecb_dec_8way_avx:
712 store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); 704 store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
713 705
714 ret; 706 ret;
707ENDPROC(serpent_ecb_dec_8way_avx)
715 708
716.align 8 709ENTRY(serpent_cbc_dec_8way_avx)
717.global serpent_cbc_dec_8way_avx
718.type serpent_cbc_dec_8way_avx,@function;
719
720serpent_cbc_dec_8way_avx:
721 /* input: 710 /* input:
722 * %rdi: ctx, CTX 711 * %rdi: ctx, CTX
723 * %rsi: dst 712 * %rsi: dst
@@ -731,12 +720,9 @@ serpent_cbc_dec_8way_avx:
731 store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); 720 store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
732 721
733 ret; 722 ret;
723ENDPROC(serpent_cbc_dec_8way_avx)
734 724
735.align 8 725ENTRY(serpent_ctr_8way_avx)
736.global serpent_ctr_8way_avx
737.type serpent_ctr_8way_avx,@function;
738
739serpent_ctr_8way_avx:
740 /* input: 726 /* input:
741 * %rdi: ctx, CTX 727 * %rdi: ctx, CTX
742 * %rsi: dst 728 * %rsi: dst
@@ -752,3 +738,4 @@ serpent_ctr_8way_avx:
752 store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 738 store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
753 739
754 ret; 740 ret;
741ENDPROC(serpent_ctr_8way_avx)
diff --git a/arch/x86/crypto/serpent-sse2-i586-asm_32.S b/arch/x86/crypto/serpent-sse2-i586-asm_32.S
index c00053d42f99..d348f1553a79 100644
--- a/arch/x86/crypto/serpent-sse2-i586-asm_32.S
+++ b/arch/x86/crypto/serpent-sse2-i586-asm_32.S
@@ -24,6 +24,8 @@
24 * 24 *
25 */ 25 */
26 26
27#include <linux/linkage.h>
28
27.file "serpent-sse2-i586-asm_32.S" 29.file "serpent-sse2-i586-asm_32.S"
28.text 30.text
29 31
@@ -510,11 +512,7 @@
510 pxor t0, x3; \ 512 pxor t0, x3; \
511 movdqu x3, (3*4*4)(out); 513 movdqu x3, (3*4*4)(out);
512 514
513.align 8 515ENTRY(__serpent_enc_blk_4way)
514.global __serpent_enc_blk_4way
515.type __serpent_enc_blk_4way,@function;
516
517__serpent_enc_blk_4way:
518 /* input: 516 /* input:
519 * arg_ctx(%esp): ctx, CTX 517 * arg_ctx(%esp): ctx, CTX
520 * arg_dst(%esp): dst 518 * arg_dst(%esp): dst
@@ -566,22 +564,19 @@ __serpent_enc_blk_4way:
566 movl arg_dst(%esp), %eax; 564 movl arg_dst(%esp), %eax;
567 565
568 cmpb $0, arg_xor(%esp); 566 cmpb $0, arg_xor(%esp);
569 jnz __enc_xor4; 567 jnz .L__enc_xor4;
570 568
571 write_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE); 569 write_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE);
572 570
573 ret; 571 ret;
574 572
575__enc_xor4: 573.L__enc_xor4:
576 xor_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE); 574 xor_blocks(%eax, RA, RB, RC, RD, RT0, RT1, RE);
577 575
578 ret; 576 ret;
577ENDPROC(__serpent_enc_blk_4way)
579 578
580.align 8 579ENTRY(serpent_dec_blk_4way)
581.global serpent_dec_blk_4way
582.type serpent_dec_blk_4way,@function;
583
584serpent_dec_blk_4way:
585 /* input: 580 /* input:
586 * arg_ctx(%esp): ctx, CTX 581 * arg_ctx(%esp): ctx, CTX
587 * arg_dst(%esp): dst 582 * arg_dst(%esp): dst
@@ -633,3 +628,4 @@ serpent_dec_blk_4way:
633 write_blocks(%eax, RC, RD, RB, RE, RT0, RT1, RA); 628 write_blocks(%eax, RC, RD, RB, RE, RT0, RT1, RA);
634 629
635 ret; 630 ret;
631ENDPROC(serpent_dec_blk_4way)
diff --git a/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S b/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S
index 3ee1ff04d3e9..acc066c7c6b2 100644
--- a/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S
+++ b/arch/x86/crypto/serpent-sse2-x86_64-asm_64.S
@@ -24,6 +24,8 @@
24 * 24 *
25 */ 25 */
26 26
27#include <linux/linkage.h>
28
27.file "serpent-sse2-x86_64-asm_64.S" 29.file "serpent-sse2-x86_64-asm_64.S"
28.text 30.text
29 31
@@ -632,11 +634,7 @@
632 pxor t0, x3; \ 634 pxor t0, x3; \
633 movdqu x3, (3*4*4)(out); 635 movdqu x3, (3*4*4)(out);
634 636
635.align 8 637ENTRY(__serpent_enc_blk_8way)
636.global __serpent_enc_blk_8way
637.type __serpent_enc_blk_8way,@function;
638
639__serpent_enc_blk_8way:
640 /* input: 638 /* input:
641 * %rdi: ctx, CTX 639 * %rdi: ctx, CTX
642 * %rsi: dst 640 * %rsi: dst
@@ -687,24 +685,21 @@ __serpent_enc_blk_8way:
687 leaq (4*4*4)(%rsi), %rax; 685 leaq (4*4*4)(%rsi), %rax;
688 686
689 testb %cl, %cl; 687 testb %cl, %cl;
690 jnz __enc_xor8; 688 jnz .L__enc_xor8;
691 689
692 write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); 690 write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
693 write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); 691 write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
694 692
695 ret; 693 ret;
696 694
697__enc_xor8: 695.L__enc_xor8:
698 xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); 696 xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
699 xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); 697 xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
700 698
701 ret; 699 ret;
700ENDPROC(__serpent_enc_blk_8way)
702 701
703.align 8 702ENTRY(serpent_dec_blk_8way)
704.global serpent_dec_blk_8way
705.type serpent_dec_blk_8way,@function;
706
707serpent_dec_blk_8way:
708 /* input: 703 /* input:
709 * %rdi: ctx, CTX 704 * %rdi: ctx, CTX
710 * %rsi: dst 705 * %rsi: dst
@@ -756,3 +751,4 @@ serpent_dec_blk_8way:
756 write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2); 751 write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2);
757 752
758 ret; 753 ret;
754ENDPROC(serpent_dec_blk_8way)
diff --git a/arch/x86/crypto/sha1_ssse3_asm.S b/arch/x86/crypto/sha1_ssse3_asm.S
index 49d6987a73d9..a4109506a5e8 100644
--- a/arch/x86/crypto/sha1_ssse3_asm.S
+++ b/arch/x86/crypto/sha1_ssse3_asm.S
@@ -28,6 +28,8 @@
28 * (at your option) any later version. 28 * (at your option) any later version.
29 */ 29 */
30 30
31#include <linux/linkage.h>
32
31#define CTX %rdi // arg1 33#define CTX %rdi // arg1
32#define BUF %rsi // arg2 34#define BUF %rsi // arg2
33#define CNT %rdx // arg3 35#define CNT %rdx // arg3
@@ -69,10 +71,8 @@
69 * param: function's name 71 * param: function's name
70 */ 72 */
71.macro SHA1_VECTOR_ASM name 73.macro SHA1_VECTOR_ASM name
72 .global \name 74 ENTRY(\name)
73 .type \name, @function 75
74 .align 32
75\name:
76 push %rbx 76 push %rbx
77 push %rbp 77 push %rbp
78 push %r12 78 push %r12
@@ -106,7 +106,7 @@
106 pop %rbx 106 pop %rbx
107 ret 107 ret
108 108
109 .size \name, .-\name 109 ENDPROC(\name)
110.endm 110.endm
111 111
112/* 112/*
diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
index ebac16bfa830..8d3e113b2c95 100644
--- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
@@ -23,6 +23,7 @@
23 * 23 *
24 */ 24 */
25 25
26#include <linux/linkage.h>
26#include "glue_helper-asm-avx.S" 27#include "glue_helper-asm-avx.S"
27 28
28.file "twofish-avx-x86_64-asm_64.S" 29.file "twofish-avx-x86_64-asm_64.S"
@@ -243,8 +244,6 @@
243 vpxor x3, wkey, x3; 244 vpxor x3, wkey, x3;
244 245
245.align 8 246.align 8
246.type __twofish_enc_blk8,@function;
247
248__twofish_enc_blk8: 247__twofish_enc_blk8:
249 /* input: 248 /* input:
250 * %rdi: ctx, CTX 249 * %rdi: ctx, CTX
@@ -284,10 +283,9 @@ __twofish_enc_blk8:
284 outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); 283 outunpack_blocks(RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
285 284
286 ret; 285 ret;
286ENDPROC(__twofish_enc_blk8)
287 287
288.align 8 288.align 8
289.type __twofish_dec_blk8,@function;
290
291__twofish_dec_blk8: 289__twofish_dec_blk8:
292 /* input: 290 /* input:
293 * %rdi: ctx, CTX 291 * %rdi: ctx, CTX
@@ -325,12 +323,9 @@ __twofish_dec_blk8:
325 outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2); 323 outunpack_blocks(RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
326 324
327 ret; 325 ret;
326ENDPROC(__twofish_dec_blk8)
328 327
329.align 8 328ENTRY(twofish_ecb_enc_8way)
330.global twofish_ecb_enc_8way
331.type twofish_ecb_enc_8way,@function;
332
333twofish_ecb_enc_8way:
334 /* input: 329 /* input:
335 * %rdi: ctx, CTX 330 * %rdi: ctx, CTX
336 * %rsi: dst 331 * %rsi: dst
@@ -346,12 +341,9 @@ twofish_ecb_enc_8way:
346 store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); 341 store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
347 342
348 ret; 343 ret;
344ENDPROC(twofish_ecb_enc_8way)
349 345
350.align 8 346ENTRY(twofish_ecb_dec_8way)
351.global twofish_ecb_dec_8way
352.type twofish_ecb_dec_8way,@function;
353
354twofish_ecb_dec_8way:
355 /* input: 347 /* input:
356 * %rdi: ctx, CTX 348 * %rdi: ctx, CTX
357 * %rsi: dst 349 * %rsi: dst
@@ -367,12 +359,9 @@ twofish_ecb_dec_8way:
367 store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); 359 store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
368 360
369 ret; 361 ret;
362ENDPROC(twofish_ecb_dec_8way)
370 363
371.align 8 364ENTRY(twofish_cbc_dec_8way)
372.global twofish_cbc_dec_8way
373.type twofish_cbc_dec_8way,@function;
374
375twofish_cbc_dec_8way:
376 /* input: 365 /* input:
377 * %rdi: ctx, CTX 366 * %rdi: ctx, CTX
378 * %rsi: dst 367 * %rsi: dst
@@ -393,12 +382,9 @@ twofish_cbc_dec_8way:
393 popq %r12; 382 popq %r12;
394 383
395 ret; 384 ret;
385ENDPROC(twofish_cbc_dec_8way)
396 386
397.align 8 387ENTRY(twofish_ctr_8way)
398.global twofish_ctr_8way
399.type twofish_ctr_8way,@function;
400
401twofish_ctr_8way:
402 /* input: 388 /* input:
403 * %rdi: ctx, CTX 389 * %rdi: ctx, CTX
404 * %rsi: dst 390 * %rsi: dst
@@ -421,3 +407,4 @@ twofish_ctr_8way:
421 popq %r12; 407 popq %r12;
422 408
423 ret; 409 ret;
410ENDPROC(twofish_ctr_8way)
diff --git a/arch/x86/crypto/twofish-i586-asm_32.S b/arch/x86/crypto/twofish-i586-asm_32.S
index 658af4bb35c9..694ea4587ba7 100644
--- a/arch/x86/crypto/twofish-i586-asm_32.S
+++ b/arch/x86/crypto/twofish-i586-asm_32.S
@@ -20,6 +20,7 @@
20.file "twofish-i586-asm.S" 20.file "twofish-i586-asm.S"
21.text 21.text
22 22
23#include <linux/linkage.h>
23#include <asm/asm-offsets.h> 24#include <asm/asm-offsets.h>
24 25
25/* return address at 0 */ 26/* return address at 0 */
@@ -219,11 +220,7 @@
219 xor %esi, d ## D;\ 220 xor %esi, d ## D;\
220 ror $1, d ## D; 221 ror $1, d ## D;
221 222
222.align 4 223ENTRY(twofish_enc_blk)
223.global twofish_enc_blk
224.global twofish_dec_blk
225
226twofish_enc_blk:
227 push %ebp /* save registers according to calling convention*/ 224 push %ebp /* save registers according to calling convention*/
228 push %ebx 225 push %ebx
229 push %esi 226 push %esi
@@ -277,8 +274,9 @@ twofish_enc_blk:
277 pop %ebp 274 pop %ebp
278 mov $1, %eax 275 mov $1, %eax
279 ret 276 ret
277ENDPROC(twofish_enc_blk)
280 278
281twofish_dec_blk: 279ENTRY(twofish_dec_blk)
282 push %ebp /* save registers according to calling convention*/ 280 push %ebp /* save registers according to calling convention*/
283 push %ebx 281 push %ebx
284 push %esi 282 push %esi
@@ -333,3 +331,4 @@ twofish_dec_blk:
333 pop %ebp 331 pop %ebp
334 mov $1, %eax 332 mov $1, %eax
335 ret 333 ret
334ENDPROC(twofish_dec_blk)
diff --git a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
index 5b012a2c5119..1c3b7ceb36d2 100644
--- a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
+++ b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
@@ -20,6 +20,8 @@
20 * 20 *
21 */ 21 */
22 22
23#include <linux/linkage.h>
24
23.file "twofish-x86_64-asm-3way.S" 25.file "twofish-x86_64-asm-3way.S"
24.text 26.text
25 27
@@ -214,11 +216,7 @@
214 rorq $32, RAB2; \ 216 rorq $32, RAB2; \
215 outunpack3(mov, RIO, 2, RAB, 2); 217 outunpack3(mov, RIO, 2, RAB, 2);
216 218
217.align 8 219ENTRY(__twofish_enc_blk_3way)
218.global __twofish_enc_blk_3way
219.type __twofish_enc_blk_3way,@function;
220
221__twofish_enc_blk_3way:
222 /* input: 220 /* input:
223 * %rdi: ctx, CTX 221 * %rdi: ctx, CTX
224 * %rsi: dst 222 * %rsi: dst
@@ -250,7 +248,7 @@ __twofish_enc_blk_3way:
250 popq %rbp; /* bool xor */ 248 popq %rbp; /* bool xor */
251 249
252 testb %bpl, %bpl; 250 testb %bpl, %bpl;
253 jnz __enc_xor3; 251 jnz .L__enc_xor3;
254 252
255 outunpack_enc3(mov); 253 outunpack_enc3(mov);
256 254
@@ -262,7 +260,7 @@ __twofish_enc_blk_3way:
262 popq %r15; 260 popq %r15;
263 ret; 261 ret;
264 262
265__enc_xor3: 263.L__enc_xor3:
266 outunpack_enc3(xor); 264 outunpack_enc3(xor);
267 265
268 popq %rbx; 266 popq %rbx;
@@ -272,11 +270,9 @@ __enc_xor3:
272 popq %r14; 270 popq %r14;
273 popq %r15; 271 popq %r15;
274 ret; 272 ret;
273ENDPROC(__twofish_enc_blk_3way)
275 274
276.global twofish_dec_blk_3way 275ENTRY(twofish_dec_blk_3way)
277.type twofish_dec_blk_3way,@function;
278
279twofish_dec_blk_3way:
280 /* input: 276 /* input:
281 * %rdi: ctx, CTX 277 * %rdi: ctx, CTX
282 * %rsi: dst 278 * %rsi: dst
@@ -313,4 +309,4 @@ twofish_dec_blk_3way:
313 popq %r14; 309 popq %r14;
314 popq %r15; 310 popq %r15;
315 ret; 311 ret;
316 312ENDPROC(twofish_dec_blk_3way)
diff --git a/arch/x86/crypto/twofish-x86_64-asm_64.S b/arch/x86/crypto/twofish-x86_64-asm_64.S
index 7bcf3fcc3668..a039d21986a2 100644
--- a/arch/x86/crypto/twofish-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-x86_64-asm_64.S
@@ -20,6 +20,7 @@
20.file "twofish-x86_64-asm.S" 20.file "twofish-x86_64-asm.S"
21.text 21.text
22 22
23#include <linux/linkage.h>
23#include <asm/asm-offsets.h> 24#include <asm/asm-offsets.h>
24 25
25#define a_offset 0 26#define a_offset 0
@@ -214,11 +215,7 @@
214 xor %r8d, d ## D;\ 215 xor %r8d, d ## D;\
215 ror $1, d ## D; 216 ror $1, d ## D;
216 217
217.align 8 218ENTRY(twofish_enc_blk)
218.global twofish_enc_blk
219.global twofish_dec_blk
220
221twofish_enc_blk:
222 pushq R1 219 pushq R1
223 220
224 /* %rdi contains the ctx address */ 221 /* %rdi contains the ctx address */
@@ -269,8 +266,9 @@ twofish_enc_blk:
269 popq R1 266 popq R1
270 movq $1,%rax 267 movq $1,%rax
271 ret 268 ret
269ENDPROC(twofish_enc_blk)
272 270
273twofish_dec_blk: 271ENTRY(twofish_dec_blk)
274 pushq R1 272 pushq R1
275 273
276 /* %rdi contains the ctx address */ 274 /* %rdi contains the ctx address */
@@ -320,3 +318,4 @@ twofish_dec_blk:
320 popq R1 318 popq R1
321 movq $1,%rax 319 movq $1,%rax
322 ret 320 ret
321ENDPROC(twofish_dec_blk)
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 0880a14834aa..05c0ce52f96d 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -353,6 +353,27 @@ config CRYPTO_CRC32C_SPARC64
353 CRC32c CRC algorithm implemented using sparc64 crypto instructions, 353 CRC32c CRC algorithm implemented using sparc64 crypto instructions,
354 when available. 354 when available.
355 355
356config CRYPTO_CRC32
357 tristate "CRC32 CRC algorithm"
358 select CRYPTO_HASH
359 select CRC32
360 help
361 CRC-32-IEEE 802.3 cyclic redundancy-check algorithm.
362 Shash crypto api wrappers to crc32_le function.
363
364config CRYPTO_CRC32_PCLMUL
365 tristate "CRC32 PCLMULQDQ hardware acceleration"
366 depends on X86
367 select CRYPTO_HASH
368 select CRC32
369 help
370 From Intel Westmere and AMD Bulldozer processor with SSE4.2
371 and PCLMULQDQ supported, the processor will support
372 CRC32 PCLMULQDQ implementation using hardware accelerated PCLMULQDQ
373 instruction. This option will create 'crc32-plcmul' module,
374 which will enable any routine to use the CRC-32-IEEE 802.3 checksum
375 and gain better performance as compared with the table implementation.
376
356config CRYPTO_GHASH 377config CRYPTO_GHASH
357 tristate "GHASH digest algorithm" 378 tristate "GHASH digest algorithm"
358 select CRYPTO_GF128MUL 379 select CRYPTO_GF128MUL
diff --git a/crypto/Makefile b/crypto/Makefile
index d59dec749804..be1a1bebbb86 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -81,6 +81,7 @@ obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o
81obj-$(CONFIG_CRYPTO_ZLIB) += zlib.o 81obj-$(CONFIG_CRYPTO_ZLIB) += zlib.o
82obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o 82obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o
83obj-$(CONFIG_CRYPTO_CRC32C) += crc32c.o 83obj-$(CONFIG_CRYPTO_CRC32C) += crc32c.o
84obj-$(CONFIG_CRYPTO_CRC32) += crc32.o
84obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o 85obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
85obj-$(CONFIG_CRYPTO_LZO) += lzo.o 86obj-$(CONFIG_CRYPTO_LZO) += lzo.o
86obj-$(CONFIG_CRYPTO_842) += 842.o 87obj-$(CONFIG_CRYPTO_842) += 842.o
diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c
index 533de9550a82..7d4a8d28277e 100644
--- a/crypto/ablkcipher.c
+++ b/crypto/ablkcipher.c
@@ -388,9 +388,9 @@ static int crypto_ablkcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
388{ 388{
389 struct crypto_report_blkcipher rblkcipher; 389 struct crypto_report_blkcipher rblkcipher;
390 390
391 snprintf(rblkcipher.type, CRYPTO_MAX_ALG_NAME, "%s", "ablkcipher"); 391 strncpy(rblkcipher.type, "ablkcipher", sizeof(rblkcipher.type));
392 snprintf(rblkcipher.geniv, CRYPTO_MAX_ALG_NAME, "%s", 392 strncpy(rblkcipher.geniv, alg->cra_ablkcipher.geniv ?: "<default>",
393 alg->cra_ablkcipher.geniv ?: "<default>"); 393 sizeof(rblkcipher.geniv));
394 394
395 rblkcipher.blocksize = alg->cra_blocksize; 395 rblkcipher.blocksize = alg->cra_blocksize;
396 rblkcipher.min_keysize = alg->cra_ablkcipher.min_keysize; 396 rblkcipher.min_keysize = alg->cra_ablkcipher.min_keysize;
@@ -469,9 +469,9 @@ static int crypto_givcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
469{ 469{
470 struct crypto_report_blkcipher rblkcipher; 470 struct crypto_report_blkcipher rblkcipher;
471 471
472 snprintf(rblkcipher.type, CRYPTO_MAX_ALG_NAME, "%s", "givcipher"); 472 strncpy(rblkcipher.type, "givcipher", sizeof(rblkcipher.type));
473 snprintf(rblkcipher.geniv, CRYPTO_MAX_ALG_NAME, "%s", 473 strncpy(rblkcipher.geniv, alg->cra_ablkcipher.geniv ?: "<built-in>",
474 alg->cra_ablkcipher.geniv ?: "<built-in>"); 474 sizeof(rblkcipher.geniv));
475 475
476 rblkcipher.blocksize = alg->cra_blocksize; 476 rblkcipher.blocksize = alg->cra_blocksize;
477 rblkcipher.min_keysize = alg->cra_ablkcipher.min_keysize; 477 rblkcipher.min_keysize = alg->cra_ablkcipher.min_keysize;
diff --git a/crypto/aead.c b/crypto/aead.c
index 0b8121ebec07..547491e35c63 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -117,9 +117,8 @@ static int crypto_aead_report(struct sk_buff *skb, struct crypto_alg *alg)
117 struct crypto_report_aead raead; 117 struct crypto_report_aead raead;
118 struct aead_alg *aead = &alg->cra_aead; 118 struct aead_alg *aead = &alg->cra_aead;
119 119
120 snprintf(raead.type, CRYPTO_MAX_ALG_NAME, "%s", "aead"); 120 strncpy(raead.type, "aead", sizeof(raead.type));
121 snprintf(raead.geniv, CRYPTO_MAX_ALG_NAME, "%s", 121 strncpy(raead.geniv, aead->geniv ?: "<built-in>", sizeof(raead.geniv));
122 aead->geniv ?: "<built-in>");
123 122
124 raead.blocksize = alg->cra_blocksize; 123 raead.blocksize = alg->cra_blocksize;
125 raead.maxauthsize = aead->maxauthsize; 124 raead.maxauthsize = aead->maxauthsize;
@@ -203,8 +202,8 @@ static int crypto_nivaead_report(struct sk_buff *skb, struct crypto_alg *alg)
203 struct crypto_report_aead raead; 202 struct crypto_report_aead raead;
204 struct aead_alg *aead = &alg->cra_aead; 203 struct aead_alg *aead = &alg->cra_aead;
205 204
206 snprintf(raead.type, CRYPTO_MAX_ALG_NAME, "%s", "nivaead"); 205 strncpy(raead.type, "nivaead", sizeof(raead.type));
207 snprintf(raead.geniv, CRYPTO_MAX_ALG_NAME, "%s", aead->geniv); 206 strncpy(raead.geniv, aead->geniv, sizeof(raead.geniv));
208 207
209 raead.blocksize = alg->cra_blocksize; 208 raead.blocksize = alg->cra_blocksize;
210 raead.maxauthsize = aead->maxauthsize; 209 raead.maxauthsize = aead->maxauthsize;
@@ -282,18 +281,16 @@ struct crypto_instance *aead_geniv_alloc(struct crypto_template *tmpl,
282 int err; 281 int err;
283 282
284 algt = crypto_get_attr_type(tb); 283 algt = crypto_get_attr_type(tb);
285 err = PTR_ERR(algt);
286 if (IS_ERR(algt)) 284 if (IS_ERR(algt))
287 return ERR_PTR(err); 285 return ERR_CAST(algt);
288 286
289 if ((algt->type ^ (CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_GENIV)) & 287 if ((algt->type ^ (CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_GENIV)) &
290 algt->mask) 288 algt->mask)
291 return ERR_PTR(-EINVAL); 289 return ERR_PTR(-EINVAL);
292 290
293 name = crypto_attr_alg_name(tb[1]); 291 name = crypto_attr_alg_name(tb[1]);
294 err = PTR_ERR(name);
295 if (IS_ERR(name)) 292 if (IS_ERR(name))
296 return ERR_PTR(err); 293 return ERR_CAST(name);
297 294
298 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 295 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
299 if (!inst) 296 if (!inst)
diff --git a/crypto/ahash.c b/crypto/ahash.c
index 3887856c2dd6..793a27f2493e 100644
--- a/crypto/ahash.c
+++ b/crypto/ahash.c
@@ -404,7 +404,7 @@ static int crypto_ahash_report(struct sk_buff *skb, struct crypto_alg *alg)
404{ 404{
405 struct crypto_report_hash rhash; 405 struct crypto_report_hash rhash;
406 406
407 snprintf(rhash.type, CRYPTO_MAX_ALG_NAME, "%s", "ahash"); 407 strncpy(rhash.type, "ahash", sizeof(rhash.type));
408 408
409 rhash.blocksize = alg->cra_blocksize; 409 rhash.blocksize = alg->cra_blocksize;
410 rhash.digestsize = __crypto_hash_alg_common(alg)->digestsize; 410 rhash.digestsize = __crypto_hash_alg_common(alg)->digestsize;
diff --git a/crypto/algapi.c b/crypto/algapi.c
index c3b9bfeeb7ff..08c57c8aec95 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -749,12 +749,10 @@ struct crypto_alg *crypto_attr_alg2(struct rtattr *rta,
749 u32 type, u32 mask) 749 u32 type, u32 mask)
750{ 750{
751 const char *name; 751 const char *name;
752 int err;
753 752
754 name = crypto_attr_alg_name(rta); 753 name = crypto_attr_alg_name(rta);
755 err = PTR_ERR(name);
756 if (IS_ERR(name)) 754 if (IS_ERR(name))
757 return ERR_PTR(err); 755 return ERR_CAST(name);
758 756
759 return crypto_find_alg(name, frontend, type, mask); 757 return crypto_find_alg(name, frontend, type, mask);
760} 758}
diff --git a/crypto/authenc.c b/crypto/authenc.c
index d0583a4489e6..ffce19de05cf 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -592,9 +592,8 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb)
592 int err; 592 int err;
593 593
594 algt = crypto_get_attr_type(tb); 594 algt = crypto_get_attr_type(tb);
595 err = PTR_ERR(algt);
596 if (IS_ERR(algt)) 595 if (IS_ERR(algt))
597 return ERR_PTR(err); 596 return ERR_CAST(algt);
598 597
599 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 598 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
600 return ERR_PTR(-EINVAL); 599 return ERR_PTR(-EINVAL);
diff --git a/crypto/authencesn.c b/crypto/authencesn.c
index 136b68b9d8d4..ab53762fc309 100644
--- a/crypto/authencesn.c
+++ b/crypto/authencesn.c
@@ -715,9 +715,8 @@ static struct crypto_instance *crypto_authenc_esn_alloc(struct rtattr **tb)
715 int err; 715 int err;
716 716
717 algt = crypto_get_attr_type(tb); 717 algt = crypto_get_attr_type(tb);
718 err = PTR_ERR(algt);
719 if (IS_ERR(algt)) 718 if (IS_ERR(algt))
720 return ERR_PTR(err); 719 return ERR_CAST(algt);
721 720
722 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 721 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
723 return ERR_PTR(-EINVAL); 722 return ERR_PTR(-EINVAL);
diff --git a/crypto/blkcipher.c b/crypto/blkcipher.c
index a8d85a1d670e..a79e7e9ab86e 100644
--- a/crypto/blkcipher.c
+++ b/crypto/blkcipher.c
@@ -499,9 +499,9 @@ static int crypto_blkcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
499{ 499{
500 struct crypto_report_blkcipher rblkcipher; 500 struct crypto_report_blkcipher rblkcipher;
501 501
502 snprintf(rblkcipher.type, CRYPTO_MAX_ALG_NAME, "%s", "blkcipher"); 502 strncpy(rblkcipher.type, "blkcipher", sizeof(rblkcipher.type));
503 snprintf(rblkcipher.geniv, CRYPTO_MAX_ALG_NAME, "%s", 503 strncpy(rblkcipher.geniv, alg->cra_blkcipher.geniv ?: "<default>",
504 alg->cra_blkcipher.geniv ?: "<default>"); 504 sizeof(rblkcipher.geniv));
505 505
506 rblkcipher.blocksize = alg->cra_blocksize; 506 rblkcipher.blocksize = alg->cra_blocksize;
507 rblkcipher.min_keysize = alg->cra_blkcipher.min_keysize; 507 rblkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
@@ -588,18 +588,16 @@ struct crypto_instance *skcipher_geniv_alloc(struct crypto_template *tmpl,
588 int err; 588 int err;
589 589
590 algt = crypto_get_attr_type(tb); 590 algt = crypto_get_attr_type(tb);
591 err = PTR_ERR(algt);
592 if (IS_ERR(algt)) 591 if (IS_ERR(algt))
593 return ERR_PTR(err); 592 return ERR_CAST(algt);
594 593
595 if ((algt->type ^ (CRYPTO_ALG_TYPE_GIVCIPHER | CRYPTO_ALG_GENIV)) & 594 if ((algt->type ^ (CRYPTO_ALG_TYPE_GIVCIPHER | CRYPTO_ALG_GENIV)) &
596 algt->mask) 595 algt->mask)
597 return ERR_PTR(-EINVAL); 596 return ERR_PTR(-EINVAL);
598 597
599 name = crypto_attr_alg_name(tb[1]); 598 name = crypto_attr_alg_name(tb[1]);
600 err = PTR_ERR(name);
601 if (IS_ERR(name)) 599 if (IS_ERR(name))
602 return ERR_PTR(err); 600 return ERR_CAST(name);
603 601
604 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 602 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
605 if (!inst) 603 if (!inst)
diff --git a/crypto/ccm.c b/crypto/ccm.c
index 32fe1bb5decb..499c91717d93 100644
--- a/crypto/ccm.c
+++ b/crypto/ccm.c
@@ -484,18 +484,16 @@ static struct crypto_instance *crypto_ccm_alloc_common(struct rtattr **tb,
484 int err; 484 int err;
485 485
486 algt = crypto_get_attr_type(tb); 486 algt = crypto_get_attr_type(tb);
487 err = PTR_ERR(algt);
488 if (IS_ERR(algt)) 487 if (IS_ERR(algt))
489 return ERR_PTR(err); 488 return ERR_CAST(algt);
490 489
491 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 490 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
492 return ERR_PTR(-EINVAL); 491 return ERR_PTR(-EINVAL);
493 492
494 cipher = crypto_alg_mod_lookup(cipher_name, CRYPTO_ALG_TYPE_CIPHER, 493 cipher = crypto_alg_mod_lookup(cipher_name, CRYPTO_ALG_TYPE_CIPHER,
495 CRYPTO_ALG_TYPE_MASK); 494 CRYPTO_ALG_TYPE_MASK);
496 err = PTR_ERR(cipher);
497 if (IS_ERR(cipher)) 495 if (IS_ERR(cipher))
498 return ERR_PTR(err); 496 return ERR_CAST(cipher);
499 497
500 err = -EINVAL; 498 err = -EINVAL;
501 if (cipher->cra_blocksize != 16) 499 if (cipher->cra_blocksize != 16)
@@ -573,15 +571,13 @@ out_put_cipher:
573 571
574static struct crypto_instance *crypto_ccm_alloc(struct rtattr **tb) 572static struct crypto_instance *crypto_ccm_alloc(struct rtattr **tb)
575{ 573{
576 int err;
577 const char *cipher_name; 574 const char *cipher_name;
578 char ctr_name[CRYPTO_MAX_ALG_NAME]; 575 char ctr_name[CRYPTO_MAX_ALG_NAME];
579 char full_name[CRYPTO_MAX_ALG_NAME]; 576 char full_name[CRYPTO_MAX_ALG_NAME];
580 577
581 cipher_name = crypto_attr_alg_name(tb[1]); 578 cipher_name = crypto_attr_alg_name(tb[1]);
582 err = PTR_ERR(cipher_name);
583 if (IS_ERR(cipher_name)) 579 if (IS_ERR(cipher_name))
584 return ERR_PTR(err); 580 return ERR_CAST(cipher_name);
585 581
586 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", 582 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)",
587 cipher_name) >= CRYPTO_MAX_ALG_NAME) 583 cipher_name) >= CRYPTO_MAX_ALG_NAME)
@@ -612,20 +608,17 @@ static struct crypto_template crypto_ccm_tmpl = {
612 608
613static struct crypto_instance *crypto_ccm_base_alloc(struct rtattr **tb) 609static struct crypto_instance *crypto_ccm_base_alloc(struct rtattr **tb)
614{ 610{
615 int err;
616 const char *ctr_name; 611 const char *ctr_name;
617 const char *cipher_name; 612 const char *cipher_name;
618 char full_name[CRYPTO_MAX_ALG_NAME]; 613 char full_name[CRYPTO_MAX_ALG_NAME];
619 614
620 ctr_name = crypto_attr_alg_name(tb[1]); 615 ctr_name = crypto_attr_alg_name(tb[1]);
621 err = PTR_ERR(ctr_name);
622 if (IS_ERR(ctr_name)) 616 if (IS_ERR(ctr_name))
623 return ERR_PTR(err); 617 return ERR_CAST(ctr_name);
624 618
625 cipher_name = crypto_attr_alg_name(tb[2]); 619 cipher_name = crypto_attr_alg_name(tb[2]);
626 err = PTR_ERR(cipher_name);
627 if (IS_ERR(cipher_name)) 620 if (IS_ERR(cipher_name))
628 return ERR_PTR(err); 621 return ERR_CAST(cipher_name);
629 622
630 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm_base(%s,%s)", 623 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm_base(%s,%s)",
631 ctr_name, cipher_name) >= CRYPTO_MAX_ALG_NAME) 624 ctr_name, cipher_name) >= CRYPTO_MAX_ALG_NAME)
@@ -760,17 +753,15 @@ static struct crypto_instance *crypto_rfc4309_alloc(struct rtattr **tb)
760 int err; 753 int err;
761 754
762 algt = crypto_get_attr_type(tb); 755 algt = crypto_get_attr_type(tb);
763 err = PTR_ERR(algt);
764 if (IS_ERR(algt)) 756 if (IS_ERR(algt))
765 return ERR_PTR(err); 757 return ERR_CAST(algt);
766 758
767 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 759 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
768 return ERR_PTR(-EINVAL); 760 return ERR_PTR(-EINVAL);
769 761
770 ccm_name = crypto_attr_alg_name(tb[1]); 762 ccm_name = crypto_attr_alg_name(tb[1]);
771 err = PTR_ERR(ccm_name);
772 if (IS_ERR(ccm_name)) 763 if (IS_ERR(ccm_name))
773 return ERR_PTR(err); 764 return ERR_CAST(ccm_name);
774 765
775 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 766 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
776 if (!inst) 767 if (!inst)
diff --git a/crypto/chainiv.c b/crypto/chainiv.c
index ba200b07449d..834d8dd3d4fc 100644
--- a/crypto/chainiv.c
+++ b/crypto/chainiv.c
@@ -291,9 +291,8 @@ static struct crypto_instance *chainiv_alloc(struct rtattr **tb)
291 int err; 291 int err;
292 292
293 algt = crypto_get_attr_type(tb); 293 algt = crypto_get_attr_type(tb);
294 err = PTR_ERR(algt);
295 if (IS_ERR(algt)) 294 if (IS_ERR(algt))
296 return ERR_PTR(err); 295 return ERR_CAST(algt);
297 296
298 err = crypto_get_default_rng(); 297 err = crypto_get_default_rng();
299 if (err) 298 if (err)
diff --git a/crypto/crc32.c b/crypto/crc32.c
new file mode 100644
index 000000000000..9d1c41569898
--- /dev/null
+++ b/crypto/crc32.c
@@ -0,0 +1,158 @@
1/* GPL HEADER START
2 *
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 only,
7 * as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License version 2 for more details (a copy is included
13 * in the LICENSE file that accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License
16 * version 2 along with this program; If not, see http://www.gnu.org/licenses
17 *
18 * Please visit http://www.xyratex.com/contact if you need additional
19 * information or have any questions.
20 *
21 * GPL HEADER END
22 */
23
24/*
25 * Copyright 2012 Xyratex Technology Limited
26 */
27
28/*
29 * This is crypto api shash wrappers to crc32_le.
30 */
31
32#include <linux/crc32.h>
33#include <crypto/internal/hash.h>
34#include <linux/init.h>
35#include <linux/module.h>
36#include <linux/string.h>
37#include <linux/kernel.h>
38
39#define CHKSUM_BLOCK_SIZE 1
40#define CHKSUM_DIGEST_SIZE 4
41
42static u32 __crc32_le(u32 crc, unsigned char const *p, size_t len)
43{
44 return crc32_le(crc, p, len);
45}
46
47/** No default init with ~0 */
48static int crc32_cra_init(struct crypto_tfm *tfm)
49{
50 u32 *key = crypto_tfm_ctx(tfm);
51
52 *key = 0;
53
54 return 0;
55}
56
57
58/*
59 * Setting the seed allows arbitrary accumulators and flexible XOR policy
60 * If your algorithm starts with ~0, then XOR with ~0 before you set
61 * the seed.
62 */
63static int crc32_setkey(struct crypto_shash *hash, const u8 *key,
64 unsigned int keylen)
65{
66 u32 *mctx = crypto_shash_ctx(hash);
67
68 if (keylen != sizeof(u32)) {
69 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
70 return -EINVAL;
71 }
72 *mctx = le32_to_cpup((__le32 *)key);
73 return 0;
74}
75
76static int crc32_init(struct shash_desc *desc)
77{
78 u32 *mctx = crypto_shash_ctx(desc->tfm);
79 u32 *crcp = shash_desc_ctx(desc);
80
81 *crcp = *mctx;
82
83 return 0;
84}
85
86static int crc32_update(struct shash_desc *desc, const u8 *data,
87 unsigned int len)
88{
89 u32 *crcp = shash_desc_ctx(desc);
90
91 *crcp = __crc32_le(*crcp, data, len);
92 return 0;
93}
94
95/* No final XOR 0xFFFFFFFF, like crc32_le */
96static int __crc32_finup(u32 *crcp, const u8 *data, unsigned int len,
97 u8 *out)
98{
99 *(__le32 *)out = cpu_to_le32(__crc32_le(*crcp, data, len));
100 return 0;
101}
102
103static int crc32_finup(struct shash_desc *desc, const u8 *data,
104 unsigned int len, u8 *out)
105{
106 return __crc32_finup(shash_desc_ctx(desc), data, len, out);
107}
108
109static int crc32_final(struct shash_desc *desc, u8 *out)
110{
111 u32 *crcp = shash_desc_ctx(desc);
112
113 *(__le32 *)out = cpu_to_le32p(crcp);
114 return 0;
115}
116
117static int crc32_digest(struct shash_desc *desc, const u8 *data,
118 unsigned int len, u8 *out)
119{
120 return __crc32_finup(crypto_shash_ctx(desc->tfm), data, len,
121 out);
122}
123static struct shash_alg alg = {
124 .setkey = crc32_setkey,
125 .init = crc32_init,
126 .update = crc32_update,
127 .final = crc32_final,
128 .finup = crc32_finup,
129 .digest = crc32_digest,
130 .descsize = sizeof(u32),
131 .digestsize = CHKSUM_DIGEST_SIZE,
132 .base = {
133 .cra_name = "crc32",
134 .cra_driver_name = "crc32-table",
135 .cra_priority = 100,
136 .cra_blocksize = CHKSUM_BLOCK_SIZE,
137 .cra_ctxsize = sizeof(u32),
138 .cra_module = THIS_MODULE,
139 .cra_init = crc32_cra_init,
140 }
141};
142
143static int __init crc32_mod_init(void)
144{
145 return crypto_register_shash(&alg);
146}
147
148static void __exit crc32_mod_fini(void)
149{
150 crypto_unregister_shash(&alg);
151}
152
153module_init(crc32_mod_init);
154module_exit(crc32_mod_fini);
155
156MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>");
157MODULE_DESCRIPTION("CRC32 calculations wrapper for lib/crc32");
158MODULE_LICENSE("GPL");
diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c
index 35d700a97d79..dfd511fb39ee 100644
--- a/crypto/crypto_user.c
+++ b/crypto/crypto_user.c
@@ -30,6 +30,8 @@
30 30
31#include "internal.h" 31#include "internal.h"
32 32
33#define null_terminated(x) (strnlen(x, sizeof(x)) < sizeof(x))
34
33static DEFINE_MUTEX(crypto_cfg_mutex); 35static DEFINE_MUTEX(crypto_cfg_mutex);
34 36
35/* The crypto netlink socket */ 37/* The crypto netlink socket */
@@ -75,7 +77,7 @@ static int crypto_report_cipher(struct sk_buff *skb, struct crypto_alg *alg)
75{ 77{
76 struct crypto_report_cipher rcipher; 78 struct crypto_report_cipher rcipher;
77 79
78 snprintf(rcipher.type, CRYPTO_MAX_ALG_NAME, "%s", "cipher"); 80 strncpy(rcipher.type, "cipher", sizeof(rcipher.type));
79 81
80 rcipher.blocksize = alg->cra_blocksize; 82 rcipher.blocksize = alg->cra_blocksize;
81 rcipher.min_keysize = alg->cra_cipher.cia_min_keysize; 83 rcipher.min_keysize = alg->cra_cipher.cia_min_keysize;
@@ -94,8 +96,7 @@ static int crypto_report_comp(struct sk_buff *skb, struct crypto_alg *alg)
94{ 96{
95 struct crypto_report_comp rcomp; 97 struct crypto_report_comp rcomp;
96 98
97 snprintf(rcomp.type, CRYPTO_MAX_ALG_NAME, "%s", "compression"); 99 strncpy(rcomp.type, "compression", sizeof(rcomp.type));
98
99 if (nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS, 100 if (nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS,
100 sizeof(struct crypto_report_comp), &rcomp)) 101 sizeof(struct crypto_report_comp), &rcomp))
101 goto nla_put_failure; 102 goto nla_put_failure;
@@ -108,12 +109,14 @@ nla_put_failure:
108static int crypto_report_one(struct crypto_alg *alg, 109static int crypto_report_one(struct crypto_alg *alg,
109 struct crypto_user_alg *ualg, struct sk_buff *skb) 110 struct crypto_user_alg *ualg, struct sk_buff *skb)
110{ 111{
111 memcpy(&ualg->cru_name, &alg->cra_name, sizeof(ualg->cru_name)); 112 strncpy(ualg->cru_name, alg->cra_name, sizeof(ualg->cru_name));
112 memcpy(&ualg->cru_driver_name, &alg->cra_driver_name, 113 strncpy(ualg->cru_driver_name, alg->cra_driver_name,
113 sizeof(ualg->cru_driver_name)); 114 sizeof(ualg->cru_driver_name));
114 memcpy(&ualg->cru_module_name, module_name(alg->cra_module), 115 strncpy(ualg->cru_module_name, module_name(alg->cra_module),
115 CRYPTO_MAX_ALG_NAME); 116 sizeof(ualg->cru_module_name));
116 117
118 ualg->cru_type = 0;
119 ualg->cru_mask = 0;
117 ualg->cru_flags = alg->cra_flags; 120 ualg->cru_flags = alg->cra_flags;
118 ualg->cru_refcnt = atomic_read(&alg->cra_refcnt); 121 ualg->cru_refcnt = atomic_read(&alg->cra_refcnt);
119 122
@@ -122,8 +125,7 @@ static int crypto_report_one(struct crypto_alg *alg,
122 if (alg->cra_flags & CRYPTO_ALG_LARVAL) { 125 if (alg->cra_flags & CRYPTO_ALG_LARVAL) {
123 struct crypto_report_larval rl; 126 struct crypto_report_larval rl;
124 127
125 snprintf(rl.type, CRYPTO_MAX_ALG_NAME, "%s", "larval"); 128 strncpy(rl.type, "larval", sizeof(rl.type));
126
127 if (nla_put(skb, CRYPTOCFGA_REPORT_LARVAL, 129 if (nla_put(skb, CRYPTOCFGA_REPORT_LARVAL,
128 sizeof(struct crypto_report_larval), &rl)) 130 sizeof(struct crypto_report_larval), &rl))
129 goto nla_put_failure; 131 goto nla_put_failure;
@@ -196,7 +198,10 @@ static int crypto_report(struct sk_buff *in_skb, struct nlmsghdr *in_nlh,
196 struct crypto_dump_info info; 198 struct crypto_dump_info info;
197 int err; 199 int err;
198 200
199 if (!p->cru_driver_name) 201 if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
202 return -EINVAL;
203
204 if (!p->cru_driver_name[0])
200 return -EINVAL; 205 return -EINVAL;
201 206
202 alg = crypto_alg_match(p, 1); 207 alg = crypto_alg_match(p, 1);
@@ -260,6 +265,9 @@ static int crypto_update_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
260 struct nlattr *priority = attrs[CRYPTOCFGA_PRIORITY_VAL]; 265 struct nlattr *priority = attrs[CRYPTOCFGA_PRIORITY_VAL];
261 LIST_HEAD(list); 266 LIST_HEAD(list);
262 267
268 if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
269 return -EINVAL;
270
263 if (priority && !strlen(p->cru_driver_name)) 271 if (priority && !strlen(p->cru_driver_name))
264 return -EINVAL; 272 return -EINVAL;
265 273
@@ -287,6 +295,9 @@ static int crypto_del_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
287 struct crypto_alg *alg; 295 struct crypto_alg *alg;
288 struct crypto_user_alg *p = nlmsg_data(nlh); 296 struct crypto_user_alg *p = nlmsg_data(nlh);
289 297
298 if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
299 return -EINVAL;
300
290 alg = crypto_alg_match(p, 1); 301 alg = crypto_alg_match(p, 1);
291 if (!alg) 302 if (!alg)
292 return -ENOENT; 303 return -ENOENT;
@@ -368,6 +379,9 @@ static int crypto_add_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
368 struct crypto_user_alg *p = nlmsg_data(nlh); 379 struct crypto_user_alg *p = nlmsg_data(nlh);
369 struct nlattr *priority = attrs[CRYPTOCFGA_PRIORITY_VAL]; 380 struct nlattr *priority = attrs[CRYPTOCFGA_PRIORITY_VAL];
370 381
382 if (!null_terminated(p->cru_name) || !null_terminated(p->cru_driver_name))
383 return -EINVAL;
384
371 if (strlen(p->cru_driver_name)) 385 if (strlen(p->cru_driver_name))
372 exact = 1; 386 exact = 1;
373 387
diff --git a/crypto/ctr.c b/crypto/ctr.c
index 1f2997cbfdd4..f2b94f27bb2c 100644
--- a/crypto/ctr.c
+++ b/crypto/ctr.c
@@ -343,17 +343,15 @@ static struct crypto_instance *crypto_rfc3686_alloc(struct rtattr **tb)
343 int err; 343 int err;
344 344
345 algt = crypto_get_attr_type(tb); 345 algt = crypto_get_attr_type(tb);
346 err = PTR_ERR(algt);
347 if (IS_ERR(algt)) 346 if (IS_ERR(algt))
348 return ERR_PTR(err); 347 return ERR_CAST(algt);
349 348
350 if ((algt->type ^ CRYPTO_ALG_TYPE_BLKCIPHER) & algt->mask) 349 if ((algt->type ^ CRYPTO_ALG_TYPE_BLKCIPHER) & algt->mask)
351 return ERR_PTR(-EINVAL); 350 return ERR_PTR(-EINVAL);
352 351
353 cipher_name = crypto_attr_alg_name(tb[1]); 352 cipher_name = crypto_attr_alg_name(tb[1]);
354 err = PTR_ERR(cipher_name);
355 if (IS_ERR(cipher_name)) 353 if (IS_ERR(cipher_name))
356 return ERR_PTR(err); 354 return ERR_CAST(cipher_name);
357 355
358 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 356 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
359 if (!inst) 357 if (!inst)
diff --git a/crypto/cts.c b/crypto/cts.c
index ccf9c5de3958..042223f8e733 100644
--- a/crypto/cts.c
+++ b/crypto/cts.c
@@ -282,9 +282,8 @@ static struct crypto_instance *crypto_cts_alloc(struct rtattr **tb)
282 282
283 alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_BLKCIPHER, 283 alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_BLKCIPHER,
284 CRYPTO_ALG_TYPE_MASK); 284 CRYPTO_ALG_TYPE_MASK);
285 err = PTR_ERR(alg);
286 if (IS_ERR(alg)) 285 if (IS_ERR(alg))
287 return ERR_PTR(err); 286 return ERR_CAST(alg);
288 287
289 inst = ERR_PTR(-EINVAL); 288 inst = ERR_PTR(-EINVAL);
290 if (!is_power_of_2(alg->cra_blocksize)) 289 if (!is_power_of_2(alg->cra_blocksize))
diff --git a/crypto/gcm.c b/crypto/gcm.c
index 1a252639ef91..137ad1ec5438 100644
--- a/crypto/gcm.c
+++ b/crypto/gcm.c
@@ -701,9 +701,8 @@ static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb,
701 int err; 701 int err;
702 702
703 algt = crypto_get_attr_type(tb); 703 algt = crypto_get_attr_type(tb);
704 err = PTR_ERR(algt);
705 if (IS_ERR(algt)) 704 if (IS_ERR(algt))
706 return ERR_PTR(err); 705 return ERR_CAST(algt);
707 706
708 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 707 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
709 return ERR_PTR(-EINVAL); 708 return ERR_PTR(-EINVAL);
@@ -711,9 +710,8 @@ static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb,
711 ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type, 710 ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type,
712 CRYPTO_ALG_TYPE_HASH, 711 CRYPTO_ALG_TYPE_HASH,
713 CRYPTO_ALG_TYPE_AHASH_MASK); 712 CRYPTO_ALG_TYPE_AHASH_MASK);
714 err = PTR_ERR(ghash_alg);
715 if (IS_ERR(ghash_alg)) 713 if (IS_ERR(ghash_alg))
716 return ERR_PTR(err); 714 return ERR_CAST(ghash_alg);
717 715
718 err = -ENOMEM; 716 err = -ENOMEM;
719 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 717 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
@@ -787,15 +785,13 @@ out_put_ghash:
787 785
788static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb) 786static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb)
789{ 787{
790 int err;
791 const char *cipher_name; 788 const char *cipher_name;
792 char ctr_name[CRYPTO_MAX_ALG_NAME]; 789 char ctr_name[CRYPTO_MAX_ALG_NAME];
793 char full_name[CRYPTO_MAX_ALG_NAME]; 790 char full_name[CRYPTO_MAX_ALG_NAME];
794 791
795 cipher_name = crypto_attr_alg_name(tb[1]); 792 cipher_name = crypto_attr_alg_name(tb[1]);
796 err = PTR_ERR(cipher_name);
797 if (IS_ERR(cipher_name)) 793 if (IS_ERR(cipher_name))
798 return ERR_PTR(err); 794 return ERR_CAST(cipher_name);
799 795
800 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >= 796 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
801 CRYPTO_MAX_ALG_NAME) 797 CRYPTO_MAX_ALG_NAME)
@@ -826,20 +822,17 @@ static struct crypto_template crypto_gcm_tmpl = {
826 822
827static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb) 823static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb)
828{ 824{
829 int err;
830 const char *ctr_name; 825 const char *ctr_name;
831 const char *ghash_name; 826 const char *ghash_name;
832 char full_name[CRYPTO_MAX_ALG_NAME]; 827 char full_name[CRYPTO_MAX_ALG_NAME];
833 828
834 ctr_name = crypto_attr_alg_name(tb[1]); 829 ctr_name = crypto_attr_alg_name(tb[1]);
835 err = PTR_ERR(ctr_name);
836 if (IS_ERR(ctr_name)) 830 if (IS_ERR(ctr_name))
837 return ERR_PTR(err); 831 return ERR_CAST(ctr_name);
838 832
839 ghash_name = crypto_attr_alg_name(tb[2]); 833 ghash_name = crypto_attr_alg_name(tb[2]);
840 err = PTR_ERR(ghash_name);
841 if (IS_ERR(ghash_name)) 834 if (IS_ERR(ghash_name))
842 return ERR_PTR(err); 835 return ERR_CAST(ghash_name);
843 836
844 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)", 837 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)",
845 ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME) 838 ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME)
@@ -971,17 +964,15 @@ static struct crypto_instance *crypto_rfc4106_alloc(struct rtattr **tb)
971 int err; 964 int err;
972 965
973 algt = crypto_get_attr_type(tb); 966 algt = crypto_get_attr_type(tb);
974 err = PTR_ERR(algt);
975 if (IS_ERR(algt)) 967 if (IS_ERR(algt))
976 return ERR_PTR(err); 968 return ERR_CAST(algt);
977 969
978 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 970 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
979 return ERR_PTR(-EINVAL); 971 return ERR_PTR(-EINVAL);
980 972
981 ccm_name = crypto_attr_alg_name(tb[1]); 973 ccm_name = crypto_attr_alg_name(tb[1]);
982 err = PTR_ERR(ccm_name);
983 if (IS_ERR(ccm_name)) 974 if (IS_ERR(ccm_name))
984 return ERR_PTR(err); 975 return ERR_CAST(ccm_name);
985 976
986 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 977 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
987 if (!inst) 978 if (!inst)
@@ -1222,17 +1213,15 @@ static struct crypto_instance *crypto_rfc4543_alloc(struct rtattr **tb)
1222 int err; 1213 int err;
1223 1214
1224 algt = crypto_get_attr_type(tb); 1215 algt = crypto_get_attr_type(tb);
1225 err = PTR_ERR(algt);
1226 if (IS_ERR(algt)) 1216 if (IS_ERR(algt))
1227 return ERR_PTR(err); 1217 return ERR_CAST(algt);
1228 1218
1229 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) 1219 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
1230 return ERR_PTR(-EINVAL); 1220 return ERR_PTR(-EINVAL);
1231 1221
1232 ccm_name = crypto_attr_alg_name(tb[1]); 1222 ccm_name = crypto_attr_alg_name(tb[1]);
1233 err = PTR_ERR(ccm_name);
1234 if (IS_ERR(ccm_name)) 1223 if (IS_ERR(ccm_name))
1235 return ERR_PTR(err); 1224 return ERR_CAST(ccm_name);
1236 1225
1237 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 1226 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
1238 if (!inst) 1227 if (!inst)
diff --git a/crypto/pcompress.c b/crypto/pcompress.c
index 04e083ff5373..7140fe70c7af 100644
--- a/crypto/pcompress.c
+++ b/crypto/pcompress.c
@@ -53,8 +53,7 @@ static int crypto_pcomp_report(struct sk_buff *skb, struct crypto_alg *alg)
53{ 53{
54 struct crypto_report_comp rpcomp; 54 struct crypto_report_comp rpcomp;
55 55
56 snprintf(rpcomp.type, CRYPTO_MAX_ALG_NAME, "%s", "pcomp"); 56 strncpy(rpcomp.type, "pcomp", sizeof(rpcomp.type));
57
58 if (nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS, 57 if (nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS,
59 sizeof(struct crypto_report_comp), &rpcomp)) 58 sizeof(struct crypto_report_comp), &rpcomp))
60 goto nla_put_failure; 59 goto nla_put_failure;
diff --git a/crypto/rng.c b/crypto/rng.c
index f3b7894dec00..e0a25c2456de 100644
--- a/crypto/rng.c
+++ b/crypto/rng.c
@@ -65,7 +65,7 @@ static int crypto_rng_report(struct sk_buff *skb, struct crypto_alg *alg)
65{ 65{
66 struct crypto_report_rng rrng; 66 struct crypto_report_rng rrng;
67 67
68 snprintf(rrng.type, CRYPTO_MAX_ALG_NAME, "%s", "rng"); 68 strncpy(rrng.type, "rng", sizeof(rrng.type));
69 69
70 rrng.seedsize = alg->cra_rng.seedsize; 70 rrng.seedsize = alg->cra_rng.seedsize;
71 71
diff --git a/crypto/seqiv.c b/crypto/seqiv.c
index 4c4491229417..f2cba4ed6f25 100644
--- a/crypto/seqiv.c
+++ b/crypto/seqiv.c
@@ -305,9 +305,8 @@ static struct crypto_instance *seqiv_alloc(struct rtattr **tb)
305 int err; 305 int err;
306 306
307 algt = crypto_get_attr_type(tb); 307 algt = crypto_get_attr_type(tb);
308 err = PTR_ERR(algt);
309 if (IS_ERR(algt)) 308 if (IS_ERR(algt))
310 return ERR_PTR(err); 309 return ERR_CAST(algt);
311 310
312 err = crypto_get_default_rng(); 311 err = crypto_get_default_rng();
313 if (err) 312 if (err)
diff --git a/crypto/shash.c b/crypto/shash.c
index f426330f1017..929058a68561 100644
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -530,7 +530,8 @@ static int crypto_shash_report(struct sk_buff *skb, struct crypto_alg *alg)
530 struct crypto_report_hash rhash; 530 struct crypto_report_hash rhash;
531 struct shash_alg *salg = __crypto_shash_alg(alg); 531 struct shash_alg *salg = __crypto_shash_alg(alg);
532 532
533 snprintf(rhash.type, CRYPTO_MAX_ALG_NAME, "%s", "shash"); 533 strncpy(rhash.type, "shash", sizeof(rhash.type));
534
534 rhash.blocksize = alg->cra_blocksize; 535 rhash.blocksize = alg->cra_blocksize;
535 rhash.digestsize = salg->digestsize; 536 rhash.digestsize = salg->digestsize;
536 537
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index edf4a0818773..efd8b20e13dc 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -2269,6 +2269,21 @@ static const struct alg_test_desc alg_test_descs[] = {
2269 } 2269 }
2270 } 2270 }
2271 }, { 2271 }, {
2272 .alg = "ecb(fcrypt)",
2273 .test = alg_test_skcipher,
2274 .suite = {
2275 .cipher = {
2276 .enc = {
2277 .vecs = fcrypt_pcbc_enc_tv_template,
2278 .count = 1
2279 },
2280 .dec = {
2281 .vecs = fcrypt_pcbc_dec_tv_template,
2282 .count = 1
2283 }
2284 }
2285 }
2286 }, {
2272 .alg = "ecb(khazad)", 2287 .alg = "ecb(khazad)",
2273 .test = alg_test_skcipher, 2288 .test = alg_test_skcipher,
2274 .suite = { 2289 .suite = {
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index c9d9d5c16f94..6f22ba51f969 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -332,7 +332,7 @@ static int atmel_aes_crypt_cpu_start(struct atmel_aes_dev *dd)
332 return -EINVAL; 332 return -EINVAL;
333 333
334 dd->nb_out_sg = atmel_aes_sg_length(dd->req, dd->out_sg); 334 dd->nb_out_sg = atmel_aes_sg_length(dd->req, dd->out_sg);
335 if (!dd->nb_in_sg) 335 if (!dd->nb_out_sg)
336 return -EINVAL; 336 return -EINVAL;
337 337
338 dd->bufcnt = sg_copy_to_buffer(dd->in_sg, dd->nb_in_sg, 338 dd->bufcnt = sg_copy_to_buffer(dd->in_sg, dd->nb_in_sg,
diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c
index a22f1a9f895f..827913d7d33a 100644
--- a/drivers/crypto/bfin_crc.c
+++ b/drivers/crypto/bfin_crc.c
@@ -694,7 +694,7 @@ out_error_dma:
694 dma_free_coherent(&pdev->dev, PAGE_SIZE, crc->sg_cpu, crc->sg_dma); 694 dma_free_coherent(&pdev->dev, PAGE_SIZE, crc->sg_cpu, crc->sg_dma);
695 free_dma(crc->dma_ch); 695 free_dma(crc->dma_ch);
696out_error_irq: 696out_error_irq:
697 free_irq(crc->irq, crc->dev); 697 free_irq(crc->irq, crc);
698out_error_unmap: 698out_error_unmap:
699 iounmap((void *)crc->regs); 699 iounmap((void *)crc->regs);
700out_error_free_mem: 700out_error_free_mem:
@@ -720,10 +720,10 @@ static int bfin_crypto_crc_remove(struct platform_device *pdev)
720 720
721 crypto_unregister_ahash(&algs); 721 crypto_unregister_ahash(&algs);
722 tasklet_kill(&crc->done_task); 722 tasklet_kill(&crc->done_task);
723 iounmap((void *)crc->regs);
724 free_dma(crc->dma_ch); 723 free_dma(crc->dma_ch);
725 if (crc->irq > 0) 724 if (crc->irq > 0)
726 free_irq(crc->irq, crc->dev); 725 free_irq(crc->irq, crc);
726 iounmap((void *)crc->regs);
727 kfree(crc); 727 kfree(crc);
728 728
729 return 0; 729 return 0;
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index e66e8ee5a9af..6aa425fe0ed5 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -5,6 +5,7 @@
5 * 5 *
6 * Copyright (c) 2010 Nokia Corporation 6 * Copyright (c) 2010 Nokia Corporation
7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com> 7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
8 * Copyright (c) 2011 Texas Instruments Incorporated
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as published 11 * it under the terms of the GNU General Public License version 2 as published
@@ -19,28 +20,39 @@
19#include <linux/init.h> 20#include <linux/init.h>
20#include <linux/errno.h> 21#include <linux/errno.h>
21#include <linux/kernel.h> 22#include <linux/kernel.h>
22#include <linux/clk.h>
23#include <linux/platform_device.h> 23#include <linux/platform_device.h>
24#include <linux/scatterlist.h> 24#include <linux/scatterlist.h>
25#include <linux/dma-mapping.h> 25#include <linux/dma-mapping.h>
26#include <linux/dmaengine.h>
27#include <linux/omap-dma.h>
28#include <linux/pm_runtime.h>
29#include <linux/of.h>
30#include <linux/of_device.h>
31#include <linux/of_address.h>
26#include <linux/io.h> 32#include <linux/io.h>
27#include <linux/crypto.h> 33#include <linux/crypto.h>
28#include <linux/interrupt.h> 34#include <linux/interrupt.h>
29#include <crypto/scatterwalk.h> 35#include <crypto/scatterwalk.h>
30#include <crypto/aes.h> 36#include <crypto/aes.h>
31 37
32#include <linux/omap-dma.h> 38#define DST_MAXBURST 4
39#define DMA_MIN (DST_MAXBURST * sizeof(u32))
33 40
34/* OMAP TRM gives bitfields as start:end, where start is the higher bit 41/* OMAP TRM gives bitfields as start:end, where start is the higher bit
35 number. For example 7:0 */ 42 number. For example 7:0 */
36#define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end)) 43#define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end))
37#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end)) 44#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
38 45
39#define AES_REG_KEY(x) (0x1C - ((x ^ 0x01) * 0x04)) 46#define AES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \
40#define AES_REG_IV(x) (0x20 + ((x) * 0x04)) 47 ((x ^ 0x01) * 0x04))
48#define AES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04))
41 49
42#define AES_REG_CTRL 0x30 50#define AES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs)
43#define AES_REG_CTRL_CTR_WIDTH (1 << 7) 51#define AES_REG_CTRL_CTR_WIDTH_MASK (3 << 7)
52#define AES_REG_CTRL_CTR_WIDTH_32 (0 << 7)
53#define AES_REG_CTRL_CTR_WIDTH_64 (1 << 7)
54#define AES_REG_CTRL_CTR_WIDTH_96 (2 << 7)
55#define AES_REG_CTRL_CTR_WIDTH_128 (3 << 7)
44#define AES_REG_CTRL_CTR (1 << 6) 56#define AES_REG_CTRL_CTR (1 << 6)
45#define AES_REG_CTRL_CBC (1 << 5) 57#define AES_REG_CTRL_CBC (1 << 5)
46#define AES_REG_CTRL_KEY_SIZE (3 << 3) 58#define AES_REG_CTRL_KEY_SIZE (3 << 3)
@@ -48,14 +60,11 @@
48#define AES_REG_CTRL_INPUT_READY (1 << 1) 60#define AES_REG_CTRL_INPUT_READY (1 << 1)
49#define AES_REG_CTRL_OUTPUT_READY (1 << 0) 61#define AES_REG_CTRL_OUTPUT_READY (1 << 0)
50 62
51#define AES_REG_DATA 0x34 63#define AES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04))
52#define AES_REG_DATA_N(x) (0x34 + ((x) * 0x04))
53 64
54#define AES_REG_REV 0x44 65#define AES_REG_REV(dd) ((dd)->pdata->rev_ofs)
55#define AES_REG_REV_MAJOR 0xF0
56#define AES_REG_REV_MINOR 0x0F
57 66
58#define AES_REG_MASK 0x48 67#define AES_REG_MASK(dd) ((dd)->pdata->mask_ofs)
59#define AES_REG_MASK_SIDLE (1 << 6) 68#define AES_REG_MASK_SIDLE (1 << 6)
60#define AES_REG_MASK_START (1 << 5) 69#define AES_REG_MASK_START (1 << 5)
61#define AES_REG_MASK_DMA_OUT_EN (1 << 3) 70#define AES_REG_MASK_DMA_OUT_EN (1 << 3)
@@ -63,8 +72,7 @@
63#define AES_REG_MASK_SOFTRESET (1 << 1) 72#define AES_REG_MASK_SOFTRESET (1 << 1)
64#define AES_REG_AUTOIDLE (1 << 0) 73#define AES_REG_AUTOIDLE (1 << 0)
65 74
66#define AES_REG_SYSSTATUS 0x4C 75#define AES_REG_LENGTH_N(x) (0x54 + ((x) * 0x04))
67#define AES_REG_SYSSTATUS_RESETDONE (1 << 0)
68 76
69#define DEFAULT_TIMEOUT (5*HZ) 77#define DEFAULT_TIMEOUT (5*HZ)
70 78
@@ -72,6 +80,7 @@
72#define FLAGS_ENCRYPT BIT(0) 80#define FLAGS_ENCRYPT BIT(0)
73#define FLAGS_CBC BIT(1) 81#define FLAGS_CBC BIT(1)
74#define FLAGS_GIV BIT(2) 82#define FLAGS_GIV BIT(2)
83#define FLAGS_CTR BIT(3)
75 84
76#define FLAGS_INIT BIT(4) 85#define FLAGS_INIT BIT(4)
77#define FLAGS_FAST BIT(5) 86#define FLAGS_FAST BIT(5)
@@ -92,11 +101,39 @@ struct omap_aes_reqctx {
92#define OMAP_AES_QUEUE_LENGTH 1 101#define OMAP_AES_QUEUE_LENGTH 1
93#define OMAP_AES_CACHE_SIZE 0 102#define OMAP_AES_CACHE_SIZE 0
94 103
104struct omap_aes_algs_info {
105 struct crypto_alg *algs_list;
106 unsigned int size;
107 unsigned int registered;
108};
109
110struct omap_aes_pdata {
111 struct omap_aes_algs_info *algs_info;
112 unsigned int algs_info_size;
113
114 void (*trigger)(struct omap_aes_dev *dd, int length);
115
116 u32 key_ofs;
117 u32 iv_ofs;
118 u32 ctrl_ofs;
119 u32 data_ofs;
120 u32 rev_ofs;
121 u32 mask_ofs;
122
123 u32 dma_enable_in;
124 u32 dma_enable_out;
125 u32 dma_start;
126
127 u32 major_mask;
128 u32 major_shift;
129 u32 minor_mask;
130 u32 minor_shift;
131};
132
95struct omap_aes_dev { 133struct omap_aes_dev {
96 struct list_head list; 134 struct list_head list;
97 unsigned long phys_base; 135 unsigned long phys_base;
98 void __iomem *io_base; 136 void __iomem *io_base;
99 struct clk *iclk;
100 struct omap_aes_ctx *ctx; 137 struct omap_aes_ctx *ctx;
101 struct device *dev; 138 struct device *dev;
102 unsigned long flags; 139 unsigned long flags;
@@ -111,20 +148,24 @@ struct omap_aes_dev {
111 struct ablkcipher_request *req; 148 struct ablkcipher_request *req;
112 size_t total; 149 size_t total;
113 struct scatterlist *in_sg; 150 struct scatterlist *in_sg;
151 struct scatterlist in_sgl;
114 size_t in_offset; 152 size_t in_offset;
115 struct scatterlist *out_sg; 153 struct scatterlist *out_sg;
154 struct scatterlist out_sgl;
116 size_t out_offset; 155 size_t out_offset;
117 156
118 size_t buflen; 157 size_t buflen;
119 void *buf_in; 158 void *buf_in;
120 size_t dma_size; 159 size_t dma_size;
121 int dma_in; 160 int dma_in;
122 int dma_lch_in; 161 struct dma_chan *dma_lch_in;
123 dma_addr_t dma_addr_in; 162 dma_addr_t dma_addr_in;
124 void *buf_out; 163 void *buf_out;
125 int dma_out; 164 int dma_out;
126 int dma_lch_out; 165 struct dma_chan *dma_lch_out;
127 dma_addr_t dma_addr_out; 166 dma_addr_t dma_addr_out;
167
168 const struct omap_aes_pdata *pdata;
128}; 169};
129 170
130/* keep registered devices data here */ 171/* keep registered devices data here */
@@ -160,19 +201,6 @@ static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,
160 omap_aes_write(dd, offset, *value); 201 omap_aes_write(dd, offset, *value);
161} 202}
162 203
163static int omap_aes_wait(struct omap_aes_dev *dd, u32 offset, u32 bit)
164{
165 unsigned long timeout = jiffies + DEFAULT_TIMEOUT;
166
167 while (!(omap_aes_read(dd, offset) & bit)) {
168 if (time_is_before_jiffies(timeout)) {
169 dev_err(dd->dev, "omap-aes timeout\n");
170 return -ETIMEDOUT;
171 }
172 }
173 return 0;
174}
175
176static int omap_aes_hw_init(struct omap_aes_dev *dd) 204static int omap_aes_hw_init(struct omap_aes_dev *dd)
177{ 205{
178 /* 206 /*
@@ -180,23 +208,9 @@ static int omap_aes_hw_init(struct omap_aes_dev *dd)
180 * It may be long delays between requests. 208 * It may be long delays between requests.
181 * Device might go to off mode to save power. 209 * Device might go to off mode to save power.
182 */ 210 */
183 clk_enable(dd->iclk); 211 pm_runtime_get_sync(dd->dev);
184 212
185 if (!(dd->flags & FLAGS_INIT)) { 213 if (!(dd->flags & FLAGS_INIT)) {
186 /* is it necessary to reset before every operation? */
187 omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_SOFTRESET,
188 AES_REG_MASK_SOFTRESET);
189 /*
190 * prevent OCP bus error (SRESP) in case an access to the module
191 * is performed while the module is coming out of soft reset
192 */
193 __asm__ __volatile__("nop");
194 __asm__ __volatile__("nop");
195
196 if (omap_aes_wait(dd, AES_REG_SYSSTATUS,
197 AES_REG_SYSSTATUS_RESETDONE))
198 return -ETIMEDOUT;
199
200 dd->flags |= FLAGS_INIT; 214 dd->flags |= FLAGS_INIT;
201 dd->err = 0; 215 dd->err = 0;
202 } 216 }
@@ -208,59 +222,75 @@ static int omap_aes_write_ctrl(struct omap_aes_dev *dd)
208{ 222{
209 unsigned int key32; 223 unsigned int key32;
210 int i, err; 224 int i, err;
211 u32 val, mask; 225 u32 val, mask = 0;
212 226
213 err = omap_aes_hw_init(dd); 227 err = omap_aes_hw_init(dd);
214 if (err) 228 if (err)
215 return err; 229 return err;
216 230
217 val = 0;
218 if (dd->dma_lch_out >= 0)
219 val |= AES_REG_MASK_DMA_OUT_EN;
220 if (dd->dma_lch_in >= 0)
221 val |= AES_REG_MASK_DMA_IN_EN;
222
223 mask = AES_REG_MASK_DMA_IN_EN | AES_REG_MASK_DMA_OUT_EN;
224
225 omap_aes_write_mask(dd, AES_REG_MASK, val, mask);
226
227 key32 = dd->ctx->keylen / sizeof(u32); 231 key32 = dd->ctx->keylen / sizeof(u32);
228 232
229 /* it seems a key should always be set even if it has not changed */ 233 /* it seems a key should always be set even if it has not changed */
230 for (i = 0; i < key32; i++) { 234 for (i = 0; i < key32; i++) {
231 omap_aes_write(dd, AES_REG_KEY(i), 235 omap_aes_write(dd, AES_REG_KEY(dd, i),
232 __le32_to_cpu(dd->ctx->key[i])); 236 __le32_to_cpu(dd->ctx->key[i]));
233 } 237 }
234 238
235 if ((dd->flags & FLAGS_CBC) && dd->req->info) 239 if ((dd->flags & (FLAGS_CBC | FLAGS_CTR)) && dd->req->info)
236 omap_aes_write_n(dd, AES_REG_IV(0), dd->req->info, 4); 240 omap_aes_write_n(dd, AES_REG_IV(dd, 0), dd->req->info, 4);
237 241
238 val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3); 242 val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
239 if (dd->flags & FLAGS_CBC) 243 if (dd->flags & FLAGS_CBC)
240 val |= AES_REG_CTRL_CBC; 244 val |= AES_REG_CTRL_CBC;
245 if (dd->flags & FLAGS_CTR) {
246 val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_32;
247 mask = AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_MASK;
248 }
241 if (dd->flags & FLAGS_ENCRYPT) 249 if (dd->flags & FLAGS_ENCRYPT)
242 val |= AES_REG_CTRL_DIRECTION; 250 val |= AES_REG_CTRL_DIRECTION;
243 251
244 mask = AES_REG_CTRL_CBC | AES_REG_CTRL_DIRECTION | 252 mask |= AES_REG_CTRL_CBC | AES_REG_CTRL_DIRECTION |
245 AES_REG_CTRL_KEY_SIZE; 253 AES_REG_CTRL_KEY_SIZE;
246 254
247 omap_aes_write_mask(dd, AES_REG_CTRL, val, mask); 255 omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, mask);
248 256
249 /* IN */ 257 return 0;
250 omap_set_dma_dest_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_CONSTANT, 258}
251 dd->phys_base + AES_REG_DATA, 0, 4);
252 259
253 omap_set_dma_dest_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4); 260static void omap_aes_dma_trigger_omap2(struct omap_aes_dev *dd, int length)
254 omap_set_dma_src_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4); 261{
262 u32 mask, val;
255 263
256 /* OUT */ 264 val = dd->pdata->dma_start;
257 omap_set_dma_src_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_CONSTANT,
258 dd->phys_base + AES_REG_DATA, 0, 4);
259 265
260 omap_set_dma_src_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4); 266 if (dd->dma_lch_out != NULL)
261 omap_set_dma_dest_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4); 267 val |= dd->pdata->dma_enable_out;
268 if (dd->dma_lch_in != NULL)
269 val |= dd->pdata->dma_enable_in;
270
271 mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
272 dd->pdata->dma_start;
273
274 omap_aes_write_mask(dd, AES_REG_MASK(dd), val, mask);
262 275
263 return 0; 276}
277
278static void omap_aes_dma_trigger_omap4(struct omap_aes_dev *dd, int length)
279{
280 omap_aes_write(dd, AES_REG_LENGTH_N(0), length);
281 omap_aes_write(dd, AES_REG_LENGTH_N(1), 0);
282
283 omap_aes_dma_trigger_omap2(dd, length);
284}
285
286static void omap_aes_dma_stop(struct omap_aes_dev *dd)
287{
288 u32 mask;
289
290 mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |
291 dd->pdata->dma_start;
292
293 omap_aes_write_mask(dd, AES_REG_MASK(dd), 0, mask);
264} 294}
265 295
266static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx) 296static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx)
@@ -284,18 +314,10 @@ static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx)
284 return dd; 314 return dd;
285} 315}
286 316
287static void omap_aes_dma_callback(int lch, u16 ch_status, void *data) 317static void omap_aes_dma_out_callback(void *data)
288{ 318{
289 struct omap_aes_dev *dd = data; 319 struct omap_aes_dev *dd = data;
290 320
291 if (ch_status != OMAP_DMA_BLOCK_IRQ) {
292 pr_err("omap-aes DMA error status: 0x%hx\n", ch_status);
293 dd->err = -EIO;
294 dd->flags &= ~FLAGS_INIT; /* request to re-initialize */
295 } else if (lch == dd->dma_lch_in) {
296 return;
297 }
298
299 /* dma_lch_out - completed */ 321 /* dma_lch_out - completed */
300 tasklet_schedule(&dd->done_task); 322 tasklet_schedule(&dd->done_task);
301} 323}
@@ -303,9 +325,10 @@ static void omap_aes_dma_callback(int lch, u16 ch_status, void *data)
303static int omap_aes_dma_init(struct omap_aes_dev *dd) 325static int omap_aes_dma_init(struct omap_aes_dev *dd)
304{ 326{
305 int err = -ENOMEM; 327 int err = -ENOMEM;
328 dma_cap_mask_t mask;
306 329
307 dd->dma_lch_out = -1; 330 dd->dma_lch_out = NULL;
308 dd->dma_lch_in = -1; 331 dd->dma_lch_in = NULL;
309 332
310 dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE); 333 dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
311 dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE); 334 dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
@@ -334,23 +357,31 @@ static int omap_aes_dma_init(struct omap_aes_dev *dd)
334 goto err_map_out; 357 goto err_map_out;
335 } 358 }
336 359
337 err = omap_request_dma(dd->dma_in, "omap-aes-rx", 360 dma_cap_zero(mask);
338 omap_aes_dma_callback, dd, &dd->dma_lch_in); 361 dma_cap_set(DMA_SLAVE, mask);
339 if (err) { 362
340 dev_err(dd->dev, "Unable to request DMA channel\n"); 363 dd->dma_lch_in = dma_request_slave_channel_compat(mask,
364 omap_dma_filter_fn,
365 &dd->dma_in,
366 dd->dev, "rx");
367 if (!dd->dma_lch_in) {
368 dev_err(dd->dev, "Unable to request in DMA channel\n");
341 goto err_dma_in; 369 goto err_dma_in;
342 } 370 }
343 err = omap_request_dma(dd->dma_out, "omap-aes-tx", 371
344 omap_aes_dma_callback, dd, &dd->dma_lch_out); 372 dd->dma_lch_out = dma_request_slave_channel_compat(mask,
345 if (err) { 373 omap_dma_filter_fn,
346 dev_err(dd->dev, "Unable to request DMA channel\n"); 374 &dd->dma_out,
375 dd->dev, "tx");
376 if (!dd->dma_lch_out) {
377 dev_err(dd->dev, "Unable to request out DMA channel\n");
347 goto err_dma_out; 378 goto err_dma_out;
348 } 379 }
349 380
350 return 0; 381 return 0;
351 382
352err_dma_out: 383err_dma_out:
353 omap_free_dma(dd->dma_lch_in); 384 dma_release_channel(dd->dma_lch_in);
354err_dma_in: 385err_dma_in:
355 dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, 386 dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
356 DMA_FROM_DEVICE); 387 DMA_FROM_DEVICE);
@@ -367,8 +398,8 @@ err_alloc:
367 398
368static void omap_aes_dma_cleanup(struct omap_aes_dev *dd) 399static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
369{ 400{
370 omap_free_dma(dd->dma_lch_out); 401 dma_release_channel(dd->dma_lch_out);
371 omap_free_dma(dd->dma_lch_in); 402 dma_release_channel(dd->dma_lch_in);
372 dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, 403 dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
373 DMA_FROM_DEVICE); 404 DMA_FROM_DEVICE);
374 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); 405 dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
@@ -426,12 +457,15 @@ static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf,
426 return off; 457 return off;
427} 458}
428 459
429static int omap_aes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in, 460static int omap_aes_crypt_dma(struct crypto_tfm *tfm,
430 dma_addr_t dma_addr_out, int length) 461 struct scatterlist *in_sg, struct scatterlist *out_sg)
431{ 462{
432 struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm); 463 struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
433 struct omap_aes_dev *dd = ctx->dd; 464 struct omap_aes_dev *dd = ctx->dd;
434 int len32; 465 struct dma_async_tx_descriptor *tx_in, *tx_out;
466 struct dma_slave_config cfg;
467 dma_addr_t dma_addr_in = sg_dma_address(in_sg);
468 int ret, length = sg_dma_len(in_sg);
435 469
436 pr_debug("len: %d\n", length); 470 pr_debug("len: %d\n", length);
437 471
@@ -441,30 +475,61 @@ static int omap_aes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
441 dma_sync_single_for_device(dd->dev, dma_addr_in, length, 475 dma_sync_single_for_device(dd->dev, dma_addr_in, length,
442 DMA_TO_DEVICE); 476 DMA_TO_DEVICE);
443 477
444 len32 = DIV_ROUND_UP(length, sizeof(u32)); 478 memset(&cfg, 0, sizeof(cfg));
479
480 cfg.src_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);
481 cfg.dst_addr = dd->phys_base + AES_REG_DATA_N(dd, 0);
482 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
483 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
484 cfg.src_maxburst = DST_MAXBURST;
485 cfg.dst_maxburst = DST_MAXBURST;
445 486
446 /* IN */ 487 /* IN */
447 omap_set_dma_transfer_params(dd->dma_lch_in, OMAP_DMA_DATA_TYPE_S32, 488 ret = dmaengine_slave_config(dd->dma_lch_in, &cfg);
448 len32, 1, OMAP_DMA_SYNC_PACKET, dd->dma_in, 489 if (ret) {
449 OMAP_DMA_DST_SYNC); 490 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
491 ret);
492 return ret;
493 }
494
495 tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, 1,
496 DMA_MEM_TO_DEV,
497 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
498 if (!tx_in) {
499 dev_err(dd->dev, "IN prep_slave_sg() failed\n");
500 return -EINVAL;
501 }
450 502
451 omap_set_dma_src_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_POST_INC, 503 /* No callback necessary */
452 dma_addr_in, 0, 0); 504 tx_in->callback_param = dd;
453 505
454 /* OUT */ 506 /* OUT */
455 omap_set_dma_transfer_params(dd->dma_lch_out, OMAP_DMA_DATA_TYPE_S32, 507 ret = dmaengine_slave_config(dd->dma_lch_out, &cfg);
456 len32, 1, OMAP_DMA_SYNC_PACKET, 508 if (ret) {
457 dd->dma_out, OMAP_DMA_SRC_SYNC); 509 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
510 ret);
511 return ret;
512 }
513
514 tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, 1,
515 DMA_DEV_TO_MEM,
516 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
517 if (!tx_out) {
518 dev_err(dd->dev, "OUT prep_slave_sg() failed\n");
519 return -EINVAL;
520 }
458 521
459 omap_set_dma_dest_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_POST_INC, 522 tx_out->callback = omap_aes_dma_out_callback;
460 dma_addr_out, 0, 0); 523 tx_out->callback_param = dd;
461 524
462 omap_start_dma(dd->dma_lch_in); 525 dmaengine_submit(tx_in);
463 omap_start_dma(dd->dma_lch_out); 526 dmaengine_submit(tx_out);
464 527
465 /* start DMA or disable idle mode */ 528 dma_async_issue_pending(dd->dma_lch_in);
466 omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_START, 529 dma_async_issue_pending(dd->dma_lch_out);
467 AES_REG_MASK_START); 530
531 /* start DMA */
532 dd->pdata->trigger(dd, length);
468 533
469 return 0; 534 return 0;
470} 535}
@@ -476,6 +541,8 @@ static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
476 int err, fast = 0, in, out; 541 int err, fast = 0, in, out;
477 size_t count; 542 size_t count;
478 dma_addr_t addr_in, addr_out; 543 dma_addr_t addr_in, addr_out;
544 struct scatterlist *in_sg, *out_sg;
545 int len32;
479 546
480 pr_debug("total: %d\n", dd->total); 547 pr_debug("total: %d\n", dd->total);
481 548
@@ -514,6 +581,9 @@ static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
514 addr_in = sg_dma_address(dd->in_sg); 581 addr_in = sg_dma_address(dd->in_sg);
515 addr_out = sg_dma_address(dd->out_sg); 582 addr_out = sg_dma_address(dd->out_sg);
516 583
584 in_sg = dd->in_sg;
585 out_sg = dd->out_sg;
586
517 dd->flags |= FLAGS_FAST; 587 dd->flags |= FLAGS_FAST;
518 588
519 } else { 589 } else {
@@ -521,6 +591,27 @@ static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
521 count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in, 591 count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in,
522 dd->buflen, dd->total, 0); 592 dd->buflen, dd->total, 0);
523 593
594 len32 = DIV_ROUND_UP(count, DMA_MIN) * DMA_MIN;
595
596 /*
597 * The data going into the AES module has been copied
598 * to a local buffer and the data coming out will go
599 * into a local buffer so set up local SG entries for
600 * both.
601 */
602 sg_init_table(&dd->in_sgl, 1);
603 dd->in_sgl.offset = dd->in_offset;
604 sg_dma_len(&dd->in_sgl) = len32;
605 sg_dma_address(&dd->in_sgl) = dd->dma_addr_in;
606
607 sg_init_table(&dd->out_sgl, 1);
608 dd->out_sgl.offset = dd->out_offset;
609 sg_dma_len(&dd->out_sgl) = len32;
610 sg_dma_address(&dd->out_sgl) = dd->dma_addr_out;
611
612 in_sg = &dd->in_sgl;
613 out_sg = &dd->out_sgl;
614
524 addr_in = dd->dma_addr_in; 615 addr_in = dd->dma_addr_in;
525 addr_out = dd->dma_addr_out; 616 addr_out = dd->dma_addr_out;
526 617
@@ -530,7 +621,7 @@ static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
530 621
531 dd->total -= count; 622 dd->total -= count;
532 623
533 err = omap_aes_crypt_dma(tfm, addr_in, addr_out, count); 624 err = omap_aes_crypt_dma(tfm, in_sg, out_sg);
534 if (err) { 625 if (err) {
535 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); 626 dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
536 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE); 627 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);
@@ -545,7 +636,7 @@ static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
545 636
546 pr_debug("err: %d\n", err); 637 pr_debug("err: %d\n", err);
547 638
548 clk_disable(dd->iclk); 639 pm_runtime_put_sync(dd->dev);
549 dd->flags &= ~FLAGS_BUSY; 640 dd->flags &= ~FLAGS_BUSY;
550 641
551 req->base.complete(&req->base, err); 642 req->base.complete(&req->base, err);
@@ -558,10 +649,10 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
558 649
559 pr_debug("total: %d\n", dd->total); 650 pr_debug("total: %d\n", dd->total);
560 651
561 omap_aes_write_mask(dd, AES_REG_MASK, 0, AES_REG_MASK_START); 652 omap_aes_dma_stop(dd);
562 653
563 omap_stop_dma(dd->dma_lch_in); 654 dmaengine_terminate_all(dd->dma_lch_in);
564 omap_stop_dma(dd->dma_lch_out); 655 dmaengine_terminate_all(dd->dma_lch_out);
565 656
566 if (dd->flags & FLAGS_FAST) { 657 if (dd->flags & FLAGS_FAST) {
567 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); 658 dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
@@ -734,6 +825,16 @@ static int omap_aes_cbc_decrypt(struct ablkcipher_request *req)
734 return omap_aes_crypt(req, FLAGS_CBC); 825 return omap_aes_crypt(req, FLAGS_CBC);
735} 826}
736 827
828static int omap_aes_ctr_encrypt(struct ablkcipher_request *req)
829{
830 return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CTR);
831}
832
833static int omap_aes_ctr_decrypt(struct ablkcipher_request *req)
834{
835 return omap_aes_crypt(req, FLAGS_CTR);
836}
837
737static int omap_aes_cra_init(struct crypto_tfm *tfm) 838static int omap_aes_cra_init(struct crypto_tfm *tfm)
738{ 839{
739 pr_debug("enter\n"); 840 pr_debug("enter\n");
@@ -750,7 +851,7 @@ static void omap_aes_cra_exit(struct crypto_tfm *tfm)
750 851
751/* ********************** ALGS ************************************ */ 852/* ********************** ALGS ************************************ */
752 853
753static struct crypto_alg algs[] = { 854static struct crypto_alg algs_ecb_cbc[] = {
754{ 855{
755 .cra_name = "ecb(aes)", 856 .cra_name = "ecb(aes)",
756 .cra_driver_name = "ecb-aes-omap", 857 .cra_driver_name = "ecb-aes-omap",
@@ -798,11 +899,213 @@ static struct crypto_alg algs[] = {
798} 899}
799}; 900};
800 901
902static struct crypto_alg algs_ctr[] = {
903{
904 .cra_name = "ctr(aes)",
905 .cra_driver_name = "ctr-aes-omap",
906 .cra_priority = 100,
907 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
908 CRYPTO_ALG_KERN_DRIVER_ONLY |
909 CRYPTO_ALG_ASYNC,
910 .cra_blocksize = AES_BLOCK_SIZE,
911 .cra_ctxsize = sizeof(struct omap_aes_ctx),
912 .cra_alignmask = 0,
913 .cra_type = &crypto_ablkcipher_type,
914 .cra_module = THIS_MODULE,
915 .cra_init = omap_aes_cra_init,
916 .cra_exit = omap_aes_cra_exit,
917 .cra_u.ablkcipher = {
918 .min_keysize = AES_MIN_KEY_SIZE,
919 .max_keysize = AES_MAX_KEY_SIZE,
920 .geniv = "eseqiv",
921 .ivsize = AES_BLOCK_SIZE,
922 .setkey = omap_aes_setkey,
923 .encrypt = omap_aes_ctr_encrypt,
924 .decrypt = omap_aes_ctr_decrypt,
925 }
926} ,
927};
928
929static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc[] = {
930 {
931 .algs_list = algs_ecb_cbc,
932 .size = ARRAY_SIZE(algs_ecb_cbc),
933 },
934};
935
936static const struct omap_aes_pdata omap_aes_pdata_omap2 = {
937 .algs_info = omap_aes_algs_info_ecb_cbc,
938 .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc),
939 .trigger = omap_aes_dma_trigger_omap2,
940 .key_ofs = 0x1c,
941 .iv_ofs = 0x20,
942 .ctrl_ofs = 0x30,
943 .data_ofs = 0x34,
944 .rev_ofs = 0x44,
945 .mask_ofs = 0x48,
946 .dma_enable_in = BIT(2),
947 .dma_enable_out = BIT(3),
948 .dma_start = BIT(5),
949 .major_mask = 0xf0,
950 .major_shift = 4,
951 .minor_mask = 0x0f,
952 .minor_shift = 0,
953};
954
955#ifdef CONFIG_OF
956static struct omap_aes_algs_info omap_aes_algs_info_ecb_cbc_ctr[] = {
957 {
958 .algs_list = algs_ecb_cbc,
959 .size = ARRAY_SIZE(algs_ecb_cbc),
960 },
961 {
962 .algs_list = algs_ctr,
963 .size = ARRAY_SIZE(algs_ctr),
964 },
965};
966
967static const struct omap_aes_pdata omap_aes_pdata_omap3 = {
968 .algs_info = omap_aes_algs_info_ecb_cbc_ctr,
969 .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr),
970 .trigger = omap_aes_dma_trigger_omap2,
971 .key_ofs = 0x1c,
972 .iv_ofs = 0x20,
973 .ctrl_ofs = 0x30,
974 .data_ofs = 0x34,
975 .rev_ofs = 0x44,
976 .mask_ofs = 0x48,
977 .dma_enable_in = BIT(2),
978 .dma_enable_out = BIT(3),
979 .dma_start = BIT(5),
980 .major_mask = 0xf0,
981 .major_shift = 4,
982 .minor_mask = 0x0f,
983 .minor_shift = 0,
984};
985
986static const struct omap_aes_pdata omap_aes_pdata_omap4 = {
987 .algs_info = omap_aes_algs_info_ecb_cbc_ctr,
988 .algs_info_size = ARRAY_SIZE(omap_aes_algs_info_ecb_cbc_ctr),
989 .trigger = omap_aes_dma_trigger_omap4,
990 .key_ofs = 0x3c,
991 .iv_ofs = 0x40,
992 .ctrl_ofs = 0x50,
993 .data_ofs = 0x60,
994 .rev_ofs = 0x80,
995 .mask_ofs = 0x84,
996 .dma_enable_in = BIT(5),
997 .dma_enable_out = BIT(6),
998 .major_mask = 0x0700,
999 .major_shift = 8,
1000 .minor_mask = 0x003f,
1001 .minor_shift = 0,
1002};
1003
1004static const struct of_device_id omap_aes_of_match[] = {
1005 {
1006 .compatible = "ti,omap2-aes",
1007 .data = &omap_aes_pdata_omap2,
1008 },
1009 {
1010 .compatible = "ti,omap3-aes",
1011 .data = &omap_aes_pdata_omap3,
1012 },
1013 {
1014 .compatible = "ti,omap4-aes",
1015 .data = &omap_aes_pdata_omap4,
1016 },
1017 {},
1018};
1019MODULE_DEVICE_TABLE(of, omap_aes_of_match);
1020
1021static int omap_aes_get_res_of(struct omap_aes_dev *dd,
1022 struct device *dev, struct resource *res)
1023{
1024 struct device_node *node = dev->of_node;
1025 const struct of_device_id *match;
1026 int err = 0;
1027
1028 match = of_match_device(of_match_ptr(omap_aes_of_match), dev);
1029 if (!match) {
1030 dev_err(dev, "no compatible OF match\n");
1031 err = -EINVAL;
1032 goto err;
1033 }
1034
1035 err = of_address_to_resource(node, 0, res);
1036 if (err < 0) {
1037 dev_err(dev, "can't translate OF node address\n");
1038 err = -EINVAL;
1039 goto err;
1040 }
1041
1042 dd->dma_out = -1; /* Dummy value that's unused */
1043 dd->dma_in = -1; /* Dummy value that's unused */
1044
1045 dd->pdata = match->data;
1046
1047err:
1048 return err;
1049}
1050#else
1051static const struct of_device_id omap_aes_of_match[] = {
1052 {},
1053};
1054
1055static int omap_aes_get_res_of(struct omap_aes_dev *dd,
1056 struct device *dev, struct resource *res)
1057{
1058 return -EINVAL;
1059}
1060#endif
1061
1062static int omap_aes_get_res_pdev(struct omap_aes_dev *dd,
1063 struct platform_device *pdev, struct resource *res)
1064{
1065 struct device *dev = &pdev->dev;
1066 struct resource *r;
1067 int err = 0;
1068
1069 /* Get the base address */
1070 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1071 if (!r) {
1072 dev_err(dev, "no MEM resource info\n");
1073 err = -ENODEV;
1074 goto err;
1075 }
1076 memcpy(res, r, sizeof(*res));
1077
1078 /* Get the DMA out channel */
1079 r = platform_get_resource(pdev, IORESOURCE_DMA, 0);
1080 if (!r) {
1081 dev_err(dev, "no DMA out resource info\n");
1082 err = -ENODEV;
1083 goto err;
1084 }
1085 dd->dma_out = r->start;
1086
1087 /* Get the DMA in channel */
1088 r = platform_get_resource(pdev, IORESOURCE_DMA, 1);
1089 if (!r) {
1090 dev_err(dev, "no DMA in resource info\n");
1091 err = -ENODEV;
1092 goto err;
1093 }
1094 dd->dma_in = r->start;
1095
1096 /* Only OMAP2/3 can be non-DT */
1097 dd->pdata = &omap_aes_pdata_omap2;
1098
1099err:
1100 return err;
1101}
1102
801static int omap_aes_probe(struct platform_device *pdev) 1103static int omap_aes_probe(struct platform_device *pdev)
802{ 1104{
803 struct device *dev = &pdev->dev; 1105 struct device *dev = &pdev->dev;
804 struct omap_aes_dev *dd; 1106 struct omap_aes_dev *dd;
805 struct resource *res; 1107 struct crypto_alg *algp;
1108 struct resource res;
806 int err = -ENOMEM, i, j; 1109 int err = -ENOMEM, i, j;
807 u32 reg; 1110 u32 reg;
808 1111
@@ -817,49 +1120,31 @@ static int omap_aes_probe(struct platform_device *pdev)
817 spin_lock_init(&dd->lock); 1120 spin_lock_init(&dd->lock);
818 crypto_init_queue(&dd->queue, OMAP_AES_QUEUE_LENGTH); 1121 crypto_init_queue(&dd->queue, OMAP_AES_QUEUE_LENGTH);
819 1122
820 /* Get the base address */ 1123 err = (dev->of_node) ? omap_aes_get_res_of(dd, dev, &res) :
821 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1124 omap_aes_get_res_pdev(dd, pdev, &res);
822 if (!res) { 1125 if (err)
823 dev_err(dev, "invalid resource type\n");
824 err = -ENODEV;
825 goto err_res;
826 }
827 dd->phys_base = res->start;
828
829 /* Get the DMA */
830 res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
831 if (!res)
832 dev_info(dev, "no DMA info\n");
833 else
834 dd->dma_out = res->start;
835
836 /* Get the DMA */
837 res = platform_get_resource(pdev, IORESOURCE_DMA, 1);
838 if (!res)
839 dev_info(dev, "no DMA info\n");
840 else
841 dd->dma_in = res->start;
842
843 /* Initializing the clock */
844 dd->iclk = clk_get(dev, "ick");
845 if (IS_ERR(dd->iclk)) {
846 dev_err(dev, "clock intialization failed.\n");
847 err = PTR_ERR(dd->iclk);
848 goto err_res; 1126 goto err_res;
849 }
850 1127
851 dd->io_base = ioremap(dd->phys_base, SZ_4K); 1128 dd->io_base = devm_request_and_ioremap(dev, &res);
852 if (!dd->io_base) { 1129 if (!dd->io_base) {
853 dev_err(dev, "can't ioremap\n"); 1130 dev_err(dev, "can't ioremap\n");
854 err = -ENOMEM; 1131 err = -ENOMEM;
855 goto err_io; 1132 goto err_res;
856 } 1133 }
1134 dd->phys_base = res.start;
1135
1136 pm_runtime_enable(dev);
1137 pm_runtime_get_sync(dev);
1138
1139 omap_aes_dma_stop(dd);
1140
1141 reg = omap_aes_read(dd, AES_REG_REV(dd));
1142
1143 pm_runtime_put_sync(dev);
857 1144
858 clk_enable(dd->iclk);
859 reg = omap_aes_read(dd, AES_REG_REV);
860 dev_info(dev, "OMAP AES hw accel rev: %u.%u\n", 1145 dev_info(dev, "OMAP AES hw accel rev: %u.%u\n",
861 (reg & AES_REG_REV_MAJOR) >> 4, reg & AES_REG_REV_MINOR); 1146 (reg & dd->pdata->major_mask) >> dd->pdata->major_shift,
862 clk_disable(dd->iclk); 1147 (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
863 1148
864 tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd); 1149 tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd);
865 tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd); 1150 tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd);
@@ -873,26 +1158,32 @@ static int omap_aes_probe(struct platform_device *pdev)
873 list_add_tail(&dd->list, &dev_list); 1158 list_add_tail(&dd->list, &dev_list);
874 spin_unlock(&list_lock); 1159 spin_unlock(&list_lock);
875 1160
876 for (i = 0; i < ARRAY_SIZE(algs); i++) { 1161 for (i = 0; i < dd->pdata->algs_info_size; i++) {
877 pr_debug("i: %d\n", i); 1162 for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
878 err = crypto_register_alg(&algs[i]); 1163 algp = &dd->pdata->algs_info[i].algs_list[j];
879 if (err) 1164
880 goto err_algs; 1165 pr_debug("reg alg: %s\n", algp->cra_name);
881 } 1166 INIT_LIST_HEAD(&algp->cra_list);
1167
1168 err = crypto_register_alg(algp);
1169 if (err)
1170 goto err_algs;
882 1171
883 pr_info("probe() done\n"); 1172 dd->pdata->algs_info[i].registered++;
1173 }
1174 }
884 1175
885 return 0; 1176 return 0;
886err_algs: 1177err_algs:
887 for (j = 0; j < i; j++) 1178 for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
888 crypto_unregister_alg(&algs[j]); 1179 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1180 crypto_unregister_alg(
1181 &dd->pdata->algs_info[i].algs_list[j]);
889 omap_aes_dma_cleanup(dd); 1182 omap_aes_dma_cleanup(dd);
890err_dma: 1183err_dma:
891 tasklet_kill(&dd->done_task); 1184 tasklet_kill(&dd->done_task);
892 tasklet_kill(&dd->queue_task); 1185 tasklet_kill(&dd->queue_task);
893 iounmap(dd->io_base); 1186 pm_runtime_disable(dev);
894err_io:
895 clk_put(dd->iclk);
896err_res: 1187err_res:
897 kfree(dd); 1188 kfree(dd);
898 dd = NULL; 1189 dd = NULL;
@@ -904,7 +1195,7 @@ err_data:
904static int omap_aes_remove(struct platform_device *pdev) 1195static int omap_aes_remove(struct platform_device *pdev)
905{ 1196{
906 struct omap_aes_dev *dd = platform_get_drvdata(pdev); 1197 struct omap_aes_dev *dd = platform_get_drvdata(pdev);
907 int i; 1198 int i, j;
908 1199
909 if (!dd) 1200 if (!dd)
910 return -ENODEV; 1201 return -ENODEV;
@@ -913,33 +1204,52 @@ static int omap_aes_remove(struct platform_device *pdev)
913 list_del(&dd->list); 1204 list_del(&dd->list);
914 spin_unlock(&list_lock); 1205 spin_unlock(&list_lock);
915 1206
916 for (i = 0; i < ARRAY_SIZE(algs); i++) 1207 for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
917 crypto_unregister_alg(&algs[i]); 1208 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1209 crypto_unregister_alg(
1210 &dd->pdata->algs_info[i].algs_list[j]);
918 1211
919 tasklet_kill(&dd->done_task); 1212 tasklet_kill(&dd->done_task);
920 tasklet_kill(&dd->queue_task); 1213 tasklet_kill(&dd->queue_task);
921 omap_aes_dma_cleanup(dd); 1214 omap_aes_dma_cleanup(dd);
922 iounmap(dd->io_base); 1215 pm_runtime_disable(dd->dev);
923 clk_put(dd->iclk);
924 kfree(dd); 1216 kfree(dd);
925 dd = NULL; 1217 dd = NULL;
926 1218
927 return 0; 1219 return 0;
928} 1220}
929 1221
1222#ifdef CONFIG_PM_SLEEP
1223static int omap_aes_suspend(struct device *dev)
1224{
1225 pm_runtime_put_sync(dev);
1226 return 0;
1227}
1228
1229static int omap_aes_resume(struct device *dev)
1230{
1231 pm_runtime_get_sync(dev);
1232 return 0;
1233}
1234#endif
1235
1236static const struct dev_pm_ops omap_aes_pm_ops = {
1237 SET_SYSTEM_SLEEP_PM_OPS(omap_aes_suspend, omap_aes_resume)
1238};
1239
930static struct platform_driver omap_aes_driver = { 1240static struct platform_driver omap_aes_driver = {
931 .probe = omap_aes_probe, 1241 .probe = omap_aes_probe,
932 .remove = omap_aes_remove, 1242 .remove = omap_aes_remove,
933 .driver = { 1243 .driver = {
934 .name = "omap-aes", 1244 .name = "omap-aes",
935 .owner = THIS_MODULE, 1245 .owner = THIS_MODULE,
1246 .pm = &omap_aes_pm_ops,
1247 .of_match_table = omap_aes_of_match,
936 }, 1248 },
937}; 1249};
938 1250
939static int __init omap_aes_mod_init(void) 1251static int __init omap_aes_mod_init(void)
940{ 1252{
941 pr_info("loading %s driver\n", "omap-aes");
942
943 return platform_driver_register(&omap_aes_driver); 1253 return platform_driver_register(&omap_aes_driver);
944} 1254}
945 1255
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 9e6947bc296f..3d1611f5aecf 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -5,6 +5,7 @@
5 * 5 *
6 * Copyright (c) 2010 Nokia Corporation 6 * Copyright (c) 2010 Nokia Corporation
7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com> 7 * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
8 * Copyright (c) 2011 Texas Instruments Incorporated
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as published 11 * it under the terms of the GNU General Public License version 2 as published
@@ -22,12 +23,18 @@
22#include <linux/errno.h> 23#include <linux/errno.h>
23#include <linux/interrupt.h> 24#include <linux/interrupt.h>
24#include <linux/kernel.h> 25#include <linux/kernel.h>
25#include <linux/clk.h>
26#include <linux/irq.h> 26#include <linux/irq.h>
27#include <linux/io.h> 27#include <linux/io.h>
28#include <linux/platform_device.h> 28#include <linux/platform_device.h>
29#include <linux/scatterlist.h> 29#include <linux/scatterlist.h>
30#include <linux/dma-mapping.h> 30#include <linux/dma-mapping.h>
31#include <linux/dmaengine.h>
32#include <linux/omap-dma.h>
33#include <linux/pm_runtime.h>
34#include <linux/of.h>
35#include <linux/of_device.h>
36#include <linux/of_address.h>
37#include <linux/of_irq.h>
31#include <linux/delay.h> 38#include <linux/delay.h>
32#include <linux/crypto.h> 39#include <linux/crypto.h>
33#include <linux/cryptohash.h> 40#include <linux/cryptohash.h>
@@ -37,19 +44,17 @@
37#include <crypto/hash.h> 44#include <crypto/hash.h>
38#include <crypto/internal/hash.h> 45#include <crypto/internal/hash.h>
39 46
40#include <linux/omap-dma.h>
41
42#ifdef CONFIG_ARCH_OMAP1
43#include <mach/irqs.h>
44#endif
45
46#define SHA_REG_DIGEST(x) (0x00 + ((x) * 0x04))
47#define SHA_REG_DIN(x) (0x1C + ((x) * 0x04))
48
49#define SHA1_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE 47#define SHA1_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE
50#define MD5_DIGEST_SIZE 16 48#define MD5_DIGEST_SIZE 16
51 49
52#define SHA_REG_DIGCNT 0x14 50#define DST_MAXBURST 16
51#define DMA_MIN (DST_MAXBURST * sizeof(u32))
52
53#define SHA_REG_IDIGEST(dd, x) ((dd)->pdata->idigest_ofs + ((x)*0x04))
54#define SHA_REG_DIN(dd, x) ((dd)->pdata->din_ofs + ((x) * 0x04))
55#define SHA_REG_DIGCNT(dd) ((dd)->pdata->digcnt_ofs)
56
57#define SHA_REG_ODIGEST(x) (0x00 + ((x) * 0x04))
53 58
54#define SHA_REG_CTRL 0x18 59#define SHA_REG_CTRL 0x18
55#define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5) 60#define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5)
@@ -59,19 +64,42 @@
59#define SHA_REG_CTRL_INPUT_READY (1 << 1) 64#define SHA_REG_CTRL_INPUT_READY (1 << 1)
60#define SHA_REG_CTRL_OUTPUT_READY (1 << 0) 65#define SHA_REG_CTRL_OUTPUT_READY (1 << 0)
61 66
62#define SHA_REG_REV 0x5C 67#define SHA_REG_REV(dd) ((dd)->pdata->rev_ofs)
63#define SHA_REG_REV_MAJOR 0xF0
64#define SHA_REG_REV_MINOR 0x0F
65 68
66#define SHA_REG_MASK 0x60 69#define SHA_REG_MASK(dd) ((dd)->pdata->mask_ofs)
67#define SHA_REG_MASK_DMA_EN (1 << 3) 70#define SHA_REG_MASK_DMA_EN (1 << 3)
68#define SHA_REG_MASK_IT_EN (1 << 2) 71#define SHA_REG_MASK_IT_EN (1 << 2)
69#define SHA_REG_MASK_SOFTRESET (1 << 1) 72#define SHA_REG_MASK_SOFTRESET (1 << 1)
70#define SHA_REG_AUTOIDLE (1 << 0) 73#define SHA_REG_AUTOIDLE (1 << 0)
71 74
72#define SHA_REG_SYSSTATUS 0x64 75#define SHA_REG_SYSSTATUS(dd) ((dd)->pdata->sysstatus_ofs)
73#define SHA_REG_SYSSTATUS_RESETDONE (1 << 0) 76#define SHA_REG_SYSSTATUS_RESETDONE (1 << 0)
74 77
78#define SHA_REG_MODE 0x44
79#define SHA_REG_MODE_HMAC_OUTER_HASH (1 << 7)
80#define SHA_REG_MODE_HMAC_KEY_PROC (1 << 5)
81#define SHA_REG_MODE_CLOSE_HASH (1 << 4)
82#define SHA_REG_MODE_ALGO_CONSTANT (1 << 3)
83#define SHA_REG_MODE_ALGO_MASK (3 << 1)
84#define SHA_REG_MODE_ALGO_MD5_128 (0 << 1)
85#define SHA_REG_MODE_ALGO_SHA1_160 (1 << 1)
86#define SHA_REG_MODE_ALGO_SHA2_224 (2 << 1)
87#define SHA_REG_MODE_ALGO_SHA2_256 (3 << 1)
88
89#define SHA_REG_LENGTH 0x48
90
91#define SHA_REG_IRQSTATUS 0x118
92#define SHA_REG_IRQSTATUS_CTX_RDY (1 << 3)
93#define SHA_REG_IRQSTATUS_PARTHASH_RDY (1 << 2)
94#define SHA_REG_IRQSTATUS_INPUT_RDY (1 << 1)
95#define SHA_REG_IRQSTATUS_OUTPUT_RDY (1 << 0)
96
97#define SHA_REG_IRQENA 0x11C
98#define SHA_REG_IRQENA_CTX_RDY (1 << 3)
99#define SHA_REG_IRQENA_PARTHASH_RDY (1 << 2)
100#define SHA_REG_IRQENA_INPUT_RDY (1 << 1)
101#define SHA_REG_IRQENA_OUTPUT_RDY (1 << 0)
102
75#define DEFAULT_TIMEOUT_INTERVAL HZ 103#define DEFAULT_TIMEOUT_INTERVAL HZ
76 104
77/* mostly device flags */ 105/* mostly device flags */
@@ -82,20 +110,33 @@
82#define FLAGS_INIT 4 110#define FLAGS_INIT 4
83#define FLAGS_CPU 5 111#define FLAGS_CPU 5
84#define FLAGS_DMA_READY 6 112#define FLAGS_DMA_READY 6
113#define FLAGS_AUTO_XOR 7
114#define FLAGS_BE32_SHA1 8
85/* context flags */ 115/* context flags */
86#define FLAGS_FINUP 16 116#define FLAGS_FINUP 16
87#define FLAGS_SG 17 117#define FLAGS_SG 17
88#define FLAGS_SHA1 18
89#define FLAGS_HMAC 19
90#define FLAGS_ERROR 20
91 118
92#define OP_UPDATE 1 119#define FLAGS_MODE_SHIFT 18
93#define OP_FINAL 2 120#define FLAGS_MODE_MASK (SHA_REG_MODE_ALGO_MASK \
121 << (FLAGS_MODE_SHIFT - 1))
122#define FLAGS_MODE_MD5 (SHA_REG_MODE_ALGO_MD5_128 \
123 << (FLAGS_MODE_SHIFT - 1))
124#define FLAGS_MODE_SHA1 (SHA_REG_MODE_ALGO_SHA1_160 \
125 << (FLAGS_MODE_SHIFT - 1))
126#define FLAGS_MODE_SHA224 (SHA_REG_MODE_ALGO_SHA2_224 \
127 << (FLAGS_MODE_SHIFT - 1))
128#define FLAGS_MODE_SHA256 (SHA_REG_MODE_ALGO_SHA2_256 \
129 << (FLAGS_MODE_SHIFT - 1))
130#define FLAGS_HMAC 20
131#define FLAGS_ERROR 21
132
133#define OP_UPDATE 1
134#define OP_FINAL 2
94 135
95#define OMAP_ALIGN_MASK (sizeof(u32)-1) 136#define OMAP_ALIGN_MASK (sizeof(u32)-1)
96#define OMAP_ALIGNED __attribute__((aligned(sizeof(u32)))) 137#define OMAP_ALIGNED __attribute__((aligned(sizeof(u32))))
97 138
98#define BUFLEN PAGE_SIZE 139#define BUFLEN PAGE_SIZE
99 140
100struct omap_sham_dev; 141struct omap_sham_dev;
101 142
@@ -104,7 +145,7 @@ struct omap_sham_reqctx {
104 unsigned long flags; 145 unsigned long flags;
105 unsigned long op; 146 unsigned long op;
106 147
107 u8 digest[SHA1_DIGEST_SIZE] OMAP_ALIGNED; 148 u8 digest[SHA256_DIGEST_SIZE] OMAP_ALIGNED;
108 size_t digcnt; 149 size_t digcnt;
109 size_t bufcnt; 150 size_t bufcnt;
110 size_t buflen; 151 size_t buflen;
@@ -112,6 +153,7 @@ struct omap_sham_reqctx {
112 153
113 /* walk state */ 154 /* walk state */
114 struct scatterlist *sg; 155 struct scatterlist *sg;
156 struct scatterlist sgl;
115 unsigned int offset; /* offset in current sg */ 157 unsigned int offset; /* offset in current sg */
116 unsigned int total; /* total request */ 158 unsigned int total; /* total request */
117 159
@@ -120,8 +162,8 @@ struct omap_sham_reqctx {
120 162
121struct omap_sham_hmac_ctx { 163struct omap_sham_hmac_ctx {
122 struct crypto_shash *shash; 164 struct crypto_shash *shash;
123 u8 ipad[SHA1_MD5_BLOCK_SIZE]; 165 u8 ipad[SHA1_MD5_BLOCK_SIZE] OMAP_ALIGNED;
124 u8 opad[SHA1_MD5_BLOCK_SIZE]; 166 u8 opad[SHA1_MD5_BLOCK_SIZE] OMAP_ALIGNED;
125}; 167};
126 168
127struct omap_sham_ctx { 169struct omap_sham_ctx {
@@ -137,22 +179,56 @@ struct omap_sham_ctx {
137 179
138#define OMAP_SHAM_QUEUE_LENGTH 1 180#define OMAP_SHAM_QUEUE_LENGTH 1
139 181
182struct omap_sham_algs_info {
183 struct ahash_alg *algs_list;
184 unsigned int size;
185 unsigned int registered;
186};
187
188struct omap_sham_pdata {
189 struct omap_sham_algs_info *algs_info;
190 unsigned int algs_info_size;
191 unsigned long flags;
192 int digest_size;
193
194 void (*copy_hash)(struct ahash_request *req, int out);
195 void (*write_ctrl)(struct omap_sham_dev *dd, size_t length,
196 int final, int dma);
197 void (*trigger)(struct omap_sham_dev *dd, size_t length);
198 int (*poll_irq)(struct omap_sham_dev *dd);
199 irqreturn_t (*intr_hdlr)(int irq, void *dev_id);
200
201 u32 odigest_ofs;
202 u32 idigest_ofs;
203 u32 din_ofs;
204 u32 digcnt_ofs;
205 u32 rev_ofs;
206 u32 mask_ofs;
207 u32 sysstatus_ofs;
208
209 u32 major_mask;
210 u32 major_shift;
211 u32 minor_mask;
212 u32 minor_shift;
213};
214
140struct omap_sham_dev { 215struct omap_sham_dev {
141 struct list_head list; 216 struct list_head list;
142 unsigned long phys_base; 217 unsigned long phys_base;
143 struct device *dev; 218 struct device *dev;
144 void __iomem *io_base; 219 void __iomem *io_base;
145 int irq; 220 int irq;
146 struct clk *iclk;
147 spinlock_t lock; 221 spinlock_t lock;
148 int err; 222 int err;
149 int dma; 223 unsigned int dma;
150 int dma_lch; 224 struct dma_chan *dma_lch;
151 struct tasklet_struct done_task; 225 struct tasklet_struct done_task;
152 226
153 unsigned long flags; 227 unsigned long flags;
154 struct crypto_queue queue; 228 struct crypto_queue queue;
155 struct ahash_request *req; 229 struct ahash_request *req;
230
231 const struct omap_sham_pdata *pdata;
156}; 232};
157 233
158struct omap_sham_drv { 234struct omap_sham_drv {
@@ -200,21 +276,44 @@ static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit)
200 return 0; 276 return 0;
201} 277}
202 278
203static void omap_sham_copy_hash(struct ahash_request *req, int out) 279static void omap_sham_copy_hash_omap2(struct ahash_request *req, int out)
204{ 280{
205 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 281 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
282 struct omap_sham_dev *dd = ctx->dd;
206 u32 *hash = (u32 *)ctx->digest; 283 u32 *hash = (u32 *)ctx->digest;
207 int i; 284 int i;
208 285
209 /* MD5 is almost unused. So copy sha1 size to reduce code */ 286 for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
210 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) {
211 if (out) 287 if (out)
212 hash[i] = omap_sham_read(ctx->dd, 288 hash[i] = omap_sham_read(dd, SHA_REG_IDIGEST(dd, i));
213 SHA_REG_DIGEST(i));
214 else 289 else
215 omap_sham_write(ctx->dd, 290 omap_sham_write(dd, SHA_REG_IDIGEST(dd, i), hash[i]);
216 SHA_REG_DIGEST(i), hash[i]); 291 }
292}
293
294static void omap_sham_copy_hash_omap4(struct ahash_request *req, int out)
295{
296 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
297 struct omap_sham_dev *dd = ctx->dd;
298 int i;
299
300 if (ctx->flags & BIT(FLAGS_HMAC)) {
301 struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
302 struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
303 struct omap_sham_hmac_ctx *bctx = tctx->base;
304 u32 *opad = (u32 *)bctx->opad;
305
306 for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
307 if (out)
308 opad[i] = omap_sham_read(dd,
309 SHA_REG_ODIGEST(i));
310 else
311 omap_sham_write(dd, SHA_REG_ODIGEST(i),
312 opad[i]);
313 }
217 } 314 }
315
316 omap_sham_copy_hash_omap2(req, out);
218} 317}
219 318
220static void omap_sham_copy_ready_hash(struct ahash_request *req) 319static void omap_sham_copy_ready_hash(struct ahash_request *req)
@@ -222,34 +321,44 @@ static void omap_sham_copy_ready_hash(struct ahash_request *req)
222 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 321 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
223 u32 *in = (u32 *)ctx->digest; 322 u32 *in = (u32 *)ctx->digest;
224 u32 *hash = (u32 *)req->result; 323 u32 *hash = (u32 *)req->result;
225 int i; 324 int i, d, big_endian = 0;
226 325
227 if (!hash) 326 if (!hash)
228 return; 327 return;
229 328
230 if (likely(ctx->flags & BIT(FLAGS_SHA1))) { 329 switch (ctx->flags & FLAGS_MODE_MASK) {
231 /* SHA1 results are in big endian */ 330 case FLAGS_MODE_MD5:
232 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) 331 d = MD5_DIGEST_SIZE / sizeof(u32);
332 break;
333 case FLAGS_MODE_SHA1:
334 /* OMAP2 SHA1 is big endian */
335 if (test_bit(FLAGS_BE32_SHA1, &ctx->dd->flags))
336 big_endian = 1;
337 d = SHA1_DIGEST_SIZE / sizeof(u32);
338 break;
339 case FLAGS_MODE_SHA224:
340 d = SHA224_DIGEST_SIZE / sizeof(u32);
341 break;
342 case FLAGS_MODE_SHA256:
343 d = SHA256_DIGEST_SIZE / sizeof(u32);
344 break;
345 default:
346 d = 0;
347 }
348
349 if (big_endian)
350 for (i = 0; i < d; i++)
233 hash[i] = be32_to_cpu(in[i]); 351 hash[i] = be32_to_cpu(in[i]);
234 } else { 352 else
235 /* MD5 results are in little endian */ 353 for (i = 0; i < d; i++)
236 for (i = 0; i < MD5_DIGEST_SIZE / sizeof(u32); i++)
237 hash[i] = le32_to_cpu(in[i]); 354 hash[i] = le32_to_cpu(in[i]);
238 }
239} 355}
240 356
241static int omap_sham_hw_init(struct omap_sham_dev *dd) 357static int omap_sham_hw_init(struct omap_sham_dev *dd)
242{ 358{
243 clk_enable(dd->iclk); 359 pm_runtime_get_sync(dd->dev);
244 360
245 if (!test_bit(FLAGS_INIT, &dd->flags)) { 361 if (!test_bit(FLAGS_INIT, &dd->flags)) {
246 omap_sham_write_mask(dd, SHA_REG_MASK,
247 SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET);
248
249 if (omap_sham_wait(dd, SHA_REG_SYSSTATUS,
250 SHA_REG_SYSSTATUS_RESETDONE))
251 return -ETIMEDOUT;
252
253 set_bit(FLAGS_INIT, &dd->flags); 362 set_bit(FLAGS_INIT, &dd->flags);
254 dd->err = 0; 363 dd->err = 0;
255 } 364 }
@@ -257,23 +366,23 @@ static int omap_sham_hw_init(struct omap_sham_dev *dd)
257 return 0; 366 return 0;
258} 367}
259 368
260static void omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length, 369static void omap_sham_write_ctrl_omap2(struct omap_sham_dev *dd, size_t length,
261 int final, int dma) 370 int final, int dma)
262{ 371{
263 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 372 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
264 u32 val = length << 5, mask; 373 u32 val = length << 5, mask;
265 374
266 if (likely(ctx->digcnt)) 375 if (likely(ctx->digcnt))
267 omap_sham_write(dd, SHA_REG_DIGCNT, ctx->digcnt); 376 omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt);
268 377
269 omap_sham_write_mask(dd, SHA_REG_MASK, 378 omap_sham_write_mask(dd, SHA_REG_MASK(dd),
270 SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0), 379 SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0),
271 SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN); 380 SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
272 /* 381 /*
273 * Setting ALGO_CONST only for the first iteration 382 * Setting ALGO_CONST only for the first iteration
274 * and CLOSE_HASH only for the last one. 383 * and CLOSE_HASH only for the last one.
275 */ 384 */
276 if (ctx->flags & BIT(FLAGS_SHA1)) 385 if ((ctx->flags & FLAGS_MODE_MASK) == FLAGS_MODE_SHA1)
277 val |= SHA_REG_CTRL_ALGO; 386 val |= SHA_REG_CTRL_ALGO;
278 if (!ctx->digcnt) 387 if (!ctx->digcnt)
279 val |= SHA_REG_CTRL_ALGO_CONST; 388 val |= SHA_REG_CTRL_ALGO_CONST;
@@ -286,6 +395,81 @@ static void omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length,
286 omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask); 395 omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask);
287} 396}
288 397
398static void omap_sham_trigger_omap2(struct omap_sham_dev *dd, size_t length)
399{
400}
401
402static int omap_sham_poll_irq_omap2(struct omap_sham_dev *dd)
403{
404 return omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY);
405}
406
407static void omap_sham_write_n(struct omap_sham_dev *dd, u32 offset,
408 u32 *value, int count)
409{
410 for (; count--; value++, offset += 4)
411 omap_sham_write(dd, offset, *value);
412}
413
414static void omap_sham_write_ctrl_omap4(struct omap_sham_dev *dd, size_t length,
415 int final, int dma)
416{
417 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
418 u32 val, mask;
419
420 /*
421 * Setting ALGO_CONST only for the first iteration and
422 * CLOSE_HASH only for the last one. Note that flags mode bits
423 * correspond to algorithm encoding in mode register.
424 */
425 val = (ctx->flags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT - 1);
426 if (!ctx->digcnt) {
427 struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
428 struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
429 struct omap_sham_hmac_ctx *bctx = tctx->base;
430
431 val |= SHA_REG_MODE_ALGO_CONSTANT;
432
433 if (ctx->flags & BIT(FLAGS_HMAC)) {
434 val |= SHA_REG_MODE_HMAC_KEY_PROC;
435 omap_sham_write_n(dd, SHA_REG_ODIGEST(0),
436 (u32 *)bctx->ipad,
437 SHA1_BLOCK_SIZE / sizeof(u32));
438 ctx->digcnt += SHA1_BLOCK_SIZE;
439 }
440 }
441
442 if (final) {
443 val |= SHA_REG_MODE_CLOSE_HASH;
444
445 if (ctx->flags & BIT(FLAGS_HMAC))
446 val |= SHA_REG_MODE_HMAC_OUTER_HASH;
447 }
448
449 mask = SHA_REG_MODE_ALGO_CONSTANT | SHA_REG_MODE_CLOSE_HASH |
450 SHA_REG_MODE_ALGO_MASK | SHA_REG_MODE_HMAC_OUTER_HASH |
451 SHA_REG_MODE_HMAC_KEY_PROC;
452
453 dev_dbg(dd->dev, "ctrl: %08x, flags: %08lx\n", val, ctx->flags);
454 omap_sham_write_mask(dd, SHA_REG_MODE, val, mask);
455 omap_sham_write(dd, SHA_REG_IRQENA, SHA_REG_IRQENA_OUTPUT_RDY);
456 omap_sham_write_mask(dd, SHA_REG_MASK(dd),
457 SHA_REG_MASK_IT_EN |
458 (dma ? SHA_REG_MASK_DMA_EN : 0),
459 SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
460}
461
462static void omap_sham_trigger_omap4(struct omap_sham_dev *dd, size_t length)
463{
464 omap_sham_write(dd, SHA_REG_LENGTH, length);
465}
466
467static int omap_sham_poll_irq_omap4(struct omap_sham_dev *dd)
468{
469 return omap_sham_wait(dd, SHA_REG_IRQSTATUS,
470 SHA_REG_IRQSTATUS_INPUT_RDY);
471}
472
289static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf, 473static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
290 size_t length, int final) 474 size_t length, int final)
291{ 475{
@@ -296,12 +480,13 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
296 dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n", 480 dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
297 ctx->digcnt, length, final); 481 ctx->digcnt, length, final);
298 482
299 omap_sham_write_ctrl(dd, length, final, 0); 483 dd->pdata->write_ctrl(dd, length, final, 0);
484 dd->pdata->trigger(dd, length);
300 485
301 /* should be non-zero before next lines to disable clocks later */ 486 /* should be non-zero before next lines to disable clocks later */
302 ctx->digcnt += length; 487 ctx->digcnt += length;
303 488
304 if (omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY)) 489 if (dd->pdata->poll_irq(dd))
305 return -ETIMEDOUT; 490 return -ETIMEDOUT;
306 491
307 if (final) 492 if (final)
@@ -312,30 +497,73 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
312 len32 = DIV_ROUND_UP(length, sizeof(u32)); 497 len32 = DIV_ROUND_UP(length, sizeof(u32));
313 498
314 for (count = 0; count < len32; count++) 499 for (count = 0; count < len32; count++)
315 omap_sham_write(dd, SHA_REG_DIN(count), buffer[count]); 500 omap_sham_write(dd, SHA_REG_DIN(dd, count), buffer[count]);
316 501
317 return -EINPROGRESS; 502 return -EINPROGRESS;
318} 503}
319 504
505static void omap_sham_dma_callback(void *param)
506{
507 struct omap_sham_dev *dd = param;
508
509 set_bit(FLAGS_DMA_READY, &dd->flags);
510 tasklet_schedule(&dd->done_task);
511}
512
320static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr, 513static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
321 size_t length, int final) 514 size_t length, int final, int is_sg)
322{ 515{
323 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 516 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
324 int len32; 517 struct dma_async_tx_descriptor *tx;
518 struct dma_slave_config cfg;
519 int len32, ret;
325 520
326 dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n", 521 dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
327 ctx->digcnt, length, final); 522 ctx->digcnt, length, final);
328 523
329 len32 = DIV_ROUND_UP(length, sizeof(u32)); 524 memset(&cfg, 0, sizeof(cfg));
525
526 cfg.dst_addr = dd->phys_base + SHA_REG_DIN(dd, 0);
527 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
528 cfg.dst_maxburst = DST_MAXBURST;
529
530 ret = dmaengine_slave_config(dd->dma_lch, &cfg);
531 if (ret) {
532 pr_err("omap-sham: can't configure dmaengine slave: %d\n", ret);
533 return ret;
534 }
535
536 len32 = DIV_ROUND_UP(length, DMA_MIN) * DMA_MIN;
537
538 if (is_sg) {
539 /*
540 * The SG entry passed in may not have the 'length' member
541 * set correctly so use a local SG entry (sgl) with the
542 * proper value for 'length' instead. If this is not done,
543 * the dmaengine may try to DMA the incorrect amount of data.
544 */
545 sg_init_table(&ctx->sgl, 1);
546 ctx->sgl.page_link = ctx->sg->page_link;
547 ctx->sgl.offset = ctx->sg->offset;
548 sg_dma_len(&ctx->sgl) = len32;
549 sg_dma_address(&ctx->sgl) = sg_dma_address(ctx->sg);
550
551 tx = dmaengine_prep_slave_sg(dd->dma_lch, &ctx->sgl, 1,
552 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
553 } else {
554 tx = dmaengine_prep_slave_single(dd->dma_lch, dma_addr, len32,
555 DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
556 }
330 557
331 omap_set_dma_transfer_params(dd->dma_lch, OMAP_DMA_DATA_TYPE_S32, len32, 558 if (!tx) {
332 1, OMAP_DMA_SYNC_PACKET, dd->dma, 559 dev_err(dd->dev, "prep_slave_sg/single() failed\n");
333 OMAP_DMA_DST_SYNC_PREFETCH); 560 return -EINVAL;
561 }
334 562
335 omap_set_dma_src_params(dd->dma_lch, 0, OMAP_DMA_AMODE_POST_INC, 563 tx->callback = omap_sham_dma_callback;
336 dma_addr, 0, 0); 564 tx->callback_param = dd;
337 565
338 omap_sham_write_ctrl(dd, length, final, 1); 566 dd->pdata->write_ctrl(dd, length, final, 1);
339 567
340 ctx->digcnt += length; 568 ctx->digcnt += length;
341 569
@@ -344,7 +572,10 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
344 572
345 set_bit(FLAGS_DMA_ACTIVE, &dd->flags); 573 set_bit(FLAGS_DMA_ACTIVE, &dd->flags);
346 574
347 omap_start_dma(dd->dma_lch); 575 dmaengine_submit(tx);
576 dma_async_issue_pending(dd->dma_lch);
577
578 dd->pdata->trigger(dd, length);
348 579
349 return -EINPROGRESS; 580 return -EINPROGRESS;
350} 581}
@@ -391,6 +622,8 @@ static int omap_sham_xmit_dma_map(struct omap_sham_dev *dd,
391 struct omap_sham_reqctx *ctx, 622 struct omap_sham_reqctx *ctx,
392 size_t length, int final) 623 size_t length, int final)
393{ 624{
625 int ret;
626
394 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen, 627 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,
395 DMA_TO_DEVICE); 628 DMA_TO_DEVICE);
396 if (dma_mapping_error(dd->dev, ctx->dma_addr)) { 629 if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
@@ -400,8 +633,12 @@ static int omap_sham_xmit_dma_map(struct omap_sham_dev *dd,
400 633
401 ctx->flags &= ~BIT(FLAGS_SG); 634 ctx->flags &= ~BIT(FLAGS_SG);
402 635
403 /* next call does not fail... so no unmap in the case of error */ 636 ret = omap_sham_xmit_dma(dd, ctx->dma_addr, length, final, 0);
404 return omap_sham_xmit_dma(dd, ctx->dma_addr, length, final); 637 if (ret != -EINPROGRESS)
638 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
639 DMA_TO_DEVICE);
640
641 return ret;
405} 642}
406 643
407static int omap_sham_update_dma_slow(struct omap_sham_dev *dd) 644static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
@@ -436,6 +673,7 @@ static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
436 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 673 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
437 unsigned int length, final, tail; 674 unsigned int length, final, tail;
438 struct scatterlist *sg; 675 struct scatterlist *sg;
676 int ret;
439 677
440 if (!ctx->total) 678 if (!ctx->total)
441 return 0; 679 return 0;
@@ -443,6 +681,15 @@ static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
443 if (ctx->bufcnt || ctx->offset) 681 if (ctx->bufcnt || ctx->offset)
444 return omap_sham_update_dma_slow(dd); 682 return omap_sham_update_dma_slow(dd);
445 683
684 /*
685 * Don't use the sg interface when the transfer size is less
686 * than the number of elements in a DMA frame. Otherwise,
687 * the dmaengine infrastructure will calculate that it needs
688 * to transfer 0 frames which ultimately fails.
689 */
690 if (ctx->total < (DST_MAXBURST * sizeof(u32)))
691 return omap_sham_update_dma_slow(dd);
692
446 dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n", 693 dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
447 ctx->digcnt, ctx->bufcnt, ctx->total); 694 ctx->digcnt, ctx->bufcnt, ctx->total);
448 695
@@ -480,8 +727,11 @@ static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
480 727
481 final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total; 728 final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
482 729
483 /* next call does not fail... so no unmap in the case of error */ 730 ret = omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final, 1);
484 return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final); 731 if (ret != -EINPROGRESS)
732 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
733
734 return ret;
485} 735}
486 736
487static int omap_sham_update_cpu(struct omap_sham_dev *dd) 737static int omap_sham_update_cpu(struct omap_sham_dev *dd)
@@ -500,7 +750,8 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
500{ 750{
501 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 751 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
502 752
503 omap_stop_dma(dd->dma_lch); 753 dmaengine_terminate_all(dd->dma_lch);
754
504 if (ctx->flags & BIT(FLAGS_SG)) { 755 if (ctx->flags & BIT(FLAGS_SG)) {
505 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); 756 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
506 if (ctx->sg->length == ctx->offset) { 757 if (ctx->sg->length == ctx->offset) {
@@ -542,18 +793,33 @@ static int omap_sham_init(struct ahash_request *req)
542 dev_dbg(dd->dev, "init: digest size: %d\n", 793 dev_dbg(dd->dev, "init: digest size: %d\n",
543 crypto_ahash_digestsize(tfm)); 794 crypto_ahash_digestsize(tfm));
544 795
545 if (crypto_ahash_digestsize(tfm) == SHA1_DIGEST_SIZE) 796 switch (crypto_ahash_digestsize(tfm)) {
546 ctx->flags |= BIT(FLAGS_SHA1); 797 case MD5_DIGEST_SIZE:
798 ctx->flags |= FLAGS_MODE_MD5;
799 break;
800 case SHA1_DIGEST_SIZE:
801 ctx->flags |= FLAGS_MODE_SHA1;
802 break;
803 case SHA224_DIGEST_SIZE:
804 ctx->flags |= FLAGS_MODE_SHA224;
805 break;
806 case SHA256_DIGEST_SIZE:
807 ctx->flags |= FLAGS_MODE_SHA256;
808 break;
809 }
547 810
548 ctx->bufcnt = 0; 811 ctx->bufcnt = 0;
549 ctx->digcnt = 0; 812 ctx->digcnt = 0;
550 ctx->buflen = BUFLEN; 813 ctx->buflen = BUFLEN;
551 814
552 if (tctx->flags & BIT(FLAGS_HMAC)) { 815 if (tctx->flags & BIT(FLAGS_HMAC)) {
553 struct omap_sham_hmac_ctx *bctx = tctx->base; 816 if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) {
817 struct omap_sham_hmac_ctx *bctx = tctx->base;
818
819 memcpy(ctx->buffer, bctx->ipad, SHA1_MD5_BLOCK_SIZE);
820 ctx->bufcnt = SHA1_MD5_BLOCK_SIZE;
821 }
554 822
555 memcpy(ctx->buffer, bctx->ipad, SHA1_MD5_BLOCK_SIZE);
556 ctx->bufcnt = SHA1_MD5_BLOCK_SIZE;
557 ctx->flags |= BIT(FLAGS_HMAC); 823 ctx->flags |= BIT(FLAGS_HMAC);
558 } 824 }
559 825
@@ -587,7 +853,7 @@ static int omap_sham_final_req(struct omap_sham_dev *dd)
587 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 853 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
588 int err = 0, use_dma = 1; 854 int err = 0, use_dma = 1;
589 855
590 if (ctx->bufcnt <= 64) 856 if (ctx->bufcnt <= DMA_MIN)
591 /* faster to handle last block with cpu */ 857 /* faster to handle last block with cpu */
592 use_dma = 0; 858 use_dma = 0;
593 859
@@ -630,7 +896,8 @@ static int omap_sham_finish(struct ahash_request *req)
630 896
631 if (ctx->digcnt) { 897 if (ctx->digcnt) {
632 omap_sham_copy_ready_hash(req); 898 omap_sham_copy_ready_hash(req);
633 if (ctx->flags & BIT(FLAGS_HMAC)) 899 if ((ctx->flags & BIT(FLAGS_HMAC)) &&
900 !test_bit(FLAGS_AUTO_XOR, &dd->flags))
634 err = omap_sham_finish_hmac(req); 901 err = omap_sham_finish_hmac(req);
635 } 902 }
636 903
@@ -645,7 +912,7 @@ static void omap_sham_finish_req(struct ahash_request *req, int err)
645 struct omap_sham_dev *dd = ctx->dd; 912 struct omap_sham_dev *dd = ctx->dd;
646 913
647 if (!err) { 914 if (!err) {
648 omap_sham_copy_hash(req, 1); 915 dd->pdata->copy_hash(req, 1);
649 if (test_bit(FLAGS_FINAL, &dd->flags)) 916 if (test_bit(FLAGS_FINAL, &dd->flags))
650 err = omap_sham_finish(req); 917 err = omap_sham_finish(req);
651 } else { 918 } else {
@@ -655,7 +922,8 @@ static void omap_sham_finish_req(struct ahash_request *req, int err)
655 /* atomic operation is not needed here */ 922 /* atomic operation is not needed here */
656 dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) | 923 dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) |
657 BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY)); 924 BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY));
658 clk_disable(dd->iclk); 925
926 pm_runtime_put_sync(dd->dev);
659 927
660 if (req->base.complete) 928 if (req->base.complete)
661 req->base.complete(&req->base, err); 929 req->base.complete(&req->base, err);
@@ -702,19 +970,9 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd,
702 if (err) 970 if (err)
703 goto err1; 971 goto err1;
704 972
705 omap_set_dma_dest_params(dd->dma_lch, 0,
706 OMAP_DMA_AMODE_CONSTANT,
707 dd->phys_base + SHA_REG_DIN(0), 0, 16);
708
709 omap_set_dma_dest_burst_mode(dd->dma_lch,
710 OMAP_DMA_DATA_BURST_16);
711
712 omap_set_dma_src_burst_mode(dd->dma_lch,
713 OMAP_DMA_DATA_BURST_4);
714
715 if (ctx->digcnt) 973 if (ctx->digcnt)
716 /* request has changed - restore hash */ 974 /* request has changed - restore hash */
717 omap_sham_copy_hash(req, 0); 975 dd->pdata->copy_hash(req, 0);
718 976
719 if (ctx->op == OP_UPDATE) { 977 if (ctx->op == OP_UPDATE) {
720 err = omap_sham_update_req(dd); 978 err = omap_sham_update_req(dd);
@@ -853,7 +1111,21 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
853 struct omap_sham_hmac_ctx *bctx = tctx->base; 1111 struct omap_sham_hmac_ctx *bctx = tctx->base;
854 int bs = crypto_shash_blocksize(bctx->shash); 1112 int bs = crypto_shash_blocksize(bctx->shash);
855 int ds = crypto_shash_digestsize(bctx->shash); 1113 int ds = crypto_shash_digestsize(bctx->shash);
1114 struct omap_sham_dev *dd = NULL, *tmp;
856 int err, i; 1115 int err, i;
1116
1117 spin_lock_bh(&sham.lock);
1118 if (!tctx->dd) {
1119 list_for_each_entry(tmp, &sham.dev_list, list) {
1120 dd = tmp;
1121 break;
1122 }
1123 tctx->dd = dd;
1124 } else {
1125 dd = tctx->dd;
1126 }
1127 spin_unlock_bh(&sham.lock);
1128
857 err = crypto_shash_setkey(tctx->fallback, key, keylen); 1129 err = crypto_shash_setkey(tctx->fallback, key, keylen);
858 if (err) 1130 if (err)
859 return err; 1131 return err;
@@ -870,11 +1142,14 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
870 } 1142 }
871 1143
872 memset(bctx->ipad + keylen, 0, bs - keylen); 1144 memset(bctx->ipad + keylen, 0, bs - keylen);
873 memcpy(bctx->opad, bctx->ipad, bs);
874 1145
875 for (i = 0; i < bs; i++) { 1146 if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) {
876 bctx->ipad[i] ^= 0x36; 1147 memcpy(bctx->opad, bctx->ipad, bs);
877 bctx->opad[i] ^= 0x5c; 1148
1149 for (i = 0; i < bs; i++) {
1150 bctx->ipad[i] ^= 0x36;
1151 bctx->opad[i] ^= 0x5c;
1152 }
878 } 1153 }
879 1154
880 return err; 1155 return err;
@@ -924,6 +1199,16 @@ static int omap_sham_cra_sha1_init(struct crypto_tfm *tfm)
924 return omap_sham_cra_init_alg(tfm, "sha1"); 1199 return omap_sham_cra_init_alg(tfm, "sha1");
925} 1200}
926 1201
1202static int omap_sham_cra_sha224_init(struct crypto_tfm *tfm)
1203{
1204 return omap_sham_cra_init_alg(tfm, "sha224");
1205}
1206
1207static int omap_sham_cra_sha256_init(struct crypto_tfm *tfm)
1208{
1209 return omap_sham_cra_init_alg(tfm, "sha256");
1210}
1211
927static int omap_sham_cra_md5_init(struct crypto_tfm *tfm) 1212static int omap_sham_cra_md5_init(struct crypto_tfm *tfm)
928{ 1213{
929 return omap_sham_cra_init_alg(tfm, "md5"); 1214 return omap_sham_cra_init_alg(tfm, "md5");
@@ -942,7 +1227,7 @@ static void omap_sham_cra_exit(struct crypto_tfm *tfm)
942 } 1227 }
943} 1228}
944 1229
945static struct ahash_alg algs[] = { 1230static struct ahash_alg algs_sha1_md5[] = {
946{ 1231{
947 .init = omap_sham_init, 1232 .init = omap_sham_init,
948 .update = omap_sham_update, 1233 .update = omap_sham_update,
@@ -1041,6 +1326,102 @@ static struct ahash_alg algs[] = {
1041} 1326}
1042}; 1327};
1043 1328
1329/* OMAP4 has some algs in addition to what OMAP2 has */
1330static struct ahash_alg algs_sha224_sha256[] = {
1331{
1332 .init = omap_sham_init,
1333 .update = omap_sham_update,
1334 .final = omap_sham_final,
1335 .finup = omap_sham_finup,
1336 .digest = omap_sham_digest,
1337 .halg.digestsize = SHA224_DIGEST_SIZE,
1338 .halg.base = {
1339 .cra_name = "sha224",
1340 .cra_driver_name = "omap-sha224",
1341 .cra_priority = 100,
1342 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1343 CRYPTO_ALG_ASYNC |
1344 CRYPTO_ALG_NEED_FALLBACK,
1345 .cra_blocksize = SHA224_BLOCK_SIZE,
1346 .cra_ctxsize = sizeof(struct omap_sham_ctx),
1347 .cra_alignmask = 0,
1348 .cra_module = THIS_MODULE,
1349 .cra_init = omap_sham_cra_init,
1350 .cra_exit = omap_sham_cra_exit,
1351 }
1352},
1353{
1354 .init = omap_sham_init,
1355 .update = omap_sham_update,
1356 .final = omap_sham_final,
1357 .finup = omap_sham_finup,
1358 .digest = omap_sham_digest,
1359 .halg.digestsize = SHA256_DIGEST_SIZE,
1360 .halg.base = {
1361 .cra_name = "sha256",
1362 .cra_driver_name = "omap-sha256",
1363 .cra_priority = 100,
1364 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1365 CRYPTO_ALG_ASYNC |
1366 CRYPTO_ALG_NEED_FALLBACK,
1367 .cra_blocksize = SHA256_BLOCK_SIZE,
1368 .cra_ctxsize = sizeof(struct omap_sham_ctx),
1369 .cra_alignmask = 0,
1370 .cra_module = THIS_MODULE,
1371 .cra_init = omap_sham_cra_init,
1372 .cra_exit = omap_sham_cra_exit,
1373 }
1374},
1375{
1376 .init = omap_sham_init,
1377 .update = omap_sham_update,
1378 .final = omap_sham_final,
1379 .finup = omap_sham_finup,
1380 .digest = omap_sham_digest,
1381 .setkey = omap_sham_setkey,
1382 .halg.digestsize = SHA224_DIGEST_SIZE,
1383 .halg.base = {
1384 .cra_name = "hmac(sha224)",
1385 .cra_driver_name = "omap-hmac-sha224",
1386 .cra_priority = 100,
1387 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1388 CRYPTO_ALG_ASYNC |
1389 CRYPTO_ALG_NEED_FALLBACK,
1390 .cra_blocksize = SHA224_BLOCK_SIZE,
1391 .cra_ctxsize = sizeof(struct omap_sham_ctx) +
1392 sizeof(struct omap_sham_hmac_ctx),
1393 .cra_alignmask = OMAP_ALIGN_MASK,
1394 .cra_module = THIS_MODULE,
1395 .cra_init = omap_sham_cra_sha224_init,
1396 .cra_exit = omap_sham_cra_exit,
1397 }
1398},
1399{
1400 .init = omap_sham_init,
1401 .update = omap_sham_update,
1402 .final = omap_sham_final,
1403 .finup = omap_sham_finup,
1404 .digest = omap_sham_digest,
1405 .setkey = omap_sham_setkey,
1406 .halg.digestsize = SHA256_DIGEST_SIZE,
1407 .halg.base = {
1408 .cra_name = "hmac(sha256)",
1409 .cra_driver_name = "omap-hmac-sha256",
1410 .cra_priority = 100,
1411 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1412 CRYPTO_ALG_ASYNC |
1413 CRYPTO_ALG_NEED_FALLBACK,
1414 .cra_blocksize = SHA256_BLOCK_SIZE,
1415 .cra_ctxsize = sizeof(struct omap_sham_ctx) +
1416 sizeof(struct omap_sham_hmac_ctx),
1417 .cra_alignmask = OMAP_ALIGN_MASK,
1418 .cra_module = THIS_MODULE,
1419 .cra_init = omap_sham_cra_sha256_init,
1420 .cra_exit = omap_sham_cra_exit,
1421 }
1422},
1423};
1424
1044static void omap_sham_done_task(unsigned long data) 1425static void omap_sham_done_task(unsigned long data)
1045{ 1426{
1046 struct omap_sham_dev *dd = (struct omap_sham_dev *)data; 1427 struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
@@ -1079,7 +1460,19 @@ finish:
1079 omap_sham_finish_req(dd->req, err); 1460 omap_sham_finish_req(dd->req, err);
1080} 1461}
1081 1462
1082static irqreturn_t omap_sham_irq(int irq, void *dev_id) 1463static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd)
1464{
1465 if (!test_bit(FLAGS_BUSY, &dd->flags)) {
1466 dev_warn(dd->dev, "Interrupt when no active requests.\n");
1467 } else {
1468 set_bit(FLAGS_OUTPUT_READY, &dd->flags);
1469 tasklet_schedule(&dd->done_task);
1470 }
1471
1472 return IRQ_HANDLED;
1473}
1474
1475static irqreturn_t omap_sham_irq_omap2(int irq, void *dev_id)
1083{ 1476{
1084 struct omap_sham_dev *dd = dev_id; 1477 struct omap_sham_dev *dd = dev_id;
1085 1478
@@ -1091,61 +1484,188 @@ static irqreturn_t omap_sham_irq(int irq, void *dev_id)
1091 SHA_REG_CTRL_OUTPUT_READY); 1484 SHA_REG_CTRL_OUTPUT_READY);
1092 omap_sham_read(dd, SHA_REG_CTRL); 1485 omap_sham_read(dd, SHA_REG_CTRL);
1093 1486
1094 if (!test_bit(FLAGS_BUSY, &dd->flags)) { 1487 return omap_sham_irq_common(dd);
1095 dev_warn(dd->dev, "Interrupt when no active requests.\n"); 1488}
1096 return IRQ_HANDLED;
1097 }
1098 1489
1099 set_bit(FLAGS_OUTPUT_READY, &dd->flags); 1490static irqreturn_t omap_sham_irq_omap4(int irq, void *dev_id)
1100 tasklet_schedule(&dd->done_task); 1491{
1492 struct omap_sham_dev *dd = dev_id;
1101 1493
1102 return IRQ_HANDLED; 1494 omap_sham_write_mask(dd, SHA_REG_MASK(dd), 0, SHA_REG_MASK_IT_EN);
1495
1496 return omap_sham_irq_common(dd);
1103} 1497}
1104 1498
1105static void omap_sham_dma_callback(int lch, u16 ch_status, void *data) 1499static struct omap_sham_algs_info omap_sham_algs_info_omap2[] = {
1500 {
1501 .algs_list = algs_sha1_md5,
1502 .size = ARRAY_SIZE(algs_sha1_md5),
1503 },
1504};
1505
1506static const struct omap_sham_pdata omap_sham_pdata_omap2 = {
1507 .algs_info = omap_sham_algs_info_omap2,
1508 .algs_info_size = ARRAY_SIZE(omap_sham_algs_info_omap2),
1509 .flags = BIT(FLAGS_BE32_SHA1),
1510 .digest_size = SHA1_DIGEST_SIZE,
1511 .copy_hash = omap_sham_copy_hash_omap2,
1512 .write_ctrl = omap_sham_write_ctrl_omap2,
1513 .trigger = omap_sham_trigger_omap2,
1514 .poll_irq = omap_sham_poll_irq_omap2,
1515 .intr_hdlr = omap_sham_irq_omap2,
1516 .idigest_ofs = 0x00,
1517 .din_ofs = 0x1c,
1518 .digcnt_ofs = 0x14,
1519 .rev_ofs = 0x5c,
1520 .mask_ofs = 0x60,
1521 .sysstatus_ofs = 0x64,
1522 .major_mask = 0xf0,
1523 .major_shift = 4,
1524 .minor_mask = 0x0f,
1525 .minor_shift = 0,
1526};
1527
1528#ifdef CONFIG_OF
1529static struct omap_sham_algs_info omap_sham_algs_info_omap4[] = {
1530 {
1531 .algs_list = algs_sha1_md5,
1532 .size = ARRAY_SIZE(algs_sha1_md5),
1533 },
1534 {
1535 .algs_list = algs_sha224_sha256,
1536 .size = ARRAY_SIZE(algs_sha224_sha256),
1537 },
1538};
1539
1540static const struct omap_sham_pdata omap_sham_pdata_omap4 = {
1541 .algs_info = omap_sham_algs_info_omap4,
1542 .algs_info_size = ARRAY_SIZE(omap_sham_algs_info_omap4),
1543 .flags = BIT(FLAGS_AUTO_XOR),
1544 .digest_size = SHA256_DIGEST_SIZE,
1545 .copy_hash = omap_sham_copy_hash_omap4,
1546 .write_ctrl = omap_sham_write_ctrl_omap4,
1547 .trigger = omap_sham_trigger_omap4,
1548 .poll_irq = omap_sham_poll_irq_omap4,
1549 .intr_hdlr = omap_sham_irq_omap4,
1550 .idigest_ofs = 0x020,
1551 .din_ofs = 0x080,
1552 .digcnt_ofs = 0x040,
1553 .rev_ofs = 0x100,
1554 .mask_ofs = 0x110,
1555 .sysstatus_ofs = 0x114,
1556 .major_mask = 0x0700,
1557 .major_shift = 8,
1558 .minor_mask = 0x003f,
1559 .minor_shift = 0,
1560};
1561
1562static const struct of_device_id omap_sham_of_match[] = {
1563 {
1564 .compatible = "ti,omap2-sham",
1565 .data = &omap_sham_pdata_omap2,
1566 },
1567 {
1568 .compatible = "ti,omap4-sham",
1569 .data = &omap_sham_pdata_omap4,
1570 },
1571 {},
1572};
1573MODULE_DEVICE_TABLE(of, omap_sham_of_match);
1574
1575static int omap_sham_get_res_of(struct omap_sham_dev *dd,
1576 struct device *dev, struct resource *res)
1106{ 1577{
1107 struct omap_sham_dev *dd = data; 1578 struct device_node *node = dev->of_node;
1579 const struct of_device_id *match;
1580 int err = 0;
1108 1581
1109 if (ch_status != OMAP_DMA_BLOCK_IRQ) { 1582 match = of_match_device(of_match_ptr(omap_sham_of_match), dev);
1110 pr_err("omap-sham DMA error status: 0x%hx\n", ch_status); 1583 if (!match) {
1111 dd->err = -EIO; 1584 dev_err(dev, "no compatible OF match\n");
1112 clear_bit(FLAGS_INIT, &dd->flags);/* request to re-initialize */ 1585 err = -EINVAL;
1586 goto err;
1113 } 1587 }
1114 1588
1115 set_bit(FLAGS_DMA_READY, &dd->flags); 1589 err = of_address_to_resource(node, 0, res);
1116 tasklet_schedule(&dd->done_task); 1590 if (err < 0) {
1591 dev_err(dev, "can't translate OF node address\n");
1592 err = -EINVAL;
1593 goto err;
1594 }
1595
1596 dd->irq = of_irq_to_resource(node, 0, NULL);
1597 if (!dd->irq) {
1598 dev_err(dev, "can't translate OF irq value\n");
1599 err = -EINVAL;
1600 goto err;
1601 }
1602
1603 dd->dma = -1; /* Dummy value that's unused */
1604 dd->pdata = match->data;
1605
1606err:
1607 return err;
1117} 1608}
1609#else
1610static const struct of_device_id omap_sham_of_match[] = {
1611 {},
1612};
1118 1613
1119static int omap_sham_dma_init(struct omap_sham_dev *dd) 1614static int omap_sham_get_res_of(struct omap_sham_dev *dd,
1615 struct device *dev, struct resource *res)
1120{ 1616{
1121 int err; 1617 return -EINVAL;
1618}
1619#endif
1122 1620
1123 dd->dma_lch = -1; 1621static int omap_sham_get_res_pdev(struct omap_sham_dev *dd,
1622 struct platform_device *pdev, struct resource *res)
1623{
1624 struct device *dev = &pdev->dev;
1625 struct resource *r;
1626 int err = 0;
1124 1627
1125 err = omap_request_dma(dd->dma, dev_name(dd->dev), 1628 /* Get the base address */
1126 omap_sham_dma_callback, dd, &dd->dma_lch); 1629 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1127 if (err) { 1630 if (!r) {
1128 dev_err(dd->dev, "Unable to request DMA channel\n"); 1631 dev_err(dev, "no MEM resource info\n");
1129 return err; 1632 err = -ENODEV;
1633 goto err;
1130 } 1634 }
1635 memcpy(res, r, sizeof(*res));
1131 1636
1132 return 0; 1637 /* Get the IRQ */
1133} 1638 dd->irq = platform_get_irq(pdev, 0);
1639 if (dd->irq < 0) {
1640 dev_err(dev, "no IRQ resource info\n");
1641 err = dd->irq;
1642 goto err;
1643 }
1134 1644
1135static void omap_sham_dma_cleanup(struct omap_sham_dev *dd) 1645 /* Get the DMA */
1136{ 1646 r = platform_get_resource(pdev, IORESOURCE_DMA, 0);
1137 if (dd->dma_lch >= 0) { 1647 if (!r) {
1138 omap_free_dma(dd->dma_lch); 1648 dev_err(dev, "no DMA resource info\n");
1139 dd->dma_lch = -1; 1649 err = -ENODEV;
1650 goto err;
1140 } 1651 }
1652 dd->dma = r->start;
1653
1654 /* Only OMAP2/3 can be non-DT */
1655 dd->pdata = &omap_sham_pdata_omap2;
1656
1657err:
1658 return err;
1141} 1659}
1142 1660
1143static int omap_sham_probe(struct platform_device *pdev) 1661static int omap_sham_probe(struct platform_device *pdev)
1144{ 1662{
1145 struct omap_sham_dev *dd; 1663 struct omap_sham_dev *dd;
1146 struct device *dev = &pdev->dev; 1664 struct device *dev = &pdev->dev;
1147 struct resource *res; 1665 struct resource res;
1666 dma_cap_mask_t mask;
1148 int err, i, j; 1667 int err, i, j;
1668 u32 rev;
1149 1669
1150 dd = kzalloc(sizeof(struct omap_sham_dev), GFP_KERNEL); 1670 dd = kzalloc(sizeof(struct omap_sham_dev), GFP_KERNEL);
1151 if (dd == NULL) { 1671 if (dd == NULL) {
@@ -1161,89 +1681,75 @@ static int omap_sham_probe(struct platform_device *pdev)
1161 tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd); 1681 tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd);
1162 crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH); 1682 crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH);
1163 1683
1164 dd->irq = -1; 1684 err = (dev->of_node) ? omap_sham_get_res_of(dd, dev, &res) :
1165 1685 omap_sham_get_res_pdev(dd, pdev, &res);
1166 /* Get the base address */ 1686 if (err)
1167 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1168 if (!res) {
1169 dev_err(dev, "no MEM resource info\n");
1170 err = -ENODEV;
1171 goto res_err;
1172 }
1173 dd->phys_base = res->start;
1174
1175 /* Get the DMA */
1176 res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
1177 if (!res) {
1178 dev_err(dev, "no DMA resource info\n");
1179 err = -ENODEV;
1180 goto res_err; 1687 goto res_err;
1181 }
1182 dd->dma = res->start;
1183 1688
1184 /* Get the IRQ */ 1689 dd->io_base = devm_request_and_ioremap(dev, &res);
1185 dd->irq = platform_get_irq(pdev, 0); 1690 if (!dd->io_base) {
1186 if (dd->irq < 0) { 1691 dev_err(dev, "can't ioremap\n");
1187 dev_err(dev, "no IRQ resource info\n"); 1692 err = -ENOMEM;
1188 err = dd->irq;
1189 goto res_err; 1693 goto res_err;
1190 } 1694 }
1695 dd->phys_base = res.start;
1191 1696
1192 err = request_irq(dd->irq, omap_sham_irq, 1697 err = request_irq(dd->irq, dd->pdata->intr_hdlr, IRQF_TRIGGER_LOW,
1193 IRQF_TRIGGER_LOW, dev_name(dev), dd); 1698 dev_name(dev), dd);
1194 if (err) { 1699 if (err) {
1195 dev_err(dev, "unable to request irq.\n"); 1700 dev_err(dev, "unable to request irq.\n");
1196 goto res_err; 1701 goto res_err;
1197 } 1702 }
1198 1703
1199 err = omap_sham_dma_init(dd); 1704 dma_cap_zero(mask);
1200 if (err) 1705 dma_cap_set(DMA_SLAVE, mask);
1201 goto dma_err;
1202 1706
1203 /* Initializing the clock */ 1707 dd->dma_lch = dma_request_slave_channel_compat(mask, omap_dma_filter_fn,
1204 dd->iclk = clk_get(dev, "ick"); 1708 &dd->dma, dev, "rx");
1205 if (IS_ERR(dd->iclk)) { 1709 if (!dd->dma_lch) {
1206 dev_err(dev, "clock intialization failed.\n"); 1710 dev_err(dev, "unable to obtain RX DMA engine channel %u\n",
1207 err = PTR_ERR(dd->iclk); 1711 dd->dma);
1208 goto clk_err; 1712 err = -ENXIO;
1713 goto dma_err;
1209 } 1714 }
1210 1715
1211 dd->io_base = ioremap(dd->phys_base, SZ_4K); 1716 dd->flags |= dd->pdata->flags;
1212 if (!dd->io_base) { 1717
1213 dev_err(dev, "can't ioremap\n"); 1718 pm_runtime_enable(dev);
1214 err = -ENOMEM; 1719 pm_runtime_get_sync(dev);
1215 goto io_err; 1720 rev = omap_sham_read(dd, SHA_REG_REV(dd));
1216 } 1721 pm_runtime_put_sync(&pdev->dev);
1217 1722
1218 clk_enable(dd->iclk);
1219 dev_info(dev, "hw accel on OMAP rev %u.%u\n", 1723 dev_info(dev, "hw accel on OMAP rev %u.%u\n",
1220 (omap_sham_read(dd, SHA_REG_REV) & SHA_REG_REV_MAJOR) >> 4, 1724 (rev & dd->pdata->major_mask) >> dd->pdata->major_shift,
1221 omap_sham_read(dd, SHA_REG_REV) & SHA_REG_REV_MINOR); 1725 (rev & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
1222 clk_disable(dd->iclk);
1223 1726
1224 spin_lock(&sham.lock); 1727 spin_lock(&sham.lock);
1225 list_add_tail(&dd->list, &sham.dev_list); 1728 list_add_tail(&dd->list, &sham.dev_list);
1226 spin_unlock(&sham.lock); 1729 spin_unlock(&sham.lock);
1227 1730
1228 for (i = 0; i < ARRAY_SIZE(algs); i++) { 1731 for (i = 0; i < dd->pdata->algs_info_size; i++) {
1229 err = crypto_register_ahash(&algs[i]); 1732 for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
1230 if (err) 1733 err = crypto_register_ahash(
1231 goto err_algs; 1734 &dd->pdata->algs_info[i].algs_list[j]);
1735 if (err)
1736 goto err_algs;
1737
1738 dd->pdata->algs_info[i].registered++;
1739 }
1232 } 1740 }
1233 1741
1234 return 0; 1742 return 0;
1235 1743
1236err_algs: 1744err_algs:
1237 for (j = 0; j < i; j++) 1745 for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1238 crypto_unregister_ahash(&algs[j]); 1746 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1239 iounmap(dd->io_base); 1747 crypto_unregister_ahash(
1240io_err: 1748 &dd->pdata->algs_info[i].algs_list[j]);
1241 clk_put(dd->iclk); 1749 pm_runtime_disable(dev);
1242clk_err: 1750 dma_release_channel(dd->dma_lch);
1243 omap_sham_dma_cleanup(dd);
1244dma_err: 1751dma_err:
1245 if (dd->irq >= 0) 1752 free_irq(dd->irq, dd);
1246 free_irq(dd->irq, dd);
1247res_err: 1753res_err:
1248 kfree(dd); 1754 kfree(dd);
1249 dd = NULL; 1755 dd = NULL;
@@ -1256,7 +1762,7 @@ data_err:
1256static int omap_sham_remove(struct platform_device *pdev) 1762static int omap_sham_remove(struct platform_device *pdev)
1257{ 1763{
1258 static struct omap_sham_dev *dd; 1764 static struct omap_sham_dev *dd;
1259 int i; 1765 int i, j;
1260 1766
1261 dd = platform_get_drvdata(pdev); 1767 dd = platform_get_drvdata(pdev);
1262 if (!dd) 1768 if (!dd)
@@ -1264,33 +1770,51 @@ static int omap_sham_remove(struct platform_device *pdev)
1264 spin_lock(&sham.lock); 1770 spin_lock(&sham.lock);
1265 list_del(&dd->list); 1771 list_del(&dd->list);
1266 spin_unlock(&sham.lock); 1772 spin_unlock(&sham.lock);
1267 for (i = 0; i < ARRAY_SIZE(algs); i++) 1773 for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
1268 crypto_unregister_ahash(&algs[i]); 1774 for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
1775 crypto_unregister_ahash(
1776 &dd->pdata->algs_info[i].algs_list[j]);
1269 tasklet_kill(&dd->done_task); 1777 tasklet_kill(&dd->done_task);
1270 iounmap(dd->io_base); 1778 pm_runtime_disable(&pdev->dev);
1271 clk_put(dd->iclk); 1779 dma_release_channel(dd->dma_lch);
1272 omap_sham_dma_cleanup(dd); 1780 free_irq(dd->irq, dd);
1273 if (dd->irq >= 0)
1274 free_irq(dd->irq, dd);
1275 kfree(dd); 1781 kfree(dd);
1276 dd = NULL; 1782 dd = NULL;
1277 1783
1278 return 0; 1784 return 0;
1279} 1785}
1280 1786
1787#ifdef CONFIG_PM_SLEEP
1788static int omap_sham_suspend(struct device *dev)
1789{
1790 pm_runtime_put_sync(dev);
1791 return 0;
1792}
1793
1794static int omap_sham_resume(struct device *dev)
1795{
1796 pm_runtime_get_sync(dev);
1797 return 0;
1798}
1799#endif
1800
1801static const struct dev_pm_ops omap_sham_pm_ops = {
1802 SET_SYSTEM_SLEEP_PM_OPS(omap_sham_suspend, omap_sham_resume)
1803};
1804
1281static struct platform_driver omap_sham_driver = { 1805static struct platform_driver omap_sham_driver = {
1282 .probe = omap_sham_probe, 1806 .probe = omap_sham_probe,
1283 .remove = omap_sham_remove, 1807 .remove = omap_sham_remove,
1284 .driver = { 1808 .driver = {
1285 .name = "omap-sham", 1809 .name = "omap-sham",
1286 .owner = THIS_MODULE, 1810 .owner = THIS_MODULE,
1811 .pm = &omap_sham_pm_ops,
1812 .of_match_table = omap_sham_of_match,
1287 }, 1813 },
1288}; 1814};
1289 1815
1290static int __init omap_sham_mod_init(void) 1816static int __init omap_sham_mod_init(void)
1291{ 1817{
1292 pr_info("loading %s driver\n", "omap-sham");
1293
1294 return platform_driver_register(&omap_sham_driver); 1818 return platform_driver_register(&omap_sham_driver);
1295} 1819}
1296 1820
diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c
index 49ad8cbade69..4b314326f48a 100644
--- a/drivers/crypto/s5p-sss.c
+++ b/drivers/crypto/s5p-sss.c
@@ -580,7 +580,7 @@ static int s5p_aes_probe(struct platform_device *pdev)
580 resource_size(res), pdev->name)) 580 resource_size(res), pdev->name))
581 return -EBUSY; 581 return -EBUSY;
582 582
583 pdata->clk = clk_get(dev, "secss"); 583 pdata->clk = devm_clk_get(dev, "secss");
584 if (IS_ERR(pdata->clk)) { 584 if (IS_ERR(pdata->clk)) {
585 dev_err(dev, "failed to find secss clock source\n"); 585 dev_err(dev, "failed to find secss clock source\n");
586 return -ENOENT; 586 return -ENOENT;
@@ -645,7 +645,6 @@ static int s5p_aes_probe(struct platform_device *pdev)
645 645
646 err_irq: 646 err_irq:
647 clk_disable(pdata->clk); 647 clk_disable(pdata->clk);
648 clk_put(pdata->clk);
649 648
650 s5p_dev = NULL; 649 s5p_dev = NULL;
651 platform_set_drvdata(pdev, NULL); 650 platform_set_drvdata(pdev, NULL);
@@ -667,7 +666,6 @@ static int s5p_aes_remove(struct platform_device *pdev)
667 tasklet_kill(&pdata->tasklet); 666 tasklet_kill(&pdata->tasklet);
668 667
669 clk_disable(pdata->clk); 668 clk_disable(pdata->clk);
670 clk_put(pdata->clk);
671 669
672 s5p_dev = NULL; 670 s5p_dev = NULL;
673 platform_set_drvdata(pdev, NULL); 671 platform_set_drvdata(pdev, NULL);
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-01 05:48:10 -0500