aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc/crypto
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc/crypto')
-rw-r--r--arch/sparc/crypto/Makefile25
-rw-r--r--arch/sparc/crypto/aes_asm.S1535
-rw-r--r--arch/sparc/crypto/aes_glue.c477
-rw-r--r--arch/sparc/crypto/camellia_asm.S563
-rw-r--r--arch/sparc/crypto/camellia_glue.c322
-rw-r--r--arch/sparc/crypto/crc32c_asm.S20
-rw-r--r--arch/sparc/crypto/crc32c_glue.c179
-rw-r--r--arch/sparc/crypto/crop_devid.c14
-rw-r--r--arch/sparc/crypto/des_asm.S418
-rw-r--r--arch/sparc/crypto/des_glue.c529
-rw-r--r--arch/sparc/crypto/md5_asm.S70
-rw-r--r--arch/sparc/crypto/md5_glue.c188
-rw-r--r--arch/sparc/crypto/opcodes.h99
-rw-r--r--arch/sparc/crypto/sha1_asm.S72
-rw-r--r--arch/sparc/crypto/sha1_glue.c183
-rw-r--r--arch/sparc/crypto/sha256_asm.S78
-rw-r--r--arch/sparc/crypto/sha256_glue.c241
-rw-r--r--arch/sparc/crypto/sha512_asm.S102
-rw-r--r--arch/sparc/crypto/sha512_glue.c226
19 files changed, 5341 insertions, 0 deletions
diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile
new file mode 100644
index 000000000000..6ae1ad5e502b
--- /dev/null
+++ b/arch/sparc/crypto/Makefile
@@ -0,0 +1,25 @@
1#
2# Arch-specific CryptoAPI modules.
3#
4
5obj-$(CONFIG_CRYPTO_SHA1_SPARC64) += sha1-sparc64.o
6obj-$(CONFIG_CRYPTO_SHA256_SPARC64) += sha256-sparc64.o
7obj-$(CONFIG_CRYPTO_SHA512_SPARC64) += sha512-sparc64.o
8obj-$(CONFIG_CRYPTO_MD5_SPARC64) += md5-sparc64.o
9
10obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o
11obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o
12obj-$(CONFIG_CRYPTO_DES_SPARC64) += camellia-sparc64.o
13
14obj-$(CONFIG_CRYPTO_CRC32C_SPARC64) += crc32c-sparc64.o
15
16sha1-sparc64-y := sha1_asm.o sha1_glue.o crop_devid.o
17sha256-sparc64-y := sha256_asm.o sha256_glue.o crop_devid.o
18sha512-sparc64-y := sha512_asm.o sha512_glue.o crop_devid.o
19md5-sparc64-y := md5_asm.o md5_glue.o crop_devid.o
20
21aes-sparc64-y := aes_asm.o aes_glue.o crop_devid.o
22des-sparc64-y := des_asm.o des_glue.o crop_devid.o
23camellia-sparc64-y := camellia_asm.o camellia_glue.o crop_devid.o
24
25crc32c-sparc64-y := crc32c_asm.o crc32c_glue.o crop_devid.o
diff --git a/arch/sparc/crypto/aes_asm.S b/arch/sparc/crypto/aes_asm.S
new file mode 100644
index 000000000000..23f6cbb910d3
--- /dev/null
+++ b/arch/sparc/crypto/aes_asm.S
@@ -0,0 +1,1535 @@
1#include <linux/linkage.h>
2#include <asm/visasm.h>
3
4#include "opcodes.h"
5
6#define ENCRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \
7 AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \
8 AES_EROUND23(KEY_BASE + 2, I0, I1, T1) \
9 AES_EROUND01(KEY_BASE + 4, T0, T1, I0) \
10 AES_EROUND23(KEY_BASE + 6, T0, T1, I1)
11
12#define ENCRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
13 AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \
14 AES_EROUND23(KEY_BASE + 2, I0, I1, T1) \
15 AES_EROUND01(KEY_BASE + 0, I2, I3, T2) \
16 AES_EROUND23(KEY_BASE + 2, I2, I3, T3) \
17 AES_EROUND01(KEY_BASE + 4, T0, T1, I0) \
18 AES_EROUND23(KEY_BASE + 6, T0, T1, I1) \
19 AES_EROUND01(KEY_BASE + 4, T2, T3, I2) \
20 AES_EROUND23(KEY_BASE + 6, T2, T3, I3)
21
22#define ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE, I0, I1, T0, T1) \
23 AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \
24 AES_EROUND23(KEY_BASE + 2, I0, I1, T1) \
25 AES_EROUND01_L(KEY_BASE + 4, T0, T1, I0) \
26 AES_EROUND23_L(KEY_BASE + 6, T0, T1, I1)
27
28#define ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
29 AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \
30 AES_EROUND23(KEY_BASE + 2, I0, I1, T1) \
31 AES_EROUND01(KEY_BASE + 0, I2, I3, T2) \
32 AES_EROUND23(KEY_BASE + 2, I2, I3, T3) \
33 AES_EROUND01_L(KEY_BASE + 4, T0, T1, I0) \
34 AES_EROUND23_L(KEY_BASE + 6, T0, T1, I1) \
35 AES_EROUND01_L(KEY_BASE + 4, T2, T3, I2) \
36 AES_EROUND23_L(KEY_BASE + 6, T2, T3, I3)
37
38 /* 10 rounds */
39#define ENCRYPT_128(KEY_BASE, I0, I1, T0, T1) \
40 ENCRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \
41 ENCRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \
42 ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
43 ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
44 ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 32, I0, I1, T0, T1)
45
46#define ENCRYPT_128_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
47 ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, T0, T1, T2, T3) \
48 ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, T0, T1, T2, T3) \
49 ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
50 ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
51 ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3)
52
53 /* 12 rounds */
54#define ENCRYPT_192(KEY_BASE, I0, I1, T0, T1) \
55 ENCRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \
56 ENCRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \
57 ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
58 ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
59 ENCRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
60 ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 40, I0, I1, T0, T1)
61
62#define ENCRYPT_192_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
63 ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, T0, T1, T2, T3) \
64 ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, T0, T1, T2, T3) \
65 ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
66 ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
67 ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3) \
68 ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 40, I0, I1, I2, I3, T0, T1, T2, T3)
69
70 /* 14 rounds */
71#define ENCRYPT_256(KEY_BASE, I0, I1, T0, T1) \
72 ENCRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \
73 ENCRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \
74 ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
75 ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
76 ENCRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
77 ENCRYPT_TWO_ROUNDS(KEY_BASE + 40, I0, I1, T0, T1) \
78 ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 48, I0, I1, T0, T1)
79
80#define ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, TMP_BASE) \
81 ENCRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, \
82 TMP_BASE + 0, TMP_BASE + 2, TMP_BASE + 4, TMP_BASE + 6)
83
84#define ENCRYPT_256_2(KEY_BASE, I0, I1, I2, I3) \
85 ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, KEY_BASE + 48) \
86 ldd [%o0 + 0xd0], %f56; \
87 ldd [%o0 + 0xd8], %f58; \
88 ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, KEY_BASE + 0) \
89 ldd [%o0 + 0xe0], %f60; \
90 ldd [%o0 + 0xe8], %f62; \
91 ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, KEY_BASE + 0) \
92 ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, KEY_BASE + 0) \
93 ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, KEY_BASE + 0) \
94 ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 40, I0, I1, I2, I3, KEY_BASE + 0) \
95 AES_EROUND01(KEY_BASE + 48, I0, I1, KEY_BASE + 0) \
96 AES_EROUND23(KEY_BASE + 50, I0, I1, KEY_BASE + 2) \
97 AES_EROUND01(KEY_BASE + 48, I2, I3, KEY_BASE + 4) \
98 AES_EROUND23(KEY_BASE + 50, I2, I3, KEY_BASE + 6) \
99 AES_EROUND01_L(KEY_BASE + 52, KEY_BASE + 0, KEY_BASE + 2, I0) \
100 AES_EROUND23_L(KEY_BASE + 54, KEY_BASE + 0, KEY_BASE + 2, I1) \
101 ldd [%o0 + 0x10], %f8; \
102 ldd [%o0 + 0x18], %f10; \
103 AES_EROUND01_L(KEY_BASE + 52, KEY_BASE + 4, KEY_BASE + 6, I2) \
104 AES_EROUND23_L(KEY_BASE + 54, KEY_BASE + 4, KEY_BASE + 6, I3) \
105 ldd [%o0 + 0x20], %f12; \
106 ldd [%o0 + 0x28], %f14;
107
108#define DECRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \
109 AES_DROUND23(KEY_BASE + 0, I0, I1, T1) \
110 AES_DROUND01(KEY_BASE + 2, I0, I1, T0) \
111 AES_DROUND23(KEY_BASE + 4, T0, T1, I1) \
112 AES_DROUND01(KEY_BASE + 6, T0, T1, I0)
113
114#define DECRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
115 AES_DROUND23(KEY_BASE + 0, I0, I1, T1) \
116 AES_DROUND01(KEY_BASE + 2, I0, I1, T0) \
117 AES_DROUND23(KEY_BASE + 0, I2, I3, T3) \
118 AES_DROUND01(KEY_BASE + 2, I2, I3, T2) \
119 AES_DROUND23(KEY_BASE + 4, T0, T1, I1) \
120 AES_DROUND01(KEY_BASE + 6, T0, T1, I0) \
121 AES_DROUND23(KEY_BASE + 4, T2, T3, I3) \
122 AES_DROUND01(KEY_BASE + 6, T2, T3, I2)
123
124#define DECRYPT_TWO_ROUNDS_LAST(KEY_BASE, I0, I1, T0, T1) \
125 AES_DROUND23(KEY_BASE + 0, I0, I1, T1) \
126 AES_DROUND01(KEY_BASE + 2, I0, I1, T0) \
127 AES_DROUND23_L(KEY_BASE + 4, T0, T1, I1) \
128 AES_DROUND01_L(KEY_BASE + 6, T0, T1, I0)
129
130#define DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
131 AES_DROUND23(KEY_BASE + 0, I0, I1, T1) \
132 AES_DROUND01(KEY_BASE + 2, I0, I1, T0) \
133 AES_DROUND23(KEY_BASE + 0, I2, I3, T3) \
134 AES_DROUND01(KEY_BASE + 2, I2, I3, T2) \
135 AES_DROUND23_L(KEY_BASE + 4, T0, T1, I1) \
136 AES_DROUND01_L(KEY_BASE + 6, T0, T1, I0) \
137 AES_DROUND23_L(KEY_BASE + 4, T2, T3, I3) \
138 AES_DROUND01_L(KEY_BASE + 6, T2, T3, I2)
139
140 /* 10 rounds */
141#define DECRYPT_128(KEY_BASE, I0, I1, T0, T1) \
142 DECRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \
143 DECRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \
144 DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
145 DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
146 DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 32, I0, I1, T0, T1)
147
148#define DECRYPT_128_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
149 DECRYPT_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, T0, T1, T2, T3) \
150 DECRYPT_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, T0, T1, T2, T3) \
151 DECRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
152 DECRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
153 DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3)
154
155 /* 12 rounds */
156#define DECRYPT_192(KEY_BASE, I0, I1, T0, T1) \
157 DECRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \
158 DECRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \
159 DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
160 DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
161 DECRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
162 DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 40, I0, I1, T0, T1)
163
164#define DECRYPT_192_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
165 DECRYPT_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, T0, T1, T2, T3) \
166 DECRYPT_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, T0, T1, T2, T3) \
167 DECRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
168 DECRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
169 DECRYPT_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3) \
170 DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 40, I0, I1, I2, I3, T0, T1, T2, T3)
171
172 /* 14 rounds */
173#define DECRYPT_256(KEY_BASE, I0, I1, T0, T1) \
174 DECRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \
175 DECRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \
176 DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
177 DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
178 DECRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
179 DECRYPT_TWO_ROUNDS(KEY_BASE + 40, I0, I1, T0, T1) \
180 DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 48, I0, I1, T0, T1)
181
182#define DECRYPT_256_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, TMP_BASE) \
183 DECRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, \
184 TMP_BASE + 0, TMP_BASE + 2, TMP_BASE + 4, TMP_BASE + 6)
185
186#define DECRYPT_256_2(KEY_BASE, I0, I1, I2, I3) \
187 DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, KEY_BASE + 48) \
188 ldd [%o0 + 0x18], %f56; \
189 ldd [%o0 + 0x10], %f58; \
190 DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, KEY_BASE + 0) \
191 ldd [%o0 + 0x08], %f60; \
192 ldd [%o0 + 0x00], %f62; \
193 DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, KEY_BASE + 0) \
194 DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, KEY_BASE + 0) \
195 DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, KEY_BASE + 0) \
196 DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 40, I0, I1, I2, I3, KEY_BASE + 0) \
197 AES_DROUND23(KEY_BASE + 48, I0, I1, KEY_BASE + 2) \
198 AES_DROUND01(KEY_BASE + 50, I0, I1, KEY_BASE + 0) \
199 AES_DROUND23(KEY_BASE + 48, I2, I3, KEY_BASE + 6) \
200 AES_DROUND01(KEY_BASE + 50, I2, I3, KEY_BASE + 4) \
201 AES_DROUND23_L(KEY_BASE + 52, KEY_BASE + 0, KEY_BASE + 2, I1) \
202 AES_DROUND01_L(KEY_BASE + 54, KEY_BASE + 0, KEY_BASE + 2, I0) \
203 ldd [%o0 + 0xd8], %f8; \
204 ldd [%o0 + 0xd0], %f10; \
205 AES_DROUND23_L(KEY_BASE + 52, KEY_BASE + 4, KEY_BASE + 6, I3) \
206 AES_DROUND01_L(KEY_BASE + 54, KEY_BASE + 4, KEY_BASE + 6, I2) \
207 ldd [%o0 + 0xc8], %f12; \
208 ldd [%o0 + 0xc0], %f14;
209
210 .align 32
211ENTRY(aes_sparc64_key_expand)
212 /* %o0=input_key, %o1=output_key, %o2=key_len */
213 VISEntry
214 ld [%o0 + 0x00], %f0
215 ld [%o0 + 0x04], %f1
216 ld [%o0 + 0x08], %f2
217 ld [%o0 + 0x0c], %f3
218
219 std %f0, [%o1 + 0x00]
220 std %f2, [%o1 + 0x08]
221 add %o1, 0x10, %o1
222
223 cmp %o2, 24
224 bl 2f
225 nop
226
227 be 1f
228 nop
229
230 /* 256-bit key expansion */
231 ld [%o0 + 0x10], %f4
232 ld [%o0 + 0x14], %f5
233 ld [%o0 + 0x18], %f6
234 ld [%o0 + 0x1c], %f7
235
236 std %f4, [%o1 + 0x00]
237 std %f6, [%o1 + 0x08]
238 add %o1, 0x10, %o1
239
240 AES_KEXPAND1(0, 6, 0x0, 8)
241 AES_KEXPAND2(2, 8, 10)
242 AES_KEXPAND0(4, 10, 12)
243 AES_KEXPAND2(6, 12, 14)
244 AES_KEXPAND1(8, 14, 0x1, 16)
245 AES_KEXPAND2(10, 16, 18)
246 AES_KEXPAND0(12, 18, 20)
247 AES_KEXPAND2(14, 20, 22)
248 AES_KEXPAND1(16, 22, 0x2, 24)
249 AES_KEXPAND2(18, 24, 26)
250 AES_KEXPAND0(20, 26, 28)
251 AES_KEXPAND2(22, 28, 30)
252 AES_KEXPAND1(24, 30, 0x3, 32)
253 AES_KEXPAND2(26, 32, 34)
254 AES_KEXPAND0(28, 34, 36)
255 AES_KEXPAND2(30, 36, 38)
256 AES_KEXPAND1(32, 38, 0x4, 40)
257 AES_KEXPAND2(34, 40, 42)
258 AES_KEXPAND0(36, 42, 44)
259 AES_KEXPAND2(38, 44, 46)
260 AES_KEXPAND1(40, 46, 0x5, 48)
261 AES_KEXPAND2(42, 48, 50)
262 AES_KEXPAND0(44, 50, 52)
263 AES_KEXPAND2(46, 52, 54)
264 AES_KEXPAND1(48, 54, 0x6, 56)
265 AES_KEXPAND2(50, 56, 58)
266
267 std %f8, [%o1 + 0x00]
268 std %f10, [%o1 + 0x08]
269 std %f12, [%o1 + 0x10]
270 std %f14, [%o1 + 0x18]
271 std %f16, [%o1 + 0x20]
272 std %f18, [%o1 + 0x28]
273 std %f20, [%o1 + 0x30]
274 std %f22, [%o1 + 0x38]
275 std %f24, [%o1 + 0x40]
276 std %f26, [%o1 + 0x48]
277 std %f28, [%o1 + 0x50]
278 std %f30, [%o1 + 0x58]
279 std %f32, [%o1 + 0x60]
280 std %f34, [%o1 + 0x68]
281 std %f36, [%o1 + 0x70]
282 std %f38, [%o1 + 0x78]
283 std %f40, [%o1 + 0x80]
284 std %f42, [%o1 + 0x88]
285 std %f44, [%o1 + 0x90]
286 std %f46, [%o1 + 0x98]
287 std %f48, [%o1 + 0xa0]
288 std %f50, [%o1 + 0xa8]
289 std %f52, [%o1 + 0xb0]
290 std %f54, [%o1 + 0xb8]
291 std %f56, [%o1 + 0xc0]
292 ba,pt %xcc, 80f
293 std %f58, [%o1 + 0xc8]
294
2951:
296 /* 192-bit key expansion */
297 ld [%o0 + 0x10], %f4
298 ld [%o0 + 0x14], %f5
299
300 std %f4, [%o1 + 0x00]
301 add %o1, 0x08, %o1
302
303 AES_KEXPAND1(0, 4, 0x0, 6)
304 AES_KEXPAND2(2, 6, 8)
305 AES_KEXPAND2(4, 8, 10)
306 AES_KEXPAND1(6, 10, 0x1, 12)
307 AES_KEXPAND2(8, 12, 14)
308 AES_KEXPAND2(10, 14, 16)
309 AES_KEXPAND1(12, 16, 0x2, 18)
310 AES_KEXPAND2(14, 18, 20)
311 AES_KEXPAND2(16, 20, 22)
312 AES_KEXPAND1(18, 22, 0x3, 24)
313 AES_KEXPAND2(20, 24, 26)
314 AES_KEXPAND2(22, 26, 28)
315 AES_KEXPAND1(24, 28, 0x4, 30)
316 AES_KEXPAND2(26, 30, 32)
317 AES_KEXPAND2(28, 32, 34)
318 AES_KEXPAND1(30, 34, 0x5, 36)
319 AES_KEXPAND2(32, 36, 38)
320 AES_KEXPAND2(34, 38, 40)
321 AES_KEXPAND1(36, 40, 0x6, 42)
322 AES_KEXPAND2(38, 42, 44)
323 AES_KEXPAND2(40, 44, 46)
324 AES_KEXPAND1(42, 46, 0x7, 48)
325 AES_KEXPAND2(44, 48, 50)
326
327 std %f6, [%o1 + 0x00]
328 std %f8, [%o1 + 0x08]
329 std %f10, [%o1 + 0x10]
330 std %f12, [%o1 + 0x18]
331 std %f14, [%o1 + 0x20]
332 std %f16, [%o1 + 0x28]
333 std %f18, [%o1 + 0x30]
334 std %f20, [%o1 + 0x38]
335 std %f22, [%o1 + 0x40]
336 std %f24, [%o1 + 0x48]
337 std %f26, [%o1 + 0x50]
338 std %f28, [%o1 + 0x58]
339 std %f30, [%o1 + 0x60]
340 std %f32, [%o1 + 0x68]
341 std %f34, [%o1 + 0x70]
342 std %f36, [%o1 + 0x78]
343 std %f38, [%o1 + 0x80]
344 std %f40, [%o1 + 0x88]
345 std %f42, [%o1 + 0x90]
346 std %f44, [%o1 + 0x98]
347 std %f46, [%o1 + 0xa0]
348 std %f48, [%o1 + 0xa8]
349 ba,pt %xcc, 80f
350 std %f50, [%o1 + 0xb0]
351
3522:
353 /* 128-bit key expansion */
354 AES_KEXPAND1(0, 2, 0x0, 4)
355 AES_KEXPAND2(2, 4, 6)
356 AES_KEXPAND1(4, 6, 0x1, 8)
357 AES_KEXPAND2(6, 8, 10)
358 AES_KEXPAND1(8, 10, 0x2, 12)
359 AES_KEXPAND2(10, 12, 14)
360 AES_KEXPAND1(12, 14, 0x3, 16)
361 AES_KEXPAND2(14, 16, 18)
362 AES_KEXPAND1(16, 18, 0x4, 20)
363 AES_KEXPAND2(18, 20, 22)
364 AES_KEXPAND1(20, 22, 0x5, 24)
365 AES_KEXPAND2(22, 24, 26)
366 AES_KEXPAND1(24, 26, 0x6, 28)
367 AES_KEXPAND2(26, 28, 30)
368 AES_KEXPAND1(28, 30, 0x7, 32)
369 AES_KEXPAND2(30, 32, 34)
370 AES_KEXPAND1(32, 34, 0x8, 36)
371 AES_KEXPAND2(34, 36, 38)
372 AES_KEXPAND1(36, 38, 0x9, 40)
373 AES_KEXPAND2(38, 40, 42)
374
375 std %f4, [%o1 + 0x00]
376 std %f6, [%o1 + 0x08]
377 std %f8, [%o1 + 0x10]
378 std %f10, [%o1 + 0x18]
379 std %f12, [%o1 + 0x20]
380 std %f14, [%o1 + 0x28]
381 std %f16, [%o1 + 0x30]
382 std %f18, [%o1 + 0x38]
383 std %f20, [%o1 + 0x40]
384 std %f22, [%o1 + 0x48]
385 std %f24, [%o1 + 0x50]
386 std %f26, [%o1 + 0x58]
387 std %f28, [%o1 + 0x60]
388 std %f30, [%o1 + 0x68]
389 std %f32, [%o1 + 0x70]
390 std %f34, [%o1 + 0x78]
391 std %f36, [%o1 + 0x80]
392 std %f38, [%o1 + 0x88]
393 std %f40, [%o1 + 0x90]
394 std %f42, [%o1 + 0x98]
39580:
396 retl
397 VISExit
398ENDPROC(aes_sparc64_key_expand)
399
400 .align 32
401ENTRY(aes_sparc64_encrypt_128)
402 /* %o0=key, %o1=input, %o2=output */
403 VISEntry
404 ld [%o1 + 0x00], %f4
405 ld [%o1 + 0x04], %f5
406 ld [%o1 + 0x08], %f6
407 ld [%o1 + 0x0c], %f7
408 ldd [%o0 + 0x00], %f8
409 ldd [%o0 + 0x08], %f10
410 ldd [%o0 + 0x10], %f12
411 ldd [%o0 + 0x18], %f14
412 ldd [%o0 + 0x20], %f16
413 ldd [%o0 + 0x28], %f18
414 ldd [%o0 + 0x30], %f20
415 ldd [%o0 + 0x38], %f22
416 ldd [%o0 + 0x40], %f24
417 ldd [%o0 + 0x48], %f26
418 ldd [%o0 + 0x50], %f28
419 ldd [%o0 + 0x58], %f30
420 ldd [%o0 + 0x60], %f32
421 ldd [%o0 + 0x68], %f34
422 ldd [%o0 + 0x70], %f36
423 ldd [%o0 + 0x78], %f38
424 ldd [%o0 + 0x80], %f40
425 ldd [%o0 + 0x88], %f42
426 ldd [%o0 + 0x90], %f44
427 ldd [%o0 + 0x98], %f46
428 ldd [%o0 + 0xa0], %f48
429 ldd [%o0 + 0xa8], %f50
430 fxor %f8, %f4, %f4
431 fxor %f10, %f6, %f6
432 ENCRYPT_128(12, 4, 6, 0, 2)
433 st %f4, [%o2 + 0x00]
434 st %f5, [%o2 + 0x04]
435 st %f6, [%o2 + 0x08]
436 st %f7, [%o2 + 0x0c]
437 retl
438 VISExit
439ENDPROC(aes_sparc64_encrypt_128)
440
441 .align 32
442ENTRY(aes_sparc64_encrypt_192)
443 /* %o0=key, %o1=input, %o2=output */
444 VISEntry
445 ld [%o1 + 0x00], %f4
446 ld [%o1 + 0x04], %f5
447 ld [%o1 + 0x08], %f6
448 ld [%o1 + 0x0c], %f7
449
450 ldd [%o0 + 0x00], %f8
451 ldd [%o0 + 0x08], %f10
452
453 fxor %f8, %f4, %f4
454 fxor %f10, %f6, %f6
455
456 ldd [%o0 + 0x10], %f8
457 ldd [%o0 + 0x18], %f10
458 ldd [%o0 + 0x20], %f12
459 ldd [%o0 + 0x28], %f14
460 add %o0, 0x20, %o0
461
462 ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
463
464 ldd [%o0 + 0x10], %f12
465 ldd [%o0 + 0x18], %f14
466 ldd [%o0 + 0x20], %f16
467 ldd [%o0 + 0x28], %f18
468 ldd [%o0 + 0x30], %f20
469 ldd [%o0 + 0x38], %f22
470 ldd [%o0 + 0x40], %f24
471 ldd [%o0 + 0x48], %f26
472 ldd [%o0 + 0x50], %f28
473 ldd [%o0 + 0x58], %f30
474 ldd [%o0 + 0x60], %f32
475 ldd [%o0 + 0x68], %f34
476 ldd [%o0 + 0x70], %f36
477 ldd [%o0 + 0x78], %f38
478 ldd [%o0 + 0x80], %f40
479 ldd [%o0 + 0x88], %f42
480 ldd [%o0 + 0x90], %f44
481 ldd [%o0 + 0x98], %f46
482 ldd [%o0 + 0xa0], %f48
483 ldd [%o0 + 0xa8], %f50
484
485
486 ENCRYPT_128(12, 4, 6, 0, 2)
487
488 st %f4, [%o2 + 0x00]
489 st %f5, [%o2 + 0x04]
490 st %f6, [%o2 + 0x08]
491 st %f7, [%o2 + 0x0c]
492
493 retl
494 VISExit
495ENDPROC(aes_sparc64_encrypt_192)
496
497 .align 32
498ENTRY(aes_sparc64_encrypt_256)
499 /* %o0=key, %o1=input, %o2=output */
500 VISEntry
501 ld [%o1 + 0x00], %f4
502 ld [%o1 + 0x04], %f5
503 ld [%o1 + 0x08], %f6
504 ld [%o1 + 0x0c], %f7
505
506 ldd [%o0 + 0x00], %f8
507 ldd [%o0 + 0x08], %f10
508
509 fxor %f8, %f4, %f4
510 fxor %f10, %f6, %f6
511
512 ldd [%o0 + 0x10], %f8
513
514 ldd [%o0 + 0x18], %f10
515 ldd [%o0 + 0x20], %f12
516 ldd [%o0 + 0x28], %f14
517 add %o0, 0x20, %o0
518
519 ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
520
521 ldd [%o0 + 0x10], %f8
522
523 ldd [%o0 + 0x18], %f10
524 ldd [%o0 + 0x20], %f12
525 ldd [%o0 + 0x28], %f14
526 add %o0, 0x20, %o0
527
528 ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
529
530 ldd [%o0 + 0x10], %f12
531 ldd [%o0 + 0x18], %f14
532 ldd [%o0 + 0x20], %f16
533 ldd [%o0 + 0x28], %f18
534 ldd [%o0 + 0x30], %f20
535 ldd [%o0 + 0x38], %f22
536 ldd [%o0 + 0x40], %f24
537 ldd [%o0 + 0x48], %f26
538 ldd [%o0 + 0x50], %f28
539 ldd [%o0 + 0x58], %f30
540 ldd [%o0 + 0x60], %f32
541 ldd [%o0 + 0x68], %f34
542 ldd [%o0 + 0x70], %f36
543 ldd [%o0 + 0x78], %f38
544 ldd [%o0 + 0x80], %f40
545 ldd [%o0 + 0x88], %f42
546 ldd [%o0 + 0x90], %f44
547 ldd [%o0 + 0x98], %f46
548 ldd [%o0 + 0xa0], %f48
549 ldd [%o0 + 0xa8], %f50
550
551 ENCRYPT_128(12, 4, 6, 0, 2)
552
553 st %f4, [%o2 + 0x00]
554 st %f5, [%o2 + 0x04]
555 st %f6, [%o2 + 0x08]
556 st %f7, [%o2 + 0x0c]
557
558 retl
559 VISExit
560ENDPROC(aes_sparc64_encrypt_256)
561
562 .align 32
563ENTRY(aes_sparc64_decrypt_128)
564 /* %o0=key, %o1=input, %o2=output */
565 VISEntry
566 ld [%o1 + 0x00], %f4
567 ld [%o1 + 0x04], %f5
568 ld [%o1 + 0x08], %f6
569 ld [%o1 + 0x0c], %f7
570 ldd [%o0 + 0xa0], %f8
571 ldd [%o0 + 0xa8], %f10
572 ldd [%o0 + 0x98], %f12
573 ldd [%o0 + 0x90], %f14
574 ldd [%o0 + 0x88], %f16
575 ldd [%o0 + 0x80], %f18
576 ldd [%o0 + 0x78], %f20
577 ldd [%o0 + 0x70], %f22
578 ldd [%o0 + 0x68], %f24
579 ldd [%o0 + 0x60], %f26
580 ldd [%o0 + 0x58], %f28
581 ldd [%o0 + 0x50], %f30
582 ldd [%o0 + 0x48], %f32
583 ldd [%o0 + 0x40], %f34
584 ldd [%o0 + 0x38], %f36
585 ldd [%o0 + 0x30], %f38
586 ldd [%o0 + 0x28], %f40
587 ldd [%o0 + 0x20], %f42
588 ldd [%o0 + 0x18], %f44
589 ldd [%o0 + 0x10], %f46
590 ldd [%o0 + 0x08], %f48
591 ldd [%o0 + 0x00], %f50
592 fxor %f8, %f4, %f4
593 fxor %f10, %f6, %f6
594 DECRYPT_128(12, 4, 6, 0, 2)
595 st %f4, [%o2 + 0x00]
596 st %f5, [%o2 + 0x04]
597 st %f6, [%o2 + 0x08]
598 st %f7, [%o2 + 0x0c]
599 retl
600 VISExit
601ENDPROC(aes_sparc64_decrypt_128)
602
603 .align 32
604ENTRY(aes_sparc64_decrypt_192)
605 /* %o0=key, %o1=input, %o2=output */
606 VISEntry
607 ld [%o1 + 0x00], %f4
608 ld [%o1 + 0x04], %f5
609 ld [%o1 + 0x08], %f6
610 ld [%o1 + 0x0c], %f7
611 ldd [%o0 + 0xc0], %f8
612 ldd [%o0 + 0xc8], %f10
613 ldd [%o0 + 0xb8], %f12
614 ldd [%o0 + 0xb0], %f14
615 ldd [%o0 + 0xa8], %f16
616 ldd [%o0 + 0xa0], %f18
617 fxor %f8, %f4, %f4
618 fxor %f10, %f6, %f6
619 ldd [%o0 + 0x98], %f20
620 ldd [%o0 + 0x90], %f22
621 ldd [%o0 + 0x88], %f24
622 ldd [%o0 + 0x80], %f26
623 DECRYPT_TWO_ROUNDS(12, 4, 6, 0, 2)
624 ldd [%o0 + 0x78], %f28
625 ldd [%o0 + 0x70], %f30
626 ldd [%o0 + 0x68], %f32
627 ldd [%o0 + 0x60], %f34
628 ldd [%o0 + 0x58], %f36
629 ldd [%o0 + 0x50], %f38
630 ldd [%o0 + 0x48], %f40
631 ldd [%o0 + 0x40], %f42
632 ldd [%o0 + 0x38], %f44
633 ldd [%o0 + 0x30], %f46
634 ldd [%o0 + 0x28], %f48
635 ldd [%o0 + 0x20], %f50
636 ldd [%o0 + 0x18], %f52
637 ldd [%o0 + 0x10], %f54
638 ldd [%o0 + 0x08], %f56
639 ldd [%o0 + 0x00], %f58
640 DECRYPT_128(20, 4, 6, 0, 2)
641 st %f4, [%o2 + 0x00]
642 st %f5, [%o2 + 0x04]
643 st %f6, [%o2 + 0x08]
644 st %f7, [%o2 + 0x0c]
645 retl
646 VISExit
647ENDPROC(aes_sparc64_decrypt_192)
648
649 .align 32
650ENTRY(aes_sparc64_decrypt_256)
651 /* %o0=key, %o1=input, %o2=output */
652 VISEntry
653 ld [%o1 + 0x00], %f4
654 ld [%o1 + 0x04], %f5
655 ld [%o1 + 0x08], %f6
656 ld [%o1 + 0x0c], %f7
657 ldd [%o0 + 0xe0], %f8
658 ldd [%o0 + 0xe8], %f10
659 ldd [%o0 + 0xd8], %f12
660 ldd [%o0 + 0xd0], %f14
661 ldd [%o0 + 0xc8], %f16
662 fxor %f8, %f4, %f4
663 ldd [%o0 + 0xc0], %f18
664 fxor %f10, %f6, %f6
665 ldd [%o0 + 0xb8], %f20
666 AES_DROUND23(12, 4, 6, 2)
667 ldd [%o0 + 0xb0], %f22
668 AES_DROUND01(14, 4, 6, 0)
669 ldd [%o0 + 0xa8], %f24
670 AES_DROUND23(16, 0, 2, 6)
671 ldd [%o0 + 0xa0], %f26
672 AES_DROUND01(18, 0, 2, 4)
673 ldd [%o0 + 0x98], %f12
674 AES_DROUND23(20, 4, 6, 2)
675 ldd [%o0 + 0x90], %f14
676 AES_DROUND01(22, 4, 6, 0)
677 ldd [%o0 + 0x88], %f16
678 AES_DROUND23(24, 0, 2, 6)
679 ldd [%o0 + 0x80], %f18
680 AES_DROUND01(26, 0, 2, 4)
681 ldd [%o0 + 0x78], %f20
682 AES_DROUND23(12, 4, 6, 2)
683 ldd [%o0 + 0x70], %f22
684 AES_DROUND01(14, 4, 6, 0)
685 ldd [%o0 + 0x68], %f24
686 AES_DROUND23(16, 0, 2, 6)
687 ldd [%o0 + 0x60], %f26
688 AES_DROUND01(18, 0, 2, 4)
689 ldd [%o0 + 0x58], %f28
690 AES_DROUND23(20, 4, 6, 2)
691 ldd [%o0 + 0x50], %f30
692 AES_DROUND01(22, 4, 6, 0)
693 ldd [%o0 + 0x48], %f32
694 AES_DROUND23(24, 0, 2, 6)
695 ldd [%o0 + 0x40], %f34
696 AES_DROUND01(26, 0, 2, 4)
697 ldd [%o0 + 0x38], %f36
698 AES_DROUND23(28, 4, 6, 2)
699 ldd [%o0 + 0x30], %f38
700 AES_DROUND01(30, 4, 6, 0)
701 ldd [%o0 + 0x28], %f40
702 AES_DROUND23(32, 0, 2, 6)
703 ldd [%o0 + 0x20], %f42
704 AES_DROUND01(34, 0, 2, 4)
705 ldd [%o0 + 0x18], %f44
706 AES_DROUND23(36, 4, 6, 2)
707 ldd [%o0 + 0x10], %f46
708 AES_DROUND01(38, 4, 6, 0)
709 ldd [%o0 + 0x08], %f48
710 AES_DROUND23(40, 0, 2, 6)
711 ldd [%o0 + 0x00], %f50
712 AES_DROUND01(42, 0, 2, 4)
713 AES_DROUND23(44, 4, 6, 2)
714 AES_DROUND01(46, 4, 6, 0)
715 AES_DROUND23_L(48, 0, 2, 6)
716 AES_DROUND01_L(50, 0, 2, 4)
717 st %f4, [%o2 + 0x00]
718 st %f5, [%o2 + 0x04]
719 st %f6, [%o2 + 0x08]
720 st %f7, [%o2 + 0x0c]
721 retl
722 VISExit
723ENDPROC(aes_sparc64_decrypt_256)
724
725 .align 32
726ENTRY(aes_sparc64_load_encrypt_keys_128)
727 /* %o0=key */
728 VISEntry
729 ldd [%o0 + 0x10], %f8
730 ldd [%o0 + 0x18], %f10
731 ldd [%o0 + 0x20], %f12
732 ldd [%o0 + 0x28], %f14
733 ldd [%o0 + 0x30], %f16
734 ldd [%o0 + 0x38], %f18
735 ldd [%o0 + 0x40], %f20
736 ldd [%o0 + 0x48], %f22
737 ldd [%o0 + 0x50], %f24
738 ldd [%o0 + 0x58], %f26
739 ldd [%o0 + 0x60], %f28
740 ldd [%o0 + 0x68], %f30
741 ldd [%o0 + 0x70], %f32
742 ldd [%o0 + 0x78], %f34
743 ldd [%o0 + 0x80], %f36
744 ldd [%o0 + 0x88], %f38
745 ldd [%o0 + 0x90], %f40
746 ldd [%o0 + 0x98], %f42
747 ldd [%o0 + 0xa0], %f44
748 retl
749 ldd [%o0 + 0xa8], %f46
750ENDPROC(aes_sparc64_load_encrypt_keys_128)
751
752 .align 32
753ENTRY(aes_sparc64_load_encrypt_keys_192)
754 /* %o0=key */
755 VISEntry
756 ldd [%o0 + 0x10], %f8
757 ldd [%o0 + 0x18], %f10
758 ldd [%o0 + 0x20], %f12
759 ldd [%o0 + 0x28], %f14
760 ldd [%o0 + 0x30], %f16
761 ldd [%o0 + 0x38], %f18
762 ldd [%o0 + 0x40], %f20
763 ldd [%o0 + 0x48], %f22
764 ldd [%o0 + 0x50], %f24
765 ldd [%o0 + 0x58], %f26
766 ldd [%o0 + 0x60], %f28
767 ldd [%o0 + 0x68], %f30
768 ldd [%o0 + 0x70], %f32
769 ldd [%o0 + 0x78], %f34
770 ldd [%o0 + 0x80], %f36
771 ldd [%o0 + 0x88], %f38
772 ldd [%o0 + 0x90], %f40
773 ldd [%o0 + 0x98], %f42
774 ldd [%o0 + 0xa0], %f44
775 ldd [%o0 + 0xa8], %f46
776 ldd [%o0 + 0xb0], %f48
777 ldd [%o0 + 0xb8], %f50
778 ldd [%o0 + 0xc0], %f52
779 retl
780 ldd [%o0 + 0xc8], %f54
781ENDPROC(aes_sparc64_load_encrypt_keys_192)
782
783 .align 32
784ENTRY(aes_sparc64_load_encrypt_keys_256)
785 /* %o0=key */
786 VISEntry
787 ldd [%o0 + 0x10], %f8
788 ldd [%o0 + 0x18], %f10
789 ldd [%o0 + 0x20], %f12
790 ldd [%o0 + 0x28], %f14
791 ldd [%o0 + 0x30], %f16
792 ldd [%o0 + 0x38], %f18
793 ldd [%o0 + 0x40], %f20
794 ldd [%o0 + 0x48], %f22
795 ldd [%o0 + 0x50], %f24
796 ldd [%o0 + 0x58], %f26
797 ldd [%o0 + 0x60], %f28
798 ldd [%o0 + 0x68], %f30
799 ldd [%o0 + 0x70], %f32
800 ldd [%o0 + 0x78], %f34
801 ldd [%o0 + 0x80], %f36
802 ldd [%o0 + 0x88], %f38
803 ldd [%o0 + 0x90], %f40
804 ldd [%o0 + 0x98], %f42
805 ldd [%o0 + 0xa0], %f44
806 ldd [%o0 + 0xa8], %f46
807 ldd [%o0 + 0xb0], %f48
808 ldd [%o0 + 0xb8], %f50
809 ldd [%o0 + 0xc0], %f52
810 ldd [%o0 + 0xc8], %f54
811 ldd [%o0 + 0xd0], %f56
812 ldd [%o0 + 0xd8], %f58
813 ldd [%o0 + 0xe0], %f60
814 retl
815 ldd [%o0 + 0xe8], %f62
816ENDPROC(aes_sparc64_load_encrypt_keys_256)
817
818 .align 32
819ENTRY(aes_sparc64_load_decrypt_keys_128)
820 /* %o0=key */
821 VISEntry
822 ldd [%o0 + 0x98], %f8
823 ldd [%o0 + 0x90], %f10
824 ldd [%o0 + 0x88], %f12
825 ldd [%o0 + 0x80], %f14
826 ldd [%o0 + 0x78], %f16
827 ldd [%o0 + 0x70], %f18
828 ldd [%o0 + 0x68], %f20
829 ldd [%o0 + 0x60], %f22
830 ldd [%o0 + 0x58], %f24
831 ldd [%o0 + 0x50], %f26
832 ldd [%o0 + 0x48], %f28
833 ldd [%o0 + 0x40], %f30
834 ldd [%o0 + 0x38], %f32
835 ldd [%o0 + 0x30], %f34
836 ldd [%o0 + 0x28], %f36
837 ldd [%o0 + 0x20], %f38
838 ldd [%o0 + 0x18], %f40
839 ldd [%o0 + 0x10], %f42
840 ldd [%o0 + 0x08], %f44
841 retl
842 ldd [%o0 + 0x00], %f46
843ENDPROC(aes_sparc64_load_decrypt_keys_128)
844
845 .align 32
846ENTRY(aes_sparc64_load_decrypt_keys_192)
847 /* %o0=key */
848 VISEntry
849 ldd [%o0 + 0xb8], %f8
850 ldd [%o0 + 0xb0], %f10
851 ldd [%o0 + 0xa8], %f12
852 ldd [%o0 + 0xa0], %f14
853 ldd [%o0 + 0x98], %f16
854 ldd [%o0 + 0x90], %f18
855 ldd [%o0 + 0x88], %f20
856 ldd [%o0 + 0x80], %f22
857 ldd [%o0 + 0x78], %f24
858 ldd [%o0 + 0x70], %f26
859 ldd [%o0 + 0x68], %f28
860 ldd [%o0 + 0x60], %f30
861 ldd [%o0 + 0x58], %f32
862 ldd [%o0 + 0x50], %f34
863 ldd [%o0 + 0x48], %f36
864 ldd [%o0 + 0x40], %f38
865 ldd [%o0 + 0x38], %f40
866 ldd [%o0 + 0x30], %f42
867 ldd [%o0 + 0x28], %f44
868 ldd [%o0 + 0x20], %f46
869 ldd [%o0 + 0x18], %f48
870 ldd [%o0 + 0x10], %f50
871 ldd [%o0 + 0x08], %f52
872 retl
873 ldd [%o0 + 0x00], %f54
874ENDPROC(aes_sparc64_load_decrypt_keys_192)
875
876 .align 32
877ENTRY(aes_sparc64_load_decrypt_keys_256)
878 /* %o0=key */
879 VISEntry
880 ldd [%o0 + 0xd8], %f8
881 ldd [%o0 + 0xd0], %f10
882 ldd [%o0 + 0xc8], %f12
883 ldd [%o0 + 0xc0], %f14
884 ldd [%o0 + 0xb8], %f16
885 ldd [%o0 + 0xb0], %f18
886 ldd [%o0 + 0xa8], %f20
887 ldd [%o0 + 0xa0], %f22
888 ldd [%o0 + 0x98], %f24
889 ldd [%o0 + 0x90], %f26
890 ldd [%o0 + 0x88], %f28
891 ldd [%o0 + 0x80], %f30
892 ldd [%o0 + 0x78], %f32
893 ldd [%o0 + 0x70], %f34
894 ldd [%o0 + 0x68], %f36
895 ldd [%o0 + 0x60], %f38
896 ldd [%o0 + 0x58], %f40
897 ldd [%o0 + 0x50], %f42
898 ldd [%o0 + 0x48], %f44
899 ldd [%o0 + 0x40], %f46
900 ldd [%o0 + 0x38], %f48
901 ldd [%o0 + 0x30], %f50
902 ldd [%o0 + 0x28], %f52
903 ldd [%o0 + 0x20], %f54
904 ldd [%o0 + 0x18], %f56
905 ldd [%o0 + 0x10], %f58
906 ldd [%o0 + 0x08], %f60
907 retl
908 ldd [%o0 + 0x00], %f62
909ENDPROC(aes_sparc64_load_decrypt_keys_256)
910
911 .align 32
912ENTRY(aes_sparc64_ecb_encrypt_128)
913 /* %o0=key, %o1=input, %o2=output, %o3=len */
914 ldx [%o0 + 0x00], %g1
915 subcc %o3, 0x10, %o3
916 be 10f
917 ldx [%o0 + 0x08], %g2
9181: ldx [%o1 + 0x00], %g3
919 ldx [%o1 + 0x08], %g7
920 ldx [%o1 + 0x10], %o4
921 ldx [%o1 + 0x18], %o5
922 xor %g1, %g3, %g3
923 xor %g2, %g7, %g7
924 MOVXTOD_G3_F4
925 MOVXTOD_G7_F6
926 xor %g1, %o4, %g3
927 xor %g2, %o5, %g7
928 MOVXTOD_G3_F60
929 MOVXTOD_G7_F62
930 ENCRYPT_128_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
931 std %f4, [%o2 + 0x00]
932 std %f6, [%o2 + 0x08]
933 std %f60, [%o2 + 0x10]
934 std %f62, [%o2 + 0x18]
935 sub %o3, 0x20, %o3
936 add %o1, 0x20, %o1
937 brgz %o3, 1b
938 add %o2, 0x20, %o2
939 brlz,pt %o3, 11f
940 nop
94110: ldx [%o1 + 0x00], %g3
942 ldx [%o1 + 0x08], %g7
943 xor %g1, %g3, %g3
944 xor %g2, %g7, %g7
945 MOVXTOD_G3_F4
946 MOVXTOD_G7_F6
947 ENCRYPT_128(8, 4, 6, 0, 2)
948 std %f4, [%o2 + 0x00]
949 std %f6, [%o2 + 0x08]
95011: retl
951 nop
952ENDPROC(aes_sparc64_ecb_encrypt_128)
953
954 .align 32
955ENTRY(aes_sparc64_ecb_encrypt_192)
956 /* %o0=key, %o1=input, %o2=output, %o3=len */
957 ldx [%o0 + 0x00], %g1
958 subcc %o3, 0x10, %o3
959 be 10f
960 ldx [%o0 + 0x08], %g2
9611: ldx [%o1 + 0x00], %g3
962 ldx [%o1 + 0x08], %g7
963 ldx [%o1 + 0x10], %o4
964 ldx [%o1 + 0x18], %o5
965 xor %g1, %g3, %g3
966 xor %g2, %g7, %g7
967 MOVXTOD_G3_F4
968 MOVXTOD_G7_F6
969 xor %g1, %o4, %g3
970 xor %g2, %o5, %g7
971 MOVXTOD_G3_F60
972 MOVXTOD_G7_F62
973 ENCRYPT_192_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
974 std %f4, [%o2 + 0x00]
975 std %f6, [%o2 + 0x08]
976 std %f60, [%o2 + 0x10]
977 std %f62, [%o2 + 0x18]
978 sub %o3, 0x20, %o3
979 add %o1, 0x20, %o1
980 brgz %o3, 1b
981 add %o2, 0x20, %o2
982 brlz,pt %o3, 11f
983 nop
98410: ldx [%o1 + 0x00], %g3
985 ldx [%o1 + 0x08], %g7
986 xor %g1, %g3, %g3
987 xor %g2, %g7, %g7
988 MOVXTOD_G3_F4
989 MOVXTOD_G7_F6
990 ENCRYPT_192(8, 4, 6, 0, 2)
991 std %f4, [%o2 + 0x00]
992 std %f6, [%o2 + 0x08]
99311: retl
994 nop
995ENDPROC(aes_sparc64_ecb_encrypt_192)
996
997 .align 32
998ENTRY(aes_sparc64_ecb_encrypt_256)
999 /* %o0=key, %o1=input, %o2=output, %o3=len */
1000 ldx [%o0 + 0x00], %g1
1001 subcc %o3, 0x10, %o3
1002 be 10f
1003 ldx [%o0 + 0x08], %g2
10041: ldx [%o1 + 0x00], %g3
1005 ldx [%o1 + 0x08], %g7
1006 ldx [%o1 + 0x10], %o4
1007 ldx [%o1 + 0x18], %o5
1008 xor %g1, %g3, %g3
1009 xor %g2, %g7, %g7
1010 MOVXTOD_G3_F4
1011 MOVXTOD_G7_F6
1012 xor %g1, %o4, %g3
1013 xor %g2, %o5, %g7
1014 MOVXTOD_G3_F0
1015 MOVXTOD_G7_F2
1016 ENCRYPT_256_2(8, 4, 6, 0, 2)
1017 std %f4, [%o2 + 0x00]
1018 std %f6, [%o2 + 0x08]
1019 std %f0, [%o2 + 0x10]
1020 std %f2, [%o2 + 0x18]
1021 sub %o3, 0x20, %o3
1022 add %o1, 0x20, %o1
1023 brgz %o3, 1b
1024 add %o2, 0x20, %o2
1025 brlz,pt %o3, 11f
1026 nop
102710: ldx [%o1 + 0x00], %g3
1028 ldx [%o1 + 0x08], %g7
1029 xor %g1, %g3, %g3
1030 xor %g2, %g7, %g7
1031 MOVXTOD_G3_F4
1032 MOVXTOD_G7_F6
1033 ENCRYPT_256(8, 4, 6, 0, 2)
1034 std %f4, [%o2 + 0x00]
1035 std %f6, [%o2 + 0x08]
103611: retl
1037 nop
1038ENDPROC(aes_sparc64_ecb_encrypt_256)
1039
1040 .align 32
1041ENTRY(aes_sparc64_ecb_decrypt_128)
1042 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
1043 ldx [%o0 - 0x10], %g1
1044 subcc %o3, 0x10, %o3
1045 be 10f
1046 ldx [%o0 - 0x08], %g2
10471: ldx [%o1 + 0x00], %g3
1048 ldx [%o1 + 0x08], %g7
1049 ldx [%o1 + 0x10], %o4
1050 ldx [%o1 + 0x18], %o5
1051 xor %g1, %g3, %g3
1052 xor %g2, %g7, %g7
1053 MOVXTOD_G3_F4
1054 MOVXTOD_G7_F6
1055 xor %g1, %o4, %g3
1056 xor %g2, %o5, %g7
1057 MOVXTOD_G3_F60
1058 MOVXTOD_G7_F62
1059 DECRYPT_128_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
1060 std %f4, [%o2 + 0x00]
1061 std %f6, [%o2 + 0x08]
1062 std %f60, [%o2 + 0x10]
1063 std %f62, [%o2 + 0x18]
1064 sub %o3, 0x20, %o3
1065 add %o1, 0x20, %o1
1066 brgz,pt %o3, 1b
1067 add %o2, 0x20, %o2
1068 brlz,pt %o3, 11f
1069 nop
107010: ldx [%o1 + 0x00], %g3
1071 ldx [%o1 + 0x08], %g7
1072 xor %g1, %g3, %g3
1073 xor %g2, %g7, %g7
1074 MOVXTOD_G3_F4
1075 MOVXTOD_G7_F6
1076 DECRYPT_128(8, 4, 6, 0, 2)
1077 std %f4, [%o2 + 0x00]
1078 std %f6, [%o2 + 0x08]
107911: retl
1080 nop
1081ENDPROC(aes_sparc64_ecb_decrypt_128)
1082
1083 .align 32
1084ENTRY(aes_sparc64_ecb_decrypt_192)
1085 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
1086 ldx [%o0 - 0x10], %g1
1087 subcc %o3, 0x10, %o3
1088 be 10f
1089 ldx [%o0 - 0x08], %g2
10901: ldx [%o1 + 0x00], %g3
1091 ldx [%o1 + 0x08], %g7
1092 ldx [%o1 + 0x10], %o4
1093 ldx [%o1 + 0x18], %o5
1094 xor %g1, %g3, %g3
1095 xor %g2, %g7, %g7
1096 MOVXTOD_G3_F4
1097 MOVXTOD_G7_F6
1098 xor %g1, %o4, %g3
1099 xor %g2, %o5, %g7
1100 MOVXTOD_G3_F60
1101 MOVXTOD_G7_F62
1102 DECRYPT_192_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
1103 std %f4, [%o2 + 0x00]
1104 std %f6, [%o2 + 0x08]
1105 std %f60, [%o2 + 0x10]
1106 std %f62, [%o2 + 0x18]
1107 sub %o3, 0x20, %o3
1108 add %o1, 0x20, %o1
1109 brgz,pt %o3, 1b
1110 add %o2, 0x20, %o2
1111 brlz,pt %o3, 11f
1112 nop
111310: ldx [%o1 + 0x00], %g3
1114 ldx [%o1 + 0x08], %g7
1115 xor %g1, %g3, %g3
1116 xor %g2, %g7, %g7
1117 MOVXTOD_G3_F4
1118 MOVXTOD_G7_F6
1119 DECRYPT_192(8, 4, 6, 0, 2)
1120 std %f4, [%o2 + 0x00]
1121 std %f6, [%o2 + 0x08]
112211: retl
1123 nop
1124ENDPROC(aes_sparc64_ecb_decrypt_192)
1125
1126 .align 32
1127ENTRY(aes_sparc64_ecb_decrypt_256)
1128 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
1129 ldx [%o0 - 0x10], %g1
1130 subcc %o3, 0x10, %o3
1131 be 10f
1132 ldx [%o0 - 0x08], %g2
1133 sub %o0, 0xf0, %o0
11341: ldx [%o1 + 0x00], %g3
1135 ldx [%o1 + 0x08], %g7
1136 ldx [%o1 + 0x10], %o4
1137 ldx [%o1 + 0x18], %o5
1138 xor %g1, %g3, %g3
1139 xor %g2, %g7, %g7
1140 MOVXTOD_G3_F4
1141 MOVXTOD_G7_F6
1142 xor %g1, %o4, %g3
1143 xor %g2, %o5, %g7
1144 MOVXTOD_G3_F0
1145 MOVXTOD_G7_F2
1146 DECRYPT_256_2(8, 4, 6, 0, 2)
1147 std %f4, [%o2 + 0x00]
1148 std %f6, [%o2 + 0x08]
1149 std %f0, [%o2 + 0x10]
1150 std %f2, [%o2 + 0x18]
1151 sub %o3, 0x20, %o3
1152 add %o1, 0x20, %o1
1153 brgz,pt %o3, 1b
1154 add %o2, 0x20, %o2
1155 brlz,pt %o3, 11f
1156 nop
115710: ldx [%o1 + 0x00], %g3
1158 ldx [%o1 + 0x08], %g7
1159 xor %g1, %g3, %g3
1160 xor %g2, %g7, %g7
1161 MOVXTOD_G3_F4
1162 MOVXTOD_G7_F6
1163 DECRYPT_256(8, 4, 6, 0, 2)
1164 std %f4, [%o2 + 0x00]
1165 std %f6, [%o2 + 0x08]
116611: retl
1167 nop
1168ENDPROC(aes_sparc64_ecb_decrypt_256)
1169
1170 .align 32
1171ENTRY(aes_sparc64_cbc_encrypt_128)
1172 /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
1173 ldd [%o4 + 0x00], %f4
1174 ldd [%o4 + 0x08], %f6
1175 ldx [%o0 + 0x00], %g1
1176 ldx [%o0 + 0x08], %g2
11771: ldx [%o1 + 0x00], %g3
1178 ldx [%o1 + 0x08], %g7
1179 add %o1, 0x10, %o1
1180 xor %g1, %g3, %g3
1181 xor %g2, %g7, %g7
1182 MOVXTOD_G3_F0
1183 MOVXTOD_G7_F2
1184 fxor %f4, %f0, %f4
1185 fxor %f6, %f2, %f6
1186 ENCRYPT_128(8, 4, 6, 0, 2)
1187 std %f4, [%o2 + 0x00]
1188 std %f6, [%o2 + 0x08]
1189 subcc %o3, 0x10, %o3
1190 bne,pt %xcc, 1b
1191 add %o2, 0x10, %o2
1192 std %f4, [%o4 + 0x00]
1193 std %f6, [%o4 + 0x08]
1194 retl
1195 nop
1196ENDPROC(aes_sparc64_cbc_encrypt_128)
1197
1198 .align 32
1199ENTRY(aes_sparc64_cbc_encrypt_192)
1200 /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
1201 ldd [%o4 + 0x00], %f4
1202 ldd [%o4 + 0x08], %f6
1203 ldx [%o0 + 0x00], %g1
1204 ldx [%o0 + 0x08], %g2
12051: ldx [%o1 + 0x00], %g3
1206 ldx [%o1 + 0x08], %g7
1207 add %o1, 0x10, %o1
1208 xor %g1, %g3, %g3
1209 xor %g2, %g7, %g7
1210 MOVXTOD_G3_F0
1211 MOVXTOD_G7_F2
1212 fxor %f4, %f0, %f4
1213 fxor %f6, %f2, %f6
1214 ENCRYPT_192(8, 4, 6, 0, 2)
1215 std %f4, [%o2 + 0x00]
1216 std %f6, [%o2 + 0x08]
1217 subcc %o3, 0x10, %o3
1218 bne,pt %xcc, 1b
1219 add %o2, 0x10, %o2
1220 std %f4, [%o4 + 0x00]
1221 std %f6, [%o4 + 0x08]
1222 retl
1223 nop
1224ENDPROC(aes_sparc64_cbc_encrypt_192)
1225
1226 .align 32
1227ENTRY(aes_sparc64_cbc_encrypt_256)
1228 /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
1229 ldd [%o4 + 0x00], %f4
1230 ldd [%o4 + 0x08], %f6
1231 ldx [%o0 + 0x00], %g1
1232 ldx [%o0 + 0x08], %g2
12331: ldx [%o1 + 0x00], %g3
1234 ldx [%o1 + 0x08], %g7
1235 add %o1, 0x10, %o1
1236 xor %g1, %g3, %g3
1237 xor %g2, %g7, %g7
1238 MOVXTOD_G3_F0
1239 MOVXTOD_G7_F2
1240 fxor %f4, %f0, %f4
1241 fxor %f6, %f2, %f6
1242 ENCRYPT_256(8, 4, 6, 0, 2)
1243 std %f4, [%o2 + 0x00]
1244 std %f6, [%o2 + 0x08]
1245 subcc %o3, 0x10, %o3
1246 bne,pt %xcc, 1b
1247 add %o2, 0x10, %o2
1248 std %f4, [%o4 + 0x00]
1249 std %f6, [%o4 + 0x08]
1250 retl
1251 nop
1252ENDPROC(aes_sparc64_cbc_encrypt_256)
1253
1254 .align 32
1255ENTRY(aes_sparc64_cbc_decrypt_128)
1256 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
1257 ldx [%o0 - 0x10], %g1
1258 ldx [%o0 - 0x08], %g2
1259 ldx [%o4 + 0x00], %o0
1260 ldx [%o4 + 0x08], %o5
12611: ldx [%o1 + 0x00], %g3
1262 ldx [%o1 + 0x08], %g7
1263 add %o1, 0x10, %o1
1264 xor %g1, %g3, %g3
1265 xor %g2, %g7, %g7
1266 MOVXTOD_G3_F4
1267 MOVXTOD_G7_F6
1268 DECRYPT_128(8, 4, 6, 0, 2)
1269 MOVXTOD_O0_F0
1270 MOVXTOD_O5_F2
1271 xor %g1, %g3, %o0
1272 xor %g2, %g7, %o5
1273 fxor %f4, %f0, %f4
1274 fxor %f6, %f2, %f6
1275 std %f4, [%o2 + 0x00]
1276 std %f6, [%o2 + 0x08]
1277 subcc %o3, 0x10, %o3
1278 bne,pt %xcc, 1b
1279 add %o2, 0x10, %o2
1280 stx %o0, [%o4 + 0x00]
1281 stx %o5, [%o4 + 0x08]
1282 retl
1283 nop
1284ENDPROC(aes_sparc64_cbc_decrypt_128)
1285
1286 .align 32
1287ENTRY(aes_sparc64_cbc_decrypt_192)
1288 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
1289 ldx [%o0 - 0x10], %g1
1290 ldx [%o0 - 0x08], %g2
1291 ldx [%o4 + 0x00], %o0
1292 ldx [%o4 + 0x08], %o5
12931: ldx [%o1 + 0x00], %g3
1294 ldx [%o1 + 0x08], %g7
1295 add %o1, 0x10, %o1
1296 xor %g1, %g3, %g3
1297 xor %g2, %g7, %g7
1298 MOVXTOD_G3_F4
1299 MOVXTOD_G7_F6
1300 DECRYPT_192(8, 4, 6, 0, 2)
1301 MOVXTOD_O0_F0
1302 MOVXTOD_O5_F2
1303 xor %g1, %g3, %o0
1304 xor %g2, %g7, %o5
1305 fxor %f4, %f0, %f4
1306 fxor %f6, %f2, %f6
1307 std %f4, [%o2 + 0x00]
1308 std %f6, [%o2 + 0x08]
1309 subcc %o3, 0x10, %o3
1310 bne,pt %xcc, 1b
1311 add %o2, 0x10, %o2
1312 stx %o0, [%o4 + 0x00]
1313 stx %o5, [%o4 + 0x08]
1314 retl
1315 nop
1316ENDPROC(aes_sparc64_cbc_decrypt_192)
1317
1318 .align 32
1319ENTRY(aes_sparc64_cbc_decrypt_256)
1320 /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
1321 ldx [%o0 - 0x10], %g1
1322 ldx [%o0 - 0x08], %g2
1323 ldx [%o4 + 0x00], %o0
1324 ldx [%o4 + 0x08], %o5
13251: ldx [%o1 + 0x00], %g3
1326 ldx [%o1 + 0x08], %g7
1327 add %o1, 0x10, %o1
1328 xor %g1, %g3, %g3
1329 xor %g2, %g7, %g7
1330 MOVXTOD_G3_F4
1331 MOVXTOD_G7_F6
1332 DECRYPT_256(8, 4, 6, 0, 2)
1333 MOVXTOD_O0_F0
1334 MOVXTOD_O5_F2
1335 xor %g1, %g3, %o0
1336 xor %g2, %g7, %o5
1337 fxor %f4, %f0, %f4
1338 fxor %f6, %f2, %f6
1339 std %f4, [%o2 + 0x00]
1340 std %f6, [%o2 + 0x08]
1341 subcc %o3, 0x10, %o3
1342 bne,pt %xcc, 1b
1343 add %o2, 0x10, %o2
1344 stx %o0, [%o4 + 0x00]
1345 stx %o5, [%o4 + 0x08]
1346 retl
1347 nop
1348ENDPROC(aes_sparc64_cbc_decrypt_256)
1349
1350 .align 32
1351ENTRY(aes_sparc64_ctr_crypt_128)
1352 /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
1353 ldx [%o4 + 0x00], %g3
1354 ldx [%o4 + 0x08], %g7
1355 subcc %o3, 0x10, %o3
1356 ldx [%o0 + 0x00], %g1
1357 be 10f
1358 ldx [%o0 + 0x08], %g2
13591: xor %g1, %g3, %o5
1360 MOVXTOD_O5_F0
1361 xor %g2, %g7, %o5
1362 MOVXTOD_O5_F2
1363 add %g7, 1, %g7
1364 add %g3, 1, %o5
1365 movrz %g7, %o5, %g3
1366 xor %g1, %g3, %o5
1367 MOVXTOD_O5_F4
1368 xor %g2, %g7, %o5
1369 MOVXTOD_O5_F6
1370 add %g7, 1, %g7
1371 add %g3, 1, %o5
1372 movrz %g7, %o5, %g3
1373 ENCRYPT_128_2(8, 0, 2, 4, 6, 56, 58, 60, 62)
1374 ldd [%o1 + 0x00], %f56
1375 ldd [%o1 + 0x08], %f58
1376 ldd [%o1 + 0x10], %f60
1377 ldd [%o1 + 0x18], %f62
1378 fxor %f56, %f0, %f56
1379 fxor %f58, %f2, %f58
1380 fxor %f60, %f4, %f60
1381 fxor %f62, %f6, %f62
1382 std %f56, [%o2 + 0x00]
1383 std %f58, [%o2 + 0x08]
1384 std %f60, [%o2 + 0x10]
1385 std %f62, [%o2 + 0x18]
1386 subcc %o3, 0x20, %o3
1387 add %o1, 0x20, %o1
1388 brgz %o3, 1b
1389 add %o2, 0x20, %o2
1390 brlz,pt %o3, 11f
1391 nop
139210: xor %g1, %g3, %o5
1393 MOVXTOD_O5_F0
1394 xor %g2, %g7, %o5
1395 MOVXTOD_O5_F2
1396 add %g7, 1, %g7
1397 add %g3, 1, %o5
1398 movrz %g7, %o5, %g3
1399 ENCRYPT_128(8, 0, 2, 4, 6)
1400 ldd [%o1 + 0x00], %f4
1401 ldd [%o1 + 0x08], %f6
1402 fxor %f4, %f0, %f4
1403 fxor %f6, %f2, %f6
1404 std %f4, [%o2 + 0x00]
1405 std %f6, [%o2 + 0x08]
140611: stx %g3, [%o4 + 0x00]
1407 retl
1408 stx %g7, [%o4 + 0x08]
1409ENDPROC(aes_sparc64_ctr_crypt_128)
1410
1411 .align 32
1412ENTRY(aes_sparc64_ctr_crypt_192)
1413 /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
1414 ldx [%o4 + 0x00], %g3
1415 ldx [%o4 + 0x08], %g7
1416 subcc %o3, 0x10, %o3
1417 ldx [%o0 + 0x00], %g1
1418 be 10f
1419 ldx [%o0 + 0x08], %g2
14201: xor %g1, %g3, %o5
1421 MOVXTOD_O5_F0
1422 xor %g2, %g7, %o5
1423 MOVXTOD_O5_F2
1424 add %g7, 1, %g7
1425 add %g3, 1, %o5
1426 movrz %g7, %o5, %g3
1427 xor %g1, %g3, %o5
1428 MOVXTOD_O5_F4
1429 xor %g2, %g7, %o5
1430 MOVXTOD_O5_F6
1431 add %g7, 1, %g7
1432 add %g3, 1, %o5
1433 movrz %g7, %o5, %g3
1434 ENCRYPT_192_2(8, 0, 2, 4, 6, 56, 58, 60, 62)
1435 ldd [%o1 + 0x00], %f56
1436 ldd [%o1 + 0x08], %f58
1437 ldd [%o1 + 0x10], %f60
1438 ldd [%o1 + 0x18], %f62
1439 fxor %f56, %f0, %f56
1440 fxor %f58, %f2, %f58
1441 fxor %f60, %f4, %f60
1442 fxor %f62, %f6, %f62
1443 std %f56, [%o2 + 0x00]
1444 std %f58, [%o2 + 0x08]
1445 std %f60, [%o2 + 0x10]
1446 std %f62, [%o2 + 0x18]
1447 subcc %o3, 0x20, %o3
1448 add %o1, 0x20, %o1
1449 brgz %o3, 1b
1450 add %o2, 0x20, %o2
1451 brlz,pt %o3, 11f
1452 nop
145310: xor %g1, %g3, %o5
1454 MOVXTOD_O5_F0
1455 xor %g2, %g7, %o5
1456 MOVXTOD_O5_F2
1457 add %g7, 1, %g7
1458 add %g3, 1, %o5
1459 movrz %g7, %o5, %g3
1460 ENCRYPT_192(8, 0, 2, 4, 6)
1461 ldd [%o1 + 0x00], %f4
1462 ldd [%o1 + 0x08], %f6
1463 fxor %f4, %f0, %f4
1464 fxor %f6, %f2, %f6
1465 std %f4, [%o2 + 0x00]
1466 std %f6, [%o2 + 0x08]
146711: stx %g3, [%o4 + 0x00]
1468 retl
1469 stx %g7, [%o4 + 0x08]
1470ENDPROC(aes_sparc64_ctr_crypt_192)
1471
1472 .align 32
1473ENTRY(aes_sparc64_ctr_crypt_256)
1474 /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
1475 ldx [%o4 + 0x00], %g3
1476 ldx [%o4 + 0x08], %g7
1477 subcc %o3, 0x10, %o3
1478 ldx [%o0 + 0x00], %g1
1479 be 10f
1480 ldx [%o0 + 0x08], %g2
14811: xor %g1, %g3, %o5
1482 MOVXTOD_O5_F0
1483 xor %g2, %g7, %o5
1484 MOVXTOD_O5_F2
1485 add %g7, 1, %g7
1486 add %g3, 1, %o5
1487 movrz %g7, %o5, %g3
1488 xor %g1, %g3, %o5
1489 MOVXTOD_O5_F4
1490 xor %g2, %g7, %o5
1491 MOVXTOD_O5_F6
1492 add %g7, 1, %g7
1493 add %g3, 1, %o5
1494 movrz %g7, %o5, %g3
1495 ENCRYPT_256_2(8, 0, 2, 4, 6)
1496 ldd [%o1 + 0x00], %f56
1497 ldd [%o1 + 0x08], %f58
1498 ldd [%o1 + 0x10], %f60
1499 ldd [%o1 + 0x18], %f62
1500 fxor %f56, %f0, %f56
1501 fxor %f58, %f2, %f58
1502 fxor %f60, %f4, %f60
1503 fxor %f62, %f6, %f62
1504 std %f56, [%o2 + 0x00]
1505 std %f58, [%o2 + 0x08]
1506 std %f60, [%o2 + 0x10]
1507 std %f62, [%o2 + 0x18]
1508 subcc %o3, 0x20, %o3
1509 add %o1, 0x20, %o1
1510 brgz %o3, 1b
1511 add %o2, 0x20, %o2
1512 brlz,pt %o3, 11f
1513 nop
1514 ldd [%o0 + 0xd0], %f56
1515 ldd [%o0 + 0xd8], %f58
1516 ldd [%o0 + 0xe0], %f60
1517 ldd [%o0 + 0xe8], %f62
151810: xor %g1, %g3, %o5
1519 MOVXTOD_O5_F0
1520 xor %g2, %g7, %o5
1521 MOVXTOD_O5_F2
1522 add %g7, 1, %g7
1523 add %g3, 1, %o5
1524 movrz %g7, %o5, %g3
1525 ENCRYPT_256(8, 0, 2, 4, 6)
1526 ldd [%o1 + 0x00], %f4
1527 ldd [%o1 + 0x08], %f6
1528 fxor %f4, %f0, %f4
1529 fxor %f6, %f2, %f6
1530 std %f4, [%o2 + 0x00]
1531 std %f6, [%o2 + 0x08]
153211: stx %g3, [%o4 + 0x00]
1533 retl
1534 stx %g7, [%o4 + 0x08]
1535ENDPROC(aes_sparc64_ctr_crypt_256)
diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
new file mode 100644
index 000000000000..8f1c9980f637
--- /dev/null
+++ b/arch/sparc/crypto/aes_glue.c
@@ -0,0 +1,477 @@
1/* Glue code for AES encryption optimized for sparc64 crypto opcodes.
2 *
3 * This is based largely upon arch/x86/crypto/aesni-intel_glue.c
4 *
5 * Copyright (C) 2008, Intel Corp.
6 * Author: Huang Ying <ying.huang@intel.com>
7 *
8 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9 * interface for 64-bit kernels.
10 * Authors: Adrian Hoban <adrian.hoban@intel.com>
11 * Gabriele Paoloni <gabriele.paoloni@intel.com>
12 * Tadeusz Struk (tadeusz.struk@intel.com)
13 * Aidan O'Mahony (aidan.o.mahony@intel.com)
14 * Copyright (c) 2010, Intel Corporation.
15 */
16
17#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18
19#include <linux/crypto.h>
20#include <linux/init.h>
21#include <linux/module.h>
22#include <linux/mm.h>
23#include <linux/types.h>
24#include <crypto/algapi.h>
25#include <crypto/aes.h>
26
27#include <asm/fpumacro.h>
28#include <asm/pstate.h>
29#include <asm/elf.h>
30
31#include "opcodes.h"
32
33struct aes_ops {
34 void (*encrypt)(const u64 *key, const u32 *input, u32 *output);
35 void (*decrypt)(const u64 *key, const u32 *input, u32 *output);
36 void (*load_encrypt_keys)(const u64 *key);
37 void (*load_decrypt_keys)(const u64 *key);
38 void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
39 unsigned int len);
40 void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,
41 unsigned int len);
42 void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,
43 unsigned int len, u64 *iv);
44 void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
45 unsigned int len, u64 *iv);
46 void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output,
47 unsigned int len, u64 *iv);
48};
49
50struct crypto_sparc64_aes_ctx {
51 struct aes_ops *ops;
52 u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];
53 u32 key_length;
54 u32 expanded_key_length;
55};
56
57extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input,
58 u32 *output);
59extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input,
60 u32 *output);
61extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input,
62 u32 *output);
63
64extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input,
65 u32 *output);
66extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input,
67 u32 *output);
68extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input,
69 u32 *output);
70
71extern void aes_sparc64_load_encrypt_keys_128(const u64 *key);
72extern void aes_sparc64_load_encrypt_keys_192(const u64 *key);
73extern void aes_sparc64_load_encrypt_keys_256(const u64 *key);
74
75extern void aes_sparc64_load_decrypt_keys_128(const u64 *key);
76extern void aes_sparc64_load_decrypt_keys_192(const u64 *key);
77extern void aes_sparc64_load_decrypt_keys_256(const u64 *key);
78
79extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input,
80 u64 *output, unsigned int len);
81extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input,
82 u64 *output, unsigned int len);
83extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input,
84 u64 *output, unsigned int len);
85
86extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input,
87 u64 *output, unsigned int len);
88extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input,
89 u64 *output, unsigned int len);
90extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input,
91 u64 *output, unsigned int len);
92
93extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input,
94 u64 *output, unsigned int len,
95 u64 *iv);
96
97extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input,
98 u64 *output, unsigned int len,
99 u64 *iv);
100
101extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input,
102 u64 *output, unsigned int len,
103 u64 *iv);
104
105extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input,
106 u64 *output, unsigned int len,
107 u64 *iv);
108
109extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input,
110 u64 *output, unsigned int len,
111 u64 *iv);
112
113extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input,
114 u64 *output, unsigned int len,
115 u64 *iv);
116
117extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input,
118 u64 *output, unsigned int len,
119 u64 *iv);
120extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input,
121 u64 *output, unsigned int len,
122 u64 *iv);
123extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input,
124 u64 *output, unsigned int len,
125 u64 *iv);
126
127struct aes_ops aes128_ops = {
128 .encrypt = aes_sparc64_encrypt_128,
129 .decrypt = aes_sparc64_decrypt_128,
130 .load_encrypt_keys = aes_sparc64_load_encrypt_keys_128,
131 .load_decrypt_keys = aes_sparc64_load_decrypt_keys_128,
132 .ecb_encrypt = aes_sparc64_ecb_encrypt_128,
133 .ecb_decrypt = aes_sparc64_ecb_decrypt_128,
134 .cbc_encrypt = aes_sparc64_cbc_encrypt_128,
135 .cbc_decrypt = aes_sparc64_cbc_decrypt_128,
136 .ctr_crypt = aes_sparc64_ctr_crypt_128,
137};
138
139struct aes_ops aes192_ops = {
140 .encrypt = aes_sparc64_encrypt_192,
141 .decrypt = aes_sparc64_decrypt_192,
142 .load_encrypt_keys = aes_sparc64_load_encrypt_keys_192,
143 .load_decrypt_keys = aes_sparc64_load_decrypt_keys_192,
144 .ecb_encrypt = aes_sparc64_ecb_encrypt_192,
145 .ecb_decrypt = aes_sparc64_ecb_decrypt_192,
146 .cbc_encrypt = aes_sparc64_cbc_encrypt_192,
147 .cbc_decrypt = aes_sparc64_cbc_decrypt_192,
148 .ctr_crypt = aes_sparc64_ctr_crypt_192,
149};
150
151struct aes_ops aes256_ops = {
152 .encrypt = aes_sparc64_encrypt_256,
153 .decrypt = aes_sparc64_decrypt_256,
154 .load_encrypt_keys = aes_sparc64_load_encrypt_keys_256,
155 .load_decrypt_keys = aes_sparc64_load_decrypt_keys_256,
156 .ecb_encrypt = aes_sparc64_ecb_encrypt_256,
157 .ecb_decrypt = aes_sparc64_ecb_decrypt_256,
158 .cbc_encrypt = aes_sparc64_cbc_encrypt_256,
159 .cbc_decrypt = aes_sparc64_cbc_decrypt_256,
160 .ctr_crypt = aes_sparc64_ctr_crypt_256,
161};
162
163extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
164 unsigned int key_len);
165
166static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
167 unsigned int key_len)
168{
169 struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
170 u32 *flags = &tfm->crt_flags;
171
172 switch (key_len) {
173 case AES_KEYSIZE_128:
174 ctx->expanded_key_length = 0xb0;
175 ctx->ops = &aes128_ops;
176 break;
177
178 case AES_KEYSIZE_192:
179 ctx->expanded_key_length = 0xd0;
180 ctx->ops = &aes192_ops;
181 break;
182
183 case AES_KEYSIZE_256:
184 ctx->expanded_key_length = 0xf0;
185 ctx->ops = &aes256_ops;
186 break;
187
188 default:
189 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
190 return -EINVAL;
191 }
192
193 aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len);
194 ctx->key_length = key_len;
195
196 return 0;
197}
198
199static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
200{
201 struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
202
203 ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
204}
205
206static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
207{
208 struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
209
210 ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
211}
212
213#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1))
214
215static int ecb_encrypt(struct blkcipher_desc *desc,
216 struct scatterlist *dst, struct scatterlist *src,
217 unsigned int nbytes)
218{
219 struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
220 struct blkcipher_walk walk;
221 int err;
222
223 blkcipher_walk_init(&walk, dst, src, nbytes);
224 err = blkcipher_walk_virt(desc, &walk);
225
226 ctx->ops->load_encrypt_keys(&ctx->key[0]);
227 while ((nbytes = walk.nbytes)) {
228 unsigned int block_len = nbytes & AES_BLOCK_MASK;
229
230 if (likely(block_len)) {
231 ctx->ops->ecb_encrypt(&ctx->key[0],
232 (const u64 *)walk.src.virt.addr,
233 (u64 *) walk.dst.virt.addr,
234 block_len);
235 }
236 nbytes &= AES_BLOCK_SIZE - 1;
237 err = blkcipher_walk_done(desc, &walk, nbytes);
238 }
239 fprs_write(0);
240 return err;
241}
242
243static int ecb_decrypt(struct blkcipher_desc *desc,
244 struct scatterlist *dst, struct scatterlist *src,
245 unsigned int nbytes)
246{
247 struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
248 struct blkcipher_walk walk;
249 u64 *key_end;
250 int err;
251
252 blkcipher_walk_init(&walk, dst, src, nbytes);
253 err = blkcipher_walk_virt(desc, &walk);
254
255 ctx->ops->load_decrypt_keys(&ctx->key[0]);
256 key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
257 while ((nbytes = walk.nbytes)) {
258 unsigned int block_len = nbytes & AES_BLOCK_MASK;
259
260 if (likely(block_len)) {
261 ctx->ops->ecb_decrypt(key_end,
262 (const u64 *) walk.src.virt.addr,
263 (u64 *) walk.dst.virt.addr, block_len);
264 }
265 nbytes &= AES_BLOCK_SIZE - 1;
266 err = blkcipher_walk_done(desc, &walk, nbytes);
267 }
268 fprs_write(0);
269
270 return err;
271}
272
273static int cbc_encrypt(struct blkcipher_desc *desc,
274 struct scatterlist *dst, struct scatterlist *src,
275 unsigned int nbytes)
276{
277 struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
278 struct blkcipher_walk walk;
279 int err;
280
281 blkcipher_walk_init(&walk, dst, src, nbytes);
282 err = blkcipher_walk_virt(desc, &walk);
283
284 ctx->ops->load_encrypt_keys(&ctx->key[0]);
285 while ((nbytes = walk.nbytes)) {
286 unsigned int block_len = nbytes & AES_BLOCK_MASK;
287
288 if (likely(block_len)) {
289 ctx->ops->cbc_encrypt(&ctx->key[0],
290 (const u64 *)walk.src.virt.addr,
291 (u64 *) walk.dst.virt.addr,
292 block_len, (u64 *) walk.iv);
293 }
294 nbytes &= AES_BLOCK_SIZE - 1;
295 err = blkcipher_walk_done(desc, &walk, nbytes);
296 }
297 fprs_write(0);
298 return err;
299}
300
301static int cbc_decrypt(struct blkcipher_desc *desc,
302 struct scatterlist *dst, struct scatterlist *src,
303 unsigned int nbytes)
304{
305 struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
306 struct blkcipher_walk walk;
307 u64 *key_end;
308 int err;
309
310 blkcipher_walk_init(&walk, dst, src, nbytes);
311 err = blkcipher_walk_virt(desc, &walk);
312
313 ctx->ops->load_decrypt_keys(&ctx->key[0]);
314 key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
315 while ((nbytes = walk.nbytes)) {
316 unsigned int block_len = nbytes & AES_BLOCK_MASK;
317
318 if (likely(block_len)) {
319 ctx->ops->cbc_decrypt(key_end,
320 (const u64 *) walk.src.virt.addr,
321 (u64 *) walk.dst.virt.addr,
322 block_len, (u64 *) walk.iv);
323 }
324 nbytes &= AES_BLOCK_SIZE - 1;
325 err = blkcipher_walk_done(desc, &walk, nbytes);
326 }
327 fprs_write(0);
328
329 return err;
330}
331
332static int ctr_crypt(struct blkcipher_desc *desc,
333 struct scatterlist *dst, struct scatterlist *src,
334 unsigned int nbytes)
335{
336 struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
337 struct blkcipher_walk walk;
338 int err;
339
340 blkcipher_walk_init(&walk, dst, src, nbytes);
341 err = blkcipher_walk_virt(desc, &walk);
342
343 ctx->ops->load_encrypt_keys(&ctx->key[0]);
344 while ((nbytes = walk.nbytes)) {
345 unsigned int block_len = nbytes & AES_BLOCK_MASK;
346
347 if (likely(block_len)) {
348 ctx->ops->ctr_crypt(&ctx->key[0],
349 (const u64 *)walk.src.virt.addr,
350 (u64 *) walk.dst.virt.addr,
351 block_len, (u64 *) walk.iv);
352 }
353 nbytes &= AES_BLOCK_SIZE - 1;
354 err = blkcipher_walk_done(desc, &walk, nbytes);
355 }
356 fprs_write(0);
357 return err;
358}
359
360static struct crypto_alg algs[] = { {
361 .cra_name = "aes",
362 .cra_driver_name = "aes-sparc64",
363 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
364 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
365 .cra_blocksize = AES_BLOCK_SIZE,
366 .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
367 .cra_alignmask = 3,
368 .cra_module = THIS_MODULE,
369 .cra_u = {
370 .cipher = {
371 .cia_min_keysize = AES_MIN_KEY_SIZE,
372 .cia_max_keysize = AES_MAX_KEY_SIZE,
373 .cia_setkey = aes_set_key,
374 .cia_encrypt = aes_encrypt,
375 .cia_decrypt = aes_decrypt
376 }
377 }
378}, {
379 .cra_name = "ecb(aes)",
380 .cra_driver_name = "ecb-aes-sparc64",
381 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
382 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
383 .cra_blocksize = AES_BLOCK_SIZE,
384 .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
385 .cra_alignmask = 7,
386 .cra_type = &crypto_blkcipher_type,
387 .cra_module = THIS_MODULE,
388 .cra_u = {
389 .blkcipher = {
390 .min_keysize = AES_MIN_KEY_SIZE,
391 .max_keysize = AES_MAX_KEY_SIZE,
392 .setkey = aes_set_key,
393 .encrypt = ecb_encrypt,
394 .decrypt = ecb_decrypt,
395 },
396 },
397}, {
398 .cra_name = "cbc(aes)",
399 .cra_driver_name = "cbc-aes-sparc64",
400 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
401 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
402 .cra_blocksize = AES_BLOCK_SIZE,
403 .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
404 .cra_alignmask = 7,
405 .cra_type = &crypto_blkcipher_type,
406 .cra_module = THIS_MODULE,
407 .cra_u = {
408 .blkcipher = {
409 .min_keysize = AES_MIN_KEY_SIZE,
410 .max_keysize = AES_MAX_KEY_SIZE,
411 .setkey = aes_set_key,
412 .encrypt = cbc_encrypt,
413 .decrypt = cbc_decrypt,
414 },
415 },
416}, {
417 .cra_name = "ctr(aes)",
418 .cra_driver_name = "ctr-aes-sparc64",
419 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
420 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
421 .cra_blocksize = AES_BLOCK_SIZE,
422 .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
423 .cra_alignmask = 7,
424 .cra_type = &crypto_blkcipher_type,
425 .cra_module = THIS_MODULE,
426 .cra_u = {
427 .blkcipher = {
428 .min_keysize = AES_MIN_KEY_SIZE,
429 .max_keysize = AES_MAX_KEY_SIZE,
430 .setkey = aes_set_key,
431 .encrypt = ctr_crypt,
432 .decrypt = ctr_crypt,
433 },
434 },
435} };
436
437static bool __init sparc64_has_aes_opcode(void)
438{
439 unsigned long cfr;
440
441 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
442 return false;
443
444 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
445 if (!(cfr & CFR_AES))
446 return false;
447
448 return true;
449}
450
451static int __init aes_sparc64_mod_init(void)
452{
453 int i;
454
455 for (i = 0; i < ARRAY_SIZE(algs); i++)
456 INIT_LIST_HEAD(&algs[i].cra_list);
457
458 if (sparc64_has_aes_opcode()) {
459 pr_info("Using sparc64 aes opcodes optimized AES implementation\n");
460 return crypto_register_algs(algs, ARRAY_SIZE(algs));
461 }
462 pr_info("sparc64 aes opcodes not available.\n");
463 return -ENODEV;
464}
465
466static void __exit aes_sparc64_mod_fini(void)
467{
468 crypto_unregister_algs(algs, ARRAY_SIZE(algs));
469}
470
471module_init(aes_sparc64_mod_init);
472module_exit(aes_sparc64_mod_fini);
473
474MODULE_LICENSE("GPL");
475MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated");
476
477MODULE_ALIAS("aes");
diff --git a/arch/sparc/crypto/camellia_asm.S b/arch/sparc/crypto/camellia_asm.S
new file mode 100644
index 000000000000..cc39553a4e43
--- /dev/null
+++ b/arch/sparc/crypto/camellia_asm.S
@@ -0,0 +1,563 @@
1#include <linux/linkage.h>
2#include <asm/visasm.h>
3
4#include "opcodes.h"
5
6#define CAMELLIA_6ROUNDS(KEY_BASE, I0, I1) \
7 CAMELLIA_F(KEY_BASE + 0, I1, I0, I1) \
8 CAMELLIA_F(KEY_BASE + 2, I0, I1, I0) \
9 CAMELLIA_F(KEY_BASE + 4, I1, I0, I1) \
10 CAMELLIA_F(KEY_BASE + 6, I0, I1, I0) \
11 CAMELLIA_F(KEY_BASE + 8, I1, I0, I1) \
12 CAMELLIA_F(KEY_BASE + 10, I0, I1, I0)
13
14#define CAMELLIA_6ROUNDS_FL_FLI(KEY_BASE, I0, I1) \
15 CAMELLIA_6ROUNDS(KEY_BASE, I0, I1) \
16 CAMELLIA_FL(KEY_BASE + 12, I0, I0) \
17 CAMELLIA_FLI(KEY_BASE + 14, I1, I1)
18
19 .data
20
21 .align 8
22SIGMA: .xword 0xA09E667F3BCC908B
23 .xword 0xB67AE8584CAA73B2
24 .xword 0xC6EF372FE94F82BE
25 .xword 0x54FF53A5F1D36F1C
26 .xword 0x10E527FADE682D1D
27 .xword 0xB05688C2B3E6C1FD
28
29 .text
30
31 .align 32
32ENTRY(camellia_sparc64_key_expand)
33 /* %o0=in_key, %o1=encrypt_key, %o2=key_len, %o3=decrypt_key */
34 VISEntry
35 ld [%o0 + 0x00], %f0 ! i0, k[0]
36 ld [%o0 + 0x04], %f1 ! i1, k[1]
37 ld [%o0 + 0x08], %f2 ! i2, k[2]
38 ld [%o0 + 0x0c], %f3 ! i3, k[3]
39 std %f0, [%o1 + 0x00] ! k[0, 1]
40 fsrc2 %f0, %f28
41 std %f2, [%o1 + 0x08] ! k[2, 3]
42 cmp %o2, 16
43 be 10f
44 fsrc2 %f2, %f30
45
46 ld [%o0 + 0x10], %f0
47 ld [%o0 + 0x14], %f1
48 std %f0, [%o1 + 0x20] ! k[8, 9]
49 cmp %o2, 24
50 fone %f10
51 be,a 1f
52 fxor %f10, %f0, %f2
53 ld [%o0 + 0x18], %f2
54 ld [%o0 + 0x1c], %f3
551:
56 std %f2, [%o1 + 0x28] ! k[10, 11]
57 fxor %f28, %f0, %f0
58 fxor %f30, %f2, %f2
59
6010:
61 sethi %hi(SIGMA), %g3
62 or %g3, %lo(SIGMA), %g3
63 ldd [%g3 + 0x00], %f16
64 ldd [%g3 + 0x08], %f18
65 ldd [%g3 + 0x10], %f20
66 ldd [%g3 + 0x18], %f22
67 ldd [%g3 + 0x20], %f24
68 ldd [%g3 + 0x28], %f26
69 CAMELLIA_F(16, 2, 0, 2)
70 CAMELLIA_F(18, 0, 2, 0)
71 fxor %f28, %f0, %f0
72 fxor %f30, %f2, %f2
73 CAMELLIA_F(20, 2, 0, 2)
74 CAMELLIA_F(22, 0, 2, 0)
75
76#define ROTL128(S01, S23, TMP1, TMP2, N) \
77 srlx S01, (64 - N), TMP1; \
78 sllx S01, N, S01; \
79 srlx S23, (64 - N), TMP2; \
80 sllx S23, N, S23; \
81 or S01, TMP2, S01; \
82 or S23, TMP1, S23
83
84 cmp %o2, 16
85 bne 1f
86 nop
87 /* 128-bit key */
88 std %f0, [%o1 + 0x10] ! k[ 4, 5]
89 std %f2, [%o1 + 0x18] ! k[ 6, 7]
90 MOVDTOX_F0_O4
91 MOVDTOX_F2_O5
92 ROTL128(%o4, %o5, %g2, %g3, 15)
93 stx %o4, [%o1 + 0x30] ! k[12, 13]
94 stx %o5, [%o1 + 0x38] ! k[14, 15]
95 ROTL128(%o4, %o5, %g2, %g3, 15)
96 stx %o4, [%o1 + 0x40] ! k[16, 17]
97 stx %o5, [%o1 + 0x48] ! k[18, 19]
98 ROTL128(%o4, %o5, %g2, %g3, 15)
99 stx %o4, [%o1 + 0x60] ! k[24, 25]
100 ROTL128(%o4, %o5, %g2, %g3, 15)
101 stx %o4, [%o1 + 0x70] ! k[28, 29]
102 stx %o5, [%o1 + 0x78] ! k[30, 31]
103 ROTL128(%o4, %o5, %g2, %g3, 34)
104 stx %o4, [%o1 + 0xa0] ! k[40, 41]
105 stx %o5, [%o1 + 0xa8] ! k[42, 43]
106 ROTL128(%o4, %o5, %g2, %g3, 17)
107 stx %o4, [%o1 + 0xc0] ! k[48, 49]
108 stx %o5, [%o1 + 0xc8] ! k[50, 51]
109
110 ldx [%o1 + 0x00], %o4 ! k[ 0, 1]
111 ldx [%o1 + 0x08], %o5 ! k[ 2, 3]
112 ROTL128(%o4, %o5, %g2, %g3, 15)
113 stx %o4, [%o1 + 0x20] ! k[ 8, 9]
114 stx %o5, [%o1 + 0x28] ! k[10, 11]
115 ROTL128(%o4, %o5, %g2, %g3, 30)
116 stx %o4, [%o1 + 0x50] ! k[20, 21]
117 stx %o5, [%o1 + 0x58] ! k[22, 23]
118 ROTL128(%o4, %o5, %g2, %g3, 15)
119 stx %o5, [%o1 + 0x68] ! k[26, 27]
120 ROTL128(%o4, %o5, %g2, %g3, 17)
121 stx %o4, [%o1 + 0x80] ! k[32, 33]
122 stx %o5, [%o1 + 0x88] ! k[34, 35]
123 ROTL128(%o4, %o5, %g2, %g3, 17)
124 stx %o4, [%o1 + 0x90] ! k[36, 37]
125 stx %o5, [%o1 + 0x98] ! k[38, 39]
126 ROTL128(%o4, %o5, %g2, %g3, 17)
127 stx %o4, [%o1 + 0xb0] ! k[44, 45]
128 stx %o5, [%o1 + 0xb8] ! k[46, 47]
129
130 ba,pt %xcc, 2f
131 mov (3 * 16 * 4), %o0
132
1331:
134 /* 192-bit or 256-bit key */
135 std %f0, [%o1 + 0x30] ! k[12, 13]
136 std %f2, [%o1 + 0x38] ! k[14, 15]
137 ldd [%o1 + 0x20], %f4 ! k[ 8, 9]
138 ldd [%o1 + 0x28], %f6 ! k[10, 11]
139 fxor %f0, %f4, %f0
140 fxor %f2, %f6, %f2
141 CAMELLIA_F(24, 2, 0, 2)
142 CAMELLIA_F(26, 0, 2, 0)
143 std %f0, [%o1 + 0x10] ! k[ 4, 5]
144 std %f2, [%o1 + 0x18] ! k[ 6, 7]
145 MOVDTOX_F0_O4
146 MOVDTOX_F2_O5
147 ROTL128(%o4, %o5, %g2, %g3, 30)
148 stx %o4, [%o1 + 0x50] ! k[20, 21]
149 stx %o5, [%o1 + 0x58] ! k[22, 23]
150 ROTL128(%o4, %o5, %g2, %g3, 30)
151 stx %o4, [%o1 + 0xa0] ! k[40, 41]
152 stx %o5, [%o1 + 0xa8] ! k[42, 43]
153 ROTL128(%o4, %o5, %g2, %g3, 51)
154 stx %o4, [%o1 + 0x100] ! k[64, 65]
155 stx %o5, [%o1 + 0x108] ! k[66, 67]
156 ldx [%o1 + 0x20], %o4 ! k[ 8, 9]
157 ldx [%o1 + 0x28], %o5 ! k[10, 11]
158 ROTL128(%o4, %o5, %g2, %g3, 15)
159 stx %o4, [%o1 + 0x20] ! k[ 8, 9]
160 stx %o5, [%o1 + 0x28] ! k[10, 11]
161 ROTL128(%o4, %o5, %g2, %g3, 15)
162 stx %o4, [%o1 + 0x40] ! k[16, 17]
163 stx %o5, [%o1 + 0x48] ! k[18, 19]
164 ROTL128(%o4, %o5, %g2, %g3, 30)
165 stx %o4, [%o1 + 0x90] ! k[36, 37]
166 stx %o5, [%o1 + 0x98] ! k[38, 39]
167 ROTL128(%o4, %o5, %g2, %g3, 34)
168 stx %o4, [%o1 + 0xd0] ! k[52, 53]
169 stx %o5, [%o1 + 0xd8] ! k[54, 55]
170 ldx [%o1 + 0x30], %o4 ! k[12, 13]
171 ldx [%o1 + 0x38], %o5 ! k[14, 15]
172 ROTL128(%o4, %o5, %g2, %g3, 15)
173 stx %o4, [%o1 + 0x30] ! k[12, 13]
174 stx %o5, [%o1 + 0x38] ! k[14, 15]
175 ROTL128(%o4, %o5, %g2, %g3, 30)
176 stx %o4, [%o1 + 0x70] ! k[28, 29]
177 stx %o5, [%o1 + 0x78] ! k[30, 31]
178 srlx %o4, 32, %g2
179 srlx %o5, 32, %g3
180 stw %o4, [%o1 + 0xc0] ! k[48]
181 stw %g3, [%o1 + 0xc4] ! k[49]
182 stw %o5, [%o1 + 0xc8] ! k[50]
183 stw %g2, [%o1 + 0xcc] ! k[51]
184 ROTL128(%o4, %o5, %g2, %g3, 49)
185 stx %o4, [%o1 + 0xe0] ! k[56, 57]
186 stx %o5, [%o1 + 0xe8] ! k[58, 59]
187 ldx [%o1 + 0x00], %o4 ! k[ 0, 1]
188 ldx [%o1 + 0x08], %o5 ! k[ 2, 3]
189 ROTL128(%o4, %o5, %g2, %g3, 45)
190 stx %o4, [%o1 + 0x60] ! k[24, 25]
191 stx %o5, [%o1 + 0x68] ! k[26, 27]
192 ROTL128(%o4, %o5, %g2, %g3, 15)
193 stx %o4, [%o1 + 0x80] ! k[32, 33]
194 stx %o5, [%o1 + 0x88] ! k[34, 35]
195 ROTL128(%o4, %o5, %g2, %g3, 17)
196 stx %o4, [%o1 + 0xb0] ! k[44, 45]
197 stx %o5, [%o1 + 0xb8] ! k[46, 47]
198 ROTL128(%o4, %o5, %g2, %g3, 34)
199 stx %o4, [%o1 + 0xf0] ! k[60, 61]
200 stx %o5, [%o1 + 0xf8] ! k[62, 63]
201 mov (4 * 16 * 4), %o0
2022:
203 add %o1, %o0, %o1
204 ldd [%o1 + 0x00], %f0
205 ldd [%o1 + 0x08], %f2
206 std %f0, [%o3 + 0x00]
207 std %f2, [%o3 + 0x08]
208 add %o3, 0x10, %o3
2091:
210 sub %o1, (16 * 4), %o1
211 ldd [%o1 + 0x38], %f0
212 ldd [%o1 + 0x30], %f2
213 ldd [%o1 + 0x28], %f4
214 ldd [%o1 + 0x20], %f6
215 ldd [%o1 + 0x18], %f8
216 ldd [%o1 + 0x10], %f10
217 std %f0, [%o3 + 0x00]
218 std %f2, [%o3 + 0x08]
219 std %f4, [%o3 + 0x10]
220 std %f6, [%o3 + 0x18]
221 std %f8, [%o3 + 0x20]
222 std %f10, [%o3 + 0x28]
223
224 ldd [%o1 + 0x08], %f0
225 ldd [%o1 + 0x00], %f2
226 std %f0, [%o3 + 0x30]
227 std %f2, [%o3 + 0x38]
228 subcc %o0, (16 * 4), %o0
229 bne,pt %icc, 1b
230 add %o3, (16 * 4), %o3
231
232 std %f2, [%o3 - 0x10]
233 std %f0, [%o3 - 0x08]
234
235 retl
236 VISExit
237ENDPROC(camellia_sparc64_key_expand)
238
239 .align 32
240ENTRY(camellia_sparc64_crypt)
241 /* %o0=key, %o1=input, %o2=output, %o3=key_len */
242 VISEntry
243
244 ld [%o1 + 0x00], %f0
245 ld [%o1 + 0x04], %f1
246 ld [%o1 + 0x08], %f2
247 ld [%o1 + 0x0c], %f3
248
249 ldd [%o0 + 0x00], %f4
250 ldd [%o0 + 0x08], %f6
251
252 cmp %o3, 16
253 fxor %f4, %f0, %f0
254 be 1f
255 fxor %f6, %f2, %f2
256
257 ldd [%o0 + 0x10], %f8
258 ldd [%o0 + 0x18], %f10
259 ldd [%o0 + 0x20], %f12
260 ldd [%o0 + 0x28], %f14
261 ldd [%o0 + 0x30], %f16
262 ldd [%o0 + 0x38], %f18
263 ldd [%o0 + 0x40], %f20
264 ldd [%o0 + 0x48], %f22
265 add %o0, 0x40, %o0
266
267 CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
268
2691:
270 ldd [%o0 + 0x10], %f8
271 ldd [%o0 + 0x18], %f10
272 ldd [%o0 + 0x20], %f12
273 ldd [%o0 + 0x28], %f14
274 ldd [%o0 + 0x30], %f16
275 ldd [%o0 + 0x38], %f18
276 ldd [%o0 + 0x40], %f20
277 ldd [%o0 + 0x48], %f22
278 ldd [%o0 + 0x50], %f24
279 ldd [%o0 + 0x58], %f26
280 ldd [%o0 + 0x60], %f28
281 ldd [%o0 + 0x68], %f30
282 ldd [%o0 + 0x70], %f32
283 ldd [%o0 + 0x78], %f34
284 ldd [%o0 + 0x80], %f36
285 ldd [%o0 + 0x88], %f38
286 ldd [%o0 + 0x90], %f40
287 ldd [%o0 + 0x98], %f42
288 ldd [%o0 + 0xa0], %f44
289 ldd [%o0 + 0xa8], %f46
290 ldd [%o0 + 0xb0], %f48
291 ldd [%o0 + 0xb8], %f50
292 ldd [%o0 + 0xc0], %f52
293 ldd [%o0 + 0xc8], %f54
294
295 CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
296 CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
297 CAMELLIA_6ROUNDS(40, 0, 2)
298 fxor %f52, %f2, %f2
299 fxor %f54, %f0, %f0
300
301 st %f2, [%o2 + 0x00]
302 st %f3, [%o2 + 0x04]
303 st %f0, [%o2 + 0x08]
304 st %f1, [%o2 + 0x0c]
305
306 retl
307 VISExit
308ENDPROC(camellia_sparc64_crypt)
309
310 .align 32
311ENTRY(camellia_sparc64_load_keys)
312 /* %o0=key, %o1=key_len */
313 VISEntry
314 ldd [%o0 + 0x00], %f4
315 ldd [%o0 + 0x08], %f6
316 ldd [%o0 + 0x10], %f8
317 ldd [%o0 + 0x18], %f10
318 ldd [%o0 + 0x20], %f12
319 ldd [%o0 + 0x28], %f14
320 ldd [%o0 + 0x30], %f16
321 ldd [%o0 + 0x38], %f18
322 ldd [%o0 + 0x40], %f20
323 ldd [%o0 + 0x48], %f22
324 ldd [%o0 + 0x50], %f24
325 ldd [%o0 + 0x58], %f26
326 ldd [%o0 + 0x60], %f28
327 ldd [%o0 + 0x68], %f30
328 ldd [%o0 + 0x70], %f32
329 ldd [%o0 + 0x78], %f34
330 ldd [%o0 + 0x80], %f36
331 ldd [%o0 + 0x88], %f38
332 ldd [%o0 + 0x90], %f40
333 ldd [%o0 + 0x98], %f42
334 ldd [%o0 + 0xa0], %f44
335 ldd [%o0 + 0xa8], %f46
336 ldd [%o0 + 0xb0], %f48
337 ldd [%o0 + 0xb8], %f50
338 ldd [%o0 + 0xc0], %f52
339 retl
340 ldd [%o0 + 0xc8], %f54
341ENDPROC(camellia_sparc64_load_keys)
342
343 .align 32
344ENTRY(camellia_sparc64_ecb_crypt_3_grand_rounds)
345 /* %o0=input, %o1=output, %o2=len, %o3=key */
3461: ldd [%o0 + 0x00], %f0
347 ldd [%o0 + 0x08], %f2
348 add %o0, 0x10, %o0
349 fxor %f4, %f0, %f0
350 fxor %f6, %f2, %f2
351 CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
352 CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
353 CAMELLIA_6ROUNDS(40, 0, 2)
354 fxor %f52, %f2, %f2
355 fxor %f54, %f0, %f0
356 std %f2, [%o1 + 0x00]
357 std %f0, [%o1 + 0x08]
358 subcc %o2, 0x10, %o2
359 bne,pt %icc, 1b
360 add %o1, 0x10, %o1
361 retl
362 nop
363ENDPROC(camellia_sparc64_ecb_crypt_3_grand_rounds)
364
365 .align 32
366ENTRY(camellia_sparc64_ecb_crypt_4_grand_rounds)
367 /* %o0=input, %o1=output, %o2=len, %o3=key */
3681: ldd [%o0 + 0x00], %f0
369 ldd [%o0 + 0x08], %f2
370 add %o0, 0x10, %o0
371 fxor %f4, %f0, %f0
372 fxor %f6, %f2, %f2
373 CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
374 ldd [%o3 + 0xd0], %f8
375 ldd [%o3 + 0xd8], %f10
376 ldd [%o3 + 0xe0], %f12
377 ldd [%o3 + 0xe8], %f14
378 ldd [%o3 + 0xf0], %f16
379 ldd [%o3 + 0xf8], %f18
380 ldd [%o3 + 0x100], %f20
381 ldd [%o3 + 0x108], %f22
382 CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
383 CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2)
384 CAMELLIA_F(8, 2, 0, 2)
385 CAMELLIA_F(10, 0, 2, 0)
386 ldd [%o3 + 0x10], %f8
387 ldd [%o3 + 0x18], %f10
388 CAMELLIA_F(12, 2, 0, 2)
389 CAMELLIA_F(14, 0, 2, 0)
390 ldd [%o3 + 0x20], %f12
391 ldd [%o3 + 0x28], %f14
392 CAMELLIA_F(16, 2, 0, 2)
393 CAMELLIA_F(18, 0, 2, 0)
394 ldd [%o3 + 0x30], %f16
395 ldd [%o3 + 0x38], %f18
396 fxor %f20, %f2, %f2
397 fxor %f22, %f0, %f0
398 ldd [%o3 + 0x40], %f20
399 ldd [%o3 + 0x48], %f22
400 std %f2, [%o1 + 0x00]
401 std %f0, [%o1 + 0x08]
402 subcc %o2, 0x10, %o2
403 bne,pt %icc, 1b
404 add %o1, 0x10, %o1
405 retl
406 nop
407ENDPROC(camellia_sparc64_ecb_crypt_4_grand_rounds)
408
409 .align 32
410ENTRY(camellia_sparc64_cbc_encrypt_3_grand_rounds)
411 /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */
412 ldd [%o4 + 0x00], %f60
413 ldd [%o4 + 0x08], %f62
4141: ldd [%o0 + 0x00], %f0
415 ldd [%o0 + 0x08], %f2
416 add %o0, 0x10, %o0
417 fxor %f60, %f0, %f0
418 fxor %f62, %f2, %f2
419 fxor %f4, %f0, %f0
420 fxor %f6, %f2, %f2
421 CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
422 CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
423 CAMELLIA_6ROUNDS(40, 0, 2)
424 fxor %f52, %f2, %f60
425 fxor %f54, %f0, %f62
426 std %f60, [%o1 + 0x00]
427 std %f62, [%o1 + 0x08]
428 subcc %o2, 0x10, %o2
429 bne,pt %icc, 1b
430 add %o1, 0x10, %o1
431 std %f60, [%o4 + 0x00]
432 retl
433 std %f62, [%o4 + 0x08]
434ENDPROC(camellia_sparc64_cbc_encrypt_3_grand_rounds)
435
436 .align 32
437ENTRY(camellia_sparc64_cbc_encrypt_4_grand_rounds)
438 /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */
439 ldd [%o4 + 0x00], %f60
440 ldd [%o4 + 0x08], %f62
4411: ldd [%o0 + 0x00], %f0
442 ldd [%o0 + 0x08], %f2
443 add %o0, 0x10, %o0
444 fxor %f60, %f0, %f0
445 fxor %f62, %f2, %f2
446 fxor %f4, %f0, %f0
447 fxor %f6, %f2, %f2
448 CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
449 ldd [%o3 + 0xd0], %f8
450 ldd [%o3 + 0xd8], %f10
451 ldd [%o3 + 0xe0], %f12
452 ldd [%o3 + 0xe8], %f14
453 ldd [%o3 + 0xf0], %f16
454 ldd [%o3 + 0xf8], %f18
455 ldd [%o3 + 0x100], %f20
456 ldd [%o3 + 0x108], %f22
457 CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
458 CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2)
459 CAMELLIA_F(8, 2, 0, 2)
460 CAMELLIA_F(10, 0, 2, 0)
461 ldd [%o3 + 0x10], %f8
462 ldd [%o3 + 0x18], %f10
463 CAMELLIA_F(12, 2, 0, 2)
464 CAMELLIA_F(14, 0, 2, 0)
465 ldd [%o3 + 0x20], %f12
466 ldd [%o3 + 0x28], %f14
467 CAMELLIA_F(16, 2, 0, 2)
468 CAMELLIA_F(18, 0, 2, 0)
469 ldd [%o3 + 0x30], %f16
470 ldd [%o3 + 0x38], %f18
471 fxor %f20, %f2, %f60
472 fxor %f22, %f0, %f62
473 ldd [%o3 + 0x40], %f20
474 ldd [%o3 + 0x48], %f22
475 std %f60, [%o1 + 0x00]
476 std %f62, [%o1 + 0x08]
477 subcc %o2, 0x10, %o2
478 bne,pt %icc, 1b
479 add %o1, 0x10, %o1
480 std %f60, [%o4 + 0x00]
481 retl
482 std %f62, [%o4 + 0x08]
483ENDPROC(camellia_sparc64_cbc_encrypt_4_grand_rounds)
484
485 .align 32
486ENTRY(camellia_sparc64_cbc_decrypt_3_grand_rounds)
487 /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */
488 ldd [%o4 + 0x00], %f60
489 ldd [%o4 + 0x08], %f62
4901: ldd [%o0 + 0x00], %f56
491 ldd [%o0 + 0x08], %f58
492 add %o0, 0x10, %o0
493 fxor %f4, %f56, %f0
494 fxor %f6, %f58, %f2
495 CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
496 CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
497 CAMELLIA_6ROUNDS(40, 0, 2)
498 fxor %f52, %f2, %f2
499 fxor %f54, %f0, %f0
500 fxor %f60, %f2, %f2
501 fxor %f62, %f0, %f0
502 fsrc2 %f56, %f60
503 fsrc2 %f58, %f62
504 std %f2, [%o1 + 0x00]
505 std %f0, [%o1 + 0x08]
506 subcc %o2, 0x10, %o2
507 bne,pt %icc, 1b
508 add %o1, 0x10, %o1
509 std %f60, [%o4 + 0x00]
510 retl
511 std %f62, [%o4 + 0x08]
512ENDPROC(camellia_sparc64_cbc_decrypt_3_grand_rounds)
513
514 .align 32
515ENTRY(camellia_sparc64_cbc_decrypt_4_grand_rounds)
516 /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */
517 ldd [%o4 + 0x00], %f60
518 ldd [%o4 + 0x08], %f62
5191: ldd [%o0 + 0x00], %f56
520 ldd [%o0 + 0x08], %f58
521 add %o0, 0x10, %o0
522 fxor %f4, %f56, %f0
523 fxor %f6, %f58, %f2
524 CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
525 ldd [%o3 + 0xd0], %f8
526 ldd [%o3 + 0xd8], %f10
527 ldd [%o3 + 0xe0], %f12
528 ldd [%o3 + 0xe8], %f14
529 ldd [%o3 + 0xf0], %f16
530 ldd [%o3 + 0xf8], %f18
531 ldd [%o3 + 0x100], %f20
532 ldd [%o3 + 0x108], %f22
533 CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
534 CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2)
535 CAMELLIA_F(8, 2, 0, 2)
536 CAMELLIA_F(10, 0, 2, 0)
537 ldd [%o3 + 0x10], %f8
538 ldd [%o3 + 0x18], %f10
539 CAMELLIA_F(12, 2, 0, 2)
540 CAMELLIA_F(14, 0, 2, 0)
541 ldd [%o3 + 0x20], %f12
542 ldd [%o3 + 0x28], %f14
543 CAMELLIA_F(16, 2, 0, 2)
544 CAMELLIA_F(18, 0, 2, 0)
545 ldd [%o3 + 0x30], %f16
546 ldd [%o3 + 0x38], %f18
547 fxor %f20, %f2, %f2
548 fxor %f22, %f0, %f0
549 ldd [%o3 + 0x40], %f20
550 ldd [%o3 + 0x48], %f22
551 fxor %f60, %f2, %f2
552 fxor %f62, %f0, %f0
553 fsrc2 %f56, %f60
554 fsrc2 %f58, %f62
555 std %f2, [%o1 + 0x00]
556 std %f0, [%o1 + 0x08]
557 subcc %o2, 0x10, %o2
558 bne,pt %icc, 1b
559 add %o1, 0x10, %o1
560 std %f60, [%o4 + 0x00]
561 retl
562 std %f62, [%o4 + 0x08]
563ENDPROC(camellia_sparc64_cbc_decrypt_4_grand_rounds)
diff --git a/arch/sparc/crypto/camellia_glue.c b/arch/sparc/crypto/camellia_glue.c
new file mode 100644
index 000000000000..42905c084299
--- /dev/null
+++ b/arch/sparc/crypto/camellia_glue.c
@@ -0,0 +1,322 @@
1/* Glue code for CAMELLIA encryption optimized for sparc64 crypto opcodes.
2 *
3 * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
4 */
5
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/crypto.h>
9#include <linux/init.h>
10#include <linux/module.h>
11#include <linux/mm.h>
12#include <linux/types.h>
13#include <crypto/algapi.h>
14
15#include <asm/fpumacro.h>
16#include <asm/pstate.h>
17#include <asm/elf.h>
18
19#include "opcodes.h"
20
21#define CAMELLIA_MIN_KEY_SIZE 16
22#define CAMELLIA_MAX_KEY_SIZE 32
23#define CAMELLIA_BLOCK_SIZE 16
24#define CAMELLIA_TABLE_BYTE_LEN 272
25
26struct camellia_sparc64_ctx {
27 u64 encrypt_key[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)];
28 u64 decrypt_key[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)];
29 int key_len;
30};
31
32extern void camellia_sparc64_key_expand(const u32 *in_key, u64 *encrypt_key,
33 unsigned int key_len, u64 *decrypt_key);
34
35static int camellia_set_key(struct crypto_tfm *tfm, const u8 *_in_key,
36 unsigned int key_len)
37{
38 struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
39 const u32 *in_key = (const u32 *) _in_key;
40 u32 *flags = &tfm->crt_flags;
41
42 if (key_len != 16 && key_len != 24 && key_len != 32) {
43 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
44 return -EINVAL;
45 }
46
47 ctx->key_len = key_len;
48
49 camellia_sparc64_key_expand(in_key, &ctx->encrypt_key[0],
50 key_len, &ctx->decrypt_key[0]);
51 return 0;
52}
53
54extern void camellia_sparc64_crypt(const u64 *key, const u32 *input,
55 u32 *output, unsigned int key_len);
56
57static void camellia_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
58{
59 struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
60
61 camellia_sparc64_crypt(&ctx->encrypt_key[0],
62 (const u32 *) src,
63 (u32 *) dst, ctx->key_len);
64}
65
66static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
67{
68 struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
69
70 camellia_sparc64_crypt(&ctx->decrypt_key[0],
71 (const u32 *) src,
72 (u32 *) dst, ctx->key_len);
73}
74
75extern void camellia_sparc64_load_keys(const u64 *key, unsigned int key_len);
76
77typedef void ecb_crypt_op(const u64 *input, u64 *output, unsigned int len,
78 const u64 *key);
79
80extern ecb_crypt_op camellia_sparc64_ecb_crypt_3_grand_rounds;
81extern ecb_crypt_op camellia_sparc64_ecb_crypt_4_grand_rounds;
82
83#define CAMELLIA_BLOCK_MASK (~(CAMELLIA_BLOCK_SIZE - 1))
84
85static int __ecb_crypt(struct blkcipher_desc *desc,
86 struct scatterlist *dst, struct scatterlist *src,
87 unsigned int nbytes, bool encrypt)
88{
89 struct camellia_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
90 struct blkcipher_walk walk;
91 ecb_crypt_op *op;
92 const u64 *key;
93 int err;
94
95 op = camellia_sparc64_ecb_crypt_3_grand_rounds;
96 if (ctx->key_len != 16)
97 op = camellia_sparc64_ecb_crypt_4_grand_rounds;
98
99 blkcipher_walk_init(&walk, dst, src, nbytes);
100 err = blkcipher_walk_virt(desc, &walk);
101
102 if (encrypt)
103 key = &ctx->encrypt_key[0];
104 else
105 key = &ctx->decrypt_key[0];
106 camellia_sparc64_load_keys(key, ctx->key_len);
107 while ((nbytes = walk.nbytes)) {
108 unsigned int block_len = nbytes & CAMELLIA_BLOCK_MASK;
109
110 if (likely(block_len)) {
111 const u64 *src64;
112 u64 *dst64;
113
114 src64 = (const u64 *)walk.src.virt.addr;
115 dst64 = (u64 *) walk.dst.virt.addr;
116 op(src64, dst64, block_len, key);
117 }
118 nbytes &= CAMELLIA_BLOCK_SIZE - 1;
119 err = blkcipher_walk_done(desc, &walk, nbytes);
120 }
121 fprs_write(0);
122 return err;
123}
124
125static int ecb_encrypt(struct blkcipher_desc *desc,
126 struct scatterlist *dst, struct scatterlist *src,
127 unsigned int nbytes)
128{
129 return __ecb_crypt(desc, dst, src, nbytes, true);
130}
131
132static int ecb_decrypt(struct blkcipher_desc *desc,
133 struct scatterlist *dst, struct scatterlist *src,
134 unsigned int nbytes)
135{
136 return __ecb_crypt(desc, dst, src, nbytes, false);
137}
138
139typedef void cbc_crypt_op(const u64 *input, u64 *output, unsigned int len,
140 const u64 *key, u64 *iv);
141
142extern cbc_crypt_op camellia_sparc64_cbc_encrypt_3_grand_rounds;
143extern cbc_crypt_op camellia_sparc64_cbc_encrypt_4_grand_rounds;
144extern cbc_crypt_op camellia_sparc64_cbc_decrypt_3_grand_rounds;
145extern cbc_crypt_op camellia_sparc64_cbc_decrypt_4_grand_rounds;
146
147static int cbc_encrypt(struct blkcipher_desc *desc,
148 struct scatterlist *dst, struct scatterlist *src,
149 unsigned int nbytes)
150{
151 struct camellia_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
152 struct blkcipher_walk walk;
153 cbc_crypt_op *op;
154 const u64 *key;
155 int err;
156
157 op = camellia_sparc64_cbc_encrypt_3_grand_rounds;
158 if (ctx->key_len != 16)
159 op = camellia_sparc64_cbc_encrypt_4_grand_rounds;
160
161 blkcipher_walk_init(&walk, dst, src, nbytes);
162 err = blkcipher_walk_virt(desc, &walk);
163
164 key = &ctx->encrypt_key[0];
165 camellia_sparc64_load_keys(key, ctx->key_len);
166 while ((nbytes = walk.nbytes)) {
167 unsigned int block_len = nbytes & CAMELLIA_BLOCK_MASK;
168
169 if (likely(block_len)) {
170 const u64 *src64;
171 u64 *dst64;
172
173 src64 = (const u64 *)walk.src.virt.addr;
174 dst64 = (u64 *) walk.dst.virt.addr;
175 op(src64, dst64, block_len, key,
176 (u64 *) walk.iv);
177 }
178 nbytes &= CAMELLIA_BLOCK_SIZE - 1;
179 err = blkcipher_walk_done(desc, &walk, nbytes);
180 }
181 fprs_write(0);
182 return err;
183}
184
185static int cbc_decrypt(struct blkcipher_desc *desc,
186 struct scatterlist *dst, struct scatterlist *src,
187 unsigned int nbytes)
188{
189 struct camellia_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
190 struct blkcipher_walk walk;
191 cbc_crypt_op *op;
192 const u64 *key;
193 int err;
194
195 op = camellia_sparc64_cbc_decrypt_3_grand_rounds;
196 if (ctx->key_len != 16)
197 op = camellia_sparc64_cbc_decrypt_4_grand_rounds;
198
199 blkcipher_walk_init(&walk, dst, src, nbytes);
200 err = blkcipher_walk_virt(desc, &walk);
201
202 key = &ctx->decrypt_key[0];
203 camellia_sparc64_load_keys(key, ctx->key_len);
204 while ((nbytes = walk.nbytes)) {
205 unsigned int block_len = nbytes & CAMELLIA_BLOCK_MASK;
206
207 if (likely(block_len)) {
208 const u64 *src64;
209 u64 *dst64;
210
211 src64 = (const u64 *)walk.src.virt.addr;
212 dst64 = (u64 *) walk.dst.virt.addr;
213 op(src64, dst64, block_len, key,
214 (u64 *) walk.iv);
215 }
216 nbytes &= CAMELLIA_BLOCK_SIZE - 1;
217 err = blkcipher_walk_done(desc, &walk, nbytes);
218 }
219 fprs_write(0);
220 return err;
221}
222
223static struct crypto_alg algs[] = { {
224 .cra_name = "camellia",
225 .cra_driver_name = "camellia-sparc64",
226 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
227 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
228 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
229 .cra_ctxsize = sizeof(struct camellia_sparc64_ctx),
230 .cra_alignmask = 3,
231 .cra_module = THIS_MODULE,
232 .cra_u = {
233 .cipher = {
234 .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE,
235 .cia_max_keysize = CAMELLIA_MAX_KEY_SIZE,
236 .cia_setkey = camellia_set_key,
237 .cia_encrypt = camellia_encrypt,
238 .cia_decrypt = camellia_decrypt
239 }
240 }
241}, {
242 .cra_name = "ecb(camellia)",
243 .cra_driver_name = "ecb-camellia-sparc64",
244 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
245 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
246 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
247 .cra_ctxsize = sizeof(struct camellia_sparc64_ctx),
248 .cra_alignmask = 7,
249 .cra_type = &crypto_blkcipher_type,
250 .cra_module = THIS_MODULE,
251 .cra_u = {
252 .blkcipher = {
253 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
254 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
255 .setkey = camellia_set_key,
256 .encrypt = ecb_encrypt,
257 .decrypt = ecb_decrypt,
258 },
259 },
260}, {
261 .cra_name = "cbc(camellia)",
262 .cra_driver_name = "cbc-camellia-sparc64",
263 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
264 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
265 .cra_blocksize = CAMELLIA_BLOCK_SIZE,
266 .cra_ctxsize = sizeof(struct camellia_sparc64_ctx),
267 .cra_alignmask = 7,
268 .cra_type = &crypto_blkcipher_type,
269 .cra_module = THIS_MODULE,
270 .cra_u = {
271 .blkcipher = {
272 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
273 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
274 .setkey = camellia_set_key,
275 .encrypt = cbc_encrypt,
276 .decrypt = cbc_decrypt,
277 },
278 },
279}
280};
281
282static bool __init sparc64_has_camellia_opcode(void)
283{
284 unsigned long cfr;
285
286 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
287 return false;
288
289 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
290 if (!(cfr & CFR_CAMELLIA))
291 return false;
292
293 return true;
294}
295
296static int __init camellia_sparc64_mod_init(void)
297{
298 int i;
299
300 for (i = 0; i < ARRAY_SIZE(algs); i++)
301 INIT_LIST_HEAD(&algs[i].cra_list);
302
303 if (sparc64_has_camellia_opcode()) {
304 pr_info("Using sparc64 camellia opcodes optimized CAMELLIA implementation\n");
305 return crypto_register_algs(algs, ARRAY_SIZE(algs));
306 }
307 pr_info("sparc64 camellia opcodes not available.\n");
308 return -ENODEV;
309}
310
311static void __exit camellia_sparc64_mod_fini(void)
312{
313 crypto_unregister_algs(algs, ARRAY_SIZE(algs));
314}
315
316module_init(camellia_sparc64_mod_init);
317module_exit(camellia_sparc64_mod_fini);
318
319MODULE_LICENSE("GPL");
320MODULE_DESCRIPTION("Camellia Cipher Algorithm, sparc64 camellia opcode accelerated");
321
322MODULE_ALIAS("aes");
diff --git a/arch/sparc/crypto/crc32c_asm.S b/arch/sparc/crypto/crc32c_asm.S
new file mode 100644
index 000000000000..2b1976e765b5
--- /dev/null
+++ b/arch/sparc/crypto/crc32c_asm.S
@@ -0,0 +1,20 @@
1#include <linux/linkage.h>
2#include <asm/visasm.h>
3#include <asm/asi.h>
4
5#include "opcodes.h"
6
7ENTRY(crc32c_sparc64)
8 /* %o0=crc32p, %o1=data_ptr, %o2=len */
9 VISEntryHalf
10 lda [%o0] ASI_PL, %f1
111: ldd [%o1], %f2
12 CRC32C(0,2,0)
13 subcc %o2, 8, %o2
14 bne,pt %icc, 1b
15 add %o1, 0x8, %o1
16 sta %f1, [%o0] ASI_PL
17 VISExitHalf
182: retl
19 nop
20ENDPROC(crc32c_sparc64)
diff --git a/arch/sparc/crypto/crc32c_glue.c b/arch/sparc/crypto/crc32c_glue.c
new file mode 100644
index 000000000000..0bd89cea8d8e
--- /dev/null
+++ b/arch/sparc/crypto/crc32c_glue.c
@@ -0,0 +1,179 @@
1/* Glue code for CRC32C optimized for sparc64 crypto opcodes.
2 *
3 * This is based largely upon arch/x86/crypto/crc32c-intel.c
4 *
5 * Copyright (C) 2008 Intel Corporation
6 * Authors: Austin Zhang <austin_zhang@linux.intel.com>
7 * Kent Liu <kent.liu@intel.com>
8 */
9
10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
12#include <linux/init.h>
13#include <linux/module.h>
14#include <linux/string.h>
15#include <linux/kernel.h>
16#include <linux/crc32.h>
17
18#include <crypto/internal/hash.h>
19
20#include <asm/pstate.h>
21#include <asm/elf.h>
22
23#include "opcodes.h"
24
25/*
26 * Setting the seed allows arbitrary accumulators and flexible XOR policy
27 * If your algorithm starts with ~0, then XOR with ~0 before you set
28 * the seed.
29 */
30static int crc32c_sparc64_setkey(struct crypto_shash *hash, const u8 *key,
31 unsigned int keylen)
32{
33 u32 *mctx = crypto_shash_ctx(hash);
34
35 if (keylen != sizeof(u32)) {
36 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
37 return -EINVAL;
38 }
39 *(__le32 *)mctx = le32_to_cpup((__le32 *)key);
40 return 0;
41}
42
43static int crc32c_sparc64_init(struct shash_desc *desc)
44{
45 u32 *mctx = crypto_shash_ctx(desc->tfm);
46 u32 *crcp = shash_desc_ctx(desc);
47
48 *crcp = *mctx;
49
50 return 0;
51}
52
53extern void crc32c_sparc64(u32 *crcp, const u64 *data, unsigned int len);
54
55static void crc32c_compute(u32 *crcp, const u64 *data, unsigned int len)
56{
57 unsigned int asm_len;
58
59 asm_len = len & ~7U;
60 if (asm_len) {
61 crc32c_sparc64(crcp, data, asm_len);
62 data += asm_len / 8;
63 len -= asm_len;
64 }
65 if (len)
66 *crcp = __crc32c_le(*crcp, (const unsigned char *) data, len);
67}
68
69static int crc32c_sparc64_update(struct shash_desc *desc, const u8 *data,
70 unsigned int len)
71{
72 u32 *crcp = shash_desc_ctx(desc);
73
74 crc32c_compute(crcp, (const u64 *) data, len);
75
76 return 0;
77}
78
79static int __crc32c_sparc64_finup(u32 *crcp, const u8 *data, unsigned int len,
80 u8 *out)
81{
82 u32 tmp = *crcp;
83
84 crc32c_compute(&tmp, (const u64 *) data, len);
85
86 *(__le32 *) out = ~cpu_to_le32(tmp);
87 return 0;
88}
89
90static int crc32c_sparc64_finup(struct shash_desc *desc, const u8 *data,
91 unsigned int len, u8 *out)
92{
93 return __crc32c_sparc64_finup(shash_desc_ctx(desc), data, len, out);
94}
95
96static int crc32c_sparc64_final(struct shash_desc *desc, u8 *out)
97{
98 u32 *crcp = shash_desc_ctx(desc);
99
100 *(__le32 *) out = ~cpu_to_le32p(crcp);
101 return 0;
102}
103
104static int crc32c_sparc64_digest(struct shash_desc *desc, const u8 *data,
105 unsigned int len, u8 *out)
106{
107 return __crc32c_sparc64_finup(crypto_shash_ctx(desc->tfm), data, len,
108 out);
109}
110
111static int crc32c_sparc64_cra_init(struct crypto_tfm *tfm)
112{
113 u32 *key = crypto_tfm_ctx(tfm);
114
115 *key = ~0;
116
117 return 0;
118}
119
120#define CHKSUM_BLOCK_SIZE 1
121#define CHKSUM_DIGEST_SIZE 4
122
123static struct shash_alg alg = {
124 .setkey = crc32c_sparc64_setkey,
125 .init = crc32c_sparc64_init,
126 .update = crc32c_sparc64_update,
127 .final = crc32c_sparc64_final,
128 .finup = crc32c_sparc64_finup,
129 .digest = crc32c_sparc64_digest,
130 .descsize = sizeof(u32),
131 .digestsize = CHKSUM_DIGEST_SIZE,
132 .base = {
133 .cra_name = "crc32c",
134 .cra_driver_name = "crc32c-sparc64",
135 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
136 .cra_blocksize = CHKSUM_BLOCK_SIZE,
137 .cra_ctxsize = sizeof(u32),
138 .cra_alignmask = 7,
139 .cra_module = THIS_MODULE,
140 .cra_init = crc32c_sparc64_cra_init,
141 }
142};
143
144static bool __init sparc64_has_crc32c_opcode(void)
145{
146 unsigned long cfr;
147
148 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
149 return false;
150
151 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
152 if (!(cfr & CFR_CRC32C))
153 return false;
154
155 return true;
156}
157
158static int __init crc32c_sparc64_mod_init(void)
159{
160 if (sparc64_has_crc32c_opcode()) {
161 pr_info("Using sparc64 crc32c opcode optimized CRC32C implementation\n");
162 return crypto_register_shash(&alg);
163 }
164 pr_info("sparc64 crc32c opcode not available.\n");
165 return -ENODEV;
166}
167
168static void __exit crc32c_sparc64_mod_fini(void)
169{
170 crypto_unregister_shash(&alg);
171}
172
173module_init(crc32c_sparc64_mod_init);
174module_exit(crc32c_sparc64_mod_fini);
175
176MODULE_LICENSE("GPL");
177MODULE_DESCRIPTION("CRC32c (Castagnoli), sparc64 crc32c opcode accelerated");
178
179MODULE_ALIAS("crc32c");
diff --git a/arch/sparc/crypto/crop_devid.c b/arch/sparc/crypto/crop_devid.c
new file mode 100644
index 000000000000..5f5724a0ae22
--- /dev/null
+++ b/arch/sparc/crypto/crop_devid.c
@@ -0,0 +1,14 @@
1#include <linux/module.h>
2#include <linux/of_device.h>
3
4/* This is a dummy device table linked into all of the crypto
5 * opcode drivers. It serves to trigger the module autoloading
6 * mechanisms in userspace which scan the OF device tree and
7 * load any modules which have device table entries that
8 * match OF device nodes.
9 */
10static const struct of_device_id crypto_opcode_match[] = {
11 { .name = "cpu", .compatible = "sun4v", },
12 {},
13};
14MODULE_DEVICE_TABLE(of, crypto_opcode_match);
diff --git a/arch/sparc/crypto/des_asm.S b/arch/sparc/crypto/des_asm.S
new file mode 100644
index 000000000000..30b6e90b28b2
--- /dev/null
+++ b/arch/sparc/crypto/des_asm.S
@@ -0,0 +1,418 @@
1#include <linux/linkage.h>
2#include <asm/visasm.h>
3
4#include "opcodes.h"
5
6 .align 32
7ENTRY(des_sparc64_key_expand)
8 /* %o0=input_key, %o1=output_key */
9 VISEntryHalf
10 ld [%o0 + 0x00], %f0
11 ld [%o0 + 0x04], %f1
12 DES_KEXPAND(0, 0, 0)
13 DES_KEXPAND(0, 1, 2)
14 DES_KEXPAND(2, 3, 6)
15 DES_KEXPAND(2, 2, 4)
16 DES_KEXPAND(6, 3, 10)
17 DES_KEXPAND(6, 2, 8)
18 DES_KEXPAND(10, 3, 14)
19 DES_KEXPAND(10, 2, 12)
20 DES_KEXPAND(14, 1, 16)
21 DES_KEXPAND(16, 3, 20)
22 DES_KEXPAND(16, 2, 18)
23 DES_KEXPAND(20, 3, 24)
24 DES_KEXPAND(20, 2, 22)
25 DES_KEXPAND(24, 3, 28)
26 DES_KEXPAND(24, 2, 26)
27 DES_KEXPAND(28, 1, 30)
28 std %f0, [%o1 + 0x00]
29 std %f2, [%o1 + 0x08]
30 std %f4, [%o1 + 0x10]
31 std %f6, [%o1 + 0x18]
32 std %f8, [%o1 + 0x20]
33 std %f10, [%o1 + 0x28]
34 std %f12, [%o1 + 0x30]
35 std %f14, [%o1 + 0x38]
36 std %f16, [%o1 + 0x40]
37 std %f18, [%o1 + 0x48]
38 std %f20, [%o1 + 0x50]
39 std %f22, [%o1 + 0x58]
40 std %f24, [%o1 + 0x60]
41 std %f26, [%o1 + 0x68]
42 std %f28, [%o1 + 0x70]
43 std %f30, [%o1 + 0x78]
44 retl
45 VISExitHalf
46ENDPROC(des_sparc64_key_expand)
47
48 .align 32
49ENTRY(des_sparc64_crypt)
50 /* %o0=key, %o1=input, %o2=output */
51 VISEntry
52 ldd [%o1 + 0x00], %f32
53 ldd [%o0 + 0x00], %f0
54 ldd [%o0 + 0x08], %f2
55 ldd [%o0 + 0x10], %f4
56 ldd [%o0 + 0x18], %f6
57 ldd [%o0 + 0x20], %f8
58 ldd [%o0 + 0x28], %f10
59 ldd [%o0 + 0x30], %f12
60 ldd [%o0 + 0x38], %f14
61 ldd [%o0 + 0x40], %f16
62 ldd [%o0 + 0x48], %f18
63 ldd [%o0 + 0x50], %f20
64 ldd [%o0 + 0x58], %f22
65 ldd [%o0 + 0x60], %f24
66 ldd [%o0 + 0x68], %f26
67 ldd [%o0 + 0x70], %f28
68 ldd [%o0 + 0x78], %f30
69 DES_IP(32, 32)
70 DES_ROUND(0, 2, 32, 32)
71 DES_ROUND(4, 6, 32, 32)
72 DES_ROUND(8, 10, 32, 32)
73 DES_ROUND(12, 14, 32, 32)
74 DES_ROUND(16, 18, 32, 32)
75 DES_ROUND(20, 22, 32, 32)
76 DES_ROUND(24, 26, 32, 32)
77 DES_ROUND(28, 30, 32, 32)
78 DES_IIP(32, 32)
79 std %f32, [%o2 + 0x00]
80 retl
81 VISExit
82ENDPROC(des_sparc64_crypt)
83
84 .align 32
85ENTRY(des_sparc64_load_keys)
86 /* %o0=key */
87 VISEntry
88 ldd [%o0 + 0x00], %f0
89 ldd [%o0 + 0x08], %f2
90 ldd [%o0 + 0x10], %f4
91 ldd [%o0 + 0x18], %f6
92 ldd [%o0 + 0x20], %f8
93 ldd [%o0 + 0x28], %f10
94 ldd [%o0 + 0x30], %f12
95 ldd [%o0 + 0x38], %f14
96 ldd [%o0 + 0x40], %f16
97 ldd [%o0 + 0x48], %f18
98 ldd [%o0 + 0x50], %f20
99 ldd [%o0 + 0x58], %f22
100 ldd [%o0 + 0x60], %f24
101 ldd [%o0 + 0x68], %f26
102 ldd [%o0 + 0x70], %f28
103 retl
104 ldd [%o0 + 0x78], %f30
105ENDPROC(des_sparc64_load_keys)
106
107 .align 32
108ENTRY(des_sparc64_ecb_crypt)
109 /* %o0=input, %o1=output, %o2=len */
1101: ldd [%o0 + 0x00], %f32
111 add %o0, 0x08, %o0
112 DES_IP(32, 32)
113 DES_ROUND(0, 2, 32, 32)
114 DES_ROUND(4, 6, 32, 32)
115 DES_ROUND(8, 10, 32, 32)
116 DES_ROUND(12, 14, 32, 32)
117 DES_ROUND(16, 18, 32, 32)
118 DES_ROUND(20, 22, 32, 32)
119 DES_ROUND(24, 26, 32, 32)
120 DES_ROUND(28, 30, 32, 32)
121 DES_IIP(32, 32)
122 std %f32, [%o1 + 0x00]
123 subcc %o2, 0x08, %o2
124 bne,pt %icc, 1b
125 add %o1, 0x08, %o1
126 retl
127 nop
128ENDPROC(des_sparc64_ecb_crypt)
129
130 .align 32
131ENTRY(des_sparc64_cbc_encrypt)
132 /* %o0=input, %o1=output, %o2=len, %o3=IV */
133 ldd [%o3 + 0x00], %f32
1341: ldd [%o0 + 0x00], %f34
135 fxor %f32, %f34, %f32
136 DES_IP(32, 32)
137 DES_ROUND(0, 2, 32, 32)
138 DES_ROUND(4, 6, 32, 32)
139 DES_ROUND(8, 10, 32, 32)
140 DES_ROUND(12, 14, 32, 32)
141 DES_ROUND(16, 18, 32, 32)
142 DES_ROUND(20, 22, 32, 32)
143 DES_ROUND(24, 26, 32, 32)
144 DES_ROUND(28, 30, 32, 32)
145 DES_IIP(32, 32)
146 std %f32, [%o1 + 0x00]
147 add %o0, 0x08, %o0
148 subcc %o2, 0x08, %o2
149 bne,pt %icc, 1b
150 add %o1, 0x08, %o1
151 retl
152 std %f32, [%o3 + 0x00]
153ENDPROC(des_sparc64_cbc_encrypt)
154
155 .align 32
156ENTRY(des_sparc64_cbc_decrypt)
157 /* %o0=input, %o1=output, %o2=len, %o3=IV */
158 ldd [%o3 + 0x00], %f34
1591: ldd [%o0 + 0x00], %f36
160 DES_IP(36, 32)
161 DES_ROUND(0, 2, 32, 32)
162 DES_ROUND(4, 6, 32, 32)
163 DES_ROUND(8, 10, 32, 32)
164 DES_ROUND(12, 14, 32, 32)
165 DES_ROUND(16, 18, 32, 32)
166 DES_ROUND(20, 22, 32, 32)
167 DES_ROUND(24, 26, 32, 32)
168 DES_ROUND(28, 30, 32, 32)
169 DES_IIP(32, 32)
170 fxor %f32, %f34, %f32
171 fsrc2 %f36, %f34
172 std %f32, [%o1 + 0x00]
173 add %o0, 0x08, %o0
174 subcc %o2, 0x08, %o2
175 bne,pt %icc, 1b
176 add %o1, 0x08, %o1
177 retl
178 std %f36, [%o3 + 0x00]
179ENDPROC(des_sparc64_cbc_decrypt)
180
181 .align 32
182ENTRY(des3_ede_sparc64_crypt)
183 /* %o0=key, %o1=input, %o2=output */
184 VISEntry
185 ldd [%o1 + 0x00], %f32
186 ldd [%o0 + 0x00], %f0
187 ldd [%o0 + 0x08], %f2
188 ldd [%o0 + 0x10], %f4
189 ldd [%o0 + 0x18], %f6
190 ldd [%o0 + 0x20], %f8
191 ldd [%o0 + 0x28], %f10
192 ldd [%o0 + 0x30], %f12
193 ldd [%o0 + 0x38], %f14
194 ldd [%o0 + 0x40], %f16
195 ldd [%o0 + 0x48], %f18
196 ldd [%o0 + 0x50], %f20
197 ldd [%o0 + 0x58], %f22
198 ldd [%o0 + 0x60], %f24
199 ldd [%o0 + 0x68], %f26
200 ldd [%o0 + 0x70], %f28
201 ldd [%o0 + 0x78], %f30
202 DES_IP(32, 32)
203 DES_ROUND(0, 2, 32, 32)
204 ldd [%o0 + 0x80], %f0
205 ldd [%o0 + 0x88], %f2
206 DES_ROUND(4, 6, 32, 32)
207 ldd [%o0 + 0x90], %f4
208 ldd [%o0 + 0x98], %f6
209 DES_ROUND(8, 10, 32, 32)
210 ldd [%o0 + 0xa0], %f8
211 ldd [%o0 + 0xa8], %f10
212 DES_ROUND(12, 14, 32, 32)
213 ldd [%o0 + 0xb0], %f12
214 ldd [%o0 + 0xb8], %f14
215 DES_ROUND(16, 18, 32, 32)
216 ldd [%o0 + 0xc0], %f16
217 ldd [%o0 + 0xc8], %f18
218 DES_ROUND(20, 22, 32, 32)
219 ldd [%o0 + 0xd0], %f20
220 ldd [%o0 + 0xd8], %f22
221 DES_ROUND(24, 26, 32, 32)
222 ldd [%o0 + 0xe0], %f24
223 ldd [%o0 + 0xe8], %f26
224 DES_ROUND(28, 30, 32, 32)
225 ldd [%o0 + 0xf0], %f28
226 ldd [%o0 + 0xf8], %f30
227 DES_IIP(32, 32)
228 DES_IP(32, 32)
229 DES_ROUND(0, 2, 32, 32)
230 ldd [%o0 + 0x100], %f0
231 ldd [%o0 + 0x108], %f2
232 DES_ROUND(4, 6, 32, 32)
233 ldd [%o0 + 0x110], %f4
234 ldd [%o0 + 0x118], %f6
235 DES_ROUND(8, 10, 32, 32)
236 ldd [%o0 + 0x120], %f8
237 ldd [%o0 + 0x128], %f10
238 DES_ROUND(12, 14, 32, 32)
239 ldd [%o0 + 0x130], %f12
240 ldd [%o0 + 0x138], %f14
241 DES_ROUND(16, 18, 32, 32)
242 ldd [%o0 + 0x140], %f16
243 ldd [%o0 + 0x148], %f18
244 DES_ROUND(20, 22, 32, 32)
245 ldd [%o0 + 0x150], %f20
246 ldd [%o0 + 0x158], %f22
247 DES_ROUND(24, 26, 32, 32)
248 ldd [%o0 + 0x160], %f24
249 ldd [%o0 + 0x168], %f26
250 DES_ROUND(28, 30, 32, 32)
251 ldd [%o0 + 0x170], %f28
252 ldd [%o0 + 0x178], %f30
253 DES_IIP(32, 32)
254 DES_IP(32, 32)
255 DES_ROUND(0, 2, 32, 32)
256 DES_ROUND(4, 6, 32, 32)
257 DES_ROUND(8, 10, 32, 32)
258 DES_ROUND(12, 14, 32, 32)
259 DES_ROUND(16, 18, 32, 32)
260 DES_ROUND(20, 22, 32, 32)
261 DES_ROUND(24, 26, 32, 32)
262 DES_ROUND(28, 30, 32, 32)
263 DES_IIP(32, 32)
264
265 std %f32, [%o2 + 0x00]
266 retl
267 VISExit
268ENDPROC(des3_ede_sparc64_crypt)
269
270 .align 32
271ENTRY(des3_ede_sparc64_load_keys)
272 /* %o0=key */
273 VISEntry
274 ldd [%o0 + 0x00], %f0
275 ldd [%o0 + 0x08], %f2
276 ldd [%o0 + 0x10], %f4
277 ldd [%o0 + 0x18], %f6
278 ldd [%o0 + 0x20], %f8
279 ldd [%o0 + 0x28], %f10
280 ldd [%o0 + 0x30], %f12
281 ldd [%o0 + 0x38], %f14
282 ldd [%o0 + 0x40], %f16
283 ldd [%o0 + 0x48], %f18
284 ldd [%o0 + 0x50], %f20
285 ldd [%o0 + 0x58], %f22
286 ldd [%o0 + 0x60], %f24
287 ldd [%o0 + 0x68], %f26
288 ldd [%o0 + 0x70], %f28
289 ldd [%o0 + 0x78], %f30
290 ldd [%o0 + 0x80], %f32
291 ldd [%o0 + 0x88], %f34
292 ldd [%o0 + 0x90], %f36
293 ldd [%o0 + 0x98], %f38
294 ldd [%o0 + 0xa0], %f40
295 ldd [%o0 + 0xa8], %f42
296 ldd [%o0 + 0xb0], %f44
297 ldd [%o0 + 0xb8], %f46
298 ldd [%o0 + 0xc0], %f48
299 ldd [%o0 + 0xc8], %f50
300 ldd [%o0 + 0xd0], %f52
301 ldd [%o0 + 0xd8], %f54
302 ldd [%o0 + 0xe0], %f56
303 retl
304 ldd [%o0 + 0xe8], %f58
305ENDPROC(des3_ede_sparc64_load_keys)
306
307#define DES3_LOOP_BODY(X) \
308 DES_IP(X, X) \
309 DES_ROUND(0, 2, X, X) \
310 DES_ROUND(4, 6, X, X) \
311 DES_ROUND(8, 10, X, X) \
312 DES_ROUND(12, 14, X, X) \
313 DES_ROUND(16, 18, X, X) \
314 ldd [%o0 + 0xf0], %f16; \
315 ldd [%o0 + 0xf8], %f18; \
316 DES_ROUND(20, 22, X, X) \
317 ldd [%o0 + 0x100], %f20; \
318 ldd [%o0 + 0x108], %f22; \
319 DES_ROUND(24, 26, X, X) \
320 ldd [%o0 + 0x110], %f24; \
321 ldd [%o0 + 0x118], %f26; \
322 DES_ROUND(28, 30, X, X) \
323 ldd [%o0 + 0x120], %f28; \
324 ldd [%o0 + 0x128], %f30; \
325 DES_IIP(X, X) \
326 DES_IP(X, X) \
327 DES_ROUND(32, 34, X, X) \
328 ldd [%o0 + 0x130], %f0; \
329 ldd [%o0 + 0x138], %f2; \
330 DES_ROUND(36, 38, X, X) \
331 ldd [%o0 + 0x140], %f4; \
332 ldd [%o0 + 0x148], %f6; \
333 DES_ROUND(40, 42, X, X) \
334 ldd [%o0 + 0x150], %f8; \
335 ldd [%o0 + 0x158], %f10; \
336 DES_ROUND(44, 46, X, X) \
337 ldd [%o0 + 0x160], %f12; \
338 ldd [%o0 + 0x168], %f14; \
339 DES_ROUND(48, 50, X, X) \
340 DES_ROUND(52, 54, X, X) \
341 DES_ROUND(56, 58, X, X) \
342 DES_ROUND(16, 18, X, X) \
343 ldd [%o0 + 0x170], %f16; \
344 ldd [%o0 + 0x178], %f18; \
345 DES_IIP(X, X) \
346 DES_IP(X, X) \
347 DES_ROUND(20, 22, X, X) \
348 ldd [%o0 + 0x50], %f20; \
349 ldd [%o0 + 0x58], %f22; \
350 DES_ROUND(24, 26, X, X) \
351 ldd [%o0 + 0x60], %f24; \
352 ldd [%o0 + 0x68], %f26; \
353 DES_ROUND(28, 30, X, X) \
354 ldd [%o0 + 0x70], %f28; \
355 ldd [%o0 + 0x78], %f30; \
356 DES_ROUND(0, 2, X, X) \
357 ldd [%o0 + 0x00], %f0; \
358 ldd [%o0 + 0x08], %f2; \
359 DES_ROUND(4, 6, X, X) \
360 ldd [%o0 + 0x10], %f4; \
361 ldd [%o0 + 0x18], %f6; \
362 DES_ROUND(8, 10, X, X) \
363 ldd [%o0 + 0x20], %f8; \
364 ldd [%o0 + 0x28], %f10; \
365 DES_ROUND(12, 14, X, X) \
366 ldd [%o0 + 0x30], %f12; \
367 ldd [%o0 + 0x38], %f14; \
368 DES_ROUND(16, 18, X, X) \
369 ldd [%o0 + 0x40], %f16; \
370 ldd [%o0 + 0x48], %f18; \
371 DES_IIP(X, X)
372
373 .align 32
374ENTRY(des3_ede_sparc64_ecb_crypt)
375 /* %o0=key, %o1=input, %o2=output, %o3=len */
3761: ldd [%o1 + 0x00], %f60
377 DES3_LOOP_BODY(60)
378 std %f60, [%o2 + 0x00]
379 subcc %o3, 0x08, %o3
380 bne,pt %icc, 1b
381 add %o2, 0x08, %o2
382 retl
383 nop
384ENDPROC(des3_ede_sparc64_ecb_crypt)
385
386 .align 32
387ENTRY(des3_ede_sparc64_cbc_encrypt)
388 /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
389 ldd [%o4 + 0x00], %f60
3901: ldd [%o1 + 0x00], %f62
391 fxor %f60, %f62, %f60
392 DES3_LOOP_BODY(60)
393 std %f60, [%o2 + 0x00]
394 add %o1, 0x08, %o1
395 subcc %o3, 0x08, %o3
396 bne,pt %icc, 1b
397 add %o2, 0x08, %o2
398 retl
399 std %f60, [%o4 + 0x00]
400ENDPROC(des3_ede_sparc64_cbc_encrypt)
401
402 .align 32
403ENTRY(des3_ede_sparc64_cbc_decrypt)
404 /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
405 ldd [%o4 + 0x00], %f62
4061: ldx [%o1 + 0x00], %g1
407 MOVXTOD_G1_F60
408 DES3_LOOP_BODY(60)
409 fxor %f62, %f60, %f60
410 MOVXTOD_G1_F62
411 std %f60, [%o2 + 0x00]
412 add %o1, 0x08, %o1
413 subcc %o3, 0x08, %o3
414 bne,pt %icc, 1b
415 add %o2, 0x08, %o2
416 retl
417 stx %g1, [%o4 + 0x00]
418ENDPROC(des3_ede_sparc64_cbc_decrypt)
diff --git a/arch/sparc/crypto/des_glue.c b/arch/sparc/crypto/des_glue.c
new file mode 100644
index 000000000000..c4940c2d3073
--- /dev/null
+++ b/arch/sparc/crypto/des_glue.c
@@ -0,0 +1,529 @@
1/* Glue code for DES encryption optimized for sparc64 crypto opcodes.
2 *
3 * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
4 */
5
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/crypto.h>
9#include <linux/init.h>
10#include <linux/module.h>
11#include <linux/mm.h>
12#include <linux/types.h>
13#include <crypto/algapi.h>
14#include <crypto/des.h>
15
16#include <asm/fpumacro.h>
17#include <asm/pstate.h>
18#include <asm/elf.h>
19
20#include "opcodes.h"
21
22struct des_sparc64_ctx {
23 u64 encrypt_expkey[DES_EXPKEY_WORDS / 2];
24 u64 decrypt_expkey[DES_EXPKEY_WORDS / 2];
25};
26
27struct des3_ede_sparc64_ctx {
28 u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
29 u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2];
30};
31
32static void encrypt_to_decrypt(u64 *d, const u64 *e)
33{
34 const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1;
35 int i;
36
37 for (i = 0; i < DES_EXPKEY_WORDS / 2; i++)
38 *d++ = *s--;
39}
40
41extern void des_sparc64_key_expand(const u32 *input_key, u64 *key);
42
43static int des_set_key(struct crypto_tfm *tfm, const u8 *key,
44 unsigned int keylen)
45{
46 struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
47 u32 *flags = &tfm->crt_flags;
48 u32 tmp[DES_EXPKEY_WORDS];
49 int ret;
50
51 /* Even though we have special instructions for key expansion,
52 * we call des_ekey() so that we don't have to write our own
53 * weak key detection code.
54 */
55 ret = des_ekey(tmp, key);
56 if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
57 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
58 return -EINVAL;
59 }
60
61 des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]);
62 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]);
63
64 return 0;
65}
66
67extern void des_sparc64_crypt(const u64 *key, const u64 *input,
68 u64 *output);
69
70static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
71{
72 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
73 const u64 *K = ctx->encrypt_expkey;
74
75 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
76}
77
78static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
79{
80 struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
81 const u64 *K = ctx->decrypt_expkey;
82
83 des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
84}
85
86extern void des_sparc64_load_keys(const u64 *key);
87
88extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output,
89 unsigned int len);
90
91#define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1))
92
93static int __ecb_crypt(struct blkcipher_desc *desc,
94 struct scatterlist *dst, struct scatterlist *src,
95 unsigned int nbytes, bool encrypt)
96{
97 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
98 struct blkcipher_walk walk;
99 int err;
100
101 blkcipher_walk_init(&walk, dst, src, nbytes);
102 err = blkcipher_walk_virt(desc, &walk);
103
104 if (encrypt)
105 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
106 else
107 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
108 while ((nbytes = walk.nbytes)) {
109 unsigned int block_len = nbytes & DES_BLOCK_MASK;
110
111 if (likely(block_len)) {
112 des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr,
113 (u64 *) walk.dst.virt.addr,
114 block_len);
115 }
116 nbytes &= DES_BLOCK_SIZE - 1;
117 err = blkcipher_walk_done(desc, &walk, nbytes);
118 }
119 fprs_write(0);
120 return err;
121}
122
123static int ecb_encrypt(struct blkcipher_desc *desc,
124 struct scatterlist *dst, struct scatterlist *src,
125 unsigned int nbytes)
126{
127 return __ecb_crypt(desc, dst, src, nbytes, true);
128}
129
130static int ecb_decrypt(struct blkcipher_desc *desc,
131 struct scatterlist *dst, struct scatterlist *src,
132 unsigned int nbytes)
133{
134 return __ecb_crypt(desc, dst, src, nbytes, false);
135}
136
137extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output,
138 unsigned int len, u64 *iv);
139
140static int cbc_encrypt(struct blkcipher_desc *desc,
141 struct scatterlist *dst, struct scatterlist *src,
142 unsigned int nbytes)
143{
144 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
145 struct blkcipher_walk walk;
146 int err;
147
148 blkcipher_walk_init(&walk, dst, src, nbytes);
149 err = blkcipher_walk_virt(desc, &walk);
150
151 des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
152 while ((nbytes = walk.nbytes)) {
153 unsigned int block_len = nbytes & DES_BLOCK_MASK;
154
155 if (likely(block_len)) {
156 des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr,
157 (u64 *) walk.dst.virt.addr,
158 block_len, (u64 *) walk.iv);
159 }
160 nbytes &= DES_BLOCK_SIZE - 1;
161 err = blkcipher_walk_done(desc, &walk, nbytes);
162 }
163 fprs_write(0);
164 return err;
165}
166
167extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output,
168 unsigned int len, u64 *iv);
169
170static int cbc_decrypt(struct blkcipher_desc *desc,
171 struct scatterlist *dst, struct scatterlist *src,
172 unsigned int nbytes)
173{
174 struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
175 struct blkcipher_walk walk;
176 int err;
177
178 blkcipher_walk_init(&walk, dst, src, nbytes);
179 err = blkcipher_walk_virt(desc, &walk);
180
181 des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
182 while ((nbytes = walk.nbytes)) {
183 unsigned int block_len = nbytes & DES_BLOCK_MASK;
184
185 if (likely(block_len)) {
186 des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr,
187 (u64 *) walk.dst.virt.addr,
188 block_len, (u64 *) walk.iv);
189 }
190 nbytes &= DES_BLOCK_SIZE - 1;
191 err = blkcipher_walk_done(desc, &walk, nbytes);
192 }
193 fprs_write(0);
194 return err;
195}
196
197static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key,
198 unsigned int keylen)
199{
200 struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm);
201 const u32 *K = (const u32 *)key;
202 u32 *flags = &tfm->crt_flags;
203 u64 k1[DES_EXPKEY_WORDS / 2];
204 u64 k2[DES_EXPKEY_WORDS / 2];
205 u64 k3[DES_EXPKEY_WORDS / 2];
206
207 if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
208 !((K[2] ^ K[4]) | (K[3] ^ K[5]))) &&
209 (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
210 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
211 return -EINVAL;
212 }
213
214 des_sparc64_key_expand((const u32 *)key, k1);
215 key += DES_KEY_SIZE;
216 des_sparc64_key_expand((const u32 *)key, k2);
217 key += DES_KEY_SIZE;
218 des_sparc64_key_expand((const u32 *)key, k3);
219
220 memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1));
221 encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]);
222 memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
223 &k3[0], sizeof(k3));
224
225 encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]);
226 memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2],
227 &k2[0], sizeof(k2));
228 encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2],
229 &k1[0]);
230
231 return 0;
232}
233
234extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input,
235 u64 *output);
236
237static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
238{
239 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
240 const u64 *K = ctx->encrypt_expkey;
241
242 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
243}
244
245static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
246{
247 struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
248 const u64 *K = ctx->decrypt_expkey;
249
250 des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst);
251}
252
253extern void des3_ede_sparc64_load_keys(const u64 *key);
254
255extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input,
256 u64 *output, unsigned int len);
257
258static int __ecb3_crypt(struct blkcipher_desc *desc,
259 struct scatterlist *dst, struct scatterlist *src,
260 unsigned int nbytes, bool encrypt)
261{
262 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
263 struct blkcipher_walk walk;
264 const u64 *K;
265 int err;
266
267 blkcipher_walk_init(&walk, dst, src, nbytes);
268 err = blkcipher_walk_virt(desc, &walk);
269
270 if (encrypt)
271 K = &ctx->encrypt_expkey[0];
272 else
273 K = &ctx->decrypt_expkey[0];
274 des3_ede_sparc64_load_keys(K);
275 while ((nbytes = walk.nbytes)) {
276 unsigned int block_len = nbytes & DES_BLOCK_MASK;
277
278 if (likely(block_len)) {
279 const u64 *src64 = (const u64 *)walk.src.virt.addr;
280 des3_ede_sparc64_ecb_crypt(K, src64,
281 (u64 *) walk.dst.virt.addr,
282 block_len);
283 }
284 nbytes &= DES_BLOCK_SIZE - 1;
285 err = blkcipher_walk_done(desc, &walk, nbytes);
286 }
287 fprs_write(0);
288 return err;
289}
290
291static int ecb3_encrypt(struct blkcipher_desc *desc,
292 struct scatterlist *dst, struct scatterlist *src,
293 unsigned int nbytes)
294{
295 return __ecb3_crypt(desc, dst, src, nbytes, true);
296}
297
298static int ecb3_decrypt(struct blkcipher_desc *desc,
299 struct scatterlist *dst, struct scatterlist *src,
300 unsigned int nbytes)
301{
302 return __ecb3_crypt(desc, dst, src, nbytes, false);
303}
304
305extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input,
306 u64 *output, unsigned int len,
307 u64 *iv);
308
309static int cbc3_encrypt(struct blkcipher_desc *desc,
310 struct scatterlist *dst, struct scatterlist *src,
311 unsigned int nbytes)
312{
313 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
314 struct blkcipher_walk walk;
315 const u64 *K;
316 int err;
317
318 blkcipher_walk_init(&walk, dst, src, nbytes);
319 err = blkcipher_walk_virt(desc, &walk);
320
321 K = &ctx->encrypt_expkey[0];
322 des3_ede_sparc64_load_keys(K);
323 while ((nbytes = walk.nbytes)) {
324 unsigned int block_len = nbytes & DES_BLOCK_MASK;
325
326 if (likely(block_len)) {
327 const u64 *src64 = (const u64 *)walk.src.virt.addr;
328 des3_ede_sparc64_cbc_encrypt(K, src64,
329 (u64 *) walk.dst.virt.addr,
330 block_len,
331 (u64 *) walk.iv);
332 }
333 nbytes &= DES_BLOCK_SIZE - 1;
334 err = blkcipher_walk_done(desc, &walk, nbytes);
335 }
336 fprs_write(0);
337 return err;
338}
339
340extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input,
341 u64 *output, unsigned int len,
342 u64 *iv);
343
344static int cbc3_decrypt(struct blkcipher_desc *desc,
345 struct scatterlist *dst, struct scatterlist *src,
346 unsigned int nbytes)
347{
348 struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
349 struct blkcipher_walk walk;
350 const u64 *K;
351 int err;
352
353 blkcipher_walk_init(&walk, dst, src, nbytes);
354 err = blkcipher_walk_virt(desc, &walk);
355
356 K = &ctx->decrypt_expkey[0];
357 des3_ede_sparc64_load_keys(K);
358 while ((nbytes = walk.nbytes)) {
359 unsigned int block_len = nbytes & DES_BLOCK_MASK;
360
361 if (likely(block_len)) {
362 const u64 *src64 = (const u64 *)walk.src.virt.addr;
363 des3_ede_sparc64_cbc_decrypt(K, src64,
364 (u64 *) walk.dst.virt.addr,
365 block_len,
366 (u64 *) walk.iv);
367 }
368 nbytes &= DES_BLOCK_SIZE - 1;
369 err = blkcipher_walk_done(desc, &walk, nbytes);
370 }
371 fprs_write(0);
372 return err;
373}
374
375static struct crypto_alg algs[] = { {
376 .cra_name = "des",
377 .cra_driver_name = "des-sparc64",
378 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
379 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
380 .cra_blocksize = DES_BLOCK_SIZE,
381 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
382 .cra_alignmask = 7,
383 .cra_module = THIS_MODULE,
384 .cra_u = {
385 .cipher = {
386 .cia_min_keysize = DES_KEY_SIZE,
387 .cia_max_keysize = DES_KEY_SIZE,
388 .cia_setkey = des_set_key,
389 .cia_encrypt = des_encrypt,
390 .cia_decrypt = des_decrypt
391 }
392 }
393}, {
394 .cra_name = "ecb(des)",
395 .cra_driver_name = "ecb-des-sparc64",
396 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
397 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
398 .cra_blocksize = DES_BLOCK_SIZE,
399 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
400 .cra_alignmask = 7,
401 .cra_type = &crypto_blkcipher_type,
402 .cra_module = THIS_MODULE,
403 .cra_u = {
404 .blkcipher = {
405 .min_keysize = DES_KEY_SIZE,
406 .max_keysize = DES_KEY_SIZE,
407 .setkey = des_set_key,
408 .encrypt = ecb_encrypt,
409 .decrypt = ecb_decrypt,
410 },
411 },
412}, {
413 .cra_name = "cbc(des)",
414 .cra_driver_name = "cbc-des-sparc64",
415 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
416 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
417 .cra_blocksize = DES_BLOCK_SIZE,
418 .cra_ctxsize = sizeof(struct des_sparc64_ctx),
419 .cra_alignmask = 7,
420 .cra_type = &crypto_blkcipher_type,
421 .cra_module = THIS_MODULE,
422 .cra_u = {
423 .blkcipher = {
424 .min_keysize = DES_KEY_SIZE,
425 .max_keysize = DES_KEY_SIZE,
426 .setkey = des_set_key,
427 .encrypt = cbc_encrypt,
428 .decrypt = cbc_decrypt,
429 },
430 },
431}, {
432 .cra_name = "des3_ede",
433 .cra_driver_name = "des3_ede-sparc64",
434 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
435 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
436 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
437 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
438 .cra_alignmask = 7,
439 .cra_module = THIS_MODULE,
440 .cra_u = {
441 .cipher = {
442 .cia_min_keysize = DES3_EDE_KEY_SIZE,
443 .cia_max_keysize = DES3_EDE_KEY_SIZE,
444 .cia_setkey = des3_ede_set_key,
445 .cia_encrypt = des3_ede_encrypt,
446 .cia_decrypt = des3_ede_decrypt
447 }
448 }
449}, {
450 .cra_name = "ecb(des3_ede)",
451 .cra_driver_name = "ecb-des3_ede-sparc64",
452 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
453 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
454 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
455 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
456 .cra_alignmask = 7,
457 .cra_type = &crypto_blkcipher_type,
458 .cra_module = THIS_MODULE,
459 .cra_u = {
460 .blkcipher = {
461 .min_keysize = DES3_EDE_KEY_SIZE,
462 .max_keysize = DES3_EDE_KEY_SIZE,
463 .setkey = des3_ede_set_key,
464 .encrypt = ecb3_encrypt,
465 .decrypt = ecb3_decrypt,
466 },
467 },
468}, {
469 .cra_name = "cbc(des3_ede)",
470 .cra_driver_name = "cbc-des3_ede-sparc64",
471 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
472 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
473 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
474 .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx),
475 .cra_alignmask = 7,
476 .cra_type = &crypto_blkcipher_type,
477 .cra_module = THIS_MODULE,
478 .cra_u = {
479 .blkcipher = {
480 .min_keysize = DES3_EDE_KEY_SIZE,
481 .max_keysize = DES3_EDE_KEY_SIZE,
482 .setkey = des3_ede_set_key,
483 .encrypt = cbc3_encrypt,
484 .decrypt = cbc3_decrypt,
485 },
486 },
487} };
488
489static bool __init sparc64_has_des_opcode(void)
490{
491 unsigned long cfr;
492
493 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
494 return false;
495
496 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
497 if (!(cfr & CFR_DES))
498 return false;
499
500 return true;
501}
502
503static int __init des_sparc64_mod_init(void)
504{
505 int i;
506
507 for (i = 0; i < ARRAY_SIZE(algs); i++)
508 INIT_LIST_HEAD(&algs[i].cra_list);
509
510 if (sparc64_has_des_opcode()) {
511 pr_info("Using sparc64 des opcodes optimized DES implementation\n");
512 return crypto_register_algs(algs, ARRAY_SIZE(algs));
513 }
514 pr_info("sparc64 des opcodes not available.\n");
515 return -ENODEV;
516}
517
518static void __exit des_sparc64_mod_fini(void)
519{
520 crypto_unregister_algs(algs, ARRAY_SIZE(algs));
521}
522
523module_init(des_sparc64_mod_init);
524module_exit(des_sparc64_mod_fini);
525
526MODULE_LICENSE("GPL");
527MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated");
528
529MODULE_ALIAS("des");
diff --git a/arch/sparc/crypto/md5_asm.S b/arch/sparc/crypto/md5_asm.S
new file mode 100644
index 000000000000..3150404e602e
--- /dev/null
+++ b/arch/sparc/crypto/md5_asm.S
@@ -0,0 +1,70 @@
1#include <linux/linkage.h>
2#include <asm/visasm.h>
3
4#include "opcodes.h"
5
6ENTRY(md5_sparc64_transform)
7 /* %o0 = digest, %o1 = data, %o2 = rounds */
8 VISEntryHalf
9 ld [%o0 + 0x00], %f0
10 ld [%o0 + 0x04], %f1
11 andcc %o1, 0x7, %g0
12 ld [%o0 + 0x08], %f2
13 bne,pn %xcc, 10f
14 ld [%o0 + 0x0c], %f3
15
161:
17 ldd [%o1 + 0x00], %f8
18 ldd [%o1 + 0x08], %f10
19 ldd [%o1 + 0x10], %f12
20 ldd [%o1 + 0x18], %f14
21 ldd [%o1 + 0x20], %f16
22 ldd [%o1 + 0x28], %f18
23 ldd [%o1 + 0x30], %f20
24 ldd [%o1 + 0x38], %f22
25
26 MD5
27
28 subcc %o2, 1, %o2
29 bne,pt %xcc, 1b
30 add %o1, 0x40, %o1
31
325:
33 st %f0, [%o0 + 0x00]
34 st %f1, [%o0 + 0x04]
35 st %f2, [%o0 + 0x08]
36 st %f3, [%o0 + 0x0c]
37 retl
38 VISExitHalf
3910:
40 alignaddr %o1, %g0, %o1
41
42 ldd [%o1 + 0x00], %f10
431:
44 ldd [%o1 + 0x08], %f12
45 ldd [%o1 + 0x10], %f14
46 ldd [%o1 + 0x18], %f16
47 ldd [%o1 + 0x20], %f18
48 ldd [%o1 + 0x28], %f20
49 ldd [%o1 + 0x30], %f22
50 ldd [%o1 + 0x38], %f24
51 ldd [%o1 + 0x40], %f26
52
53 faligndata %f10, %f12, %f8
54 faligndata %f12, %f14, %f10
55 faligndata %f14, %f16, %f12
56 faligndata %f16, %f18, %f14
57 faligndata %f18, %f20, %f16
58 faligndata %f20, %f22, %f18
59 faligndata %f22, %f24, %f20
60 faligndata %f24, %f26, %f22
61
62 MD5
63
64 subcc %o2, 1, %o2
65 fsrc2 %f26, %f10
66 bne,pt %xcc, 1b
67 add %o1, 0x40, %o1
68
69 ba,a,pt %xcc, 5b
70ENDPROC(md5_sparc64_transform)
diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c
new file mode 100644
index 000000000000..603d723038ce
--- /dev/null
+++ b/arch/sparc/crypto/md5_glue.c
@@ -0,0 +1,188 @@
1/* Glue code for MD5 hashing optimized for sparc64 crypto opcodes.
2 *
3 * This is based largely upon arch/x86/crypto/sha1_ssse3_glue.c
4 * and crypto/md5.c which are:
5 *
6 * Copyright (c) Alan Smithee.
7 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
8 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
9 * Copyright (c) Mathias Krause <minipli@googlemail.com>
10 * Copyright (c) Cryptoapi developers.
11 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
12 */
13
14#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15
16#include <crypto/internal/hash.h>
17#include <linux/init.h>
18#include <linux/module.h>
19#include <linux/mm.h>
20#include <linux/cryptohash.h>
21#include <linux/types.h>
22#include <crypto/md5.h>
23
24#include <asm/pstate.h>
25#include <asm/elf.h>
26
27#include "opcodes.h"
28
29asmlinkage void md5_sparc64_transform(u32 *digest, const char *data,
30 unsigned int rounds);
31
32static int md5_sparc64_init(struct shash_desc *desc)
33{
34 struct md5_state *mctx = shash_desc_ctx(desc);
35
36 mctx->hash[0] = cpu_to_le32(0x67452301);
37 mctx->hash[1] = cpu_to_le32(0xefcdab89);
38 mctx->hash[2] = cpu_to_le32(0x98badcfe);
39 mctx->hash[3] = cpu_to_le32(0x10325476);
40 mctx->byte_count = 0;
41
42 return 0;
43}
44
45static void __md5_sparc64_update(struct md5_state *sctx, const u8 *data,
46 unsigned int len, unsigned int partial)
47{
48 unsigned int done = 0;
49
50 sctx->byte_count += len;
51 if (partial) {
52 done = MD5_HMAC_BLOCK_SIZE - partial;
53 memcpy((u8 *)sctx->block + partial, data, done);
54 md5_sparc64_transform(sctx->hash, (u8 *)sctx->block, 1);
55 }
56 if (len - done >= MD5_HMAC_BLOCK_SIZE) {
57 const unsigned int rounds = (len - done) / MD5_HMAC_BLOCK_SIZE;
58
59 md5_sparc64_transform(sctx->hash, data + done, rounds);
60 done += rounds * MD5_HMAC_BLOCK_SIZE;
61 }
62
63 memcpy(sctx->block, data + done, len - done);
64}
65
66static int md5_sparc64_update(struct shash_desc *desc, const u8 *data,
67 unsigned int len)
68{
69 struct md5_state *sctx = shash_desc_ctx(desc);
70 unsigned int partial = sctx->byte_count % MD5_HMAC_BLOCK_SIZE;
71
72 /* Handle the fast case right here */
73 if (partial + len < MD5_HMAC_BLOCK_SIZE) {
74 sctx->byte_count += len;
75 memcpy((u8 *)sctx->block + partial, data, len);
76 } else
77 __md5_sparc64_update(sctx, data, len, partial);
78
79 return 0;
80}
81
82/* Add padding and return the message digest. */
83static int md5_sparc64_final(struct shash_desc *desc, u8 *out)
84{
85 struct md5_state *sctx = shash_desc_ctx(desc);
86 unsigned int i, index, padlen;
87 u32 *dst = (u32 *)out;
88 __le64 bits;
89 static const u8 padding[MD5_HMAC_BLOCK_SIZE] = { 0x80, };
90
91 bits = cpu_to_le64(sctx->byte_count << 3);
92
93 /* Pad out to 56 mod 64 and append length */
94 index = sctx->byte_count % MD5_HMAC_BLOCK_SIZE;
95 padlen = (index < 56) ? (56 - index) : ((MD5_HMAC_BLOCK_SIZE+56) - index);
96
97 /* We need to fill a whole block for __md5_sparc64_update() */
98 if (padlen <= 56) {
99 sctx->byte_count += padlen;
100 memcpy((u8 *)sctx->block + index, padding, padlen);
101 } else {
102 __md5_sparc64_update(sctx, padding, padlen, index);
103 }
104 __md5_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56);
105
106 /* Store state in digest */
107 for (i = 0; i < MD5_HASH_WORDS; i++)
108 dst[i] = sctx->hash[i];
109
110 /* Wipe context */
111 memset(sctx, 0, sizeof(*sctx));
112
113 return 0;
114}
115
116static int md5_sparc64_export(struct shash_desc *desc, void *out)
117{
118 struct md5_state *sctx = shash_desc_ctx(desc);
119
120 memcpy(out, sctx, sizeof(*sctx));
121
122 return 0;
123}
124
125static int md5_sparc64_import(struct shash_desc *desc, const void *in)
126{
127 struct md5_state *sctx = shash_desc_ctx(desc);
128
129 memcpy(sctx, in, sizeof(*sctx));
130
131 return 0;
132}
133
134static struct shash_alg alg = {
135 .digestsize = MD5_DIGEST_SIZE,
136 .init = md5_sparc64_init,
137 .update = md5_sparc64_update,
138 .final = md5_sparc64_final,
139 .export = md5_sparc64_export,
140 .import = md5_sparc64_import,
141 .descsize = sizeof(struct md5_state),
142 .statesize = sizeof(struct md5_state),
143 .base = {
144 .cra_name = "md5",
145 .cra_driver_name= "md5-sparc64",
146 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
147 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
148 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
149 .cra_module = THIS_MODULE,
150 }
151};
152
153static bool __init sparc64_has_md5_opcode(void)
154{
155 unsigned long cfr;
156
157 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
158 return false;
159
160 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
161 if (!(cfr & CFR_MD5))
162 return false;
163
164 return true;
165}
166
167static int __init md5_sparc64_mod_init(void)
168{
169 if (sparc64_has_md5_opcode()) {
170 pr_info("Using sparc64 md5 opcode optimized MD5 implementation\n");
171 return crypto_register_shash(&alg);
172 }
173 pr_info("sparc64 md5 opcode not available.\n");
174 return -ENODEV;
175}
176
177static void __exit md5_sparc64_mod_fini(void)
178{
179 crypto_unregister_shash(&alg);
180}
181
182module_init(md5_sparc64_mod_init);
183module_exit(md5_sparc64_mod_fini);
184
185MODULE_LICENSE("GPL");
186MODULE_DESCRIPTION("MD5 Secure Hash Algorithm, sparc64 md5 opcode accelerated");
187
188MODULE_ALIAS("md5");
diff --git a/arch/sparc/crypto/opcodes.h b/arch/sparc/crypto/opcodes.h
new file mode 100644
index 000000000000..19cbaea6976f
--- /dev/null
+++ b/arch/sparc/crypto/opcodes.h
@@ -0,0 +1,99 @@
1#ifndef _OPCODES_H
2#define _OPCODES_H
3
4#define SPARC_CR_OPCODE_PRIORITY 300
5
6#define F3F(x,y,z) (((x)<<30)|((y)<<19)|((z)<<5))
7
8#define FPD_ENCODE(x) (((x) >> 5) | ((x) & ~(0x20)))
9
10#define RS1(x) (FPD_ENCODE(x) << 14)
11#define RS2(x) (FPD_ENCODE(x) << 0)
12#define RS3(x) (FPD_ENCODE(x) << 9)
13#define RD(x) (FPD_ENCODE(x) << 25)
14#define IMM5_0(x) ((x) << 0)
15#define IMM5_9(x) ((x) << 9)
16
17#define CRC32C(a,b,c) \
18 .word (F3F(2,0x36,0x147)|RS1(a)|RS2(b)|RD(c));
19
20#define MD5 \
21 .word 0x81b02800;
22#define SHA1 \
23 .word 0x81b02820;
24#define SHA256 \
25 .word 0x81b02840;
26#define SHA512 \
27 .word 0x81b02860;
28
29#define AES_EROUND01(a,b,c,d) \
30 .word (F3F(2, 0x19, 0)|RS1(a)|RS2(b)|RS3(c)|RD(d));
31#define AES_EROUND23(a,b,c,d) \
32 .word (F3F(2, 0x19, 1)|RS1(a)|RS2(b)|RS3(c)|RD(d));
33#define AES_DROUND01(a,b,c,d) \
34 .word (F3F(2, 0x19, 2)|RS1(a)|RS2(b)|RS3(c)|RD(d));
35#define AES_DROUND23(a,b,c,d) \
36 .word (F3F(2, 0x19, 3)|RS1(a)|RS2(b)|RS3(c)|RD(d));
37#define AES_EROUND01_L(a,b,c,d) \
38 .word (F3F(2, 0x19, 4)|RS1(a)|RS2(b)|RS3(c)|RD(d));
39#define AES_EROUND23_L(a,b,c,d) \
40 .word (F3F(2, 0x19, 5)|RS1(a)|RS2(b)|RS3(c)|RD(d));
41#define AES_DROUND01_L(a,b,c,d) \
42 .word (F3F(2, 0x19, 6)|RS1(a)|RS2(b)|RS3(c)|RD(d));
43#define AES_DROUND23_L(a,b,c,d) \
44 .word (F3F(2, 0x19, 7)|RS1(a)|RS2(b)|RS3(c)|RD(d));
45#define AES_KEXPAND1(a,b,c,d) \
46 .word (F3F(2, 0x19, 8)|RS1(a)|RS2(b)|IMM5_9(c)|RD(d));
47#define AES_KEXPAND0(a,b,c) \
48 .word (F3F(2, 0x36, 0x130)|RS1(a)|RS2(b)|RD(c));
49#define AES_KEXPAND2(a,b,c) \
50 .word (F3F(2, 0x36, 0x131)|RS1(a)|RS2(b)|RD(c));
51
52#define DES_IP(a,b) \
53 .word (F3F(2, 0x36, 0x134)|RS1(a)|RD(b));
54#define DES_IIP(a,b) \
55 .word (F3F(2, 0x36, 0x135)|RS1(a)|RD(b));
56#define DES_KEXPAND(a,b,c) \
57 .word (F3F(2, 0x36, 0x136)|RS1(a)|IMM5_0(b)|RD(c));
58#define DES_ROUND(a,b,c,d) \
59 .word (F3F(2, 0x19, 0x009)|RS1(a)|RS2(b)|RS3(c)|RD(d));
60
61#define CAMELLIA_F(a,b,c,d) \
62 .word (F3F(2, 0x19, 0x00c)|RS1(a)|RS2(b)|RS3(c)|RD(d));
63#define CAMELLIA_FL(a,b,c) \
64 .word (F3F(2, 0x36, 0x13c)|RS1(a)|RS2(b)|RD(c));
65#define CAMELLIA_FLI(a,b,c) \
66 .word (F3F(2, 0x36, 0x13d)|RS1(a)|RS2(b)|RD(c));
67
68#define MOVDTOX_F0_O4 \
69 .word 0x99b02200
70#define MOVDTOX_F2_O5 \
71 .word 0x9bb02202
72#define MOVXTOD_G1_F60 \
73 .word 0xbbb02301
74#define MOVXTOD_G1_F62 \
75 .word 0xbfb02301
76#define MOVXTOD_G3_F4 \
77 .word 0x89b02303;
78#define MOVXTOD_G7_F6 \
79 .word 0x8db02307;
80#define MOVXTOD_G3_F0 \
81 .word 0x81b02303;
82#define MOVXTOD_G7_F2 \
83 .word 0x85b02307;
84#define MOVXTOD_O0_F0 \
85 .word 0x81b02308;
86#define MOVXTOD_O5_F0 \
87 .word 0x81b0230d;
88#define MOVXTOD_O5_F2 \
89 .word 0x85b0230d;
90#define MOVXTOD_O5_F4 \
91 .word 0x89b0230d;
92#define MOVXTOD_O5_F6 \
93 .word 0x8db0230d;
94#define MOVXTOD_G3_F60 \
95 .word 0xbbb02303;
96#define MOVXTOD_G7_F62 \
97 .word 0xbfb02307;
98
99#endif /* _OPCODES_H */
diff --git a/arch/sparc/crypto/sha1_asm.S b/arch/sparc/crypto/sha1_asm.S
new file mode 100644
index 000000000000..219d10c5ae0e
--- /dev/null
+++ b/arch/sparc/crypto/sha1_asm.S
@@ -0,0 +1,72 @@
1#include <linux/linkage.h>
2#include <asm/visasm.h>
3
4#include "opcodes.h"
5
6ENTRY(sha1_sparc64_transform)
7 /* %o0 = digest, %o1 = data, %o2 = rounds */
8 VISEntryHalf
9 ld [%o0 + 0x00], %f0
10 ld [%o0 + 0x04], %f1
11 ld [%o0 + 0x08], %f2
12 andcc %o1, 0x7, %g0
13 ld [%o0 + 0x0c], %f3
14 bne,pn %xcc, 10f
15 ld [%o0 + 0x10], %f4
16
171:
18 ldd [%o1 + 0x00], %f8
19 ldd [%o1 + 0x08], %f10
20 ldd [%o1 + 0x10], %f12
21 ldd [%o1 + 0x18], %f14
22 ldd [%o1 + 0x20], %f16
23 ldd [%o1 + 0x28], %f18
24 ldd [%o1 + 0x30], %f20
25 ldd [%o1 + 0x38], %f22
26
27 SHA1
28
29 subcc %o2, 1, %o2
30 bne,pt %xcc, 1b
31 add %o1, 0x40, %o1
32
335:
34 st %f0, [%o0 + 0x00]
35 st %f1, [%o0 + 0x04]
36 st %f2, [%o0 + 0x08]
37 st %f3, [%o0 + 0x0c]
38 st %f4, [%o0 + 0x10]
39 retl
40 VISExitHalf
4110:
42 alignaddr %o1, %g0, %o1
43
44 ldd [%o1 + 0x00], %f10
451:
46 ldd [%o1 + 0x08], %f12
47 ldd [%o1 + 0x10], %f14
48 ldd [%o1 + 0x18], %f16
49 ldd [%o1 + 0x20], %f18
50 ldd [%o1 + 0x28], %f20
51 ldd [%o1 + 0x30], %f22
52 ldd [%o1 + 0x38], %f24
53 ldd [%o1 + 0x40], %f26
54
55 faligndata %f10, %f12, %f8
56 faligndata %f12, %f14, %f10
57 faligndata %f14, %f16, %f12
58 faligndata %f16, %f18, %f14
59 faligndata %f18, %f20, %f16
60 faligndata %f20, %f22, %f18
61 faligndata %f22, %f24, %f20
62 faligndata %f24, %f26, %f22
63
64 SHA1
65
66 subcc %o2, 1, %o2
67 fsrc2 %f26, %f10
68 bne,pt %xcc, 1b
69 add %o1, 0x40, %o1
70
71 ba,a,pt %xcc, 5b
72ENDPROC(sha1_sparc64_transform)
diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c
new file mode 100644
index 000000000000..2bbb20bee9f1
--- /dev/null
+++ b/arch/sparc/crypto/sha1_glue.c
@@ -0,0 +1,183 @@
1/* Glue code for SHA1 hashing optimized for sparc64 crypto opcodes.
2 *
3 * This is based largely upon arch/x86/crypto/sha1_ssse3_glue.c
4 *
5 * Copyright (c) Alan Smithee.
6 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
7 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
8 * Copyright (c) Mathias Krause <minipli@googlemail.com>
9 */
10
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13#include <crypto/internal/hash.h>
14#include <linux/init.h>
15#include <linux/module.h>
16#include <linux/mm.h>
17#include <linux/cryptohash.h>
18#include <linux/types.h>
19#include <crypto/sha.h>
20
21#include <asm/pstate.h>
22#include <asm/elf.h>
23
24#include "opcodes.h"
25
26asmlinkage void sha1_sparc64_transform(u32 *digest, const char *data,
27 unsigned int rounds);
28
29static int sha1_sparc64_init(struct shash_desc *desc)
30{
31 struct sha1_state *sctx = shash_desc_ctx(desc);
32
33 *sctx = (struct sha1_state){
34 .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
35 };
36
37 return 0;
38}
39
40static void __sha1_sparc64_update(struct sha1_state *sctx, const u8 *data,
41 unsigned int len, unsigned int partial)
42{
43 unsigned int done = 0;
44
45 sctx->count += len;
46 if (partial) {
47 done = SHA1_BLOCK_SIZE - partial;
48 memcpy(sctx->buffer + partial, data, done);
49 sha1_sparc64_transform(sctx->state, sctx->buffer, 1);
50 }
51 if (len - done >= SHA1_BLOCK_SIZE) {
52 const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE;
53
54 sha1_sparc64_transform(sctx->state, data + done, rounds);
55 done += rounds * SHA1_BLOCK_SIZE;
56 }
57
58 memcpy(sctx->buffer, data + done, len - done);
59}
60
61static int sha1_sparc64_update(struct shash_desc *desc, const u8 *data,
62 unsigned int len)
63{
64 struct sha1_state *sctx = shash_desc_ctx(desc);
65 unsigned int partial = sctx->count % SHA1_BLOCK_SIZE;
66
67 /* Handle the fast case right here */
68 if (partial + len < SHA1_BLOCK_SIZE) {
69 sctx->count += len;
70 memcpy(sctx->buffer + partial, data, len);
71 } else
72 __sha1_sparc64_update(sctx, data, len, partial);
73
74 return 0;
75}
76
77/* Add padding and return the message digest. */
78static int sha1_sparc64_final(struct shash_desc *desc, u8 *out)
79{
80 struct sha1_state *sctx = shash_desc_ctx(desc);
81 unsigned int i, index, padlen;
82 __be32 *dst = (__be32 *)out;
83 __be64 bits;
84 static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, };
85
86 bits = cpu_to_be64(sctx->count << 3);
87
88 /* Pad out to 56 mod 64 and append length */
89 index = sctx->count % SHA1_BLOCK_SIZE;
90 padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index);
91
92 /* We need to fill a whole block for __sha1_sparc64_update() */
93 if (padlen <= 56) {
94 sctx->count += padlen;
95 memcpy(sctx->buffer + index, padding, padlen);
96 } else {
97 __sha1_sparc64_update(sctx, padding, padlen, index);
98 }
99 __sha1_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56);
100
101 /* Store state in digest */
102 for (i = 0; i < 5; i++)
103 dst[i] = cpu_to_be32(sctx->state[i]);
104
105 /* Wipe context */
106 memset(sctx, 0, sizeof(*sctx));
107
108 return 0;
109}
110
111static int sha1_sparc64_export(struct shash_desc *desc, void *out)
112{
113 struct sha1_state *sctx = shash_desc_ctx(desc);
114
115 memcpy(out, sctx, sizeof(*sctx));
116
117 return 0;
118}
119
120static int sha1_sparc64_import(struct shash_desc *desc, const void *in)
121{
122 struct sha1_state *sctx = shash_desc_ctx(desc);
123
124 memcpy(sctx, in, sizeof(*sctx));
125
126 return 0;
127}
128
129static struct shash_alg alg = {
130 .digestsize = SHA1_DIGEST_SIZE,
131 .init = sha1_sparc64_init,
132 .update = sha1_sparc64_update,
133 .final = sha1_sparc64_final,
134 .export = sha1_sparc64_export,
135 .import = sha1_sparc64_import,
136 .descsize = sizeof(struct sha1_state),
137 .statesize = sizeof(struct sha1_state),
138 .base = {
139 .cra_name = "sha1",
140 .cra_driver_name= "sha1-sparc64",
141 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
142 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
143 .cra_blocksize = SHA1_BLOCK_SIZE,
144 .cra_module = THIS_MODULE,
145 }
146};
147
148static bool __init sparc64_has_sha1_opcode(void)
149{
150 unsigned long cfr;
151
152 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
153 return false;
154
155 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
156 if (!(cfr & CFR_SHA1))
157 return false;
158
159 return true;
160}
161
162static int __init sha1_sparc64_mod_init(void)
163{
164 if (sparc64_has_sha1_opcode()) {
165 pr_info("Using sparc64 sha1 opcode optimized SHA-1 implementation\n");
166 return crypto_register_shash(&alg);
167 }
168 pr_info("sparc64 sha1 opcode not available.\n");
169 return -ENODEV;
170}
171
172static void __exit sha1_sparc64_mod_fini(void)
173{
174 crypto_unregister_shash(&alg);
175}
176
177module_init(sha1_sparc64_mod_init);
178module_exit(sha1_sparc64_mod_fini);
179
180MODULE_LICENSE("GPL");
181MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, sparc64 sha1 opcode accelerated");
182
183MODULE_ALIAS("sha1");
diff --git a/arch/sparc/crypto/sha256_asm.S b/arch/sparc/crypto/sha256_asm.S
new file mode 100644
index 000000000000..b5f3d5826eb4
--- /dev/null
+++ b/arch/sparc/crypto/sha256_asm.S
@@ -0,0 +1,78 @@
1#include <linux/linkage.h>
2#include <asm/visasm.h>
3
4#include "opcodes.h"
5
6ENTRY(sha256_sparc64_transform)
7 /* %o0 = digest, %o1 = data, %o2 = rounds */
8 VISEntryHalf
9 ld [%o0 + 0x00], %f0
10 ld [%o0 + 0x04], %f1
11 ld [%o0 + 0x08], %f2
12 ld [%o0 + 0x0c], %f3
13 ld [%o0 + 0x10], %f4
14 ld [%o0 + 0x14], %f5
15 andcc %o1, 0x7, %g0
16 ld [%o0 + 0x18], %f6
17 bne,pn %xcc, 10f
18 ld [%o0 + 0x1c], %f7
19
201:
21 ldd [%o1 + 0x00], %f8
22 ldd [%o1 + 0x08], %f10
23 ldd [%o1 + 0x10], %f12
24 ldd [%o1 + 0x18], %f14
25 ldd [%o1 + 0x20], %f16
26 ldd [%o1 + 0x28], %f18
27 ldd [%o1 + 0x30], %f20
28 ldd [%o1 + 0x38], %f22
29
30 SHA256
31
32 subcc %o2, 1, %o2
33 bne,pt %xcc, 1b
34 add %o1, 0x40, %o1
35
365:
37 st %f0, [%o0 + 0x00]
38 st %f1, [%o0 + 0x04]
39 st %f2, [%o0 + 0x08]
40 st %f3, [%o0 + 0x0c]
41 st %f4, [%o0 + 0x10]
42 st %f5, [%o0 + 0x14]
43 st %f6, [%o0 + 0x18]
44 st %f7, [%o0 + 0x1c]
45 retl
46 VISExitHalf
4710:
48 alignaddr %o1, %g0, %o1
49
50 ldd [%o1 + 0x00], %f10
511:
52 ldd [%o1 + 0x08], %f12
53 ldd [%o1 + 0x10], %f14
54 ldd [%o1 + 0x18], %f16
55 ldd [%o1 + 0x20], %f18
56 ldd [%o1 + 0x28], %f20
57 ldd [%o1 + 0x30], %f22
58 ldd [%o1 + 0x38], %f24
59 ldd [%o1 + 0x40], %f26
60
61 faligndata %f10, %f12, %f8
62 faligndata %f12, %f14, %f10
63 faligndata %f14, %f16, %f12
64 faligndata %f16, %f18, %f14
65 faligndata %f18, %f20, %f16
66 faligndata %f20, %f22, %f18
67 faligndata %f22, %f24, %f20
68 faligndata %f24, %f26, %f22
69
70 SHA256
71
72 subcc %o2, 1, %o2
73 fsrc2 %f26, %f10
74 bne,pt %xcc, 1b
75 add %o1, 0x40, %o1
76
77 ba,a,pt %xcc, 5b
78ENDPROC(sha256_sparc64_transform)
diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c
new file mode 100644
index 000000000000..591e656bd891
--- /dev/null
+++ b/arch/sparc/crypto/sha256_glue.c
@@ -0,0 +1,241 @@
1/* Glue code for SHA256 hashing optimized for sparc64 crypto opcodes.
2 *
3 * This is based largely upon crypto/sha256_generic.c
4 *
5 * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com>
6 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
7 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
8 * SHA224 Support Copyright 2007 Intel Corporation <jonathan.lynch@intel.com>
9 */
10
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13#include <crypto/internal/hash.h>
14#include <linux/init.h>
15#include <linux/module.h>
16#include <linux/mm.h>
17#include <linux/cryptohash.h>
18#include <linux/types.h>
19#include <crypto/sha.h>
20
21#include <asm/pstate.h>
22#include <asm/elf.h>
23
24#include "opcodes.h"
25
26asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data,
27 unsigned int rounds);
28
29static int sha224_sparc64_init(struct shash_desc *desc)
30{
31 struct sha256_state *sctx = shash_desc_ctx(desc);
32 sctx->state[0] = SHA224_H0;
33 sctx->state[1] = SHA224_H1;
34 sctx->state[2] = SHA224_H2;
35 sctx->state[3] = SHA224_H3;
36 sctx->state[4] = SHA224_H4;
37 sctx->state[5] = SHA224_H5;
38 sctx->state[6] = SHA224_H6;
39 sctx->state[7] = SHA224_H7;
40 sctx->count = 0;
41
42 return 0;
43}
44
45static int sha256_sparc64_init(struct shash_desc *desc)
46{
47 struct sha256_state *sctx = shash_desc_ctx(desc);
48 sctx->state[0] = SHA256_H0;
49 sctx->state[1] = SHA256_H1;
50 sctx->state[2] = SHA256_H2;
51 sctx->state[3] = SHA256_H3;
52 sctx->state[4] = SHA256_H4;
53 sctx->state[5] = SHA256_H5;
54 sctx->state[6] = SHA256_H6;
55 sctx->state[7] = SHA256_H7;
56 sctx->count = 0;
57
58 return 0;
59}
60
61static void __sha256_sparc64_update(struct sha256_state *sctx, const u8 *data,
62 unsigned int len, unsigned int partial)
63{
64 unsigned int done = 0;
65
66 sctx->count += len;
67 if (partial) {
68 done = SHA256_BLOCK_SIZE - partial;
69 memcpy(sctx->buf + partial, data, done);
70 sha256_sparc64_transform(sctx->state, sctx->buf, 1);
71 }
72 if (len - done >= SHA256_BLOCK_SIZE) {
73 const unsigned int rounds = (len - done) / SHA256_BLOCK_SIZE;
74
75 sha256_sparc64_transform(sctx->state, data + done, rounds);
76 done += rounds * SHA256_BLOCK_SIZE;
77 }
78
79 memcpy(sctx->buf, data + done, len - done);
80}
81
82static int sha256_sparc64_update(struct shash_desc *desc, const u8 *data,
83 unsigned int len)
84{
85 struct sha256_state *sctx = shash_desc_ctx(desc);
86 unsigned int partial = sctx->count % SHA256_BLOCK_SIZE;
87
88 /* Handle the fast case right here */
89 if (partial + len < SHA256_BLOCK_SIZE) {
90 sctx->count += len;
91 memcpy(sctx->buf + partial, data, len);
92 } else
93 __sha256_sparc64_update(sctx, data, len, partial);
94
95 return 0;
96}
97
98static int sha256_sparc64_final(struct shash_desc *desc, u8 *out)
99{
100 struct sha256_state *sctx = shash_desc_ctx(desc);
101 unsigned int i, index, padlen;
102 __be32 *dst = (__be32 *)out;
103 __be64 bits;
104 static const u8 padding[SHA256_BLOCK_SIZE] = { 0x80, };
105
106 bits = cpu_to_be64(sctx->count << 3);
107
108 /* Pad out to 56 mod 64 and append length */
109 index = sctx->count % SHA256_BLOCK_SIZE;
110 padlen = (index < 56) ? (56 - index) : ((SHA256_BLOCK_SIZE+56) - index);
111
112 /* We need to fill a whole block for __sha256_sparc64_update() */
113 if (padlen <= 56) {
114 sctx->count += padlen;
115 memcpy(sctx->buf + index, padding, padlen);
116 } else {
117 __sha256_sparc64_update(sctx, padding, padlen, index);
118 }
119 __sha256_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56);
120
121 /* Store state in digest */
122 for (i = 0; i < 8; i++)
123 dst[i] = cpu_to_be32(sctx->state[i]);
124
125 /* Wipe context */
126 memset(sctx, 0, sizeof(*sctx));
127
128 return 0;
129}
130
131static int sha224_sparc64_final(struct shash_desc *desc, u8 *hash)
132{
133 u8 D[SHA256_DIGEST_SIZE];
134
135 sha256_sparc64_final(desc, D);
136
137 memcpy(hash, D, SHA224_DIGEST_SIZE);
138 memset(D, 0, SHA256_DIGEST_SIZE);
139
140 return 0;
141}
142
143static int sha256_sparc64_export(struct shash_desc *desc, void *out)
144{
145 struct sha256_state *sctx = shash_desc_ctx(desc);
146
147 memcpy(out, sctx, sizeof(*sctx));
148 return 0;
149}
150
151static int sha256_sparc64_import(struct shash_desc *desc, const void *in)
152{
153 struct sha256_state *sctx = shash_desc_ctx(desc);
154
155 memcpy(sctx, in, sizeof(*sctx));
156 return 0;
157}
158
159static struct shash_alg sha256 = {
160 .digestsize = SHA256_DIGEST_SIZE,
161 .init = sha256_sparc64_init,
162 .update = sha256_sparc64_update,
163 .final = sha256_sparc64_final,
164 .export = sha256_sparc64_export,
165 .import = sha256_sparc64_import,
166 .descsize = sizeof(struct sha256_state),
167 .statesize = sizeof(struct sha256_state),
168 .base = {
169 .cra_name = "sha256",
170 .cra_driver_name= "sha256-sparc64",
171 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
172 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
173 .cra_blocksize = SHA256_BLOCK_SIZE,
174 .cra_module = THIS_MODULE,
175 }
176};
177
178static struct shash_alg sha224 = {
179 .digestsize = SHA224_DIGEST_SIZE,
180 .init = sha224_sparc64_init,
181 .update = sha256_sparc64_update,
182 .final = sha224_sparc64_final,
183 .descsize = sizeof(struct sha256_state),
184 .base = {
185 .cra_name = "sha224",
186 .cra_driver_name= "sha224-sparc64",
187 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
188 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
189 .cra_blocksize = SHA224_BLOCK_SIZE,
190 .cra_module = THIS_MODULE,
191 }
192};
193
194static bool __init sparc64_has_sha256_opcode(void)
195{
196 unsigned long cfr;
197
198 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
199 return false;
200
201 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
202 if (!(cfr & CFR_SHA256))
203 return false;
204
205 return true;
206}
207
208static int __init sha256_sparc64_mod_init(void)
209{
210 if (sparc64_has_sha256_opcode()) {
211 int ret = crypto_register_shash(&sha224);
212 if (ret < 0)
213 return ret;
214
215 ret = crypto_register_shash(&sha256);
216 if (ret < 0) {
217 crypto_unregister_shash(&sha224);
218 return ret;
219 }
220
221 pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n");
222 return 0;
223 }
224 pr_info("sparc64 sha256 opcode not available.\n");
225 return -ENODEV;
226}
227
228static void __exit sha256_sparc64_mod_fini(void)
229{
230 crypto_unregister_shash(&sha224);
231 crypto_unregister_shash(&sha256);
232}
233
234module_init(sha256_sparc64_mod_init);
235module_exit(sha256_sparc64_mod_fini);
236
237MODULE_LICENSE("GPL");
238MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm, sparc64 sha256 opcode accelerated");
239
240MODULE_ALIAS("sha224");
241MODULE_ALIAS("sha256");
diff --git a/arch/sparc/crypto/sha512_asm.S b/arch/sparc/crypto/sha512_asm.S
new file mode 100644
index 000000000000..54bfba713c0e
--- /dev/null
+++ b/arch/sparc/crypto/sha512_asm.S
@@ -0,0 +1,102 @@
1#include <linux/linkage.h>
2#include <asm/visasm.h>
3
4#include "opcodes.h"
5
6ENTRY(sha512_sparc64_transform)
7 /* %o0 = digest, %o1 = data, %o2 = rounds */
8 VISEntry
9 ldd [%o0 + 0x00], %f0
10 ldd [%o0 + 0x08], %f2
11 ldd [%o0 + 0x10], %f4
12 ldd [%o0 + 0x18], %f6
13 ldd [%o0 + 0x20], %f8
14 ldd [%o0 + 0x28], %f10
15 andcc %o1, 0x7, %g0
16 ldd [%o0 + 0x30], %f12
17 bne,pn %xcc, 10f
18 ldd [%o0 + 0x38], %f14
19
201:
21 ldd [%o1 + 0x00], %f16
22 ldd [%o1 + 0x08], %f18
23 ldd [%o1 + 0x10], %f20
24 ldd [%o1 + 0x18], %f22
25 ldd [%o1 + 0x20], %f24
26 ldd [%o1 + 0x28], %f26
27 ldd [%o1 + 0x30], %f28
28 ldd [%o1 + 0x38], %f30
29 ldd [%o1 + 0x40], %f32
30 ldd [%o1 + 0x48], %f34
31 ldd [%o1 + 0x50], %f36
32 ldd [%o1 + 0x58], %f38
33 ldd [%o1 + 0x60], %f40
34 ldd [%o1 + 0x68], %f42
35 ldd [%o1 + 0x70], %f44
36 ldd [%o1 + 0x78], %f46
37
38 SHA512
39
40 subcc %o2, 1, %o2
41 bne,pt %xcc, 1b
42 add %o1, 0x80, %o1
43
445:
45 std %f0, [%o0 + 0x00]
46 std %f2, [%o0 + 0x08]
47 std %f4, [%o0 + 0x10]
48 std %f6, [%o0 + 0x18]
49 std %f8, [%o0 + 0x20]
50 std %f10, [%o0 + 0x28]
51 std %f12, [%o0 + 0x30]
52 std %f14, [%o0 + 0x38]
53 retl
54 VISExit
5510:
56 alignaddr %o1, %g0, %o1
57
58 ldd [%o1 + 0x00], %f18
591:
60 ldd [%o1 + 0x08], %f20
61 ldd [%o1 + 0x10], %f22
62 ldd [%o1 + 0x18], %f24
63 ldd [%o1 + 0x20], %f26
64 ldd [%o1 + 0x28], %f28
65 ldd [%o1 + 0x30], %f30
66 ldd [%o1 + 0x38], %f32
67 ldd [%o1 + 0x40], %f34
68 ldd [%o1 + 0x48], %f36
69 ldd [%o1 + 0x50], %f38
70 ldd [%o1 + 0x58], %f40
71 ldd [%o1 + 0x60], %f42
72 ldd [%o1 + 0x68], %f44
73 ldd [%o1 + 0x70], %f46
74 ldd [%o1 + 0x78], %f48
75 ldd [%o1 + 0x80], %f50
76
77 faligndata %f18, %f20, %f16
78 faligndata %f20, %f22, %f18
79 faligndata %f22, %f24, %f20
80 faligndata %f24, %f26, %f22
81 faligndata %f26, %f28, %f24
82 faligndata %f28, %f30, %f26
83 faligndata %f30, %f32, %f28
84 faligndata %f32, %f34, %f30
85 faligndata %f34, %f36, %f32
86 faligndata %f36, %f38, %f34
87 faligndata %f38, %f40, %f36
88 faligndata %f40, %f42, %f38
89 faligndata %f42, %f44, %f40
90 faligndata %f44, %f46, %f42
91 faligndata %f46, %f48, %f44
92 faligndata %f48, %f50, %f46
93
94 SHA512
95
96 subcc %o2, 1, %o2
97 fsrc2 %f50, %f18
98 bne,pt %xcc, 1b
99 add %o1, 0x80, %o1
100
101 ba,a,pt %xcc, 5b
102ENDPROC(sha512_sparc64_transform)
diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c
new file mode 100644
index 000000000000..486f0a2b7001
--- /dev/null
+++ b/arch/sparc/crypto/sha512_glue.c
@@ -0,0 +1,226 @@
1/* Glue code for SHA512 hashing optimized for sparc64 crypto opcodes.
2 *
3 * This is based largely upon crypto/sha512_generic.c
4 *
5 * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com>
6 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
7 * Copyright (c) 2003 Kyle McMartin <kyle@debian.org>
8 */
9
10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
11
12#include <crypto/internal/hash.h>
13#include <linux/init.h>
14#include <linux/module.h>
15#include <linux/mm.h>
16#include <linux/cryptohash.h>
17#include <linux/types.h>
18#include <crypto/sha.h>
19
20#include <asm/pstate.h>
21#include <asm/elf.h>
22
23#include "opcodes.h"
24
25asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data,
26 unsigned int rounds);
27
28static int sha512_sparc64_init(struct shash_desc *desc)
29{
30 struct sha512_state *sctx = shash_desc_ctx(desc);
31 sctx->state[0] = SHA512_H0;
32 sctx->state[1] = SHA512_H1;
33 sctx->state[2] = SHA512_H2;
34 sctx->state[3] = SHA512_H3;
35 sctx->state[4] = SHA512_H4;
36 sctx->state[5] = SHA512_H5;
37 sctx->state[6] = SHA512_H6;
38 sctx->state[7] = SHA512_H7;
39 sctx->count[0] = sctx->count[1] = 0;
40
41 return 0;
42}
43
44static int sha384_sparc64_init(struct shash_desc *desc)
45{
46 struct sha512_state *sctx = shash_desc_ctx(desc);
47 sctx->state[0] = SHA384_H0;
48 sctx->state[1] = SHA384_H1;
49 sctx->state[2] = SHA384_H2;
50 sctx->state[3] = SHA384_H3;
51 sctx->state[4] = SHA384_H4;
52 sctx->state[5] = SHA384_H5;
53 sctx->state[6] = SHA384_H6;
54 sctx->state[7] = SHA384_H7;
55 sctx->count[0] = sctx->count[1] = 0;
56
57 return 0;
58}
59
60static void __sha512_sparc64_update(struct sha512_state *sctx, const u8 *data,
61 unsigned int len, unsigned int partial)
62{
63 unsigned int done = 0;
64
65 if ((sctx->count[0] += len) < len)
66 sctx->count[1]++;
67 if (partial) {
68 done = SHA512_BLOCK_SIZE - partial;
69 memcpy(sctx->buf + partial, data, done);
70 sha512_sparc64_transform(sctx->state, sctx->buf, 1);
71 }
72 if (len - done >= SHA512_BLOCK_SIZE) {
73 const unsigned int rounds = (len - done) / SHA512_BLOCK_SIZE;
74
75 sha512_sparc64_transform(sctx->state, data + done, rounds);
76 done += rounds * SHA512_BLOCK_SIZE;
77 }
78
79 memcpy(sctx->buf, data + done, len - done);
80}
81
82static int sha512_sparc64_update(struct shash_desc *desc, const u8 *data,
83 unsigned int len)
84{
85 struct sha512_state *sctx = shash_desc_ctx(desc);
86 unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE;
87
88 /* Handle the fast case right here */
89 if (partial + len < SHA512_BLOCK_SIZE) {
90 if ((sctx->count[0] += len) < len)
91 sctx->count[1]++;
92 memcpy(sctx->buf + partial, data, len);
93 } else
94 __sha512_sparc64_update(sctx, data, len, partial);
95
96 return 0;
97}
98
99static int sha512_sparc64_final(struct shash_desc *desc, u8 *out)
100{
101 struct sha512_state *sctx = shash_desc_ctx(desc);
102 unsigned int i, index, padlen;
103 __be64 *dst = (__be64 *)out;
104 __be64 bits[2];
105 static const u8 padding[SHA512_BLOCK_SIZE] = { 0x80, };
106
107 /* Save number of bits */
108 bits[1] = cpu_to_be64(sctx->count[0] << 3);
109 bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61);
110
111 /* Pad out to 112 mod 128 and append length */
112 index = sctx->count[0] % SHA512_BLOCK_SIZE;
113 padlen = (index < 112) ? (112 - index) : ((SHA512_BLOCK_SIZE+112) - index);
114
115 /* We need to fill a whole block for __sha512_sparc64_update() */
116 if (padlen <= 112) {
117 if ((sctx->count[0] += padlen) < padlen)
118 sctx->count[1]++;
119 memcpy(sctx->buf + index, padding, padlen);
120 } else {
121 __sha512_sparc64_update(sctx, padding, padlen, index);
122 }
123 __sha512_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 112);
124
125 /* Store state in digest */
126 for (i = 0; i < 8; i++)
127 dst[i] = cpu_to_be64(sctx->state[i]);
128
129 /* Wipe context */
130 memset(sctx, 0, sizeof(*sctx));
131
132 return 0;
133}
134
135static int sha384_sparc64_final(struct shash_desc *desc, u8 *hash)
136{
137 u8 D[64];
138
139 sha512_sparc64_final(desc, D);
140
141 memcpy(hash, D, 48);
142 memset(D, 0, 64);
143
144 return 0;
145}
146
147static struct shash_alg sha512 = {
148 .digestsize = SHA512_DIGEST_SIZE,
149 .init = sha512_sparc64_init,
150 .update = sha512_sparc64_update,
151 .final = sha512_sparc64_final,
152 .descsize = sizeof(struct sha512_state),
153 .base = {
154 .cra_name = "sha512",
155 .cra_driver_name= "sha512-sparc64",
156 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
157 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
158 .cra_blocksize = SHA512_BLOCK_SIZE,
159 .cra_module = THIS_MODULE,
160 }
161};
162
163static struct shash_alg sha384 = {
164 .digestsize = SHA384_DIGEST_SIZE,
165 .init = sha384_sparc64_init,
166 .update = sha512_sparc64_update,
167 .final = sha384_sparc64_final,
168 .descsize = sizeof(struct sha512_state),
169 .base = {
170 .cra_name = "sha384",
171 .cra_driver_name= "sha384-sparc64",
172 .cra_priority = SPARC_CR_OPCODE_PRIORITY,
173 .cra_flags = CRYPTO_ALG_TYPE_SHASH,
174 .cra_blocksize = SHA384_BLOCK_SIZE,
175 .cra_module = THIS_MODULE,
176 }
177};
178
179static bool __init sparc64_has_sha512_opcode(void)
180{
181 unsigned long cfr;
182
183 if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
184 return false;
185
186 __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
187 if (!(cfr & CFR_SHA512))
188 return false;
189
190 return true;
191}
192
193static int __init sha512_sparc64_mod_init(void)
194{
195 if (sparc64_has_sha512_opcode()) {
196 int ret = crypto_register_shash(&sha384);
197 if (ret < 0)
198 return ret;
199
200 ret = crypto_register_shash(&sha512);
201 if (ret < 0) {
202 crypto_unregister_shash(&sha384);
203 return ret;
204 }
205
206 pr_info("Using sparc64 sha512 opcode optimized SHA-512/SHA-384 implementation\n");
207 return 0;
208 }
209 pr_info("sparc64 sha512 opcode not available.\n");
210 return -ENODEV;
211}
212
213static void __exit sha512_sparc64_mod_fini(void)
214{
215 crypto_unregister_shash(&sha384);
216 crypto_unregister_shash(&sha512);
217}
218
219module_init(sha512_sparc64_mod_init);
220module_exit(sha512_sparc64_mod_fini);
221
222MODULE_LICENSE("GPL");
223MODULE_DESCRIPTION("SHA-384 and SHA-512 Secure Hash Algorithm, sparc64 sha512 opcode accelerated");
224
225MODULE_ALIAS("sha384");
226MODULE_ALIAS("sha512");
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-05 07:30:05 -0500