aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
Diffstat (limited to 'drivers')
-rw-r--r--drivers/crypto/Kconfig26
-rw-r--r--drivers/crypto/Makefile2
-rw-r--r--drivers/crypto/hifn_795x.c367
-rw-r--r--drivers/crypto/ixp4xx_crypto.c1506
-rw-r--r--drivers/crypto/padlock-aes.c4
-rw-r--r--drivers/crypto/padlock-sha.c4
-rw-r--r--drivers/crypto/talitos.c1597
-rw-r--r--drivers/crypto/talitos.h199
8 files changed, 3515 insertions, 190 deletions
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 43b71b69daa5..e522144cba3a 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -174,4 +174,30 @@ config CRYPTO_DEV_HIFN_795X_RNG
174 Select this option if you want to enable the random number generator 174 Select this option if you want to enable the random number generator
175 on the HIFN 795x crypto adapters. 175 on the HIFN 795x crypto adapters.
176 176
177config CRYPTO_DEV_TALITOS
178 tristate "Talitos Freescale Security Engine (SEC)"
179 select CRYPTO_ALGAPI
180 select CRYPTO_AUTHENC
181 select HW_RANDOM
182 depends on FSL_SOC
183 help
184 Say 'Y' here to use the Freescale Security Engine (SEC)
185 to offload cryptographic algorithm computation.
186
187 The Freescale SEC is present on PowerQUICC 'E' processors, such
188 as the MPC8349E and MPC8548E.
189
190 To compile this driver as a module, choose M here: the module
191 will be called talitos.
192
193config CRYPTO_DEV_IXP4XX
194 tristate "Driver for IXP4xx crypto hardware acceleration"
195 depends on ARCH_IXP4XX
196 select CRYPTO_DES
197 select CRYPTO_ALGAPI
198 select CRYPTO_AUTHENC
199 select CRYPTO_BLKCIPHER
200 help
201 Driver for the IXP4xx NPE crypto engine.
202
177endif # CRYPTO_HW 203endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index c0327f0dadc5..73557b2968d3 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -2,3 +2,5 @@ obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
2obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o 2obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
3obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o 3obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
4obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o 4obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
5obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
6obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o
diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c
index 81f3f950cd7d..4d22b21bd3e3 100644
--- a/drivers/crypto/hifn_795x.c
+++ b/drivers/crypto/hifn_795x.c
@@ -29,7 +29,6 @@
29#include <linux/dma-mapping.h> 29#include <linux/dma-mapping.h>
30#include <linux/scatterlist.h> 30#include <linux/scatterlist.h>
31#include <linux/highmem.h> 31#include <linux/highmem.h>
32#include <linux/interrupt.h>
33#include <linux/crypto.h> 32#include <linux/crypto.h>
34#include <linux/hw_random.h> 33#include <linux/hw_random.h>
35#include <linux/ktime.h> 34#include <linux/ktime.h>
@@ -369,7 +368,9 @@ static atomic_t hifn_dev_number;
369#define HIFN_D_DST_RSIZE 80*4 368#define HIFN_D_DST_RSIZE 80*4
370#define HIFN_D_RES_RSIZE 24*4 369#define HIFN_D_RES_RSIZE 24*4
371 370
372#define HIFN_QUEUE_LENGTH HIFN_D_CMD_RSIZE-5 371#define HIFN_D_DST_DALIGN 4
372
373#define HIFN_QUEUE_LENGTH HIFN_D_CMD_RSIZE-1
373 374
374#define AES_MIN_KEY_SIZE 16 375#define AES_MIN_KEY_SIZE 16
375#define AES_MAX_KEY_SIZE 32 376#define AES_MAX_KEY_SIZE 32
@@ -535,10 +536,10 @@ struct hifn_crypt_command
535 */ 536 */
536struct hifn_mac_command 537struct hifn_mac_command
537{ 538{
538 volatile u16 masks; 539 volatile __le16 masks;
539 volatile u16 header_skip; 540 volatile __le16 header_skip;
540 volatile u16 source_count; 541 volatile __le16 source_count;
541 volatile u16 reserved; 542 volatile __le16 reserved;
542}; 543};
543 544
544#define HIFN_MAC_CMD_ALG_MASK 0x0001 545#define HIFN_MAC_CMD_ALG_MASK 0x0001
@@ -564,10 +565,10 @@ struct hifn_mac_command
564 565
565struct hifn_comp_command 566struct hifn_comp_command
566{ 567{
567 volatile u16 masks; 568 volatile __le16 masks;
568 volatile u16 header_skip; 569 volatile __le16 header_skip;
569 volatile u16 source_count; 570 volatile __le16 source_count;
570 volatile u16 reserved; 571 volatile __le16 reserved;
571}; 572};
572 573
573#define HIFN_COMP_CMD_SRCLEN_M 0xc000 574#define HIFN_COMP_CMD_SRCLEN_M 0xc000
@@ -583,10 +584,10 @@ struct hifn_comp_command
583 584
584struct hifn_base_result 585struct hifn_base_result
585{ 586{
586 volatile u16 flags; 587 volatile __le16 flags;
587 volatile u16 session; 588 volatile __le16 session;
588 volatile u16 src_cnt; /* 15:0 of source count */ 589 volatile __le16 src_cnt; /* 15:0 of source count */
589 volatile u16 dst_cnt; /* 15:0 of dest count */ 590 volatile __le16 dst_cnt; /* 15:0 of dest count */
590}; 591};
591 592
592#define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */ 593#define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */
@@ -597,8 +598,8 @@ struct hifn_base_result
597 598
598struct hifn_comp_result 599struct hifn_comp_result
599{ 600{
600 volatile u16 flags; 601 volatile __le16 flags;
601 volatile u16 crc; 602 volatile __le16 crc;
602}; 603};
603 604
604#define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */ 605#define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */
@@ -609,8 +610,8 @@ struct hifn_comp_result
609 610
610struct hifn_mac_result 611struct hifn_mac_result
611{ 612{
612 volatile u16 flags; 613 volatile __le16 flags;
613 volatile u16 reserved; 614 volatile __le16 reserved;
614 /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */ 615 /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */
615}; 616};
616 617
@@ -619,8 +620,8 @@ struct hifn_mac_result
619 620
620struct hifn_crypt_result 621struct hifn_crypt_result
621{ 622{
622 volatile u16 flags; 623 volatile __le16 flags;
623 volatile u16 reserved; 624 volatile __le16 reserved;
624}; 625};
625 626
626#define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */ 627#define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */
@@ -686,12 +687,12 @@ static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg)
686 687
687static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val) 688static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
688{ 689{
689 writel(val, dev->bar[0] + reg); 690 writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg);
690} 691}
691 692
692static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val) 693static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)
693{ 694{
694 writel(val, dev->bar[1] + reg); 695 writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg);
695} 696}
696 697
697static void hifn_wait_puc(struct hifn_device *dev) 698static void hifn_wait_puc(struct hifn_device *dev)
@@ -894,7 +895,7 @@ static int hifn_enable_crypto(struct hifn_device *dev)
894 char *offtbl = NULL; 895 char *offtbl = NULL;
895 int i; 896 int i;
896 897
897 for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) { 898 for (i = 0; i < ARRAY_SIZE(pci2id); i++) {
898 if (pci2id[i].pci_vendor == dev->pdev->vendor && 899 if (pci2id[i].pci_vendor == dev->pdev->vendor &&
899 pci2id[i].pci_prod == dev->pdev->device) { 900 pci2id[i].pci_prod == dev->pdev->device) {
900 offtbl = pci2id[i].card_id; 901 offtbl = pci2id[i].card_id;
@@ -1037,14 +1038,14 @@ static void hifn_init_registers(struct hifn_device *dev)
1037 hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); 1038 hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
1038 1039
1039 /* write all 4 ring address registers */ 1040 /* write all 4 ring address registers */
1040 hifn_write_1(dev, HIFN_1_DMA_CRAR, __cpu_to_le32(dptr + 1041 hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr +
1041 offsetof(struct hifn_dma, cmdr[0]))); 1042 offsetof(struct hifn_dma, cmdr[0]));
1042 hifn_write_1(dev, HIFN_1_DMA_SRAR, __cpu_to_le32(dptr + 1043 hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr +
1043 offsetof(struct hifn_dma, srcr[0]))); 1044 offsetof(struct hifn_dma, srcr[0]));
1044 hifn_write_1(dev, HIFN_1_DMA_DRAR, __cpu_to_le32(dptr + 1045 hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr +
1045 offsetof(struct hifn_dma, dstr[0]))); 1046 offsetof(struct hifn_dma, dstr[0]));
1046 hifn_write_1(dev, HIFN_1_DMA_RRAR, __cpu_to_le32(dptr + 1047 hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr +
1047 offsetof(struct hifn_dma, resr[0]))); 1048 offsetof(struct hifn_dma, resr[0]));
1048 1049
1049 mdelay(2); 1050 mdelay(2);
1050#if 0 1051#if 0
@@ -1166,109 +1167,15 @@ static int hifn_setup_crypto_command(struct hifn_device *dev,
1166 return cmd_len; 1167 return cmd_len;
1167} 1168}
1168 1169
1169static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, 1170static int hifn_setup_cmd_desc(struct hifn_device *dev,
1170 unsigned int offset, unsigned int size) 1171 struct hifn_context *ctx, void *priv, unsigned int nbytes)
1171{
1172 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1173 int idx;
1174 dma_addr_t addr;
1175
1176 addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
1177
1178 idx = dma->srci;
1179
1180 dma->srcr[idx].p = __cpu_to_le32(addr);
1181 dma->srcr[idx].l = __cpu_to_le32(size) | HIFN_D_VALID |
1182 HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST;
1183
1184 if (++idx == HIFN_D_SRC_RSIZE) {
1185 dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1186 HIFN_D_JUMP |
1187 HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1188 idx = 0;
1189 }
1190
1191 dma->srci = idx;
1192 dma->srcu++;
1193
1194 if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
1195 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1196 dev->flags |= HIFN_FLAG_SRC_BUSY;
1197 }
1198
1199 return size;
1200}
1201
1202static void hifn_setup_res_desc(struct hifn_device *dev)
1203{
1204 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1205
1206 dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
1207 HIFN_D_VALID | HIFN_D_LAST);
1208 /*
1209 * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
1210 * HIFN_D_LAST | HIFN_D_NOINVALID);
1211 */
1212
1213 if (++dma->resi == HIFN_D_RES_RSIZE) {
1214 dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
1215 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1216 dma->resi = 0;
1217 }
1218
1219 dma->resu++;
1220
1221 if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
1222 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1223 dev->flags |= HIFN_FLAG_RES_BUSY;
1224 }
1225}
1226
1227static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
1228 unsigned offset, unsigned size)
1229{
1230 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1231 int idx;
1232 dma_addr_t addr;
1233
1234 addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
1235
1236 idx = dma->dsti;
1237 dma->dstr[idx].p = __cpu_to_le32(addr);
1238 dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
1239 HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST);
1240
1241 if (++idx == HIFN_D_DST_RSIZE) {
1242 dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1243 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1244 HIFN_D_LAST | HIFN_D_NOINVALID);
1245 idx = 0;
1246 }
1247 dma->dsti = idx;
1248 dma->dstu++;
1249
1250 if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
1251 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1252 dev->flags |= HIFN_FLAG_DST_BUSY;
1253 }
1254}
1255
1256static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned int soff,
1257 struct page *dpage, unsigned int doff, unsigned int nbytes, void *priv,
1258 struct hifn_context *ctx)
1259{ 1172{
1260 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; 1173 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1261 int cmd_len, sa_idx; 1174 int cmd_len, sa_idx;
1262 u8 *buf, *buf_pos; 1175 u8 *buf, *buf_pos;
1263 u16 mask; 1176 u16 mask;
1264 1177
1265 dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, nbytes: %u, priv: %p, ctx: %p.\n", 1178 sa_idx = dma->cmdi;
1266 dev->name, spage, soff, dpage, doff, nbytes, priv, ctx);
1267
1268 sa_idx = dma->resi;
1269
1270 hifn_setup_src_desc(dev, spage, soff, nbytes);
1271
1272 buf_pos = buf = dma->command_bufs[dma->cmdi]; 1179 buf_pos = buf = dma->command_bufs[dma->cmdi];
1273 1180
1274 mask = 0; 1181 mask = 0;
@@ -1370,16 +1277,113 @@ static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned
1370 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); 1277 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
1371 dev->flags |= HIFN_FLAG_CMD_BUSY; 1278 dev->flags |= HIFN_FLAG_CMD_BUSY;
1372 } 1279 }
1373
1374 hifn_setup_dst_desc(dev, dpage, doff, nbytes);
1375 hifn_setup_res_desc(dev);
1376
1377 return 0; 1280 return 0;
1378 1281
1379err_out: 1282err_out:
1380 return -EINVAL; 1283 return -EINVAL;
1381} 1284}
1382 1285
1286static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
1287 unsigned int offset, unsigned int size)
1288{
1289 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1290 int idx;
1291 dma_addr_t addr;
1292
1293 addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
1294
1295 idx = dma->srci;
1296
1297 dma->srcr[idx].p = __cpu_to_le32(addr);
1298 dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
1299 HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1300
1301 if (++idx == HIFN_D_SRC_RSIZE) {
1302 dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1303 HIFN_D_JUMP |
1304 HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1305 idx = 0;
1306 }
1307
1308 dma->srci = idx;
1309 dma->srcu++;
1310
1311 if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
1312 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1313 dev->flags |= HIFN_FLAG_SRC_BUSY;
1314 }
1315
1316 return size;
1317}
1318
1319static void hifn_setup_res_desc(struct hifn_device *dev)
1320{
1321 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1322
1323 dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
1324 HIFN_D_VALID | HIFN_D_LAST);
1325 /*
1326 * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
1327 * HIFN_D_LAST);
1328 */
1329
1330 if (++dma->resi == HIFN_D_RES_RSIZE) {
1331 dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
1332 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1333 dma->resi = 0;
1334 }
1335
1336 dma->resu++;
1337
1338 if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
1339 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1340 dev->flags |= HIFN_FLAG_RES_BUSY;
1341 }
1342}
1343
1344static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
1345 unsigned offset, unsigned size)
1346{
1347 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1348 int idx;
1349 dma_addr_t addr;
1350
1351 addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
1352
1353 idx = dma->dsti;
1354 dma->dstr[idx].p = __cpu_to_le32(addr);
1355 dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
1356 HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1357
1358 if (++idx == HIFN_D_DST_RSIZE) {
1359 dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1360 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1361 HIFN_D_LAST);
1362 idx = 0;
1363 }
1364 dma->dsti = idx;
1365 dma->dstu++;
1366
1367 if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
1368 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1369 dev->flags |= HIFN_FLAG_DST_BUSY;
1370 }
1371}
1372
1373static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned int soff,
1374 struct page *dpage, unsigned int doff, unsigned int nbytes, void *priv,
1375 struct hifn_context *ctx)
1376{
1377 dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, nbytes: %u, priv: %p, ctx: %p.\n",
1378 dev->name, spage, soff, dpage, doff, nbytes, priv, ctx);
1379
1380 hifn_setup_src_desc(dev, spage, soff, nbytes);
1381 hifn_setup_cmd_desc(dev, ctx, priv, nbytes);
1382 hifn_setup_dst_desc(dev, dpage, doff, nbytes);
1383 hifn_setup_res_desc(dev);
1384 return 0;
1385}
1386
1383static int ablkcipher_walk_init(struct ablkcipher_walk *w, 1387static int ablkcipher_walk_init(struct ablkcipher_walk *w,
1384 int num, gfp_t gfp_flags) 1388 int num, gfp_t gfp_flags)
1385{ 1389{
@@ -1431,7 +1435,7 @@ static int ablkcipher_add(void *daddr, unsigned int *drestp, struct scatterlist
1431 return -EINVAL; 1435 return -EINVAL;
1432 1436
1433 while (size) { 1437 while (size) {
1434 copy = min(drest, src->length); 1438 copy = min(drest, min(size, src->length));
1435 1439
1436 saddr = kmap_atomic(sg_page(src), KM_SOFTIRQ1); 1440 saddr = kmap_atomic(sg_page(src), KM_SOFTIRQ1);
1437 memcpy(daddr, saddr + src->offset, copy); 1441 memcpy(daddr, saddr + src->offset, copy);
@@ -1458,10 +1462,6 @@ static int ablkcipher_add(void *daddr, unsigned int *drestp, struct scatterlist
1458static int ablkcipher_walk(struct ablkcipher_request *req, 1462static int ablkcipher_walk(struct ablkcipher_request *req,
1459 struct ablkcipher_walk *w) 1463 struct ablkcipher_walk *w)
1460{ 1464{
1461 unsigned blocksize =
1462 crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
1463 unsigned alignmask =
1464 crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
1465 struct scatterlist *src, *dst, *t; 1465 struct scatterlist *src, *dst, *t;
1466 void *daddr; 1466 void *daddr;
1467 unsigned int nbytes = req->nbytes, offset, copy, diff; 1467 unsigned int nbytes = req->nbytes, offset, copy, diff;
@@ -1477,16 +1477,14 @@ static int ablkcipher_walk(struct ablkcipher_request *req,
1477 dst = &req->dst[idx]; 1477 dst = &req->dst[idx];
1478 1478
1479 dprintk("\n%s: slen: %u, dlen: %u, soff: %u, doff: %u, offset: %u, " 1479 dprintk("\n%s: slen: %u, dlen: %u, soff: %u, doff: %u, offset: %u, "
1480 "blocksize: %u, nbytes: %u.\n", 1480 "nbytes: %u.\n",
1481 __func__, src->length, dst->length, src->offset, 1481 __func__, src->length, dst->length, src->offset,
1482 dst->offset, offset, blocksize, nbytes); 1482 dst->offset, offset, nbytes);
1483 1483
1484 if (src->length & (blocksize - 1) || 1484 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
1485 src->offset & (alignmask - 1) || 1485 !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) ||
1486 dst->length & (blocksize - 1) || 1486 offset) {
1487 dst->offset & (alignmask - 1) || 1487 unsigned slen = min(src->length - offset, nbytes);
1488 offset) {
1489 unsigned slen = src->length - offset;
1490 unsigned dlen = PAGE_SIZE; 1488 unsigned dlen = PAGE_SIZE;
1491 1489
1492 t = &w->cache[idx]; 1490 t = &w->cache[idx];
@@ -1498,8 +1496,8 @@ static int ablkcipher_walk(struct ablkcipher_request *req,
1498 1496
1499 idx += err; 1497 idx += err;
1500 1498
1501 copy = slen & ~(blocksize - 1); 1499 copy = slen & ~(HIFN_D_DST_DALIGN - 1);
1502 diff = slen & (blocksize - 1); 1500 diff = slen & (HIFN_D_DST_DALIGN - 1);
1503 1501
1504 if (dlen < nbytes) { 1502 if (dlen < nbytes) {
1505 /* 1503 /*
@@ -1507,7 +1505,7 @@ static int ablkcipher_walk(struct ablkcipher_request *req,
1507 * to put there additional blocksized chunk, 1505 * to put there additional blocksized chunk,
1508 * so we mark that page as containing only 1506 * so we mark that page as containing only
1509 * blocksize aligned chunks: 1507 * blocksize aligned chunks:
1510 * t->length = (slen & ~(blocksize - 1)); 1508 * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1));
1511 * and increase number of bytes to be processed 1509 * and increase number of bytes to be processed
1512 * in next chunk: 1510 * in next chunk:
1513 * nbytes += diff; 1511 * nbytes += diff;
@@ -1544,7 +1542,7 @@ static int ablkcipher_walk(struct ablkcipher_request *req,
1544 1542
1545 kunmap_atomic(daddr, KM_SOFTIRQ0); 1543 kunmap_atomic(daddr, KM_SOFTIRQ0);
1546 } else { 1544 } else {
1547 nbytes -= src->length; 1545 nbytes -= min(src->length, nbytes);
1548 idx++; 1546 idx++;
1549 } 1547 }
1550 1548
@@ -1563,14 +1561,10 @@ static int hifn_setup_session(struct ablkcipher_request *req)
1563 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); 1561 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1564 struct hifn_device *dev = ctx->dev; 1562 struct hifn_device *dev = ctx->dev;
1565 struct page *spage, *dpage; 1563 struct page *spage, *dpage;
1566 unsigned long soff, doff, flags; 1564 unsigned long soff, doff, dlen, flags;
1567 unsigned int nbytes = req->nbytes, idx = 0, len; 1565 unsigned int nbytes = req->nbytes, idx = 0, len;
1568 int err = -EINVAL, sg_num; 1566 int err = -EINVAL, sg_num;
1569 struct scatterlist *src, *dst, *t; 1567 struct scatterlist *src, *dst, *t;
1570 unsigned blocksize =
1571 crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
1572 unsigned alignmask =
1573 crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
1574 1568
1575 if (ctx->iv && !ctx->ivsize && ctx->mode != ACRYPTO_MODE_ECB) 1569 if (ctx->iv && !ctx->ivsize && ctx->mode != ACRYPTO_MODE_ECB)
1576 goto err_out_exit; 1570 goto err_out_exit;
@@ -1578,17 +1572,14 @@ static int hifn_setup_session(struct ablkcipher_request *req)
1578 ctx->walk.flags = 0; 1572 ctx->walk.flags = 0;
1579 1573
1580 while (nbytes) { 1574 while (nbytes) {
1581 src = &req->src[idx];
1582 dst = &req->dst[idx]; 1575 dst = &req->dst[idx];
1576 dlen = min(dst->length, nbytes);
1583 1577
1584 if (src->length & (blocksize - 1) || 1578 if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
1585 src->offset & (alignmask - 1) || 1579 !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN))
1586 dst->length & (blocksize - 1) ||
1587 dst->offset & (alignmask - 1)) {
1588 ctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; 1580 ctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
1589 }
1590 1581
1591 nbytes -= src->length; 1582 nbytes -= dlen;
1592 idx++; 1583 idx++;
1593 } 1584 }
1594 1585
@@ -1602,7 +1593,10 @@ static int hifn_setup_session(struct ablkcipher_request *req)
1602 idx = 0; 1593 idx = 0;
1603 1594
1604 sg_num = ablkcipher_walk(req, &ctx->walk); 1595 sg_num = ablkcipher_walk(req, &ctx->walk);
1605 1596 if (sg_num < 0) {
1597 err = sg_num;
1598 goto err_out_exit;
1599 }
1606 atomic_set(&ctx->sg_num, sg_num); 1600 atomic_set(&ctx->sg_num, sg_num);
1607 1601
1608 spin_lock_irqsave(&dev->lock, flags); 1602 spin_lock_irqsave(&dev->lock, flags);
@@ -1640,7 +1634,7 @@ static int hifn_setup_session(struct ablkcipher_request *req)
1640 if (err) 1634 if (err)
1641 goto err_out; 1635 goto err_out;
1642 1636
1643 nbytes -= len; 1637 nbytes -= min(len, nbytes);
1644 } 1638 }
1645 1639
1646 dev->active = HIFN_DEFAULT_ACTIVE_NUM; 1640 dev->active = HIFN_DEFAULT_ACTIVE_NUM;
@@ -1651,7 +1645,7 @@ static int hifn_setup_session(struct ablkcipher_request *req)
1651err_out: 1645err_out:
1652 spin_unlock_irqrestore(&dev->lock, flags); 1646 spin_unlock_irqrestore(&dev->lock, flags);
1653err_out_exit: 1647err_out_exit:
1654 if (err && printk_ratelimit()) 1648 if (err)
1655 dprintk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, " 1649 dprintk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
1656 "type: %u, err: %d.\n", 1650 "type: %u, err: %d.\n",
1657 dev->name, ctx->iv, ctx->ivsize, 1651 dev->name, ctx->iv, ctx->ivsize,
@@ -1745,8 +1739,7 @@ static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset
1745 return -EINVAL; 1739 return -EINVAL;
1746 1740
1747 while (size) { 1741 while (size) {
1748 1742 copy = min(srest, min(dst->length, size));
1749 copy = min(dst->length, srest);
1750 1743
1751 daddr = kmap_atomic(sg_page(dst), KM_IRQ0); 1744 daddr = kmap_atomic(sg_page(dst), KM_IRQ0);
1752 memcpy(daddr + dst->offset + offset, saddr, copy); 1745 memcpy(daddr + dst->offset + offset, saddr, copy);
@@ -1803,7 +1796,7 @@ static void hifn_process_ready(struct ablkcipher_request *req, int error)
1803 sg_page(dst), dst->length, nbytes); 1796 sg_page(dst), dst->length, nbytes);
1804 1797
1805 if (!t->length) { 1798 if (!t->length) {
1806 nbytes -= dst->length; 1799 nbytes -= min(dst->length, nbytes);
1807 idx++; 1800 idx++;
1808 continue; 1801 continue;
1809 } 1802 }
@@ -2202,9 +2195,9 @@ static int hifn_setup_crypto(struct ablkcipher_request *req, u8 op,
2202 return err; 2195 return err;
2203 2196
2204 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) 2197 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
2205 err = hifn_process_queue(dev); 2198 hifn_process_queue(dev);
2206 2199
2207 return err; 2200 return -EINPROGRESS;
2208} 2201}
2209 2202
2210/* 2203/*
@@ -2364,7 +2357,7 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2364 * 3DES ECB, CBC, CFB and OFB modes. 2357 * 3DES ECB, CBC, CFB and OFB modes.
2365 */ 2358 */
2366 { 2359 {
2367 .name = "cfb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, 2360 .name = "cfb(des3_ede)", .drv_name = "cfb-3des", .bsize = 8,
2368 .ablkcipher = { 2361 .ablkcipher = {
2369 .min_keysize = HIFN_3DES_KEY_LENGTH, 2362 .min_keysize = HIFN_3DES_KEY_LENGTH,
2370 .max_keysize = HIFN_3DES_KEY_LENGTH, 2363 .max_keysize = HIFN_3DES_KEY_LENGTH,
@@ -2374,7 +2367,7 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2374 }, 2367 },
2375 }, 2368 },
2376 { 2369 {
2377 .name = "ofb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, 2370 .name = "ofb(des3_ede)", .drv_name = "ofb-3des", .bsize = 8,
2378 .ablkcipher = { 2371 .ablkcipher = {
2379 .min_keysize = HIFN_3DES_KEY_LENGTH, 2372 .min_keysize = HIFN_3DES_KEY_LENGTH,
2380 .max_keysize = HIFN_3DES_KEY_LENGTH, 2373 .max_keysize = HIFN_3DES_KEY_LENGTH,
@@ -2384,8 +2377,9 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2384 }, 2377 },
2385 }, 2378 },
2386 { 2379 {
2387 .name = "cbc(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, 2380 .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8,
2388 .ablkcipher = { 2381 .ablkcipher = {
2382 .ivsize = HIFN_IV_LENGTH,
2389 .min_keysize = HIFN_3DES_KEY_LENGTH, 2383 .min_keysize = HIFN_3DES_KEY_LENGTH,
2390 .max_keysize = HIFN_3DES_KEY_LENGTH, 2384 .max_keysize = HIFN_3DES_KEY_LENGTH,
2391 .setkey = hifn_setkey, 2385 .setkey = hifn_setkey,
@@ -2394,7 +2388,7 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2394 }, 2388 },
2395 }, 2389 },
2396 { 2390 {
2397 .name = "ecb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, 2391 .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8,
2398 .ablkcipher = { 2392 .ablkcipher = {
2399 .min_keysize = HIFN_3DES_KEY_LENGTH, 2393 .min_keysize = HIFN_3DES_KEY_LENGTH,
2400 .max_keysize = HIFN_3DES_KEY_LENGTH, 2394 .max_keysize = HIFN_3DES_KEY_LENGTH,
@@ -2408,7 +2402,7 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2408 * DES ECB, CBC, CFB and OFB modes. 2402 * DES ECB, CBC, CFB and OFB modes.
2409 */ 2403 */
2410 { 2404 {
2411 .name = "cfb(des)", .drv_name = "hifn-des", .bsize = 8, 2405 .name = "cfb(des)", .drv_name = "cfb-des", .bsize = 8,
2412 .ablkcipher = { 2406 .ablkcipher = {
2413 .min_keysize = HIFN_DES_KEY_LENGTH, 2407 .min_keysize = HIFN_DES_KEY_LENGTH,
2414 .max_keysize = HIFN_DES_KEY_LENGTH, 2408 .max_keysize = HIFN_DES_KEY_LENGTH,
@@ -2418,7 +2412,7 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2418 }, 2412 },
2419 }, 2413 },
2420 { 2414 {
2421 .name = "ofb(des)", .drv_name = "hifn-des", .bsize = 8, 2415 .name = "ofb(des)", .drv_name = "ofb-des", .bsize = 8,
2422 .ablkcipher = { 2416 .ablkcipher = {
2423 .min_keysize = HIFN_DES_KEY_LENGTH, 2417 .min_keysize = HIFN_DES_KEY_LENGTH,
2424 .max_keysize = HIFN_DES_KEY_LENGTH, 2418 .max_keysize = HIFN_DES_KEY_LENGTH,
@@ -2428,8 +2422,9 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2428 }, 2422 },
2429 }, 2423 },
2430 { 2424 {
2431 .name = "cbc(des)", .drv_name = "hifn-des", .bsize = 8, 2425 .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8,
2432 .ablkcipher = { 2426 .ablkcipher = {
2427 .ivsize = HIFN_IV_LENGTH,
2433 .min_keysize = HIFN_DES_KEY_LENGTH, 2428 .min_keysize = HIFN_DES_KEY_LENGTH,
2434 .max_keysize = HIFN_DES_KEY_LENGTH, 2429 .max_keysize = HIFN_DES_KEY_LENGTH,
2435 .setkey = hifn_setkey, 2430 .setkey = hifn_setkey,
@@ -2438,7 +2433,7 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2438 }, 2433 },
2439 }, 2434 },
2440 { 2435 {
2441 .name = "ecb(des)", .drv_name = "hifn-des", .bsize = 8, 2436 .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8,
2442 .ablkcipher = { 2437 .ablkcipher = {
2443 .min_keysize = HIFN_DES_KEY_LENGTH, 2438 .min_keysize = HIFN_DES_KEY_LENGTH,
2444 .max_keysize = HIFN_DES_KEY_LENGTH, 2439 .max_keysize = HIFN_DES_KEY_LENGTH,
@@ -2452,7 +2447,7 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2452 * AES ECB, CBC, CFB and OFB modes. 2447 * AES ECB, CBC, CFB and OFB modes.
2453 */ 2448 */
2454 { 2449 {
2455 .name = "ecb(aes)", .drv_name = "hifn-aes", .bsize = 16, 2450 .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16,
2456 .ablkcipher = { 2451 .ablkcipher = {
2457 .min_keysize = AES_MIN_KEY_SIZE, 2452 .min_keysize = AES_MIN_KEY_SIZE,
2458 .max_keysize = AES_MAX_KEY_SIZE, 2453 .max_keysize = AES_MAX_KEY_SIZE,
@@ -2462,8 +2457,9 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2462 }, 2457 },
2463 }, 2458 },
2464 { 2459 {
2465 .name = "cbc(aes)", .drv_name = "hifn-aes", .bsize = 16, 2460 .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16,
2466 .ablkcipher = { 2461 .ablkcipher = {
2462 .ivsize = HIFN_AES_IV_LENGTH,
2467 .min_keysize = AES_MIN_KEY_SIZE, 2463 .min_keysize = AES_MIN_KEY_SIZE,
2468 .max_keysize = AES_MAX_KEY_SIZE, 2464 .max_keysize = AES_MAX_KEY_SIZE,
2469 .setkey = hifn_setkey, 2465 .setkey = hifn_setkey,
@@ -2472,7 +2468,7 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2472 }, 2468 },
2473 }, 2469 },
2474 { 2470 {
2475 .name = "cfb(aes)", .drv_name = "hifn-aes", .bsize = 16, 2471 .name = "cfb(aes)", .drv_name = "cfb-aes", .bsize = 16,
2476 .ablkcipher = { 2472 .ablkcipher = {
2477 .min_keysize = AES_MIN_KEY_SIZE, 2473 .min_keysize = AES_MIN_KEY_SIZE,
2478 .max_keysize = AES_MAX_KEY_SIZE, 2474 .max_keysize = AES_MAX_KEY_SIZE,
@@ -2482,7 +2478,7 @@ static struct hifn_alg_template hifn_alg_templates[] = {
2482 }, 2478 },
2483 }, 2479 },
2484 { 2480 {
2485 .name = "ofb(aes)", .drv_name = "hifn-aes", .bsize = 16, 2481 .name = "ofb(aes)", .drv_name = "ofb-aes", .bsize = 16,
2486 .ablkcipher = { 2482 .ablkcipher = {
2487 .min_keysize = AES_MIN_KEY_SIZE, 2483 .min_keysize = AES_MIN_KEY_SIZE,
2488 .max_keysize = AES_MAX_KEY_SIZE, 2484 .max_keysize = AES_MAX_KEY_SIZE,
@@ -2514,15 +2510,14 @@ static int hifn_alg_alloc(struct hifn_device *dev, struct hifn_alg_template *t)
2514 return -ENOMEM; 2510 return -ENOMEM;
2515 2511
2516 snprintf(alg->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name); 2512 snprintf(alg->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name);
2517 snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", t->drv_name); 2513 snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-%s",
2514 t->drv_name, dev->name);
2518 2515
2519 alg->alg.cra_priority = 300; 2516 alg->alg.cra_priority = 300;
2520 alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; 2517 alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
2521 alg->alg.cra_blocksize = t->bsize; 2518 alg->alg.cra_blocksize = t->bsize;
2522 alg->alg.cra_ctxsize = sizeof(struct hifn_context); 2519 alg->alg.cra_ctxsize = sizeof(struct hifn_context);
2523 alg->alg.cra_alignmask = 15; 2520 alg->alg.cra_alignmask = 0;
2524 if (t->bsize == 8)
2525 alg->alg.cra_alignmask = 3;
2526 alg->alg.cra_type = &crypto_ablkcipher_type; 2521 alg->alg.cra_type = &crypto_ablkcipher_type;
2527 alg->alg.cra_module = THIS_MODULE; 2522 alg->alg.cra_module = THIS_MODULE;
2528 alg->alg.cra_u.ablkcipher = t->ablkcipher; 2523 alg->alg.cra_u.ablkcipher = t->ablkcipher;
diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c
new file mode 100644
index 000000000000..42a107fe9233
--- /dev/null
+++ b/drivers/crypto/ixp4xx_crypto.c
@@ -0,0 +1,1506 @@
1/*
2 * Intel IXP4xx NPE-C crypto driver
3 *
4 * Copyright (C) 2008 Christian Hohnstaedt <chohnstaedt@innominate.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of version 2 of the GNU General Public License
8 * as published by the Free Software Foundation.
9 *
10 */
11
12#include <linux/platform_device.h>
13#include <linux/dma-mapping.h>
14#include <linux/dmapool.h>
15#include <linux/crypto.h>
16#include <linux/kernel.h>
17#include <linux/rtnetlink.h>
18#include <linux/interrupt.h>
19#include <linux/spinlock.h>
20
21#include <crypto/ctr.h>
22#include <crypto/des.h>
23#include <crypto/aes.h>
24#include <crypto/sha.h>
25#include <crypto/algapi.h>
26#include <crypto/aead.h>
27#include <crypto/authenc.h>
28#include <crypto/scatterwalk.h>
29
30#include <asm/arch/npe.h>
31#include <asm/arch/qmgr.h>
32
33#define MAX_KEYLEN 32
34
35/* hash: cfgword + 2 * digestlen; crypt: keylen + cfgword */
36#define NPE_CTX_LEN 80
37#define AES_BLOCK128 16
38
39#define NPE_OP_HASH_VERIFY 0x01
40#define NPE_OP_CCM_ENABLE 0x04
41#define NPE_OP_CRYPT_ENABLE 0x08
42#define NPE_OP_HASH_ENABLE 0x10
43#define NPE_OP_NOT_IN_PLACE 0x20
44#define NPE_OP_HMAC_DISABLE 0x40
45#define NPE_OP_CRYPT_ENCRYPT 0x80
46
47#define NPE_OP_CCM_GEN_MIC 0xcc
48#define NPE_OP_HASH_GEN_ICV 0x50
49#define NPE_OP_ENC_GEN_KEY 0xc9
50
51#define MOD_ECB 0x0000
52#define MOD_CTR 0x1000
53#define MOD_CBC_ENC 0x2000
54#define MOD_CBC_DEC 0x3000
55#define MOD_CCM_ENC 0x4000
56#define MOD_CCM_DEC 0x5000
57
58#define KEYLEN_128 4
59#define KEYLEN_192 6
60#define KEYLEN_256 8
61
62#define CIPH_DECR 0x0000
63#define CIPH_ENCR 0x0400
64
65#define MOD_DES 0x0000
66#define MOD_TDEA2 0x0100
67#define MOD_3DES 0x0200
68#define MOD_AES 0x0800
69#define MOD_AES128 (0x0800 | KEYLEN_128)
70#define MOD_AES192 (0x0900 | KEYLEN_192)
71#define MOD_AES256 (0x0a00 | KEYLEN_256)
72
73#define MAX_IVLEN 16
74#define NPE_ID 2 /* NPE C */
75#define NPE_QLEN 16
76/* Space for registering when the first
77 * NPE_QLEN crypt_ctl are busy */
78#define NPE_QLEN_TOTAL 64
79
80#define SEND_QID 29
81#define RECV_QID 30
82
83#define CTL_FLAG_UNUSED 0x0000
84#define CTL_FLAG_USED 0x1000
85#define CTL_FLAG_PERFORM_ABLK 0x0001
86#define CTL_FLAG_GEN_ICV 0x0002
87#define CTL_FLAG_GEN_REVAES 0x0004
88#define CTL_FLAG_PERFORM_AEAD 0x0008
89#define CTL_FLAG_MASK 0x000f
90
91#define HMAC_IPAD_VALUE 0x36
92#define HMAC_OPAD_VALUE 0x5C
93#define HMAC_PAD_BLOCKLEN SHA1_BLOCK_SIZE
94
95#define MD5_DIGEST_SIZE 16
96
97struct buffer_desc {
98 u32 phys_next;
99 u16 buf_len;
100 u16 pkt_len;
101 u32 phys_addr;
102 u32 __reserved[4];
103 struct buffer_desc *next;
104};
105
106struct crypt_ctl {
107 u8 mode; /* NPE_OP_* operation mode */
108 u8 init_len;
109 u16 reserved;
110 u8 iv[MAX_IVLEN]; /* IV for CBC mode or CTR IV for CTR mode */
111 u32 icv_rev_aes; /* icv or rev aes */
112 u32 src_buf;
113 u32 dst_buf;
114 u16 auth_offs; /* Authentication start offset */
115 u16 auth_len; /* Authentication data length */
116 u16 crypt_offs; /* Cryption start offset */
117 u16 crypt_len; /* Cryption data length */
118 u32 aadAddr; /* Additional Auth Data Addr for CCM mode */
119 u32 crypto_ctx; /* NPE Crypto Param structure address */
120
121 /* Used by Host: 4*4 bytes*/
122 unsigned ctl_flags;
123 union {
124 struct ablkcipher_request *ablk_req;
125 struct aead_request *aead_req;
126 struct crypto_tfm *tfm;
127 } data;
128 struct buffer_desc *regist_buf;
129 u8 *regist_ptr;
130};
131
132struct ablk_ctx {
133 struct buffer_desc *src;
134 struct buffer_desc *dst;
135 unsigned src_nents;
136 unsigned dst_nents;
137};
138
139struct aead_ctx {
140 struct buffer_desc *buffer;
141 unsigned short assoc_nents;
142 unsigned short src_nents;
143 struct scatterlist ivlist;
144 /* used when the hmac is not on one sg entry */
145 u8 *hmac_virt;
146 int encrypt;
147};
148
149struct ix_hash_algo {
150 u32 cfgword;
151 unsigned char *icv;
152};
153
154struct ix_sa_dir {
155 unsigned char *npe_ctx;
156 dma_addr_t npe_ctx_phys;
157 int npe_ctx_idx;
158 u8 npe_mode;
159};
160
161struct ixp_ctx {
162 struct ix_sa_dir encrypt;
163 struct ix_sa_dir decrypt;
164 int authkey_len;
165 u8 authkey[MAX_KEYLEN];
166 int enckey_len;
167 u8 enckey[MAX_KEYLEN];
168 u8 salt[MAX_IVLEN];
169 u8 nonce[CTR_RFC3686_NONCE_SIZE];
170 unsigned salted;
171 atomic_t configuring;
172 struct completion completion;
173};
174
175struct ixp_alg {
176 struct crypto_alg crypto;
177 const struct ix_hash_algo *hash;
178 u32 cfg_enc;
179 u32 cfg_dec;
180
181 int registered;
182};
183
184static const struct ix_hash_algo hash_alg_md5 = {
185 .cfgword = 0xAA010004,
186 .icv = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
187 "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
188};
189static const struct ix_hash_algo hash_alg_sha1 = {
190 .cfgword = 0x00000005,
191 .icv = "\x67\x45\x23\x01\xEF\xCD\xAB\x89\x98\xBA"
192 "\xDC\xFE\x10\x32\x54\x76\xC3\xD2\xE1\xF0",
193};
194
195static struct npe *npe_c;
196static struct dma_pool *buffer_pool = NULL;
197static struct dma_pool *ctx_pool = NULL;
198
199static struct crypt_ctl *crypt_virt = NULL;
200static dma_addr_t crypt_phys;
201
202static int support_aes = 1;
203
204static void dev_release(struct device *dev)
205{
206 return;
207}
208
209#define DRIVER_NAME "ixp4xx_crypto"
210static struct platform_device pseudo_dev = {
211 .name = DRIVER_NAME,
212 .id = 0,
213 .num_resources = 0,
214 .dev = {
215 .coherent_dma_mask = DMA_32BIT_MASK,
216 .release = dev_release,
217 }
218};
219
220static struct device *dev = &pseudo_dev.dev;
221
222static inline dma_addr_t crypt_virt2phys(struct crypt_ctl *virt)
223{
224 return crypt_phys + (virt - crypt_virt) * sizeof(struct crypt_ctl);
225}
226
227static inline struct crypt_ctl *crypt_phys2virt(dma_addr_t phys)
228{
229 return crypt_virt + (phys - crypt_phys) / sizeof(struct crypt_ctl);
230}
231
232static inline u32 cipher_cfg_enc(struct crypto_tfm *tfm)
233{
234 return container_of(tfm->__crt_alg, struct ixp_alg,crypto)->cfg_enc;
235}
236
237static inline u32 cipher_cfg_dec(struct crypto_tfm *tfm)
238{
239 return container_of(tfm->__crt_alg, struct ixp_alg,crypto)->cfg_dec;
240}
241
242static inline const struct ix_hash_algo *ix_hash(struct crypto_tfm *tfm)
243{
244 return container_of(tfm->__crt_alg, struct ixp_alg, crypto)->hash;
245}
246
247static int setup_crypt_desc(void)
248{
249 BUILD_BUG_ON(sizeof(struct crypt_ctl) != 64);
250 crypt_virt = dma_alloc_coherent(dev,
251 NPE_QLEN * sizeof(struct crypt_ctl),
252 &crypt_phys, GFP_KERNEL);
253 if (!crypt_virt)
254 return -ENOMEM;
255 memset(crypt_virt, 0, NPE_QLEN * sizeof(struct crypt_ctl));
256 return 0;
257}
258
259static spinlock_t desc_lock;
260static struct crypt_ctl *get_crypt_desc(void)
261{
262 int i;
263 static int idx = 0;
264 unsigned long flags;
265
266 spin_lock_irqsave(&desc_lock, flags);
267
268 if (unlikely(!crypt_virt))
269 setup_crypt_desc();
270 if (unlikely(!crypt_virt)) {
271 spin_unlock_irqrestore(&desc_lock, flags);
272 return NULL;
273 }
274 i = idx;
275 if (crypt_virt[i].ctl_flags == CTL_FLAG_UNUSED) {
276 if (++idx >= NPE_QLEN)
277 idx = 0;
278 crypt_virt[i].ctl_flags = CTL_FLAG_USED;
279 spin_unlock_irqrestore(&desc_lock, flags);
280 return crypt_virt +i;
281 } else {
282 spin_unlock_irqrestore(&desc_lock, flags);
283 return NULL;
284 }
285}
286
287static spinlock_t emerg_lock;
288static struct crypt_ctl *get_crypt_desc_emerg(void)
289{
290 int i;
291 static int idx = NPE_QLEN;
292 struct crypt_ctl *desc;
293 unsigned long flags;
294
295 desc = get_crypt_desc();
296 if (desc)
297 return desc;
298 if (unlikely(!crypt_virt))
299 return NULL;
300
301 spin_lock_irqsave(&emerg_lock, flags);
302 i = idx;
303 if (crypt_virt[i].ctl_flags == CTL_FLAG_UNUSED) {
304 if (++idx >= NPE_QLEN_TOTAL)
305 idx = NPE_QLEN;
306 crypt_virt[i].ctl_flags = CTL_FLAG_USED;
307 spin_unlock_irqrestore(&emerg_lock, flags);
308 return crypt_virt +i;
309 } else {
310 spin_unlock_irqrestore(&emerg_lock, flags);
311 return NULL;
312 }
313}
314
315static void free_buf_chain(struct buffer_desc *buf, u32 phys)
316{
317 while (buf) {
318 struct buffer_desc *buf1;
319 u32 phys1;
320
321 buf1 = buf->next;
322 phys1 = buf->phys_next;
323 dma_pool_free(buffer_pool, buf, phys);
324 buf = buf1;
325 phys = phys1;
326 }
327}
328
329static struct tasklet_struct crypto_done_tasklet;
330
331static void finish_scattered_hmac(struct crypt_ctl *crypt)
332{
333 struct aead_request *req = crypt->data.aead_req;
334 struct aead_ctx *req_ctx = aead_request_ctx(req);
335 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
336 int authsize = crypto_aead_authsize(tfm);
337 int decryptlen = req->cryptlen - authsize;
338
339 if (req_ctx->encrypt) {
340 scatterwalk_map_and_copy(req_ctx->hmac_virt,
341 req->src, decryptlen, authsize, 1);
342 }
343 dma_pool_free(buffer_pool, req_ctx->hmac_virt, crypt->icv_rev_aes);
344}
345
346static void one_packet(dma_addr_t phys)
347{
348 struct crypt_ctl *crypt;
349 struct ixp_ctx *ctx;
350 int failed;
351 enum dma_data_direction src_direction = DMA_BIDIRECTIONAL;
352
353 failed = phys & 0x1 ? -EBADMSG : 0;
354 phys &= ~0x3;
355 crypt = crypt_phys2virt(phys);
356
357 switch (crypt->ctl_flags & CTL_FLAG_MASK) {
358 case CTL_FLAG_PERFORM_AEAD: {
359 struct aead_request *req = crypt->data.aead_req;
360 struct aead_ctx *req_ctx = aead_request_ctx(req);
361 dma_unmap_sg(dev, req->assoc, req_ctx->assoc_nents,
362 DMA_TO_DEVICE);
363 dma_unmap_sg(dev, &req_ctx->ivlist, 1, DMA_BIDIRECTIONAL);
364 dma_unmap_sg(dev, req->src, req_ctx->src_nents,
365 DMA_BIDIRECTIONAL);
366
367 free_buf_chain(req_ctx->buffer, crypt->src_buf);
368 if (req_ctx->hmac_virt) {
369 finish_scattered_hmac(crypt);
370 }
371 req->base.complete(&req->base, failed);
372 break;
373 }
374 case CTL_FLAG_PERFORM_ABLK: {
375 struct ablkcipher_request *req = crypt->data.ablk_req;
376 struct ablk_ctx *req_ctx = ablkcipher_request_ctx(req);
377 int nents;
378 if (req_ctx->dst) {
379 nents = req_ctx->dst_nents;
380 dma_unmap_sg(dev, req->dst, nents, DMA_FROM_DEVICE);
381 free_buf_chain(req_ctx->dst, crypt->dst_buf);
382 src_direction = DMA_TO_DEVICE;
383 }
384 nents = req_ctx->src_nents;
385 dma_unmap_sg(dev, req->src, nents, src_direction);
386 free_buf_chain(req_ctx->src, crypt->src_buf);
387 req->base.complete(&req->base, failed);
388 break;
389 }
390 case CTL_FLAG_GEN_ICV:
391 ctx = crypto_tfm_ctx(crypt->data.tfm);
392 dma_pool_free(ctx_pool, crypt->regist_ptr,
393 crypt->regist_buf->phys_addr);
394 dma_pool_free(buffer_pool, crypt->regist_buf, crypt->src_buf);
395 if (atomic_dec_and_test(&ctx->configuring))
396 complete(&ctx->completion);
397 break;
398 case CTL_FLAG_GEN_REVAES:
399 ctx = crypto_tfm_ctx(crypt->data.tfm);
400 *(u32*)ctx->decrypt.npe_ctx &= cpu_to_be32(~CIPH_ENCR);
401 if (atomic_dec_and_test(&ctx->configuring))
402 complete(&ctx->completion);
403 break;
404 default:
405 BUG();
406 }
407 crypt->ctl_flags = CTL_FLAG_UNUSED;
408}
409
410static void irqhandler(void *_unused)
411{
412 tasklet_schedule(&crypto_done_tasklet);
413}
414
415static void crypto_done_action(unsigned long arg)
416{
417 int i;
418
419 for(i=0; i<4; i++) {
420 dma_addr_t phys = qmgr_get_entry(RECV_QID);
421 if (!phys)
422 return;
423 one_packet(phys);
424 }
425 tasklet_schedule(&crypto_done_tasklet);
426}
427
428static int init_ixp_crypto(void)
429{
430 int ret = -ENODEV;
431
432 if (! ( ~(*IXP4XX_EXP_CFG2) & (IXP4XX_FEATURE_HASH |
433 IXP4XX_FEATURE_AES | IXP4XX_FEATURE_DES))) {
434 printk(KERN_ERR "ixp_crypto: No HW crypto available\n");
435 return ret;
436 }
437 npe_c = npe_request(NPE_ID);
438 if (!npe_c)
439 return ret;
440
441 if (!npe_running(npe_c)) {
442 npe_load_firmware(npe_c, npe_name(npe_c), dev);
443 }
444
445 /* buffer_pool will also be used to sometimes store the hmac,
446 * so assure it is large enough
447 */
448 BUILD_BUG_ON(SHA1_DIGEST_SIZE > sizeof(struct buffer_desc));
449 buffer_pool = dma_pool_create("buffer", dev,
450 sizeof(struct buffer_desc), 32, 0);
451 ret = -ENOMEM;
452 if (!buffer_pool) {
453 goto err;
454 }
455 ctx_pool = dma_pool_create("context", dev,
456 NPE_CTX_LEN, 16, 0);
457 if (!ctx_pool) {
458 goto err;
459 }
460 ret = qmgr_request_queue(SEND_QID, NPE_QLEN_TOTAL, 0, 0);
461 if (ret)
462 goto err;
463 ret = qmgr_request_queue(RECV_QID, NPE_QLEN, 0, 0);
464 if (ret) {
465 qmgr_release_queue(SEND_QID);
466 goto err;
467 }
468 qmgr_set_irq(RECV_QID, QUEUE_IRQ_SRC_NOT_EMPTY, irqhandler, NULL);
469 tasklet_init(&crypto_done_tasklet, crypto_done_action, 0);
470
471 qmgr_enable_irq(RECV_QID);
472 return 0;
473err:
474 if (ctx_pool)
475 dma_pool_destroy(ctx_pool);
476 if (buffer_pool)
477 dma_pool_destroy(buffer_pool);
478 npe_release(npe_c);
479 return ret;
480}
481
482static void release_ixp_crypto(void)
483{
484 qmgr_disable_irq(RECV_QID);
485 tasklet_kill(&crypto_done_tasklet);
486
487 qmgr_release_queue(SEND_QID);
488 qmgr_release_queue(RECV_QID);
489
490 dma_pool_destroy(ctx_pool);
491 dma_pool_destroy(buffer_pool);
492
493 npe_release(npe_c);
494
495 if (crypt_virt) {
496 dma_free_coherent(dev,
497 NPE_QLEN_TOTAL * sizeof( struct crypt_ctl),
498 crypt_virt, crypt_phys);
499 }
500 return;
501}
502
503static void reset_sa_dir(struct ix_sa_dir *dir)
504{
505 memset(dir->npe_ctx, 0, NPE_CTX_LEN);
506 dir->npe_ctx_idx = 0;
507 dir->npe_mode = 0;
508}
509
510static int init_sa_dir(struct ix_sa_dir *dir)
511{
512 dir->npe_ctx = dma_pool_alloc(ctx_pool, GFP_KERNEL, &dir->npe_ctx_phys);
513 if (!dir->npe_ctx) {
514 return -ENOMEM;
515 }
516 reset_sa_dir(dir);
517 return 0;
518}
519
520static void free_sa_dir(struct ix_sa_dir *dir)
521{
522 memset(dir->npe_ctx, 0, NPE_CTX_LEN);
523 dma_pool_free(ctx_pool, dir->npe_ctx, dir->npe_ctx_phys);
524}
525
526static int init_tfm(struct crypto_tfm *tfm)
527{
528 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
529 int ret;
530
531 atomic_set(&ctx->configuring, 0);
532 ret = init_sa_dir(&ctx->encrypt);
533 if (ret)
534 return ret;
535 ret = init_sa_dir(&ctx->decrypt);
536 if (ret) {
537 free_sa_dir(&ctx->encrypt);
538 }
539 return ret;
540}
541
542static int init_tfm_ablk(struct crypto_tfm *tfm)
543{
544 tfm->crt_ablkcipher.reqsize = sizeof(struct ablk_ctx);
545 return init_tfm(tfm);
546}
547
548static int init_tfm_aead(struct crypto_tfm *tfm)
549{
550 tfm->crt_aead.reqsize = sizeof(struct aead_ctx);
551 return init_tfm(tfm);
552}
553
554static void exit_tfm(struct crypto_tfm *tfm)
555{
556 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
557 free_sa_dir(&ctx->encrypt);
558 free_sa_dir(&ctx->decrypt);
559}
560
561static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target,
562 int init_len, u32 ctx_addr, const u8 *key, int key_len)
563{
564 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
565 struct crypt_ctl *crypt;
566 struct buffer_desc *buf;
567 int i;
568 u8 *pad;
569 u32 pad_phys, buf_phys;
570
571 BUILD_BUG_ON(NPE_CTX_LEN < HMAC_PAD_BLOCKLEN);
572 pad = dma_pool_alloc(ctx_pool, GFP_KERNEL, &pad_phys);
573 if (!pad)
574 return -ENOMEM;
575 buf = dma_pool_alloc(buffer_pool, GFP_KERNEL, &buf_phys);
576 if (!buf) {
577 dma_pool_free(ctx_pool, pad, pad_phys);
578 return -ENOMEM;
579 }
580 crypt = get_crypt_desc_emerg();
581 if (!crypt) {
582 dma_pool_free(ctx_pool, pad, pad_phys);
583 dma_pool_free(buffer_pool, buf, buf_phys);
584 return -EAGAIN;
585 }
586
587 memcpy(pad, key, key_len);
588 memset(pad + key_len, 0, HMAC_PAD_BLOCKLEN - key_len);
589 for (i = 0; i < HMAC_PAD_BLOCKLEN; i++) {
590 pad[i] ^= xpad;
591 }
592
593 crypt->data.tfm = tfm;
594 crypt->regist_ptr = pad;
595 crypt->regist_buf = buf;
596
597 crypt->auth_offs = 0;
598 crypt->auth_len = HMAC_PAD_BLOCKLEN;
599 crypt->crypto_ctx = ctx_addr;
600 crypt->src_buf = buf_phys;
601 crypt->icv_rev_aes = target;
602 crypt->mode = NPE_OP_HASH_GEN_ICV;
603 crypt->init_len = init_len;
604 crypt->ctl_flags |= CTL_FLAG_GEN_ICV;
605
606 buf->next = 0;
607 buf->buf_len = HMAC_PAD_BLOCKLEN;
608 buf->pkt_len = 0;
609 buf->phys_addr = pad_phys;
610
611 atomic_inc(&ctx->configuring);
612 qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
613 BUG_ON(qmgr_stat_overflow(SEND_QID));
614 return 0;
615}
616
617static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned authsize,
618 const u8 *key, int key_len, unsigned digest_len)
619{
620 u32 itarget, otarget, npe_ctx_addr;
621 unsigned char *cinfo;
622 int init_len, ret = 0;
623 u32 cfgword;
624 struct ix_sa_dir *dir;
625 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
626 const struct ix_hash_algo *algo;
627
628 dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
629 cinfo = dir->npe_ctx + dir->npe_ctx_idx;
630 algo = ix_hash(tfm);
631
632 /* write cfg word to cryptinfo */
633 cfgword = algo->cfgword | ( authsize << 6); /* (authsize/4) << 8 */
634 *(u32*)cinfo = cpu_to_be32(cfgword);
635 cinfo += sizeof(cfgword);
636
637 /* write ICV to cryptinfo */
638 memcpy(cinfo, algo->icv, digest_len);
639 cinfo += digest_len;
640
641 itarget = dir->npe_ctx_phys + dir->npe_ctx_idx
642 + sizeof(algo->cfgword);
643 otarget = itarget + digest_len;
644 init_len = cinfo - (dir->npe_ctx + dir->npe_ctx_idx);
645 npe_ctx_addr = dir->npe_ctx_phys + dir->npe_ctx_idx;
646
647 dir->npe_ctx_idx += init_len;
648 dir->npe_mode |= NPE_OP_HASH_ENABLE;
649
650 if (!encrypt)
651 dir->npe_mode |= NPE_OP_HASH_VERIFY;
652
653 ret = register_chain_var(tfm, HMAC_OPAD_VALUE, otarget,
654 init_len, npe_ctx_addr, key, key_len);
655 if (ret)
656 return ret;
657 return register_chain_var(tfm, HMAC_IPAD_VALUE, itarget,
658 init_len, npe_ctx_addr, key, key_len);
659}
660
661static int gen_rev_aes_key(struct crypto_tfm *tfm)
662{
663 struct crypt_ctl *crypt;
664 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
665 struct ix_sa_dir *dir = &ctx->decrypt;
666
667 crypt = get_crypt_desc_emerg();
668 if (!crypt) {
669 return -EAGAIN;
670 }
671 *(u32*)dir->npe_ctx |= cpu_to_be32(CIPH_ENCR);
672
673 crypt->data.tfm = tfm;
674 crypt->crypt_offs = 0;
675 crypt->crypt_len = AES_BLOCK128;
676 crypt->src_buf = 0;
677 crypt->crypto_ctx = dir->npe_ctx_phys;
678 crypt->icv_rev_aes = dir->npe_ctx_phys + sizeof(u32);
679 crypt->mode = NPE_OP_ENC_GEN_KEY;
680 crypt->init_len = dir->npe_ctx_idx;
681 crypt->ctl_flags |= CTL_FLAG_GEN_REVAES;
682
683 atomic_inc(&ctx->configuring);
684 qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
685 BUG_ON(qmgr_stat_overflow(SEND_QID));
686 return 0;
687}
688
689static int setup_cipher(struct crypto_tfm *tfm, int encrypt,
690 const u8 *key, int key_len)
691{
692 u8 *cinfo;
693 u32 cipher_cfg;
694 u32 keylen_cfg = 0;
695 struct ix_sa_dir *dir;
696 struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
697 u32 *flags = &tfm->crt_flags;
698
699 dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
700 cinfo = dir->npe_ctx;
701
702 if (encrypt) {
703 cipher_cfg = cipher_cfg_enc(tfm);
704 dir->npe_mode |= NPE_OP_CRYPT_ENCRYPT;
705 } else {
706 cipher_cfg = cipher_cfg_dec(tfm);
707 }
708 if (cipher_cfg & MOD_AES) {
709 switch (key_len) {
710 case 16: keylen_cfg = MOD_AES128 | KEYLEN_128; break;
711 case 24: keylen_cfg = MOD_AES192 | KEYLEN_192; break;
712 case 32: keylen_cfg = MOD_AES256 | KEYLEN_256; break;
713 default:
714 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
715 return -EINVAL;
716 }
717 cipher_cfg |= keylen_cfg;
718 } else if (cipher_cfg & MOD_3DES) {
719 const u32 *K = (const u32 *)key;
720 if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
721 !((K[2] ^ K[4]) | (K[3] ^ K[5]))))
722 {
723 *flags |= CRYPTO_TFM_RES_BAD_KEY_SCHED;
724 return -EINVAL;
725 }
726 } else {
727 u32 tmp[DES_EXPKEY_WORDS];
728 if (des_ekey(tmp, key) == 0) {
729 *flags |= CRYPTO_TFM_RES_WEAK_KEY;
730 }
731 }
732 /* write cfg word to cryptinfo */
733 *(u32*)cinfo = cpu_to_be32(cipher_cfg);
734 cinfo += sizeof(cipher_cfg);
735
736 /* write cipher key to cryptinfo */
737 memcpy(cinfo, key, key_len);
738 /* NPE wants keylen set to DES3_EDE_KEY_SIZE even for single DES */
739 if (key_len < DES3_EDE_KEY_SIZE && !(cipher_cfg & MOD_AES)) {
740 memset(cinfo + key_len, 0, DES3_EDE_KEY_SIZE -key_len);
741 key_len = DES3_EDE_KEY_SIZE;
742 }
743 dir->npe_ctx_idx = sizeof(cipher_cfg) + key_len;
744 dir->npe_mode |= NPE_OP_CRYPT_ENABLE;
745 if ((cipher_cfg & MOD_AES) && !encrypt) {
746 return gen_rev_aes_key(tfm);
747 }
748 return 0;
749}
750
751static int count_sg(struct scatterlist *sg, int nbytes)
752{
753 int i;
754 for (i = 0; nbytes > 0; i++, sg = sg_next(sg))
755 nbytes -= sg->length;
756 return i;
757}
758
759static struct buffer_desc *chainup_buffers(struct scatterlist *sg,
760 unsigned nbytes, struct buffer_desc *buf, gfp_t flags)
761{
762 int nents = 0;
763
764 while (nbytes > 0) {
765 struct buffer_desc *next_buf;
766 u32 next_buf_phys;
767 unsigned len = min(nbytes, sg_dma_len(sg));
768
769 nents++;
770 nbytes -= len;
771 if (!buf->phys_addr) {
772 buf->phys_addr = sg_dma_address(sg);
773 buf->buf_len = len;
774 buf->next = NULL;
775 buf->phys_next = 0;
776 goto next;
777 }
778 /* Two consecutive chunks on one page may be handled by the old
779 * buffer descriptor, increased by the length of the new one
780 */
781 if (sg_dma_address(sg) == buf->phys_addr + buf->buf_len) {
782 buf->buf_len += len;
783 goto next;
784 }
785 next_buf = dma_pool_alloc(buffer_pool, flags, &next_buf_phys);
786 if (!next_buf)
787 return NULL;
788 buf->next = next_buf;
789 buf->phys_next = next_buf_phys;
790
791 buf = next_buf;
792 buf->next = NULL;
793 buf->phys_next = 0;
794 buf->phys_addr = sg_dma_address(sg);
795 buf->buf_len = len;
796next:
797 if (nbytes > 0) {
798 sg = sg_next(sg);
799 }
800 }
801 return buf;
802}
803
804static int ablk_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
805 unsigned int key_len)
806{
807 struct ixp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
808 u32 *flags = &tfm->base.crt_flags;
809 int ret;
810
811 init_completion(&ctx->completion);
812 atomic_inc(&ctx->configuring);
813
814 reset_sa_dir(&ctx->encrypt);
815 reset_sa_dir(&ctx->decrypt);
816
817 ctx->encrypt.npe_mode = NPE_OP_HMAC_DISABLE;
818 ctx->decrypt.npe_mode = NPE_OP_HMAC_DISABLE;
819
820 ret = setup_cipher(&tfm->base, 0, key, key_len);
821 if (ret)
822 goto out;
823 ret = setup_cipher(&tfm->base, 1, key, key_len);
824 if (ret)
825 goto out;
826
827 if (*flags & CRYPTO_TFM_RES_WEAK_KEY) {
828 if (*flags & CRYPTO_TFM_REQ_WEAK_KEY) {
829 ret = -EINVAL;
830 } else {
831 *flags &= ~CRYPTO_TFM_RES_WEAK_KEY;
832 }
833 }
834out:
835 if (!atomic_dec_and_test(&ctx->configuring))
836 wait_for_completion(&ctx->completion);
837 return ret;
838}
839
840static int ablk_rfc3686_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
841 unsigned int key_len)
842{
843 struct ixp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
844
845 /* the nonce is stored in bytes at end of key */
846 if (key_len < CTR_RFC3686_NONCE_SIZE)
847 return -EINVAL;
848
849 memcpy(ctx->nonce, key + (key_len - CTR_RFC3686_NONCE_SIZE),
850 CTR_RFC3686_NONCE_SIZE);
851
852 key_len -= CTR_RFC3686_NONCE_SIZE;
853 return ablk_setkey(tfm, key, key_len);
854}
855
856static int ablk_perform(struct ablkcipher_request *req, int encrypt)
857{
858 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
859 struct ixp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
860 unsigned ivsize = crypto_ablkcipher_ivsize(tfm);
861 int ret = -ENOMEM;
862 struct ix_sa_dir *dir;
863 struct crypt_ctl *crypt;
864 unsigned int nbytes = req->nbytes, nents;
865 enum dma_data_direction src_direction = DMA_BIDIRECTIONAL;
866 struct ablk_ctx *req_ctx = ablkcipher_request_ctx(req);
867 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
868 GFP_KERNEL : GFP_ATOMIC;
869
870 if (qmgr_stat_full(SEND_QID))
871 return -EAGAIN;
872 if (atomic_read(&ctx->configuring))
873 return -EAGAIN;
874
875 dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
876
877 crypt = get_crypt_desc();
878 if (!crypt)
879 return ret;
880
881 crypt->data.ablk_req = req;
882 crypt->crypto_ctx = dir->npe_ctx_phys;
883 crypt->mode = dir->npe_mode;
884 crypt->init_len = dir->npe_ctx_idx;
885
886 crypt->crypt_offs = 0;
887 crypt->crypt_len = nbytes;
888
889 BUG_ON(ivsize && !req->info);
890 memcpy(crypt->iv, req->info, ivsize);
891 if (req->src != req->dst) {
892 crypt->mode |= NPE_OP_NOT_IN_PLACE;
893 nents = count_sg(req->dst, nbytes);
894 /* This was never tested by Intel
895 * for more than one dst buffer, I think. */
896 BUG_ON(nents != 1);
897 req_ctx->dst_nents = nents;
898 dma_map_sg(dev, req->dst, nents, DMA_FROM_DEVICE);
899 req_ctx->dst = dma_pool_alloc(buffer_pool, flags,&crypt->dst_buf);
900 if (!req_ctx->dst)
901 goto unmap_sg_dest;
902 req_ctx->dst->phys_addr = 0;
903 if (!chainup_buffers(req->dst, nbytes, req_ctx->dst, flags))
904 goto free_buf_dest;
905 src_direction = DMA_TO_DEVICE;
906 } else {
907 req_ctx->dst = NULL;
908 req_ctx->dst_nents = 0;
909 }
910 nents = count_sg(req->src, nbytes);
911 req_ctx->src_nents = nents;
912 dma_map_sg(dev, req->src, nents, src_direction);
913
914 req_ctx->src = dma_pool_alloc(buffer_pool, flags, &crypt->src_buf);
915 if (!req_ctx->src)
916 goto unmap_sg_src;
917 req_ctx->src->phys_addr = 0;
918 if (!chainup_buffers(req->src, nbytes, req_ctx->src, flags))
919 goto free_buf_src;
920
921 crypt->ctl_flags |= CTL_FLAG_PERFORM_ABLK;
922 qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
923 BUG_ON(qmgr_stat_overflow(SEND_QID));
924 return -EINPROGRESS;
925
926free_buf_src:
927 free_buf_chain(req_ctx->src, crypt->src_buf);
928unmap_sg_src:
929 dma_unmap_sg(dev, req->src, req_ctx->src_nents, src_direction);
930free_buf_dest:
931 if (req->src != req->dst) {
932 free_buf_chain(req_ctx->dst, crypt->dst_buf);
933unmap_sg_dest:
934 dma_unmap_sg(dev, req->src, req_ctx->dst_nents,
935 DMA_FROM_DEVICE);
936 }
937 crypt->ctl_flags = CTL_FLAG_UNUSED;
938 return ret;
939}
940
941static int ablk_encrypt(struct ablkcipher_request *req)
942{
943 return ablk_perform(req, 1);
944}
945
946static int ablk_decrypt(struct ablkcipher_request *req)
947{
948 return ablk_perform(req, 0);
949}
950
951static int ablk_rfc3686_crypt(struct ablkcipher_request *req)
952{
953 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
954 struct ixp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
955 u8 iv[CTR_RFC3686_BLOCK_SIZE];
956 u8 *info = req->info;
957 int ret;
958
959 /* set up counter block */
960 memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
961 memcpy(iv + CTR_RFC3686_NONCE_SIZE, info, CTR_RFC3686_IV_SIZE);
962
963 /* initialize counter portion of counter block */
964 *(__be32 *)(iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) =
965 cpu_to_be32(1);
966
967 req->info = iv;
968 ret = ablk_perform(req, 1);
969 req->info = info;
970 return ret;
971}
972
973static int hmac_inconsistent(struct scatterlist *sg, unsigned start,
974 unsigned int nbytes)
975{
976 int offset = 0;
977
978 if (!nbytes)
979 return 0;
980
981 for (;;) {
982 if (start < offset + sg->length)
983 break;
984
985 offset += sg->length;
986 sg = sg_next(sg);
987 }
988 return (start + nbytes > offset + sg->length);
989}
990
991static int aead_perform(struct aead_request *req, int encrypt,
992 int cryptoffset, int eff_cryptlen, u8 *iv)
993{
994 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
995 struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
996 unsigned ivsize = crypto_aead_ivsize(tfm);
997 unsigned authsize = crypto_aead_authsize(tfm);
998 int ret = -ENOMEM;
999 struct ix_sa_dir *dir;
1000 struct crypt_ctl *crypt;
1001 unsigned int cryptlen, nents;
1002 struct buffer_desc *buf;
1003 struct aead_ctx *req_ctx = aead_request_ctx(req);
1004 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
1005 GFP_KERNEL : GFP_ATOMIC;
1006
1007 if (qmgr_stat_full(SEND_QID))
1008 return -EAGAIN;
1009 if (atomic_read(&ctx->configuring))
1010 return -EAGAIN;
1011
1012 if (encrypt) {
1013 dir = &ctx->encrypt;
1014 cryptlen = req->cryptlen;
1015 } else {
1016 dir = &ctx->decrypt;
1017 /* req->cryptlen includes the authsize when decrypting */
1018 cryptlen = req->cryptlen -authsize;
1019 eff_cryptlen -= authsize;
1020 }
1021 crypt = get_crypt_desc();
1022 if (!crypt)
1023 return ret;
1024
1025 crypt->data.aead_req = req;
1026 crypt->crypto_ctx = dir->npe_ctx_phys;
1027 crypt->mode = dir->npe_mode;
1028 crypt->init_len = dir->npe_ctx_idx;
1029
1030 crypt->crypt_offs = cryptoffset;
1031 crypt->crypt_len = eff_cryptlen;
1032
1033 crypt->auth_offs = 0;
1034 crypt->auth_len = req->assoclen + ivsize + cryptlen;
1035 BUG_ON(ivsize && !req->iv);
1036 memcpy(crypt->iv, req->iv, ivsize);
1037
1038 if (req->src != req->dst) {
1039 BUG(); /* -ENOTSUP because of my lazyness */
1040 }
1041
1042 req_ctx->buffer = dma_pool_alloc(buffer_pool, flags, &crypt->src_buf);
1043 if (!req_ctx->buffer)
1044 goto out;
1045 req_ctx->buffer->phys_addr = 0;
1046 /* ASSOC data */
1047 nents = count_sg(req->assoc, req->assoclen);
1048 req_ctx->assoc_nents = nents;
1049 dma_map_sg(dev, req->assoc, nents, DMA_TO_DEVICE);
1050 buf = chainup_buffers(req->assoc, req->assoclen, req_ctx->buffer,flags);
1051 if (!buf)
1052 goto unmap_sg_assoc;
1053 /* IV */
1054 sg_init_table(&req_ctx->ivlist, 1);
1055 sg_set_buf(&req_ctx->ivlist, iv, ivsize);
1056 dma_map_sg(dev, &req_ctx->ivlist, 1, DMA_BIDIRECTIONAL);
1057 buf = chainup_buffers(&req_ctx->ivlist, ivsize, buf, flags);
1058 if (!buf)
1059 goto unmap_sg_iv;
1060 if (unlikely(hmac_inconsistent(req->src, cryptlen, authsize))) {
1061 /* The 12 hmac bytes are scattered,
1062 * we need to copy them into a safe buffer */
1063 req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags,
1064 &crypt->icv_rev_aes);
1065 if (unlikely(!req_ctx->hmac_virt))
1066 goto unmap_sg_iv;
1067 if (!encrypt) {
1068 scatterwalk_map_and_copy(req_ctx->hmac_virt,
1069 req->src, cryptlen, authsize, 0);
1070 }
1071 req_ctx->encrypt = encrypt;
1072 } else {
1073 req_ctx->hmac_virt = NULL;
1074 }
1075 /* Crypt */
1076 nents = count_sg(req->src, cryptlen + authsize);
1077 req_ctx->src_nents = nents;
1078 dma_map_sg(dev, req->src, nents, DMA_BIDIRECTIONAL);
1079 buf = chainup_buffers(req->src, cryptlen + authsize, buf, flags);
1080 if (!buf)
1081 goto unmap_sg_src;
1082 if (!req_ctx->hmac_virt) {
1083 crypt->icv_rev_aes = buf->phys_addr + buf->buf_len - authsize;
1084 }
1085 crypt->ctl_flags |= CTL_FLAG_PERFORM_AEAD;
1086 qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
1087 BUG_ON(qmgr_stat_overflow(SEND_QID));
1088 return -EINPROGRESS;
1089unmap_sg_src:
1090 dma_unmap_sg(dev, req->src, req_ctx->src_nents, DMA_BIDIRECTIONAL);
1091 if (req_ctx->hmac_virt) {
1092 dma_pool_free(buffer_pool, req_ctx->hmac_virt,
1093 crypt->icv_rev_aes);
1094 }
1095unmap_sg_iv:
1096 dma_unmap_sg(dev, &req_ctx->ivlist, 1, DMA_BIDIRECTIONAL);
1097unmap_sg_assoc:
1098 dma_unmap_sg(dev, req->assoc, req_ctx->assoc_nents, DMA_TO_DEVICE);
1099 free_buf_chain(req_ctx->buffer, crypt->src_buf);
1100out:
1101 crypt->ctl_flags = CTL_FLAG_UNUSED;
1102 return ret;
1103}
1104
1105static int aead_setup(struct crypto_aead *tfm, unsigned int authsize)
1106{
1107 struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
1108 u32 *flags = &tfm->base.crt_flags;
1109 unsigned digest_len = crypto_aead_alg(tfm)->maxauthsize;
1110 int ret;
1111
1112 if (!ctx->enckey_len && !ctx->authkey_len)
1113 return 0;
1114 init_completion(&ctx->completion);
1115 atomic_inc(&ctx->configuring);
1116
1117 reset_sa_dir(&ctx->encrypt);
1118 reset_sa_dir(&ctx->decrypt);
1119
1120 ret = setup_cipher(&tfm->base, 0, ctx->enckey, ctx->enckey_len);
1121 if (ret)
1122 goto out;
1123 ret = setup_cipher(&tfm->base, 1, ctx->enckey, ctx->enckey_len);
1124 if (ret)
1125 goto out;
1126 ret = setup_auth(&tfm->base, 0, authsize, ctx->authkey,
1127 ctx->authkey_len, digest_len);
1128 if (ret)
1129 goto out;
1130 ret = setup_auth(&tfm->base, 1, authsize, ctx->authkey,
1131 ctx->authkey_len, digest_len);
1132 if (ret)
1133 goto out;
1134
1135 if (*flags & CRYPTO_TFM_RES_WEAK_KEY) {
1136 if (*flags & CRYPTO_TFM_REQ_WEAK_KEY) {
1137 ret = -EINVAL;
1138 goto out;
1139 } else {
1140 *flags &= ~CRYPTO_TFM_RES_WEAK_KEY;
1141 }
1142 }
1143out:
1144 if (!atomic_dec_and_test(&ctx->configuring))
1145 wait_for_completion(&ctx->completion);
1146 return ret;
1147}
1148
1149static int aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
1150{
1151 int max = crypto_aead_alg(tfm)->maxauthsize >> 2;
1152
1153 if ((authsize>>2) < 1 || (authsize>>2) > max || (authsize & 3))
1154 return -EINVAL;
1155 return aead_setup(tfm, authsize);
1156}
1157
1158static int aead_setkey(struct crypto_aead *tfm, const u8 *key,
1159 unsigned int keylen)
1160{
1161 struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
1162 struct rtattr *rta = (struct rtattr *)key;
1163 struct crypto_authenc_key_param *param;
1164
1165 if (!RTA_OK(rta, keylen))
1166 goto badkey;
1167 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
1168 goto badkey;
1169 if (RTA_PAYLOAD(rta) < sizeof(*param))
1170 goto badkey;
1171
1172 param = RTA_DATA(rta);
1173 ctx->enckey_len = be32_to_cpu(param->enckeylen);
1174
1175 key += RTA_ALIGN(rta->rta_len);
1176 keylen -= RTA_ALIGN(rta->rta_len);
1177
1178 if (keylen < ctx->enckey_len)
1179 goto badkey;
1180
1181 ctx->authkey_len = keylen - ctx->enckey_len;
1182 memcpy(ctx->enckey, key + ctx->authkey_len, ctx->enckey_len);
1183 memcpy(ctx->authkey, key, ctx->authkey_len);
1184
1185 return aead_setup(tfm, crypto_aead_authsize(tfm));
1186badkey:
1187 ctx->enckey_len = 0;
1188 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1189 return -EINVAL;
1190}
1191
1192static int aead_encrypt(struct aead_request *req)
1193{
1194 unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
1195 return aead_perform(req, 1, req->assoclen + ivsize,
1196 req->cryptlen, req->iv);
1197}
1198
1199static int aead_decrypt(struct aead_request *req)
1200{
1201 unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
1202 return aead_perform(req, 0, req->assoclen + ivsize,
1203 req->cryptlen, req->iv);
1204}
1205
1206static int aead_givencrypt(struct aead_givcrypt_request *req)
1207{
1208 struct crypto_aead *tfm = aead_givcrypt_reqtfm(req);
1209 struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
1210 unsigned len, ivsize = crypto_aead_ivsize(tfm);
1211 __be64 seq;
1212
1213 /* copied from eseqiv.c */
1214 if (!ctx->salted) {
1215 get_random_bytes(ctx->salt, ivsize);
1216 ctx->salted = 1;
1217 }
1218 memcpy(req->areq.iv, ctx->salt, ivsize);
1219 len = ivsize;
1220 if (ivsize > sizeof(u64)) {
1221 memset(req->giv, 0, ivsize - sizeof(u64));
1222 len = sizeof(u64);
1223 }
1224 seq = cpu_to_be64(req->seq);
1225 memcpy(req->giv + ivsize - len, &seq, len);
1226 return aead_perform(&req->areq, 1, req->areq.assoclen,
1227 req->areq.cryptlen +ivsize, req->giv);
1228}
1229
1230static struct ixp_alg ixp4xx_algos[] = {
1231{
1232 .crypto = {
1233 .cra_name = "cbc(des)",
1234 .cra_blocksize = DES_BLOCK_SIZE,
1235 .cra_u = { .ablkcipher = {
1236 .min_keysize = DES_KEY_SIZE,
1237 .max_keysize = DES_KEY_SIZE,
1238 .ivsize = DES_BLOCK_SIZE,
1239 .geniv = "eseqiv",
1240 }
1241 }
1242 },
1243 .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
1244 .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
1245
1246}, {
1247 .crypto = {
1248 .cra_name = "ecb(des)",
1249 .cra_blocksize = DES_BLOCK_SIZE,
1250 .cra_u = { .ablkcipher = {
1251 .min_keysize = DES_KEY_SIZE,
1252 .max_keysize = DES_KEY_SIZE,
1253 }
1254 }
1255 },
1256 .cfg_enc = CIPH_ENCR | MOD_DES | MOD_ECB | KEYLEN_192,
1257 .cfg_dec = CIPH_DECR | MOD_DES | MOD_ECB | KEYLEN_192,
1258}, {
1259 .crypto = {
1260 .cra_name = "cbc(des3_ede)",
1261 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1262 .cra_u = { .ablkcipher = {
1263 .min_keysize = DES3_EDE_KEY_SIZE,
1264 .max_keysize = DES3_EDE_KEY_SIZE,
1265 .ivsize = DES3_EDE_BLOCK_SIZE,
1266 .geniv = "eseqiv",
1267 }
1268 }
1269 },
1270 .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
1271 .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
1272}, {
1273 .crypto = {
1274 .cra_name = "ecb(des3_ede)",
1275 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1276 .cra_u = { .ablkcipher = {
1277 .min_keysize = DES3_EDE_KEY_SIZE,
1278 .max_keysize = DES3_EDE_KEY_SIZE,
1279 }
1280 }
1281 },
1282 .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_ECB | KEYLEN_192,
1283 .cfg_dec = CIPH_DECR | MOD_3DES | MOD_ECB | KEYLEN_192,
1284}, {
1285 .crypto = {
1286 .cra_name = "cbc(aes)",
1287 .cra_blocksize = AES_BLOCK_SIZE,
1288 .cra_u = { .ablkcipher = {
1289 .min_keysize = AES_MIN_KEY_SIZE,
1290 .max_keysize = AES_MAX_KEY_SIZE,
1291 .ivsize = AES_BLOCK_SIZE,
1292 .geniv = "eseqiv",
1293 }
1294 }
1295 },
1296 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
1297 .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
1298}, {
1299 .crypto = {
1300 .cra_name = "ecb(aes)",
1301 .cra_blocksize = AES_BLOCK_SIZE,
1302 .cra_u = { .ablkcipher = {
1303 .min_keysize = AES_MIN_KEY_SIZE,
1304 .max_keysize = AES_MAX_KEY_SIZE,
1305 }
1306 }
1307 },
1308 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_ECB,
1309 .cfg_dec = CIPH_DECR | MOD_AES | MOD_ECB,
1310}, {
1311 .crypto = {
1312 .cra_name = "ctr(aes)",
1313 .cra_blocksize = AES_BLOCK_SIZE,
1314 .cra_u = { .ablkcipher = {
1315 .min_keysize = AES_MIN_KEY_SIZE,
1316 .max_keysize = AES_MAX_KEY_SIZE,
1317 .ivsize = AES_BLOCK_SIZE,
1318 .geniv = "eseqiv",
1319 }
1320 }
1321 },
1322 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR,
1323 .cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR,
1324}, {
1325 .crypto = {
1326 .cra_name = "rfc3686(ctr(aes))",
1327 .cra_blocksize = AES_BLOCK_SIZE,
1328 .cra_u = { .ablkcipher = {
1329 .min_keysize = AES_MIN_KEY_SIZE,
1330 .max_keysize = AES_MAX_KEY_SIZE,
1331 .ivsize = AES_BLOCK_SIZE,
1332 .geniv = "eseqiv",
1333 .setkey = ablk_rfc3686_setkey,
1334 .encrypt = ablk_rfc3686_crypt,
1335 .decrypt = ablk_rfc3686_crypt }
1336 }
1337 },
1338 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR,
1339 .cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR,
1340}, {
1341 .crypto = {
1342 .cra_name = "authenc(hmac(md5),cbc(des))",
1343 .cra_blocksize = DES_BLOCK_SIZE,
1344 .cra_u = { .aead = {
1345 .ivsize = DES_BLOCK_SIZE,
1346 .maxauthsize = MD5_DIGEST_SIZE,
1347 }
1348 }
1349 },
1350 .hash = &hash_alg_md5,
1351 .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
1352 .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
1353}, {
1354 .crypto = {
1355 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
1356 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1357 .cra_u = { .aead = {
1358 .ivsize = DES3_EDE_BLOCK_SIZE,
1359 .maxauthsize = MD5_DIGEST_SIZE,
1360 }
1361 }
1362 },
1363 .hash = &hash_alg_md5,
1364 .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
1365 .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
1366}, {
1367 .crypto = {
1368 .cra_name = "authenc(hmac(sha1),cbc(des))",
1369 .cra_blocksize = DES_BLOCK_SIZE,
1370 .cra_u = { .aead = {
1371 .ivsize = DES_BLOCK_SIZE,
1372 .maxauthsize = SHA1_DIGEST_SIZE,
1373 }
1374 }
1375 },
1376 .hash = &hash_alg_sha1,
1377 .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
1378 .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
1379}, {
1380 .crypto = {
1381 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
1382 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1383 .cra_u = { .aead = {
1384 .ivsize = DES3_EDE_BLOCK_SIZE,
1385 .maxauthsize = SHA1_DIGEST_SIZE,
1386 }
1387 }
1388 },
1389 .hash = &hash_alg_sha1,
1390 .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
1391 .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
1392}, {
1393 .crypto = {
1394 .cra_name = "authenc(hmac(md5),cbc(aes))",
1395 .cra_blocksize = AES_BLOCK_SIZE,
1396 .cra_u = { .aead = {
1397 .ivsize = AES_BLOCK_SIZE,
1398 .maxauthsize = MD5_DIGEST_SIZE,
1399 }
1400 }
1401 },
1402 .hash = &hash_alg_md5,
1403 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
1404 .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
1405}, {
1406 .crypto = {
1407 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1408 .cra_blocksize = AES_BLOCK_SIZE,
1409 .cra_u = { .aead = {
1410 .ivsize = AES_BLOCK_SIZE,
1411 .maxauthsize = SHA1_DIGEST_SIZE,
1412 }
1413 }
1414 },
1415 .hash = &hash_alg_sha1,
1416 .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
1417 .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
1418} };
1419
1420#define IXP_POSTFIX "-ixp4xx"
1421static int __init ixp_module_init(void)
1422{
1423 int num = ARRAY_SIZE(ixp4xx_algos);
1424 int i,err ;
1425
1426 if (platform_device_register(&pseudo_dev))
1427 return -ENODEV;
1428
1429 spin_lock_init(&desc_lock);
1430 spin_lock_init(&emerg_lock);
1431
1432 err = init_ixp_crypto();
1433 if (err) {
1434 platform_device_unregister(&pseudo_dev);
1435 return err;
1436 }
1437 for (i=0; i< num; i++) {
1438 struct crypto_alg *cra = &ixp4xx_algos[i].crypto;
1439
1440 if (snprintf(cra->cra_driver_name, CRYPTO_MAX_ALG_NAME,
1441 "%s"IXP_POSTFIX, cra->cra_name) >=
1442 CRYPTO_MAX_ALG_NAME)
1443 {
1444 continue;
1445 }
1446 if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES)) {
1447 continue;
1448 }
1449 if (!ixp4xx_algos[i].hash) {
1450 /* block ciphers */
1451 cra->cra_type = &crypto_ablkcipher_type;
1452 cra->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1453 CRYPTO_ALG_ASYNC;
1454 if (!cra->cra_ablkcipher.setkey)
1455 cra->cra_ablkcipher.setkey = ablk_setkey;
1456 if (!cra->cra_ablkcipher.encrypt)
1457 cra->cra_ablkcipher.encrypt = ablk_encrypt;
1458 if (!cra->cra_ablkcipher.decrypt)
1459 cra->cra_ablkcipher.decrypt = ablk_decrypt;
1460 cra->cra_init = init_tfm_ablk;
1461 } else {
1462 /* authenc */
1463 cra->cra_type = &crypto_aead_type;
1464 cra->cra_flags = CRYPTO_ALG_TYPE_AEAD |
1465 CRYPTO_ALG_ASYNC;
1466 cra->cra_aead.setkey = aead_setkey;
1467 cra->cra_aead.setauthsize = aead_setauthsize;
1468 cra->cra_aead.encrypt = aead_encrypt;
1469 cra->cra_aead.decrypt = aead_decrypt;
1470 cra->cra_aead.givencrypt = aead_givencrypt;
1471 cra->cra_init = init_tfm_aead;
1472 }
1473 cra->cra_ctxsize = sizeof(struct ixp_ctx);
1474 cra->cra_module = THIS_MODULE;
1475 cra->cra_alignmask = 3;
1476 cra->cra_priority = 300;
1477 cra->cra_exit = exit_tfm;
1478 if (crypto_register_alg(cra))
1479 printk(KERN_ERR "Failed to register '%s'\n",
1480 cra->cra_name);
1481 else
1482 ixp4xx_algos[i].registered = 1;
1483 }
1484 return 0;
1485}
1486
1487static void __exit ixp_module_exit(void)
1488{
1489 int num = ARRAY_SIZE(ixp4xx_algos);
1490 int i;
1491
1492 for (i=0; i< num; i++) {
1493 if (ixp4xx_algos[i].registered)
1494 crypto_unregister_alg(&ixp4xx_algos[i].crypto);
1495 }
1496 release_ixp_crypto();
1497 platform_device_unregister(&pseudo_dev);
1498}
1499
1500module_init(ixp_module_init);
1501module_exit(ixp_module_exit);
1502
1503MODULE_LICENSE("GPL");
1504MODULE_AUTHOR("Christian Hohnstaedt <chohnstaedt@innominate.com>");
1505MODULE_DESCRIPTION("IXP4xx hardware crypto");
1506
diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c
index bb30eb9b93ef..54a2a166e566 100644
--- a/drivers/crypto/padlock-aes.c
+++ b/drivers/crypto/padlock-aes.c
@@ -385,12 +385,12 @@ static int __init padlock_init(void)
385 int ret; 385 int ret;
386 386
387 if (!cpu_has_xcrypt) { 387 if (!cpu_has_xcrypt) {
388 printk(KERN_ERR PFX "VIA PadLock not detected.\n"); 388 printk(KERN_NOTICE PFX "VIA PadLock not detected.\n");
389 return -ENODEV; 389 return -ENODEV;
390 } 390 }
391 391
392 if (!cpu_has_xcrypt_enabled) { 392 if (!cpu_has_xcrypt_enabled) {
393 printk(KERN_ERR PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n"); 393 printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
394 return -ENODEV; 394 return -ENODEV;
395 } 395 }
396 396
diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c
index c666b4e0933e..40d5680fa013 100644
--- a/drivers/crypto/padlock-sha.c
+++ b/drivers/crypto/padlock-sha.c
@@ -254,12 +254,12 @@ static int __init padlock_init(void)
254 int rc = -ENODEV; 254 int rc = -ENODEV;
255 255
256 if (!cpu_has_phe) { 256 if (!cpu_has_phe) {
257 printk(KERN_ERR PFX "VIA PadLock Hash Engine not detected.\n"); 257 printk(KERN_NOTICE PFX "VIA PadLock Hash Engine not detected.\n");
258 return -ENODEV; 258 return -ENODEV;
259 } 259 }
260 260
261 if (!cpu_has_phe_enabled) { 261 if (!cpu_has_phe_enabled) {
262 printk(KERN_ERR PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n"); 262 printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
263 return -ENODEV; 263 return -ENODEV;
264 } 264 }
265 265
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
new file mode 100644
index 000000000000..b11943dadefd
--- /dev/null
+++ b/drivers/crypto/talitos.c
@@ -0,0 +1,1597 @@
1/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
4 * Copyright (c) 2008 Freescale Semiconductor, Inc.
5 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28#include <linux/kernel.h>
29#include <linux/module.h>
30#include <linux/mod_devicetable.h>
31#include <linux/device.h>
32#include <linux/interrupt.h>
33#include <linux/crypto.h>
34#include <linux/hw_random.h>
35#include <linux/of_platform.h>
36#include <linux/dma-mapping.h>
37#include <linux/io.h>
38#include <linux/spinlock.h>
39#include <linux/rtnetlink.h>
40
41#include <crypto/algapi.h>
42#include <crypto/aes.h>
43#include <crypto/des.h>
44#include <crypto/sha.h>
45#include <crypto/aead.h>
46#include <crypto/authenc.h>
47
48#include "talitos.h"
49
50#define TALITOS_TIMEOUT 100000
51#define TALITOS_MAX_DATA_LEN 65535
52
53#define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
54#define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
55#define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
56
57/* descriptor pointer entry */
58struct talitos_ptr {
59 __be16 len; /* length */
60 u8 j_extent; /* jump to sg link table and/or extent */
61 u8 eptr; /* extended address */
62 __be32 ptr; /* address */
63};
64
65/* descriptor */
66struct talitos_desc {
67 __be32 hdr; /* header high bits */
68 __be32 hdr_lo; /* header low bits */
69 struct talitos_ptr ptr[7]; /* ptr/len pair array */
70};
71
72/**
73 * talitos_request - descriptor submission request
74 * @desc: descriptor pointer (kernel virtual)
75 * @dma_desc: descriptor's physical bus address
76 * @callback: whom to call when descriptor processing is done
77 * @context: caller context (optional)
78 */
79struct talitos_request {
80 struct talitos_desc *desc;
81 dma_addr_t dma_desc;
82 void (*callback) (struct device *dev, struct talitos_desc *desc,
83 void *context, int error);
84 void *context;
85};
86
87struct talitos_private {
88 struct device *dev;
89 struct of_device *ofdev;
90 void __iomem *reg;
91 int irq;
92
93 /* SEC version geometry (from device tree node) */
94 unsigned int num_channels;
95 unsigned int chfifo_len;
96 unsigned int exec_units;
97 unsigned int desc_types;
98
99 /* next channel to be assigned next incoming descriptor */
100 atomic_t last_chan;
101
102 /* per-channel request fifo */
103 struct talitos_request **fifo;
104
105 /*
106 * length of the request fifo
107 * fifo_len is chfifo_len rounded up to next power of 2
108 * so we can use bitwise ops to wrap
109 */
110 unsigned int fifo_len;
111
112 /* per-channel index to next free descriptor request */
113 int *head;
114
115 /* per-channel index to next in-progress/done descriptor request */
116 int *tail;
117
118 /* per-channel request submission (head) and release (tail) locks */
119 spinlock_t *head_lock;
120 spinlock_t *tail_lock;
121
122 /* request callback tasklet */
123 struct tasklet_struct done_task;
124 struct tasklet_struct error_task;
125
126 /* list of registered algorithms */
127 struct list_head alg_list;
128
129 /* hwrng device */
130 struct hwrng rng;
131};
132
133/*
134 * map virtual single (contiguous) pointer to h/w descriptor pointer
135 */
136static void map_single_talitos_ptr(struct device *dev,
137 struct talitos_ptr *talitos_ptr,
138 unsigned short len, void *data,
139 unsigned char extent,
140 enum dma_data_direction dir)
141{
142 talitos_ptr->len = cpu_to_be16(len);
143 talitos_ptr->ptr = cpu_to_be32(dma_map_single(dev, data, len, dir));
144 talitos_ptr->j_extent = extent;
145}
146
147/*
148 * unmap bus single (contiguous) h/w descriptor pointer
149 */
150static void unmap_single_talitos_ptr(struct device *dev,
151 struct talitos_ptr *talitos_ptr,
152 enum dma_data_direction dir)
153{
154 dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr),
155 be16_to_cpu(talitos_ptr->len), dir);
156}
157
158static int reset_channel(struct device *dev, int ch)
159{
160 struct talitos_private *priv = dev_get_drvdata(dev);
161 unsigned int timeout = TALITOS_TIMEOUT;
162
163 setbits32(priv->reg + TALITOS_CCCR(ch), TALITOS_CCCR_RESET);
164
165 while ((in_be32(priv->reg + TALITOS_CCCR(ch)) & TALITOS_CCCR_RESET)
166 && --timeout)
167 cpu_relax();
168
169 if (timeout == 0) {
170 dev_err(dev, "failed to reset channel %d\n", ch);
171 return -EIO;
172 }
173
174 /* set done writeback and IRQ */
175 setbits32(priv->reg + TALITOS_CCCR_LO(ch), TALITOS_CCCR_LO_CDWE |
176 TALITOS_CCCR_LO_CDIE);
177
178 return 0;
179}
180
181static int reset_device(struct device *dev)
182{
183 struct talitos_private *priv = dev_get_drvdata(dev);
184 unsigned int timeout = TALITOS_TIMEOUT;
185
186 setbits32(priv->reg + TALITOS_MCR, TALITOS_MCR_SWR);
187
188 while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR)
189 && --timeout)
190 cpu_relax();
191
192 if (timeout == 0) {
193 dev_err(dev, "failed to reset device\n");
194 return -EIO;
195 }
196
197 return 0;
198}
199
200/*
201 * Reset and initialize the device
202 */
203static int init_device(struct device *dev)
204{
205 struct talitos_private *priv = dev_get_drvdata(dev);
206 int ch, err;
207
208 /*
209 * Master reset
210 * errata documentation: warning: certain SEC interrupts
211 * are not fully cleared by writing the MCR:SWR bit,
212 * set bit twice to completely reset
213 */
214 err = reset_device(dev);
215 if (err)
216 return err;
217
218 err = reset_device(dev);
219 if (err)
220 return err;
221
222 /* reset channels */
223 for (ch = 0; ch < priv->num_channels; ch++) {
224 err = reset_channel(dev, ch);
225 if (err)
226 return err;
227 }
228
229 /* enable channel done and error interrupts */
230 setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
231 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
232
233 return 0;
234}
235
236/**
237 * talitos_submit - submits a descriptor to the device for processing
238 * @dev: the SEC device to be used
239 * @desc: the descriptor to be processed by the device
240 * @callback: whom to call when processing is complete
241 * @context: a handle for use by caller (optional)
242 *
243 * desc must contain valid dma-mapped (bus physical) address pointers.
244 * callback must check err and feedback in descriptor header
245 * for device processing status.
246 */
247static int talitos_submit(struct device *dev, struct talitos_desc *desc,
248 void (*callback)(struct device *dev,
249 struct talitos_desc *desc,
250 void *context, int error),
251 void *context)
252{
253 struct talitos_private *priv = dev_get_drvdata(dev);
254 struct talitos_request *request;
255 unsigned long flags, ch;
256 int head;
257
258 /* select done notification */
259 desc->hdr |= DESC_HDR_DONE_NOTIFY;
260
261 /* emulate SEC's round-robin channel fifo polling scheme */
262 ch = atomic_inc_return(&priv->last_chan) & (priv->num_channels - 1);
263
264 spin_lock_irqsave(&priv->head_lock[ch], flags);
265
266 head = priv->head[ch];
267 request = &priv->fifo[ch][head];
268
269 if (request->desc) {
270 /* request queue is full */
271 spin_unlock_irqrestore(&priv->head_lock[ch], flags);
272 return -EAGAIN;
273 }
274
275 /* map descriptor and save caller data */
276 request->dma_desc = dma_map_single(dev, desc, sizeof(*desc),
277 DMA_BIDIRECTIONAL);
278 request->callback = callback;
279 request->context = context;
280
281 /* increment fifo head */
282 priv->head[ch] = (priv->head[ch] + 1) & (priv->fifo_len - 1);
283
284 smp_wmb();
285 request->desc = desc;
286
287 /* GO! */
288 wmb();
289 out_be32(priv->reg + TALITOS_FF_LO(ch), request->dma_desc);
290
291 spin_unlock_irqrestore(&priv->head_lock[ch], flags);
292
293 return -EINPROGRESS;
294}
295
296/*
297 * process what was done, notify callback of error if not
298 */
299static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
300{
301 struct talitos_private *priv = dev_get_drvdata(dev);
302 struct talitos_request *request, saved_req;
303 unsigned long flags;
304 int tail, status;
305
306 spin_lock_irqsave(&priv->tail_lock[ch], flags);
307
308 tail = priv->tail[ch];
309 while (priv->fifo[ch][tail].desc) {
310 request = &priv->fifo[ch][tail];
311
312 /* descriptors with their done bits set don't get the error */
313 rmb();
314 if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
315 status = 0;
316 else
317 if (!error)
318 break;
319 else
320 status = error;
321
322 dma_unmap_single(dev, request->dma_desc,
323 sizeof(struct talitos_desc), DMA_BIDIRECTIONAL);
324
325 /* copy entries so we can call callback outside lock */
326 saved_req.desc = request->desc;
327 saved_req.callback = request->callback;
328 saved_req.context = request->context;
329
330 /* release request entry in fifo */
331 smp_wmb();
332 request->desc = NULL;
333
334 /* increment fifo tail */
335 priv->tail[ch] = (tail + 1) & (priv->fifo_len - 1);
336
337 spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
338 saved_req.callback(dev, saved_req.desc, saved_req.context,
339 status);
340 /* channel may resume processing in single desc error case */
341 if (error && !reset_ch && status == error)
342 return;
343 spin_lock_irqsave(&priv->tail_lock[ch], flags);
344 tail = priv->tail[ch];
345 }
346
347 spin_unlock_irqrestore(&priv->tail_lock[ch], flags);
348}
349
350/*
351 * process completed requests for channels that have done status
352 */
353static void talitos_done(unsigned long data)
354{
355 struct device *dev = (struct device *)data;
356 struct talitos_private *priv = dev_get_drvdata(dev);
357 int ch;
358
359 for (ch = 0; ch < priv->num_channels; ch++)
360 flush_channel(dev, ch, 0, 0);
361}
362
363/*
364 * locate current (offending) descriptor
365 */
366static struct talitos_desc *current_desc(struct device *dev, int ch)
367{
368 struct talitos_private *priv = dev_get_drvdata(dev);
369 int tail = priv->tail[ch];
370 dma_addr_t cur_desc;
371
372 cur_desc = in_be32(priv->reg + TALITOS_CDPR_LO(ch));
373
374 while (priv->fifo[ch][tail].dma_desc != cur_desc) {
375 tail = (tail + 1) & (priv->fifo_len - 1);
376 if (tail == priv->tail[ch]) {
377 dev_err(dev, "couldn't locate current descriptor\n");
378 return NULL;
379 }
380 }
381
382 return priv->fifo[ch][tail].desc;
383}
384
385/*
386 * user diagnostics; report root cause of error based on execution unit status
387 */
388static void report_eu_error(struct device *dev, int ch, struct talitos_desc *desc)
389{
390 struct talitos_private *priv = dev_get_drvdata(dev);
391 int i;
392
393 switch (desc->hdr & DESC_HDR_SEL0_MASK) {
394 case DESC_HDR_SEL0_AFEU:
395 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
396 in_be32(priv->reg + TALITOS_AFEUISR),
397 in_be32(priv->reg + TALITOS_AFEUISR_LO));
398 break;
399 case DESC_HDR_SEL0_DEU:
400 dev_err(dev, "DEUISR 0x%08x_%08x\n",
401 in_be32(priv->reg + TALITOS_DEUISR),
402 in_be32(priv->reg + TALITOS_DEUISR_LO));
403 break;
404 case DESC_HDR_SEL0_MDEUA:
405 case DESC_HDR_SEL0_MDEUB:
406 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
407 in_be32(priv->reg + TALITOS_MDEUISR),
408 in_be32(priv->reg + TALITOS_MDEUISR_LO));
409 break;
410 case DESC_HDR_SEL0_RNG:
411 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
412 in_be32(priv->reg + TALITOS_RNGUISR),
413 in_be32(priv->reg + TALITOS_RNGUISR_LO));
414 break;
415 case DESC_HDR_SEL0_PKEU:
416 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
417 in_be32(priv->reg + TALITOS_PKEUISR),
418 in_be32(priv->reg + TALITOS_PKEUISR_LO));
419 break;
420 case DESC_HDR_SEL0_AESU:
421 dev_err(dev, "AESUISR 0x%08x_%08x\n",
422 in_be32(priv->reg + TALITOS_AESUISR),
423 in_be32(priv->reg + TALITOS_AESUISR_LO));
424 break;
425 case DESC_HDR_SEL0_CRCU:
426 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
427 in_be32(priv->reg + TALITOS_CRCUISR),
428 in_be32(priv->reg + TALITOS_CRCUISR_LO));
429 break;
430 case DESC_HDR_SEL0_KEU:
431 dev_err(dev, "KEUISR 0x%08x_%08x\n",
432 in_be32(priv->reg + TALITOS_KEUISR),
433 in_be32(priv->reg + TALITOS_KEUISR_LO));
434 break;
435 }
436
437 switch (desc->hdr & DESC_HDR_SEL1_MASK) {
438 case DESC_HDR_SEL1_MDEUA:
439 case DESC_HDR_SEL1_MDEUB:
440 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
441 in_be32(priv->reg + TALITOS_MDEUISR),
442 in_be32(priv->reg + TALITOS_MDEUISR_LO));
443 break;
444 case DESC_HDR_SEL1_CRCU:
445 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
446 in_be32(priv->reg + TALITOS_CRCUISR),
447 in_be32(priv->reg + TALITOS_CRCUISR_LO));
448 break;
449 }
450
451 for (i = 0; i < 8; i++)
452 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
453 in_be32(priv->reg + TALITOS_DESCBUF(ch) + 8*i),
454 in_be32(priv->reg + TALITOS_DESCBUF_LO(ch) + 8*i));
455}
456
457/*
458 * recover from error interrupts
459 */
460static void talitos_error(unsigned long data)
461{
462 struct device *dev = (struct device *)data;
463 struct talitos_private *priv = dev_get_drvdata(dev);
464 unsigned int timeout = TALITOS_TIMEOUT;
465 int ch, error, reset_dev = 0, reset_ch = 0;
466 u32 isr, isr_lo, v, v_lo;
467
468 isr = in_be32(priv->reg + TALITOS_ISR);
469 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);
470
471 for (ch = 0; ch < priv->num_channels; ch++) {
472 /* skip channels without errors */
473 if (!(isr & (1 << (ch * 2 + 1))))
474 continue;
475
476 error = -EINVAL;
477
478 v = in_be32(priv->reg + TALITOS_CCPSR(ch));
479 v_lo = in_be32(priv->reg + TALITOS_CCPSR_LO(ch));
480
481 if (v_lo & TALITOS_CCPSR_LO_DOF) {
482 dev_err(dev, "double fetch fifo overflow error\n");
483 error = -EAGAIN;
484 reset_ch = 1;
485 }
486 if (v_lo & TALITOS_CCPSR_LO_SOF) {
487 /* h/w dropped descriptor */
488 dev_err(dev, "single fetch fifo overflow error\n");
489 error = -EAGAIN;
490 }
491 if (v_lo & TALITOS_CCPSR_LO_MDTE)
492 dev_err(dev, "master data transfer error\n");
493 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
494 dev_err(dev, "s/g data length zero error\n");
495 if (v_lo & TALITOS_CCPSR_LO_FPZ)
496 dev_err(dev, "fetch pointer zero error\n");
497 if (v_lo & TALITOS_CCPSR_LO_IDH)
498 dev_err(dev, "illegal descriptor header error\n");
499 if (v_lo & TALITOS_CCPSR_LO_IEU)
500 dev_err(dev, "invalid execution unit error\n");
501 if (v_lo & TALITOS_CCPSR_LO_EU)
502 report_eu_error(dev, ch, current_desc(dev, ch));
503 if (v_lo & TALITOS_CCPSR_LO_GB)
504 dev_err(dev, "gather boundary error\n");
505 if (v_lo & TALITOS_CCPSR_LO_GRL)
506 dev_err(dev, "gather return/length error\n");
507 if (v_lo & TALITOS_CCPSR_LO_SB)
508 dev_err(dev, "scatter boundary error\n");
509 if (v_lo & TALITOS_CCPSR_LO_SRL)
510 dev_err(dev, "scatter return/length error\n");
511
512 flush_channel(dev, ch, error, reset_ch);
513
514 if (reset_ch) {
515 reset_channel(dev, ch);
516 } else {
517 setbits32(priv->reg + TALITOS_CCCR(ch),
518 TALITOS_CCCR_CONT);
519 setbits32(priv->reg + TALITOS_CCCR_LO(ch), 0);
520 while ((in_be32(priv->reg + TALITOS_CCCR(ch)) &
521 TALITOS_CCCR_CONT) && --timeout)
522 cpu_relax();
523 if (timeout == 0) {
524 dev_err(dev, "failed to restart channel %d\n",
525 ch);
526 reset_dev = 1;
527 }
528 }
529 }
530 if (reset_dev || isr & ~TALITOS_ISR_CHERR || isr_lo) {
531 dev_err(dev, "done overflow, internal time out, or rngu error: "
532 "ISR 0x%08x_%08x\n", isr, isr_lo);
533
534 /* purge request queues */
535 for (ch = 0; ch < priv->num_channels; ch++)
536 flush_channel(dev, ch, -EIO, 1);
537
538 /* reset and reinitialize the device */
539 init_device(dev);
540 }
541}
542
543static irqreturn_t talitos_interrupt(int irq, void *data)
544{
545 struct device *dev = data;
546 struct talitos_private *priv = dev_get_drvdata(dev);
547 u32 isr, isr_lo;
548
549 isr = in_be32(priv->reg + TALITOS_ISR);
550 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);
551
552 /* ack */
553 out_be32(priv->reg + TALITOS_ICR, isr);
554 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);
555
556 if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo))
557 talitos_error((unsigned long)data);
558 else
559 if (likely(isr & TALITOS_ISR_CHDONE))
560 tasklet_schedule(&priv->done_task);
561
562 return (isr || isr_lo) ? IRQ_HANDLED : IRQ_NONE;
563}
564
565/*
566 * hwrng
567 */
568static int talitos_rng_data_present(struct hwrng *rng, int wait)
569{
570 struct device *dev = (struct device *)rng->priv;
571 struct talitos_private *priv = dev_get_drvdata(dev);
572 u32 ofl;
573 int i;
574
575 for (i = 0; i < 20; i++) {
576 ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) &
577 TALITOS_RNGUSR_LO_OFL;
578 if (ofl || !wait)
579 break;
580 udelay(10);
581 }
582
583 return !!ofl;
584}
585
586static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
587{
588 struct device *dev = (struct device *)rng->priv;
589 struct talitos_private *priv = dev_get_drvdata(dev);
590
591 /* rng fifo requires 64-bit accesses */
592 *data = in_be32(priv->reg + TALITOS_RNGU_FIFO);
593 *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO);
594
595 return sizeof(u32);
596}
597
598static int talitos_rng_init(struct hwrng *rng)
599{
600 struct device *dev = (struct device *)rng->priv;
601 struct talitos_private *priv = dev_get_drvdata(dev);
602 unsigned int timeout = TALITOS_TIMEOUT;
603
604 setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR);
605 while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD)
606 && --timeout)
607 cpu_relax();
608 if (timeout == 0) {
609 dev_err(dev, "failed to reset rng hw\n");
610 return -ENODEV;
611 }
612
613 /* start generating */
614 setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0);
615
616 return 0;
617}
618
619static int talitos_register_rng(struct device *dev)
620{
621 struct talitos_private *priv = dev_get_drvdata(dev);
622
623 priv->rng.name = dev_driver_string(dev),
624 priv->rng.init = talitos_rng_init,
625 priv->rng.data_present = talitos_rng_data_present,
626 priv->rng.data_read = talitos_rng_data_read,
627 priv->rng.priv = (unsigned long)dev;
628
629 return hwrng_register(&priv->rng);
630}
631
632static void talitos_unregister_rng(struct device *dev)
633{
634 struct talitos_private *priv = dev_get_drvdata(dev);
635
636 hwrng_unregister(&priv->rng);
637}
638
639/*
640 * crypto alg
641 */
642#define TALITOS_CRA_PRIORITY 3000
643#define TALITOS_MAX_KEY_SIZE 64
644#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
645
646#define MD5_DIGEST_SIZE 16
647
648struct talitos_ctx {
649 struct device *dev;
650 __be32 desc_hdr_template;
651 u8 key[TALITOS_MAX_KEY_SIZE];
652 u8 iv[TALITOS_MAX_IV_LENGTH];
653 unsigned int keylen;
654 unsigned int enckeylen;
655 unsigned int authkeylen;
656 unsigned int authsize;
657};
658
659static int aead_authenc_setauthsize(struct crypto_aead *authenc,
660 unsigned int authsize)
661{
662 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
663
664 ctx->authsize = authsize;
665
666 return 0;
667}
668
669static int aead_authenc_setkey(struct crypto_aead *authenc,
670 const u8 *key, unsigned int keylen)
671{
672 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
673 struct rtattr *rta = (void *)key;
674 struct crypto_authenc_key_param *param;
675 unsigned int authkeylen;
676 unsigned int enckeylen;
677
678 if (!RTA_OK(rta, keylen))
679 goto badkey;
680
681 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
682 goto badkey;
683
684 if (RTA_PAYLOAD(rta) < sizeof(*param))
685 goto badkey;
686
687 param = RTA_DATA(rta);
688 enckeylen = be32_to_cpu(param->enckeylen);
689
690 key += RTA_ALIGN(rta->rta_len);
691 keylen -= RTA_ALIGN(rta->rta_len);
692
693 if (keylen < enckeylen)
694 goto badkey;
695
696 authkeylen = keylen - enckeylen;
697
698 if (keylen > TALITOS_MAX_KEY_SIZE)
699 goto badkey;
700
701 memcpy(&ctx->key, key, keylen);
702
703 ctx->keylen = keylen;
704 ctx->enckeylen = enckeylen;
705 ctx->authkeylen = authkeylen;
706
707 return 0;
708
709badkey:
710 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
711 return -EINVAL;
712}
713
714/*
715 * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor
716 * @src_nents: number of segments in input scatterlist
717 * @dst_nents: number of segments in output scatterlist
718 * @dma_len: length of dma mapped link_tbl space
719 * @dma_link_tbl: bus physical address of link_tbl
720 * @desc: h/w descriptor
721 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
722 *
723 * if decrypting (with authcheck), or either one of src_nents or dst_nents
724 * is greater than 1, an integrity check value is concatenated to the end
725 * of link_tbl data
726 */
727struct ipsec_esp_edesc {
728 int src_nents;
729 int dst_nents;
730 int dma_len;
731 dma_addr_t dma_link_tbl;
732 struct talitos_desc desc;
733 struct talitos_ptr link_tbl[0];
734};
735
736static void ipsec_esp_unmap(struct device *dev,
737 struct ipsec_esp_edesc *edesc,
738 struct aead_request *areq)
739{
740 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
741 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
742 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
743 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
744
745 dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE);
746
747 if (areq->src != areq->dst) {
748 dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1,
749 DMA_TO_DEVICE);
750 dma_unmap_sg(dev, areq->dst, edesc->dst_nents ? : 1,
751 DMA_FROM_DEVICE);
752 } else {
753 dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1,
754 DMA_BIDIRECTIONAL);
755 }
756
757 if (edesc->dma_len)
758 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
759 DMA_BIDIRECTIONAL);
760}
761
762/*
763 * ipsec_esp descriptor callbacks
764 */
765static void ipsec_esp_encrypt_done(struct device *dev,
766 struct talitos_desc *desc, void *context,
767 int err)
768{
769 struct aead_request *areq = context;
770 struct ipsec_esp_edesc *edesc =
771 container_of(desc, struct ipsec_esp_edesc, desc);
772 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
773 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
774 struct scatterlist *sg;
775 void *icvdata;
776
777 ipsec_esp_unmap(dev, edesc, areq);
778
779 /* copy the generated ICV to dst */
780 if (edesc->dma_len) {
781 icvdata = &edesc->link_tbl[edesc->src_nents +
782 edesc->dst_nents + 1];
783 sg = sg_last(areq->dst, edesc->dst_nents);
784 memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
785 icvdata, ctx->authsize);
786 }
787
788 kfree(edesc);
789
790 aead_request_complete(areq, err);
791}
792
793static void ipsec_esp_decrypt_done(struct device *dev,
794 struct talitos_desc *desc, void *context,
795 int err)
796{
797 struct aead_request *req = context;
798 struct ipsec_esp_edesc *edesc =
799 container_of(desc, struct ipsec_esp_edesc, desc);
800 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
801 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
802 struct scatterlist *sg;
803 void *icvdata;
804
805 ipsec_esp_unmap(dev, edesc, req);
806
807 if (!err) {
808 /* auth check */
809 if (edesc->dma_len)
810 icvdata = &edesc->link_tbl[edesc->src_nents +
811 edesc->dst_nents + 1];
812 else
813 icvdata = &edesc->link_tbl[0];
814
815 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
816 err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
817 ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
818 }
819
820 kfree(edesc);
821
822 aead_request_complete(req, err);
823}
824
825/*
826 * convert scatterlist to SEC h/w link table format
827 * stop at cryptlen bytes
828 */
829static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
830 int cryptlen, struct talitos_ptr *link_tbl_ptr)
831{
832 int n_sg = sg_count;
833
834 while (n_sg--) {
835 link_tbl_ptr->ptr = cpu_to_be32(sg_dma_address(sg));
836 link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
837 link_tbl_ptr->j_extent = 0;
838 link_tbl_ptr++;
839 cryptlen -= sg_dma_len(sg);
840 sg = sg_next(sg);
841 }
842
843 /* adjust (decrease) last one (or two) entry's len to cryptlen */
844 link_tbl_ptr--;
845 while (link_tbl_ptr->len <= (-cryptlen)) {
846 /* Empty this entry, and move to previous one */
847 cryptlen += be16_to_cpu(link_tbl_ptr->len);
848 link_tbl_ptr->len = 0;
849 sg_count--;
850 link_tbl_ptr--;
851 }
852 link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len)
853 + cryptlen);
854
855 /* tag end of link table */
856 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
857
858 return sg_count;
859}
860
861/*
862 * fill in and submit ipsec_esp descriptor
863 */
864static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
865 u8 *giv, u64 seq,
866 void (*callback) (struct device *dev,
867 struct talitos_desc *desc,
868 void *context, int error))
869{
870 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
871 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
872 struct device *dev = ctx->dev;
873 struct talitos_desc *desc = &edesc->desc;
874 unsigned int cryptlen = areq->cryptlen;
875 unsigned int authsize = ctx->authsize;
876 unsigned int ivsize;
877 int sg_count;
878
879 /* hmac key */
880 map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
881 0, DMA_TO_DEVICE);
882 /* hmac data */
883 map_single_talitos_ptr(dev, &desc->ptr[1], sg_virt(areq->src) -
884 sg_virt(areq->assoc), sg_virt(areq->assoc), 0,
885 DMA_TO_DEVICE);
886 /* cipher iv */
887 ivsize = crypto_aead_ivsize(aead);
888 map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0,
889 DMA_TO_DEVICE);
890
891 /* cipher key */
892 map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
893 (char *)&ctx->key + ctx->authkeylen, 0,
894 DMA_TO_DEVICE);
895
896 /*
897 * cipher in
898 * map and adjust cipher len to aead request cryptlen.
899 * extent is bytes of HMAC postpended to ciphertext,
900 * typically 12 for ipsec
901 */
902 desc->ptr[4].len = cpu_to_be16(cryptlen);
903 desc->ptr[4].j_extent = authsize;
904
905 if (areq->src == areq->dst)
906 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1,
907 DMA_BIDIRECTIONAL);
908 else
909 sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1,
910 DMA_TO_DEVICE);
911
912 if (sg_count == 1) {
913 desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
914 } else {
915 sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,
916 &edesc->link_tbl[0]);
917 if (sg_count > 1) {
918 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
919 desc->ptr[4].ptr = cpu_to_be32(edesc->dma_link_tbl);
920 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
921 edesc->dma_len, DMA_BIDIRECTIONAL);
922 } else {
923 /* Only one segment now, so no link tbl needed */
924 desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
925 }
926 }
927
928 /* cipher out */
929 desc->ptr[5].len = cpu_to_be16(cryptlen);
930 desc->ptr[5].j_extent = authsize;
931
932 if (areq->src != areq->dst) {
933 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1,
934 DMA_FROM_DEVICE);
935 }
936
937 if (sg_count == 1) {
938 desc->ptr[5].ptr = cpu_to_be32(sg_dma_address(areq->dst));
939 } else {
940 struct talitos_ptr *link_tbl_ptr =
941 &edesc->link_tbl[edesc->src_nents];
942 struct scatterlist *sg;
943
944 desc->ptr[5].ptr = cpu_to_be32((struct talitos_ptr *)
945 edesc->dma_link_tbl +
946 edesc->src_nents);
947 if (areq->src == areq->dst) {
948 memcpy(link_tbl_ptr, &edesc->link_tbl[0],
949 edesc->src_nents * sizeof(struct talitos_ptr));
950 } else {
951 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
952 link_tbl_ptr);
953 }
954 link_tbl_ptr += sg_count - 1;
955
956 /* handle case where sg_last contains the ICV exclusively */
957 sg = sg_last(areq->dst, edesc->dst_nents);
958 if (sg->length == ctx->authsize)
959 link_tbl_ptr--;
960
961 link_tbl_ptr->j_extent = 0;
962 link_tbl_ptr++;
963 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
964 link_tbl_ptr->len = cpu_to_be16(authsize);
965
966 /* icv data follows link tables */
967 link_tbl_ptr->ptr = cpu_to_be32((struct talitos_ptr *)
968 edesc->dma_link_tbl +
969 edesc->src_nents +
970 edesc->dst_nents + 1);
971
972 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
973 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
974 edesc->dma_len, DMA_BIDIRECTIONAL);
975 }
976
977 /* iv out */
978 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0,
979 DMA_FROM_DEVICE);
980
981 return talitos_submit(dev, desc, callback, areq);
982}
983
984
985/*
986 * derive number of elements in scatterlist
987 */
988static int sg_count(struct scatterlist *sg_list, int nbytes)
989{
990 struct scatterlist *sg = sg_list;
991 int sg_nents = 0;
992
993 while (nbytes) {
994 sg_nents++;
995 nbytes -= sg->length;
996 sg = sg_next(sg);
997 }
998
999 return sg_nents;
1000}
1001
1002/*
1003 * allocate and map the ipsec_esp extended descriptor
1004 */
1005static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq,
1006 int icv_stashing)
1007{
1008 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1009 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1010 struct ipsec_esp_edesc *edesc;
1011 int src_nents, dst_nents, alloc_len, dma_len;
1012
1013 if (areq->cryptlen + ctx->authsize > TALITOS_MAX_DATA_LEN) {
1014 dev_err(ctx->dev, "cryptlen exceeds h/w max limit\n");
1015 return ERR_PTR(-EINVAL);
1016 }
1017
1018 src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize);
1019 src_nents = (src_nents == 1) ? 0 : src_nents;
1020
1021 if (areq->dst == areq->src) {
1022 dst_nents = src_nents;
1023 } else {
1024 dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize);
1025 dst_nents = (dst_nents == 1) ? 0 : src_nents;
1026 }
1027
1028 /*
1029 * allocate space for base edesc plus the link tables,
1030 * allowing for a separate entry for the generated ICV (+ 1),
1031 * and the ICV data itself
1032 */
1033 alloc_len = sizeof(struct ipsec_esp_edesc);
1034 if (src_nents || dst_nents) {
1035 dma_len = (src_nents + dst_nents + 1) *
1036 sizeof(struct talitos_ptr) + ctx->authsize;
1037 alloc_len += dma_len;
1038 } else {
1039 dma_len = 0;
1040 alloc_len += icv_stashing ? ctx->authsize : 0;
1041 }
1042
1043 edesc = kmalloc(alloc_len, GFP_DMA);
1044 if (!edesc) {
1045 dev_err(ctx->dev, "could not allocate edescriptor\n");
1046 return ERR_PTR(-ENOMEM);
1047 }
1048
1049 edesc->src_nents = src_nents;
1050 edesc->dst_nents = dst_nents;
1051 edesc->dma_len = dma_len;
1052 edesc->dma_link_tbl = dma_map_single(ctx->dev, &edesc->link_tbl[0],
1053 edesc->dma_len, DMA_BIDIRECTIONAL);
1054
1055 return edesc;
1056}
1057
1058static int aead_authenc_encrypt(struct aead_request *req)
1059{
1060 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1061 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1062 struct ipsec_esp_edesc *edesc;
1063
1064 /* allocate extended descriptor */
1065 edesc = ipsec_esp_edesc_alloc(req, 0);
1066 if (IS_ERR(edesc))
1067 return PTR_ERR(edesc);
1068
1069 /* set encrypt */
1070 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1071
1072 return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done);
1073}
1074
1075static int aead_authenc_decrypt(struct aead_request *req)
1076{
1077 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1078 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1079 unsigned int authsize = ctx->authsize;
1080 struct ipsec_esp_edesc *edesc;
1081 struct scatterlist *sg;
1082 void *icvdata;
1083
1084 req->cryptlen -= authsize;
1085
1086 /* allocate extended descriptor */
1087 edesc = ipsec_esp_edesc_alloc(req, 1);
1088 if (IS_ERR(edesc))
1089 return PTR_ERR(edesc);
1090
1091 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1092 if (edesc->dma_len)
1093 icvdata = &edesc->link_tbl[edesc->src_nents +
1094 edesc->dst_nents + 1];
1095 else
1096 icvdata = &edesc->link_tbl[0];
1097
1098 sg = sg_last(req->src, edesc->src_nents ? : 1);
1099
1100 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
1101 ctx->authsize);
1102
1103 /* decrypt */
1104 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1105
1106 return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_done);
1107}
1108
1109static int aead_authenc_givencrypt(
1110 struct aead_givcrypt_request *req)
1111{
1112 struct aead_request *areq = &req->areq;
1113 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1114 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1115 struct ipsec_esp_edesc *edesc;
1116
1117 /* allocate extended descriptor */
1118 edesc = ipsec_esp_edesc_alloc(areq, 0);
1119 if (IS_ERR(edesc))
1120 return PTR_ERR(edesc);
1121
1122 /* set encrypt */
1123 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1124
1125 memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
1126
1127 return ipsec_esp(edesc, areq, req->giv, req->seq,
1128 ipsec_esp_encrypt_done);
1129}
1130
1131struct talitos_alg_template {
1132 char name[CRYPTO_MAX_ALG_NAME];
1133 char driver_name[CRYPTO_MAX_ALG_NAME];
1134 unsigned int blocksize;
1135 struct aead_alg aead;
1136 struct device *dev;
1137 __be32 desc_hdr_template;
1138};
1139
1140static struct talitos_alg_template driver_algs[] = {
1141 /* single-pass ipsec_esp descriptor */
1142 {
1143 .name = "authenc(hmac(sha1),cbc(aes))",
1144 .driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
1145 .blocksize = AES_BLOCK_SIZE,
1146 .aead = {
1147 .setkey = aead_authenc_setkey,
1148 .setauthsize = aead_authenc_setauthsize,
1149 .encrypt = aead_authenc_encrypt,
1150 .decrypt = aead_authenc_decrypt,
1151 .givencrypt = aead_authenc_givencrypt,
1152 .geniv = "<built-in>",
1153 .ivsize = AES_BLOCK_SIZE,
1154 .maxauthsize = SHA1_DIGEST_SIZE,
1155 },
1156 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1157 DESC_HDR_SEL0_AESU |
1158 DESC_HDR_MODE0_AESU_CBC |
1159 DESC_HDR_SEL1_MDEUA |
1160 DESC_HDR_MODE1_MDEU_INIT |
1161 DESC_HDR_MODE1_MDEU_PAD |
1162 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1163 },
1164 {
1165 .name = "authenc(hmac(sha1),cbc(des3_ede))",
1166 .driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
1167 .blocksize = DES3_EDE_BLOCK_SIZE,
1168 .aead = {
1169 .setkey = aead_authenc_setkey,
1170 .setauthsize = aead_authenc_setauthsize,
1171 .encrypt = aead_authenc_encrypt,
1172 .decrypt = aead_authenc_decrypt,
1173 .givencrypt = aead_authenc_givencrypt,
1174 .geniv = "<built-in>",
1175 .ivsize = DES3_EDE_BLOCK_SIZE,
1176 .maxauthsize = SHA1_DIGEST_SIZE,
1177 },
1178 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1179 DESC_HDR_SEL0_DEU |
1180 DESC_HDR_MODE0_DEU_CBC |
1181 DESC_HDR_MODE0_DEU_3DES |
1182 DESC_HDR_SEL1_MDEUA |
1183 DESC_HDR_MODE1_MDEU_INIT |
1184 DESC_HDR_MODE1_MDEU_PAD |
1185 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1186 },
1187 {
1188 .name = "authenc(hmac(sha256),cbc(aes))",
1189 .driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
1190 .blocksize = AES_BLOCK_SIZE,
1191 .aead = {
1192 .setkey = aead_authenc_setkey,
1193 .setauthsize = aead_authenc_setauthsize,
1194 .encrypt = aead_authenc_encrypt,
1195 .decrypt = aead_authenc_decrypt,
1196 .givencrypt = aead_authenc_givencrypt,
1197 .geniv = "<built-in>",
1198 .ivsize = AES_BLOCK_SIZE,
1199 .maxauthsize = SHA256_DIGEST_SIZE,
1200 },
1201 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1202 DESC_HDR_SEL0_AESU |
1203 DESC_HDR_MODE0_AESU_CBC |
1204 DESC_HDR_SEL1_MDEUA |
1205 DESC_HDR_MODE1_MDEU_INIT |
1206 DESC_HDR_MODE1_MDEU_PAD |
1207 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1208 },
1209 {
1210 .name = "authenc(hmac(sha256),cbc(des3_ede))",
1211 .driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
1212 .blocksize = DES3_EDE_BLOCK_SIZE,
1213 .aead = {
1214 .setkey = aead_authenc_setkey,
1215 .setauthsize = aead_authenc_setauthsize,
1216 .encrypt = aead_authenc_encrypt,
1217 .decrypt = aead_authenc_decrypt,
1218 .givencrypt = aead_authenc_givencrypt,
1219 .geniv = "<built-in>",
1220 .ivsize = DES3_EDE_BLOCK_SIZE,
1221 .maxauthsize = SHA256_DIGEST_SIZE,
1222 },
1223 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1224 DESC_HDR_SEL0_DEU |
1225 DESC_HDR_MODE0_DEU_CBC |
1226 DESC_HDR_MODE0_DEU_3DES |
1227 DESC_HDR_SEL1_MDEUA |
1228 DESC_HDR_MODE1_MDEU_INIT |
1229 DESC_HDR_MODE1_MDEU_PAD |
1230 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1231 },
1232 {
1233 .name = "authenc(hmac(md5),cbc(aes))",
1234 .driver_name = "authenc-hmac-md5-cbc-aes-talitos",
1235 .blocksize = AES_BLOCK_SIZE,
1236 .aead = {
1237 .setkey = aead_authenc_setkey,
1238 .setauthsize = aead_authenc_setauthsize,
1239 .encrypt = aead_authenc_encrypt,
1240 .decrypt = aead_authenc_decrypt,
1241 .givencrypt = aead_authenc_givencrypt,
1242 .geniv = "<built-in>",
1243 .ivsize = AES_BLOCK_SIZE,
1244 .maxauthsize = MD5_DIGEST_SIZE,
1245 },
1246 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1247 DESC_HDR_SEL0_AESU |
1248 DESC_HDR_MODE0_AESU_CBC |
1249 DESC_HDR_SEL1_MDEUA |
1250 DESC_HDR_MODE1_MDEU_INIT |
1251 DESC_HDR_MODE1_MDEU_PAD |
1252 DESC_HDR_MODE1_MDEU_MD5_HMAC,
1253 },
1254 {
1255 .name = "authenc(hmac(md5),cbc(des3_ede))",
1256 .driver_name = "authenc-hmac-md5-cbc-3des-talitos",
1257 .blocksize = DES3_EDE_BLOCK_SIZE,
1258 .aead = {
1259 .setkey = aead_authenc_setkey,
1260 .setauthsize = aead_authenc_setauthsize,
1261 .encrypt = aead_authenc_encrypt,
1262 .decrypt = aead_authenc_decrypt,
1263 .givencrypt = aead_authenc_givencrypt,
1264 .geniv = "<built-in>",
1265 .ivsize = DES3_EDE_BLOCK_SIZE,
1266 .maxauthsize = MD5_DIGEST_SIZE,
1267 },
1268 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1269 DESC_HDR_SEL0_DEU |
1270 DESC_HDR_MODE0_DEU_CBC |
1271 DESC_HDR_MODE0_DEU_3DES |
1272 DESC_HDR_SEL1_MDEUA |
1273 DESC_HDR_MODE1_MDEU_INIT |
1274 DESC_HDR_MODE1_MDEU_PAD |
1275 DESC_HDR_MODE1_MDEU_MD5_HMAC,
1276 }
1277};
1278
1279struct talitos_crypto_alg {
1280 struct list_head entry;
1281 struct device *dev;
1282 __be32 desc_hdr_template;
1283 struct crypto_alg crypto_alg;
1284};
1285
1286static int talitos_cra_init(struct crypto_tfm *tfm)
1287{
1288 struct crypto_alg *alg = tfm->__crt_alg;
1289 struct talitos_crypto_alg *talitos_alg =
1290 container_of(alg, struct talitos_crypto_alg, crypto_alg);
1291 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
1292
1293 /* update context with ptr to dev */
1294 ctx->dev = talitos_alg->dev;
1295 /* copy descriptor header template value */
1296 ctx->desc_hdr_template = talitos_alg->desc_hdr_template;
1297
1298 /* random first IV */
1299 get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
1300
1301 return 0;
1302}
1303
1304/*
1305 * given the alg's descriptor header template, determine whether descriptor
1306 * type and primary/secondary execution units required match the hw
1307 * capabilities description provided in the device tree node.
1308 */
1309static int hw_supports(struct device *dev, __be32 desc_hdr_template)
1310{
1311 struct talitos_private *priv = dev_get_drvdata(dev);
1312 int ret;
1313
1314 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
1315 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
1316
1317 if (SECONDARY_EU(desc_hdr_template))
1318 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
1319 & priv->exec_units);
1320
1321 return ret;
1322}
1323
1324static int __devexit talitos_remove(struct of_device *ofdev)
1325{
1326 struct device *dev = &ofdev->dev;
1327 struct talitos_private *priv = dev_get_drvdata(dev);
1328 struct talitos_crypto_alg *t_alg, *n;
1329 int i;
1330
1331 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
1332 crypto_unregister_alg(&t_alg->crypto_alg);
1333 list_del(&t_alg->entry);
1334 kfree(t_alg);
1335 }
1336
1337 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
1338 talitos_unregister_rng(dev);
1339
1340 kfree(priv->tail);
1341 kfree(priv->head);
1342
1343 if (priv->fifo)
1344 for (i = 0; i < priv->num_channels; i++)
1345 kfree(priv->fifo[i]);
1346
1347 kfree(priv->fifo);
1348 kfree(priv->head_lock);
1349 kfree(priv->tail_lock);
1350
1351 if (priv->irq != NO_IRQ) {
1352 free_irq(priv->irq, dev);
1353 irq_dispose_mapping(priv->irq);
1354 }
1355
1356 tasklet_kill(&priv->done_task);
1357 tasklet_kill(&priv->error_task);
1358
1359 iounmap(priv->reg);
1360
1361 dev_set_drvdata(dev, NULL);
1362
1363 kfree(priv);
1364
1365 return 0;
1366}
1367
1368static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
1369 struct talitos_alg_template
1370 *template)
1371{
1372 struct talitos_crypto_alg *t_alg;
1373 struct crypto_alg *alg;
1374
1375 t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
1376 if (!t_alg)
1377 return ERR_PTR(-ENOMEM);
1378
1379 alg = &t_alg->crypto_alg;
1380
1381 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
1382 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1383 template->driver_name);
1384 alg->cra_module = THIS_MODULE;
1385 alg->cra_init = talitos_cra_init;
1386 alg->cra_priority = TALITOS_CRA_PRIORITY;
1387 alg->cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
1388 alg->cra_blocksize = template->blocksize;
1389 alg->cra_alignmask = 0;
1390 alg->cra_type = &crypto_aead_type;
1391 alg->cra_ctxsize = sizeof(struct talitos_ctx);
1392 alg->cra_u.aead = template->aead;
1393
1394 t_alg->desc_hdr_template = template->desc_hdr_template;
1395 t_alg->dev = dev;
1396
1397 return t_alg;
1398}
1399
1400static int talitos_probe(struct of_device *ofdev,
1401 const struct of_device_id *match)
1402{
1403 struct device *dev = &ofdev->dev;
1404 struct device_node *np = ofdev->node;
1405 struct talitos_private *priv;
1406 const unsigned int *prop;
1407 int i, err;
1408
1409 priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
1410 if (!priv)
1411 return -ENOMEM;
1412
1413 dev_set_drvdata(dev, priv);
1414
1415 priv->ofdev = ofdev;
1416
1417 tasklet_init(&priv->done_task, talitos_done, (unsigned long)dev);
1418 tasklet_init(&priv->error_task, talitos_error, (unsigned long)dev);
1419
1420 priv->irq = irq_of_parse_and_map(np, 0);
1421
1422 if (priv->irq == NO_IRQ) {
1423 dev_err(dev, "failed to map irq\n");
1424 err = -EINVAL;
1425 goto err_out;
1426 }
1427
1428 /* get the irq line */
1429 err = request_irq(priv->irq, talitos_interrupt, 0,
1430 dev_driver_string(dev), dev);
1431 if (err) {
1432 dev_err(dev, "failed to request irq %d\n", priv->irq);
1433 irq_dispose_mapping(priv->irq);
1434 priv->irq = NO_IRQ;
1435 goto err_out;
1436 }
1437
1438 priv->reg = of_iomap(np, 0);
1439 if (!priv->reg) {
1440 dev_err(dev, "failed to of_iomap\n");
1441 err = -ENOMEM;
1442 goto err_out;
1443 }
1444
1445 /* get SEC version capabilities from device tree */
1446 prop = of_get_property(np, "fsl,num-channels", NULL);
1447 if (prop)
1448 priv->num_channels = *prop;
1449
1450 prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
1451 if (prop)
1452 priv->chfifo_len = *prop;
1453
1454 prop = of_get_property(np, "fsl,exec-units-mask", NULL);
1455 if (prop)
1456 priv->exec_units = *prop;
1457
1458 prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
1459 if (prop)
1460 priv->desc_types = *prop;
1461
1462 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
1463 !priv->exec_units || !priv->desc_types) {
1464 dev_err(dev, "invalid property data in device tree node\n");
1465 err = -EINVAL;
1466 goto err_out;
1467 }
1468
1469 of_node_put(np);
1470 np = NULL;
1471
1472 priv->head_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
1473 GFP_KERNEL);
1474 priv->tail_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels,
1475 GFP_KERNEL);
1476 if (!priv->head_lock || !priv->tail_lock) {
1477 dev_err(dev, "failed to allocate fifo locks\n");
1478 err = -ENOMEM;
1479 goto err_out;
1480 }
1481
1482 for (i = 0; i < priv->num_channels; i++) {
1483 spin_lock_init(&priv->head_lock[i]);
1484 spin_lock_init(&priv->tail_lock[i]);
1485 }
1486
1487 priv->fifo = kmalloc(sizeof(struct talitos_request *) *
1488 priv->num_channels, GFP_KERNEL);
1489 if (!priv->fifo) {
1490 dev_err(dev, "failed to allocate request fifo\n");
1491 err = -ENOMEM;
1492 goto err_out;
1493 }
1494
1495 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
1496
1497 for (i = 0; i < priv->num_channels; i++) {
1498 priv->fifo[i] = kzalloc(sizeof(struct talitos_request) *
1499 priv->fifo_len, GFP_KERNEL);
1500 if (!priv->fifo[i]) {
1501 dev_err(dev, "failed to allocate request fifo %d\n", i);
1502 err = -ENOMEM;
1503 goto err_out;
1504 }
1505 }
1506
1507 priv->head = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
1508 priv->tail = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL);
1509 if (!priv->head || !priv->tail) {
1510 dev_err(dev, "failed to allocate request index space\n");
1511 err = -ENOMEM;
1512 goto err_out;
1513 }
1514
1515 /* reset and initialize the h/w */
1516 err = init_device(dev);
1517 if (err) {
1518 dev_err(dev, "failed to initialize device\n");
1519 goto err_out;
1520 }
1521
1522 /* register the RNG, if available */
1523 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
1524 err = talitos_register_rng(dev);
1525 if (err) {
1526 dev_err(dev, "failed to register hwrng: %d\n", err);
1527 goto err_out;
1528 } else
1529 dev_info(dev, "hwrng\n");
1530 }
1531
1532 /* register crypto algorithms the device supports */
1533 INIT_LIST_HEAD(&priv->alg_list);
1534
1535 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
1536 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
1537 struct talitos_crypto_alg *t_alg;
1538
1539 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
1540 if (IS_ERR(t_alg)) {
1541 err = PTR_ERR(t_alg);
1542 goto err_out;
1543 }
1544
1545 err = crypto_register_alg(&t_alg->crypto_alg);
1546 if (err) {
1547 dev_err(dev, "%s alg registration failed\n",
1548 t_alg->crypto_alg.cra_driver_name);
1549 kfree(t_alg);
1550 } else {
1551 list_add_tail(&t_alg->entry, &priv->alg_list);
1552 dev_info(dev, "%s\n",
1553 t_alg->crypto_alg.cra_driver_name);
1554 }
1555 }
1556 }
1557
1558 return 0;
1559
1560err_out:
1561 talitos_remove(ofdev);
1562 if (np)
1563 of_node_put(np);
1564
1565 return err;
1566}
1567
1568static struct of_device_id talitos_match[] = {
1569 {
1570 .compatible = "fsl,sec2.0",
1571 },
1572 {},
1573};
1574MODULE_DEVICE_TABLE(of, talitos_match);
1575
1576static struct of_platform_driver talitos_driver = {
1577 .name = "talitos",
1578 .match_table = talitos_match,
1579 .probe = talitos_probe,
1580 .remove = __devexit_p(talitos_remove),
1581};
1582
1583static int __init talitos_init(void)
1584{
1585 return of_register_platform_driver(&talitos_driver);
1586}
1587module_init(talitos_init);
1588
1589static void __exit talitos_exit(void)
1590{
1591 of_unregister_platform_driver(&talitos_driver);
1592}
1593module_exit(talitos_exit);
1594
1595MODULE_LICENSE("GPL");
1596MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
1597MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h
new file mode 100644
index 000000000000..c48a405abf70
--- /dev/null
+++ b/drivers/crypto/talitos.h
@@ -0,0 +1,199 @@
1/*
2 * Freescale SEC (talitos) device register and descriptor header defines
3 *
4 * Copyright (c) 2006-2008 Freescale Semiconductor, Inc.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 * derived from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 */
30
31/*
32 * TALITOS_xxx_LO addresses point to the low data bits (32-63) of the register
33 */
34
35/* global register offset addresses */
36#define TALITOS_MCR 0x1030 /* master control register */
37#define TALITOS_MCR_LO 0x1038
38#define TALITOS_MCR_SWR 0x1 /* s/w reset */
39#define TALITOS_IMR 0x1008 /* interrupt mask register */
40#define TALITOS_IMR_INIT 0x10fff /* enable channel IRQs */
41#define TALITOS_IMR_LO 0x100C
42#define TALITOS_IMR_LO_INIT 0x20000 /* allow RNGU error IRQs */
43#define TALITOS_ISR 0x1010 /* interrupt status register */
44#define TALITOS_ISR_CHERR 0xaa /* channel errors mask */
45#define TALITOS_ISR_CHDONE 0x55 /* channel done mask */
46#define TALITOS_ISR_LO 0x1014
47#define TALITOS_ICR 0x1018 /* interrupt clear register */
48#define TALITOS_ICR_LO 0x101C
49
50/* channel register address stride */
51#define TALITOS_CH_STRIDE 0x100
52
53/* channel configuration register */
54#define TALITOS_CCCR(ch) (ch * TALITOS_CH_STRIDE + 0x1108)
55#define TALITOS_CCCR_CONT 0x2 /* channel continue */
56#define TALITOS_CCCR_RESET 0x1 /* channel reset */
57#define TALITOS_CCCR_LO(ch) (ch * TALITOS_CH_STRIDE + 0x110c)
58#define TALITOS_CCCR_LO_CDWE 0x10 /* chan. done writeback enab. */
59#define TALITOS_CCCR_LO_NT 0x4 /* notification type */
60#define TALITOS_CCCR_LO_CDIE 0x2 /* channel done IRQ enable */
61
62/* CCPSR: channel pointer status register */
63#define TALITOS_CCPSR(ch) (ch * TALITOS_CH_STRIDE + 0x1110)
64#define TALITOS_CCPSR_LO(ch) (ch * TALITOS_CH_STRIDE + 0x1114)
65#define TALITOS_CCPSR_LO_DOF 0x8000 /* double FF write oflow error */
66#define TALITOS_CCPSR_LO_SOF 0x4000 /* single FF write oflow error */
67#define TALITOS_CCPSR_LO_MDTE 0x2000 /* master data transfer error */
68#define TALITOS_CCPSR_LO_SGDLZ 0x1000 /* s/g data len zero error */
69#define TALITOS_CCPSR_LO_FPZ 0x0800 /* fetch ptr zero error */
70#define TALITOS_CCPSR_LO_IDH 0x0400 /* illegal desc hdr error */
71#define TALITOS_CCPSR_LO_IEU 0x0200 /* invalid EU error */
72#define TALITOS_CCPSR_LO_EU 0x0100 /* EU error detected */
73#define TALITOS_CCPSR_LO_GB 0x0080 /* gather boundary error */
74#define TALITOS_CCPSR_LO_GRL 0x0040 /* gather return/length error */
75#define TALITOS_CCPSR_LO_SB 0x0020 /* scatter boundary error */
76#define TALITOS_CCPSR_LO_SRL 0x0010 /* scatter return/length error */
77
78/* channel fetch fifo register */
79#define TALITOS_FF(ch) (ch * TALITOS_CH_STRIDE + 0x1148)
80#define TALITOS_FF_LO(ch) (ch * TALITOS_CH_STRIDE + 0x114c)
81
82/* current descriptor pointer register */
83#define TALITOS_CDPR(ch) (ch * TALITOS_CH_STRIDE + 0x1140)
84#define TALITOS_CDPR_LO(ch) (ch * TALITOS_CH_STRIDE + 0x1144)
85
86/* descriptor buffer register */
87#define TALITOS_DESCBUF(ch) (ch * TALITOS_CH_STRIDE + 0x1180)
88#define TALITOS_DESCBUF_LO(ch) (ch * TALITOS_CH_STRIDE + 0x1184)
89
90/* gather link table */
91#define TALITOS_GATHER(ch) (ch * TALITOS_CH_STRIDE + 0x11c0)
92#define TALITOS_GATHER_LO(ch) (ch * TALITOS_CH_STRIDE + 0x11c4)
93
94/* scatter link table */
95#define TALITOS_SCATTER(ch) (ch * TALITOS_CH_STRIDE + 0x11e0)
96#define TALITOS_SCATTER_LO(ch) (ch * TALITOS_CH_STRIDE + 0x11e4)
97
98/* execution unit interrupt status registers */
99#define TALITOS_DEUISR 0x2030 /* DES unit */
100#define TALITOS_DEUISR_LO 0x2034
101#define TALITOS_AESUISR 0x4030 /* AES unit */
102#define TALITOS_AESUISR_LO 0x4034
103#define TALITOS_MDEUISR 0x6030 /* message digest unit */
104#define TALITOS_MDEUISR_LO 0x6034
105#define TALITOS_AFEUISR 0x8030 /* arc4 unit */
106#define TALITOS_AFEUISR_LO 0x8034
107#define TALITOS_RNGUISR 0xa030 /* random number unit */
108#define TALITOS_RNGUISR_LO 0xa034
109#define TALITOS_RNGUSR 0xa028 /* rng status */
110#define TALITOS_RNGUSR_LO 0xa02c
111#define TALITOS_RNGUSR_LO_RD 0x1 /* reset done */
112#define TALITOS_RNGUSR_LO_OFL 0xff0000/* output FIFO length */
113#define TALITOS_RNGUDSR 0xa010 /* data size */
114#define TALITOS_RNGUDSR_LO 0xa014
115#define TALITOS_RNGU_FIFO 0xa800 /* output FIFO */
116#define TALITOS_RNGU_FIFO_LO 0xa804 /* output FIFO */
117#define TALITOS_RNGURCR 0xa018 /* reset control */
118#define TALITOS_RNGURCR_LO 0xa01c
119#define TALITOS_RNGURCR_LO_SR 0x1 /* software reset */
120#define TALITOS_PKEUISR 0xc030 /* public key unit */
121#define TALITOS_PKEUISR_LO 0xc034
122#define TALITOS_KEUISR 0xe030 /* kasumi unit */
123#define TALITOS_KEUISR_LO 0xe034
124#define TALITOS_CRCUISR 0xf030 /* cyclic redundancy check unit*/
125#define TALITOS_CRCUISR_LO 0xf034
126
127/*
128 * talitos descriptor header (hdr) bits
129 */
130
131/* written back when done */
132#define DESC_HDR_DONE __constant_cpu_to_be32(0xff000000)
133
134/* primary execution unit select */
135#define DESC_HDR_SEL0_MASK __constant_cpu_to_be32(0xf0000000)
136#define DESC_HDR_SEL0_AFEU __constant_cpu_to_be32(0x10000000)
137#define DESC_HDR_SEL0_DEU __constant_cpu_to_be32(0x20000000)
138#define DESC_HDR_SEL0_MDEUA __constant_cpu_to_be32(0x30000000)
139#define DESC_HDR_SEL0_MDEUB __constant_cpu_to_be32(0xb0000000)
140#define DESC_HDR_SEL0_RNG __constant_cpu_to_be32(0x40000000)
141#define DESC_HDR_SEL0_PKEU __constant_cpu_to_be32(0x50000000)
142#define DESC_HDR_SEL0_AESU __constant_cpu_to_be32(0x60000000)
143#define DESC_HDR_SEL0_KEU __constant_cpu_to_be32(0x70000000)
144#define DESC_HDR_SEL0_CRCU __constant_cpu_to_be32(0x80000000)
145
146/* primary execution unit mode (MODE0) and derivatives */
147#define DESC_HDR_MODE0_ENCRYPT __constant_cpu_to_be32(0x00100000)
148#define DESC_HDR_MODE0_AESU_CBC __constant_cpu_to_be32(0x00200000)
149#define DESC_HDR_MODE0_DEU_CBC __constant_cpu_to_be32(0x00400000)
150#define DESC_HDR_MODE0_DEU_3DES __constant_cpu_to_be32(0x00200000)
151#define DESC_HDR_MODE0_MDEU_INIT __constant_cpu_to_be32(0x01000000)
152#define DESC_HDR_MODE0_MDEU_HMAC __constant_cpu_to_be32(0x00800000)
153#define DESC_HDR_MODE0_MDEU_PAD __constant_cpu_to_be32(0x00400000)
154#define DESC_HDR_MODE0_MDEU_MD5 __constant_cpu_to_be32(0x00200000)
155#define DESC_HDR_MODE0_MDEU_SHA256 __constant_cpu_to_be32(0x00100000)
156#define DESC_HDR_MODE0_MDEU_SHA1 __constant_cpu_to_be32(0x00000000)
157#define DESC_HDR_MODE0_MDEU_MD5_HMAC (DESC_HDR_MODE0_MDEU_MD5 | \
158 DESC_HDR_MODE0_MDEU_HMAC)
159#define DESC_HDR_MODE0_MDEU_SHA256_HMAC (DESC_HDR_MODE0_MDEU_SHA256 | \
160 DESC_HDR_MODE0_MDEU_HMAC)
161#define DESC_HDR_MODE0_MDEU_SHA1_HMAC (DESC_HDR_MODE0_MDEU_SHA1 | \
162 DESC_HDR_MODE0_MDEU_HMAC)
163
164/* secondary execution unit select (SEL1) */
165#define DESC_HDR_SEL1_MASK __constant_cpu_to_be32(0x000f0000)
166#define DESC_HDR_SEL1_MDEUA __constant_cpu_to_be32(0x00030000)
167#define DESC_HDR_SEL1_MDEUB __constant_cpu_to_be32(0x000b0000)
168#define DESC_HDR_SEL1_CRCU __constant_cpu_to_be32(0x00080000)
169
170/* secondary execution unit mode (MODE1) and derivatives */
171#define DESC_HDR_MODE1_MDEU_INIT __constant_cpu_to_be32(0x00001000)
172#define DESC_HDR_MODE1_MDEU_HMAC __constant_cpu_to_be32(0x00000800)
173#define DESC_HDR_MODE1_MDEU_PAD __constant_cpu_to_be32(0x00000400)
174#define DESC_HDR_MODE1_MDEU_MD5 __constant_cpu_to_be32(0x00000200)
175#define DESC_HDR_MODE1_MDEU_SHA256 __constant_cpu_to_be32(0x00000100)
176#define DESC_HDR_MODE1_MDEU_SHA1 __constant_cpu_to_be32(0x00000000)
177#define DESC_HDR_MODE1_MDEU_MD5_HMAC (DESC_HDR_MODE1_MDEU_MD5 | \
178 DESC_HDR_MODE1_MDEU_HMAC)
179#define DESC_HDR_MODE1_MDEU_SHA256_HMAC (DESC_HDR_MODE1_MDEU_SHA256 | \
180 DESC_HDR_MODE1_MDEU_HMAC)
181#define DESC_HDR_MODE1_MDEU_SHA1_HMAC (DESC_HDR_MODE1_MDEU_SHA1 | \
182 DESC_HDR_MODE1_MDEU_HMAC)
183
184/* direction of overall data flow (DIR) */
185#define DESC_HDR_DIR_INBOUND __constant_cpu_to_be32(0x00000002)
186
187/* request done notification (DN) */
188#define DESC_HDR_DONE_NOTIFY __constant_cpu_to_be32(0x00000001)
189
190/* descriptor types */
191#define DESC_HDR_TYPE_AESU_CTR_NONSNOOP __constant_cpu_to_be32(0 << 3)
192#define DESC_HDR_TYPE_IPSEC_ESP __constant_cpu_to_be32(1 << 3)
193#define DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU __constant_cpu_to_be32(2 << 3)
194#define DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU __constant_cpu_to_be32(4 << 3)
195
196/* link table extent field bits */
197#define DESC_PTR_LNKTBL_JUMP 0x80
198#define DESC_PTR_LNKTBL_RETURN 0x02
199#define DESC_PTR_LNKTBL_NEXT 0x01
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-14 07:15:51 -0500