aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2011-07-24 12:05:32 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2011-07-24 12:05:32 -0400
commita23a334bd547e9462d9ca4a74608519a1e928848 (patch)
treee3d4f4423130f0d74f141c9bbd0c0874690e38b3 /drivers
parenta642285014df03b8f320399d515bf3b779af07ac (diff)
parentacdca31dba86c4f426460aa000d13930a00549b7 (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
* git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (34 commits) crypto: caam - ablkcipher support crypto: caam - faster aead implementation crypto: caam - structure renaming crypto: caam - shorter names crypto: talitos - don't bad_key in ablkcipher setkey crypto: talitos - remove unused giv from ablkcipher methods crypto: talitos - don't set done notification in hot path crypto: talitos - ensure request ordering within a single tfm crypto: gf128mul - fix call to memset() crypto: s390 - support hardware accelerated SHA-224 crypto: algif_hash - Handle initial af_alg_make_sg error correctly crypto: sha1_generic - use SHA1_BLOCK_SIZE hwrng: ppc4xx - add support for ppc4xx TRNG crypto: crypto4xx - Perform read/modify/write on device control register crypto: caam - fix build warning when DEBUG_FS not configured crypto: arc4 - Fixed coding style issues crypto: crc32c - Fixed coding style issue crypto: omap-sham - do not schedule tasklet if there is no active requests crypto: omap-sham - clear device flags when finishing request crypto: omap-sham - irq handler must not clear error code ...
Diffstat (limited to 'drivers')
-rw-r--r--drivers/char/hw_random/Kconfig12
-rw-r--r--drivers/char/hw_random/Makefile1
-rw-r--r--drivers/char/hw_random/nomadik-rng.c3
-rw-r--r--drivers/char/hw_random/omap-rng.c6
-rw-r--r--drivers/char/hw_random/ppc4xx-rng.c156
-rw-r--r--drivers/char/hw_random/timeriomem-rng.c3
-rw-r--r--drivers/crypto/amcc/crypto4xx_core.c5
-rw-r--r--drivers/crypto/caam/caamalg.c1832
-rw-r--r--drivers/crypto/caam/compat.h1
-rw-r--r--drivers/crypto/caam/ctrl.c4
-rw-r--r--drivers/crypto/caam/desc_constr.h58
-rw-r--r--drivers/crypto/omap-sham.c180
-rw-r--r--drivers/crypto/talitos.c47
13 files changed, 1756 insertions, 552 deletions
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index a60043b3e409..1d2ebc7a4947 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -210,3 +210,15 @@ config HW_RANDOM_PICOXCELL
210 module will be called picoxcell-rng. 210 module will be called picoxcell-rng.
211 211
212 If unsure, say Y. 212 If unsure, say Y.
213
214config HW_RANDOM_PPC4XX
215 tristate "PowerPC 4xx generic true random number generator support"
216 depends on HW_RANDOM && PPC && 4xx
217 ---help---
218 This driver provides the kernel-side support for the TRNG hardware
219 found in the security function of some PowerPC 4xx SoCs.
220
221 To compile this driver as a module, choose M here: the
222 module will be called ppc4xx-rng.
223
224 If unsure, say N.
diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile
index 3db4eb8b19c0..c88f244c8a71 100644
--- a/drivers/char/hw_random/Makefile
+++ b/drivers/char/hw_random/Makefile
@@ -20,3 +20,4 @@ obj-$(CONFIG_HW_RANDOM_MXC_RNGA) += mxc-rnga.o
20obj-$(CONFIG_HW_RANDOM_OCTEON) += octeon-rng.o 20obj-$(CONFIG_HW_RANDOM_OCTEON) += octeon-rng.o
21obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o 21obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o
22obj-$(CONFIG_HW_RANDOM_PICOXCELL) += picoxcell-rng.o 22obj-$(CONFIG_HW_RANDOM_PICOXCELL) += picoxcell-rng.o
23obj-$(CONFIG_HW_RANDOM_PPC4XX) += ppc4xx-rng.o
diff --git a/drivers/char/hw_random/nomadik-rng.c b/drivers/char/hw_random/nomadik-rng.c
index dd1d143eb8ea..52e08ca3ccd7 100644
--- a/drivers/char/hw_random/nomadik-rng.c
+++ b/drivers/char/hw_random/nomadik-rng.c
@@ -55,7 +55,7 @@ static int nmk_rng_probe(struct amba_device *dev, const struct amba_id *id)
55 55
56 ret = amba_request_regions(dev, dev->dev.init_name); 56 ret = amba_request_regions(dev, dev->dev.init_name);
57 if (ret) 57 if (ret)
58 return ret; 58 goto out_clk;
59 ret = -ENOMEM; 59 ret = -ENOMEM;
60 base = ioremap(dev->res.start, resource_size(&dev->res)); 60 base = ioremap(dev->res.start, resource_size(&dev->res));
61 if (!base) 61 if (!base)
@@ -70,6 +70,7 @@ out_unmap:
70 iounmap(base); 70 iounmap(base);
71out_release: 71out_release:
72 amba_release_regions(dev); 72 amba_release_regions(dev);
73out_clk:
73 clk_disable(rng_clk); 74 clk_disable(rng_clk);
74 clk_put(rng_clk); 75 clk_put(rng_clk);
75 return ret; 76 return ret;
diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c
index 2cc755a64302..b757fac3cd1f 100644
--- a/drivers/char/hw_random/omap-rng.c
+++ b/drivers/char/hw_random/omap-rng.c
@@ -113,8 +113,10 @@ static int __devinit omap_rng_probe(struct platform_device *pdev)
113 113
114 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 114 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
115 115
116 if (!res) 116 if (!res) {
117 return -ENOENT; 117 ret = -ENOENT;
118 goto err_region;
119 }
118 120
119 if (!request_mem_region(res->start, resource_size(res), pdev->name)) { 121 if (!request_mem_region(res->start, resource_size(res), pdev->name)) {
120 ret = -EBUSY; 122 ret = -EBUSY;
diff --git a/drivers/char/hw_random/ppc4xx-rng.c b/drivers/char/hw_random/ppc4xx-rng.c
new file mode 100644
index 000000000000..b8afa6a4ff67
--- /dev/null
+++ b/drivers/char/hw_random/ppc4xx-rng.c
@@ -0,0 +1,156 @@
1/*
2 * Generic PowerPC 44x RNG driver
3 *
4 * Copyright 2011 IBM Corporation
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation; version 2 of the License.
9 */
10
11#include <linux/module.h>
12#include <linux/kernel.h>
13#include <linux/platform_device.h>
14#include <linux/hw_random.h>
15#include <linux/delay.h>
16#include <linux/of_platform.h>
17#include <asm/io.h>
18
19#define PPC4XX_TRNG_DEV_CTRL 0x60080
20
21#define PPC4XX_TRNGE 0x00020000
22#define PPC4XX_TRNG_CTRL 0x0008
23#define PPC4XX_TRNG_CTRL_DALM 0x20
24#define PPC4XX_TRNG_STAT 0x0004
25#define PPC4XX_TRNG_STAT_B 0x1
26#define PPC4XX_TRNG_DATA 0x0000
27
28#define MODULE_NAME "ppc4xx_rng"
29
30static int ppc4xx_rng_data_present(struct hwrng *rng, int wait)
31{
32 void __iomem *rng_regs = (void __iomem *) rng->priv;
33 int busy, i, present = 0;
34
35 for (i = 0; i < 20; i++) {
36 busy = (in_le32(rng_regs + PPC4XX_TRNG_STAT) & PPC4XX_TRNG_STAT_B);
37 if (!busy || !wait) {
38 present = 1;
39 break;
40 }
41 udelay(10);
42 }
43 return present;
44}
45
46static int ppc4xx_rng_data_read(struct hwrng *rng, u32 *data)
47{
48 void __iomem *rng_regs = (void __iomem *) rng->priv;
49 *data = in_le32(rng_regs + PPC4XX_TRNG_DATA);
50 return 4;
51}
52
53static int ppc4xx_rng_enable(int enable)
54{
55 struct device_node *ctrl;
56 void __iomem *ctrl_reg;
57 int err = 0;
58 u32 val;
59
60 /* Find the main crypto device node and map it to turn the TRNG on */
61 ctrl = of_find_compatible_node(NULL, NULL, "amcc,ppc4xx-crypto");
62 if (!ctrl)
63 return -ENODEV;
64
65 ctrl_reg = of_iomap(ctrl, 0);
66 if (!ctrl_reg) {
67 err = -ENODEV;
68 goto out;
69 }
70
71 val = in_le32(ctrl_reg + PPC4XX_TRNG_DEV_CTRL);
72
73 if (enable)
74 val |= PPC4XX_TRNGE;
75 else
76 val = val & ~PPC4XX_TRNGE;
77
78 out_le32(ctrl_reg + PPC4XX_TRNG_DEV_CTRL, val);
79 iounmap(ctrl_reg);
80
81out:
82 of_node_put(ctrl);
83
84 return err;
85}
86
87static struct hwrng ppc4xx_rng = {
88 .name = MODULE_NAME,
89 .data_present = ppc4xx_rng_data_present,
90 .data_read = ppc4xx_rng_data_read,
91};
92
93static int __devinit ppc4xx_rng_probe(struct platform_device *dev)
94{
95 void __iomem *rng_regs;
96 int err = 0;
97
98 rng_regs = of_iomap(dev->dev.of_node, 0);
99 if (!rng_regs)
100 return -ENODEV;
101
102 err = ppc4xx_rng_enable(1);
103 if (err)
104 return err;
105
106 out_le32(rng_regs + PPC4XX_TRNG_CTRL, PPC4XX_TRNG_CTRL_DALM);
107 ppc4xx_rng.priv = (unsigned long) rng_regs;
108
109 err = hwrng_register(&ppc4xx_rng);
110
111 return err;
112}
113
114static int __devexit ppc4xx_rng_remove(struct platform_device *dev)
115{
116 void __iomem *rng_regs = (void __iomem *) ppc4xx_rng.priv;
117
118 hwrng_unregister(&ppc4xx_rng);
119 ppc4xx_rng_enable(0);
120 iounmap(rng_regs);
121
122 return 0;
123}
124
125static struct of_device_id ppc4xx_rng_match[] = {
126 { .compatible = "ppc4xx-rng", },
127 { .compatible = "amcc,ppc460ex-rng", },
128 { .compatible = "amcc,ppc440epx-rng", },
129 {},
130};
131
132static struct platform_driver ppc4xx_rng_driver = {
133 .driver = {
134 .name = MODULE_NAME,
135 .owner = THIS_MODULE,
136 .of_match_table = ppc4xx_rng_match,
137 },
138 .probe = ppc4xx_rng_probe,
139 .remove = ppc4xx_rng_remove,
140};
141
142static int __init ppc4xx_rng_init(void)
143{
144 return platform_driver_register(&ppc4xx_rng_driver);
145}
146module_init(ppc4xx_rng_init);
147
148static void __exit ppc4xx_rng_exit(void)
149{
150 platform_driver_unregister(&ppc4xx_rng_driver);
151}
152module_exit(ppc4xx_rng_exit);
153
154MODULE_LICENSE("GPL");
155MODULE_AUTHOR("Josh Boyer <jwboyer@linux.vnet.ibm.com>");
156MODULE_DESCRIPTION("HW RNG driver for PPC 4xx processors");
diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c
index a94e930575f2..a8428e6f64a9 100644
--- a/drivers/char/hw_random/timeriomem-rng.c
+++ b/drivers/char/hw_random/timeriomem-rng.c
@@ -100,8 +100,7 @@ static int __devinit timeriomem_rng_probe(struct platform_device *pdev)
100 100
101 timeriomem_rng_data = pdev->dev.platform_data; 101 timeriomem_rng_data = pdev->dev.platform_data;
102 102
103 timeriomem_rng_data->address = ioremap(res->start, 103 timeriomem_rng_data->address = ioremap(res->start, resource_size(res));
104 res->end - res->start + 1);
105 if (!timeriomem_rng_data->address) 104 if (!timeriomem_rng_data->address)
106 return -EIO; 105 return -EIO;
107 106
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index 18912521a7a5..1d103f997dc2 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -51,6 +51,7 @@ static void crypto4xx_hw_init(struct crypto4xx_device *dev)
51 union ce_io_threshold io_threshold; 51 union ce_io_threshold io_threshold;
52 u32 rand_num; 52 u32 rand_num;
53 union ce_pe_dma_cfg pe_dma_cfg; 53 union ce_pe_dma_cfg pe_dma_cfg;
54 u32 device_ctrl;
54 55
55 writel(PPC4XX_BYTE_ORDER, dev->ce_base + CRYPTO4XX_BYTE_ORDER_CFG); 56 writel(PPC4XX_BYTE_ORDER, dev->ce_base + CRYPTO4XX_BYTE_ORDER_CFG);
56 /* setup pe dma, include reset sg, pdr and pe, then release reset */ 57 /* setup pe dma, include reset sg, pdr and pe, then release reset */
@@ -84,7 +85,9 @@ static void crypto4xx_hw_init(struct crypto4xx_device *dev)
84 writel(ring_size.w, dev->ce_base + CRYPTO4XX_RING_SIZE); 85 writel(ring_size.w, dev->ce_base + CRYPTO4XX_RING_SIZE);
85 ring_ctrl.w = 0; 86 ring_ctrl.w = 0;
86 writel(ring_ctrl.w, dev->ce_base + CRYPTO4XX_RING_CTRL); 87 writel(ring_ctrl.w, dev->ce_base + CRYPTO4XX_RING_CTRL);
87 writel(PPC4XX_DC_3DES_EN, dev->ce_base + CRYPTO4XX_DEVICE_CTRL); 88 device_ctrl = readl(dev->ce_base + CRYPTO4XX_DEVICE_CTRL);
89 device_ctrl |= PPC4XX_DC_3DES_EN;
90 writel(device_ctrl, dev->ce_base + CRYPTO4XX_DEVICE_CTRL);
88 writel(dev->gdr_pa, dev->ce_base + CRYPTO4XX_GATH_RING_BASE); 91 writel(dev->gdr_pa, dev->ce_base + CRYPTO4XX_GATH_RING_BASE);
89 writel(dev->sdr_pa, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE); 92 writel(dev->sdr_pa, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE);
90 part_ring_size.w = 0; 93 part_ring_size.w = 0;
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 676d957c22b0..4159265b453b 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -62,10 +62,22 @@
62#define CAAM_MAX_IV_LENGTH 16 62#define CAAM_MAX_IV_LENGTH 16
63 63
64/* length of descriptors text */ 64/* length of descriptors text */
65#define DESC_AEAD_SHARED_TEXT_LEN 4 65#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 3 + CAAM_PTR_SZ * 3)
66#define DESC_AEAD_ENCRYPT_TEXT_LEN 21 66
67#define DESC_AEAD_DECRYPT_TEXT_LEN 24 67#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ)
68#define DESC_AEAD_GIVENCRYPT_TEXT_LEN 27 68#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ)
69#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ)
70#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
71
72#define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ)
73#define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \
74 20 * CAAM_CMD_SZ)
75#define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \
76 15 * CAAM_CMD_SZ)
77
78#define DESC_MAX_USED_BYTES (DESC_AEAD_GIVENC_LEN + \
79 CAAM_MAX_KEY_SIZE)
80#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
69 81
70#ifdef DEBUG 82#ifdef DEBUG
71/* for print_hex_dumps with line references */ 83/* for print_hex_dumps with line references */
@@ -76,30 +88,366 @@
76#define debug(format, arg...) 88#define debug(format, arg...)
77#endif 89#endif
78 90
91/* Set DK bit in class 1 operation if shared */
92static inline void append_dec_op1(u32 *desc, u32 type)
93{
94 u32 *jump_cmd, *uncond_jump_cmd;
95
96 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
97 append_operation(desc, type | OP_ALG_AS_INITFINAL |
98 OP_ALG_DECRYPT);
99 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
100 set_jump_tgt_here(desc, jump_cmd);
101 append_operation(desc, type | OP_ALG_AS_INITFINAL |
102 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
103 set_jump_tgt_here(desc, uncond_jump_cmd);
104}
105
106/*
107 * Wait for completion of class 1 key loading before allowing
108 * error propagation
109 */
110static inline void append_dec_shr_done(u32 *desc)
111{
112 u32 *jump_cmd;
113
114 jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TEST_ALL);
115 set_jump_tgt_here(desc, jump_cmd);
116 append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD);
117}
118
119/*
120 * For aead functions, read payload and write payload,
121 * both of which are specified in req->src and req->dst
122 */
123static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
124{
125 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
126 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
127 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
128}
129
130/*
131 * For aead encrypt and decrypt, read iv for both classes
132 */
133static inline void aead_append_ld_iv(u32 *desc, int ivsize)
134{
135 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
136 LDST_CLASS_1_CCB | ivsize);
137 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize);
138}
139
140/*
141 * For ablkcipher encrypt and decrypt, read from req->src and
142 * write to req->dst
143 */
144static inline void ablkcipher_append_src_dst(u32 *desc)
145{
146 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); \
147 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); \
148 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | \
149 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); \
150 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); \
151}
152
153/*
154 * If all data, including src (with assoc and iv) or dst (with iv only) are
155 * contiguous
156 */
157#define GIV_SRC_CONTIG 1
158#define GIV_DST_CONTIG (1 << 1)
159
79/* 160/*
80 * per-session context 161 * per-session context
81 */ 162 */
82struct caam_ctx { 163struct caam_ctx {
83 struct device *jrdev; 164 struct device *jrdev;
84 u32 *sh_desc; 165 u32 sh_desc_enc[DESC_MAX_USED_LEN];
85 dma_addr_t shared_desc_phys; 166 u32 sh_desc_dec[DESC_MAX_USED_LEN];
167 u32 sh_desc_givenc[DESC_MAX_USED_LEN];
168 dma_addr_t sh_desc_enc_dma;
169 dma_addr_t sh_desc_dec_dma;
170 dma_addr_t sh_desc_givenc_dma;
86 u32 class1_alg_type; 171 u32 class1_alg_type;
87 u32 class2_alg_type; 172 u32 class2_alg_type;
88 u32 alg_op; 173 u32 alg_op;
89 u8 *key; 174 u8 key[CAAM_MAX_KEY_SIZE];
90 dma_addr_t key_phys; 175 dma_addr_t key_dma;
91 unsigned int enckeylen; 176 unsigned int enckeylen;
92 unsigned int split_key_len; 177 unsigned int split_key_len;
93 unsigned int split_key_pad_len; 178 unsigned int split_key_pad_len;
94 unsigned int authsize; 179 unsigned int authsize;
95}; 180};
96 181
97static int aead_authenc_setauthsize(struct crypto_aead *authenc, 182static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
183 int keys_fit_inline)
184{
185 if (keys_fit_inline) {
186 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
187 ctx->split_key_len, CLASS_2 |
188 KEY_DEST_MDHA_SPLIT | KEY_ENC);
189 append_key_as_imm(desc, (void *)ctx->key +
190 ctx->split_key_pad_len, ctx->enckeylen,
191 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
192 } else {
193 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
194 KEY_DEST_MDHA_SPLIT | KEY_ENC);
195 append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
196 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
197 }
198}
199
200static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
201 int keys_fit_inline)
202{
203 u32 *key_jump_cmd;
204
205 init_sh_desc(desc, HDR_SHARE_WAIT);
206
207 /* Skip if already shared */
208 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
209 JUMP_COND_SHRD);
210
211 append_key_aead(desc, ctx, keys_fit_inline);
212
213 set_jump_tgt_here(desc, key_jump_cmd);
214
215 /* Propagate errors from shared to job descriptor */
216 append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD);
217}
218
219static int aead_set_sh_desc(struct crypto_aead *aead)
220{
221 struct aead_tfm *tfm = &aead->base.crt_aead;
222 struct caam_ctx *ctx = crypto_aead_ctx(aead);
223 struct device *jrdev = ctx->jrdev;
224 bool keys_fit_inline = 0;
225 u32 *key_jump_cmd, *jump_cmd;
226 u32 geniv, moveiv;
227 u32 *desc;
228
229 if (!ctx->enckeylen || !ctx->authsize)
230 return 0;
231
232 /*
233 * Job Descriptor and Shared Descriptors
234 * must all fit into the 64-word Descriptor h/w Buffer
235 */
236 if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN +
237 ctx->split_key_pad_len + ctx->enckeylen <=
238 CAAM_DESC_BYTES_MAX)
239 keys_fit_inline = 1;
240
241 /* aead_encrypt shared descriptor */
242 desc = ctx->sh_desc_enc;
243
244 init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
245
246 /* Class 2 operation */
247 append_operation(desc, ctx->class2_alg_type |
248 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
249
250 /* cryptlen = seqoutlen - authsize */
251 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
252
253 /* assoclen + cryptlen = seqinlen - ivsize */
254 append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
255
256 /* assoclen + cryptlen = (assoclen + cryptlen) - cryptlen */
257 append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ);
258
259 /* read assoc before reading payload */
260 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
261 KEY_VLF);
262 aead_append_ld_iv(desc, tfm->ivsize);
263
264 /* Class 1 operation */
265 append_operation(desc, ctx->class1_alg_type |
266 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
267
268 /* Read and write cryptlen bytes */
269 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
270 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
271 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
272
273 /* Write ICV */
274 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
275 LDST_SRCDST_BYTE_CONTEXT);
276
277 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
278 desc_bytes(desc),
279 DMA_TO_DEVICE);
280 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
281 dev_err(jrdev, "unable to map shared descriptor\n");
282 return -ENOMEM;
283 }
284#ifdef DEBUG
285 print_hex_dump(KERN_ERR, "aead enc shdesc@"xstr(__LINE__)": ",
286 DUMP_PREFIX_ADDRESS, 16, 4, desc,
287 desc_bytes(desc), 1);
288#endif
289
290 /*
291 * Job Descriptor and Shared Descriptors
292 * must all fit into the 64-word Descriptor h/w Buffer
293 */
294 if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN +
295 ctx->split_key_pad_len + ctx->enckeylen <=
296 CAAM_DESC_BYTES_MAX)
297 keys_fit_inline = 1;
298
299 desc = ctx->sh_desc_dec;
300
301 /* aead_decrypt shared descriptor */
302 init_sh_desc(desc, HDR_SHARE_WAIT);
303
304 /* Skip if already shared */
305 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
306 JUMP_COND_SHRD);
307
308 append_key_aead(desc, ctx, keys_fit_inline);
309
310 /* Only propagate error immediately if shared */
311 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
312 set_jump_tgt_here(desc, key_jump_cmd);
313 append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD);
314 set_jump_tgt_here(desc, jump_cmd);
315
316 /* Class 2 operation */
317 append_operation(desc, ctx->class2_alg_type |
318 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
319
320 /* assoclen + cryptlen = seqinlen - ivsize */
321 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
322 ctx->authsize + tfm->ivsize)
323 /* assoclen = (assoclen + cryptlen) - cryptlen */
324 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
325 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
326
327 /* read assoc before reading payload */
328 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
329 KEY_VLF);
330
331 aead_append_ld_iv(desc, tfm->ivsize);
332
333 append_dec_op1(desc, ctx->class1_alg_type);
334
335 /* Read and write cryptlen bytes */
336 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
337 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
338 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
339
340 /* Load ICV */
341 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
342 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
343 append_dec_shr_done(desc);
344
345 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
346 desc_bytes(desc),
347 DMA_TO_DEVICE);
348 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
349 dev_err(jrdev, "unable to map shared descriptor\n");
350 return -ENOMEM;
351 }
352#ifdef DEBUG
353 print_hex_dump(KERN_ERR, "aead dec shdesc@"xstr(__LINE__)": ",
354 DUMP_PREFIX_ADDRESS, 16, 4, desc,
355 desc_bytes(desc), 1);
356#endif
357
358 /*
359 * Job Descriptor and Shared Descriptors
360 * must all fit into the 64-word Descriptor h/w Buffer
361 */
362 if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN +
363 ctx->split_key_pad_len + ctx->enckeylen <=
364 CAAM_DESC_BYTES_MAX)
365 keys_fit_inline = 1;
366
367 /* aead_givencrypt shared descriptor */
368 desc = ctx->sh_desc_givenc;
369
370 init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
371
372 /* Generate IV */
373 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
374 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
375 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
376 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
377 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
378 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
379 append_move(desc, MOVE_SRC_INFIFO |
380 MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT));
381 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
382
383 /* Copy IV to class 1 context */
384 append_move(desc, MOVE_SRC_CLASS1CTX |
385 MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT));
386
387 /* Return to encryption */
388 append_operation(desc, ctx->class2_alg_type |
389 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
390
391 /* ivsize + cryptlen = seqoutlen - authsize */
392 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
393
394 /* assoclen = seqinlen - (ivsize + cryptlen) */
395 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
396
397 /* read assoc before reading payload */
398 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
399 KEY_VLF);
400
401 /* Copy iv from class 1 ctx to class 2 fifo*/
402 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
403 NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
404 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
405 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
406 append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB |
407 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
408
409 /* Class 1 operation */
410 append_operation(desc, ctx->class1_alg_type |
411 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
412
413 /* Will write ivsize + cryptlen */
414 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
415
416 /* Not need to reload iv */
417 append_seq_fifo_load(desc, tfm->ivsize,
418 FIFOLD_CLASS_SKIP);
419
420 /* Will read cryptlen */
421 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
422 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
423
424 /* Write ICV */
425 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
426 LDST_SRCDST_BYTE_CONTEXT);
427
428 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
429 desc_bytes(desc),
430 DMA_TO_DEVICE);
431 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
432 dev_err(jrdev, "unable to map shared descriptor\n");
433 return -ENOMEM;
434 }
435#ifdef DEBUG
436 print_hex_dump(KERN_ERR, "aead givenc shdesc@"xstr(__LINE__)": ",
437 DUMP_PREFIX_ADDRESS, 16, 4, desc,
438 desc_bytes(desc), 1);
439#endif
440
441 return 0;
442}
443
444static int aead_setauthsize(struct crypto_aead *authenc,
98 unsigned int authsize) 445 unsigned int authsize)
99{ 446{
100 struct caam_ctx *ctx = crypto_aead_ctx(authenc); 447 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
101 448
102 ctx->authsize = authsize; 449 ctx->authsize = authsize;
450 aead_set_sh_desc(authenc);
103 451
104 return 0; 452 return 0;
105} 453}
@@ -117,6 +465,7 @@ static void split_key_done(struct device *dev, u32 *desc, u32 err,
117#ifdef DEBUG 465#ifdef DEBUG
118 dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 466 dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
119#endif 467#endif
468
120 if (err) { 469 if (err) {
121 char tmp[CAAM_ERROR_STR_MAX]; 470 char tmp[CAAM_ERROR_STR_MAX];
122 471
@@ -220,73 +569,7 @@ static u32 gen_split_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen)
220 return ret; 569 return ret;
221} 570}
222 571
223static int build_sh_desc_ipsec(struct caam_ctx *ctx) 572static int aead_setkey(struct crypto_aead *aead,
224{
225 struct device *jrdev = ctx->jrdev;
226 u32 *sh_desc;
227 u32 *jump_cmd;
228 bool keys_fit_inline = 0;
229
230 /*
231 * largest Job Descriptor and its Shared Descriptor
232 * must both fit into the 64-word Descriptor h/w Buffer
233 */
234 if ((DESC_AEAD_GIVENCRYPT_TEXT_LEN +
235 DESC_AEAD_SHARED_TEXT_LEN) * CAAM_CMD_SZ +
236 ctx->split_key_pad_len + ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
237 keys_fit_inline = 1;
238
239 /* build shared descriptor for this session */
240 sh_desc = kmalloc(CAAM_CMD_SZ * DESC_AEAD_SHARED_TEXT_LEN +
241 (keys_fit_inline ?
242 ctx->split_key_pad_len + ctx->enckeylen :
243 CAAM_PTR_SZ * 2), GFP_DMA | GFP_KERNEL);
244 if (!sh_desc) {
245 dev_err(jrdev, "could not allocate shared descriptor\n");
246 return -ENOMEM;
247 }
248
249 init_sh_desc(sh_desc, HDR_SAVECTX | HDR_SHARE_SERIAL);
250
251 jump_cmd = append_jump(sh_desc, CLASS_BOTH | JUMP_TEST_ALL |
252 JUMP_COND_SHRD | JUMP_COND_SELF);
253
254 /*
255 * process keys, starting with class 2/authentication.
256 */
257 if (keys_fit_inline) {
258 append_key_as_imm(sh_desc, ctx->key, ctx->split_key_pad_len,
259 ctx->split_key_len,
260 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
261
262 append_key_as_imm(sh_desc, (void *)ctx->key +
263 ctx->split_key_pad_len, ctx->enckeylen,
264 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
265 } else {
266 append_key(sh_desc, ctx->key_phys, ctx->split_key_len, CLASS_2 |
267 KEY_DEST_MDHA_SPLIT | KEY_ENC);
268 append_key(sh_desc, ctx->key_phys + ctx->split_key_pad_len,
269 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
270 }
271
272 /* update jump cmd now that we are at the jump target */
273 set_jump_tgt_here(sh_desc, jump_cmd);
274
275 ctx->shared_desc_phys = dma_map_single(jrdev, sh_desc,
276 desc_bytes(sh_desc),
277 DMA_TO_DEVICE);
278 if (dma_mapping_error(jrdev, ctx->shared_desc_phys)) {
279 dev_err(jrdev, "unable to map shared descriptor\n");
280 kfree(sh_desc);
281 return -ENOMEM;
282 }
283
284 ctx->sh_desc = sh_desc;
285
286 return 0;
287}
288
289static int aead_authenc_setkey(struct crypto_aead *aead,
290 const u8 *key, unsigned int keylen) 573 const u8 *key, unsigned int keylen)
291{ 574{
292 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ 575 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
@@ -326,27 +609,19 @@ static int aead_authenc_setkey(struct crypto_aead *aead,
326 print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ", 609 print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
327 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); 610 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
328#endif 611#endif
329 ctx->key = kmalloc(ctx->split_key_pad_len + enckeylen,
330 GFP_KERNEL | GFP_DMA);
331 if (!ctx->key) {
332 dev_err(jrdev, "could not allocate key output memory\n");
333 return -ENOMEM;
334 }
335 612
336 ret = gen_split_key(ctx, key, authkeylen); 613 ret = gen_split_key(ctx, key, authkeylen);
337 if (ret) { 614 if (ret) {
338 kfree(ctx->key);
339 goto badkey; 615 goto badkey;
340 } 616 }
341 617
342 /* postpend encryption key to auth split key */ 618 /* postpend encryption key to auth split key */
343 memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen); 619 memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen);
344 620
345 ctx->key_phys = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + 621 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
346 enckeylen, DMA_TO_DEVICE); 622 enckeylen, DMA_TO_DEVICE);
347 if (dma_mapping_error(jrdev, ctx->key_phys)) { 623 if (dma_mapping_error(jrdev, ctx->key_dma)) {
348 dev_err(jrdev, "unable to map key i/o memory\n"); 624 dev_err(jrdev, "unable to map key i/o memory\n");
349 kfree(ctx->key);
350 return -ENOMEM; 625 return -ENOMEM;
351 } 626 }
352#ifdef DEBUG 627#ifdef DEBUG
@@ -357,11 +632,10 @@ static int aead_authenc_setkey(struct crypto_aead *aead,
357 632
358 ctx->enckeylen = enckeylen; 633 ctx->enckeylen = enckeylen;
359 634
360 ret = build_sh_desc_ipsec(ctx); 635 ret = aead_set_sh_desc(aead);
361 if (ret) { 636 if (ret) {
362 dma_unmap_single(jrdev, ctx->key_phys, ctx->split_key_pad_len + 637 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
363 enckeylen, DMA_TO_DEVICE); 638 enckeylen, DMA_TO_DEVICE);
364 kfree(ctx->key);
365 } 639 }
366 640
367 return ret; 641 return ret;
@@ -370,6 +644,119 @@ badkey:
370 return -EINVAL; 644 return -EINVAL;
371} 645}
372 646
647static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
648 const u8 *key, unsigned int keylen)
649{
650 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
651 struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher;
652 struct device *jrdev = ctx->jrdev;
653 int ret = 0;
654 u32 *key_jump_cmd, *jump_cmd;
655 u32 *desc;
656
657#ifdef DEBUG
658 print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
659 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
660#endif
661
662 memcpy(ctx->key, key, keylen);
663 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
664 DMA_TO_DEVICE);
665 if (dma_mapping_error(jrdev, ctx->key_dma)) {
666 dev_err(jrdev, "unable to map key i/o memory\n");
667 return -ENOMEM;
668 }
669 ctx->enckeylen = keylen;
670
671 /* ablkcipher_encrypt shared descriptor */
672 desc = ctx->sh_desc_enc;
673 init_sh_desc(desc, HDR_SHARE_WAIT);
674 /* Skip if already shared */
675 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
676 JUMP_COND_SHRD);
677
678 /* Load class1 key only */
679 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
680 ctx->enckeylen, CLASS_1 |
681 KEY_DEST_CLASS_REG);
682
683 set_jump_tgt_here(desc, key_jump_cmd);
684
685 /* Propagate errors from shared to job descriptor */
686 append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD);
687
688 /* Load iv */
689 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
690 LDST_CLASS_1_CCB | tfm->ivsize);
691
692 /* Load operation */
693 append_operation(desc, ctx->class1_alg_type |
694 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
695
696 /* Perform operation */
697 ablkcipher_append_src_dst(desc);
698
699 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
700 desc_bytes(desc),
701 DMA_TO_DEVICE);
702 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
703 dev_err(jrdev, "unable to map shared descriptor\n");
704 return -ENOMEM;
705 }
706#ifdef DEBUG
707 print_hex_dump(KERN_ERR, "ablkcipher enc shdesc@"xstr(__LINE__)": ",
708 DUMP_PREFIX_ADDRESS, 16, 4, desc,
709 desc_bytes(desc), 1);
710#endif
711 /* ablkcipher_decrypt shared descriptor */
712 desc = ctx->sh_desc_dec;
713
714 init_sh_desc(desc, HDR_SHARE_WAIT);
715 /* Skip if already shared */
716 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
717 JUMP_COND_SHRD);
718
719 /* Load class1 key only */
720 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
721 ctx->enckeylen, CLASS_1 |
722 KEY_DEST_CLASS_REG);
723
724 /* For aead, only propagate error immediately if shared */
725 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
726 set_jump_tgt_here(desc, key_jump_cmd);
727 append_cmd(desc, SET_OK_PROP_ERRORS | CMD_LOAD);
728 set_jump_tgt_here(desc, jump_cmd);
729
730 /* load IV */
731 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
732 LDST_CLASS_1_CCB | tfm->ivsize);
733
734 /* Choose operation */
735 append_dec_op1(desc, ctx->class1_alg_type);
736
737 /* Perform operation */
738 ablkcipher_append_src_dst(desc);
739
740 /* Wait for key to load before allowing propagating error */
741 append_dec_shr_done(desc);
742
743 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
744 desc_bytes(desc),
745 DMA_TO_DEVICE);
746 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
747 dev_err(jrdev, "unable to map shared descriptor\n");
748 return -ENOMEM;
749 }
750
751#ifdef DEBUG
752 print_hex_dump(KERN_ERR, "ablkcipher dec shdesc@"xstr(__LINE__)": ",
753 DUMP_PREFIX_ADDRESS, 16, 4, desc,
754 desc_bytes(desc), 1);
755#endif
756
757 return ret;
758}
759
373struct link_tbl_entry { 760struct link_tbl_entry {
374 u64 ptr; 761 u64 ptr;
375 u32 len; 762 u32 len;
@@ -379,64 +766,109 @@ struct link_tbl_entry {
379}; 766};
380 767
381/* 768/*
382 * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor 769 * aead_edesc - s/w-extended aead descriptor
770 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
383 * @src_nents: number of segments in input scatterlist 771 * @src_nents: number of segments in input scatterlist
384 * @dst_nents: number of segments in output scatterlist 772 * @dst_nents: number of segments in output scatterlist
385 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist 773 * @iv_dma: dma address of iv for checking continuity and link table
386 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) 774 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
387 * @link_tbl_bytes: length of dma mapped link_tbl space 775 * @link_tbl_bytes: length of dma mapped link_tbl space
388 * @link_tbl_dma: bus physical mapped address of h/w link table 776 * @link_tbl_dma: bus physical mapped address of h/w link table
389 * @hw_desc: the h/w job descriptor followed by any referenced link tables 777 * @hw_desc: the h/w job descriptor followed by any referenced link tables
390 */ 778 */
391struct ipsec_esp_edesc { 779struct aead_edesc {
392 int assoc_nents; 780 int assoc_nents;
393 int src_nents; 781 int src_nents;
394 int dst_nents; 782 int dst_nents;
783 dma_addr_t iv_dma;
395 int link_tbl_bytes; 784 int link_tbl_bytes;
396 dma_addr_t link_tbl_dma; 785 dma_addr_t link_tbl_dma;
397 struct link_tbl_entry *link_tbl; 786 struct link_tbl_entry *link_tbl;
398 u32 hw_desc[0]; 787 u32 hw_desc[0];
399}; 788};
400 789
401static void ipsec_esp_unmap(struct device *dev, 790/*
402 struct ipsec_esp_edesc *edesc, 791 * ablkcipher_edesc - s/w-extended ablkcipher descriptor
403 struct aead_request *areq) 792 * @src_nents: number of segments in input scatterlist
404{ 793 * @dst_nents: number of segments in output scatterlist
405 dma_unmap_sg(dev, areq->assoc, edesc->assoc_nents, DMA_TO_DEVICE); 794 * @iv_dma: dma address of iv for checking continuity and link table
795 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
796 * @link_tbl_bytes: length of dma mapped link_tbl space
797 * @link_tbl_dma: bus physical mapped address of h/w link table
798 * @hw_desc: the h/w job descriptor followed by any referenced link tables
799 */
800struct ablkcipher_edesc {
801 int src_nents;
802 int dst_nents;
803 dma_addr_t iv_dma;
804 int link_tbl_bytes;
805 dma_addr_t link_tbl_dma;
806 struct link_tbl_entry *link_tbl;
807 u32 hw_desc[0];
808};
406 809
407 if (unlikely(areq->dst != areq->src)) { 810static void caam_unmap(struct device *dev, struct scatterlist *src,
408 dma_unmap_sg(dev, areq->src, edesc->src_nents, 811 struct scatterlist *dst, int src_nents, int dst_nents,
409 DMA_TO_DEVICE); 812 dma_addr_t iv_dma, int ivsize, dma_addr_t link_tbl_dma,
410 dma_unmap_sg(dev, areq->dst, edesc->dst_nents, 813 int link_tbl_bytes)
411 DMA_FROM_DEVICE); 814{
815 if (unlikely(dst != src)) {
816 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
817 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
412 } else { 818 } else {
413 dma_unmap_sg(dev, areq->src, edesc->src_nents, 819 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
414 DMA_BIDIRECTIONAL);
415 } 820 }
416 821
417 if (edesc->link_tbl_bytes) 822 if (iv_dma)
418 dma_unmap_single(dev, edesc->link_tbl_dma, 823 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
419 edesc->link_tbl_bytes, 824 if (link_tbl_bytes)
825 dma_unmap_single(dev, link_tbl_dma, link_tbl_bytes,
420 DMA_TO_DEVICE); 826 DMA_TO_DEVICE);
421} 827}
422 828
423/* 829static void aead_unmap(struct device *dev,
424 * ipsec_esp descriptor callbacks 830 struct aead_edesc *edesc,
425 */ 831 struct aead_request *req)
426static void ipsec_esp_encrypt_done(struct device *jrdev, u32 *desc, u32 err, 832{
833 struct crypto_aead *aead = crypto_aead_reqtfm(req);
834 int ivsize = crypto_aead_ivsize(aead);
835
836 dma_unmap_sg(dev, req->assoc, edesc->assoc_nents, DMA_TO_DEVICE);
837
838 caam_unmap(dev, req->src, req->dst,
839 edesc->src_nents, edesc->dst_nents,
840 edesc->iv_dma, ivsize, edesc->link_tbl_dma,
841 edesc->link_tbl_bytes);
842}
843
844static void ablkcipher_unmap(struct device *dev,
845 struct ablkcipher_edesc *edesc,
846 struct ablkcipher_request *req)
847{
848 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
849 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
850
851 caam_unmap(dev, req->src, req->dst,
852 edesc->src_nents, edesc->dst_nents,
853 edesc->iv_dma, ivsize, edesc->link_tbl_dma,
854 edesc->link_tbl_bytes);
855}
856
857static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
427 void *context) 858 void *context)
428{ 859{
429 struct aead_request *areq = context; 860 struct aead_request *req = context;
430 struct ipsec_esp_edesc *edesc; 861 struct aead_edesc *edesc;
431#ifdef DEBUG 862#ifdef DEBUG
432 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 863 struct crypto_aead *aead = crypto_aead_reqtfm(req);
433 int ivsize = crypto_aead_ivsize(aead);
434 struct caam_ctx *ctx = crypto_aead_ctx(aead); 864 struct caam_ctx *ctx = crypto_aead_ctx(aead);
865 int ivsize = crypto_aead_ivsize(aead);
435 866
436 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 867 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
437#endif 868#endif
438 edesc = (struct ipsec_esp_edesc *)((char *)desc - 869
439 offsetof(struct ipsec_esp_edesc, hw_desc)); 870 edesc = (struct aead_edesc *)((char *)desc -
871 offsetof(struct aead_edesc, hw_desc));
440 872
441 if (err) { 873 if (err) {
442 char tmp[CAAM_ERROR_STR_MAX]; 874 char tmp[CAAM_ERROR_STR_MAX];
@@ -444,39 +876,50 @@ static void ipsec_esp_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
444 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 876 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
445 } 877 }
446 878
447 ipsec_esp_unmap(jrdev, edesc, areq); 879 aead_unmap(jrdev, edesc, req);
448 880
449#ifdef DEBUG 881#ifdef DEBUG
450 print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", 882 print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
451 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc), 883 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
452 areq->assoclen , 1); 884 req->assoclen , 1);
453 print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ", 885 print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
454 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize, 886 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize,
455 edesc->src_nents ? 100 : ivsize, 1); 887 edesc->src_nents ? 100 : ivsize, 1);
456 print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ", 888 print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
457 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src), 889 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
458 edesc->src_nents ? 100 : areq->cryptlen + 890 edesc->src_nents ? 100 : req->cryptlen +
459 ctx->authsize + 4, 1); 891 ctx->authsize + 4, 1);
460#endif 892#endif
461 893
462 kfree(edesc); 894 kfree(edesc);
463 895
464 aead_request_complete(areq, err); 896 aead_request_complete(req, err);
465} 897}
466 898
467static void ipsec_esp_decrypt_done(struct device *jrdev, u32 *desc, u32 err, 899static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
468 void *context) 900 void *context)
469{ 901{
470 struct aead_request *areq = context; 902 struct aead_request *req = context;
471 struct ipsec_esp_edesc *edesc; 903 struct aead_edesc *edesc;
472#ifdef DEBUG 904#ifdef DEBUG
473 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 905 struct crypto_aead *aead = crypto_aead_reqtfm(req);
474 struct caam_ctx *ctx = crypto_aead_ctx(aead); 906 struct caam_ctx *ctx = crypto_aead_ctx(aead);
907 int ivsize = crypto_aead_ivsize(aead);
475 908
476 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); 909 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
477#endif 910#endif
478 edesc = (struct ipsec_esp_edesc *)((char *)desc - 911
479 offsetof(struct ipsec_esp_edesc, hw_desc)); 912 edesc = (struct aead_edesc *)((char *)desc -
913 offsetof(struct aead_edesc, hw_desc));
914
915#ifdef DEBUG
916 print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
917 DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
918 ivsize, 1);
919 print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
920 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst),
921 req->cryptlen, 1);
922#endif
480 923
481 if (err) { 924 if (err) {
482 char tmp[CAAM_ERROR_STR_MAX]; 925 char tmp[CAAM_ERROR_STR_MAX];
@@ -484,7 +927,7 @@ static void ipsec_esp_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
484 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); 927 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
485 } 928 }
486 929
487 ipsec_esp_unmap(jrdev, edesc, areq); 930 aead_unmap(jrdev, edesc, req);
488 931
489 /* 932 /*
490 * verify hw auth check passed else return -EBADMSG 933 * verify hw auth check passed else return -EBADMSG
@@ -495,255 +938,413 @@ static void ipsec_esp_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
495#ifdef DEBUG 938#ifdef DEBUG
496 print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ", 939 print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ",
497 DUMP_PREFIX_ADDRESS, 16, 4, 940 DUMP_PREFIX_ADDRESS, 16, 4,
498 ((char *)sg_virt(areq->assoc) - sizeof(struct iphdr)), 941 ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)),
499 sizeof(struct iphdr) + areq->assoclen + 942 sizeof(struct iphdr) + req->assoclen +
500 ((areq->cryptlen > 1500) ? 1500 : areq->cryptlen) + 943 ((req->cryptlen > 1500) ? 1500 : req->cryptlen) +
501 ctx->authsize + 36, 1); 944 ctx->authsize + 36, 1);
502 if (!err && edesc->link_tbl_bytes) { 945 if (!err && edesc->link_tbl_bytes) {
503 struct scatterlist *sg = sg_last(areq->src, edesc->src_nents); 946 struct scatterlist *sg = sg_last(req->src, edesc->src_nents);
504 print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ", 947 print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ",
505 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg), 948 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
506 sg->length + ctx->authsize + 16, 1); 949 sg->length + ctx->authsize + 16, 1);
507 } 950 }
508#endif 951#endif
952
509 kfree(edesc); 953 kfree(edesc);
510 954
511 aead_request_complete(areq, err); 955 aead_request_complete(req, err);
956}
957
958static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
959 void *context)
960{
961 struct ablkcipher_request *req = context;
962 struct ablkcipher_edesc *edesc;
963#ifdef DEBUG
964 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
965 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
966
967 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
968#endif
969
970 edesc = (struct ablkcipher_edesc *)((char *)desc -
971 offsetof(struct ablkcipher_edesc, hw_desc));
972
973 if (err) {
974 char tmp[CAAM_ERROR_STR_MAX];
975
976 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
977 }
978
979#ifdef DEBUG
980 print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
981 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
982 edesc->src_nents > 1 ? 100 : ivsize, 1);
983 print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
984 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
985 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
986#endif
987
988 ablkcipher_unmap(jrdev, edesc, req);
989 kfree(edesc);
990
991 ablkcipher_request_complete(req, err);
992}
993
994static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
995 void *context)
996{
997 struct ablkcipher_request *req = context;
998 struct ablkcipher_edesc *edesc;
999#ifdef DEBUG
1000 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1001 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1002
1003 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1004#endif
1005
1006 edesc = (struct ablkcipher_edesc *)((char *)desc -
1007 offsetof(struct ablkcipher_edesc, hw_desc));
1008 if (err) {
1009 char tmp[CAAM_ERROR_STR_MAX];
1010
1011 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
1012 }
1013
1014#ifdef DEBUG
1015 print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
1016 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1017 ivsize, 1);
1018 print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
1019 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1020 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
1021#endif
1022
1023 ablkcipher_unmap(jrdev, edesc, req);
1024 kfree(edesc);
1025
1026 ablkcipher_request_complete(req, err);
1027}
1028
1029static void sg_to_link_tbl_one(struct link_tbl_entry *link_tbl_ptr,
1030 dma_addr_t dma, u32 len, u32 offset)
1031{
1032 link_tbl_ptr->ptr = dma;
1033 link_tbl_ptr->len = len;
1034 link_tbl_ptr->reserved = 0;
1035 link_tbl_ptr->buf_pool_id = 0;
1036 link_tbl_ptr->offset = offset;
1037#ifdef DEBUG
1038 print_hex_dump(KERN_ERR, "link_tbl_ptr@"xstr(__LINE__)": ",
1039 DUMP_PREFIX_ADDRESS, 16, 4, link_tbl_ptr,
1040 sizeof(struct link_tbl_entry), 1);
1041#endif
512} 1042}
513 1043
514/* 1044/*
515 * convert scatterlist to h/w link table format 1045 * convert scatterlist to h/w link table format
516 * scatterlist must have been previously dma mapped 1046 * but does not have final bit; instead, returns last entry
517 */ 1047 */
518static void sg_to_link_tbl(struct scatterlist *sg, int sg_count, 1048static struct link_tbl_entry *sg_to_link_tbl(struct scatterlist *sg,
519 struct link_tbl_entry *link_tbl_ptr, u32 offset) 1049 int sg_count, struct link_tbl_entry
1050 *link_tbl_ptr, u32 offset)
520{ 1051{
521 while (sg_count) { 1052 while (sg_count) {
522 link_tbl_ptr->ptr = sg_dma_address(sg); 1053 sg_to_link_tbl_one(link_tbl_ptr, sg_dma_address(sg),
523 link_tbl_ptr->len = sg_dma_len(sg); 1054 sg_dma_len(sg), offset);
524 link_tbl_ptr->reserved = 0;
525 link_tbl_ptr->buf_pool_id = 0;
526 link_tbl_ptr->offset = offset;
527 link_tbl_ptr++; 1055 link_tbl_ptr++;
528 sg = sg_next(sg); 1056 sg = sg_next(sg);
529 sg_count--; 1057 sg_count--;
530 } 1058 }
1059 return link_tbl_ptr - 1;
1060}
531 1061
532 /* set Final bit (marks end of link table) */ 1062/*
533 link_tbl_ptr--; 1063 * convert scatterlist to h/w link table format
1064 * scatterlist must have been previously dma mapped
1065 */
1066static void sg_to_link_tbl_last(struct scatterlist *sg, int sg_count,
1067 struct link_tbl_entry *link_tbl_ptr, u32 offset)
1068{
1069 link_tbl_ptr = sg_to_link_tbl(sg, sg_count, link_tbl_ptr, offset);
534 link_tbl_ptr->len |= 0x40000000; 1070 link_tbl_ptr->len |= 0x40000000;
535} 1071}
536 1072
537/* 1073/*
538 * fill in and submit ipsec_esp job descriptor 1074 * Fill in aead job descriptor
539 */ 1075 */
540static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, 1076static void init_aead_job(u32 *sh_desc, dma_addr_t ptr,
541 u32 encrypt, 1077 struct aead_edesc *edesc,
542 void (*callback) (struct device *dev, u32 *desc, 1078 struct aead_request *req,
543 u32 err, void *context)) 1079 bool all_contig, bool encrypt)
544{ 1080{
545 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1081 struct crypto_aead *aead = crypto_aead_reqtfm(req);
546 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1082 struct caam_ctx *ctx = crypto_aead_ctx(aead);
547 struct device *jrdev = ctx->jrdev;
548 u32 *desc = edesc->hw_desc, options;
549 int ret, sg_count, assoc_sg_count;
550 int ivsize = crypto_aead_ivsize(aead); 1083 int ivsize = crypto_aead_ivsize(aead);
551 int authsize = ctx->authsize; 1084 int authsize = ctx->authsize;
552 dma_addr_t ptr, dst_dma, src_dma; 1085 u32 *desc = edesc->hw_desc;
553#ifdef DEBUG 1086 u32 out_options = 0, in_options;
554 u32 *sh_desc = ctx->sh_desc; 1087 dma_addr_t dst_dma, src_dma;
1088 int len, link_tbl_index = 0;
555 1089
1090#ifdef DEBUG
556 debug("assoclen %d cryptlen %d authsize %d\n", 1091 debug("assoclen %d cryptlen %d authsize %d\n",
557 areq->assoclen, areq->cryptlen, authsize); 1092 req->assoclen, req->cryptlen, authsize);
558 print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ", 1093 print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
559 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc), 1094 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
560 areq->assoclen , 1); 1095 req->assoclen , 1);
561 print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ", 1096 print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
562 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize, 1097 DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
563 edesc->src_nents ? 100 : ivsize, 1); 1098 edesc->src_nents ? 100 : ivsize, 1);
564 print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ", 1099 print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
565 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src), 1100 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
566 edesc->src_nents ? 100 : areq->cryptlen + authsize, 1); 1101 edesc->src_nents ? 100 : req->cryptlen, 1);
567 print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ", 1102 print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
568 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, 1103 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
569 desc_bytes(sh_desc), 1); 1104 desc_bytes(sh_desc), 1);
570#endif 1105#endif
571 assoc_sg_count = dma_map_sg(jrdev, areq->assoc, edesc->assoc_nents ?: 1,
572 DMA_TO_DEVICE);
573 if (areq->src == areq->dst)
574 sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1,
575 DMA_BIDIRECTIONAL);
576 else
577 sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1,
578 DMA_TO_DEVICE);
579 1106
580 /* start auth operation */ 1107 len = desc_len(sh_desc);
581 append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL | 1108 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
582 (encrypt ? : OP_ALG_ICV_ON));
583 1109
584 /* Load FIFO with data for Class 2 CHA */ 1110 if (all_contig) {
585 options = FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG; 1111 src_dma = sg_dma_address(req->assoc);
586 if (!edesc->assoc_nents) { 1112 in_options = 0;
587 ptr = sg_dma_address(areq->assoc);
588 } else { 1113 } else {
589 sg_to_link_tbl(areq->assoc, edesc->assoc_nents, 1114 src_dma = edesc->link_tbl_dma;
590 edesc->link_tbl, 0); 1115 link_tbl_index += (edesc->assoc_nents ? : 1) + 1 +
591 ptr = edesc->link_tbl_dma; 1116 (edesc->src_nents ? : 1);
592 options |= LDST_SGF; 1117 in_options = LDST_SGF;
593 } 1118 }
594 append_fifo_load(desc, ptr, areq->assoclen, options); 1119 if (encrypt)
595 1120 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize +
596 /* copy iv from cipher/class1 input context to class2 infifo */ 1121 req->cryptlen - authsize, in_options);
597 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize); 1122 else
598 1123 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize +
599 if (!encrypt) { 1124 req->cryptlen, in_options);
600 u32 *jump_cmd, *uncond_jump_cmd;
601
602 /* JUMP if shared */
603 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
604 1125
605 /* start class 1 (cipher) operation, non-shared version */ 1126 if (likely(req->src == req->dst)) {
606 append_operation(desc, ctx->class1_alg_type | 1127 if (all_contig) {
607 OP_ALG_AS_INITFINAL); 1128 dst_dma = sg_dma_address(req->src);
1129 } else {
1130 dst_dma = src_dma + sizeof(struct link_tbl_entry) *
1131 ((edesc->assoc_nents ? : 1) + 1);
1132 out_options = LDST_SGF;
1133 }
1134 } else {
1135 if (!edesc->dst_nents) {
1136 dst_dma = sg_dma_address(req->dst);
1137 } else {
1138 dst_dma = edesc->link_tbl_dma +
1139 link_tbl_index *
1140 sizeof(struct link_tbl_entry);
1141 out_options = LDST_SGF;
1142 }
1143 }
1144 if (encrypt)
1145 append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options);
1146 else
1147 append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize,
1148 out_options);
1149}
608 1150
609 uncond_jump_cmd = append_jump(desc, 0); 1151/*
1152 * Fill in aead givencrypt job descriptor
1153 */
1154static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr,
1155 struct aead_edesc *edesc,
1156 struct aead_request *req,
1157 int contig)
1158{
1159 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1160 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1161 int ivsize = crypto_aead_ivsize(aead);
1162 int authsize = ctx->authsize;
1163 u32 *desc = edesc->hw_desc;
1164 u32 out_options = 0, in_options;
1165 dma_addr_t dst_dma, src_dma;
1166 int len, link_tbl_index = 0;
610 1167
611 set_jump_tgt_here(desc, jump_cmd); 1168#ifdef DEBUG
1169 debug("assoclen %d cryptlen %d authsize %d\n",
1170 req->assoclen, req->cryptlen, authsize);
1171 print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
1172 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
1173 req->assoclen , 1);
1174 print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
1175 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
1176 print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
1177 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1178 edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
1179 print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
1180 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
1181 desc_bytes(sh_desc), 1);
1182#endif
612 1183
613 /* start class 1 (cipher) operation, shared version */ 1184 len = desc_len(sh_desc);
614 append_operation(desc, ctx->class1_alg_type | 1185 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
615 OP_ALG_AS_INITFINAL | OP_ALG_AAI_DK);
616 set_jump_tgt_here(desc, uncond_jump_cmd);
617 } else
618 append_operation(desc, ctx->class1_alg_type |
619 OP_ALG_AS_INITFINAL | encrypt);
620 1186
621 /* load payload & instruct to class2 to snoop class 1 if encrypting */ 1187 if (contig & GIV_SRC_CONTIG) {
622 options = 0; 1188 src_dma = sg_dma_address(req->assoc);
623 if (!edesc->src_nents) { 1189 in_options = 0;
624 src_dma = sg_dma_address(areq->src);
625 } else { 1190 } else {
626 sg_to_link_tbl(areq->src, edesc->src_nents, edesc->link_tbl + 1191 src_dma = edesc->link_tbl_dma;
627 edesc->assoc_nents, 0); 1192 link_tbl_index += edesc->assoc_nents + 1 + edesc->src_nents;
628 src_dma = edesc->link_tbl_dma + edesc->assoc_nents * 1193 in_options = LDST_SGF;
629 sizeof(struct link_tbl_entry);
630 options |= LDST_SGF;
631 } 1194 }
632 append_seq_in_ptr(desc, src_dma, areq->cryptlen + authsize, options); 1195 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize +
633 append_seq_fifo_load(desc, areq->cryptlen, FIFOLD_CLASS_BOTH | 1196 req->cryptlen - authsize, in_options);
634 FIFOLD_TYPE_LASTBOTH | 1197
635 (encrypt ? FIFOLD_TYPE_MSG1OUT2 1198 if (contig & GIV_DST_CONTIG) {
636 : FIFOLD_TYPE_MSG)); 1199 dst_dma = edesc->iv_dma;
637
638 /* specify destination */
639 if (areq->src == areq->dst) {
640 dst_dma = src_dma;
641 } else { 1200 } else {
642 sg_count = dma_map_sg(jrdev, areq->dst, edesc->dst_nents ? : 1, 1201 if (likely(req->src == req->dst)) {
643 DMA_FROM_DEVICE); 1202 dst_dma = src_dma + sizeof(struct link_tbl_entry) *
644 if (!edesc->dst_nents) { 1203 edesc->assoc_nents;
645 dst_dma = sg_dma_address(areq->dst); 1204 out_options = LDST_SGF;
646 options = 0;
647 } else { 1205 } else {
648 sg_to_link_tbl(areq->dst, edesc->dst_nents, 1206 dst_dma = edesc->link_tbl_dma +
649 edesc->link_tbl + edesc->assoc_nents + 1207 link_tbl_index *
650 edesc->src_nents, 0);
651 dst_dma = edesc->link_tbl_dma + (edesc->assoc_nents +
652 edesc->src_nents) *
653 sizeof(struct link_tbl_entry); 1208 sizeof(struct link_tbl_entry);
654 options = LDST_SGF; 1209 out_options = LDST_SGF;
655 } 1210 }
656 } 1211 }
657 append_seq_out_ptr(desc, dst_dma, areq->cryptlen + authsize, options);
658 append_seq_fifo_store(desc, areq->cryptlen, FIFOST_TYPE_MESSAGE_DATA);
659 1212
660 /* ICV */ 1213 append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen, out_options);
661 if (encrypt) 1214}
662 append_seq_store(desc, authsize, LDST_CLASS_2_CCB | 1215
663 LDST_SRCDST_BYTE_CONTEXT); 1216/*
664 else 1217 * Fill in ablkcipher job descriptor
665 append_seq_fifo_load(desc, authsize, FIFOLD_CLASS_CLASS2 | 1218 */
666 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); 1219static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
1220 struct ablkcipher_edesc *edesc,
1221 struct ablkcipher_request *req,
1222 bool iv_contig)
1223{
1224 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1225 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1226 u32 *desc = edesc->hw_desc;
1227 u32 out_options = 0, in_options;
1228 dma_addr_t dst_dma, src_dma;
1229 int len, link_tbl_index = 0;
667 1230
668#ifdef DEBUG 1231#ifdef DEBUG
669 debug("job_desc_len %d\n", desc_len(desc)); 1232 print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
670 print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ", 1233 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
671 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc) , 1); 1234 ivsize, 1);
672 print_hex_dump(KERN_ERR, "jdlinkt@"xstr(__LINE__)": ", 1235 print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
673 DUMP_PREFIX_ADDRESS, 16, 4, edesc->link_tbl, 1236 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
674 edesc->link_tbl_bytes, 1); 1237 edesc->src_nents ? 100 : req->nbytes, 1);
675#endif 1238#endif
676 1239
677 ret = caam_jr_enqueue(jrdev, desc, callback, areq); 1240 len = desc_len(sh_desc);
678 if (!ret) 1241 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
679 ret = -EINPROGRESS; 1242
680 else { 1243 if (iv_contig) {
681 ipsec_esp_unmap(jrdev, edesc, areq); 1244 src_dma = edesc->iv_dma;
682 kfree(edesc); 1245 in_options = 0;
1246 } else {
1247 src_dma = edesc->link_tbl_dma;
1248 link_tbl_index += (iv_contig ? 0 : 1) + edesc->src_nents;
1249 in_options = LDST_SGF;
683 } 1250 }
1251 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
684 1252
685 return ret; 1253 if (likely(req->src == req->dst)) {
1254 if (!edesc->src_nents && iv_contig) {
1255 dst_dma = sg_dma_address(req->src);
1256 } else {
1257 dst_dma = edesc->link_tbl_dma +
1258 sizeof(struct link_tbl_entry);
1259 out_options = LDST_SGF;
1260 }
1261 } else {
1262 if (!edesc->dst_nents) {
1263 dst_dma = sg_dma_address(req->dst);
1264 } else {
1265 dst_dma = edesc->link_tbl_dma +
1266 link_tbl_index * sizeof(struct link_tbl_entry);
1267 out_options = LDST_SGF;
1268 }
1269 }
1270 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
686} 1271}
687 1272
688/* 1273/*
689 * derive number of elements in scatterlist 1274 * derive number of elements in scatterlist
690 */ 1275 */
691static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained) 1276static int sg_count(struct scatterlist *sg_list, int nbytes)
692{ 1277{
693 struct scatterlist *sg = sg_list; 1278 struct scatterlist *sg = sg_list;
694 int sg_nents = 0; 1279 int sg_nents = 0;
695 1280
696 *chained = 0;
697 while (nbytes > 0) { 1281 while (nbytes > 0) {
698 sg_nents++; 1282 sg_nents++;
699 nbytes -= sg->length; 1283 nbytes -= sg->length;
700 if (!sg_is_last(sg) && (sg + 1)->length == 0) 1284 if (!sg_is_last(sg) && (sg + 1)->length == 0)
701 *chained = 1; 1285 BUG(); /* Not support chaining */
702 sg = scatterwalk_sg_next(sg); 1286 sg = scatterwalk_sg_next(sg);
703 } 1287 }
704 1288
1289 if (likely(sg_nents == 1))
1290 return 0;
1291
705 return sg_nents; 1292 return sg_nents;
706} 1293}
707 1294
708/* 1295/*
709 * allocate and map the ipsec_esp extended descriptor 1296 * allocate and map the aead extended descriptor
710 */ 1297 */
711static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq, 1298static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
712 int desc_bytes) 1299 int desc_bytes, bool *all_contig_ptr)
713{ 1300{
714 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1301 struct crypto_aead *aead = crypto_aead_reqtfm(req);
715 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1302 struct caam_ctx *ctx = crypto_aead_ctx(aead);
716 struct device *jrdev = ctx->jrdev; 1303 struct device *jrdev = ctx->jrdev;
717 gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : 1304 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
718 GFP_ATOMIC; 1305 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
719 int assoc_nents, src_nents, dst_nents = 0, chained, link_tbl_bytes; 1306 int assoc_nents, src_nents, dst_nents = 0;
720 struct ipsec_esp_edesc *edesc; 1307 struct aead_edesc *edesc;
721 1308 dma_addr_t iv_dma = 0;
722 assoc_nents = sg_count(areq->assoc, areq->assoclen, &chained); 1309 int sgc;
723 BUG_ON(chained); 1310 bool all_contig = true;
724 if (likely(assoc_nents == 1)) 1311 int ivsize = crypto_aead_ivsize(aead);
725 assoc_nents = 0; 1312 int link_tbl_index, link_tbl_len = 0, link_tbl_bytes;
726 1313
727 src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize, 1314 assoc_nents = sg_count(req->assoc, req->assoclen);
728 &chained); 1315 src_nents = sg_count(req->src, req->cryptlen);
729 BUG_ON(chained); 1316
730 if (src_nents == 1) 1317 if (unlikely(req->dst != req->src))
731 src_nents = 0; 1318 dst_nents = sg_count(req->dst, req->cryptlen);
732 1319
733 if (unlikely(areq->dst != areq->src)) { 1320 sgc = dma_map_sg(jrdev, req->assoc, assoc_nents ? : 1,
734 dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize, 1321 DMA_BIDIRECTIONAL);
735 &chained); 1322 if (likely(req->src == req->dst)) {
736 BUG_ON(chained); 1323 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
737 if (dst_nents == 1) 1324 DMA_BIDIRECTIONAL);
738 dst_nents = 0; 1325 } else {
1326 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1327 DMA_TO_DEVICE);
1328 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
1329 DMA_FROM_DEVICE);
739 } 1330 }
740 1331
741 link_tbl_bytes = (assoc_nents + src_nents + dst_nents) * 1332 /* Check if data are contiguous */
742 sizeof(struct link_tbl_entry); 1333 iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE);
743 debug("link_tbl_bytes %d\n", link_tbl_bytes); 1334 if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen !=
1335 iv_dma || src_nents || iv_dma + ivsize !=
1336 sg_dma_address(req->src)) {
1337 all_contig = false;
1338 assoc_nents = assoc_nents ? : 1;
1339 src_nents = src_nents ? : 1;
1340 link_tbl_len = assoc_nents + 1 + src_nents;
1341 }
1342 link_tbl_len += dst_nents;
1343
1344 link_tbl_bytes = link_tbl_len * sizeof(struct link_tbl_entry);
744 1345
745 /* allocate space for base edesc and hw desc commands, link tables */ 1346 /* allocate space for base edesc and hw desc commands, link tables */
746 edesc = kmalloc(sizeof(struct ipsec_esp_edesc) + desc_bytes + 1347 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
747 link_tbl_bytes, GFP_DMA | flags); 1348 link_tbl_bytes, GFP_DMA | flags);
748 if (!edesc) { 1349 if (!edesc) {
749 dev_err(jrdev, "could not allocate extended descriptor\n"); 1350 dev_err(jrdev, "could not allocate extended descriptor\n");
@@ -753,142 +1354,450 @@ static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq,
753 edesc->assoc_nents = assoc_nents; 1354 edesc->assoc_nents = assoc_nents;
754 edesc->src_nents = src_nents; 1355 edesc->src_nents = src_nents;
755 edesc->dst_nents = dst_nents; 1356 edesc->dst_nents = dst_nents;
756 edesc->link_tbl = (void *)edesc + sizeof(struct ipsec_esp_edesc) + 1357 edesc->iv_dma = iv_dma;
1358 edesc->link_tbl_bytes = link_tbl_bytes;
1359 edesc->link_tbl = (void *)edesc + sizeof(struct aead_edesc) +
757 desc_bytes; 1360 desc_bytes;
758 edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl, 1361 edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl,
759 link_tbl_bytes, DMA_TO_DEVICE); 1362 link_tbl_bytes, DMA_TO_DEVICE);
760 edesc->link_tbl_bytes = link_tbl_bytes; 1363 *all_contig_ptr = all_contig;
1364
1365 link_tbl_index = 0;
1366 if (!all_contig) {
1367 sg_to_link_tbl(req->assoc,
1368 (assoc_nents ? : 1),
1369 edesc->link_tbl +
1370 link_tbl_index, 0);
1371 link_tbl_index += assoc_nents ? : 1;
1372 sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index,
1373 iv_dma, ivsize, 0);
1374 link_tbl_index += 1;
1375 sg_to_link_tbl_last(req->src,
1376 (src_nents ? : 1),
1377 edesc->link_tbl +
1378 link_tbl_index, 0);
1379 link_tbl_index += src_nents ? : 1;
1380 }
1381 if (dst_nents) {
1382 sg_to_link_tbl_last(req->dst, dst_nents,
1383 edesc->link_tbl + link_tbl_index, 0);
1384 }
761 1385
762 return edesc; 1386 return edesc;
763} 1387}
764 1388
765static int aead_authenc_encrypt(struct aead_request *areq) 1389static int aead_encrypt(struct aead_request *req)
766{ 1390{
767 struct ipsec_esp_edesc *edesc; 1391 struct aead_edesc *edesc;
768 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1392 struct crypto_aead *aead = crypto_aead_reqtfm(req);
769 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1393 struct caam_ctx *ctx = crypto_aead_ctx(aead);
770 struct device *jrdev = ctx->jrdev; 1394 struct device *jrdev = ctx->jrdev;
771 int ivsize = crypto_aead_ivsize(aead); 1395 bool all_contig;
772 u32 *desc; 1396 u32 *desc;
773 dma_addr_t iv_dma; 1397 int ret = 0;
1398
1399 req->cryptlen += ctx->authsize;
774 1400
775 /* allocate extended descriptor */ 1401 /* allocate extended descriptor */
776 edesc = ipsec_esp_edesc_alloc(areq, DESC_AEAD_ENCRYPT_TEXT_LEN * 1402 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
777 CAAM_CMD_SZ); 1403 CAAM_CMD_SZ, &all_contig);
778 if (IS_ERR(edesc)) 1404 if (IS_ERR(edesc))
779 return PTR_ERR(edesc); 1405 return PTR_ERR(edesc);
780 1406
781 desc = edesc->hw_desc; 1407 /* Create and submit job descriptor */
782 1408 init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req,
783 /* insert shared descriptor pointer */ 1409 all_contig, true);
784 init_job_desc_shared(desc, ctx->shared_desc_phys, 1410#ifdef DEBUG
785 desc_len(ctx->sh_desc), HDR_SHARE_DEFER); 1411 print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
786 1412 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
787 iv_dma = dma_map_single(jrdev, areq->iv, ivsize, DMA_TO_DEVICE); 1413 desc_bytes(edesc->hw_desc), 1);
788 /* check dma error */ 1414#endif
789 1415
790 append_load(desc, iv_dma, ivsize, 1416 desc = edesc->hw_desc;
791 LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); 1417 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1418 if (!ret) {
1419 ret = -EINPROGRESS;
1420 } else {
1421 aead_unmap(jrdev, edesc, req);
1422 kfree(edesc);
1423 }
792 1424
793 return ipsec_esp(edesc, areq, OP_ALG_ENCRYPT, ipsec_esp_encrypt_done); 1425 return ret;
794} 1426}
795 1427
796static int aead_authenc_decrypt(struct aead_request *req) 1428static int aead_decrypt(struct aead_request *req)
797{ 1429{
1430 struct aead_edesc *edesc;
798 struct crypto_aead *aead = crypto_aead_reqtfm(req); 1431 struct crypto_aead *aead = crypto_aead_reqtfm(req);
799 int ivsize = crypto_aead_ivsize(aead);
800 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1432 struct caam_ctx *ctx = crypto_aead_ctx(aead);
801 struct device *jrdev = ctx->jrdev; 1433 struct device *jrdev = ctx->jrdev;
802 struct ipsec_esp_edesc *edesc; 1434 bool all_contig;
803 u32 *desc; 1435 u32 *desc;
804 dma_addr_t iv_dma; 1436 int ret = 0;
805
806 req->cryptlen -= ctx->authsize;
807 1437
808 /* allocate extended descriptor */ 1438 /* allocate extended descriptor */
809 edesc = ipsec_esp_edesc_alloc(req, DESC_AEAD_DECRYPT_TEXT_LEN * 1439 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
810 CAAM_CMD_SZ); 1440 CAAM_CMD_SZ, &all_contig);
811 if (IS_ERR(edesc)) 1441 if (IS_ERR(edesc))
812 return PTR_ERR(edesc); 1442 return PTR_ERR(edesc);
813 1443
1444#ifdef DEBUG
1445 print_hex_dump(KERN_ERR, "dec src@"xstr(__LINE__)": ",
1446 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1447 req->cryptlen, 1);
1448#endif
1449
1450 /* Create and submit job descriptor*/
1451 init_aead_job(ctx->sh_desc_dec,
1452 ctx->sh_desc_dec_dma, edesc, req, all_contig, false);
1453#ifdef DEBUG
1454 print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
1455 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1456 desc_bytes(edesc->hw_desc), 1);
1457#endif
1458
814 desc = edesc->hw_desc; 1459 desc = edesc->hw_desc;
1460 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
1461 if (!ret) {
1462 ret = -EINPROGRESS;
1463 } else {
1464 aead_unmap(jrdev, edesc, req);
1465 kfree(edesc);
1466 }
815 1467
816 /* insert shared descriptor pointer */ 1468 return ret;
817 init_job_desc_shared(desc, ctx->shared_desc_phys, 1469}
818 desc_len(ctx->sh_desc), HDR_SHARE_DEFER);
819 1470
820 iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); 1471/*
821 /* check dma error */ 1472 * allocate and map the aead extended descriptor for aead givencrypt
1473 */
1474static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request
1475 *greq, int desc_bytes,
1476 u32 *contig_ptr)
1477{
1478 struct aead_request *req = &greq->areq;
1479 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1480 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1481 struct device *jrdev = ctx->jrdev;
1482 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1483 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1484 int assoc_nents, src_nents, dst_nents = 0;
1485 struct aead_edesc *edesc;
1486 dma_addr_t iv_dma = 0;
1487 int sgc;
1488 u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG;
1489 int ivsize = crypto_aead_ivsize(aead);
1490 int link_tbl_index, link_tbl_len = 0, link_tbl_bytes;
1491
1492 assoc_nents = sg_count(req->assoc, req->assoclen);
1493 src_nents = sg_count(req->src, req->cryptlen);
822 1494
823 append_load(desc, iv_dma, ivsize, 1495 if (unlikely(req->dst != req->src))
824 LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); 1496 dst_nents = sg_count(req->dst, req->cryptlen);
825 1497
826 return ipsec_esp(edesc, req, !OP_ALG_ENCRYPT, ipsec_esp_decrypt_done); 1498 sgc = dma_map_sg(jrdev, req->assoc, assoc_nents ? : 1,
1499 DMA_BIDIRECTIONAL);
1500 if (likely(req->src == req->dst)) {
1501 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1502 DMA_BIDIRECTIONAL);
1503 } else {
1504 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1505 DMA_TO_DEVICE);
1506 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
1507 DMA_FROM_DEVICE);
1508 }
1509
1510 /* Check if data are contiguous */
1511 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
1512 if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen !=
1513 iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src))
1514 contig &= ~GIV_SRC_CONTIG;
1515 if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst))
1516 contig &= ~GIV_DST_CONTIG;
1517 if (unlikely(req->src != req->dst)) {
1518 dst_nents = dst_nents ? : 1;
1519 link_tbl_len += 1;
1520 }
1521 if (!(contig & GIV_SRC_CONTIG)) {
1522 assoc_nents = assoc_nents ? : 1;
1523 src_nents = src_nents ? : 1;
1524 link_tbl_len += assoc_nents + 1 + src_nents;
1525 if (likely(req->src == req->dst))
1526 contig &= ~GIV_DST_CONTIG;
1527 }
1528 link_tbl_len += dst_nents;
1529
1530 link_tbl_bytes = link_tbl_len * sizeof(struct link_tbl_entry);
1531
1532 /* allocate space for base edesc and hw desc commands, link tables */
1533 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
1534 link_tbl_bytes, GFP_DMA | flags);
1535 if (!edesc) {
1536 dev_err(jrdev, "could not allocate extended descriptor\n");
1537 return ERR_PTR(-ENOMEM);
1538 }
1539
1540 edesc->assoc_nents = assoc_nents;
1541 edesc->src_nents = src_nents;
1542 edesc->dst_nents = dst_nents;
1543 edesc->iv_dma = iv_dma;
1544 edesc->link_tbl_bytes = link_tbl_bytes;
1545 edesc->link_tbl = (void *)edesc + sizeof(struct aead_edesc) +
1546 desc_bytes;
1547 edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl,
1548 link_tbl_bytes, DMA_TO_DEVICE);
1549 *contig_ptr = contig;
1550
1551 link_tbl_index = 0;
1552 if (!(contig & GIV_SRC_CONTIG)) {
1553 sg_to_link_tbl(req->assoc, assoc_nents,
1554 edesc->link_tbl +
1555 link_tbl_index, 0);
1556 link_tbl_index += assoc_nents;
1557 sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index,
1558 iv_dma, ivsize, 0);
1559 link_tbl_index += 1;
1560 sg_to_link_tbl_last(req->src, src_nents,
1561 edesc->link_tbl +
1562 link_tbl_index, 0);
1563 link_tbl_index += src_nents;
1564 }
1565 if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) {
1566 sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index,
1567 iv_dma, ivsize, 0);
1568 link_tbl_index += 1;
1569 sg_to_link_tbl_last(req->dst, dst_nents,
1570 edesc->link_tbl + link_tbl_index, 0);
1571 }
1572
1573 return edesc;
827} 1574}
828 1575
829static int aead_authenc_givencrypt(struct aead_givcrypt_request *req) 1576static int aead_givencrypt(struct aead_givcrypt_request *areq)
830{ 1577{
831 struct aead_request *areq = &req->areq; 1578 struct aead_request *req = &areq->areq;
832 struct ipsec_esp_edesc *edesc; 1579 struct aead_edesc *edesc;
833 struct crypto_aead *aead = crypto_aead_reqtfm(areq); 1580 struct crypto_aead *aead = crypto_aead_reqtfm(req);
834 struct caam_ctx *ctx = crypto_aead_ctx(aead); 1581 struct caam_ctx *ctx = crypto_aead_ctx(aead);
835 struct device *jrdev = ctx->jrdev; 1582 struct device *jrdev = ctx->jrdev;
836 int ivsize = crypto_aead_ivsize(aead); 1583 u32 contig;
837 dma_addr_t iv_dma;
838 u32 *desc; 1584 u32 *desc;
1585 int ret = 0;
839 1586
840 iv_dma = dma_map_single(jrdev, req->giv, ivsize, DMA_FROM_DEVICE); 1587 req->cryptlen += ctx->authsize;
841
842 debug("%s: giv %p\n", __func__, req->giv);
843 1588
844 /* allocate extended descriptor */ 1589 /* allocate extended descriptor */
845 edesc = ipsec_esp_edesc_alloc(areq, DESC_AEAD_GIVENCRYPT_TEXT_LEN * 1590 edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN *
846 CAAM_CMD_SZ); 1591 CAAM_CMD_SZ, &contig);
1592
847 if (IS_ERR(edesc)) 1593 if (IS_ERR(edesc))
848 return PTR_ERR(edesc); 1594 return PTR_ERR(edesc);
849 1595
1596#ifdef DEBUG
1597 print_hex_dump(KERN_ERR, "giv src@"xstr(__LINE__)": ",
1598 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1599 req->cryptlen, 1);
1600#endif
1601
1602 /* Create and submit job descriptor*/
1603 init_aead_giv_job(ctx->sh_desc_givenc,
1604 ctx->sh_desc_givenc_dma, edesc, req, contig);
1605#ifdef DEBUG
1606 print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
1607 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1608 desc_bytes(edesc->hw_desc), 1);
1609#endif
1610
850 desc = edesc->hw_desc; 1611 desc = edesc->hw_desc;
1612 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1613 if (!ret) {
1614 ret = -EINPROGRESS;
1615 } else {
1616 aead_unmap(jrdev, edesc, req);
1617 kfree(edesc);
1618 }
851 1619
852 /* insert shared descriptor pointer */ 1620 return ret;
853 init_job_desc_shared(desc, ctx->shared_desc_phys, 1621}
854 desc_len(ctx->sh_desc), HDR_SHARE_DEFER);
855 1622
856 /* 1623/*
857 * LOAD IMM Info FIFO 1624 * allocate and map the ablkcipher extended descriptor for ablkcipher
858 * to DECO, Last, Padding, Random, Message, 16 bytes 1625 */
859 */ 1626static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
860 append_load_imm_u32(desc, NFIFOENTRY_DEST_DECO | NFIFOENTRY_LC1 | 1627 *req, int desc_bytes,
861 NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DTYPE_MSG | 1628 bool *iv_contig_out)
862 NFIFOENTRY_PTYPE_RND | ivsize, 1629{
863 LDST_SRCDST_WORD_INFO_FIFO); 1630 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1631 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1632 struct device *jrdev = ctx->jrdev;
1633 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1634 CRYPTO_TFM_REQ_MAY_SLEEP)) ?
1635 GFP_KERNEL : GFP_ATOMIC;
1636 int src_nents, dst_nents = 0, link_tbl_bytes;
1637 struct ablkcipher_edesc *edesc;
1638 dma_addr_t iv_dma = 0;
1639 bool iv_contig = false;
1640 int sgc;
1641 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1642 int link_tbl_index;
1643
1644 src_nents = sg_count(req->src, req->nbytes);
1645
1646 if (unlikely(req->dst != req->src))
1647 dst_nents = sg_count(req->dst, req->nbytes);
1648
1649 if (likely(req->src == req->dst)) {
1650 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1651 DMA_BIDIRECTIONAL);
1652 } else {
1653 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
1654 DMA_TO_DEVICE);
1655 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
1656 DMA_FROM_DEVICE);
1657 }
864 1658
865 /* 1659 /*
866 * disable info fifo entries since the above serves as the entry 1660 * Check if iv can be contiguous with source and destination.
867 * this way, the MOVE command won't generate an entry. 1661 * If so, include it. If not, create scatterlist.
868 * Note that this isn't required in more recent versions of
869 * SEC as a MOVE that doesn't do info FIFO entries is available.
870 */ 1662 */
871 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); 1663 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
1664 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
1665 iv_contig = true;
1666 else
1667 src_nents = src_nents ? : 1;
1668 link_tbl_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
1669 sizeof(struct link_tbl_entry);
872 1670
873 /* MOVE DECO Alignment -> C1 Context 16 bytes */ 1671 /* allocate space for base edesc and hw desc commands, link tables */
874 append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | ivsize); 1672 edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes +
1673 link_tbl_bytes, GFP_DMA | flags);
1674 if (!edesc) {
1675 dev_err(jrdev, "could not allocate extended descriptor\n");
1676 return ERR_PTR(-ENOMEM);
1677 }
875 1678
876 /* re-enable info fifo entries */ 1679 edesc->src_nents = src_nents;
877 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); 1680 edesc->dst_nents = dst_nents;
1681 edesc->link_tbl_bytes = link_tbl_bytes;
1682 edesc->link_tbl = (void *)edesc + sizeof(struct ablkcipher_edesc) +
1683 desc_bytes;
1684
1685 link_tbl_index = 0;
1686 if (!iv_contig) {
1687 sg_to_link_tbl_one(edesc->link_tbl, iv_dma, ivsize, 0);
1688 sg_to_link_tbl_last(req->src, src_nents,
1689 edesc->link_tbl + 1, 0);
1690 link_tbl_index += 1 + src_nents;
1691 }
1692
1693 if (unlikely(dst_nents)) {
1694 sg_to_link_tbl_last(req->dst, dst_nents,
1695 edesc->link_tbl + link_tbl_index, 0);
1696 }
1697
1698 edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl,
1699 link_tbl_bytes, DMA_TO_DEVICE);
1700 edesc->iv_dma = iv_dma;
1701
1702#ifdef DEBUG
1703 print_hex_dump(KERN_ERR, "ablkcipher link_tbl@"xstr(__LINE__)": ",
1704 DUMP_PREFIX_ADDRESS, 16, 4, edesc->link_tbl,
1705 link_tbl_bytes, 1);
1706#endif
1707
1708 *iv_contig_out = iv_contig;
1709 return edesc;
1710}
1711
1712static int ablkcipher_encrypt(struct ablkcipher_request *req)
1713{
1714 struct ablkcipher_edesc *edesc;
1715 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1716 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1717 struct device *jrdev = ctx->jrdev;
1718 bool iv_contig;
1719 u32 *desc;
1720 int ret = 0;
1721
1722 /* allocate extended descriptor */
1723 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1724 CAAM_CMD_SZ, &iv_contig);
1725 if (IS_ERR(edesc))
1726 return PTR_ERR(edesc);
878 1727
879 /* MOVE C1 Context -> OFIFO 16 bytes */ 1728 /* Create and submit job descriptor*/
880 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | ivsize); 1729 init_ablkcipher_job(ctx->sh_desc_enc,
1730 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
1731#ifdef DEBUG
1732 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ",
1733 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1734 desc_bytes(edesc->hw_desc), 1);
1735#endif
1736 desc = edesc->hw_desc;
1737 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
881 1738
882 append_fifo_store(desc, iv_dma, ivsize, FIFOST_TYPE_MESSAGE_DATA); 1739 if (!ret) {
1740 ret = -EINPROGRESS;
1741 } else {
1742 ablkcipher_unmap(jrdev, edesc, req);
1743 kfree(edesc);
1744 }
883 1745
884 return ipsec_esp(edesc, areq, OP_ALG_ENCRYPT, ipsec_esp_encrypt_done); 1746 return ret;
885} 1747}
886 1748
1749static int ablkcipher_decrypt(struct ablkcipher_request *req)
1750{
1751 struct ablkcipher_edesc *edesc;
1752 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1753 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1754 struct device *jrdev = ctx->jrdev;
1755 bool iv_contig;
1756 u32 *desc;
1757 int ret = 0;
1758
1759 /* allocate extended descriptor */
1760 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1761 CAAM_CMD_SZ, &iv_contig);
1762 if (IS_ERR(edesc))
1763 return PTR_ERR(edesc);
1764
1765 /* Create and submit job descriptor*/
1766 init_ablkcipher_job(ctx->sh_desc_dec,
1767 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
1768 desc = edesc->hw_desc;
1769#ifdef DEBUG
1770 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ",
1771 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1772 desc_bytes(edesc->hw_desc), 1);
1773#endif
1774
1775 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
1776 if (!ret) {
1777 ret = -EINPROGRESS;
1778 } else {
1779 ablkcipher_unmap(jrdev, edesc, req);
1780 kfree(edesc);
1781 }
1782
1783 return ret;
1784}
1785
1786#define template_aead template_u.aead
1787#define template_ablkcipher template_u.ablkcipher
887struct caam_alg_template { 1788struct caam_alg_template {
888 char name[CRYPTO_MAX_ALG_NAME]; 1789 char name[CRYPTO_MAX_ALG_NAME];
889 char driver_name[CRYPTO_MAX_ALG_NAME]; 1790 char driver_name[CRYPTO_MAX_ALG_NAME];
890 unsigned int blocksize; 1791 unsigned int blocksize;
891 struct aead_alg aead; 1792 u32 type;
1793 union {
1794 struct ablkcipher_alg ablkcipher;
1795 struct aead_alg aead;
1796 struct blkcipher_alg blkcipher;
1797 struct cipher_alg cipher;
1798 struct compress_alg compress;
1799 struct rng_alg rng;
1800 } template_u;
892 u32 class1_alg_type; 1801 u32 class1_alg_type;
893 u32 class2_alg_type; 1802 u32 class2_alg_type;
894 u32 alg_op; 1803 u32 alg_op;
@@ -900,12 +1809,13 @@ static struct caam_alg_template driver_algs[] = {
900 .name = "authenc(hmac(sha1),cbc(aes))", 1809 .name = "authenc(hmac(sha1),cbc(aes))",
901 .driver_name = "authenc-hmac-sha1-cbc-aes-caam", 1810 .driver_name = "authenc-hmac-sha1-cbc-aes-caam",
902 .blocksize = AES_BLOCK_SIZE, 1811 .blocksize = AES_BLOCK_SIZE,
903 .aead = { 1812 .type = CRYPTO_ALG_TYPE_AEAD,
904 .setkey = aead_authenc_setkey, 1813 .template_aead = {
905 .setauthsize = aead_authenc_setauthsize, 1814 .setkey = aead_setkey,
906 .encrypt = aead_authenc_encrypt, 1815 .setauthsize = aead_setauthsize,
907 .decrypt = aead_authenc_decrypt, 1816 .encrypt = aead_encrypt,
908 .givencrypt = aead_authenc_givencrypt, 1817 .decrypt = aead_decrypt,
1818 .givencrypt = aead_givencrypt,
909 .geniv = "<built-in>", 1819 .geniv = "<built-in>",
910 .ivsize = AES_BLOCK_SIZE, 1820 .ivsize = AES_BLOCK_SIZE,
911 .maxauthsize = SHA1_DIGEST_SIZE, 1821 .maxauthsize = SHA1_DIGEST_SIZE,
@@ -918,12 +1828,13 @@ static struct caam_alg_template driver_algs[] = {
918 .name = "authenc(hmac(sha256),cbc(aes))", 1828 .name = "authenc(hmac(sha256),cbc(aes))",
919 .driver_name = "authenc-hmac-sha256-cbc-aes-caam", 1829 .driver_name = "authenc-hmac-sha256-cbc-aes-caam",
920 .blocksize = AES_BLOCK_SIZE, 1830 .blocksize = AES_BLOCK_SIZE,
921 .aead = { 1831 .type = CRYPTO_ALG_TYPE_AEAD,
922 .setkey = aead_authenc_setkey, 1832 .template_aead = {
923 .setauthsize = aead_authenc_setauthsize, 1833 .setkey = aead_setkey,
924 .encrypt = aead_authenc_encrypt, 1834 .setauthsize = aead_setauthsize,
925 .decrypt = aead_authenc_decrypt, 1835 .encrypt = aead_encrypt,
926 .givencrypt = aead_authenc_givencrypt, 1836 .decrypt = aead_decrypt,
1837 .givencrypt = aead_givencrypt,
927 .geniv = "<built-in>", 1838 .geniv = "<built-in>",
928 .ivsize = AES_BLOCK_SIZE, 1839 .ivsize = AES_BLOCK_SIZE,
929 .maxauthsize = SHA256_DIGEST_SIZE, 1840 .maxauthsize = SHA256_DIGEST_SIZE,
@@ -937,12 +1848,13 @@ static struct caam_alg_template driver_algs[] = {
937 .name = "authenc(hmac(sha512),cbc(aes))", 1848 .name = "authenc(hmac(sha512),cbc(aes))",
938 .driver_name = "authenc-hmac-sha512-cbc-aes-caam", 1849 .driver_name = "authenc-hmac-sha512-cbc-aes-caam",
939 .blocksize = AES_BLOCK_SIZE, 1850 .blocksize = AES_BLOCK_SIZE,
940 .aead = { 1851 .type = CRYPTO_ALG_TYPE_AEAD,
941 .setkey = aead_authenc_setkey, 1852 .template_aead = {
942 .setauthsize = aead_authenc_setauthsize, 1853 .setkey = aead_setkey,
943 .encrypt = aead_authenc_encrypt, 1854 .setauthsize = aead_setauthsize,
944 .decrypt = aead_authenc_decrypt, 1855 .encrypt = aead_encrypt,
945 .givencrypt = aead_authenc_givencrypt, 1856 .decrypt = aead_decrypt,
1857 .givencrypt = aead_givencrypt,
946 .geniv = "<built-in>", 1858 .geniv = "<built-in>",
947 .ivsize = AES_BLOCK_SIZE, 1859 .ivsize = AES_BLOCK_SIZE,
948 .maxauthsize = SHA512_DIGEST_SIZE, 1860 .maxauthsize = SHA512_DIGEST_SIZE,
@@ -956,12 +1868,13 @@ static struct caam_alg_template driver_algs[] = {
956 .name = "authenc(hmac(sha1),cbc(des3_ede))", 1868 .name = "authenc(hmac(sha1),cbc(des3_ede))",
957 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam", 1869 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam",
958 .blocksize = DES3_EDE_BLOCK_SIZE, 1870 .blocksize = DES3_EDE_BLOCK_SIZE,
959 .aead = { 1871 .type = CRYPTO_ALG_TYPE_AEAD,
960 .setkey = aead_authenc_setkey, 1872 .template_aead = {
961 .setauthsize = aead_authenc_setauthsize, 1873 .setkey = aead_setkey,
962 .encrypt = aead_authenc_encrypt, 1874 .setauthsize = aead_setauthsize,
963 .decrypt = aead_authenc_decrypt, 1875 .encrypt = aead_encrypt,
964 .givencrypt = aead_authenc_givencrypt, 1876 .decrypt = aead_decrypt,
1877 .givencrypt = aead_givencrypt,
965 .geniv = "<built-in>", 1878 .geniv = "<built-in>",
966 .ivsize = DES3_EDE_BLOCK_SIZE, 1879 .ivsize = DES3_EDE_BLOCK_SIZE,
967 .maxauthsize = SHA1_DIGEST_SIZE, 1880 .maxauthsize = SHA1_DIGEST_SIZE,
@@ -974,12 +1887,13 @@ static struct caam_alg_template driver_algs[] = {
974 .name = "authenc(hmac(sha256),cbc(des3_ede))", 1887 .name = "authenc(hmac(sha256),cbc(des3_ede))",
975 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam", 1888 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam",
976 .blocksize = DES3_EDE_BLOCK_SIZE, 1889 .blocksize = DES3_EDE_BLOCK_SIZE,
977 .aead = { 1890 .type = CRYPTO_ALG_TYPE_AEAD,
978 .setkey = aead_authenc_setkey, 1891 .template_aead = {
979 .setauthsize = aead_authenc_setauthsize, 1892 .setkey = aead_setkey,
980 .encrypt = aead_authenc_encrypt, 1893 .setauthsize = aead_setauthsize,
981 .decrypt = aead_authenc_decrypt, 1894 .encrypt = aead_encrypt,
982 .givencrypt = aead_authenc_givencrypt, 1895 .decrypt = aead_decrypt,
1896 .givencrypt = aead_givencrypt,
983 .geniv = "<built-in>", 1897 .geniv = "<built-in>",
984 .ivsize = DES3_EDE_BLOCK_SIZE, 1898 .ivsize = DES3_EDE_BLOCK_SIZE,
985 .maxauthsize = SHA256_DIGEST_SIZE, 1899 .maxauthsize = SHA256_DIGEST_SIZE,
@@ -993,12 +1907,13 @@ static struct caam_alg_template driver_algs[] = {
993 .name = "authenc(hmac(sha512),cbc(des3_ede))", 1907 .name = "authenc(hmac(sha512),cbc(des3_ede))",
994 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam", 1908 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam",
995 .blocksize = DES3_EDE_BLOCK_SIZE, 1909 .blocksize = DES3_EDE_BLOCK_SIZE,
996 .aead = { 1910 .type = CRYPTO_ALG_TYPE_AEAD,
997 .setkey = aead_authenc_setkey, 1911 .template_aead = {
998 .setauthsize = aead_authenc_setauthsize, 1912 .setkey = aead_setkey,
999 .encrypt = aead_authenc_encrypt, 1913 .setauthsize = aead_setauthsize,
1000 .decrypt = aead_authenc_decrypt, 1914 .encrypt = aead_encrypt,
1001 .givencrypt = aead_authenc_givencrypt, 1915 .decrypt = aead_decrypt,
1916 .givencrypt = aead_givencrypt,
1002 .geniv = "<built-in>", 1917 .geniv = "<built-in>",
1003 .ivsize = DES3_EDE_BLOCK_SIZE, 1918 .ivsize = DES3_EDE_BLOCK_SIZE,
1004 .maxauthsize = SHA512_DIGEST_SIZE, 1919 .maxauthsize = SHA512_DIGEST_SIZE,
@@ -1012,12 +1927,13 @@ static struct caam_alg_template driver_algs[] = {
1012 .name = "authenc(hmac(sha1),cbc(des))", 1927 .name = "authenc(hmac(sha1),cbc(des))",
1013 .driver_name = "authenc-hmac-sha1-cbc-des-caam", 1928 .driver_name = "authenc-hmac-sha1-cbc-des-caam",
1014 .blocksize = DES_BLOCK_SIZE, 1929 .blocksize = DES_BLOCK_SIZE,
1015 .aead = { 1930 .type = CRYPTO_ALG_TYPE_AEAD,
1016 .setkey = aead_authenc_setkey, 1931 .template_aead = {
1017 .setauthsize = aead_authenc_setauthsize, 1932 .setkey = aead_setkey,
1018 .encrypt = aead_authenc_encrypt, 1933 .setauthsize = aead_setauthsize,
1019 .decrypt = aead_authenc_decrypt, 1934 .encrypt = aead_encrypt,
1020 .givencrypt = aead_authenc_givencrypt, 1935 .decrypt = aead_decrypt,
1936 .givencrypt = aead_givencrypt,
1021 .geniv = "<built-in>", 1937 .geniv = "<built-in>",
1022 .ivsize = DES_BLOCK_SIZE, 1938 .ivsize = DES_BLOCK_SIZE,
1023 .maxauthsize = SHA1_DIGEST_SIZE, 1939 .maxauthsize = SHA1_DIGEST_SIZE,
@@ -1030,12 +1946,13 @@ static struct caam_alg_template driver_algs[] = {
1030 .name = "authenc(hmac(sha256),cbc(des))", 1946 .name = "authenc(hmac(sha256),cbc(des))",
1031 .driver_name = "authenc-hmac-sha256-cbc-des-caam", 1947 .driver_name = "authenc-hmac-sha256-cbc-des-caam",
1032 .blocksize = DES_BLOCK_SIZE, 1948 .blocksize = DES_BLOCK_SIZE,
1033 .aead = { 1949 .type = CRYPTO_ALG_TYPE_AEAD,
1034 .setkey = aead_authenc_setkey, 1950 .template_aead = {
1035 .setauthsize = aead_authenc_setauthsize, 1951 .setkey = aead_setkey,
1036 .encrypt = aead_authenc_encrypt, 1952 .setauthsize = aead_setauthsize,
1037 .decrypt = aead_authenc_decrypt, 1953 .encrypt = aead_encrypt,
1038 .givencrypt = aead_authenc_givencrypt, 1954 .decrypt = aead_decrypt,
1955 .givencrypt = aead_givencrypt,
1039 .geniv = "<built-in>", 1956 .geniv = "<built-in>",
1040 .ivsize = DES_BLOCK_SIZE, 1957 .ivsize = DES_BLOCK_SIZE,
1041 .maxauthsize = SHA256_DIGEST_SIZE, 1958 .maxauthsize = SHA256_DIGEST_SIZE,
@@ -1049,12 +1966,13 @@ static struct caam_alg_template driver_algs[] = {
1049 .name = "authenc(hmac(sha512),cbc(des))", 1966 .name = "authenc(hmac(sha512),cbc(des))",
1050 .driver_name = "authenc-hmac-sha512-cbc-des-caam", 1967 .driver_name = "authenc-hmac-sha512-cbc-des-caam",
1051 .blocksize = DES_BLOCK_SIZE, 1968 .blocksize = DES_BLOCK_SIZE,
1052 .aead = { 1969 .type = CRYPTO_ALG_TYPE_AEAD,
1053 .setkey = aead_authenc_setkey, 1970 .template_aead = {
1054 .setauthsize = aead_authenc_setauthsize, 1971 .setkey = aead_setkey,
1055 .encrypt = aead_authenc_encrypt, 1972 .setauthsize = aead_setauthsize,
1056 .decrypt = aead_authenc_decrypt, 1973 .encrypt = aead_encrypt,
1057 .givencrypt = aead_authenc_givencrypt, 1974 .decrypt = aead_decrypt,
1975 .givencrypt = aead_givencrypt,
1058 .geniv = "<built-in>", 1976 .geniv = "<built-in>",
1059 .ivsize = DES_BLOCK_SIZE, 1977 .ivsize = DES_BLOCK_SIZE,
1060 .maxauthsize = SHA512_DIGEST_SIZE, 1978 .maxauthsize = SHA512_DIGEST_SIZE,
@@ -1064,6 +1982,55 @@ static struct caam_alg_template driver_algs[] = {
1064 OP_ALG_AAI_HMAC_PRECOMP, 1982 OP_ALG_AAI_HMAC_PRECOMP,
1065 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, 1983 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
1066 }, 1984 },
1985 /* ablkcipher descriptor */
1986 {
1987 .name = "cbc(aes)",
1988 .driver_name = "cbc-aes-caam",
1989 .blocksize = AES_BLOCK_SIZE,
1990 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
1991 .template_ablkcipher = {
1992 .setkey = ablkcipher_setkey,
1993 .encrypt = ablkcipher_encrypt,
1994 .decrypt = ablkcipher_decrypt,
1995 .geniv = "eseqiv",
1996 .min_keysize = AES_MIN_KEY_SIZE,
1997 .max_keysize = AES_MAX_KEY_SIZE,
1998 .ivsize = AES_BLOCK_SIZE,
1999 },
2000 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2001 },
2002 {
2003 .name = "cbc(des3_ede)",
2004 .driver_name = "cbc-3des-caam",
2005 .blocksize = DES3_EDE_BLOCK_SIZE,
2006 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2007 .template_ablkcipher = {
2008 .setkey = ablkcipher_setkey,
2009 .encrypt = ablkcipher_encrypt,
2010 .decrypt = ablkcipher_decrypt,
2011 .geniv = "eseqiv",
2012 .min_keysize = DES3_EDE_KEY_SIZE,
2013 .max_keysize = DES3_EDE_KEY_SIZE,
2014 .ivsize = DES3_EDE_BLOCK_SIZE,
2015 },
2016 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2017 },
2018 {
2019 .name = "cbc(des)",
2020 .driver_name = "cbc-des-caam",
2021 .blocksize = DES_BLOCK_SIZE,
2022 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2023 .template_ablkcipher = {
2024 .setkey = ablkcipher_setkey,
2025 .encrypt = ablkcipher_encrypt,
2026 .decrypt = ablkcipher_decrypt,
2027 .geniv = "eseqiv",
2028 .min_keysize = DES_KEY_SIZE,
2029 .max_keysize = DES_KEY_SIZE,
2030 .ivsize = DES_BLOCK_SIZE,
2031 },
2032 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2033 }
1067}; 2034};
1068 2035
1069struct caam_crypto_alg { 2036struct caam_crypto_alg {
@@ -1102,16 +2069,19 @@ static void caam_cra_exit(struct crypto_tfm *tfm)
1102{ 2069{
1103 struct caam_ctx *ctx = crypto_tfm_ctx(tfm); 2070 struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
1104 2071
1105 if (!dma_mapping_error(ctx->jrdev, ctx->shared_desc_phys)) 2072 if (ctx->sh_desc_enc_dma &&
1106 dma_unmap_single(ctx->jrdev, ctx->shared_desc_phys, 2073 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
1107 desc_bytes(ctx->sh_desc), DMA_TO_DEVICE); 2074 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
1108 kfree(ctx->sh_desc); 2075 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
1109 2076 if (ctx->sh_desc_dec_dma &&
1110 if (!dma_mapping_error(ctx->jrdev, ctx->key_phys)) 2077 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
1111 dma_unmap_single(ctx->jrdev, ctx->key_phys, 2078 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
1112 ctx->split_key_pad_len + ctx->enckeylen, 2079 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
2080 if (ctx->sh_desc_givenc_dma &&
2081 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
2082 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
2083 desc_bytes(ctx->sh_desc_givenc),
1113 DMA_TO_DEVICE); 2084 DMA_TO_DEVICE);
1114 kfree(ctx->key);
1115} 2085}
1116 2086
1117static void __exit caam_algapi_exit(void) 2087static void __exit caam_algapi_exit(void)
@@ -1175,12 +2145,20 @@ static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev,
1175 alg->cra_init = caam_cra_init; 2145 alg->cra_init = caam_cra_init;
1176 alg->cra_exit = caam_cra_exit; 2146 alg->cra_exit = caam_cra_exit;
1177 alg->cra_priority = CAAM_CRA_PRIORITY; 2147 alg->cra_priority = CAAM_CRA_PRIORITY;
1178 alg->cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
1179 alg->cra_blocksize = template->blocksize; 2148 alg->cra_blocksize = template->blocksize;
1180 alg->cra_alignmask = 0; 2149 alg->cra_alignmask = 0;
1181 alg->cra_type = &crypto_aead_type;
1182 alg->cra_ctxsize = sizeof(struct caam_ctx); 2150 alg->cra_ctxsize = sizeof(struct caam_ctx);
1183 alg->cra_u.aead = template->aead; 2151 alg->cra_flags = CRYPTO_ALG_ASYNC | template->type;
2152 switch (template->type) {
2153 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2154 alg->cra_type = &crypto_ablkcipher_type;
2155 alg->cra_ablkcipher = template->template_ablkcipher;
2156 break;
2157 case CRYPTO_ALG_TYPE_AEAD:
2158 alg->cra_type = &crypto_aead_type;
2159 alg->cra_aead = template->template_aead;
2160 break;
2161 }
1184 2162
1185 t_alg->class1_alg_type = template->class1_alg_type; 2163 t_alg->class1_alg_type = template->class1_alg_type;
1186 t_alg->class2_alg_type = template->class2_alg_type; 2164 t_alg->class2_alg_type = template->class2_alg_type;
diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h
index 950450346f70..d38f2afaa966 100644
--- a/drivers/crypto/caam/compat.h
+++ b/drivers/crypto/caam/compat.h
@@ -31,5 +31,6 @@
31#include <crypto/aead.h> 31#include <crypto/aead.h>
32#include <crypto/authenc.h> 32#include <crypto/authenc.h>
33#include <crypto/scatterwalk.h> 33#include <crypto/scatterwalk.h>
34#include <crypto/internal/skcipher.h>
34 35
35#endif /* !defined(CAAM_COMPAT_H) */ 36#endif /* !defined(CAAM_COMPAT_H) */
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index 9009713a3c2e..fc2d9ed22470 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -52,9 +52,11 @@ static int caam_probe(struct platform_device *pdev)
52 struct caam_ctrl __iomem *ctrl; 52 struct caam_ctrl __iomem *ctrl;
53 struct caam_full __iomem *topregs; 53 struct caam_full __iomem *topregs;
54 struct caam_drv_private *ctrlpriv; 54 struct caam_drv_private *ctrlpriv;
55 struct caam_perfmon *perfmon;
56 struct caam_deco **deco; 55 struct caam_deco **deco;
57 u32 deconum; 56 u32 deconum;
57#ifdef CONFIG_DEBUG_FS
58 struct caam_perfmon *perfmon;
59#endif
58 60
59 ctrlpriv = kzalloc(sizeof(struct caam_drv_private), GFP_KERNEL); 61 ctrlpriv = kzalloc(sizeof(struct caam_drv_private), GFP_KERNEL);
60 if (!ctrlpriv) 62 if (!ctrlpriv)
diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h
index 46915800c26f..0991323cf3fd 100644
--- a/drivers/crypto/caam/desc_constr.h
+++ b/drivers/crypto/caam/desc_constr.h
@@ -9,7 +9,7 @@
9#define IMMEDIATE (1 << 23) 9#define IMMEDIATE (1 << 23)
10#define CAAM_CMD_SZ sizeof(u32) 10#define CAAM_CMD_SZ sizeof(u32)
11#define CAAM_PTR_SZ sizeof(dma_addr_t) 11#define CAAM_PTR_SZ sizeof(dma_addr_t)
12#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * 64) 12#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * MAX_CAAM_DESCSIZE)
13 13
14#ifdef DEBUG 14#ifdef DEBUG
15#define PRINT_POS do { printk(KERN_DEBUG "%02d: %s\n", desc_len(desc),\ 15#define PRINT_POS do { printk(KERN_DEBUG "%02d: %s\n", desc_len(desc),\
@@ -18,6 +18,9 @@
18#define PRINT_POS 18#define PRINT_POS
19#endif 19#endif
20 20
21#define SET_OK_PROP_ERRORS (IMMEDIATE | LDST_CLASS_DECO | \
22 LDST_SRCDST_WORD_DECOCTRL | \
23 (LDOFF_CHG_SHARE_OK_PROP << LDST_OFFSET_SHIFT))
21#define DISABLE_AUTO_INFO_FIFO (IMMEDIATE | LDST_CLASS_DECO | \ 24#define DISABLE_AUTO_INFO_FIFO (IMMEDIATE | LDST_CLASS_DECO | \
22 LDST_SRCDST_WORD_DECOCTRL | \ 25 LDST_SRCDST_WORD_DECOCTRL | \
23 (LDOFF_DISABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT)) 26 (LDOFF_DISABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT))
@@ -203,3 +206,56 @@ static inline void append_##cmd##_imm_##type(u32 *desc, type immediate, \
203 append_cmd(desc, immediate); \ 206 append_cmd(desc, immediate); \
204} 207}
205APPEND_CMD_RAW_IMM(load, LOAD, u32); 208APPEND_CMD_RAW_IMM(load, LOAD, u32);
209
210/*
211 * Append math command. Only the last part of destination and source need to
212 * be specified
213 */
214#define APPEND_MATH(op, desc, dest, src_0, src_1, len) \
215append_cmd(desc, CMD_MATH | MATH_FUN_##op | MATH_DEST_##dest | \
216 MATH_SRC0_##src_0 | MATH_SRC1_##src_1 | (u32) (len & MATH_LEN_MASK));
217
218#define append_math_add(desc, dest, src0, src1, len) \
219 APPEND_MATH(ADD, desc, dest, src0, src1, len)
220#define append_math_sub(desc, dest, src0, src1, len) \
221 APPEND_MATH(SUB, desc, dest, src0, src1, len)
222#define append_math_add_c(desc, dest, src0, src1, len) \
223 APPEND_MATH(ADDC, desc, dest, src0, src1, len)
224#define append_math_sub_b(desc, dest, src0, src1, len) \
225 APPEND_MATH(SUBB, desc, dest, src0, src1, len)
226#define append_math_and(desc, dest, src0, src1, len) \
227 APPEND_MATH(AND, desc, dest, src0, src1, len)
228#define append_math_or(desc, dest, src0, src1, len) \
229 APPEND_MATH(OR, desc, dest, src0, src1, len)
230#define append_math_xor(desc, dest, src0, src1, len) \
231 APPEND_MATH(XOR, desc, dest, src0, src1, len)
232#define append_math_lshift(desc, dest, src0, src1, len) \
233 APPEND_MATH(LSHIFT, desc, dest, src0, src1, len)
234#define append_math_rshift(desc, dest, src0, src1, len) \
235 APPEND_MATH(RSHIFT, desc, dest, src0, src1, len)
236
237/* Exactly one source is IMM. Data is passed in as u32 value */
238#define APPEND_MATH_IMM_u32(op, desc, dest, src_0, src_1, data) \
239do { \
240 APPEND_MATH(op, desc, dest, src_0, src_1, CAAM_CMD_SZ); \
241 append_cmd(desc, data); \
242} while (0);
243
244#define append_math_add_imm_u32(desc, dest, src0, src1, data) \
245 APPEND_MATH_IMM_u32(ADD, desc, dest, src0, src1, data)
246#define append_math_sub_imm_u32(desc, dest, src0, src1, data) \
247 APPEND_MATH_IMM_u32(SUB, desc, dest, src0, src1, data)
248#define append_math_add_c_imm_u32(desc, dest, src0, src1, data) \
249 APPEND_MATH_IMM_u32(ADDC, desc, dest, src0, src1, data)
250#define append_math_sub_b_imm_u32(desc, dest, src0, src1, data) \
251 APPEND_MATH_IMM_u32(SUBB, desc, dest, src0, src1, data)
252#define append_math_and_imm_u32(desc, dest, src0, src1, data) \
253 APPEND_MATH_IMM_u32(AND, desc, dest, src0, src1, data)
254#define append_math_or_imm_u32(desc, dest, src0, src1, data) \
255 APPEND_MATH_IMM_u32(OR, desc, dest, src0, src1, data)
256#define append_math_xor_imm_u32(desc, dest, src0, src1, data) \
257 APPEND_MATH_IMM_u32(XOR, desc, dest, src0, src1, data)
258#define append_math_lshift_imm_u32(desc, dest, src0, src1, data) \
259 APPEND_MATH_IMM_u32(LSHIFT, desc, dest, src0, src1, data)
260#define append_math_rshift_imm_u32(desc, dest, src0, src1, data) \
261 APPEND_MATH_IMM_u32(RSHIFT, desc, dest, src0, src1, data)
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index ba8f1ea84c5e..6399a8f1938a 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -72,17 +72,20 @@
72 72
73#define DEFAULT_TIMEOUT_INTERVAL HZ 73#define DEFAULT_TIMEOUT_INTERVAL HZ
74 74
75#define FLAGS_FINUP 0x0002 75/* mostly device flags */
76#define FLAGS_FINAL 0x0004 76#define FLAGS_BUSY 0
77#define FLAGS_SG 0x0008 77#define FLAGS_FINAL 1
78#define FLAGS_SHA1 0x0010 78#define FLAGS_DMA_ACTIVE 2
79#define FLAGS_DMA_ACTIVE 0x0020 79#define FLAGS_OUTPUT_READY 3
80#define FLAGS_OUTPUT_READY 0x0040 80#define FLAGS_INIT 4
81#define FLAGS_INIT 0x0100 81#define FLAGS_CPU 5
82#define FLAGS_CPU 0x0200 82#define FLAGS_DMA_READY 6
83#define FLAGS_HMAC 0x0400 83/* context flags */
84#define FLAGS_ERROR 0x0800 84#define FLAGS_FINUP 16
85#define FLAGS_BUSY 0x1000 85#define FLAGS_SG 17
86#define FLAGS_SHA1 18
87#define FLAGS_HMAC 19
88#define FLAGS_ERROR 20
86 89
87#define OP_UPDATE 1 90#define OP_UPDATE 1
88#define OP_FINAL 2 91#define OP_FINAL 2
@@ -144,7 +147,6 @@ struct omap_sham_dev {
144 int dma; 147 int dma;
145 int dma_lch; 148 int dma_lch;
146 struct tasklet_struct done_task; 149 struct tasklet_struct done_task;
147 struct tasklet_struct queue_task;
148 150
149 unsigned long flags; 151 unsigned long flags;
150 struct crypto_queue queue; 152 struct crypto_queue queue;
@@ -223,7 +225,7 @@ static void omap_sham_copy_ready_hash(struct ahash_request *req)
223 if (!hash) 225 if (!hash)
224 return; 226 return;
225 227
226 if (likely(ctx->flags & FLAGS_SHA1)) { 228 if (likely(ctx->flags & BIT(FLAGS_SHA1))) {
227 /* SHA1 results are in big endian */ 229 /* SHA1 results are in big endian */
228 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) 230 for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++)
229 hash[i] = be32_to_cpu(in[i]); 231 hash[i] = be32_to_cpu(in[i]);
@@ -238,7 +240,7 @@ static int omap_sham_hw_init(struct omap_sham_dev *dd)
238{ 240{
239 clk_enable(dd->iclk); 241 clk_enable(dd->iclk);
240 242
241 if (!(dd->flags & FLAGS_INIT)) { 243 if (!test_bit(FLAGS_INIT, &dd->flags)) {
242 omap_sham_write_mask(dd, SHA_REG_MASK, 244 omap_sham_write_mask(dd, SHA_REG_MASK,
243 SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET); 245 SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET);
244 246
@@ -246,7 +248,7 @@ static int omap_sham_hw_init(struct omap_sham_dev *dd)
246 SHA_REG_SYSSTATUS_RESETDONE)) 248 SHA_REG_SYSSTATUS_RESETDONE))
247 return -ETIMEDOUT; 249 return -ETIMEDOUT;
248 250
249 dd->flags |= FLAGS_INIT; 251 set_bit(FLAGS_INIT, &dd->flags);
250 dd->err = 0; 252 dd->err = 0;
251 } 253 }
252 254
@@ -269,7 +271,7 @@ static void omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length,
269 * Setting ALGO_CONST only for the first iteration 271 * Setting ALGO_CONST only for the first iteration
270 * and CLOSE_HASH only for the last one. 272 * and CLOSE_HASH only for the last one.
271 */ 273 */
272 if (ctx->flags & FLAGS_SHA1) 274 if (ctx->flags & BIT(FLAGS_SHA1))
273 val |= SHA_REG_CTRL_ALGO; 275 val |= SHA_REG_CTRL_ALGO;
274 if (!ctx->digcnt) 276 if (!ctx->digcnt)
275 val |= SHA_REG_CTRL_ALGO_CONST; 277 val |= SHA_REG_CTRL_ALGO_CONST;
@@ -301,7 +303,9 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
301 return -ETIMEDOUT; 303 return -ETIMEDOUT;
302 304
303 if (final) 305 if (final)
304 ctx->flags |= FLAGS_FINAL; /* catch last interrupt */ 306 set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
307
308 set_bit(FLAGS_CPU, &dd->flags);
305 309
306 len32 = DIV_ROUND_UP(length, sizeof(u32)); 310 len32 = DIV_ROUND_UP(length, sizeof(u32));
307 311
@@ -334,9 +338,9 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
334 ctx->digcnt += length; 338 ctx->digcnt += length;
335 339
336 if (final) 340 if (final)
337 ctx->flags |= FLAGS_FINAL; /* catch last interrupt */ 341 set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
338 342
339 dd->flags |= FLAGS_DMA_ACTIVE; 343 set_bit(FLAGS_DMA_ACTIVE, &dd->flags);
340 344
341 omap_start_dma(dd->dma_lch); 345 omap_start_dma(dd->dma_lch);
342 346
@@ -392,7 +396,7 @@ static int omap_sham_xmit_dma_map(struct omap_sham_dev *dd,
392 return -EINVAL; 396 return -EINVAL;
393 } 397 }
394 398
395 ctx->flags &= ~FLAGS_SG; 399 ctx->flags &= ~BIT(FLAGS_SG);
396 400
397 /* next call does not fail... so no unmap in the case of error */ 401 /* next call does not fail... so no unmap in the case of error */
398 return omap_sham_xmit_dma(dd, ctx->dma_addr, length, final); 402 return omap_sham_xmit_dma(dd, ctx->dma_addr, length, final);
@@ -406,7 +410,7 @@ static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
406 410
407 omap_sham_append_sg(ctx); 411 omap_sham_append_sg(ctx);
408 412
409 final = (ctx->flags & FLAGS_FINUP) && !ctx->total; 413 final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
410 414
411 dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: %d, final: %d\n", 415 dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: %d, final: %d\n",
412 ctx->bufcnt, ctx->digcnt, final); 416 ctx->bufcnt, ctx->digcnt, final);
@@ -452,7 +456,7 @@ static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
452 length = min(ctx->total, sg->length); 456 length = min(ctx->total, sg->length);
453 457
454 if (sg_is_last(sg)) { 458 if (sg_is_last(sg)) {
455 if (!(ctx->flags & FLAGS_FINUP)) { 459 if (!(ctx->flags & BIT(FLAGS_FINUP))) {
456 /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */ 460 /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */
457 tail = length & (SHA1_MD5_BLOCK_SIZE - 1); 461 tail = length & (SHA1_MD5_BLOCK_SIZE - 1);
458 /* without finup() we need one block to close hash */ 462 /* without finup() we need one block to close hash */
@@ -467,12 +471,12 @@ static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
467 return -EINVAL; 471 return -EINVAL;
468 } 472 }
469 473
470 ctx->flags |= FLAGS_SG; 474 ctx->flags |= BIT(FLAGS_SG);
471 475
472 ctx->total -= length; 476 ctx->total -= length;
473 ctx->offset = length; /* offset where to start slow */ 477 ctx->offset = length; /* offset where to start slow */
474 478
475 final = (ctx->flags & FLAGS_FINUP) && !ctx->total; 479 final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
476 480
477 /* next call does not fail... so no unmap in the case of error */ 481 /* next call does not fail... so no unmap in the case of error */
478 return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final); 482 return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final);
@@ -495,7 +499,7 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
495 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req); 499 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
496 500
497 omap_stop_dma(dd->dma_lch); 501 omap_stop_dma(dd->dma_lch);
498 if (ctx->flags & FLAGS_SG) { 502 if (ctx->flags & BIT(FLAGS_SG)) {
499 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE); 503 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
500 if (ctx->sg->length == ctx->offset) { 504 if (ctx->sg->length == ctx->offset) {
501 ctx->sg = sg_next(ctx->sg); 505 ctx->sg = sg_next(ctx->sg);
@@ -537,18 +541,18 @@ static int omap_sham_init(struct ahash_request *req)
537 crypto_ahash_digestsize(tfm)); 541 crypto_ahash_digestsize(tfm));
538 542
539 if (crypto_ahash_digestsize(tfm) == SHA1_DIGEST_SIZE) 543 if (crypto_ahash_digestsize(tfm) == SHA1_DIGEST_SIZE)
540 ctx->flags |= FLAGS_SHA1; 544 ctx->flags |= BIT(FLAGS_SHA1);
541 545
542 ctx->bufcnt = 0; 546 ctx->bufcnt = 0;
543 ctx->digcnt = 0; 547 ctx->digcnt = 0;
544 ctx->buflen = BUFLEN; 548 ctx->buflen = BUFLEN;
545 549
546 if (tctx->flags & FLAGS_HMAC) { 550 if (tctx->flags & BIT(FLAGS_HMAC)) {
547 struct omap_sham_hmac_ctx *bctx = tctx->base; 551 struct omap_sham_hmac_ctx *bctx = tctx->base;
548 552
549 memcpy(ctx->buffer, bctx->ipad, SHA1_MD5_BLOCK_SIZE); 553 memcpy(ctx->buffer, bctx->ipad, SHA1_MD5_BLOCK_SIZE);
550 ctx->bufcnt = SHA1_MD5_BLOCK_SIZE; 554 ctx->bufcnt = SHA1_MD5_BLOCK_SIZE;
551 ctx->flags |= FLAGS_HMAC; 555 ctx->flags |= BIT(FLAGS_HMAC);
552 } 556 }
553 557
554 return 0; 558 return 0;
@@ -562,9 +566,9 @@ static int omap_sham_update_req(struct omap_sham_dev *dd)
562 int err; 566 int err;
563 567
564 dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n", 568 dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n",
565 ctx->total, ctx->digcnt, (ctx->flags & FLAGS_FINUP) != 0); 569 ctx->total, ctx->digcnt, (ctx->flags & BIT(FLAGS_FINUP)) != 0);
566 570
567 if (ctx->flags & FLAGS_CPU) 571 if (ctx->flags & BIT(FLAGS_CPU))
568 err = omap_sham_update_cpu(dd); 572 err = omap_sham_update_cpu(dd);
569 else 573 else
570 err = omap_sham_update_dma_start(dd); 574 err = omap_sham_update_dma_start(dd);
@@ -624,7 +628,7 @@ static int omap_sham_finish(struct ahash_request *req)
624 628
625 if (ctx->digcnt) { 629 if (ctx->digcnt) {
626 omap_sham_copy_ready_hash(req); 630 omap_sham_copy_ready_hash(req);
627 if (ctx->flags & FLAGS_HMAC) 631 if (ctx->flags & BIT(FLAGS_HMAC))
628 err = omap_sham_finish_hmac(req); 632 err = omap_sham_finish_hmac(req);
629 } 633 }
630 634
@@ -639,18 +643,23 @@ static void omap_sham_finish_req(struct ahash_request *req, int err)
639 struct omap_sham_dev *dd = ctx->dd; 643 struct omap_sham_dev *dd = ctx->dd;
640 644
641 if (!err) { 645 if (!err) {
642 omap_sham_copy_hash(ctx->dd->req, 1); 646 omap_sham_copy_hash(req, 1);
643 if (ctx->flags & FLAGS_FINAL) 647 if (test_bit(FLAGS_FINAL, &dd->flags))
644 err = omap_sham_finish(req); 648 err = omap_sham_finish(req);
645 } else { 649 } else {
646 ctx->flags |= FLAGS_ERROR; 650 ctx->flags |= BIT(FLAGS_ERROR);
647 } 651 }
648 652
653 /* atomic operation is not needed here */
654 dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) |
655 BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY));
649 clk_disable(dd->iclk); 656 clk_disable(dd->iclk);
650 dd->flags &= ~FLAGS_BUSY;
651 657
652 if (req->base.complete) 658 if (req->base.complete)
653 req->base.complete(&req->base, err); 659 req->base.complete(&req->base, err);
660
661 /* handle new request */
662 tasklet_schedule(&dd->done_task);
654} 663}
655 664
656static int omap_sham_handle_queue(struct omap_sham_dev *dd, 665static int omap_sham_handle_queue(struct omap_sham_dev *dd,
@@ -658,21 +667,20 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd,
658{ 667{
659 struct crypto_async_request *async_req, *backlog; 668 struct crypto_async_request *async_req, *backlog;
660 struct omap_sham_reqctx *ctx; 669 struct omap_sham_reqctx *ctx;
661 struct ahash_request *prev_req;
662 unsigned long flags; 670 unsigned long flags;
663 int err = 0, ret = 0; 671 int err = 0, ret = 0;
664 672
665 spin_lock_irqsave(&dd->lock, flags); 673 spin_lock_irqsave(&dd->lock, flags);
666 if (req) 674 if (req)
667 ret = ahash_enqueue_request(&dd->queue, req); 675 ret = ahash_enqueue_request(&dd->queue, req);
668 if (dd->flags & FLAGS_BUSY) { 676 if (test_bit(FLAGS_BUSY, &dd->flags)) {
669 spin_unlock_irqrestore(&dd->lock, flags); 677 spin_unlock_irqrestore(&dd->lock, flags);
670 return ret; 678 return ret;
671 } 679 }
672 backlog = crypto_get_backlog(&dd->queue); 680 backlog = crypto_get_backlog(&dd->queue);
673 async_req = crypto_dequeue_request(&dd->queue); 681 async_req = crypto_dequeue_request(&dd->queue);
674 if (async_req) 682 if (async_req)
675 dd->flags |= FLAGS_BUSY; 683 set_bit(FLAGS_BUSY, &dd->flags);
676 spin_unlock_irqrestore(&dd->lock, flags); 684 spin_unlock_irqrestore(&dd->lock, flags);
677 685
678 if (!async_req) 686 if (!async_req)
@@ -682,16 +690,12 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd,
682 backlog->complete(backlog, -EINPROGRESS); 690 backlog->complete(backlog, -EINPROGRESS);
683 691
684 req = ahash_request_cast(async_req); 692 req = ahash_request_cast(async_req);
685
686 prev_req = dd->req;
687 dd->req = req; 693 dd->req = req;
688
689 ctx = ahash_request_ctx(req); 694 ctx = ahash_request_ctx(req);
690 695
691 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n", 696 dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
692 ctx->op, req->nbytes); 697 ctx->op, req->nbytes);
693 698
694
695 err = omap_sham_hw_init(dd); 699 err = omap_sham_hw_init(dd);
696 if (err) 700 if (err)
697 goto err1; 701 goto err1;
@@ -712,18 +716,16 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd,
712 716
713 if (ctx->op == OP_UPDATE) { 717 if (ctx->op == OP_UPDATE) {
714 err = omap_sham_update_req(dd); 718 err = omap_sham_update_req(dd);
715 if (err != -EINPROGRESS && (ctx->flags & FLAGS_FINUP)) 719 if (err != -EINPROGRESS && (ctx->flags & BIT(FLAGS_FINUP)))
716 /* no final() after finup() */ 720 /* no final() after finup() */
717 err = omap_sham_final_req(dd); 721 err = omap_sham_final_req(dd);
718 } else if (ctx->op == OP_FINAL) { 722 } else if (ctx->op == OP_FINAL) {
719 err = omap_sham_final_req(dd); 723 err = omap_sham_final_req(dd);
720 } 724 }
721err1: 725err1:
722 if (err != -EINPROGRESS) { 726 if (err != -EINPROGRESS)
723 /* done_task will not finish it, so do it here */ 727 /* done_task will not finish it, so do it here */
724 omap_sham_finish_req(req, err); 728 omap_sham_finish_req(req, err);
725 tasklet_schedule(&dd->queue_task);
726 }
727 729
728 dev_dbg(dd->dev, "exit, err: %d\n", err); 730 dev_dbg(dd->dev, "exit, err: %d\n", err);
729 731
@@ -752,7 +754,7 @@ static int omap_sham_update(struct ahash_request *req)
752 ctx->sg = req->src; 754 ctx->sg = req->src;
753 ctx->offset = 0; 755 ctx->offset = 0;
754 756
755 if (ctx->flags & FLAGS_FINUP) { 757 if (ctx->flags & BIT(FLAGS_FINUP)) {
756 if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 9) { 758 if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 9) {
757 /* 759 /*
758 * OMAP HW accel works only with buffers >= 9 760 * OMAP HW accel works only with buffers >= 9
@@ -765,7 +767,7 @@ static int omap_sham_update(struct ahash_request *req)
765 /* 767 /*
766 * faster to use CPU for short transfers 768 * faster to use CPU for short transfers
767 */ 769 */
768 ctx->flags |= FLAGS_CPU; 770 ctx->flags |= BIT(FLAGS_CPU);
769 } 771 }
770 } else if (ctx->bufcnt + ctx->total < ctx->buflen) { 772 } else if (ctx->bufcnt + ctx->total < ctx->buflen) {
771 omap_sham_append_sg(ctx); 773 omap_sham_append_sg(ctx);
@@ -802,9 +804,9 @@ static int omap_sham_final(struct ahash_request *req)
802{ 804{
803 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 805 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
804 806
805 ctx->flags |= FLAGS_FINUP; 807 ctx->flags |= BIT(FLAGS_FINUP);
806 808
807 if (ctx->flags & FLAGS_ERROR) 809 if (ctx->flags & BIT(FLAGS_ERROR))
808 return 0; /* uncompleted hash is not needed */ 810 return 0; /* uncompleted hash is not needed */
809 811
810 /* OMAP HW accel works only with buffers >= 9 */ 812 /* OMAP HW accel works only with buffers >= 9 */
@@ -823,7 +825,7 @@ static int omap_sham_finup(struct ahash_request *req)
823 struct omap_sham_reqctx *ctx = ahash_request_ctx(req); 825 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
824 int err1, err2; 826 int err1, err2;
825 827
826 ctx->flags |= FLAGS_FINUP; 828 ctx->flags |= BIT(FLAGS_FINUP);
827 829
828 err1 = omap_sham_update(req); 830 err1 = omap_sham_update(req);
829 if (err1 == -EINPROGRESS || err1 == -EBUSY) 831 if (err1 == -EINPROGRESS || err1 == -EBUSY)
@@ -895,7 +897,7 @@ static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
895 897
896 if (alg_base) { 898 if (alg_base) {
897 struct omap_sham_hmac_ctx *bctx = tctx->base; 899 struct omap_sham_hmac_ctx *bctx = tctx->base;
898 tctx->flags |= FLAGS_HMAC; 900 tctx->flags |= BIT(FLAGS_HMAC);
899 bctx->shash = crypto_alloc_shash(alg_base, 0, 901 bctx->shash = crypto_alloc_shash(alg_base, 0,
900 CRYPTO_ALG_NEED_FALLBACK); 902 CRYPTO_ALG_NEED_FALLBACK);
901 if (IS_ERR(bctx->shash)) { 903 if (IS_ERR(bctx->shash)) {
@@ -932,7 +934,7 @@ static void omap_sham_cra_exit(struct crypto_tfm *tfm)
932 crypto_free_shash(tctx->fallback); 934 crypto_free_shash(tctx->fallback);
933 tctx->fallback = NULL; 935 tctx->fallback = NULL;
934 936
935 if (tctx->flags & FLAGS_HMAC) { 937 if (tctx->flags & BIT(FLAGS_HMAC)) {
936 struct omap_sham_hmac_ctx *bctx = tctx->base; 938 struct omap_sham_hmac_ctx *bctx = tctx->base;
937 crypto_free_shash(bctx->shash); 939 crypto_free_shash(bctx->shash);
938 } 940 }
@@ -1036,51 +1038,46 @@ static struct ahash_alg algs[] = {
1036static void omap_sham_done_task(unsigned long data) 1038static void omap_sham_done_task(unsigned long data)
1037{ 1039{
1038 struct omap_sham_dev *dd = (struct omap_sham_dev *)data; 1040 struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
1039 struct ahash_request *req = dd->req; 1041 int err = 0;
1040 struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
1041 int ready = 0, err = 0;
1042 1042
1043 if (ctx->flags & FLAGS_OUTPUT_READY) { 1043 if (!test_bit(FLAGS_BUSY, &dd->flags)) {
1044 ctx->flags &= ~FLAGS_OUTPUT_READY; 1044 omap_sham_handle_queue(dd, NULL);
1045 ready = 1; 1045 return;
1046 } 1046 }
1047 1047
1048 if (dd->flags & FLAGS_DMA_ACTIVE) { 1048 if (test_bit(FLAGS_CPU, &dd->flags)) {
1049 dd->flags &= ~FLAGS_DMA_ACTIVE; 1049 if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags))
1050 omap_sham_update_dma_stop(dd); 1050 goto finish;
1051 if (!dd->err) 1051 } else if (test_bit(FLAGS_DMA_READY, &dd->flags)) {
1052 if (test_and_clear_bit(FLAGS_DMA_ACTIVE, &dd->flags)) {
1053 omap_sham_update_dma_stop(dd);
1054 if (dd->err) {
1055 err = dd->err;
1056 goto finish;
1057 }
1058 }
1059 if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) {
1060 /* hash or semi-hash ready */
1061 clear_bit(FLAGS_DMA_READY, &dd->flags);
1052 err = omap_sham_update_dma_start(dd); 1062 err = omap_sham_update_dma_start(dd);
1063 if (err != -EINPROGRESS)
1064 goto finish;
1065 }
1053 } 1066 }
1054 1067
1055 err = dd->err ? : err; 1068 return;
1056
1057 if (err != -EINPROGRESS && (ready || err)) {
1058 dev_dbg(dd->dev, "update done: err: %d\n", err);
1059 /* finish curent request */
1060 omap_sham_finish_req(req, err);
1061 /* start new request */
1062 omap_sham_handle_queue(dd, NULL);
1063 }
1064}
1065
1066static void omap_sham_queue_task(unsigned long data)
1067{
1068 struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
1069 1069
1070 omap_sham_handle_queue(dd, NULL); 1070finish:
1071 dev_dbg(dd->dev, "update done: err: %d\n", err);
1072 /* finish curent request */
1073 omap_sham_finish_req(dd->req, err);
1071} 1074}
1072 1075
1073static irqreturn_t omap_sham_irq(int irq, void *dev_id) 1076static irqreturn_t omap_sham_irq(int irq, void *dev_id)
1074{ 1077{
1075 struct omap_sham_dev *dd = dev_id; 1078 struct omap_sham_dev *dd = dev_id;
1076 struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
1077
1078 if (!ctx) {
1079 dev_err(dd->dev, "unknown interrupt.\n");
1080 return IRQ_HANDLED;
1081 }
1082 1079
1083 if (unlikely(ctx->flags & FLAGS_FINAL)) 1080 if (unlikely(test_bit(FLAGS_FINAL, &dd->flags)))
1084 /* final -> allow device to go to power-saving mode */ 1081 /* final -> allow device to go to power-saving mode */
1085 omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH); 1082 omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH);
1086 1083
@@ -1088,8 +1085,12 @@ static irqreturn_t omap_sham_irq(int irq, void *dev_id)
1088 SHA_REG_CTRL_OUTPUT_READY); 1085 SHA_REG_CTRL_OUTPUT_READY);
1089 omap_sham_read(dd, SHA_REG_CTRL); 1086 omap_sham_read(dd, SHA_REG_CTRL);
1090 1087
1091 ctx->flags |= FLAGS_OUTPUT_READY; 1088 if (!test_bit(FLAGS_BUSY, &dd->flags)) {
1092 dd->err = 0; 1089 dev_warn(dd->dev, "Interrupt when no active requests.\n");
1090 return IRQ_HANDLED;
1091 }
1092
1093 set_bit(FLAGS_OUTPUT_READY, &dd->flags);
1093 tasklet_schedule(&dd->done_task); 1094 tasklet_schedule(&dd->done_task);
1094 1095
1095 return IRQ_HANDLED; 1096 return IRQ_HANDLED;
@@ -1102,9 +1103,10 @@ static void omap_sham_dma_callback(int lch, u16 ch_status, void *data)
1102 if (ch_status != OMAP_DMA_BLOCK_IRQ) { 1103 if (ch_status != OMAP_DMA_BLOCK_IRQ) {
1103 pr_err("omap-sham DMA error status: 0x%hx\n", ch_status); 1104 pr_err("omap-sham DMA error status: 0x%hx\n", ch_status);
1104 dd->err = -EIO; 1105 dd->err = -EIO;
1105 dd->flags &= ~FLAGS_INIT; /* request to re-initialize */ 1106 clear_bit(FLAGS_INIT, &dd->flags);/* request to re-initialize */
1106 } 1107 }
1107 1108
1109 set_bit(FLAGS_DMA_READY, &dd->flags);
1108 tasklet_schedule(&dd->done_task); 1110 tasklet_schedule(&dd->done_task);
1109} 1111}
1110 1112
@@ -1151,7 +1153,6 @@ static int __devinit omap_sham_probe(struct platform_device *pdev)
1151 INIT_LIST_HEAD(&dd->list); 1153 INIT_LIST_HEAD(&dd->list);
1152 spin_lock_init(&dd->lock); 1154 spin_lock_init(&dd->lock);
1153 tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd); 1155 tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd);
1154 tasklet_init(&dd->queue_task, omap_sham_queue_task, (unsigned long)dd);
1155 crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH); 1156 crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH);
1156 1157
1157 dd->irq = -1; 1158 dd->irq = -1;
@@ -1260,7 +1261,6 @@ static int __devexit omap_sham_remove(struct platform_device *pdev)
1260 for (i = 0; i < ARRAY_SIZE(algs); i++) 1261 for (i = 0; i < ARRAY_SIZE(algs); i++)
1261 crypto_unregister_ahash(&algs[i]); 1262 crypto_unregister_ahash(&algs[i]);
1262 tasklet_kill(&dd->done_task); 1263 tasklet_kill(&dd->done_task);
1263 tasklet_kill(&dd->queue_task);
1264 iounmap(dd->io_base); 1264 iounmap(dd->io_base);
1265 clk_put(dd->iclk); 1265 clk_put(dd->iclk);
1266 omap_sham_dma_cleanup(dd); 1266 omap_sham_dma_cleanup(dd);
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index 854e2632f9a6..8a0bb417aa11 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -1,7 +1,7 @@
1/* 1/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver 2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 * 3 *
4 * Copyright (c) 2008-2010 Freescale Semiconductor, Inc. 4 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
5 * 5 *
6 * Scatterlist Crypto API glue code copied from files with the following: 6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au> 7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
@@ -282,6 +282,7 @@ static int init_device(struct device *dev)
282/** 282/**
283 * talitos_submit - submits a descriptor to the device for processing 283 * talitos_submit - submits a descriptor to the device for processing
284 * @dev: the SEC device to be used 284 * @dev: the SEC device to be used
285 * @ch: the SEC device channel to be used
285 * @desc: the descriptor to be processed by the device 286 * @desc: the descriptor to be processed by the device
286 * @callback: whom to call when processing is complete 287 * @callback: whom to call when processing is complete
287 * @context: a handle for use by caller (optional) 288 * @context: a handle for use by caller (optional)
@@ -290,7 +291,7 @@ static int init_device(struct device *dev)
290 * callback must check err and feedback in descriptor header 291 * callback must check err and feedback in descriptor header
291 * for device processing status. 292 * for device processing status.
292 */ 293 */
293static int talitos_submit(struct device *dev, struct talitos_desc *desc, 294static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
294 void (*callback)(struct device *dev, 295 void (*callback)(struct device *dev,
295 struct talitos_desc *desc, 296 struct talitos_desc *desc,
296 void *context, int error), 297 void *context, int error),
@@ -298,15 +299,9 @@ static int talitos_submit(struct device *dev, struct talitos_desc *desc,
298{ 299{
299 struct talitos_private *priv = dev_get_drvdata(dev); 300 struct talitos_private *priv = dev_get_drvdata(dev);
300 struct talitos_request *request; 301 struct talitos_request *request;
301 unsigned long flags, ch; 302 unsigned long flags;
302 int head; 303 int head;
303 304
304 /* select done notification */
305 desc->hdr |= DESC_HDR_DONE_NOTIFY;
306
307 /* emulate SEC's round-robin channel fifo polling scheme */
308 ch = atomic_inc_return(&priv->last_chan) & (priv->num_channels - 1);
309
310 spin_lock_irqsave(&priv->chan[ch].head_lock, flags); 305 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
311 306
312 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { 307 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
@@ -706,6 +701,7 @@ static void talitos_unregister_rng(struct device *dev)
706 701
707struct talitos_ctx { 702struct talitos_ctx {
708 struct device *dev; 703 struct device *dev;
704 int ch;
709 __be32 desc_hdr_template; 705 __be32 desc_hdr_template;
710 u8 key[TALITOS_MAX_KEY_SIZE]; 706 u8 key[TALITOS_MAX_KEY_SIZE];
711 u8 iv[TALITOS_MAX_IV_LENGTH]; 707 u8 iv[TALITOS_MAX_IV_LENGTH];
@@ -1117,7 +1113,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1117 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0, 1113 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0,
1118 DMA_FROM_DEVICE); 1114 DMA_FROM_DEVICE);
1119 1115
1120 ret = talitos_submit(dev, desc, callback, areq); 1116 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1121 if (ret != -EINPROGRESS) { 1117 if (ret != -EINPROGRESS) {
1122 ipsec_esp_unmap(dev, edesc, areq); 1118 ipsec_esp_unmap(dev, edesc, areq);
1123 kfree(edesc); 1119 kfree(edesc);
@@ -1382,22 +1378,11 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1382 const u8 *key, unsigned int keylen) 1378 const u8 *key, unsigned int keylen)
1383{ 1379{
1384 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); 1380 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1385 struct ablkcipher_alg *alg = crypto_ablkcipher_alg(cipher);
1386
1387 if (keylen > TALITOS_MAX_KEY_SIZE)
1388 goto badkey;
1389
1390 if (keylen < alg->min_keysize || keylen > alg->max_keysize)
1391 goto badkey;
1392 1381
1393 memcpy(&ctx->key, key, keylen); 1382 memcpy(&ctx->key, key, keylen);
1394 ctx->keylen = keylen; 1383 ctx->keylen = keylen;
1395 1384
1396 return 0; 1385 return 0;
1397
1398badkey:
1399 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1400 return -EINVAL;
1401} 1386}
1402 1387
1403static void common_nonsnoop_unmap(struct device *dev, 1388static void common_nonsnoop_unmap(struct device *dev,
@@ -1433,7 +1418,6 @@ static void ablkcipher_done(struct device *dev,
1433 1418
1434static int common_nonsnoop(struct talitos_edesc *edesc, 1419static int common_nonsnoop(struct talitos_edesc *edesc,
1435 struct ablkcipher_request *areq, 1420 struct ablkcipher_request *areq,
1436 u8 *giv,
1437 void (*callback) (struct device *dev, 1421 void (*callback) (struct device *dev,
1438 struct talitos_desc *desc, 1422 struct talitos_desc *desc,
1439 void *context, int error)) 1423 void *context, int error))
@@ -1453,7 +1437,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc,
1453 1437
1454 /* cipher iv */ 1438 /* cipher iv */
1455 ivsize = crypto_ablkcipher_ivsize(cipher); 1439 ivsize = crypto_ablkcipher_ivsize(cipher);
1456 map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, giv ?: areq->info, 0, 1440 map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, areq->info, 0,
1457 DMA_TO_DEVICE); 1441 DMA_TO_DEVICE);
1458 1442
1459 /* cipher key */ 1443 /* cipher key */
@@ -1524,7 +1508,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc,
1524 to_talitos_ptr(&desc->ptr[6], 0); 1508 to_talitos_ptr(&desc->ptr[6], 0);
1525 desc->ptr[6].j_extent = 0; 1509 desc->ptr[6].j_extent = 0;
1526 1510
1527 ret = talitos_submit(dev, desc, callback, areq); 1511 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1528 if (ret != -EINPROGRESS) { 1512 if (ret != -EINPROGRESS) {
1529 common_nonsnoop_unmap(dev, edesc, areq); 1513 common_nonsnoop_unmap(dev, edesc, areq);
1530 kfree(edesc); 1514 kfree(edesc);
@@ -1556,7 +1540,7 @@ static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1556 /* set encrypt */ 1540 /* set encrypt */
1557 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; 1541 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1558 1542
1559 return common_nonsnoop(edesc, areq, NULL, ablkcipher_done); 1543 return common_nonsnoop(edesc, areq, ablkcipher_done);
1560} 1544}
1561 1545
1562static int ablkcipher_decrypt(struct ablkcipher_request *areq) 1546static int ablkcipher_decrypt(struct ablkcipher_request *areq)
@@ -1572,7 +1556,7 @@ static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1572 1556
1573 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; 1557 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1574 1558
1575 return common_nonsnoop(edesc, areq, NULL, ablkcipher_done); 1559 return common_nonsnoop(edesc, areq, ablkcipher_done);
1576} 1560}
1577 1561
1578static void common_nonsnoop_hash_unmap(struct device *dev, 1562static void common_nonsnoop_hash_unmap(struct device *dev,
@@ -1703,7 +1687,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1703 /* last DWORD empty */ 1687 /* last DWORD empty */
1704 desc->ptr[6] = zero_entry; 1688 desc->ptr[6] = zero_entry;
1705 1689
1706 ret = talitos_submit(dev, desc, callback, areq); 1690 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1707 if (ret != -EINPROGRESS) { 1691 if (ret != -EINPROGRESS) {
1708 common_nonsnoop_hash_unmap(dev, edesc, areq); 1692 common_nonsnoop_hash_unmap(dev, edesc, areq);
1709 kfree(edesc); 1693 kfree(edesc);
@@ -2244,6 +2228,7 @@ static int talitos_cra_init(struct crypto_tfm *tfm)
2244 struct crypto_alg *alg = tfm->__crt_alg; 2228 struct crypto_alg *alg = tfm->__crt_alg;
2245 struct talitos_crypto_alg *talitos_alg; 2229 struct talitos_crypto_alg *talitos_alg;
2246 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); 2230 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2231 struct talitos_private *priv;
2247 2232
2248 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH) 2233 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2249 talitos_alg = container_of(__crypto_ahash_alg(alg), 2234 talitos_alg = container_of(__crypto_ahash_alg(alg),
@@ -2256,9 +2241,17 @@ static int talitos_cra_init(struct crypto_tfm *tfm)
2256 /* update context with ptr to dev */ 2241 /* update context with ptr to dev */
2257 ctx->dev = talitos_alg->dev; 2242 ctx->dev = talitos_alg->dev;
2258 2243
2244 /* assign SEC channel to tfm in round-robin fashion */
2245 priv = dev_get_drvdata(ctx->dev);
2246 ctx->ch = atomic_inc_return(&priv->last_chan) &
2247 (priv->num_channels - 1);
2248
2259 /* copy descriptor header template value */ 2249 /* copy descriptor header template value */
2260 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template; 2250 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2261 2251
2252 /* select done notification */
2253 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2254
2262 return 0; 2255 return 0;
2263} 2256}
2264 2257
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-14 01:11:42 -0400