aboutsummaryrefslogtreecommitdiffstats
path: root/crypto
diff options
context:
space:
mode:
authorHerbert Xu <herbert@gondor.apana.org.au>2009-12-01 02:16:22 -0500
committerHerbert Xu <herbert@gondor.apana.org.au>2009-12-01 02:16:22 -0500
commit838632438145ac6863377eb12d8b8eef9c55d288 (patch)
treefbb0757df837f3c75a99c518a3596c38daef162d /crypto
parent9996508b3353063f2d6c48c1a28a84543d72d70b (diff)
parent29e553631b2a0d4eebd23db630572e1027a9967a (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6
Diffstat (limited to 'crypto')
-rw-r--r--crypto/aead.c1
-rw-r--r--crypto/async_tx/Kconfig14
-rw-r--r--crypto/async_tx/Makefile3
-rw-r--r--crypto/async_tx/async_memcpy.c44
-rw-r--r--crypto/async_tx/async_memset.c43
-rw-r--r--crypto/async_tx/async_pq.c415
-rw-r--r--crypto/async_tx/async_raid6_recov.c500
-rw-r--r--crypto/async_tx/async_tx.c87
-rw-r--r--crypto/async_tx/async_xor.c238
-rw-r--r--crypto/async_tx/raid6test.c240
-rw-r--r--crypto/gcm.c107
11 files changed, 1445 insertions, 247 deletions
diff --git a/crypto/aead.c b/crypto/aead.c
index d9aa733db164..0a55da70845e 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -18,6 +18,7 @@
18#include <linux/kernel.h> 18#include <linux/kernel.h>
19#include <linux/module.h> 19#include <linux/module.h>
20#include <linux/rtnetlink.h> 20#include <linux/rtnetlink.h>
21#include <linux/sched.h>
21#include <linux/slab.h> 22#include <linux/slab.h>
22#include <linux/seq_file.h> 23#include <linux/seq_file.h>
23 24
diff --git a/crypto/async_tx/Kconfig b/crypto/async_tx/Kconfig
index d8fb39145986..e28e276ac611 100644
--- a/crypto/async_tx/Kconfig
+++ b/crypto/async_tx/Kconfig
@@ -14,3 +14,17 @@ config ASYNC_MEMSET
14 tristate 14 tristate
15 select ASYNC_CORE 15 select ASYNC_CORE
16 16
17config ASYNC_PQ
18 tristate
19 select ASYNC_CORE
20
21config ASYNC_RAID6_RECOV
22 tristate
23 select ASYNC_CORE
24 select ASYNC_PQ
25
26config ASYNC_TX_DISABLE_PQ_VAL_DMA
27 bool
28
29config ASYNC_TX_DISABLE_XOR_VAL_DMA
30 bool
diff --git a/crypto/async_tx/Makefile b/crypto/async_tx/Makefile
index 27baa7d52fbc..d1e0e6f72bc1 100644
--- a/crypto/async_tx/Makefile
+++ b/crypto/async_tx/Makefile
@@ -2,3 +2,6 @@ obj-$(CONFIG_ASYNC_CORE) += async_tx.o
2obj-$(CONFIG_ASYNC_MEMCPY) += async_memcpy.o 2obj-$(CONFIG_ASYNC_MEMCPY) += async_memcpy.o
3obj-$(CONFIG_ASYNC_MEMSET) += async_memset.o 3obj-$(CONFIG_ASYNC_MEMSET) += async_memset.o
4obj-$(CONFIG_ASYNC_XOR) += async_xor.o 4obj-$(CONFIG_ASYNC_XOR) += async_xor.o
5obj-$(CONFIG_ASYNC_PQ) += async_pq.o
6obj-$(CONFIG_ASYNC_RAID6_RECOV) += async_raid6_recov.o
7obj-$(CONFIG_ASYNC_RAID6_TEST) += raid6test.o
diff --git a/crypto/async_tx/async_memcpy.c b/crypto/async_tx/async_memcpy.c
index ddccfb01c416..0ec1fb69d4ea 100644
--- a/crypto/async_tx/async_memcpy.c
+++ b/crypto/async_tx/async_memcpy.c
@@ -33,28 +33,31 @@
33 * async_memcpy - attempt to copy memory with a dma engine. 33 * async_memcpy - attempt to copy memory with a dma engine.
34 * @dest: destination page 34 * @dest: destination page
35 * @src: src page 35 * @src: src page
36 * @offset: offset in pages to start transaction 36 * @dest_offset: offset into 'dest' to start transaction
37 * @src_offset: offset into 'src' to start transaction
37 * @len: length in bytes 38 * @len: length in bytes
38 * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK, 39 * @submit: submission / completion modifiers
39 * @depend_tx: memcpy depends on the result of this transaction 40 *
40 * @cb_fn: function to call when the memcpy completes 41 * honored flags: ASYNC_TX_ACK
41 * @cb_param: parameter to pass to the callback routine
42 */ 42 */
43struct dma_async_tx_descriptor * 43struct dma_async_tx_descriptor *
44async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, 44async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
45 unsigned int src_offset, size_t len, enum async_tx_flags flags, 45 unsigned int src_offset, size_t len,
46 struct dma_async_tx_descriptor *depend_tx, 46 struct async_submit_ctl *submit)
47 dma_async_tx_callback cb_fn, void *cb_param)
48{ 47{
49 struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_MEMCPY, 48 struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY,
50 &dest, 1, &src, 1, len); 49 &dest, 1, &src, 1, len);
51 struct dma_device *device = chan ? chan->device : NULL; 50 struct dma_device *device = chan ? chan->device : NULL;
52 struct dma_async_tx_descriptor *tx = NULL; 51 struct dma_async_tx_descriptor *tx = NULL;
53 52
54 if (device) { 53 if (device && is_dma_copy_aligned(device, src_offset, dest_offset, len)) {
55 dma_addr_t dma_dest, dma_src; 54 dma_addr_t dma_dest, dma_src;
56 unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; 55 unsigned long dma_prep_flags = 0;
57 56
57 if (submit->cb_fn)
58 dma_prep_flags |= DMA_PREP_INTERRUPT;
59 if (submit->flags & ASYNC_TX_FENCE)
60 dma_prep_flags |= DMA_PREP_FENCE;
58 dma_dest = dma_map_page(device->dev, dest, dest_offset, len, 61 dma_dest = dma_map_page(device->dev, dest, dest_offset, len,
59 DMA_FROM_DEVICE); 62 DMA_FROM_DEVICE);
60 63
@@ -67,13 +70,13 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
67 70
68 if (tx) { 71 if (tx) {
69 pr_debug("%s: (async) len: %zu\n", __func__, len); 72 pr_debug("%s: (async) len: %zu\n", __func__, len);
70 async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); 73 async_tx_submit(chan, tx, submit);
71 } else { 74 } else {
72 void *dest_buf, *src_buf; 75 void *dest_buf, *src_buf;
73 pr_debug("%s: (sync) len: %zu\n", __func__, len); 76 pr_debug("%s: (sync) len: %zu\n", __func__, len);
74 77
75 /* wait for any prerequisite operations */ 78 /* wait for any prerequisite operations */
76 async_tx_quiesce(&depend_tx); 79 async_tx_quiesce(&submit->depend_tx);
77 80
78 dest_buf = kmap_atomic(dest, KM_USER0) + dest_offset; 81 dest_buf = kmap_atomic(dest, KM_USER0) + dest_offset;
79 src_buf = kmap_atomic(src, KM_USER1) + src_offset; 82 src_buf = kmap_atomic(src, KM_USER1) + src_offset;
@@ -83,26 +86,13 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
83 kunmap_atomic(dest_buf, KM_USER0); 86 kunmap_atomic(dest_buf, KM_USER0);
84 kunmap_atomic(src_buf, KM_USER1); 87 kunmap_atomic(src_buf, KM_USER1);
85 88
86 async_tx_sync_epilog(cb_fn, cb_param); 89 async_tx_sync_epilog(submit);
87 } 90 }
88 91
89 return tx; 92 return tx;
90} 93}
91EXPORT_SYMBOL_GPL(async_memcpy); 94EXPORT_SYMBOL_GPL(async_memcpy);
92 95
93static int __init async_memcpy_init(void)
94{
95 return 0;
96}
97
98static void __exit async_memcpy_exit(void)
99{
100 do { } while (0);
101}
102
103module_init(async_memcpy_init);
104module_exit(async_memcpy_exit);
105
106MODULE_AUTHOR("Intel Corporation"); 96MODULE_AUTHOR("Intel Corporation");
107MODULE_DESCRIPTION("asynchronous memcpy api"); 97MODULE_DESCRIPTION("asynchronous memcpy api");
108MODULE_LICENSE("GPL"); 98MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_memset.c b/crypto/async_tx/async_memset.c
index 5b5eb99bb244..58e4a8752aee 100644
--- a/crypto/async_tx/async_memset.c
+++ b/crypto/async_tx/async_memset.c
@@ -35,26 +35,26 @@
35 * @val: fill value 35 * @val: fill value
36 * @offset: offset in pages to start transaction 36 * @offset: offset in pages to start transaction
37 * @len: length in bytes 37 * @len: length in bytes
38 * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK 38 *
39 * @depend_tx: memset depends on the result of this transaction 39 * honored flags: ASYNC_TX_ACK
40 * @cb_fn: function to call when the memcpy completes
41 * @cb_param: parameter to pass to the callback routine
42 */ 40 */
43struct dma_async_tx_descriptor * 41struct dma_async_tx_descriptor *
44async_memset(struct page *dest, int val, unsigned int offset, 42async_memset(struct page *dest, int val, unsigned int offset, size_t len,
45 size_t len, enum async_tx_flags flags, 43 struct async_submit_ctl *submit)
46 struct dma_async_tx_descriptor *depend_tx,
47 dma_async_tx_callback cb_fn, void *cb_param)
48{ 44{
49 struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_MEMSET, 45 struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMSET,
50 &dest, 1, NULL, 0, len); 46 &dest, 1, NULL, 0, len);
51 struct dma_device *device = chan ? chan->device : NULL; 47 struct dma_device *device = chan ? chan->device : NULL;
52 struct dma_async_tx_descriptor *tx = NULL; 48 struct dma_async_tx_descriptor *tx = NULL;
53 49
54 if (device) { 50 if (device && is_dma_fill_aligned(device, offset, 0, len)) {
55 dma_addr_t dma_dest; 51 dma_addr_t dma_dest;
56 unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; 52 unsigned long dma_prep_flags = 0;
57 53
54 if (submit->cb_fn)
55 dma_prep_flags |= DMA_PREP_INTERRUPT;
56 if (submit->flags & ASYNC_TX_FENCE)
57 dma_prep_flags |= DMA_PREP_FENCE;
58 dma_dest = dma_map_page(device->dev, dest, offset, len, 58 dma_dest = dma_map_page(device->dev, dest, offset, len,
59 DMA_FROM_DEVICE); 59 DMA_FROM_DEVICE);
60 60
@@ -64,38 +64,25 @@ async_memset(struct page *dest, int val, unsigned int offset,
64 64
65 if (tx) { 65 if (tx) {
66 pr_debug("%s: (async) len: %zu\n", __func__, len); 66 pr_debug("%s: (async) len: %zu\n", __func__, len);
67 async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); 67 async_tx_submit(chan, tx, submit);
68 } else { /* run the memset synchronously */ 68 } else { /* run the memset synchronously */
69 void *dest_buf; 69 void *dest_buf;
70 pr_debug("%s: (sync) len: %zu\n", __func__, len); 70 pr_debug("%s: (sync) len: %zu\n", __func__, len);
71 71
72 dest_buf = (void *) (((char *) page_address(dest)) + offset); 72 dest_buf = page_address(dest) + offset;
73 73
74 /* wait for any prerequisite operations */ 74 /* wait for any prerequisite operations */
75 async_tx_quiesce(&depend_tx); 75 async_tx_quiesce(&submit->depend_tx);
76 76
77 memset(dest_buf, val, len); 77 memset(dest_buf, val, len);
78 78
79 async_tx_sync_epilog(cb_fn, cb_param); 79 async_tx_sync_epilog(submit);
80 } 80 }
81 81
82 return tx; 82 return tx;
83} 83}
84EXPORT_SYMBOL_GPL(async_memset); 84EXPORT_SYMBOL_GPL(async_memset);
85 85
86static int __init async_memset_init(void)
87{
88 return 0;
89}
90
91static void __exit async_memset_exit(void)
92{
93 do { } while (0);
94}
95
96module_init(async_memset_init);
97module_exit(async_memset_exit);
98
99MODULE_AUTHOR("Intel Corporation"); 86MODULE_AUTHOR("Intel Corporation");
100MODULE_DESCRIPTION("asynchronous memset api"); 87MODULE_DESCRIPTION("asynchronous memset api");
101MODULE_LICENSE("GPL"); 88MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c
new file mode 100644
index 000000000000..ec87f53d5059
--- /dev/null
+++ b/crypto/async_tx/async_pq.c
@@ -0,0 +1,415 @@
1/*
2 * Copyright(c) 2007 Yuri Tikhonov <yur@emcraft.com>
3 * Copyright(c) 2009 Intel Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the Free
7 * Software Foundation; either version 2 of the License, or (at your option)
8 * any later version.
9 *
10 * This program is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * this program; if not, write to the Free Software Foundation, Inc., 59
17 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * The full GNU General Public License is included in this distribution in the
20 * file called COPYING.
21 */
22#include <linux/kernel.h>
23#include <linux/interrupt.h>
24#include <linux/dma-mapping.h>
25#include <linux/raid/pq.h>
26#include <linux/async_tx.h>
27
28/**
29 * pq_scribble_page - space to hold throwaway P or Q buffer for
30 * synchronous gen_syndrome
31 */
32static struct page *pq_scribble_page;
33
34/* the struct page *blocks[] parameter passed to async_gen_syndrome()
35 * and async_syndrome_val() contains the 'P' destination address at
36 * blocks[disks-2] and the 'Q' destination address at blocks[disks-1]
37 *
38 * note: these are macros as they are used as lvalues
39 */
40#define P(b, d) (b[d-2])
41#define Q(b, d) (b[d-1])
42
43/**
44 * do_async_gen_syndrome - asynchronously calculate P and/or Q
45 */
46static __async_inline struct dma_async_tx_descriptor *
47do_async_gen_syndrome(struct dma_chan *chan, struct page **blocks,
48 const unsigned char *scfs, unsigned int offset, int disks,
49 size_t len, dma_addr_t *dma_src,
50 struct async_submit_ctl *submit)
51{
52 struct dma_async_tx_descriptor *tx = NULL;
53 struct dma_device *dma = chan->device;
54 enum dma_ctrl_flags dma_flags = 0;
55 enum async_tx_flags flags_orig = submit->flags;
56 dma_async_tx_callback cb_fn_orig = submit->cb_fn;
57 dma_async_tx_callback cb_param_orig = submit->cb_param;
58 int src_cnt = disks - 2;
59 unsigned char coefs[src_cnt];
60 unsigned short pq_src_cnt;
61 dma_addr_t dma_dest[2];
62 int src_off = 0;
63 int idx;
64 int i;
65
66 /* DMAs use destinations as sources, so use BIDIRECTIONAL mapping */
67 if (P(blocks, disks))
68 dma_dest[0] = dma_map_page(dma->dev, P(blocks, disks), offset,
69 len, DMA_BIDIRECTIONAL);
70 else
71 dma_flags |= DMA_PREP_PQ_DISABLE_P;
72 if (Q(blocks, disks))
73 dma_dest[1] = dma_map_page(dma->dev, Q(blocks, disks), offset,
74 len, DMA_BIDIRECTIONAL);
75 else
76 dma_flags |= DMA_PREP_PQ_DISABLE_Q;
77
78 /* convert source addresses being careful to collapse 'empty'
79 * sources and update the coefficients accordingly
80 */
81 for (i = 0, idx = 0; i < src_cnt; i++) {
82 if (blocks[i] == NULL)
83 continue;
84 dma_src[idx] = dma_map_page(dma->dev, blocks[i], offset, len,
85 DMA_TO_DEVICE);
86 coefs[idx] = scfs[i];
87 idx++;
88 }
89 src_cnt = idx;
90
91 while (src_cnt > 0) {
92 submit->flags = flags_orig;
93 pq_src_cnt = min(src_cnt, dma_maxpq(dma, dma_flags));
94 /* if we are submitting additional pqs, leave the chain open,
95 * clear the callback parameters, and leave the destination
96 * buffers mapped
97 */
98 if (src_cnt > pq_src_cnt) {
99 submit->flags &= ~ASYNC_TX_ACK;
100 submit->flags |= ASYNC_TX_FENCE;
101 dma_flags |= DMA_COMPL_SKIP_DEST_UNMAP;
102 submit->cb_fn = NULL;
103 submit->cb_param = NULL;
104 } else {
105 dma_flags &= ~DMA_COMPL_SKIP_DEST_UNMAP;
106 submit->cb_fn = cb_fn_orig;
107 submit->cb_param = cb_param_orig;
108 if (cb_fn_orig)
109 dma_flags |= DMA_PREP_INTERRUPT;
110 }
111 if (submit->flags & ASYNC_TX_FENCE)
112 dma_flags |= DMA_PREP_FENCE;
113
114 /* Since we have clobbered the src_list we are committed
115 * to doing this asynchronously. Drivers force forward
116 * progress in case they can not provide a descriptor
117 */
118 for (;;) {
119 tx = dma->device_prep_dma_pq(chan, dma_dest,
120 &dma_src[src_off],
121 pq_src_cnt,
122 &coefs[src_off], len,
123 dma_flags);
124 if (likely(tx))
125 break;
126 async_tx_quiesce(&submit->depend_tx);
127 dma_async_issue_pending(chan);
128 }
129
130 async_tx_submit(chan, tx, submit);
131 submit->depend_tx = tx;
132
133 /* drop completed sources */
134 src_cnt -= pq_src_cnt;
135 src_off += pq_src_cnt;
136
137 dma_flags |= DMA_PREP_CONTINUE;
138 }
139
140 return tx;
141}
142
143/**
144 * do_sync_gen_syndrome - synchronously calculate a raid6 syndrome
145 */
146static void
147do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
148 size_t len, struct async_submit_ctl *submit)
149{
150 void **srcs;
151 int i;
152
153 if (submit->scribble)
154 srcs = submit->scribble;
155 else
156 srcs = (void **) blocks;
157
158 for (i = 0; i < disks; i++) {
159 if (blocks[i] == NULL) {
160 BUG_ON(i > disks - 3); /* P or Q can't be zero */
161 srcs[i] = (void*)raid6_empty_zero_page;
162 } else
163 srcs[i] = page_address(blocks[i]) + offset;
164 }
165 raid6_call.gen_syndrome(disks, len, srcs);
166 async_tx_sync_epilog(submit);
167}
168
169/**
170 * async_gen_syndrome - asynchronously calculate a raid6 syndrome
171 * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1
172 * @offset: common offset into each block (src and dest) to start transaction
173 * @disks: number of blocks (including missing P or Q, see below)
174 * @len: length of operation in bytes
175 * @submit: submission/completion modifiers
176 *
177 * General note: This routine assumes a field of GF(2^8) with a
178 * primitive polynomial of 0x11d and a generator of {02}.
179 *
180 * 'disks' note: callers can optionally omit either P or Q (but not
181 * both) from the calculation by setting blocks[disks-2] or
182 * blocks[disks-1] to NULL. When P or Q is omitted 'len' must be <=
183 * PAGE_SIZE as a temporary buffer of this size is used in the
184 * synchronous path. 'disks' always accounts for both destination
185 * buffers. If any source buffers (blocks[i] where i < disks - 2) are
186 * set to NULL those buffers will be replaced with the raid6_zero_page
187 * in the synchronous path and omitted in the hardware-asynchronous
188 * path.
189 *
190 * 'blocks' note: if submit->scribble is NULL then the contents of
191 * 'blocks' may be overwritten to perform address conversions
192 * (dma_map_page() or page_address()).
193 */
194struct dma_async_tx_descriptor *
195async_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
196 size_t len, struct async_submit_ctl *submit)
197{
198 int src_cnt = disks - 2;
199 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
200 &P(blocks, disks), 2,
201 blocks, src_cnt, len);
202 struct dma_device *device = chan ? chan->device : NULL;
203 dma_addr_t *dma_src = NULL;
204
205 BUG_ON(disks > 255 || !(P(blocks, disks) || Q(blocks, disks)));
206
207 if (submit->scribble)
208 dma_src = submit->scribble;
209 else if (sizeof(dma_addr_t) <= sizeof(struct page *))
210 dma_src = (dma_addr_t *) blocks;
211
212 if (dma_src && device &&
213 (src_cnt <= dma_maxpq(device, 0) ||
214 dma_maxpq(device, DMA_PREP_CONTINUE) > 0) &&
215 is_dma_pq_aligned(device, offset, 0, len)) {
216 /* run the p+q asynchronously */
217 pr_debug("%s: (async) disks: %d len: %zu\n",
218 __func__, disks, len);
219 return do_async_gen_syndrome(chan, blocks, raid6_gfexp, offset,
220 disks, len, dma_src, submit);
221 }
222
223 /* run the pq synchronously */
224 pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len);
225
226 /* wait for any prerequisite operations */
227 async_tx_quiesce(&submit->depend_tx);
228
229 if (!P(blocks, disks)) {
230 P(blocks, disks) = pq_scribble_page;
231 BUG_ON(len + offset > PAGE_SIZE);
232 }
233 if (!Q(blocks, disks)) {
234 Q(blocks, disks) = pq_scribble_page;
235 BUG_ON(len + offset > PAGE_SIZE);
236 }
237 do_sync_gen_syndrome(blocks, offset, disks, len, submit);
238
239 return NULL;
240}
241EXPORT_SYMBOL_GPL(async_gen_syndrome);
242
243static inline struct dma_chan *
244pq_val_chan(struct async_submit_ctl *submit, struct page **blocks, int disks, size_t len)
245{
246 #ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
247 return NULL;
248 #endif
249 return async_tx_find_channel(submit, DMA_PQ_VAL, NULL, 0, blocks,
250 disks, len);
251}
252
253/**
254 * async_syndrome_val - asynchronously validate a raid6 syndrome
255 * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1
256 * @offset: common offset into each block (src and dest) to start transaction
257 * @disks: number of blocks (including missing P or Q, see below)
258 * @len: length of operation in bytes
259 * @pqres: on val failure SUM_CHECK_P_RESULT and/or SUM_CHECK_Q_RESULT are set
260 * @spare: temporary result buffer for the synchronous case
261 * @submit: submission / completion modifiers
262 *
263 * The same notes from async_gen_syndrome apply to the 'blocks',
264 * and 'disks' parameters of this routine. The synchronous path
265 * requires a temporary result buffer and submit->scribble to be
266 * specified.
267 */
268struct dma_async_tx_descriptor *
269async_syndrome_val(struct page **blocks, unsigned int offset, int disks,
270 size_t len, enum sum_check_flags *pqres, struct page *spare,
271 struct async_submit_ctl *submit)
272{
273 struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len);
274 struct dma_device *device = chan ? chan->device : NULL;
275 struct dma_async_tx_descriptor *tx;
276 unsigned char coefs[disks-2];
277 enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0;
278 dma_addr_t *dma_src = NULL;
279 int src_cnt = 0;
280
281 BUG_ON(disks < 4);
282
283 if (submit->scribble)
284 dma_src = submit->scribble;
285 else if (sizeof(dma_addr_t) <= sizeof(struct page *))
286 dma_src = (dma_addr_t *) blocks;
287
288 if (dma_src && device && disks <= dma_maxpq(device, 0) &&
289 is_dma_pq_aligned(device, offset, 0, len)) {
290 struct device *dev = device->dev;
291 dma_addr_t *pq = &dma_src[disks-2];
292 int i;
293
294 pr_debug("%s: (async) disks: %d len: %zu\n",
295 __func__, disks, len);
296 if (!P(blocks, disks))
297 dma_flags |= DMA_PREP_PQ_DISABLE_P;
298 else
299 pq[0] = dma_map_page(dev, P(blocks, disks),
300 offset, len,
301 DMA_TO_DEVICE);
302 if (!Q(blocks, disks))
303 dma_flags |= DMA_PREP_PQ_DISABLE_Q;
304 else
305 pq[1] = dma_map_page(dev, Q(blocks, disks),
306 offset, len,
307 DMA_TO_DEVICE);
308
309 if (submit->flags & ASYNC_TX_FENCE)
310 dma_flags |= DMA_PREP_FENCE;
311 for (i = 0; i < disks-2; i++)
312 if (likely(blocks[i])) {
313 dma_src[src_cnt] = dma_map_page(dev, blocks[i],
314 offset, len,
315 DMA_TO_DEVICE);
316 coefs[src_cnt] = raid6_gfexp[i];
317 src_cnt++;
318 }
319
320 for (;;) {
321 tx = device->device_prep_dma_pq_val(chan, pq, dma_src,
322 src_cnt,
323 coefs,
324 len, pqres,
325 dma_flags);
326 if (likely(tx))
327 break;
328 async_tx_quiesce(&submit->depend_tx);
329 dma_async_issue_pending(chan);
330 }
331 async_tx_submit(chan, tx, submit);
332
333 return tx;
334 } else {
335 struct page *p_src = P(blocks, disks);
336 struct page *q_src = Q(blocks, disks);
337 enum async_tx_flags flags_orig = submit->flags;
338 dma_async_tx_callback cb_fn_orig = submit->cb_fn;
339 void *scribble = submit->scribble;
340 void *cb_param_orig = submit->cb_param;
341 void *p, *q, *s;
342
343 pr_debug("%s: (sync) disks: %d len: %zu\n",
344 __func__, disks, len);
345
346 /* caller must provide a temporary result buffer and
347 * allow the input parameters to be preserved
348 */
349 BUG_ON(!spare || !scribble);
350
351 /* wait for any prerequisite operations */
352 async_tx_quiesce(&submit->depend_tx);
353
354 /* recompute p and/or q into the temporary buffer and then
355 * check to see the result matches the current value
356 */
357 tx = NULL;
358 *pqres = 0;
359 if (p_src) {
360 init_async_submit(submit, ASYNC_TX_XOR_ZERO_DST, NULL,
361 NULL, NULL, scribble);
362 tx = async_xor(spare, blocks, offset, disks-2, len, submit);
363 async_tx_quiesce(&tx);
364 p = page_address(p_src) + offset;
365 s = page_address(spare) + offset;
366 *pqres |= !!memcmp(p, s, len) << SUM_CHECK_P;
367 }
368
369 if (q_src) {
370 P(blocks, disks) = NULL;
371 Q(blocks, disks) = spare;
372 init_async_submit(submit, 0, NULL, NULL, NULL, scribble);
373 tx = async_gen_syndrome(blocks, offset, disks, len, submit);
374 async_tx_quiesce(&tx);
375 q = page_address(q_src) + offset;
376 s = page_address(spare) + offset;
377 *pqres |= !!memcmp(q, s, len) << SUM_CHECK_Q;
378 }
379
380 /* restore P, Q and submit */
381 P(blocks, disks) = p_src;
382 Q(blocks, disks) = q_src;
383
384 submit->cb_fn = cb_fn_orig;
385 submit->cb_param = cb_param_orig;
386 submit->flags = flags_orig;
387 async_tx_sync_epilog(submit);
388
389 return NULL;
390 }
391}
392EXPORT_SYMBOL_GPL(async_syndrome_val);
393
394static int __init async_pq_init(void)
395{
396 pq_scribble_page = alloc_page(GFP_KERNEL);
397
398 if (pq_scribble_page)
399 return 0;
400
401 pr_err("%s: failed to allocate required spare page\n", __func__);
402
403 return -ENOMEM;
404}
405
406static void __exit async_pq_exit(void)
407{
408 put_page(pq_scribble_page);
409}
410
411module_init(async_pq_init);
412module_exit(async_pq_exit);
413
414MODULE_DESCRIPTION("asynchronous raid6 syndrome generation/validation");
415MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c
new file mode 100644
index 000000000000..943f2abac9b4
--- /dev/null
+++ b/crypto/async_tx/async_raid6_recov.c
@@ -0,0 +1,500 @@
1/*
2 * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
3 * Copyright(c) 2009 Intel Corporation
4 *
5 * based on raid6recov.c:
6 * Copyright 2002 H. Peter Anvin
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc., 51
20 * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
21 *
22 */
23#include <linux/kernel.h>
24#include <linux/interrupt.h>
25#include <linux/dma-mapping.h>
26#include <linux/raid/pq.h>
27#include <linux/async_tx.h>
28
29static struct dma_async_tx_descriptor *
30async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
31 size_t len, struct async_submit_ctl *submit)
32{
33 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
34 &dest, 1, srcs, 2, len);
35 struct dma_device *dma = chan ? chan->device : NULL;
36 const u8 *amul, *bmul;
37 u8 ax, bx;
38 u8 *a, *b, *c;
39
40 if (dma) {
41 dma_addr_t dma_dest[2];
42 dma_addr_t dma_src[2];
43 struct device *dev = dma->dev;
44 struct dma_async_tx_descriptor *tx;
45 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
46
47 if (submit->flags & ASYNC_TX_FENCE)
48 dma_flags |= DMA_PREP_FENCE;
49 dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
50 dma_src[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
51 dma_src[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
52 tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 2, coef,
53 len, dma_flags);
54 if (tx) {
55 async_tx_submit(chan, tx, submit);
56 return tx;
57 }
58
59 /* could not get a descriptor, unmap and fall through to
60 * the synchronous path
61 */
62 dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL);
63 dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
64 dma_unmap_page(dev, dma_src[1], len, DMA_TO_DEVICE);
65 }
66
67 /* run the operation synchronously */
68 async_tx_quiesce(&submit->depend_tx);
69 amul = raid6_gfmul[coef[0]];
70 bmul = raid6_gfmul[coef[1]];
71 a = page_address(srcs[0]);
72 b = page_address(srcs[1]);
73 c = page_address(dest);
74
75 while (len--) {
76 ax = amul[*a++];
77 bx = bmul[*b++];
78 *c++ = ax ^ bx;
79 }
80
81 return NULL;
82}
83
84static struct dma_async_tx_descriptor *
85async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
86 struct async_submit_ctl *submit)
87{
88 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
89 &dest, 1, &src, 1, len);
90 struct dma_device *dma = chan ? chan->device : NULL;
91 const u8 *qmul; /* Q multiplier table */
92 u8 *d, *s;
93
94 if (dma) {
95 dma_addr_t dma_dest[2];
96 dma_addr_t dma_src[1];
97 struct device *dev = dma->dev;
98 struct dma_async_tx_descriptor *tx;
99 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
100
101 if (submit->flags & ASYNC_TX_FENCE)
102 dma_flags |= DMA_PREP_FENCE;
103 dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
104 dma_src[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
105 tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 1, &coef,
106 len, dma_flags);
107 if (tx) {
108 async_tx_submit(chan, tx, submit);
109 return tx;
110 }
111
112 /* could not get a descriptor, unmap and fall through to
113 * the synchronous path
114 */
115 dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL);
116 dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
117 }
118
119 /* no channel available, or failed to allocate a descriptor, so
120 * perform the operation synchronously
121 */
122 async_tx_quiesce(&submit->depend_tx);
123 qmul = raid6_gfmul[coef];
124 d = page_address(dest);
125 s = page_address(src);
126
127 while (len--)
128 *d++ = qmul[*s++];
129
130 return NULL;
131}
132
133static struct dma_async_tx_descriptor *
134__2data_recov_4(int disks, size_t bytes, int faila, int failb,
135 struct page **blocks, struct async_submit_ctl *submit)
136{
137 struct dma_async_tx_descriptor *tx = NULL;
138 struct page *p, *q, *a, *b;
139 struct page *srcs[2];
140 unsigned char coef[2];
141 enum async_tx_flags flags = submit->flags;
142 dma_async_tx_callback cb_fn = submit->cb_fn;
143 void *cb_param = submit->cb_param;
144 void *scribble = submit->scribble;
145
146 p = blocks[disks-2];
147 q = blocks[disks-1];
148
149 a = blocks[faila];
150 b = blocks[failb];
151
152 /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
153 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
154 srcs[0] = p;
155 srcs[1] = q;
156 coef[0] = raid6_gfexi[failb-faila];
157 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
158 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
159 tx = async_sum_product(b, srcs, coef, bytes, submit);
160
161 /* Dy = P+Pxy+Dx */
162 srcs[0] = p;
163 srcs[1] = b;
164 init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
165 cb_param, scribble);
166 tx = async_xor(a, srcs, 0, 2, bytes, submit);
167
168 return tx;
169
170}
171
172static struct dma_async_tx_descriptor *
173__2data_recov_5(int disks, size_t bytes, int faila, int failb,
174 struct page **blocks, struct async_submit_ctl *submit)
175{
176 struct dma_async_tx_descriptor *tx = NULL;
177 struct page *p, *q, *g, *dp, *dq;
178 struct page *srcs[2];
179 unsigned char coef[2];
180 enum async_tx_flags flags = submit->flags;
181 dma_async_tx_callback cb_fn = submit->cb_fn;
182 void *cb_param = submit->cb_param;
183 void *scribble = submit->scribble;
184 int good_srcs, good, i;
185
186 good_srcs = 0;
187 good = -1;
188 for (i = 0; i < disks-2; i++) {
189 if (blocks[i] == NULL)
190 continue;
191 if (i == faila || i == failb)
192 continue;
193 good = i;
194 good_srcs++;
195 }
196 BUG_ON(good_srcs > 1);
197
198 p = blocks[disks-2];
199 q = blocks[disks-1];
200 g = blocks[good];
201
202 /* Compute syndrome with zero for the missing data pages
203 * Use the dead data pages as temporary storage for delta p and
204 * delta q
205 */
206 dp = blocks[faila];
207 dq = blocks[failb];
208
209 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
210 tx = async_memcpy(dp, g, 0, 0, bytes, submit);
211 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
212 tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
213
214 /* compute P + Pxy */
215 srcs[0] = dp;
216 srcs[1] = p;
217 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
218 NULL, NULL, scribble);
219 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
220
221 /* compute Q + Qxy */
222 srcs[0] = dq;
223 srcs[1] = q;
224 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
225 NULL, NULL, scribble);
226 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
227
228 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
229 srcs[0] = dp;
230 srcs[1] = dq;
231 coef[0] = raid6_gfexi[failb-faila];
232 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
233 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
234 tx = async_sum_product(dq, srcs, coef, bytes, submit);
235
236 /* Dy = P+Pxy+Dx */
237 srcs[0] = dp;
238 srcs[1] = dq;
239 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
240 cb_param, scribble);
241 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
242
243 return tx;
244}
245
246static struct dma_async_tx_descriptor *
247__2data_recov_n(int disks, size_t bytes, int faila, int failb,
248 struct page **blocks, struct async_submit_ctl *submit)
249{
250 struct dma_async_tx_descriptor *tx = NULL;
251 struct page *p, *q, *dp, *dq;
252 struct page *srcs[2];
253 unsigned char coef[2];
254 enum async_tx_flags flags = submit->flags;
255 dma_async_tx_callback cb_fn = submit->cb_fn;
256 void *cb_param = submit->cb_param;
257 void *scribble = submit->scribble;
258
259 p = blocks[disks-2];
260 q = blocks[disks-1];
261
262 /* Compute syndrome with zero for the missing data pages
263 * Use the dead data pages as temporary storage for
264 * delta p and delta q
265 */
266 dp = blocks[faila];
267 blocks[faila] = NULL;
268 blocks[disks-2] = dp;
269 dq = blocks[failb];
270 blocks[failb] = NULL;
271 blocks[disks-1] = dq;
272
273 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
274 tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
275
276 /* Restore pointer table */
277 blocks[faila] = dp;
278 blocks[failb] = dq;
279 blocks[disks-2] = p;
280 blocks[disks-1] = q;
281
282 /* compute P + Pxy */
283 srcs[0] = dp;
284 srcs[1] = p;
285 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
286 NULL, NULL, scribble);
287 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
288
289 /* compute Q + Qxy */
290 srcs[0] = dq;
291 srcs[1] = q;
292 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
293 NULL, NULL, scribble);
294 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
295
296 /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
297 srcs[0] = dp;
298 srcs[1] = dq;
299 coef[0] = raid6_gfexi[failb-faila];
300 coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
301 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
302 tx = async_sum_product(dq, srcs, coef, bytes, submit);
303
304 /* Dy = P+Pxy+Dx */
305 srcs[0] = dp;
306 srcs[1] = dq;
307 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
308 cb_param, scribble);
309 tx = async_xor(dp, srcs, 0, 2, bytes, submit);
310
311 return tx;
312}
313
314/**
315 * async_raid6_2data_recov - asynchronously calculate two missing data blocks
316 * @disks: number of disks in the RAID-6 array
317 * @bytes: block size
318 * @faila: first failed drive index
319 * @failb: second failed drive index
320 * @blocks: array of source pointers where the last two entries are p and q
321 * @submit: submission/completion modifiers
322 */
323struct dma_async_tx_descriptor *
324async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
325 struct page **blocks, struct async_submit_ctl *submit)
326{
327 int non_zero_srcs, i;
328
329 BUG_ON(faila == failb);
330 if (failb < faila)
331 swap(faila, failb);
332
333 pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
334
335 /* we need to preserve the contents of 'blocks' for the async
336 * case, so punt to synchronous if a scribble buffer is not available
337 */
338 if (!submit->scribble) {
339 void **ptrs = (void **) blocks;
340
341 async_tx_quiesce(&submit->depend_tx);
342 for (i = 0; i < disks; i++)
343 if (blocks[i] == NULL)
344 ptrs[i] = (void *) raid6_empty_zero_page;
345 else
346 ptrs[i] = page_address(blocks[i]);
347
348 raid6_2data_recov(disks, bytes, faila, failb, ptrs);
349
350 async_tx_sync_epilog(submit);
351
352 return NULL;
353 }
354
355 non_zero_srcs = 0;
356 for (i = 0; i < disks-2 && non_zero_srcs < 4; i++)
357 if (blocks[i])
358 non_zero_srcs++;
359 switch (non_zero_srcs) {
360 case 0:
361 case 1:
362 /* There must be at least 2 sources - the failed devices. */
363 BUG();
364
365 case 2:
366 /* dma devices do not uniformly understand a zero source pq
367 * operation (in contrast to the synchronous case), so
368 * explicitly handle the special case of a 4 disk array with
369 * both data disks missing.
370 */
371 return __2data_recov_4(disks, bytes, faila, failb, blocks, submit);
372 case 3:
373 /* dma devices do not uniformly understand a single
374 * source pq operation (in contrast to the synchronous
375 * case), so explicitly handle the special case of a 5 disk
376 * array with 2 of 3 data disks missing.
377 */
378 return __2data_recov_5(disks, bytes, faila, failb, blocks, submit);
379 default:
380 return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
381 }
382}
383EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
384
385/**
386 * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
387 * @disks: number of disks in the RAID-6 array
388 * @bytes: block size
389 * @faila: failed drive index
390 * @blocks: array of source pointers where the last two entries are p and q
391 * @submit: submission/completion modifiers
392 */
393struct dma_async_tx_descriptor *
394async_raid6_datap_recov(int disks, size_t bytes, int faila,
395 struct page **blocks, struct async_submit_ctl *submit)
396{
397 struct dma_async_tx_descriptor *tx = NULL;
398 struct page *p, *q, *dq;
399 u8 coef;
400 enum async_tx_flags flags = submit->flags;
401 dma_async_tx_callback cb_fn = submit->cb_fn;
402 void *cb_param = submit->cb_param;
403 void *scribble = submit->scribble;
404 int good_srcs, good, i;
405 struct page *srcs[2];
406
407 pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
408
409 /* we need to preserve the contents of 'blocks' for the async
410 * case, so punt to synchronous if a scribble buffer is not available
411 */
412 if (!scribble) {
413 void **ptrs = (void **) blocks;
414
415 async_tx_quiesce(&submit->depend_tx);
416 for (i = 0; i < disks; i++)
417 if (blocks[i] == NULL)
418 ptrs[i] = (void*)raid6_empty_zero_page;
419 else
420 ptrs[i] = page_address(blocks[i]);
421
422 raid6_datap_recov(disks, bytes, faila, ptrs);
423
424 async_tx_sync_epilog(submit);
425
426 return NULL;
427 }
428
429 good_srcs = 0;
430 good = -1;
431 for (i = 0; i < disks-2; i++) {
432 if (i == faila)
433 continue;
434 if (blocks[i]) {
435 good = i;
436 good_srcs++;
437 if (good_srcs > 1)
438 break;
439 }
440 }
441 BUG_ON(good_srcs == 0);
442
443 p = blocks[disks-2];
444 q = blocks[disks-1];
445
446 /* Compute syndrome with zero for the missing data page
447 * Use the dead data page as temporary storage for delta q
448 */
449 dq = blocks[faila];
450 blocks[faila] = NULL;
451 blocks[disks-1] = dq;
452
453 /* in the 4-disk case we only need to perform a single source
454 * multiplication with the one good data block.
455 */
456 if (good_srcs == 1) {
457 struct page *g = blocks[good];
458
459 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
460 scribble);
461 tx = async_memcpy(p, g, 0, 0, bytes, submit);
462
463 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
464 scribble);
465 tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
466 } else {
467 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
468 scribble);
469 tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
470 }
471
472 /* Restore pointer table */
473 blocks[faila] = dq;
474 blocks[disks-1] = q;
475
476 /* calculate g^{-faila} */
477 coef = raid6_gfinv[raid6_gfexp[faila]];
478
479 srcs[0] = dq;
480 srcs[1] = q;
481 init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
482 NULL, NULL, scribble);
483 tx = async_xor(dq, srcs, 0, 2, bytes, submit);
484
485 init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
486 tx = async_mult(dq, dq, coef, bytes, submit);
487
488 srcs[0] = p;
489 srcs[1] = dq;
490 init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
491 cb_param, scribble);
492 tx = async_xor(p, srcs, 0, 2, bytes, submit);
493
494 return tx;
495}
496EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
497
498MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
499MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
500MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c
index 06eb6cc09fef..f9cdf04fe7c0 100644
--- a/crypto/async_tx/async_tx.c
+++ b/crypto/async_tx/async_tx.c
@@ -42,16 +42,21 @@ static void __exit async_tx_exit(void)
42 async_dmaengine_put(); 42 async_dmaengine_put();
43} 43}
44 44
45module_init(async_tx_init);
46module_exit(async_tx_exit);
47
45/** 48/**
46 * __async_tx_find_channel - find a channel to carry out the operation or let 49 * __async_tx_find_channel - find a channel to carry out the operation or let
47 * the transaction execute synchronously 50 * the transaction execute synchronously
48 * @depend_tx: transaction dependency 51 * @submit: transaction dependency and submission modifiers
49 * @tx_type: transaction type 52 * @tx_type: transaction type
50 */ 53 */
51struct dma_chan * 54struct dma_chan *
52__async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx, 55__async_tx_find_channel(struct async_submit_ctl *submit,
53 enum dma_transaction_type tx_type) 56 enum dma_transaction_type tx_type)
54{ 57{
58 struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
59
55 /* see if we can keep the chain on one channel */ 60 /* see if we can keep the chain on one channel */
56 if (depend_tx && 61 if (depend_tx &&
57 dma_has_cap(tx_type, depend_tx->chan->device->cap_mask)) 62 dma_has_cap(tx_type, depend_tx->chan->device->cap_mask))
@@ -59,17 +64,6 @@ __async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx,
59 return async_dma_find_channel(tx_type); 64 return async_dma_find_channel(tx_type);
60} 65}
61EXPORT_SYMBOL_GPL(__async_tx_find_channel); 66EXPORT_SYMBOL_GPL(__async_tx_find_channel);
62#else
63static int __init async_tx_init(void)
64{
65 printk(KERN_INFO "async_tx: api initialized (sync-only)\n");
66 return 0;
67}
68
69static void __exit async_tx_exit(void)
70{
71 do { } while (0);
72}
73#endif 67#endif
74 68
75 69
@@ -83,10 +77,14 @@ static void
83async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx, 77async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
84 struct dma_async_tx_descriptor *tx) 78 struct dma_async_tx_descriptor *tx)
85{ 79{
86 struct dma_chan *chan; 80 struct dma_chan *chan = depend_tx->chan;
87 struct dma_device *device; 81 struct dma_device *device = chan->device;
88 struct dma_async_tx_descriptor *intr_tx = (void *) ~0; 82 struct dma_async_tx_descriptor *intr_tx = (void *) ~0;
89 83
84 #ifdef CONFIG_ASYNC_TX_DISABLE_CHANNEL_SWITCH
85 BUG();
86 #endif
87
90 /* first check to see if we can still append to depend_tx */ 88 /* first check to see if we can still append to depend_tx */
91 spin_lock_bh(&depend_tx->lock); 89 spin_lock_bh(&depend_tx->lock);
92 if (depend_tx->parent && depend_tx->chan == tx->chan) { 90 if (depend_tx->parent && depend_tx->chan == tx->chan) {
@@ -96,11 +94,11 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
96 } 94 }
97 spin_unlock_bh(&depend_tx->lock); 95 spin_unlock_bh(&depend_tx->lock);
98 96
99 if (!intr_tx) 97 /* attached dependency, flush the parent channel */
98 if (!intr_tx) {
99 device->device_issue_pending(chan);
100 return; 100 return;
101 101 }
102 chan = depend_tx->chan;
103 device = chan->device;
104 102
105 /* see if we can schedule an interrupt 103 /* see if we can schedule an interrupt
106 * otherwise poll for completion 104 * otherwise poll for completion
@@ -134,6 +132,7 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
134 intr_tx->tx_submit(intr_tx); 132 intr_tx->tx_submit(intr_tx);
135 async_tx_ack(intr_tx); 133 async_tx_ack(intr_tx);
136 } 134 }
135 device->device_issue_pending(chan);
137 } else { 136 } else {
138 if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR) 137 if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
139 panic("%s: DMA_ERROR waiting for depend_tx\n", 138 panic("%s: DMA_ERROR waiting for depend_tx\n",
@@ -144,13 +143,14 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
144 143
145 144
146/** 145/**
147 * submit_disposition - while holding depend_tx->lock we must avoid submitting 146 * submit_disposition - flags for routing an incoming operation
148 * new operations to prevent a circular locking dependency with
149 * drivers that already hold a channel lock when calling
150 * async_tx_run_dependencies.
151 * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock 147 * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock
152 * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch 148 * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch
153 * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly 149 * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly
150 *
151 * while holding depend_tx->lock we must avoid submitting new operations
152 * to prevent a circular locking dependency with drivers that already
153 * hold a channel lock when calling async_tx_run_dependencies.
154 */ 154 */
155enum submit_disposition { 155enum submit_disposition {
156 ASYNC_TX_SUBMITTED, 156 ASYNC_TX_SUBMITTED,
@@ -160,11 +160,12 @@ enum submit_disposition {
160 160
161void 161void
162async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, 162async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,
163 enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx, 163 struct async_submit_ctl *submit)
164 dma_async_tx_callback cb_fn, void *cb_param)
165{ 164{
166 tx->callback = cb_fn; 165 struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
167 tx->callback_param = cb_param; 166
167 tx->callback = submit->cb_fn;
168 tx->callback_param = submit->cb_param;
168 169
169 if (depend_tx) { 170 if (depend_tx) {
170 enum submit_disposition s; 171 enum submit_disposition s;
@@ -220,30 +221,29 @@ async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,
220 tx->tx_submit(tx); 221 tx->tx_submit(tx);
221 } 222 }
222 223
223 if (flags & ASYNC_TX_ACK) 224 if (submit->flags & ASYNC_TX_ACK)
224 async_tx_ack(tx); 225 async_tx_ack(tx);
225 226
226 if (depend_tx && (flags & ASYNC_TX_DEP_ACK)) 227 if (depend_tx)
227 async_tx_ack(depend_tx); 228 async_tx_ack(depend_tx);
228} 229}
229EXPORT_SYMBOL_GPL(async_tx_submit); 230EXPORT_SYMBOL_GPL(async_tx_submit);
230 231
231/** 232/**
232 * async_trigger_callback - schedules the callback function to be run after 233 * async_trigger_callback - schedules the callback function to be run
233 * any dependent operations have been completed. 234 * @submit: submission and completion parameters
234 * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK 235 *
235 * @depend_tx: 'callback' requires the completion of this transaction 236 * honored flags: ASYNC_TX_ACK
236 * @cb_fn: function to call after depend_tx completes 237 *
237 * @cb_param: parameter to pass to the callback routine 238 * The callback is run after any dependent operations have completed.
238 */ 239 */
239struct dma_async_tx_descriptor * 240struct dma_async_tx_descriptor *
240async_trigger_callback(enum async_tx_flags flags, 241async_trigger_callback(struct async_submit_ctl *submit)
241 struct dma_async_tx_descriptor *depend_tx,
242 dma_async_tx_callback cb_fn, void *cb_param)
243{ 242{
244 struct dma_chan *chan; 243 struct dma_chan *chan;
245 struct dma_device *device; 244 struct dma_device *device;
246 struct dma_async_tx_descriptor *tx; 245 struct dma_async_tx_descriptor *tx;
246 struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
247 247
248 if (depend_tx) { 248 if (depend_tx) {
249 chan = depend_tx->chan; 249 chan = depend_tx->chan;
@@ -262,14 +262,14 @@ async_trigger_callback(enum async_tx_flags flags,
262 if (tx) { 262 if (tx) {
263 pr_debug("%s: (async)\n", __func__); 263 pr_debug("%s: (async)\n", __func__);
264 264
265 async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); 265 async_tx_submit(chan, tx, submit);
266 } else { 266 } else {
267 pr_debug("%s: (sync)\n", __func__); 267 pr_debug("%s: (sync)\n", __func__);
268 268
269 /* wait for any prerequisite operations */ 269 /* wait for any prerequisite operations */
270 async_tx_quiesce(&depend_tx); 270 async_tx_quiesce(&submit->depend_tx);
271 271
272 async_tx_sync_epilog(cb_fn, cb_param); 272 async_tx_sync_epilog(submit);
273 } 273 }
274 274
275 return tx; 275 return tx;
@@ -295,9 +295,6 @@ void async_tx_quiesce(struct dma_async_tx_descriptor **tx)
295} 295}
296EXPORT_SYMBOL_GPL(async_tx_quiesce); 296EXPORT_SYMBOL_GPL(async_tx_quiesce);
297 297
298module_init(async_tx_init);
299module_exit(async_tx_exit);
300
301MODULE_AUTHOR("Intel Corporation"); 298MODULE_AUTHOR("Intel Corporation");
302MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API"); 299MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API");
303MODULE_LICENSE("GPL"); 300MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c
index 90dd3f8bd283..079ae8ca590b 100644
--- a/crypto/async_tx/async_xor.c
+++ b/crypto/async_tx/async_xor.c
@@ -33,55 +33,57 @@
33/* do_async_xor - dma map the pages and perform the xor with an engine */ 33/* do_async_xor - dma map the pages and perform the xor with an engine */
34static __async_inline struct dma_async_tx_descriptor * 34static __async_inline struct dma_async_tx_descriptor *
35do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, 35do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
36 unsigned int offset, int src_cnt, size_t len, 36 unsigned int offset, int src_cnt, size_t len, dma_addr_t *dma_src,
37 enum async_tx_flags flags, 37 struct async_submit_ctl *submit)
38 struct dma_async_tx_descriptor *depend_tx,
39 dma_async_tx_callback cb_fn, void *cb_param)
40{ 38{
41 struct dma_device *dma = chan->device; 39 struct dma_device *dma = chan->device;
42 dma_addr_t *dma_src = (dma_addr_t *) src_list;
43 struct dma_async_tx_descriptor *tx = NULL; 40 struct dma_async_tx_descriptor *tx = NULL;
44 int src_off = 0; 41 int src_off = 0;
45 int i; 42 int i;
46 dma_async_tx_callback _cb_fn; 43 dma_async_tx_callback cb_fn_orig = submit->cb_fn;
47 void *_cb_param; 44 void *cb_param_orig = submit->cb_param;
48 enum async_tx_flags async_flags; 45 enum async_tx_flags flags_orig = submit->flags;
49 enum dma_ctrl_flags dma_flags; 46 enum dma_ctrl_flags dma_flags;
50 int xor_src_cnt; 47 int xor_src_cnt = 0;
51 dma_addr_t dma_dest; 48 dma_addr_t dma_dest;
52 49
53 /* map the dest bidrectional in case it is re-used as a source */ 50 /* map the dest bidrectional in case it is re-used as a source */
54 dma_dest = dma_map_page(dma->dev, dest, offset, len, DMA_BIDIRECTIONAL); 51 dma_dest = dma_map_page(dma->dev, dest, offset, len, DMA_BIDIRECTIONAL);
55 for (i = 0; i < src_cnt; i++) { 52 for (i = 0; i < src_cnt; i++) {
56 /* only map the dest once */ 53 /* only map the dest once */
54 if (!src_list[i])
55 continue;
57 if (unlikely(src_list[i] == dest)) { 56 if (unlikely(src_list[i] == dest)) {
58 dma_src[i] = dma_dest; 57 dma_src[xor_src_cnt++] = dma_dest;
59 continue; 58 continue;
60 } 59 }
61 dma_src[i] = dma_map_page(dma->dev, src_list[i], offset, 60 dma_src[xor_src_cnt++] = dma_map_page(dma->dev, src_list[i], offset,
62 len, DMA_TO_DEVICE); 61 len, DMA_TO_DEVICE);
63 } 62 }
63 src_cnt = xor_src_cnt;
64 64
65 while (src_cnt) { 65 while (src_cnt) {
66 async_flags = flags; 66 submit->flags = flags_orig;
67 dma_flags = 0; 67 dma_flags = 0;
68 xor_src_cnt = min(src_cnt, dma->max_xor); 68 xor_src_cnt = min(src_cnt, (int)dma->max_xor);
69 /* if we are submitting additional xors, leave the chain open, 69 /* if we are submitting additional xors, leave the chain open,
70 * clear the callback parameters, and leave the destination 70 * clear the callback parameters, and leave the destination
71 * buffer mapped 71 * buffer mapped
72 */ 72 */
73 if (src_cnt > xor_src_cnt) { 73 if (src_cnt > xor_src_cnt) {
74 async_flags &= ~ASYNC_TX_ACK; 74 submit->flags &= ~ASYNC_TX_ACK;
75 submit->flags |= ASYNC_TX_FENCE;
75 dma_flags = DMA_COMPL_SKIP_DEST_UNMAP; 76 dma_flags = DMA_COMPL_SKIP_DEST_UNMAP;
76 _cb_fn = NULL; 77 submit->cb_fn = NULL;
77 _cb_param = NULL; 78 submit->cb_param = NULL;
78 } else { 79 } else {
79 _cb_fn = cb_fn; 80 submit->cb_fn = cb_fn_orig;
80 _cb_param = cb_param; 81 submit->cb_param = cb_param_orig;
81 } 82 }
82 if (_cb_fn) 83 if (submit->cb_fn)
83 dma_flags |= DMA_PREP_INTERRUPT; 84 dma_flags |= DMA_PREP_INTERRUPT;
84 85 if (submit->flags & ASYNC_TX_FENCE)
86 dma_flags |= DMA_PREP_FENCE;
85 /* Since we have clobbered the src_list we are committed 87 /* Since we have clobbered the src_list we are committed
86 * to doing this asynchronously. Drivers force forward progress 88 * to doing this asynchronously. Drivers force forward progress
87 * in case they can not provide a descriptor 89 * in case they can not provide a descriptor
@@ -90,7 +92,7 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
90 xor_src_cnt, len, dma_flags); 92 xor_src_cnt, len, dma_flags);
91 93
92 if (unlikely(!tx)) 94 if (unlikely(!tx))
93 async_tx_quiesce(&depend_tx); 95 async_tx_quiesce(&submit->depend_tx);
94 96
95 /* spin wait for the preceeding transactions to complete */ 97 /* spin wait for the preceeding transactions to complete */
96 while (unlikely(!tx)) { 98 while (unlikely(!tx)) {
@@ -101,11 +103,8 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
101 dma_flags); 103 dma_flags);
102 } 104 }
103 105
104 async_tx_submit(chan, tx, async_flags, depend_tx, _cb_fn, 106 async_tx_submit(chan, tx, submit);
105 _cb_param); 107 submit->depend_tx = tx;
106
107 depend_tx = tx;
108 flags |= ASYNC_TX_DEP_ACK;
109 108
110 if (src_cnt > xor_src_cnt) { 109 if (src_cnt > xor_src_cnt) {
111 /* drop completed sources */ 110 /* drop completed sources */
@@ -124,23 +123,28 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
124 123
125static void 124static void
126do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, 125do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset,
127 int src_cnt, size_t len, enum async_tx_flags flags, 126 int src_cnt, size_t len, struct async_submit_ctl *submit)
128 dma_async_tx_callback cb_fn, void *cb_param)
129{ 127{
130 int i; 128 int i;
131 int xor_src_cnt; 129 int xor_src_cnt = 0;
132 int src_off = 0; 130 int src_off = 0;
133 void *dest_buf; 131 void *dest_buf;
134 void **srcs = (void **) src_list; 132 void **srcs;
135 133
136 /* reuse the 'src_list' array to convert to buffer pointers */ 134 if (submit->scribble)
137 for (i = 0; i < src_cnt; i++) 135 srcs = submit->scribble;
138 srcs[i] = page_address(src_list[i]) + offset; 136 else
137 srcs = (void **) src_list;
139 138
139 /* convert to buffer pointers */
140 for (i = 0; i < src_cnt; i++)
141 if (src_list[i])
142 srcs[xor_src_cnt++] = page_address(src_list[i]) + offset;
143 src_cnt = xor_src_cnt;
140 /* set destination address */ 144 /* set destination address */
141 dest_buf = page_address(dest) + offset; 145 dest_buf = page_address(dest) + offset;
142 146
143 if (flags & ASYNC_TX_XOR_ZERO_DST) 147 if (submit->flags & ASYNC_TX_XOR_ZERO_DST)
144 memset(dest_buf, 0, len); 148 memset(dest_buf, 0, len);
145 149
146 while (src_cnt > 0) { 150 while (src_cnt > 0) {
@@ -153,61 +157,70 @@ do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset,
153 src_off += xor_src_cnt; 157 src_off += xor_src_cnt;
154 } 158 }
155 159
156 async_tx_sync_epilog(cb_fn, cb_param); 160 async_tx_sync_epilog(submit);
157} 161}
158 162
159/** 163/**
160 * async_xor - attempt to xor a set of blocks with a dma engine. 164 * async_xor - attempt to xor a set of blocks with a dma engine.
161 * xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST
162 * flag must be set to not include dest data in the calculation. The
163 * assumption with dma eninges is that they only use the destination
164 * buffer as a source when it is explicity specified in the source list.
165 * @dest: destination page 165 * @dest: destination page
166 * @src_list: array of source pages (if the dest is also a source it must be 166 * @src_list: array of source pages
167 * at index zero). The contents of this array may be overwritten. 167 * @offset: common src/dst offset to start transaction
168 * @offset: offset in pages to start transaction
169 * @src_cnt: number of source pages 168 * @src_cnt: number of source pages
170 * @len: length in bytes 169 * @len: length in bytes
171 * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST, 170 * @submit: submission / completion modifiers
172 * ASYNC_TX_ACK, ASYNC_TX_DEP_ACK 171 *
173 * @depend_tx: xor depends on the result of this transaction. 172 * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
174 * @cb_fn: function to call when the xor completes 173 *
175 * @cb_param: parameter to pass to the callback routine 174 * xor_blocks always uses the dest as a source so the
175 * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in
176 * the calculation. The assumption with dma eninges is that they only
177 * use the destination buffer as a source when it is explicity specified
178 * in the source list.
179 *
180 * src_list note: if the dest is also a source it must be at index zero.
181 * The contents of this array will be overwritten if a scribble region
182 * is not specified.
176 */ 183 */
177struct dma_async_tx_descriptor * 184struct dma_async_tx_descriptor *
178async_xor(struct page *dest, struct page **src_list, unsigned int offset, 185async_xor(struct page *dest, struct page **src_list, unsigned int offset,
179 int src_cnt, size_t len, enum async_tx_flags flags, 186 int src_cnt, size_t len, struct async_submit_ctl *submit)
180 struct dma_async_tx_descriptor *depend_tx,
181 dma_async_tx_callback cb_fn, void *cb_param)
182{ 187{
183 struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_XOR, 188 struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR,
184 &dest, 1, src_list, 189 &dest, 1, src_list,
185 src_cnt, len); 190 src_cnt, len);
191 dma_addr_t *dma_src = NULL;
192
186 BUG_ON(src_cnt <= 1); 193 BUG_ON(src_cnt <= 1);
187 194
188 if (chan) { 195 if (submit->scribble)
196 dma_src = submit->scribble;
197 else if (sizeof(dma_addr_t) <= sizeof(struct page *))
198 dma_src = (dma_addr_t *) src_list;
199
200 if (dma_src && chan && is_dma_xor_aligned(chan->device, offset, 0, len)) {
189 /* run the xor asynchronously */ 201 /* run the xor asynchronously */
190 pr_debug("%s (async): len: %zu\n", __func__, len); 202 pr_debug("%s (async): len: %zu\n", __func__, len);
191 203
192 return do_async_xor(chan, dest, src_list, offset, src_cnt, len, 204 return do_async_xor(chan, dest, src_list, offset, src_cnt, len,
193 flags, depend_tx, cb_fn, cb_param); 205 dma_src, submit);
194 } else { 206 } else {
195 /* run the xor synchronously */ 207 /* run the xor synchronously */
196 pr_debug("%s (sync): len: %zu\n", __func__, len); 208 pr_debug("%s (sync): len: %zu\n", __func__, len);
209 WARN_ONCE(chan, "%s: no space for dma address conversion\n",
210 __func__);
197 211
198 /* in the sync case the dest is an implied source 212 /* in the sync case the dest is an implied source
199 * (assumes the dest is the first source) 213 * (assumes the dest is the first source)
200 */ 214 */
201 if (flags & ASYNC_TX_XOR_DROP_DST) { 215 if (submit->flags & ASYNC_TX_XOR_DROP_DST) {
202 src_cnt--; 216 src_cnt--;
203 src_list++; 217 src_list++;
204 } 218 }
205 219
206 /* wait for any prerequisite operations */ 220 /* wait for any prerequisite operations */
207 async_tx_quiesce(&depend_tx); 221 async_tx_quiesce(&submit->depend_tx);
208 222
209 do_sync_xor(dest, src_list, offset, src_cnt, len, 223 do_sync_xor(dest, src_list, offset, src_cnt, len, submit);
210 flags, cb_fn, cb_param);
211 224
212 return NULL; 225 return NULL;
213 } 226 }
@@ -221,105 +234,104 @@ static int page_is_zero(struct page *p, unsigned int offset, size_t len)
221 memcmp(a, a + 4, len - 4) == 0); 234 memcmp(a, a + 4, len - 4) == 0);
222} 235}
223 236
237static inline struct dma_chan *
238xor_val_chan(struct async_submit_ctl *submit, struct page *dest,
239 struct page **src_list, int src_cnt, size_t len)
240{
241 #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
242 return NULL;
243 #endif
244 return async_tx_find_channel(submit, DMA_XOR_VAL, &dest, 1, src_list,
245 src_cnt, len);
246}
247
224/** 248/**
225 * async_xor_zero_sum - attempt a xor parity check with a dma engine. 249 * async_xor_val - attempt a xor parity check with a dma engine.
226 * @dest: destination page used if the xor is performed synchronously 250 * @dest: destination page used if the xor is performed synchronously
227 * @src_list: array of source pages. The dest page must be listed as a source 251 * @src_list: array of source pages
228 * at index zero. The contents of this array may be overwritten.
229 * @offset: offset in pages to start transaction 252 * @offset: offset in pages to start transaction
230 * @src_cnt: number of source pages 253 * @src_cnt: number of source pages
231 * @len: length in bytes 254 * @len: length in bytes
232 * @result: 0 if sum == 0 else non-zero 255 * @result: 0 if sum == 0 else non-zero
233 * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK 256 * @submit: submission / completion modifiers
234 * @depend_tx: xor depends on the result of this transaction. 257 *
235 * @cb_fn: function to call when the xor completes 258 * honored flags: ASYNC_TX_ACK
236 * @cb_param: parameter to pass to the callback routine 259 *
260 * src_list note: if the dest is also a source it must be at index zero.
261 * The contents of this array will be overwritten if a scribble region
262 * is not specified.
237 */ 263 */
238struct dma_async_tx_descriptor * 264struct dma_async_tx_descriptor *
239async_xor_zero_sum(struct page *dest, struct page **src_list, 265async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
240 unsigned int offset, int src_cnt, size_t len, 266 int src_cnt, size_t len, enum sum_check_flags *result,
241 u32 *result, enum async_tx_flags flags, 267 struct async_submit_ctl *submit)
242 struct dma_async_tx_descriptor *depend_tx,
243 dma_async_tx_callback cb_fn, void *cb_param)
244{ 268{
245 struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_ZERO_SUM, 269 struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len);
246 &dest, 1, src_list,
247 src_cnt, len);
248 struct dma_device *device = chan ? chan->device : NULL; 270 struct dma_device *device = chan ? chan->device : NULL;
249 struct dma_async_tx_descriptor *tx = NULL; 271 struct dma_async_tx_descriptor *tx = NULL;
272 dma_addr_t *dma_src = NULL;
250 273
251 BUG_ON(src_cnt <= 1); 274 BUG_ON(src_cnt <= 1);
252 275
253 if (device && src_cnt <= device->max_xor) { 276 if (submit->scribble)
254 dma_addr_t *dma_src = (dma_addr_t *) src_list; 277 dma_src = submit->scribble;
255 unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; 278 else if (sizeof(dma_addr_t) <= sizeof(struct page *))
279 dma_src = (dma_addr_t *) src_list;
280
281 if (dma_src && device && src_cnt <= device->max_xor &&
282 is_dma_xor_aligned(device, offset, 0, len)) {
283 unsigned long dma_prep_flags = 0;
256 int i; 284 int i;
257 285
258 pr_debug("%s: (async) len: %zu\n", __func__, len); 286 pr_debug("%s: (async) len: %zu\n", __func__, len);
259 287
288 if (submit->cb_fn)
289 dma_prep_flags |= DMA_PREP_INTERRUPT;
290 if (submit->flags & ASYNC_TX_FENCE)
291 dma_prep_flags |= DMA_PREP_FENCE;
260 for (i = 0; i < src_cnt; i++) 292 for (i = 0; i < src_cnt; i++)
261 dma_src[i] = dma_map_page(device->dev, src_list[i], 293 dma_src[i] = dma_map_page(device->dev, src_list[i],
262 offset, len, DMA_TO_DEVICE); 294 offset, len, DMA_TO_DEVICE);
263 295
264 tx = device->device_prep_dma_zero_sum(chan, dma_src, src_cnt, 296 tx = device->device_prep_dma_xor_val(chan, dma_src, src_cnt,
265 len, result, 297 len, result,
266 dma_prep_flags); 298 dma_prep_flags);
267 if (unlikely(!tx)) { 299 if (unlikely(!tx)) {
268 async_tx_quiesce(&depend_tx); 300 async_tx_quiesce(&submit->depend_tx);
269 301
270 while (!tx) { 302 while (!tx) {
271 dma_async_issue_pending(chan); 303 dma_async_issue_pending(chan);
272 tx = device->device_prep_dma_zero_sum(chan, 304 tx = device->device_prep_dma_xor_val(chan,
273 dma_src, src_cnt, len, result, 305 dma_src, src_cnt, len, result,
274 dma_prep_flags); 306 dma_prep_flags);
275 } 307 }
276 } 308 }
277 309
278 async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); 310 async_tx_submit(chan, tx, submit);
279 } else { 311 } else {
280 unsigned long xor_flags = flags; 312 enum async_tx_flags flags_orig = submit->flags;
281 313
282 pr_debug("%s: (sync) len: %zu\n", __func__, len); 314 pr_debug("%s: (sync) len: %zu\n", __func__, len);
315 WARN_ONCE(device && src_cnt <= device->max_xor,
316 "%s: no space for dma address conversion\n",
317 __func__);
283 318
284 xor_flags |= ASYNC_TX_XOR_DROP_DST; 319 submit->flags |= ASYNC_TX_XOR_DROP_DST;
285 xor_flags &= ~ASYNC_TX_ACK; 320 submit->flags &= ~ASYNC_TX_ACK;
286 321
287 tx = async_xor(dest, src_list, offset, src_cnt, len, xor_flags, 322 tx = async_xor(dest, src_list, offset, src_cnt, len, submit);
288 depend_tx, NULL, NULL);
289 323
290 async_tx_quiesce(&tx); 324 async_tx_quiesce(&tx);
291 325
292 *result = page_is_zero(dest, offset, len) ? 0 : 1; 326 *result = !page_is_zero(dest, offset, len) << SUM_CHECK_P;
293 327
294 async_tx_sync_epilog(cb_fn, cb_param); 328 async_tx_sync_epilog(submit);
329 submit->flags = flags_orig;
295 } 330 }
296 331
297 return tx; 332 return tx;
298} 333}
299EXPORT_SYMBOL_GPL(async_xor_zero_sum); 334EXPORT_SYMBOL_GPL(async_xor_val);
300
301static int __init async_xor_init(void)
302{
303 #ifdef CONFIG_ASYNC_TX_DMA
304 /* To conserve stack space the input src_list (array of page pointers)
305 * is reused to hold the array of dma addresses passed to the driver.
306 * This conversion is only possible when dma_addr_t is less than the
307 * the size of a pointer. HIGHMEM64G is known to violate this
308 * assumption.
309 */
310 BUILD_BUG_ON(sizeof(dma_addr_t) > sizeof(struct page *));
311 #endif
312
313 return 0;
314}
315
316static void __exit async_xor_exit(void)
317{
318 do { } while (0);
319}
320
321module_init(async_xor_init);
322module_exit(async_xor_exit);
323 335
324MODULE_AUTHOR("Intel Corporation"); 336MODULE_AUTHOR("Intel Corporation");
325MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api"); 337MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api");
diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c
new file mode 100644
index 000000000000..3ec27c7e62ea
--- /dev/null
+++ b/crypto/async_tx/raid6test.c
@@ -0,0 +1,240 @@
1/*
2 * asynchronous raid6 recovery self test
3 * Copyright (c) 2009, Intel Corporation.
4 *
5 * based on drivers/md/raid6test/test.c:
6 * Copyright 2002-2007 H. Peter Anvin
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License,
10 * version 2, as published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * more details.
16 *
17 * You should have received a copy of the GNU General Public License along with
18 * this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 */
22#include <linux/async_tx.h>
23#include <linux/random.h>
24
25#undef pr
26#define pr(fmt, args...) pr_info("raid6test: " fmt, ##args)
27
28#define NDISKS 16 /* Including P and Q */
29
30static struct page *dataptrs[NDISKS];
31static addr_conv_t addr_conv[NDISKS];
32static struct page *data[NDISKS+3];
33static struct page *spare;
34static struct page *recovi;
35static struct page *recovj;
36
37static void callback(void *param)
38{
39 struct completion *cmp = param;
40
41 complete(cmp);
42}
43
44static void makedata(int disks)
45{
46 int i, j;
47
48 for (i = 0; i < disks; i++) {
49 for (j = 0; j < PAGE_SIZE/sizeof(u32); j += sizeof(u32)) {
50 u32 *p = page_address(data[i]) + j;
51
52 *p = random32();
53 }
54
55 dataptrs[i] = data[i];
56 }
57}
58
59static char disk_type(int d, int disks)
60{
61 if (d == disks - 2)
62 return 'P';
63 else if (d == disks - 1)
64 return 'Q';
65 else
66 return 'D';
67}
68
69/* Recover two failed blocks. */
70static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, struct page **ptrs)
71{
72 struct async_submit_ctl submit;
73 struct completion cmp;
74 struct dma_async_tx_descriptor *tx = NULL;
75 enum sum_check_flags result = ~0;
76
77 if (faila > failb)
78 swap(faila, failb);
79
80 if (failb == disks-1) {
81 if (faila == disks-2) {
82 /* P+Q failure. Just rebuild the syndrome. */
83 init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv);
84 tx = async_gen_syndrome(ptrs, 0, disks, bytes, &submit);
85 } else {
86 struct page *blocks[disks];
87 struct page *dest;
88 int count = 0;
89 int i;
90
91 /* data+Q failure. Reconstruct data from P,
92 * then rebuild syndrome
93 */
94 for (i = disks; i-- ; ) {
95 if (i == faila || i == failb)
96 continue;
97 blocks[count++] = ptrs[i];
98 }
99 dest = ptrs[faila];
100 init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL,
101 NULL, NULL, addr_conv);
102 tx = async_xor(dest, blocks, 0, count, bytes, &submit);
103
104 init_async_submit(&submit, 0, tx, NULL, NULL, addr_conv);
105 tx = async_gen_syndrome(ptrs, 0, disks, bytes, &submit);
106 }
107 } else {
108 if (failb == disks-2) {
109 /* data+P failure. */
110 init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv);
111 tx = async_raid6_datap_recov(disks, bytes, faila, ptrs, &submit);
112 } else {
113 /* data+data failure. */
114 init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv);
115 tx = async_raid6_2data_recov(disks, bytes, faila, failb, ptrs, &submit);
116 }
117 }
118 init_completion(&cmp);
119 init_async_submit(&submit, ASYNC_TX_ACK, tx, callback, &cmp, addr_conv);
120 tx = async_syndrome_val(ptrs, 0, disks, bytes, &result, spare, &submit);
121 async_tx_issue_pending(tx);
122
123 if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0)
124 pr("%s: timeout! (faila: %d failb: %d disks: %d)\n",
125 __func__, faila, failb, disks);
126
127 if (result != 0)
128 pr("%s: validation failure! faila: %d failb: %d sum_check_flags: %x\n",
129 __func__, faila, failb, result);
130}
131
132static int test_disks(int i, int j, int disks)
133{
134 int erra, errb;
135
136 memset(page_address(recovi), 0xf0, PAGE_SIZE);
137 memset(page_address(recovj), 0xba, PAGE_SIZE);
138
139 dataptrs[i] = recovi;
140 dataptrs[j] = recovj;
141
142 raid6_dual_recov(disks, PAGE_SIZE, i, j, dataptrs);
143
144 erra = memcmp(page_address(data[i]), page_address(recovi), PAGE_SIZE);
145 errb = memcmp(page_address(data[j]), page_address(recovj), PAGE_SIZE);
146
147 pr("%s(%d, %d): faila=%3d(%c) failb=%3d(%c) %s\n",
148 __func__, i, j, i, disk_type(i, disks), j, disk_type(j, disks),
149 (!erra && !errb) ? "OK" : !erra ? "ERRB" : !errb ? "ERRA" : "ERRAB");
150
151 dataptrs[i] = data[i];
152 dataptrs[j] = data[j];
153
154 return erra || errb;
155}
156
157static int test(int disks, int *tests)
158{
159 struct dma_async_tx_descriptor *tx;
160 struct async_submit_ctl submit;
161 struct completion cmp;
162 int err = 0;
163 int i, j;
164
165 recovi = data[disks];
166 recovj = data[disks+1];
167 spare = data[disks+2];
168
169 makedata(disks);
170
171 /* Nuke syndromes */
172 memset(page_address(data[disks-2]), 0xee, PAGE_SIZE);
173 memset(page_address(data[disks-1]), 0xee, PAGE_SIZE);
174
175 /* Generate assumed good syndrome */
176 init_completion(&cmp);
177 init_async_submit(&submit, ASYNC_TX_ACK, NULL, callback, &cmp, addr_conv);
178 tx = async_gen_syndrome(dataptrs, 0, disks, PAGE_SIZE, &submit);
179 async_tx_issue_pending(tx);
180
181 if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0) {
182 pr("error: initial gen_syndrome(%d) timed out\n", disks);
183 return 1;
184 }
185
186 pr("testing the %d-disk case...\n", disks);
187 for (i = 0; i < disks-1; i++)
188 for (j = i+1; j < disks; j++) {
189 (*tests)++;
190 err += test_disks(i, j, disks);
191 }
192
193 return err;
194}
195
196
197static int raid6_test(void)
198{
199 int err = 0;
200 int tests = 0;
201 int i;
202
203 for (i = 0; i < NDISKS+3; i++) {
204 data[i] = alloc_page(GFP_KERNEL);
205 if (!data[i]) {
206 while (i--)
207 put_page(data[i]);
208 return -ENOMEM;
209 }
210 }
211
212 /* the 4-disk and 5-disk cases are special for the recovery code */
213 if (NDISKS > 4)
214 err += test(4, &tests);
215 if (NDISKS > 5)
216 err += test(5, &tests);
217 err += test(NDISKS, &tests);
218
219 pr("\n");
220 pr("complete (%d tests, %d failure%s)\n",
221 tests, err, err == 1 ? "" : "s");
222
223 for (i = 0; i < NDISKS+3; i++)
224 put_page(data[i]);
225
226 return 0;
227}
228
229static void raid6_test_exit(void)
230{
231}
232
233/* when compiled-in wait for drivers to load first (assumes dma drivers
234 * are also compliled-in)
235 */
236late_initcall(raid6_test);
237module_exit(raid6_test_exit);
238MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
239MODULE_DESCRIPTION("asynchronous RAID-6 recovery self tests");
240MODULE_LICENSE("GPL");
diff --git a/crypto/gcm.c b/crypto/gcm.c
index 5fc3292483ef..c6547130624c 100644
--- a/crypto/gcm.c
+++ b/crypto/gcm.c
@@ -40,7 +40,7 @@ struct crypto_rfc4106_ctx {
40struct crypto_gcm_ghash_ctx { 40struct crypto_gcm_ghash_ctx {
41 unsigned int cryptlen; 41 unsigned int cryptlen;
42 struct scatterlist *src; 42 struct scatterlist *src;
43 crypto_completion_t complete; 43 void (*complete)(struct aead_request *req, int err);
44}; 44};
45 45
46struct crypto_gcm_req_priv_ctx { 46struct crypto_gcm_req_priv_ctx {
@@ -267,23 +267,26 @@ static int gcm_hash_final(struct aead_request *req,
267 return crypto_ahash_final(ahreq); 267 return crypto_ahash_final(ahreq);
268} 268}
269 269
270static void gcm_hash_final_done(struct crypto_async_request *areq, 270static void __gcm_hash_final_done(struct aead_request *req, int err)
271 int err)
272{ 271{
273 struct aead_request *req = areq->data;
274 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 272 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
275 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 273 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
276 274
277 if (!err) 275 if (!err)
278 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); 276 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
279 277
280 gctx->complete(areq, err); 278 gctx->complete(req, err);
281} 279}
282 280
283static void gcm_hash_len_done(struct crypto_async_request *areq, 281static void gcm_hash_final_done(struct crypto_async_request *areq, int err)
284 int err)
285{ 282{
286 struct aead_request *req = areq->data; 283 struct aead_request *req = areq->data;
284
285 __gcm_hash_final_done(req, err);
286}
287
288static void __gcm_hash_len_done(struct aead_request *req, int err)
289{
287 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 290 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
288 291
289 if (!err) { 292 if (!err) {
@@ -292,13 +295,18 @@ static void gcm_hash_len_done(struct crypto_async_request *areq,
292 return; 295 return;
293 } 296 }
294 297
295 gcm_hash_final_done(areq, err); 298 __gcm_hash_final_done(req, err);
296} 299}
297 300
298static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq, 301static void gcm_hash_len_done(struct crypto_async_request *areq, int err)
299 int err)
300{ 302{
301 struct aead_request *req = areq->data; 303 struct aead_request *req = areq->data;
304
305 __gcm_hash_len_done(req, err);
306}
307
308static void __gcm_hash_crypt_remain_done(struct aead_request *req, int err)
309{
302 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 310 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
303 311
304 if (!err) { 312 if (!err) {
@@ -307,13 +315,19 @@ static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq,
307 return; 315 return;
308 } 316 }
309 317
310 gcm_hash_len_done(areq, err); 318 __gcm_hash_len_done(req, err);
311} 319}
312 320
313static void gcm_hash_crypt_done(struct crypto_async_request *areq, 321static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq,
314 int err) 322 int err)
315{ 323{
316 struct aead_request *req = areq->data; 324 struct aead_request *req = areq->data;
325
326 __gcm_hash_crypt_remain_done(req, err);
327}
328
329static void __gcm_hash_crypt_done(struct aead_request *req, int err)
330{
317 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 331 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
318 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 332 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
319 unsigned int remain; 333 unsigned int remain;
@@ -327,13 +341,18 @@ static void gcm_hash_crypt_done(struct crypto_async_request *areq,
327 return; 341 return;
328 } 342 }
329 343
330 gcm_hash_crypt_remain_done(areq, err); 344 __gcm_hash_crypt_remain_done(req, err);
331} 345}
332 346
333static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq, 347static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err)
334 int err)
335{ 348{
336 struct aead_request *req = areq->data; 349 struct aead_request *req = areq->data;
350
351 __gcm_hash_crypt_done(req, err);
352}
353
354static void __gcm_hash_assoc_remain_done(struct aead_request *req, int err)
355{
337 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 356 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
338 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 357 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
339 crypto_completion_t complete; 358 crypto_completion_t complete;
@@ -350,15 +369,21 @@ static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq,
350 } 369 }
351 370
352 if (remain) 371 if (remain)
353 gcm_hash_crypt_done(areq, err); 372 __gcm_hash_crypt_done(req, err);
354 else 373 else
355 gcm_hash_crypt_remain_done(areq, err); 374 __gcm_hash_crypt_remain_done(req, err);
356} 375}
357 376
358static void gcm_hash_assoc_done(struct crypto_async_request *areq, 377static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq,
359 int err) 378 int err)
360{ 379{
361 struct aead_request *req = areq->data; 380 struct aead_request *req = areq->data;
381
382 __gcm_hash_assoc_remain_done(req, err);
383}
384
385static void __gcm_hash_assoc_done(struct aead_request *req, int err)
386{
362 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 387 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
363 unsigned int remain; 388 unsigned int remain;
364 389
@@ -371,13 +396,18 @@ static void gcm_hash_assoc_done(struct crypto_async_request *areq,
371 return; 396 return;
372 } 397 }
373 398
374 gcm_hash_assoc_remain_done(areq, err); 399 __gcm_hash_assoc_remain_done(req, err);
375} 400}
376 401
377static void gcm_hash_init_done(struct crypto_async_request *areq, 402static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err)
378 int err)
379{ 403{
380 struct aead_request *req = areq->data; 404 struct aead_request *req = areq->data;
405
406 __gcm_hash_assoc_done(req, err);
407}
408
409static void __gcm_hash_init_done(struct aead_request *req, int err)
410{
381 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 411 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
382 crypto_completion_t complete; 412 crypto_completion_t complete;
383 unsigned int remain = 0; 413 unsigned int remain = 0;
@@ -393,9 +423,16 @@ static void gcm_hash_init_done(struct crypto_async_request *areq,
393 } 423 }
394 424
395 if (remain) 425 if (remain)
396 gcm_hash_assoc_done(areq, err); 426 __gcm_hash_assoc_done(req, err);
397 else 427 else
398 gcm_hash_assoc_remain_done(areq, err); 428 __gcm_hash_assoc_remain_done(req, err);
429}
430
431static void gcm_hash_init_done(struct crypto_async_request *areq, int err)
432{
433 struct aead_request *req = areq->data;
434
435 __gcm_hash_init_done(req, err);
399} 436}
400 437
401static int gcm_hash(struct aead_request *req, 438static int gcm_hash(struct aead_request *req,
@@ -457,10 +494,8 @@ static void gcm_enc_copy_hash(struct aead_request *req,
457 crypto_aead_authsize(aead), 1); 494 crypto_aead_authsize(aead), 1);
458} 495}
459 496
460static void gcm_enc_hash_done(struct crypto_async_request *areq, 497static void gcm_enc_hash_done(struct aead_request *req, int err)
461 int err)
462{ 498{
463 struct aead_request *req = areq->data;
464 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 499 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
465 500
466 if (!err) 501 if (!err)
@@ -469,8 +504,7 @@ static void gcm_enc_hash_done(struct crypto_async_request *areq,
469 aead_request_complete(req, err); 504 aead_request_complete(req, err);
470} 505}
471 506
472static void gcm_encrypt_done(struct crypto_async_request *areq, 507static void gcm_encrypt_done(struct crypto_async_request *areq, int err)
473 int err)
474{ 508{
475 struct aead_request *req = areq->data; 509 struct aead_request *req = areq->data;
476 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 510 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
@@ -479,9 +513,13 @@ static void gcm_encrypt_done(struct crypto_async_request *areq,
479 err = gcm_hash(req, pctx); 513 err = gcm_hash(req, pctx);
480 if (err == -EINPROGRESS || err == -EBUSY) 514 if (err == -EINPROGRESS || err == -EBUSY)
481 return; 515 return;
516 else if (!err) {
517 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
518 gcm_enc_copy_hash(req, pctx);
519 }
482 } 520 }
483 521
484 gcm_enc_hash_done(areq, err); 522 aead_request_complete(req, err);
485} 523}
486 524
487static int crypto_gcm_encrypt(struct aead_request *req) 525static int crypto_gcm_encrypt(struct aead_request *req)
@@ -538,9 +576,8 @@ static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
538 aead_request_complete(req, err); 576 aead_request_complete(req, err);
539} 577}
540 578
541static void gcm_dec_hash_done(struct crypto_async_request *areq, int err) 579static void gcm_dec_hash_done(struct aead_request *req, int err)
542{ 580{
543 struct aead_request *req = areq->data;
544 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); 581 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
545 struct ablkcipher_request *abreq = &pctx->u.abreq; 582 struct ablkcipher_request *abreq = &pctx->u.abreq;
546 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; 583 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
@@ -552,9 +589,11 @@ static void gcm_dec_hash_done(struct crypto_async_request *areq, int err)
552 err = crypto_ablkcipher_decrypt(abreq); 589 err = crypto_ablkcipher_decrypt(abreq);
553 if (err == -EINPROGRESS || err == -EBUSY) 590 if (err == -EINPROGRESS || err == -EBUSY)
554 return; 591 return;
592 else if (!err)
593 err = crypto_gcm_verify(req, pctx);
555 } 594 }
556 595
557 gcm_decrypt_done(areq, err); 596 aead_request_complete(req, err);
558} 597}
559 598
560static int crypto_gcm_decrypt(struct aead_request *req) 599static int crypto_gcm_decrypt(struct aead_request *req)
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-09 19:43:05 -0500