diff options
Diffstat (limited to 'crypto/async_tx/async_xor.c')
-rw-r--r-- | crypto/async_tx/async_xor.c | 199 |
1 files changed, 96 insertions, 103 deletions
diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c index 95fe2c8d6c5..56b5f98da46 100644 --- a/crypto/async_tx/async_xor.c +++ b/crypto/async_tx/async_xor.c | |||
@@ -33,19 +33,16 @@ | |||
33 | /* do_async_xor - dma map the pages and perform the xor with an engine */ | 33 | /* do_async_xor - dma map the pages and perform the xor with an engine */ |
34 | static __async_inline struct dma_async_tx_descriptor * | 34 | static __async_inline struct dma_async_tx_descriptor * |
35 | do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, | 35 | do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, |
36 | unsigned int offset, int src_cnt, size_t len, | 36 | unsigned int offset, int src_cnt, size_t len, dma_addr_t *dma_src, |
37 | enum async_tx_flags flags, | 37 | struct async_submit_ctl *submit) |
38 | struct dma_async_tx_descriptor *depend_tx, | ||
39 | dma_async_tx_callback cb_fn, void *cb_param) | ||
40 | { | 38 | { |
41 | struct dma_device *dma = chan->device; | 39 | struct dma_device *dma = chan->device; |
42 | dma_addr_t *dma_src = (dma_addr_t *) src_list; | ||
43 | struct dma_async_tx_descriptor *tx = NULL; | 40 | struct dma_async_tx_descriptor *tx = NULL; |
44 | int src_off = 0; | 41 | int src_off = 0; |
45 | int i; | 42 | int i; |
46 | dma_async_tx_callback _cb_fn; | 43 | dma_async_tx_callback cb_fn_orig = submit->cb_fn; |
47 | void *_cb_param; | 44 | void *cb_param_orig = submit->cb_param; |
48 | enum async_tx_flags async_flags; | 45 | enum async_tx_flags flags_orig = submit->flags; |
49 | enum dma_ctrl_flags dma_flags; | 46 | enum dma_ctrl_flags dma_flags; |
50 | int xor_src_cnt; | 47 | int xor_src_cnt; |
51 | dma_addr_t dma_dest; | 48 | dma_addr_t dma_dest; |
@@ -63,23 +60,23 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, | |||
63 | } | 60 | } |
64 | 61 | ||
65 | while (src_cnt) { | 62 | while (src_cnt) { |
66 | async_flags = flags; | 63 | submit->flags = flags_orig; |
67 | dma_flags = 0; | 64 | dma_flags = 0; |
68 | xor_src_cnt = min(src_cnt, dma->max_xor); | 65 | xor_src_cnt = min(src_cnt, (int)dma->max_xor); |
69 | /* if we are submitting additional xors, leave the chain open, | 66 | /* if we are submitting additional xors, leave the chain open, |
70 | * clear the callback parameters, and leave the destination | 67 | * clear the callback parameters, and leave the destination |
71 | * buffer mapped | 68 | * buffer mapped |
72 | */ | 69 | */ |
73 | if (src_cnt > xor_src_cnt) { | 70 | if (src_cnt > xor_src_cnt) { |
74 | async_flags &= ~ASYNC_TX_ACK; | 71 | submit->flags &= ~ASYNC_TX_ACK; |
75 | dma_flags = DMA_COMPL_SKIP_DEST_UNMAP; | 72 | dma_flags = DMA_COMPL_SKIP_DEST_UNMAP; |
76 | _cb_fn = NULL; | 73 | submit->cb_fn = NULL; |
77 | _cb_param = NULL; | 74 | submit->cb_param = NULL; |
78 | } else { | 75 | } else { |
79 | _cb_fn = cb_fn; | 76 | submit->cb_fn = cb_fn_orig; |
80 | _cb_param = cb_param; | 77 | submit->cb_param = cb_param_orig; |
81 | } | 78 | } |
82 | if (_cb_fn) | 79 | if (submit->cb_fn) |
83 | dma_flags |= DMA_PREP_INTERRUPT; | 80 | dma_flags |= DMA_PREP_INTERRUPT; |
84 | 81 | ||
85 | /* Since we have clobbered the src_list we are committed | 82 | /* Since we have clobbered the src_list we are committed |
@@ -90,7 +87,7 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, | |||
90 | xor_src_cnt, len, dma_flags); | 87 | xor_src_cnt, len, dma_flags); |
91 | 88 | ||
92 | if (unlikely(!tx)) | 89 | if (unlikely(!tx)) |
93 | async_tx_quiesce(&depend_tx); | 90 | async_tx_quiesce(&submit->depend_tx); |
94 | 91 | ||
95 | /* spin wait for the preceeding transactions to complete */ | 92 | /* spin wait for the preceeding transactions to complete */ |
96 | while (unlikely(!tx)) { | 93 | while (unlikely(!tx)) { |
@@ -101,11 +98,8 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, | |||
101 | dma_flags); | 98 | dma_flags); |
102 | } | 99 | } |
103 | 100 | ||
104 | async_tx_submit(chan, tx, async_flags, depend_tx, _cb_fn, | 101 | async_tx_submit(chan, tx, submit); |
105 | _cb_param); | 102 | submit->depend_tx = tx; |
106 | |||
107 | depend_tx = tx; | ||
108 | flags |= ASYNC_TX_DEP_ACK; | ||
109 | 103 | ||
110 | if (src_cnt > xor_src_cnt) { | 104 | if (src_cnt > xor_src_cnt) { |
111 | /* drop completed sources */ | 105 | /* drop completed sources */ |
@@ -124,23 +118,27 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, | |||
124 | 118 | ||
125 | static void | 119 | static void |
126 | do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, | 120 | do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, |
127 | int src_cnt, size_t len, enum async_tx_flags flags, | 121 | int src_cnt, size_t len, struct async_submit_ctl *submit) |
128 | dma_async_tx_callback cb_fn, void *cb_param) | ||
129 | { | 122 | { |
130 | int i; | 123 | int i; |
131 | int xor_src_cnt; | 124 | int xor_src_cnt; |
132 | int src_off = 0; | 125 | int src_off = 0; |
133 | void *dest_buf; | 126 | void *dest_buf; |
134 | void **srcs = (void **) src_list; | 127 | void **srcs; |
128 | |||
129 | if (submit->scribble) | ||
130 | srcs = submit->scribble; | ||
131 | else | ||
132 | srcs = (void **) src_list; | ||
135 | 133 | ||
136 | /* reuse the 'src_list' array to convert to buffer pointers */ | 134 | /* convert to buffer pointers */ |
137 | for (i = 0; i < src_cnt; i++) | 135 | for (i = 0; i < src_cnt; i++) |
138 | srcs[i] = page_address(src_list[i]) + offset; | 136 | srcs[i] = page_address(src_list[i]) + offset; |
139 | 137 | ||
140 | /* set destination address */ | 138 | /* set destination address */ |
141 | dest_buf = page_address(dest) + offset; | 139 | dest_buf = page_address(dest) + offset; |
142 | 140 | ||
143 | if (flags & ASYNC_TX_XOR_ZERO_DST) | 141 | if (submit->flags & ASYNC_TX_XOR_ZERO_DST) |
144 | memset(dest_buf, 0, len); | 142 | memset(dest_buf, 0, len); |
145 | 143 | ||
146 | while (src_cnt > 0) { | 144 | while (src_cnt > 0) { |
@@ -153,61 +151,70 @@ do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, | |||
153 | src_off += xor_src_cnt; | 151 | src_off += xor_src_cnt; |
154 | } | 152 | } |
155 | 153 | ||
156 | async_tx_sync_epilog(cb_fn, cb_param); | 154 | async_tx_sync_epilog(submit); |
157 | } | 155 | } |
158 | 156 | ||
159 | /** | 157 | /** |
160 | * async_xor - attempt to xor a set of blocks with a dma engine. | 158 | * async_xor - attempt to xor a set of blocks with a dma engine. |
161 | * xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST | ||
162 | * flag must be set to not include dest data in the calculation. The | ||
163 | * assumption with dma eninges is that they only use the destination | ||
164 | * buffer as a source when it is explicity specified in the source list. | ||
165 | * @dest: destination page | 159 | * @dest: destination page |
166 | * @src_list: array of source pages (if the dest is also a source it must be | 160 | * @src_list: array of source pages |
167 | * at index zero). The contents of this array may be overwritten. | 161 | * @offset: common src/dst offset to start transaction |
168 | * @offset: offset in pages to start transaction | ||
169 | * @src_cnt: number of source pages | 162 | * @src_cnt: number of source pages |
170 | * @len: length in bytes | 163 | * @len: length in bytes |
171 | * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST, | 164 | * @submit: submission / completion modifiers |
172 | * ASYNC_TX_ACK, ASYNC_TX_DEP_ACK | 165 | * |
173 | * @depend_tx: xor depends on the result of this transaction. | 166 | * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST |
174 | * @cb_fn: function to call when the xor completes | 167 | * |
175 | * @cb_param: parameter to pass to the callback routine | 168 | * xor_blocks always uses the dest as a source so the |
169 | * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in | ||
170 | * the calculation. The assumption with dma eninges is that they only | ||
171 | * use the destination buffer as a source when it is explicity specified | ||
172 | * in the source list. | ||
173 | * | ||
174 | * src_list note: if the dest is also a source it must be at index zero. | ||
175 | * The contents of this array will be overwritten if a scribble region | ||
176 | * is not specified. | ||
176 | */ | 177 | */ |
177 | struct dma_async_tx_descriptor * | 178 | struct dma_async_tx_descriptor * |
178 | async_xor(struct page *dest, struct page **src_list, unsigned int offset, | 179 | async_xor(struct page *dest, struct page **src_list, unsigned int offset, |
179 | int src_cnt, size_t len, enum async_tx_flags flags, | 180 | int src_cnt, size_t len, struct async_submit_ctl *submit) |
180 | struct dma_async_tx_descriptor *depend_tx, | ||
181 | dma_async_tx_callback cb_fn, void *cb_param) | ||
182 | { | 181 | { |
183 | struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_XOR, | 182 | struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR, |
184 | &dest, 1, src_list, | 183 | &dest, 1, src_list, |
185 | src_cnt, len); | 184 | src_cnt, len); |
185 | dma_addr_t *dma_src = NULL; | ||
186 | |||
186 | BUG_ON(src_cnt <= 1); | 187 | BUG_ON(src_cnt <= 1); |
187 | 188 | ||
188 | if (chan) { | 189 | if (submit->scribble) |
190 | dma_src = submit->scribble; | ||
191 | else if (sizeof(dma_addr_t) <= sizeof(struct page *)) | ||
192 | dma_src = (dma_addr_t *) src_list; | ||
193 | |||
194 | if (dma_src && chan) { | ||
189 | /* run the xor asynchronously */ | 195 | /* run the xor asynchronously */ |
190 | pr_debug("%s (async): len: %zu\n", __func__, len); | 196 | pr_debug("%s (async): len: %zu\n", __func__, len); |
191 | 197 | ||
192 | return do_async_xor(chan, dest, src_list, offset, src_cnt, len, | 198 | return do_async_xor(chan, dest, src_list, offset, src_cnt, len, |
193 | flags, depend_tx, cb_fn, cb_param); | 199 | dma_src, submit); |
194 | } else { | 200 | } else { |
195 | /* run the xor synchronously */ | 201 | /* run the xor synchronously */ |
196 | pr_debug("%s (sync): len: %zu\n", __func__, len); | 202 | pr_debug("%s (sync): len: %zu\n", __func__, len); |
203 | WARN_ONCE(chan, "%s: no space for dma address conversion\n", | ||
204 | __func__); | ||
197 | 205 | ||
198 | /* in the sync case the dest is an implied source | 206 | /* in the sync case the dest is an implied source |
199 | * (assumes the dest is the first source) | 207 | * (assumes the dest is the first source) |
200 | */ | 208 | */ |
201 | if (flags & ASYNC_TX_XOR_DROP_DST) { | 209 | if (submit->flags & ASYNC_TX_XOR_DROP_DST) { |
202 | src_cnt--; | 210 | src_cnt--; |
203 | src_list++; | 211 | src_list++; |
204 | } | 212 | } |
205 | 213 | ||
206 | /* wait for any prerequisite operations */ | 214 | /* wait for any prerequisite operations */ |
207 | async_tx_quiesce(&depend_tx); | 215 | async_tx_quiesce(&submit->depend_tx); |
208 | 216 | ||
209 | do_sync_xor(dest, src_list, offset, src_cnt, len, | 217 | do_sync_xor(dest, src_list, offset, src_cnt, len, submit); |
210 | flags, cb_fn, cb_param); | ||
211 | 218 | ||
212 | return NULL; | 219 | return NULL; |
213 | } | 220 | } |
@@ -222,104 +229,90 @@ static int page_is_zero(struct page *p, unsigned int offset, size_t len) | |||
222 | } | 229 | } |
223 | 230 | ||
224 | /** | 231 | /** |
225 | * async_xor_zero_sum - attempt a xor parity check with a dma engine. | 232 | * async_xor_val - attempt a xor parity check with a dma engine. |
226 | * @dest: destination page used if the xor is performed synchronously | 233 | * @dest: destination page used if the xor is performed synchronously |
227 | * @src_list: array of source pages. The dest page must be listed as a source | 234 | * @src_list: array of source pages |
228 | * at index zero. The contents of this array may be overwritten. | ||
229 | * @offset: offset in pages to start transaction | 235 | * @offset: offset in pages to start transaction |
230 | * @src_cnt: number of source pages | 236 | * @src_cnt: number of source pages |
231 | * @len: length in bytes | 237 | * @len: length in bytes |
232 | * @result: 0 if sum == 0 else non-zero | 238 | * @result: 0 if sum == 0 else non-zero |
233 | * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK | 239 | * @submit: submission / completion modifiers |
234 | * @depend_tx: xor depends on the result of this transaction. | 240 | * |
235 | * @cb_fn: function to call when the xor completes | 241 | * honored flags: ASYNC_TX_ACK |
236 | * @cb_param: parameter to pass to the callback routine | 242 | * |
243 | * src_list note: if the dest is also a source it must be at index zero. | ||
244 | * The contents of this array will be overwritten if a scribble region | ||
245 | * is not specified. | ||
237 | */ | 246 | */ |
238 | struct dma_async_tx_descriptor * | 247 | struct dma_async_tx_descriptor * |
239 | async_xor_zero_sum(struct page *dest, struct page **src_list, | 248 | async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, |
240 | unsigned int offset, int src_cnt, size_t len, | 249 | int src_cnt, size_t len, enum sum_check_flags *result, |
241 | u32 *result, enum async_tx_flags flags, | 250 | struct async_submit_ctl *submit) |
242 | struct dma_async_tx_descriptor *depend_tx, | ||
243 | dma_async_tx_callback cb_fn, void *cb_param) | ||
244 | { | 251 | { |
245 | struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_ZERO_SUM, | 252 | struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR_VAL, |
246 | &dest, 1, src_list, | 253 | &dest, 1, src_list, |
247 | src_cnt, len); | 254 | src_cnt, len); |
248 | struct dma_device *device = chan ? chan->device : NULL; | 255 | struct dma_device *device = chan ? chan->device : NULL; |
249 | struct dma_async_tx_descriptor *tx = NULL; | 256 | struct dma_async_tx_descriptor *tx = NULL; |
257 | dma_addr_t *dma_src = NULL; | ||
250 | 258 | ||
251 | BUG_ON(src_cnt <= 1); | 259 | BUG_ON(src_cnt <= 1); |
252 | 260 | ||
253 | if (device && src_cnt <= device->max_xor) { | 261 | if (submit->scribble) |
254 | dma_addr_t *dma_src = (dma_addr_t *) src_list; | 262 | dma_src = submit->scribble; |
255 | unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; | 263 | else if (sizeof(dma_addr_t) <= sizeof(struct page *)) |
264 | dma_src = (dma_addr_t *) src_list; | ||
265 | |||
266 | if (dma_src && device && src_cnt <= device->max_xor) { | ||
267 | unsigned long dma_prep_flags; | ||
256 | int i; | 268 | int i; |
257 | 269 | ||
258 | pr_debug("%s: (async) len: %zu\n", __func__, len); | 270 | pr_debug("%s: (async) len: %zu\n", __func__, len); |
259 | 271 | ||
272 | dma_prep_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0; | ||
260 | for (i = 0; i < src_cnt; i++) | 273 | for (i = 0; i < src_cnt; i++) |
261 | dma_src[i] = dma_map_page(device->dev, src_list[i], | 274 | dma_src[i] = dma_map_page(device->dev, src_list[i], |
262 | offset, len, DMA_TO_DEVICE); | 275 | offset, len, DMA_TO_DEVICE); |
263 | 276 | ||
264 | tx = device->device_prep_dma_zero_sum(chan, dma_src, src_cnt, | 277 | tx = device->device_prep_dma_xor_val(chan, dma_src, src_cnt, |
265 | len, result, | 278 | len, result, |
266 | dma_prep_flags); | 279 | dma_prep_flags); |
267 | if (unlikely(!tx)) { | 280 | if (unlikely(!tx)) { |
268 | async_tx_quiesce(&depend_tx); | 281 | async_tx_quiesce(&submit->depend_tx); |
269 | 282 | ||
270 | while (!tx) { | 283 | while (!tx) { |
271 | dma_async_issue_pending(chan); | 284 | dma_async_issue_pending(chan); |
272 | tx = device->device_prep_dma_zero_sum(chan, | 285 | tx = device->device_prep_dma_xor_val(chan, |
273 | dma_src, src_cnt, len, result, | 286 | dma_src, src_cnt, len, result, |
274 | dma_prep_flags); | 287 | dma_prep_flags); |
275 | } | 288 | } |
276 | } | 289 | } |
277 | 290 | ||
278 | async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); | 291 | async_tx_submit(chan, tx, submit); |
279 | } else { | 292 | } else { |
280 | unsigned long xor_flags = flags; | 293 | enum async_tx_flags flags_orig = submit->flags; |
281 | 294 | ||
282 | pr_debug("%s: (sync) len: %zu\n", __func__, len); | 295 | pr_debug("%s: (sync) len: %zu\n", __func__, len); |
296 | WARN_ONCE(device && src_cnt <= device->max_xor, | ||
297 | "%s: no space for dma address conversion\n", | ||
298 | __func__); | ||
283 | 299 | ||
284 | xor_flags |= ASYNC_TX_XOR_DROP_DST; | 300 | submit->flags |= ASYNC_TX_XOR_DROP_DST; |
285 | xor_flags &= ~ASYNC_TX_ACK; | 301 | submit->flags &= ~ASYNC_TX_ACK; |
286 | 302 | ||
287 | tx = async_xor(dest, src_list, offset, src_cnt, len, xor_flags, | 303 | tx = async_xor(dest, src_list, offset, src_cnt, len, submit); |
288 | depend_tx, NULL, NULL); | ||
289 | 304 | ||
290 | async_tx_quiesce(&tx); | 305 | async_tx_quiesce(&tx); |
291 | 306 | ||
292 | *result = page_is_zero(dest, offset, len) ? 0 : 1; | 307 | *result = !page_is_zero(dest, offset, len) << SUM_CHECK_P; |
293 | 308 | ||
294 | async_tx_sync_epilog(cb_fn, cb_param); | 309 | async_tx_sync_epilog(submit); |
310 | submit->flags = flags_orig; | ||
295 | } | 311 | } |
296 | 312 | ||
297 | return tx; | 313 | return tx; |
298 | } | 314 | } |
299 | EXPORT_SYMBOL_GPL(async_xor_zero_sum); | 315 | EXPORT_SYMBOL_GPL(async_xor_val); |
300 | |||
301 | static int __init async_xor_init(void) | ||
302 | { | ||
303 | #ifdef CONFIG_DMA_ENGINE | ||
304 | /* To conserve stack space the input src_list (array of page pointers) | ||
305 | * is reused to hold the array of dma addresses passed to the driver. | ||
306 | * This conversion is only possible when dma_addr_t is less than the | ||
307 | * the size of a pointer. HIGHMEM64G is known to violate this | ||
308 | * assumption. | ||
309 | */ | ||
310 | BUILD_BUG_ON(sizeof(dma_addr_t) > sizeof(struct page *)); | ||
311 | #endif | ||
312 | |||
313 | return 0; | ||
314 | } | ||
315 | |||
316 | static void __exit async_xor_exit(void) | ||
317 | { | ||
318 | do { } while (0); | ||
319 | } | ||
320 | |||
321 | module_init(async_xor_init); | ||
322 | module_exit(async_xor_exit); | ||
323 | 316 | ||
324 | MODULE_AUTHOR("Intel Corporation"); | 317 | MODULE_AUTHOR("Intel Corporation"); |
325 | MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api"); | 318 | MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api"); |