aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/dmatest.txt72
-rw-r--r--arch/arm/include/asm/hardware/iop3xx-adma.h30
-rw-r--r--arch/arm/include/asm/hardware/iop_adma.h4
-rw-r--r--arch/arm/mach-iop13xx/include/mach/adma.h26
-rw-r--r--crypto/async_tx/async_memcpy.c37
-rw-r--r--crypto/async_tx/async_pq.c174
-rw-r--r--crypto/async_tx/async_raid6_recov.c61
-rw-r--r--crypto/async_tx/async_xor.c123
-rw-r--r--crypto/async_tx/raid6test.c10
-rw-r--r--drivers/ata/pata_arasan_cf.c3
-rw-r--r--drivers/dma/amba-pl08x.c32
-rw-r--r--drivers/dma/at_hdmac.c26
-rw-r--r--drivers/dma/dmaengine.c262
-rw-r--r--drivers/dma/dmatest.c915
-rw-r--r--drivers/dma/dw/core.c25
-rw-r--r--drivers/dma/ep93xx_dma.c30
-rw-r--r--drivers/dma/fsldma.c17
-rw-r--r--drivers/dma/ioat/dma.c49
-rw-r--r--drivers/dma/ioat/dma.h14
-rw-r--r--drivers/dma/ioat/dma_v2.c2
-rw-r--r--drivers/dma/ioat/dma_v2.h1
-rw-r--r--drivers/dma/ioat/dma_v3.c315
-rw-r--r--drivers/dma/ioat/pci.c20
-rw-r--r--drivers/dma/iop-adma.c97
-rw-r--r--drivers/dma/mv_xor.c52
-rw-r--r--drivers/dma/mv_xor.h25
-rw-r--r--drivers/dma/pl330.c2
-rw-r--r--drivers/dma/ppc4xx/adma.c270
-rw-r--r--drivers/dma/timb_dma.c37
-rw-r--r--drivers/dma/txx9dmac.c25
-rw-r--r--drivers/media/platform/m2m-deinterlace.c3
-rw-r--r--drivers/media/platform/timblogiw.c2
-rw-r--r--drivers/misc/carma/carma-fpga.c3
-rw-r--r--drivers/mtd/nand/atmel_nand.c3
-rw-r--r--drivers/mtd/nand/fsmc_nand.c2
-rw-r--r--drivers/net/ethernet/micrel/ks8842.c6
-rw-r--r--drivers/ntb/ntb_transport.c86
-rw-r--r--drivers/spi/spi-dw-mid.c4
-rw-r--r--include/linux/dmaengine.h64
39 files changed, 1018 insertions, 1911 deletions
diff --git a/Documentation/dmatest.txt b/Documentation/dmatest.txt
index a2b5663eae26..dd77a81bdb80 100644
--- a/Documentation/dmatest.txt
+++ b/Documentation/dmatest.txt
@@ -15,39 +15,48 @@ be built as module or inside kernel. Let's consider those cases.
15 15
16 Part 2 - When dmatest is built as a module... 16 Part 2 - When dmatest is built as a module...
17 17
18After mounting debugfs and loading the module, the /sys/kernel/debug/dmatest
19folder with nodes will be created. There are two important files located. First
20is the 'run' node that controls run and stop phases of the test, and the second
21one, 'results', is used to get the test case results.
22
23Note that in this case test will not run on load automatically.
24
25Example of usage: 18Example of usage:
19 % modprobe dmatest channel=dma0chan0 timeout=2000 iterations=1 run=1
20
21...or:
22 % modprobe dmatest
26 % echo dma0chan0 > /sys/module/dmatest/parameters/channel 23 % echo dma0chan0 > /sys/module/dmatest/parameters/channel
27 % echo 2000 > /sys/module/dmatest/parameters/timeout 24 % echo 2000 > /sys/module/dmatest/parameters/timeout
28 % echo 1 > /sys/module/dmatest/parameters/iterations 25 % echo 1 > /sys/module/dmatest/parameters/iterations
29 % echo 1 > /sys/kernel/debug/dmatest/run 26 % echo 1 > /sys/module/dmatest/parameters/run
27
28...or on the kernel command line:
29
30 dmatest.channel=dma0chan0 dmatest.timeout=2000 dmatest.iterations=1 dmatest.run=1
30 31
31Hint: available channel list could be extracted by running the following 32Hint: available channel list could be extracted by running the following
32command: 33command:
33 % ls -1 /sys/class/dma/ 34 % ls -1 /sys/class/dma/
34 35
35After a while you will start to get messages about current status or error like 36Once started a message like "dmatest: Started 1 threads using dma0chan0" is
36in the original code. 37emitted. After that only test failure messages are reported until the test
38stops.
37 39
38Note that running a new test will not stop any in progress test. 40Note that running a new test will not stop any in progress test.
39 41
40The following command should return actual state of the test. 42The following command returns the state of the test.
41 % cat /sys/kernel/debug/dmatest/run 43 % cat /sys/module/dmatest/parameters/run
42 44
43To wait for test done the user may perform a busy loop that checks the state. 45To wait for test completion userpace can poll 'run' until it is false, or use
44 46the wait parameter. Specifying 'wait=1' when loading the module causes module
45 % while [ $(cat /sys/kernel/debug/dmatest/run) = "Y" ] 47initialization to pause until a test run has completed, while reading
46 > do 48/sys/module/dmatest/parameters/wait waits for any running test to complete
47 > echo -n "." 49before returning. For example, the following scripts wait for 42 tests
48 > sleep 1 50to complete before exiting. Note that if 'iterations' is set to 'infinite' then
49 > done 51waiting is disabled.
50 > echo 52
53Example:
54 % modprobe dmatest run=1 iterations=42 wait=1
55 % modprobe -r dmatest
56...or:
57 % modprobe dmatest run=1 iterations=42
58 % cat /sys/module/dmatest/parameters/wait
59 % modprobe -r dmatest
51 60
52 Part 3 - When built-in in the kernel... 61 Part 3 - When built-in in the kernel...
53 62
@@ -62,21 +71,22 @@ case. You always could check them at run-time by running
62 71
63 Part 4 - Gathering the test results 72 Part 4 - Gathering the test results
64 73
65The module provides a storage for the test results in the memory. The gathered 74Test results are printed to the kernel log buffer with the format:
66data could be used after test is done.
67 75
68The special file 'results' in the debugfs represents gathered data of the in 76"dmatest: result <channel>: <test id>: '<error msg>' with src_off=<val> dst_off=<val> len=<val> (<err code>)"
69progress test. The messages collected are printed to the kernel log as well.
70 77
71Example of output: 78Example of output:
72 % cat /sys/kernel/debug/dmatest/results 79 % dmesg | tail -n 1
73 dma0chan0-copy0: #1: No errors with src_off=0x7bf dst_off=0x8ad len=0x3fea (0) 80 dmatest: result dma0chan0-copy0: #1: No errors with src_off=0x7bf dst_off=0x8ad len=0x3fea (0)
74 81
75The message format is unified across the different types of errors. A number in 82The message format is unified across the different types of errors. A number in
76the parens represents additional information, e.g. error code, error counter, 83the parens represents additional information, e.g. error code, error counter,
77or status. 84or status. A test thread also emits a summary line at completion listing the
85number of tests executed, number that failed, and a result code.
78 86
79Comparison between buffers is stored to the dedicated structure. 87Example:
88 % dmesg | tail -n 1
89 dmatest: dma0chan0-copy0: summary 1 test, 0 failures 1000 iops 100000 KB/s (0)
80 90
81Note that the verify result is now accessible only via file 'results' in the 91The details of a data miscompare error are also emitted, but do not follow the
82debugfs. 92above format.
diff --git a/arch/arm/include/asm/hardware/iop3xx-adma.h b/arch/arm/include/asm/hardware/iop3xx-adma.h
index 9b28f1243bdc..240b29ef17db 100644
--- a/arch/arm/include/asm/hardware/iop3xx-adma.h
+++ b/arch/arm/include/asm/hardware/iop3xx-adma.h
@@ -393,36 +393,6 @@ static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt,
393 return slot_cnt; 393 return slot_cnt;
394} 394}
395 395
396static inline int iop_desc_is_pq(struct iop_adma_desc_slot *desc)
397{
398 return 0;
399}
400
401static inline u32 iop_desc_get_dest_addr(struct iop_adma_desc_slot *desc,
402 struct iop_adma_chan *chan)
403{
404 union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
405
406 switch (chan->device->id) {
407 case DMA0_ID:
408 case DMA1_ID:
409 return hw_desc.dma->dest_addr;
410 case AAU_ID:
411 return hw_desc.aau->dest_addr;
412 default:
413 BUG();
414 }
415 return 0;
416}
417
418
419static inline u32 iop_desc_get_qdest_addr(struct iop_adma_desc_slot *desc,
420 struct iop_adma_chan *chan)
421{
422 BUG();
423 return 0;
424}
425
426static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc, 396static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc,
427 struct iop_adma_chan *chan) 397 struct iop_adma_chan *chan)
428{ 398{
diff --git a/arch/arm/include/asm/hardware/iop_adma.h b/arch/arm/include/asm/hardware/iop_adma.h
index 122f86d8c991..250760e08103 100644
--- a/arch/arm/include/asm/hardware/iop_adma.h
+++ b/arch/arm/include/asm/hardware/iop_adma.h
@@ -82,8 +82,6 @@ struct iop_adma_chan {
82 * @slot_cnt: total slots used in an transaction (group of operations) 82 * @slot_cnt: total slots used in an transaction (group of operations)
83 * @slots_per_op: number of slots per operation 83 * @slots_per_op: number of slots per operation
84 * @idx: pool index 84 * @idx: pool index
85 * @unmap_src_cnt: number of xor sources
86 * @unmap_len: transaction bytecount
87 * @tx_list: list of descriptors that are associated with one operation 85 * @tx_list: list of descriptors that are associated with one operation
88 * @async_tx: support for the async_tx api 86 * @async_tx: support for the async_tx api
89 * @group_list: list of slots that make up a multi-descriptor transaction 87 * @group_list: list of slots that make up a multi-descriptor transaction
@@ -99,8 +97,6 @@ struct iop_adma_desc_slot {
99 u16 slot_cnt; 97 u16 slot_cnt;
100 u16 slots_per_op; 98 u16 slots_per_op;
101 u16 idx; 99 u16 idx;
102 u16 unmap_src_cnt;
103 size_t unmap_len;
104 struct list_head tx_list; 100 struct list_head tx_list;
105 struct dma_async_tx_descriptor async_tx; 101 struct dma_async_tx_descriptor async_tx;
106 union { 102 union {
diff --git a/arch/arm/mach-iop13xx/include/mach/adma.h b/arch/arm/mach-iop13xx/include/mach/adma.h
index 6d3782d85a9f..a86fd0ed7757 100644
--- a/arch/arm/mach-iop13xx/include/mach/adma.h
+++ b/arch/arm/mach-iop13xx/include/mach/adma.h
@@ -218,20 +218,6 @@ iop_chan_xor_slot_count(size_t len, int src_cnt, int *slots_per_op)
218#define iop_chan_pq_slot_count iop_chan_xor_slot_count 218#define iop_chan_pq_slot_count iop_chan_xor_slot_count
219#define iop_chan_pq_zero_sum_slot_count iop_chan_xor_slot_count 219#define iop_chan_pq_zero_sum_slot_count iop_chan_xor_slot_count
220 220
221static inline u32 iop_desc_get_dest_addr(struct iop_adma_desc_slot *desc,
222 struct iop_adma_chan *chan)
223{
224 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
225 return hw_desc->dest_addr;
226}
227
228static inline u32 iop_desc_get_qdest_addr(struct iop_adma_desc_slot *desc,
229 struct iop_adma_chan *chan)
230{
231 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
232 return hw_desc->q_dest_addr;
233}
234
235static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc, 221static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc,
236 struct iop_adma_chan *chan) 222 struct iop_adma_chan *chan)
237{ 223{
@@ -350,18 +336,6 @@ iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt,
350 hw_desc->desc_ctrl = u_desc_ctrl.value; 336 hw_desc->desc_ctrl = u_desc_ctrl.value;
351} 337}
352 338
353static inline int iop_desc_is_pq(struct iop_adma_desc_slot *desc)
354{
355 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
356 union {
357 u32 value;
358 struct iop13xx_adma_desc_ctrl field;
359 } u_desc_ctrl;
360
361 u_desc_ctrl.value = hw_desc->desc_ctrl;
362 return u_desc_ctrl.field.pq_xfer_en;
363}
364
365static inline void 339static inline void
366iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt, 340iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
367 unsigned long flags) 341 unsigned long flags)
diff --git a/crypto/async_tx/async_memcpy.c b/crypto/async_tx/async_memcpy.c
index 9e62feffb374..f8c0b8dbeb75 100644
--- a/crypto/async_tx/async_memcpy.c
+++ b/crypto/async_tx/async_memcpy.c
@@ -50,33 +50,36 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
50 &dest, 1, &src, 1, len); 50 &dest, 1, &src, 1, len);
51 struct dma_device *device = chan ? chan->device : NULL; 51 struct dma_device *device = chan ? chan->device : NULL;
52 struct dma_async_tx_descriptor *tx = NULL; 52 struct dma_async_tx_descriptor *tx = NULL;
53 struct dmaengine_unmap_data *unmap = NULL;
53 54
54 if (device && is_dma_copy_aligned(device, src_offset, dest_offset, len)) { 55 if (device)
55 dma_addr_t dma_dest, dma_src; 56 unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOIO);
57
58 if (unmap && is_dma_copy_aligned(device, src_offset, dest_offset, len)) {
56 unsigned long dma_prep_flags = 0; 59 unsigned long dma_prep_flags = 0;
57 60
58 if (submit->cb_fn) 61 if (submit->cb_fn)
59 dma_prep_flags |= DMA_PREP_INTERRUPT; 62 dma_prep_flags |= DMA_PREP_INTERRUPT;
60 if (submit->flags & ASYNC_TX_FENCE) 63 if (submit->flags & ASYNC_TX_FENCE)
61 dma_prep_flags |= DMA_PREP_FENCE; 64 dma_prep_flags |= DMA_PREP_FENCE;
62 dma_dest = dma_map_page(device->dev, dest, dest_offset, len, 65
63 DMA_FROM_DEVICE); 66 unmap->to_cnt = 1;
64 67 unmap->addr[0] = dma_map_page(device->dev, src, src_offset, len,
65 dma_src = dma_map_page(device->dev, src, src_offset, len, 68 DMA_TO_DEVICE);
66 DMA_TO_DEVICE); 69 unmap->from_cnt = 1;
67 70 unmap->addr[1] = dma_map_page(device->dev, dest, dest_offset, len,
68 tx = device->device_prep_dma_memcpy(chan, dma_dest, dma_src, 71 DMA_FROM_DEVICE);
69 len, dma_prep_flags); 72 unmap->len = len;
70 if (!tx) { 73
71 dma_unmap_page(device->dev, dma_dest, len, 74 tx = device->device_prep_dma_memcpy(chan, unmap->addr[1],
72 DMA_FROM_DEVICE); 75 unmap->addr[0], len,
73 dma_unmap_page(device->dev, dma_src, len, 76 dma_prep_flags);
74 DMA_TO_DEVICE);
75 }
76 } 77 }
77 78
78 if (tx) { 79 if (tx) {
79 pr_debug("%s: (async) len: %zu\n", __func__, len); 80 pr_debug("%s: (async) len: %zu\n", __func__, len);
81
82 dma_set_unmap(tx, unmap);
80 async_tx_submit(chan, tx, submit); 83 async_tx_submit(chan, tx, submit);
81 } else { 84 } else {
82 void *dest_buf, *src_buf; 85 void *dest_buf, *src_buf;
@@ -96,6 +99,8 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
96 async_tx_sync_epilog(submit); 99 async_tx_sync_epilog(submit);
97 } 100 }
98 101
102 dmaengine_unmap_put(unmap);
103
99 return tx; 104 return tx;
100} 105}
101EXPORT_SYMBOL_GPL(async_memcpy); 106EXPORT_SYMBOL_GPL(async_memcpy);
diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c
index 91d5d385899e..d05327caf69d 100644
--- a/crypto/async_tx/async_pq.c
+++ b/crypto/async_tx/async_pq.c
@@ -46,49 +46,24 @@ static struct page *pq_scribble_page;
46 * do_async_gen_syndrome - asynchronously calculate P and/or Q 46 * do_async_gen_syndrome - asynchronously calculate P and/or Q
47 */ 47 */
48static __async_inline struct dma_async_tx_descriptor * 48static __async_inline struct dma_async_tx_descriptor *
49do_async_gen_syndrome(struct dma_chan *chan, struct page **blocks, 49do_async_gen_syndrome(struct dma_chan *chan,
50 const unsigned char *scfs, unsigned int offset, int disks, 50 const unsigned char *scfs, int disks,
51 size_t len, dma_addr_t *dma_src, 51 struct dmaengine_unmap_data *unmap,
52 enum dma_ctrl_flags dma_flags,
52 struct async_submit_ctl *submit) 53 struct async_submit_ctl *submit)
53{ 54{
54 struct dma_async_tx_descriptor *tx = NULL; 55 struct dma_async_tx_descriptor *tx = NULL;
55 struct dma_device *dma = chan->device; 56 struct dma_device *dma = chan->device;
56 enum dma_ctrl_flags dma_flags = 0;
57 enum async_tx_flags flags_orig = submit->flags; 57 enum async_tx_flags flags_orig = submit->flags;
58 dma_async_tx_callback cb_fn_orig = submit->cb_fn; 58 dma_async_tx_callback cb_fn_orig = submit->cb_fn;
59 dma_async_tx_callback cb_param_orig = submit->cb_param; 59 dma_async_tx_callback cb_param_orig = submit->cb_param;
60 int src_cnt = disks - 2; 60 int src_cnt = disks - 2;
61 unsigned char coefs[src_cnt];
62 unsigned short pq_src_cnt; 61 unsigned short pq_src_cnt;
63 dma_addr_t dma_dest[2]; 62 dma_addr_t dma_dest[2];
64 int src_off = 0; 63 int src_off = 0;
65 int idx;
66 int i;
67 64
68 /* DMAs use destinations as sources, so use BIDIRECTIONAL mapping */ 65 if (submit->flags & ASYNC_TX_FENCE)
69 if (P(blocks, disks)) 66 dma_flags |= DMA_PREP_FENCE;
70 dma_dest[0] = dma_map_page(dma->dev, P(blocks, disks), offset,
71 len, DMA_BIDIRECTIONAL);
72 else
73 dma_flags |= DMA_PREP_PQ_DISABLE_P;
74 if (Q(blocks, disks))
75 dma_dest[1] = dma_map_page(dma->dev, Q(blocks, disks), offset,
76 len, DMA_BIDIRECTIONAL);
77 else
78 dma_flags |= DMA_PREP_PQ_DISABLE_Q;
79
80 /* convert source addresses being careful to collapse 'empty'
81 * sources and update the coefficients accordingly
82 */
83 for (i = 0, idx = 0; i < src_cnt; i++) {
84 if (blocks[i] == NULL)
85 continue;
86 dma_src[idx] = dma_map_page(dma->dev, blocks[i], offset, len,
87 DMA_TO_DEVICE);
88 coefs[idx] = scfs[i];
89 idx++;
90 }
91 src_cnt = idx;
92 67
93 while (src_cnt > 0) { 68 while (src_cnt > 0) {
94 submit->flags = flags_orig; 69 submit->flags = flags_orig;
@@ -100,28 +75,25 @@ do_async_gen_syndrome(struct dma_chan *chan, struct page **blocks,
100 if (src_cnt > pq_src_cnt) { 75 if (src_cnt > pq_src_cnt) {
101 submit->flags &= ~ASYNC_TX_ACK; 76 submit->flags &= ~ASYNC_TX_ACK;
102 submit->flags |= ASYNC_TX_FENCE; 77 submit->flags |= ASYNC_TX_FENCE;
103 dma_flags |= DMA_COMPL_SKIP_DEST_UNMAP;
104 submit->cb_fn = NULL; 78 submit->cb_fn = NULL;
105 submit->cb_param = NULL; 79 submit->cb_param = NULL;
106 } else { 80 } else {
107 dma_flags &= ~DMA_COMPL_SKIP_DEST_UNMAP;
108 submit->cb_fn = cb_fn_orig; 81 submit->cb_fn = cb_fn_orig;
109 submit->cb_param = cb_param_orig; 82 submit->cb_param = cb_param_orig;
110 if (cb_fn_orig) 83 if (cb_fn_orig)
111 dma_flags |= DMA_PREP_INTERRUPT; 84 dma_flags |= DMA_PREP_INTERRUPT;
112 } 85 }
113 if (submit->flags & ASYNC_TX_FENCE)
114 dma_flags |= DMA_PREP_FENCE;
115 86
116 /* Since we have clobbered the src_list we are committed 87 /* Drivers force forward progress in case they can not provide
117 * to doing this asynchronously. Drivers force forward 88 * a descriptor
118 * progress in case they can not provide a descriptor
119 */ 89 */
120 for (;;) { 90 for (;;) {
91 dma_dest[0] = unmap->addr[disks - 2];
92 dma_dest[1] = unmap->addr[disks - 1];
121 tx = dma->device_prep_dma_pq(chan, dma_dest, 93 tx = dma->device_prep_dma_pq(chan, dma_dest,
122 &dma_src[src_off], 94 &unmap->addr[src_off],
123 pq_src_cnt, 95 pq_src_cnt,
124 &coefs[src_off], len, 96 &scfs[src_off], unmap->len,
125 dma_flags); 97 dma_flags);
126 if (likely(tx)) 98 if (likely(tx))
127 break; 99 break;
@@ -129,6 +101,7 @@ do_async_gen_syndrome(struct dma_chan *chan, struct page **blocks,
129 dma_async_issue_pending(chan); 101 dma_async_issue_pending(chan);
130 } 102 }
131 103
104 dma_set_unmap(tx, unmap);
132 async_tx_submit(chan, tx, submit); 105 async_tx_submit(chan, tx, submit);
133 submit->depend_tx = tx; 106 submit->depend_tx = tx;
134 107
@@ -188,10 +161,6 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
188 * set to NULL those buffers will be replaced with the raid6_zero_page 161 * set to NULL those buffers will be replaced with the raid6_zero_page
189 * in the synchronous path and omitted in the hardware-asynchronous 162 * in the synchronous path and omitted in the hardware-asynchronous
190 * path. 163 * path.
191 *
192 * 'blocks' note: if submit->scribble is NULL then the contents of
193 * 'blocks' may be overwritten to perform address conversions
194 * (dma_map_page() or page_address()).
195 */ 164 */
196struct dma_async_tx_descriptor * 165struct dma_async_tx_descriptor *
197async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, 166async_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
@@ -202,26 +171,69 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
202 &P(blocks, disks), 2, 171 &P(blocks, disks), 2,
203 blocks, src_cnt, len); 172 blocks, src_cnt, len);
204 struct dma_device *device = chan ? chan->device : NULL; 173 struct dma_device *device = chan ? chan->device : NULL;
205 dma_addr_t *dma_src = NULL; 174 struct dmaengine_unmap_data *unmap = NULL;
206 175
207 BUG_ON(disks > 255 || !(P(blocks, disks) || Q(blocks, disks))); 176 BUG_ON(disks > 255 || !(P(blocks, disks) || Q(blocks, disks)));
208 177
209 if (submit->scribble) 178 if (device)
210 dma_src = submit->scribble; 179 unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOIO);
211 else if (sizeof(dma_addr_t) <= sizeof(struct page *))
212 dma_src = (dma_addr_t *) blocks;
213 180
214 if (dma_src && device && 181 if (unmap &&
215 (src_cnt <= dma_maxpq(device, 0) || 182 (src_cnt <= dma_maxpq(device, 0) ||
216 dma_maxpq(device, DMA_PREP_CONTINUE) > 0) && 183 dma_maxpq(device, DMA_PREP_CONTINUE) > 0) &&
217 is_dma_pq_aligned(device, offset, 0, len)) { 184 is_dma_pq_aligned(device, offset, 0, len)) {
185 struct dma_async_tx_descriptor *tx;
186 enum dma_ctrl_flags dma_flags = 0;
187 unsigned char coefs[src_cnt];
188 int i, j;
189
218 /* run the p+q asynchronously */ 190 /* run the p+q asynchronously */
219 pr_debug("%s: (async) disks: %d len: %zu\n", 191 pr_debug("%s: (async) disks: %d len: %zu\n",
220 __func__, disks, len); 192 __func__, disks, len);
221 return do_async_gen_syndrome(chan, blocks, raid6_gfexp, offset, 193
222 disks, len, dma_src, submit); 194 /* convert source addresses being careful to collapse 'empty'
195 * sources and update the coefficients accordingly
196 */
197 unmap->len = len;
198 for (i = 0, j = 0; i < src_cnt; i++) {
199 if (blocks[i] == NULL)
200 continue;
201 unmap->addr[j] = dma_map_page(device->dev, blocks[i], offset,
202 len, DMA_TO_DEVICE);
203 coefs[j] = raid6_gfexp[i];
204 unmap->to_cnt++;
205 j++;
206 }
207
208 /*
209 * DMAs use destinations as sources,
210 * so use BIDIRECTIONAL mapping
211 */
212 unmap->bidi_cnt++;
213 if (P(blocks, disks))
214 unmap->addr[j++] = dma_map_page(device->dev, P(blocks, disks),
215 offset, len, DMA_BIDIRECTIONAL);
216 else {
217 unmap->addr[j++] = 0;
218 dma_flags |= DMA_PREP_PQ_DISABLE_P;
219 }
220
221 unmap->bidi_cnt++;
222 if (Q(blocks, disks))
223 unmap->addr[j++] = dma_map_page(device->dev, Q(blocks, disks),
224 offset, len, DMA_BIDIRECTIONAL);
225 else {
226 unmap->addr[j++] = 0;
227 dma_flags |= DMA_PREP_PQ_DISABLE_Q;
228 }
229
230 tx = do_async_gen_syndrome(chan, coefs, j, unmap, dma_flags, submit);
231 dmaengine_unmap_put(unmap);
232 return tx;
223 } 233 }
224 234
235 dmaengine_unmap_put(unmap);
236
225 /* run the pq synchronously */ 237 /* run the pq synchronously */
226 pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len); 238 pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len);
227 239
@@ -277,50 +289,60 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks,
277 struct dma_async_tx_descriptor *tx; 289 struct dma_async_tx_descriptor *tx;
278 unsigned char coefs[disks-2]; 290 unsigned char coefs[disks-2];
279 enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0; 291 enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0;
280 dma_addr_t *dma_src = NULL; 292 struct dmaengine_unmap_data *unmap = NULL;
281 int src_cnt = 0;
282 293
283 BUG_ON(disks < 4); 294 BUG_ON(disks < 4);
284 295
285 if (submit->scribble) 296 if (device)
286 dma_src = submit->scribble; 297 unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOIO);
287 else if (sizeof(dma_addr_t) <= sizeof(struct page *))
288 dma_src = (dma_addr_t *) blocks;
289 298
290 if (dma_src && device && disks <= dma_maxpq(device, 0) && 299 if (unmap && disks <= dma_maxpq(device, 0) &&
291 is_dma_pq_aligned(device, offset, 0, len)) { 300 is_dma_pq_aligned(device, offset, 0, len)) {
292 struct device *dev = device->dev; 301 struct device *dev = device->dev;
293 dma_addr_t *pq = &dma_src[disks-2]; 302 dma_addr_t pq[2];
294 int i; 303 int i, j = 0, src_cnt = 0;
295 304
296 pr_debug("%s: (async) disks: %d len: %zu\n", 305 pr_debug("%s: (async) disks: %d len: %zu\n",
297 __func__, disks, len); 306 __func__, disks, len);
298 if (!P(blocks, disks)) 307
308 unmap->len = len;
309 for (i = 0; i < disks-2; i++)
310 if (likely(blocks[i])) {
311 unmap->addr[j] = dma_map_page(dev, blocks[i],
312 offset, len,
313 DMA_TO_DEVICE);
314 coefs[j] = raid6_gfexp[i];
315 unmap->to_cnt++;
316 src_cnt++;
317 j++;
318 }
319
320 if (!P(blocks, disks)) {
321 pq[0] = 0;
299 dma_flags |= DMA_PREP_PQ_DISABLE_P; 322 dma_flags |= DMA_PREP_PQ_DISABLE_P;
300 else 323 } else {
301 pq[0] = dma_map_page(dev, P(blocks, disks), 324 pq[0] = dma_map_page(dev, P(blocks, disks),
302 offset, len, 325 offset, len,
303 DMA_TO_DEVICE); 326 DMA_TO_DEVICE);
304 if (!Q(blocks, disks)) 327 unmap->addr[j++] = pq[0];
328 unmap->to_cnt++;
329 }
330 if (!Q(blocks, disks)) {
331 pq[1] = 0;
305 dma_flags |= DMA_PREP_PQ_DISABLE_Q; 332 dma_flags |= DMA_PREP_PQ_DISABLE_Q;
306 else 333 } else {
307 pq[1] = dma_map_page(dev, Q(blocks, disks), 334 pq[1] = dma_map_page(dev, Q(blocks, disks),
308 offset, len, 335 offset, len,
309 DMA_TO_DEVICE); 336 DMA_TO_DEVICE);
337 unmap->addr[j++] = pq[1];
338 unmap->to_cnt++;
339 }
310 340
311 if (submit->flags & ASYNC_TX_FENCE) 341 if (submit->flags & ASYNC_TX_FENCE)
312 dma_flags |= DMA_PREP_FENCE; 342 dma_flags |= DMA_PREP_FENCE;
313 for (i = 0; i < disks-2; i++)
314 if (likely(blocks[i])) {
315 dma_src[src_cnt] = dma_map_page(dev, blocks[i],
316 offset, len,
317 DMA_TO_DEVICE);
318 coefs[src_cnt] = raid6_gfexp[i];
319 src_cnt++;
320 }
321
322 for (;;) { 343 for (;;) {
323 tx = device->device_prep_dma_pq_val(chan, pq, dma_src, 344 tx = device->device_prep_dma_pq_val(chan, pq,
345 unmap->addr,
324 src_cnt, 346 src_cnt,
325 coefs, 347 coefs,
326 len, pqres, 348 len, pqres,
@@ -330,6 +352,8 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks,
330 async_tx_quiesce(&submit->depend_tx); 352 async_tx_quiesce(&submit->depend_tx);
331 dma_async_issue_pending(chan); 353 dma_async_issue_pending(chan);
332 } 354 }
355
356 dma_set_unmap(tx, unmap);
333 async_tx_submit(chan, tx, submit); 357 async_tx_submit(chan, tx, submit);
334 358
335 return tx; 359 return tx;
diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c
index a9f08a6a582e..934a84981495 100644
--- a/crypto/async_tx/async_raid6_recov.c
+++ b/crypto/async_tx/async_raid6_recov.c
@@ -26,6 +26,7 @@
26#include <linux/dma-mapping.h> 26#include <linux/dma-mapping.h>
27#include <linux/raid/pq.h> 27#include <linux/raid/pq.h>
28#include <linux/async_tx.h> 28#include <linux/async_tx.h>
29#include <linux/dmaengine.h>
29 30
30static struct dma_async_tx_descriptor * 31static struct dma_async_tx_descriptor *
31async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef, 32async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
@@ -34,35 +35,45 @@ async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
34 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, 35 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
35 &dest, 1, srcs, 2, len); 36 &dest, 1, srcs, 2, len);
36 struct dma_device *dma = chan ? chan->device : NULL; 37 struct dma_device *dma = chan ? chan->device : NULL;
38 struct dmaengine_unmap_data *unmap = NULL;
37 const u8 *amul, *bmul; 39 const u8 *amul, *bmul;
38 u8 ax, bx; 40 u8 ax, bx;
39 u8 *a, *b, *c; 41 u8 *a, *b, *c;
40 42
41 if (dma) { 43 if (dma)
42 dma_addr_t dma_dest[2]; 44 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOIO);
43 dma_addr_t dma_src[2]; 45
46 if (unmap) {
44 struct device *dev = dma->dev; 47 struct device *dev = dma->dev;
48 dma_addr_t pq[2];
45 struct dma_async_tx_descriptor *tx; 49 struct dma_async_tx_descriptor *tx;
46 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; 50 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
47 51
48 if (submit->flags & ASYNC_TX_FENCE) 52 if (submit->flags & ASYNC_TX_FENCE)
49 dma_flags |= DMA_PREP_FENCE; 53 dma_flags |= DMA_PREP_FENCE;
50 dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); 54 unmap->addr[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
51 dma_src[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE); 55 unmap->addr[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
52 dma_src[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE); 56 unmap->to_cnt = 2;
53 tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 2, coef, 57
58 unmap->addr[2] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
59 unmap->bidi_cnt = 1;
60 /* engine only looks at Q, but expects it to follow P */
61 pq[1] = unmap->addr[2];
62
63 unmap->len = len;
64 tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef,
54 len, dma_flags); 65 len, dma_flags);
55 if (tx) { 66 if (tx) {
67 dma_set_unmap(tx, unmap);
56 async_tx_submit(chan, tx, submit); 68 async_tx_submit(chan, tx, submit);
69 dmaengine_unmap_put(unmap);
57 return tx; 70 return tx;
58 } 71 }
59 72
60 /* could not get a descriptor, unmap and fall through to 73 /* could not get a descriptor, unmap and fall through to
61 * the synchronous path 74 * the synchronous path
62 */ 75 */
63 dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL); 76 dmaengine_unmap_put(unmap);
64 dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
65 dma_unmap_page(dev, dma_src[1], len, DMA_TO_DEVICE);
66 } 77 }
67 78
68 /* run the operation synchronously */ 79 /* run the operation synchronously */
@@ -89,23 +100,38 @@ async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
89 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, 100 struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
90 &dest, 1, &src, 1, len); 101 &dest, 1, &src, 1, len);
91 struct dma_device *dma = chan ? chan->device : NULL; 102 struct dma_device *dma = chan ? chan->device : NULL;
103 struct dmaengine_unmap_data *unmap = NULL;
92 const u8 *qmul; /* Q multiplier table */ 104 const u8 *qmul; /* Q multiplier table */
93 u8 *d, *s; 105 u8 *d, *s;
94 106
95 if (dma) { 107 if (dma)
108 unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOIO);
109
110 if (unmap) {
96 dma_addr_t dma_dest[2]; 111 dma_addr_t dma_dest[2];
97 dma_addr_t dma_src[1];
98 struct device *dev = dma->dev; 112 struct device *dev = dma->dev;
99 struct dma_async_tx_descriptor *tx; 113 struct dma_async_tx_descriptor *tx;
100 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; 114 enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
101 115
102 if (submit->flags & ASYNC_TX_FENCE) 116 if (submit->flags & ASYNC_TX_FENCE)
103 dma_flags |= DMA_PREP_FENCE; 117 dma_flags |= DMA_PREP_FENCE;
104 dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); 118 unmap->addr[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
105 dma_src[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE); 119 unmap->to_cnt++;
106 tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 1, &coef, 120 unmap->addr[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
107 len, dma_flags); 121 dma_dest[1] = unmap->addr[1];
122 unmap->bidi_cnt++;
123 unmap->len = len;
124
125 /* this looks funny, but the engine looks for Q at
126 * dma_dest[1] and ignores dma_dest[0] as a dest
127 * due to DMA_PREP_PQ_DISABLE_P
128 */
129 tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr,
130 1, &coef, len, dma_flags);
131
108 if (tx) { 132 if (tx) {
133 dma_set_unmap(tx, unmap);
134 dmaengine_unmap_put(unmap);
109 async_tx_submit(chan, tx, submit); 135 async_tx_submit(chan, tx, submit);
110 return tx; 136 return tx;
111 } 137 }
@@ -113,8 +139,7 @@ async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
113 /* could not get a descriptor, unmap and fall through to 139 /* could not get a descriptor, unmap and fall through to
114 * the synchronous path 140 * the synchronous path
115 */ 141 */
116 dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL); 142 dmaengine_unmap_put(unmap);
117 dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
118 } 143 }
119 144
120 /* no channel available, or failed to allocate a descriptor, so 145 /* no channel available, or failed to allocate a descriptor, so
diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c
index 8ade0a0481c6..3c562f5a60bb 100644
--- a/crypto/async_tx/async_xor.c
+++ b/crypto/async_tx/async_xor.c
@@ -33,48 +33,31 @@
33 33
34/* do_async_xor - dma map the pages and perform the xor with an engine */ 34/* do_async_xor - dma map the pages and perform the xor with an engine */
35static __async_inline struct dma_async_tx_descriptor * 35static __async_inline struct dma_async_tx_descriptor *
36do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, 36do_async_xor(struct dma_chan *chan, struct dmaengine_unmap_data *unmap,
37 unsigned int offset, int src_cnt, size_t len, dma_addr_t *dma_src,
38 struct async_submit_ctl *submit) 37 struct async_submit_ctl *submit)
39{ 38{
40 struct dma_device *dma = chan->device; 39 struct dma_device *dma = chan->device;
41 struct dma_async_tx_descriptor *tx = NULL; 40 struct dma_async_tx_descriptor *tx = NULL;
42 int src_off = 0;
43 int i;
44 dma_async_tx_callback cb_fn_orig = submit->cb_fn; 41 dma_async_tx_callback cb_fn_orig = submit->cb_fn;
45 void *cb_param_orig = submit->cb_param; 42 void *cb_param_orig = submit->cb_param;
46 enum async_tx_flags flags_orig = submit->flags; 43 enum async_tx_flags flags_orig = submit->flags;
47 enum dma_ctrl_flags dma_flags; 44 enum dma_ctrl_flags dma_flags = 0;
48 int xor_src_cnt = 0; 45 int src_cnt = unmap->to_cnt;
49 dma_addr_t dma_dest; 46 int xor_src_cnt;
50 47 dma_addr_t dma_dest = unmap->addr[unmap->to_cnt];
51 /* map the dest bidrectional in case it is re-used as a source */ 48 dma_addr_t *src_list = unmap->addr;
52 dma_dest = dma_map_page(dma->dev, dest, offset, len, DMA_BIDIRECTIONAL);
53 for (i = 0; i < src_cnt; i++) {
54 /* only map the dest once */
55 if (!src_list[i])
56 continue;
57 if (unlikely(src_list[i] == dest)) {
58 dma_src[xor_src_cnt++] = dma_dest;
59 continue;
60 }
61 dma_src[xor_src_cnt++] = dma_map_page(dma->dev, src_list[i], offset,
62 len, DMA_TO_DEVICE);
63 }
64 src_cnt = xor_src_cnt;
65 49
66 while (src_cnt) { 50 while (src_cnt) {
51 dma_addr_t tmp;
52
67 submit->flags = flags_orig; 53 submit->flags = flags_orig;
68 dma_flags = 0;
69 xor_src_cnt = min(src_cnt, (int)dma->max_xor); 54 xor_src_cnt = min(src_cnt, (int)dma->max_xor);
70 /* if we are submitting additional xors, leave the chain open, 55 /* if we are submitting additional xors, leave the chain open
71 * clear the callback parameters, and leave the destination 56 * and clear the callback parameters
72 * buffer mapped
73 */ 57 */
74 if (src_cnt > xor_src_cnt) { 58 if (src_cnt > xor_src_cnt) {
75 submit->flags &= ~ASYNC_TX_ACK; 59 submit->flags &= ~ASYNC_TX_ACK;
76 submit->flags |= ASYNC_TX_FENCE; 60 submit->flags |= ASYNC_TX_FENCE;
77 dma_flags = DMA_COMPL_SKIP_DEST_UNMAP;
78 submit->cb_fn = NULL; 61 submit->cb_fn = NULL;
79 submit->cb_param = NULL; 62 submit->cb_param = NULL;
80 } else { 63 } else {
@@ -85,12 +68,18 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
85 dma_flags |= DMA_PREP_INTERRUPT; 68 dma_flags |= DMA_PREP_INTERRUPT;
86 if (submit->flags & ASYNC_TX_FENCE) 69 if (submit->flags & ASYNC_TX_FENCE)
87 dma_flags |= DMA_PREP_FENCE; 70 dma_flags |= DMA_PREP_FENCE;
88 /* Since we have clobbered the src_list we are committed 71
89 * to doing this asynchronously. Drivers force forward progress 72 /* Drivers force forward progress in case they can not provide a
90 * in case they can not provide a descriptor 73 * descriptor
91 */ 74 */
92 tx = dma->device_prep_dma_xor(chan, dma_dest, &dma_src[src_off], 75 tmp = src_list[0];
93 xor_src_cnt, len, dma_flags); 76 if (src_list > unmap->addr)
77 src_list[0] = dma_dest;
78 tx = dma->device_prep_dma_xor(chan, dma_dest, src_list,
79 xor_src_cnt, unmap->len,
80 dma_flags);
81 src_list[0] = tmp;
82
94 83
95 if (unlikely(!tx)) 84 if (unlikely(!tx))
96 async_tx_quiesce(&submit->depend_tx); 85 async_tx_quiesce(&submit->depend_tx);
@@ -99,22 +88,21 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
99 while (unlikely(!tx)) { 88 while (unlikely(!tx)) {
100 dma_async_issue_pending(chan); 89 dma_async_issue_pending(chan);
101 tx = dma->device_prep_dma_xor(chan, dma_dest, 90 tx = dma->device_prep_dma_xor(chan, dma_dest,
102 &dma_src[src_off], 91 src_list,
103 xor_src_cnt, len, 92 xor_src_cnt, unmap->len,
104 dma_flags); 93 dma_flags);
105 } 94 }
106 95
96 dma_set_unmap(tx, unmap);
107 async_tx_submit(chan, tx, submit); 97 async_tx_submit(chan, tx, submit);
108 submit->depend_tx = tx; 98 submit->depend_tx = tx;
109 99
110 if (src_cnt > xor_src_cnt) { 100 if (src_cnt > xor_src_cnt) {
111 /* drop completed sources */ 101 /* drop completed sources */
112 src_cnt -= xor_src_cnt; 102 src_cnt -= xor_src_cnt;
113 src_off += xor_src_cnt;
114
115 /* use the intermediate result a source */ 103 /* use the intermediate result a source */
116 dma_src[--src_off] = dma_dest;
117 src_cnt++; 104 src_cnt++;
105 src_list += xor_src_cnt - 1;
118 } else 106 } else
119 break; 107 break;
120 } 108 }
@@ -189,22 +177,40 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset,
189 struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR, 177 struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR,
190 &dest, 1, src_list, 178 &dest, 1, src_list,
191 src_cnt, len); 179 src_cnt, len);
192 dma_addr_t *dma_src = NULL; 180 struct dma_device *device = chan ? chan->device : NULL;
181 struct dmaengine_unmap_data *unmap = NULL;
193 182
194 BUG_ON(src_cnt <= 1); 183 BUG_ON(src_cnt <= 1);
195 184
196 if (submit->scribble) 185 if (device)
197 dma_src = submit->scribble; 186 unmap = dmaengine_get_unmap_data(device->dev, src_cnt+1, GFP_NOIO);
198 else if (sizeof(dma_addr_t) <= sizeof(struct page *)) 187
199 dma_src = (dma_addr_t *) src_list; 188 if (unmap && is_dma_xor_aligned(device, offset, 0, len)) {
189 struct dma_async_tx_descriptor *tx;
190 int i, j;
200 191
201 if (dma_src && chan && is_dma_xor_aligned(chan->device, offset, 0, len)) {
202 /* run the xor asynchronously */ 192 /* run the xor asynchronously */
203 pr_debug("%s (async): len: %zu\n", __func__, len); 193 pr_debug("%s (async): len: %zu\n", __func__, len);
204 194
205 return do_async_xor(chan, dest, src_list, offset, src_cnt, len, 195 unmap->len = len;
206 dma_src, submit); 196 for (i = 0, j = 0; i < src_cnt; i++) {
197 if (!src_list[i])
198 continue;
199 unmap->to_cnt++;
200 unmap->addr[j++] = dma_map_page(device->dev, src_list[i],
201 offset, len, DMA_TO_DEVICE);
202 }
203
204 /* map it bidirectional as it may be re-used as a source */
205 unmap->addr[j] = dma_map_page(device->dev, dest, offset, len,
206 DMA_BIDIRECTIONAL);
207 unmap->bidi_cnt = 1;
208
209 tx = do_async_xor(chan, unmap, submit);
210 dmaengine_unmap_put(unmap);
211 return tx;
207 } else { 212 } else {
213 dmaengine_unmap_put(unmap);
208 /* run the xor synchronously */ 214 /* run the xor synchronously */
209 pr_debug("%s (sync): len: %zu\n", __func__, len); 215 pr_debug("%s (sync): len: %zu\n", __func__, len);
210 WARN_ONCE(chan, "%s: no space for dma address conversion\n", 216 WARN_ONCE(chan, "%s: no space for dma address conversion\n",
@@ -268,16 +274,14 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
268 struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len); 274 struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len);
269 struct dma_device *device = chan ? chan->device : NULL; 275 struct dma_device *device = chan ? chan->device : NULL;
270 struct dma_async_tx_descriptor *tx = NULL; 276 struct dma_async_tx_descriptor *tx = NULL;
271 dma_addr_t *dma_src = NULL; 277 struct dmaengine_unmap_data *unmap = NULL;
272 278
273 BUG_ON(src_cnt <= 1); 279 BUG_ON(src_cnt <= 1);
274 280
275 if (submit->scribble) 281 if (device)
276 dma_src = submit->scribble; 282 unmap = dmaengine_get_unmap_data(device->dev, src_cnt, GFP_NOIO);
277 else if (sizeof(dma_addr_t) <= sizeof(struct page *))
278 dma_src = (dma_addr_t *) src_list;
279 283
280 if (dma_src && device && src_cnt <= device->max_xor && 284 if (unmap && src_cnt <= device->max_xor &&
281 is_dma_xor_aligned(device, offset, 0, len)) { 285 is_dma_xor_aligned(device, offset, 0, len)) {
282 unsigned long dma_prep_flags = 0; 286 unsigned long dma_prep_flags = 0;
283 int i; 287 int i;
@@ -288,11 +292,15 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
288 dma_prep_flags |= DMA_PREP_INTERRUPT; 292 dma_prep_flags |= DMA_PREP_INTERRUPT;
289 if (submit->flags & ASYNC_TX_FENCE) 293 if (submit->flags & ASYNC_TX_FENCE)
290 dma_prep_flags |= DMA_PREP_FENCE; 294 dma_prep_flags |= DMA_PREP_FENCE;
291 for (i = 0; i < src_cnt; i++)
292 dma_src[i] = dma_map_page(device->dev, src_list[i],
293 offset, len, DMA_TO_DEVICE);
294 295
295 tx = device->device_prep_dma_xor_val(chan, dma_src, src_cnt, 296 for (i = 0; i < src_cnt; i++) {
297 unmap->addr[i] = dma_map_page(device->dev, src_list[i],
298 offset, len, DMA_TO_DEVICE);
299 unmap->to_cnt++;
300 }
301 unmap->len = len;
302
303 tx = device->device_prep_dma_xor_val(chan, unmap->addr, src_cnt,
296 len, result, 304 len, result,
297 dma_prep_flags); 305 dma_prep_flags);
298 if (unlikely(!tx)) { 306 if (unlikely(!tx)) {
@@ -301,11 +309,11 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
301 while (!tx) { 309 while (!tx) {
302 dma_async_issue_pending(chan); 310 dma_async_issue_pending(chan);
303 tx = device->device_prep_dma_xor_val(chan, 311 tx = device->device_prep_dma_xor_val(chan,
304 dma_src, src_cnt, len, result, 312 unmap->addr, src_cnt, len, result,
305 dma_prep_flags); 313 dma_prep_flags);
306 } 314 }
307 } 315 }
308 316 dma_set_unmap(tx, unmap);
309 async_tx_submit(chan, tx, submit); 317 async_tx_submit(chan, tx, submit);
310 } else { 318 } else {
311 enum async_tx_flags flags_orig = submit->flags; 319 enum async_tx_flags flags_orig = submit->flags;
@@ -327,6 +335,7 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
327 async_tx_sync_epilog(submit); 335 async_tx_sync_epilog(submit);
328 submit->flags = flags_orig; 336 submit->flags = flags_orig;
329 } 337 }
338 dmaengine_unmap_put(unmap);
330 339
331 return tx; 340 return tx;
332} 341}
diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c
index 4a92bac744dc..dad95f45b88f 100644
--- a/crypto/async_tx/raid6test.c
+++ b/crypto/async_tx/raid6test.c
@@ -28,7 +28,7 @@
28#undef pr 28#undef pr
29#define pr(fmt, args...) pr_info("raid6test: " fmt, ##args) 29#define pr(fmt, args...) pr_info("raid6test: " fmt, ##args)
30 30
31#define NDISKS 16 /* Including P and Q */ 31#define NDISKS 64 /* Including P and Q */
32 32
33static struct page *dataptrs[NDISKS]; 33static struct page *dataptrs[NDISKS];
34static addr_conv_t addr_conv[NDISKS]; 34static addr_conv_t addr_conv[NDISKS];
@@ -219,6 +219,14 @@ static int raid6_test(void)
219 err += test(11, &tests); 219 err += test(11, &tests);
220 err += test(12, &tests); 220 err += test(12, &tests);
221 } 221 }
222
223 /* the 24 disk case is special for ioatdma as it is the boudary point
224 * at which it needs to switch from 8-source ops to 16-source
225 * ops for continuation (assumes DMA_HAS_PQ_CONTINUE is not set)
226 */
227 if (NDISKS > 24)
228 err += test(24, &tests);
229
222 err += test(NDISKS, &tests); 230 err += test(NDISKS, &tests);
223 231
224 pr("\n"); 232 pr("\n");
diff --git a/drivers/ata/pata_arasan_cf.c b/drivers/ata/pata_arasan_cf.c
index 853f610af28f..e88690ebfd82 100644
--- a/drivers/ata/pata_arasan_cf.c
+++ b/drivers/ata/pata_arasan_cf.c
@@ -396,8 +396,7 @@ dma_xfer(struct arasan_cf_dev *acdev, dma_addr_t src, dma_addr_t dest, u32 len)
396 struct dma_async_tx_descriptor *tx; 396 struct dma_async_tx_descriptor *tx;
397 struct dma_chan *chan = acdev->dma_chan; 397 struct dma_chan *chan = acdev->dma_chan;
398 dma_cookie_t cookie; 398 dma_cookie_t cookie;
399 unsigned long flags = DMA_PREP_INTERRUPT | DMA_COMPL_SKIP_SRC_UNMAP | 399 unsigned long flags = DMA_PREP_INTERRUPT;
400 DMA_COMPL_SKIP_DEST_UNMAP;
401 int ret = 0; 400 int ret = 0;
402 401
403 tx = chan->device->device_prep_dma_memcpy(chan, dest, src, len, flags); 402 tx = chan->device->device_prep_dma_memcpy(chan, dest, src, len, flags);
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index 4ee6533108f6..f2917e2035ba 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -1164,42 +1164,12 @@ static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
1164 kfree(txd); 1164 kfree(txd);
1165} 1165}
1166 1166
1167static void pl08x_unmap_buffers(struct pl08x_txd *txd)
1168{
1169 struct device *dev = txd->vd.tx.chan->device->dev;
1170 struct pl08x_sg *dsg;
1171
1172 if (!(txd->vd.tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1173 if (txd->vd.tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
1174 list_for_each_entry(dsg, &txd->dsg_list, node)
1175 dma_unmap_single(dev, dsg->src_addr, dsg->len,
1176 DMA_TO_DEVICE);
1177 else {
1178 list_for_each_entry(dsg, &txd->dsg_list, node)
1179 dma_unmap_page(dev, dsg->src_addr, dsg->len,
1180 DMA_TO_DEVICE);
1181 }
1182 }
1183 if (!(txd->vd.tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1184 if (txd->vd.tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
1185 list_for_each_entry(dsg, &txd->dsg_list, node)
1186 dma_unmap_single(dev, dsg->dst_addr, dsg->len,
1187 DMA_FROM_DEVICE);
1188 else
1189 list_for_each_entry(dsg, &txd->dsg_list, node)
1190 dma_unmap_page(dev, dsg->dst_addr, dsg->len,
1191 DMA_FROM_DEVICE);
1192 }
1193}
1194
1195static void pl08x_desc_free(struct virt_dma_desc *vd) 1167static void pl08x_desc_free(struct virt_dma_desc *vd)
1196{ 1168{
1197 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1169 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
1198 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan); 1170 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan);
1199 1171
1200 if (!plchan->slave) 1172 dma_descriptor_unmap(txd);
1201 pl08x_unmap_buffers(txd);
1202
1203 if (!txd->done) 1173 if (!txd->done)
1204 pl08x_release_mux(plchan); 1174 pl08x_release_mux(plchan);
1205 1175
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index 1ef74579447d..e2c04dc81e2a 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -344,31 +344,7 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
344 /* move myself to free_list */ 344 /* move myself to free_list */
345 list_move(&desc->desc_node, &atchan->free_list); 345 list_move(&desc->desc_node, &atchan->free_list);
346 346
347 /* unmap dma addresses (not on slave channels) */ 347 dma_descriptor_unmap(txd);
348 if (!atchan->chan_common.private) {
349 struct device *parent = chan2parent(&atchan->chan_common);
350 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
351 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
352 dma_unmap_single(parent,
353 desc->lli.daddr,
354 desc->len, DMA_FROM_DEVICE);
355 else
356 dma_unmap_page(parent,
357 desc->lli.daddr,
358 desc->len, DMA_FROM_DEVICE);
359 }
360 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
361 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
362 dma_unmap_single(parent,
363 desc->lli.saddr,
364 desc->len, DMA_TO_DEVICE);
365 else
366 dma_unmap_page(parent,
367 desc->lli.saddr,
368 desc->len, DMA_TO_DEVICE);
369 }
370 }
371
372 /* for cyclic transfers, 348 /* for cyclic transfers,
373 * no need to replay callback function while stopping */ 349 * no need to replay callback function while stopping */
374 if (!atc_chan_is_cyclic(atchan)) { 350 if (!atc_chan_is_cyclic(atchan)) {
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 81d876528c70..ea806bdc12ef 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -65,6 +65,7 @@
65#include <linux/acpi.h> 65#include <linux/acpi.h>
66#include <linux/acpi_dma.h> 66#include <linux/acpi_dma.h>
67#include <linux/of_dma.h> 67#include <linux/of_dma.h>
68#include <linux/mempool.h>
68 69
69static DEFINE_MUTEX(dma_list_mutex); 70static DEFINE_MUTEX(dma_list_mutex);
70static DEFINE_IDR(dma_idr); 71static DEFINE_IDR(dma_idr);
@@ -901,98 +902,132 @@ void dma_async_device_unregister(struct dma_device *device)
901} 902}
902EXPORT_SYMBOL(dma_async_device_unregister); 903EXPORT_SYMBOL(dma_async_device_unregister);
903 904
904/** 905struct dmaengine_unmap_pool {
905 * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses 906 struct kmem_cache *cache;
906 * @chan: DMA channel to offload copy to 907 const char *name;
907 * @dest: destination address (virtual) 908 mempool_t *pool;
908 * @src: source address (virtual) 909 size_t size;
909 * @len: length 910};
910 *
911 * Both @dest and @src must be mappable to a bus address according to the
912 * DMA mapping API rules for streaming mappings.
913 * Both @dest and @src must stay memory resident (kernel memory or locked
914 * user space pages).
915 */
916dma_cookie_t
917dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest,
918 void *src, size_t len)
919{
920 struct dma_device *dev = chan->device;
921 struct dma_async_tx_descriptor *tx;
922 dma_addr_t dma_dest, dma_src;
923 dma_cookie_t cookie;
924 unsigned long flags;
925 911
926 dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE); 912#define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) }
927 dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE); 913static struct dmaengine_unmap_pool unmap_pool[] = {
928 flags = DMA_CTRL_ACK | 914 __UNMAP_POOL(2),
929 DMA_COMPL_SRC_UNMAP_SINGLE | 915 #if IS_ENABLED(CONFIG_ASYNC_TX_DMA)
930 DMA_COMPL_DEST_UNMAP_SINGLE; 916 __UNMAP_POOL(16),
931 tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); 917 __UNMAP_POOL(128),
918 __UNMAP_POOL(256),
919 #endif
920};
932 921
933 if (!tx) { 922static struct dmaengine_unmap_pool *__get_unmap_pool(int nr)
934 dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); 923{
935 dma_unmap_single(dev->dev, dma_dest, len, DMA_FROM_DEVICE); 924 int order = get_count_order(nr);
936 return -ENOMEM; 925
926 switch (order) {
927 case 0 ... 1:
928 return &unmap_pool[0];
929 case 2 ... 4:
930 return &unmap_pool[1];
931 case 5 ... 7:
932 return &unmap_pool[2];
933 case 8:
934 return &unmap_pool[3];
935 default:
936 BUG();
937 return NULL;
937 } 938 }
939}
938 940
939 tx->callback = NULL; 941static void dmaengine_unmap(struct kref *kref)
940 cookie = tx->tx_submit(tx); 942{
943 struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref);
944 struct device *dev = unmap->dev;
945 int cnt, i;
946
947 cnt = unmap->to_cnt;
948 for (i = 0; i < cnt; i++)
949 dma_unmap_page(dev, unmap->addr[i], unmap->len,
950 DMA_TO_DEVICE);
951 cnt += unmap->from_cnt;
952 for (; i < cnt; i++)
953 dma_unmap_page(dev, unmap->addr[i], unmap->len,
954 DMA_FROM_DEVICE);
955 cnt += unmap->bidi_cnt;
956 for (; i < cnt; i++) {
957 if (unmap->addr[i] == 0)
958 continue;
959 dma_unmap_page(dev, unmap->addr[i], unmap->len,
960 DMA_BIDIRECTIONAL);
961 }
962 mempool_free(unmap, __get_unmap_pool(cnt)->pool);
963}
941 964
942 preempt_disable(); 965void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)
943 __this_cpu_add(chan->local->bytes_transferred, len); 966{
944 __this_cpu_inc(chan->local->memcpy_count); 967 if (unmap)
945 preempt_enable(); 968 kref_put(&unmap->kref, dmaengine_unmap);
969}
970EXPORT_SYMBOL_GPL(dmaengine_unmap_put);
946 971
947 return cookie; 972static void dmaengine_destroy_unmap_pool(void)
973{
974 int i;
975
976 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
977 struct dmaengine_unmap_pool *p = &unmap_pool[i];
978
979 if (p->pool)
980 mempool_destroy(p->pool);
981 p->pool = NULL;
982 if (p->cache)
983 kmem_cache_destroy(p->cache);
984 p->cache = NULL;
985 }
948} 986}
949EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
950 987
951/** 988static int __init dmaengine_init_unmap_pool(void)
952 * dma_async_memcpy_buf_to_pg - offloaded copy from address to page
953 * @chan: DMA channel to offload copy to
954 * @page: destination page
955 * @offset: offset in page to copy to
956 * @kdata: source address (virtual)
957 * @len: length
958 *
959 * Both @page/@offset and @kdata must be mappable to a bus address according
960 * to the DMA mapping API rules for streaming mappings.
961 * Both @page/@offset and @kdata must stay memory resident (kernel memory or
962 * locked user space pages)
963 */
964dma_cookie_t
965dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page,
966 unsigned int offset, void *kdata, size_t len)
967{ 989{
968 struct dma_device *dev = chan->device; 990 int i;
969 struct dma_async_tx_descriptor *tx;
970 dma_addr_t dma_dest, dma_src;
971 dma_cookie_t cookie;
972 unsigned long flags;
973 991
974 dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE); 992 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
975 dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE); 993 struct dmaengine_unmap_pool *p = &unmap_pool[i];
976 flags = DMA_CTRL_ACK | DMA_COMPL_SRC_UNMAP_SINGLE; 994 size_t size;
977 tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags);
978 995
979 if (!tx) { 996 size = sizeof(struct dmaengine_unmap_data) +
980 dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); 997 sizeof(dma_addr_t) * p->size;
981 dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); 998
982 return -ENOMEM; 999 p->cache = kmem_cache_create(p->name, size, 0,
1000 SLAB_HWCACHE_ALIGN, NULL);
1001 if (!p->cache)
1002 break;
1003 p->pool = mempool_create_slab_pool(1, p->cache);
1004 if (!p->pool)
1005 break;
983 } 1006 }
984 1007
985 tx->callback = NULL; 1008 if (i == ARRAY_SIZE(unmap_pool))
986 cookie = tx->tx_submit(tx); 1009 return 0;
987 1010
988 preempt_disable(); 1011 dmaengine_destroy_unmap_pool();
989 __this_cpu_add(chan->local->bytes_transferred, len); 1012 return -ENOMEM;
990 __this_cpu_inc(chan->local->memcpy_count); 1013}
991 preempt_enable();
992 1014
993 return cookie; 1015struct dmaengine_unmap_data *
1016dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)
1017{
1018 struct dmaengine_unmap_data *unmap;
1019
1020 unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags);
1021 if (!unmap)
1022 return NULL;
1023
1024 memset(unmap, 0, sizeof(*unmap));
1025 kref_init(&unmap->kref);
1026 unmap->dev = dev;
1027
1028 return unmap;
994} 1029}
995EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg); 1030EXPORT_SYMBOL(dmaengine_get_unmap_data);
996 1031
997/** 1032/**
998 * dma_async_memcpy_pg_to_pg - offloaded copy from page to page 1033 * dma_async_memcpy_pg_to_pg - offloaded copy from page to page
@@ -1015,24 +1050,33 @@ dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg,
1015{ 1050{
1016 struct dma_device *dev = chan->device; 1051 struct dma_device *dev = chan->device;
1017 struct dma_async_tx_descriptor *tx; 1052 struct dma_async_tx_descriptor *tx;
1018 dma_addr_t dma_dest, dma_src; 1053 struct dmaengine_unmap_data *unmap;
1019 dma_cookie_t cookie; 1054 dma_cookie_t cookie;
1020 unsigned long flags; 1055 unsigned long flags;
1021 1056
1022 dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE); 1057 unmap = dmaengine_get_unmap_data(dev->dev, 2, GFP_NOIO);
1023 dma_dest = dma_map_page(dev->dev, dest_pg, dest_off, len, 1058 if (!unmap)
1024 DMA_FROM_DEVICE); 1059 return -ENOMEM;
1060
1061 unmap->to_cnt = 1;
1062 unmap->from_cnt = 1;
1063 unmap->addr[0] = dma_map_page(dev->dev, src_pg, src_off, len,
1064 DMA_TO_DEVICE);
1065 unmap->addr[1] = dma_map_page(dev->dev, dest_pg, dest_off, len,
1066 DMA_FROM_DEVICE);
1067 unmap->len = len;
1025 flags = DMA_CTRL_ACK; 1068 flags = DMA_CTRL_ACK;
1026 tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); 1069 tx = dev->device_prep_dma_memcpy(chan, unmap->addr[1], unmap->addr[0],
1070 len, flags);
1027 1071
1028 if (!tx) { 1072 if (!tx) {
1029 dma_unmap_page(dev->dev, dma_src, len, DMA_TO_DEVICE); 1073 dmaengine_unmap_put(unmap);
1030 dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE);
1031 return -ENOMEM; 1074 return -ENOMEM;
1032 } 1075 }
1033 1076
1034 tx->callback = NULL; 1077 dma_set_unmap(tx, unmap);
1035 cookie = tx->tx_submit(tx); 1078 cookie = tx->tx_submit(tx);
1079 dmaengine_unmap_put(unmap);
1036 1080
1037 preempt_disable(); 1081 preempt_disable();
1038 __this_cpu_add(chan->local->bytes_transferred, len); 1082 __this_cpu_add(chan->local->bytes_transferred, len);
@@ -1043,6 +1087,52 @@ dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg,
1043} 1087}
1044EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg); 1088EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg);
1045 1089
1090/**
1091 * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
1092 * @chan: DMA channel to offload copy to
1093 * @dest: destination address (virtual)
1094 * @src: source address (virtual)
1095 * @len: length
1096 *
1097 * Both @dest and @src must be mappable to a bus address according to the
1098 * DMA mapping API rules for streaming mappings.
1099 * Both @dest and @src must stay memory resident (kernel memory or locked
1100 * user space pages).
1101 */
1102dma_cookie_t
1103dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest,
1104 void *src, size_t len)
1105{
1106 return dma_async_memcpy_pg_to_pg(chan, virt_to_page(dest),
1107 (unsigned long) dest & ~PAGE_MASK,
1108 virt_to_page(src),
1109 (unsigned long) src & ~PAGE_MASK, len);
1110}
1111EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
1112
1113/**
1114 * dma_async_memcpy_buf_to_pg - offloaded copy from address to page
1115 * @chan: DMA channel to offload copy to
1116 * @page: destination page
1117 * @offset: offset in page to copy to
1118 * @kdata: source address (virtual)
1119 * @len: length
1120 *
1121 * Both @page/@offset and @kdata must be mappable to a bus address according
1122 * to the DMA mapping API rules for streaming mappings.
1123 * Both @page/@offset and @kdata must stay memory resident (kernel memory or
1124 * locked user space pages)
1125 */
1126dma_cookie_t
1127dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page,
1128 unsigned int offset, void *kdata, size_t len)
1129{
1130 return dma_async_memcpy_pg_to_pg(chan, page, offset,
1131 virt_to_page(kdata),
1132 (unsigned long) kdata & ~PAGE_MASK, len);
1133}
1134EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg);
1135
1046void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, 1136void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
1047 struct dma_chan *chan) 1137 struct dma_chan *chan)
1048{ 1138{
@@ -1116,6 +1206,10 @@ EXPORT_SYMBOL_GPL(dma_run_dependencies);
1116 1206
1117static int __init dma_bus_init(void) 1207static int __init dma_bus_init(void)
1118{ 1208{
1209 int err = dmaengine_init_unmap_pool();
1210
1211 if (err)
1212 return err;
1119 return class_register(&dma_devclass); 1213 return class_register(&dma_devclass);
1120} 1214}
1121arch_initcall(dma_bus_init); 1215arch_initcall(dma_bus_init);
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index 59e287f56dfc..20f9a3aaf926 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -8,6 +8,8 @@
8 * it under the terms of the GNU General Public License version 2 as 8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation. 9 * published by the Free Software Foundation.
10 */ 10 */
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
11#include <linux/delay.h> 13#include <linux/delay.h>
12#include <linux/dma-mapping.h> 14#include <linux/dma-mapping.h>
13#include <linux/dmaengine.h> 15#include <linux/dmaengine.h>
@@ -19,10 +21,6 @@
19#include <linux/random.h> 21#include <linux/random.h>
20#include <linux/slab.h> 22#include <linux/slab.h>
21#include <linux/wait.h> 23#include <linux/wait.h>
22#include <linux/ctype.h>
23#include <linux/debugfs.h>
24#include <linux/uaccess.h>
25#include <linux/seq_file.h>
26 24
27static unsigned int test_buf_size = 16384; 25static unsigned int test_buf_size = 16384;
28module_param(test_buf_size, uint, S_IRUGO | S_IWUSR); 26module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
@@ -68,92 +66,13 @@ module_param(timeout, uint, S_IRUGO | S_IWUSR);
68MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " 66MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
69 "Pass -1 for infinite timeout"); 67 "Pass -1 for infinite timeout");
70 68
71/* Maximum amount of mismatched bytes in buffer to print */ 69static bool noverify;
72#define MAX_ERROR_COUNT 32 70module_param(noverify, bool, S_IRUGO | S_IWUSR);
73 71MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
74/*
75 * Initialization patterns. All bytes in the source buffer has bit 7
76 * set, all bytes in the destination buffer has bit 7 cleared.
77 *
78 * Bit 6 is set for all bytes which are to be copied by the DMA
79 * engine. Bit 5 is set for all bytes which are to be overwritten by
80 * the DMA engine.
81 *
82 * The remaining bits are the inverse of a counter which increments by
83 * one for each byte address.
84 */
85#define PATTERN_SRC 0x80
86#define PATTERN_DST 0x00
87#define PATTERN_COPY 0x40
88#define PATTERN_OVERWRITE 0x20
89#define PATTERN_COUNT_MASK 0x1f
90
91enum dmatest_error_type {
92 DMATEST_ET_OK,
93 DMATEST_ET_MAP_SRC,
94 DMATEST_ET_MAP_DST,
95 DMATEST_ET_PREP,
96 DMATEST_ET_SUBMIT,
97 DMATEST_ET_TIMEOUT,
98 DMATEST_ET_DMA_ERROR,
99 DMATEST_ET_DMA_IN_PROGRESS,
100 DMATEST_ET_VERIFY,
101 DMATEST_ET_VERIFY_BUF,
102};
103
104struct dmatest_verify_buffer {
105 unsigned int index;
106 u8 expected;
107 u8 actual;
108};
109
110struct dmatest_verify_result {
111 unsigned int error_count;
112 struct dmatest_verify_buffer data[MAX_ERROR_COUNT];
113 u8 pattern;
114 bool is_srcbuf;
115};
116
117struct dmatest_thread_result {
118 struct list_head node;
119 unsigned int n;
120 unsigned int src_off;
121 unsigned int dst_off;
122 unsigned int len;
123 enum dmatest_error_type type;
124 union {
125 unsigned long data;
126 dma_cookie_t cookie;
127 enum dma_status status;
128 int error;
129 struct dmatest_verify_result *vr;
130 };
131};
132
133struct dmatest_result {
134 struct list_head node;
135 char *name;
136 struct list_head results;
137};
138
139struct dmatest_info;
140
141struct dmatest_thread {
142 struct list_head node;
143 struct dmatest_info *info;
144 struct task_struct *task;
145 struct dma_chan *chan;
146 u8 **srcs;
147 u8 **dsts;
148 enum dma_transaction_type type;
149 bool done;
150};
151 72
152struct dmatest_chan { 73static bool verbose;
153 struct list_head node; 74module_param(verbose, bool, S_IRUGO | S_IWUSR);
154 struct dma_chan *chan; 75MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
155 struct list_head threads;
156};
157 76
158/** 77/**
159 * struct dmatest_params - test parameters. 78 * struct dmatest_params - test parameters.
@@ -177,6 +96,7 @@ struct dmatest_params {
177 unsigned int xor_sources; 96 unsigned int xor_sources;
178 unsigned int pq_sources; 97 unsigned int pq_sources;
179 int timeout; 98 int timeout;
99 bool noverify;
180}; 100};
181 101
182/** 102/**
@@ -184,7 +104,7 @@ struct dmatest_params {
184 * @params: test parameters 104 * @params: test parameters
185 * @lock: access protection to the fields of this structure 105 * @lock: access protection to the fields of this structure
186 */ 106 */
187struct dmatest_info { 107static struct dmatest_info {
188 /* Test parameters */ 108 /* Test parameters */
189 struct dmatest_params params; 109 struct dmatest_params params;
190 110
@@ -192,16 +112,95 @@ struct dmatest_info {
192 struct list_head channels; 112 struct list_head channels;
193 unsigned int nr_channels; 113 unsigned int nr_channels;
194 struct mutex lock; 114 struct mutex lock;
115 bool did_init;
116} test_info = {
117 .channels = LIST_HEAD_INIT(test_info.channels),
118 .lock = __MUTEX_INITIALIZER(test_info.lock),
119};
120
121static int dmatest_run_set(const char *val, const struct kernel_param *kp);
122static int dmatest_run_get(char *val, const struct kernel_param *kp);
123static struct kernel_param_ops run_ops = {
124 .set = dmatest_run_set,
125 .get = dmatest_run_get,
126};
127static bool dmatest_run;
128module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
129MODULE_PARM_DESC(run, "Run the test (default: false)");
130
131/* Maximum amount of mismatched bytes in buffer to print */
132#define MAX_ERROR_COUNT 32
133
134/*
135 * Initialization patterns. All bytes in the source buffer has bit 7
136 * set, all bytes in the destination buffer has bit 7 cleared.
137 *
138 * Bit 6 is set for all bytes which are to be copied by the DMA
139 * engine. Bit 5 is set for all bytes which are to be overwritten by
140 * the DMA engine.
141 *
142 * The remaining bits are the inverse of a counter which increments by
143 * one for each byte address.
144 */
145#define PATTERN_SRC 0x80
146#define PATTERN_DST 0x00
147#define PATTERN_COPY 0x40
148#define PATTERN_OVERWRITE 0x20
149#define PATTERN_COUNT_MASK 0x1f
195 150
196 /* debugfs related stuff */ 151struct dmatest_thread {
197 struct dentry *root; 152 struct list_head node;
153 struct dmatest_info *info;
154 struct task_struct *task;
155 struct dma_chan *chan;
156 u8 **srcs;
157 u8 **dsts;
158 enum dma_transaction_type type;
159 bool done;
160};
198 161
199 /* Test results */ 162struct dmatest_chan {
200 struct list_head results; 163 struct list_head node;
201 struct mutex results_lock; 164 struct dma_chan *chan;
165 struct list_head threads;
202}; 166};
203 167
204static struct dmatest_info test_info; 168static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
169static bool wait;
170
171static bool is_threaded_test_run(struct dmatest_info *info)
172{
173 struct dmatest_chan *dtc;
174
175 list_for_each_entry(dtc, &info->channels, node) {
176 struct dmatest_thread *thread;
177
178 list_for_each_entry(thread, &dtc->threads, node) {
179 if (!thread->done)
180 return true;
181 }
182 }
183
184 return false;
185}
186
187static int dmatest_wait_get(char *val, const struct kernel_param *kp)
188{
189 struct dmatest_info *info = &test_info;
190 struct dmatest_params *params = &info->params;
191
192 if (params->iterations)
193 wait_event(thread_wait, !is_threaded_test_run(info));
194 wait = true;
195 return param_get_bool(val, kp);
196}
197
198static struct kernel_param_ops wait_ops = {
199 .get = dmatest_wait_get,
200 .set = param_set_bool,
201};
202module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
203MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
205 204
206static bool dmatest_match_channel(struct dmatest_params *params, 205static bool dmatest_match_channel(struct dmatest_params *params,
207 struct dma_chan *chan) 206 struct dma_chan *chan)
@@ -223,7 +222,7 @@ static unsigned long dmatest_random(void)
223{ 222{
224 unsigned long buf; 223 unsigned long buf;
225 224
226 get_random_bytes(&buf, sizeof(buf)); 225 prandom_bytes(&buf, sizeof(buf));
227 return buf; 226 return buf;
228} 227}
229 228
@@ -262,9 +261,31 @@ static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
262 } 261 }
263} 262}
264 263
265static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs, 264static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
266 unsigned int start, unsigned int end, unsigned int counter, 265 unsigned int counter, bool is_srcbuf)
267 u8 pattern, bool is_srcbuf) 266{
267 u8 diff = actual ^ pattern;
268 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK);
269 const char *thread_name = current->comm;
270
271 if (is_srcbuf)
272 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
273 thread_name, index, expected, actual);
274 else if ((pattern & PATTERN_COPY)
275 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
276 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
277 thread_name, index, expected, actual);
278 else if (diff & PATTERN_SRC)
279 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
280 thread_name, index, expected, actual);
281 else
282 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
283 thread_name, index, expected, actual);
284}
285
286static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
287 unsigned int end, unsigned int counter, u8 pattern,
288 bool is_srcbuf)
268{ 289{
269 unsigned int i; 290 unsigned int i;
270 unsigned int error_count = 0; 291 unsigned int error_count = 0;
@@ -272,7 +293,6 @@ static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs,
272 u8 expected; 293 u8 expected;
273 u8 *buf; 294 u8 *buf;
274 unsigned int counter_orig = counter; 295 unsigned int counter_orig = counter;
275 struct dmatest_verify_buffer *vb;
276 296
277 for (; (buf = *bufs); bufs++) { 297 for (; (buf = *bufs); bufs++) {
278 counter = counter_orig; 298 counter = counter_orig;
@@ -280,12 +300,9 @@ static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs,
280 actual = buf[i]; 300 actual = buf[i];
281 expected = pattern | (~counter & PATTERN_COUNT_MASK); 301 expected = pattern | (~counter & PATTERN_COUNT_MASK);
282 if (actual != expected) { 302 if (actual != expected) {
283 if (error_count < MAX_ERROR_COUNT && vr) { 303 if (error_count < MAX_ERROR_COUNT)
284 vb = &vr->data[error_count]; 304 dmatest_mismatch(actual, pattern, i,
285 vb->index = i; 305 counter, is_srcbuf);
286 vb->expected = expected;
287 vb->actual = actual;
288 }
289 error_count++; 306 error_count++;
290 } 307 }
291 counter++; 308 counter++;
@@ -293,7 +310,7 @@ static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs,
293 } 310 }
294 311
295 if (error_count > MAX_ERROR_COUNT) 312 if (error_count > MAX_ERROR_COUNT)
296 pr_warning("%s: %u errors suppressed\n", 313 pr_warn("%s: %u errors suppressed\n",
297 current->comm, error_count - MAX_ERROR_COUNT); 314 current->comm, error_count - MAX_ERROR_COUNT);
298 315
299 return error_count; 316 return error_count;
@@ -313,20 +330,6 @@ static void dmatest_callback(void *arg)
313 wake_up_all(done->wait); 330 wake_up_all(done->wait);
314} 331}
315 332
316static inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len,
317 unsigned int count)
318{
319 while (count--)
320 dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE);
321}
322
323static inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len,
324 unsigned int count)
325{
326 while (count--)
327 dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL);
328}
329
330static unsigned int min_odd(unsigned int x, unsigned int y) 333static unsigned int min_odd(unsigned int x, unsigned int y)
331{ 334{
332 unsigned int val = min(x, y); 335 unsigned int val = min(x, y);
@@ -334,172 +337,49 @@ static unsigned int min_odd(unsigned int x, unsigned int y)
334 return val % 2 ? val : val - 1; 337 return val % 2 ? val : val - 1;
335} 338}
336 339
337static char *verify_result_get_one(struct dmatest_verify_result *vr, 340static void result(const char *err, unsigned int n, unsigned int src_off,
338 unsigned int i) 341 unsigned int dst_off, unsigned int len, unsigned long data)
339{ 342{
340 struct dmatest_verify_buffer *vb = &vr->data[i]; 343 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)",
341 u8 diff = vb->actual ^ vr->pattern; 344 current->comm, n, err, src_off, dst_off, len, data);
342 static char buf[512];
343 char *msg;
344
345 if (vr->is_srcbuf)
346 msg = "srcbuf overwritten!";
347 else if ((vr->pattern & PATTERN_COPY)
348 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
349 msg = "dstbuf not copied!";
350 else if (diff & PATTERN_SRC)
351 msg = "dstbuf was copied!";
352 else
353 msg = "dstbuf mismatch!";
354
355 snprintf(buf, sizeof(buf) - 1, "%s [0x%x] Expected %02x, got %02x", msg,
356 vb->index, vb->expected, vb->actual);
357
358 return buf;
359} 345}
360 346
361static char *thread_result_get(const char *name, 347static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
362 struct dmatest_thread_result *tr) 348 unsigned int dst_off, unsigned int len,
349 unsigned long data)
363{ 350{
364 static const char * const messages[] = { 351 pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)",
365 [DMATEST_ET_OK] = "No errors", 352 current->comm, n, err, src_off, dst_off, len, data);
366 [DMATEST_ET_MAP_SRC] = "src mapping error",
367 [DMATEST_ET_MAP_DST] = "dst mapping error",
368 [DMATEST_ET_PREP] = "prep error",
369 [DMATEST_ET_SUBMIT] = "submit error",
370 [DMATEST_ET_TIMEOUT] = "test timed out",
371 [DMATEST_ET_DMA_ERROR] =
372 "got completion callback (DMA_ERROR)",
373 [DMATEST_ET_DMA_IN_PROGRESS] =
374 "got completion callback (DMA_IN_PROGRESS)",
375 [DMATEST_ET_VERIFY] = "errors",
376 [DMATEST_ET_VERIFY_BUF] = "verify errors",
377 };
378 static char buf[512];
379
380 snprintf(buf, sizeof(buf) - 1,
381 "%s: #%u: %s with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)",
382 name, tr->n, messages[tr->type], tr->src_off, tr->dst_off,
383 tr->len, tr->data);
384
385 return buf;
386} 353}
387 354
388static int thread_result_add(struct dmatest_info *info, 355#define verbose_result(err, n, src_off, dst_off, len, data) ({ \
389 struct dmatest_result *r, enum dmatest_error_type type, 356 if (verbose) \
390 unsigned int n, unsigned int src_off, unsigned int dst_off, 357 result(err, n, src_off, dst_off, len, data); \
391 unsigned int len, unsigned long data) 358 else \
392{ 359 dbg_result(err, n, src_off, dst_off, len, data); \
393 struct dmatest_thread_result *tr; 360})
394
395 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
396 if (!tr)
397 return -ENOMEM;
398
399 tr->type = type;
400 tr->n = n;
401 tr->src_off = src_off;
402 tr->dst_off = dst_off;
403 tr->len = len;
404 tr->data = data;
405 361
406 mutex_lock(&info->results_lock); 362static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
407 list_add_tail(&tr->node, &r->results);
408 mutex_unlock(&info->results_lock);
409
410 if (tr->type == DMATEST_ET_OK)
411 pr_debug("%s\n", thread_result_get(r->name, tr));
412 else
413 pr_warn("%s\n", thread_result_get(r->name, tr));
414
415 return 0;
416}
417
418static unsigned int verify_result_add(struct dmatest_info *info,
419 struct dmatest_result *r, unsigned int n,
420 unsigned int src_off, unsigned int dst_off, unsigned int len,
421 u8 **bufs, int whence, unsigned int counter, u8 pattern,
422 bool is_srcbuf)
423{ 363{
424 struct dmatest_verify_result *vr; 364 unsigned long long per_sec = 1000000;
425 unsigned int error_count;
426 unsigned int buf_off = is_srcbuf ? src_off : dst_off;
427 unsigned int start, end;
428
429 if (whence < 0) {
430 start = 0;
431 end = buf_off;
432 } else if (whence > 0) {
433 start = buf_off + len;
434 end = info->params.buf_size;
435 } else {
436 start = buf_off;
437 end = buf_off + len;
438 }
439 365
440 vr = kmalloc(sizeof(*vr), GFP_KERNEL); 366 if (runtime <= 0)
441 if (!vr) { 367 return 0;
442 pr_warn("dmatest: No memory to store verify result\n");
443 return dmatest_verify(NULL, bufs, start, end, counter, pattern,
444 is_srcbuf);
445 }
446
447 vr->pattern = pattern;
448 vr->is_srcbuf = is_srcbuf;
449
450 error_count = dmatest_verify(vr, bufs, start, end, counter, pattern,
451 is_srcbuf);
452 if (error_count) {
453 vr->error_count = error_count;
454 thread_result_add(info, r, DMATEST_ET_VERIFY_BUF, n, src_off,
455 dst_off, len, (unsigned long)vr);
456 return error_count;
457 }
458
459 kfree(vr);
460 return 0;
461}
462
463static void result_free(struct dmatest_info *info, const char *name)
464{
465 struct dmatest_result *r, *_r;
466
467 mutex_lock(&info->results_lock);
468 list_for_each_entry_safe(r, _r, &info->results, node) {
469 struct dmatest_thread_result *tr, *_tr;
470
471 if (name && strcmp(r->name, name))
472 continue;
473
474 list_for_each_entry_safe(tr, _tr, &r->results, node) {
475 if (tr->type == DMATEST_ET_VERIFY_BUF)
476 kfree(tr->vr);
477 list_del(&tr->node);
478 kfree(tr);
479 }
480 368
481 kfree(r->name); 369 /* drop precision until runtime is 32-bits */
482 list_del(&r->node); 370 while (runtime > UINT_MAX) {
483 kfree(r); 371 runtime >>= 1;
372 per_sec <<= 1;
484 } 373 }
485 374
486 mutex_unlock(&info->results_lock); 375 per_sec *= val;
376 do_div(per_sec, runtime);
377 return per_sec;
487} 378}
488 379
489static struct dmatest_result *result_init(struct dmatest_info *info, 380static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
490 const char *name)
491{ 381{
492 struct dmatest_result *r; 382 return dmatest_persec(runtime, len >> 10);
493
494 r = kzalloc(sizeof(*r), GFP_KERNEL);
495 if (r) {
496 r->name = kstrdup(name, GFP_KERNEL);
497 INIT_LIST_HEAD(&r->results);
498 mutex_lock(&info->results_lock);
499 list_add_tail(&r->node, &info->results);
500 mutex_unlock(&info->results_lock);
501 }
502 return r;
503} 383}
504 384
505/* 385/*
@@ -525,7 +405,6 @@ static int dmatest_func(void *data)
525 struct dmatest_params *params; 405 struct dmatest_params *params;
526 struct dma_chan *chan; 406 struct dma_chan *chan;
527 struct dma_device *dev; 407 struct dma_device *dev;
528 const char *thread_name;
529 unsigned int src_off, dst_off, len; 408 unsigned int src_off, dst_off, len;
530 unsigned int error_count; 409 unsigned int error_count;
531 unsigned int failed_tests = 0; 410 unsigned int failed_tests = 0;
@@ -538,9 +417,10 @@ static int dmatest_func(void *data)
538 int src_cnt; 417 int src_cnt;
539 int dst_cnt; 418 int dst_cnt;
540 int i; 419 int i;
541 struct dmatest_result *result; 420 ktime_t ktime;
421 s64 runtime = 0;
422 unsigned long long total_len = 0;
542 423
543 thread_name = current->comm;
544 set_freezable(); 424 set_freezable();
545 425
546 ret = -ENOMEM; 426 ret = -ENOMEM;
@@ -570,10 +450,6 @@ static int dmatest_func(void *data)
570 } else 450 } else
571 goto err_thread_type; 451 goto err_thread_type;
572 452
573 result = result_init(info, thread_name);
574 if (!result)
575 goto err_srcs;
576
577 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); 453 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
578 if (!thread->srcs) 454 if (!thread->srcs)
579 goto err_srcs; 455 goto err_srcs;
@@ -597,17 +473,17 @@ static int dmatest_func(void *data)
597 set_user_nice(current, 10); 473 set_user_nice(current, 10);
598 474
599 /* 475 /*
600 * src buffers are freed by the DMAEngine code with dma_unmap_single() 476 * src and dst buffers are freed by ourselves below
601 * dst buffers are freed by ourselves below
602 */ 477 */
603 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT 478 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
604 | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE;
605 479
480 ktime = ktime_get();
606 while (!kthread_should_stop() 481 while (!kthread_should_stop()
607 && !(params->iterations && total_tests >= params->iterations)) { 482 && !(params->iterations && total_tests >= params->iterations)) {
608 struct dma_async_tx_descriptor *tx = NULL; 483 struct dma_async_tx_descriptor *tx = NULL;
609 dma_addr_t dma_srcs[src_cnt]; 484 struct dmaengine_unmap_data *um;
610 dma_addr_t dma_dsts[dst_cnt]; 485 dma_addr_t srcs[src_cnt];
486 dma_addr_t *dsts;
611 u8 align = 0; 487 u8 align = 0;
612 488
613 total_tests++; 489 total_tests++;
@@ -626,81 +502,103 @@ static int dmatest_func(void *data)
626 break; 502 break;
627 } 503 }
628 504
629 len = dmatest_random() % params->buf_size + 1; 505 if (params->noverify) {
506 len = params->buf_size;
507 src_off = 0;
508 dst_off = 0;
509 } else {
510 len = dmatest_random() % params->buf_size + 1;
511 len = (len >> align) << align;
512 if (!len)
513 len = 1 << align;
514 src_off = dmatest_random() % (params->buf_size - len + 1);
515 dst_off = dmatest_random() % (params->buf_size - len + 1);
516
517 src_off = (src_off >> align) << align;
518 dst_off = (dst_off >> align) << align;
519
520 dmatest_init_srcs(thread->srcs, src_off, len,
521 params->buf_size);
522 dmatest_init_dsts(thread->dsts, dst_off, len,
523 params->buf_size);
524 }
525
630 len = (len >> align) << align; 526 len = (len >> align) << align;
631 if (!len) 527 if (!len)
632 len = 1 << align; 528 len = 1 << align;
633 src_off = dmatest_random() % (params->buf_size - len + 1); 529 total_len += len;
634 dst_off = dmatest_random() % (params->buf_size - len + 1);
635 530
636 src_off = (src_off >> align) << align; 531 um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt,
637 dst_off = (dst_off >> align) << align; 532 GFP_KERNEL);
638 533 if (!um) {
639 dmatest_init_srcs(thread->srcs, src_off, len, params->buf_size); 534 failed_tests++;
640 dmatest_init_dsts(thread->dsts, dst_off, len, params->buf_size); 535 result("unmap data NULL", total_tests,
536 src_off, dst_off, len, ret);
537 continue;
538 }
641 539
540 um->len = params->buf_size;
642 for (i = 0; i < src_cnt; i++) { 541 for (i = 0; i < src_cnt; i++) {
643 u8 *buf = thread->srcs[i] + src_off; 542 unsigned long buf = (unsigned long) thread->srcs[i];
644 543 struct page *pg = virt_to_page(buf);
645 dma_srcs[i] = dma_map_single(dev->dev, buf, len, 544 unsigned pg_off = buf & ~PAGE_MASK;
646 DMA_TO_DEVICE); 545
647 ret = dma_mapping_error(dev->dev, dma_srcs[i]); 546 um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
547 um->len, DMA_TO_DEVICE);
548 srcs[i] = um->addr[i] + src_off;
549 ret = dma_mapping_error(dev->dev, um->addr[i]);
648 if (ret) { 550 if (ret) {
649 unmap_src(dev->dev, dma_srcs, len, i); 551 dmaengine_unmap_put(um);
650 thread_result_add(info, result, 552 result("src mapping error", total_tests,
651 DMATEST_ET_MAP_SRC, 553 src_off, dst_off, len, ret);
652 total_tests, src_off, dst_off,
653 len, ret);
654 failed_tests++; 554 failed_tests++;
655 continue; 555 continue;
656 } 556 }
557 um->to_cnt++;
657 } 558 }
658 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 559 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
560 dsts = &um->addr[src_cnt];
659 for (i = 0; i < dst_cnt; i++) { 561 for (i = 0; i < dst_cnt; i++) {
660 dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], 562 unsigned long buf = (unsigned long) thread->dsts[i];
661 params->buf_size, 563 struct page *pg = virt_to_page(buf);
662 DMA_BIDIRECTIONAL); 564 unsigned pg_off = buf & ~PAGE_MASK;
663 ret = dma_mapping_error(dev->dev, dma_dsts[i]); 565
566 dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
567 DMA_BIDIRECTIONAL);
568 ret = dma_mapping_error(dev->dev, dsts[i]);
664 if (ret) { 569 if (ret) {
665 unmap_src(dev->dev, dma_srcs, len, src_cnt); 570 dmaengine_unmap_put(um);
666 unmap_dst(dev->dev, dma_dsts, params->buf_size, 571 result("dst mapping error", total_tests,
667 i); 572 src_off, dst_off, len, ret);
668 thread_result_add(info, result,
669 DMATEST_ET_MAP_DST,
670 total_tests, src_off, dst_off,
671 len, ret);
672 failed_tests++; 573 failed_tests++;
673 continue; 574 continue;
674 } 575 }
576 um->bidi_cnt++;
675 } 577 }
676 578
677 if (thread->type == DMA_MEMCPY) 579 if (thread->type == DMA_MEMCPY)
678 tx = dev->device_prep_dma_memcpy(chan, 580 tx = dev->device_prep_dma_memcpy(chan,
679 dma_dsts[0] + dst_off, 581 dsts[0] + dst_off,
680 dma_srcs[0], len, 582 srcs[0], len, flags);
681 flags);
682 else if (thread->type == DMA_XOR) 583 else if (thread->type == DMA_XOR)
683 tx = dev->device_prep_dma_xor(chan, 584 tx = dev->device_prep_dma_xor(chan,
684 dma_dsts[0] + dst_off, 585 dsts[0] + dst_off,
685 dma_srcs, src_cnt, 586 srcs, src_cnt,
686 len, flags); 587 len, flags);
687 else if (thread->type == DMA_PQ) { 588 else if (thread->type == DMA_PQ) {
688 dma_addr_t dma_pq[dst_cnt]; 589 dma_addr_t dma_pq[dst_cnt];
689 590
690 for (i = 0; i < dst_cnt; i++) 591 for (i = 0; i < dst_cnt; i++)
691 dma_pq[i] = dma_dsts[i] + dst_off; 592 dma_pq[i] = dsts[i] + dst_off;
692 tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, 593 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
693 src_cnt, pq_coefs, 594 src_cnt, pq_coefs,
694 len, flags); 595 len, flags);
695 } 596 }
696 597
697 if (!tx) { 598 if (!tx) {
698 unmap_src(dev->dev, dma_srcs, len, src_cnt); 599 dmaengine_unmap_put(um);
699 unmap_dst(dev->dev, dma_dsts, params->buf_size, 600 result("prep error", total_tests, src_off,
700 dst_cnt); 601 dst_off, len, ret);
701 thread_result_add(info, result, DMATEST_ET_PREP,
702 total_tests, src_off, dst_off,
703 len, 0);
704 msleep(100); 602 msleep(100);
705 failed_tests++; 603 failed_tests++;
706 continue; 604 continue;
@@ -712,9 +610,9 @@ static int dmatest_func(void *data)
712 cookie = tx->tx_submit(tx); 610 cookie = tx->tx_submit(tx);
713 611
714 if (dma_submit_error(cookie)) { 612 if (dma_submit_error(cookie)) {
715 thread_result_add(info, result, DMATEST_ET_SUBMIT, 613 dmaengine_unmap_put(um);
716 total_tests, src_off, dst_off, 614 result("submit error", total_tests, src_off,
717 len, cookie); 615 dst_off, len, ret);
718 msleep(100); 616 msleep(100);
719 failed_tests++; 617 failed_tests++;
720 continue; 618 continue;
@@ -735,59 +633,59 @@ static int dmatest_func(void *data)
735 * free it this time?" dancing. For now, just 633 * free it this time?" dancing. For now, just
736 * leave it dangling. 634 * leave it dangling.
737 */ 635 */
738 thread_result_add(info, result, DMATEST_ET_TIMEOUT, 636 dmaengine_unmap_put(um);
739 total_tests, src_off, dst_off, 637 result("test timed out", total_tests, src_off, dst_off,
740 len, 0); 638 len, 0);
741 failed_tests++; 639 failed_tests++;
742 continue; 640 continue;
743 } else if (status != DMA_COMPLETE) { 641 } else if (status != DMA_COMPLETE) {
744 enum dmatest_error_type type = (status == DMA_ERROR) ? 642 dmaengine_unmap_put(um);
745 DMATEST_ET_DMA_ERROR : DMATEST_ET_DMA_IN_PROGRESS; 643 result(status == DMA_ERROR ?
746 thread_result_add(info, result, type, 644 "completion error status" :
747 total_tests, src_off, dst_off, 645 "completion busy status", total_tests, src_off,
748 len, status); 646 dst_off, len, ret);
749 failed_tests++; 647 failed_tests++;
750 continue; 648 continue;
751 } 649 }
752 650
753 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ 651 dmaengine_unmap_put(um);
754 unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt);
755 652
756 error_count = 0; 653 if (params->noverify) {
654 verbose_result("test passed", total_tests, src_off,
655 dst_off, len, 0);
656 continue;
657 }
757 658
758 pr_debug("%s: verifying source buffer...\n", thread_name); 659 pr_debug("%s: verifying source buffer...\n", current->comm);
759 error_count += verify_result_add(info, result, total_tests, 660 error_count = dmatest_verify(thread->srcs, 0, src_off,
760 src_off, dst_off, len, thread->srcs, -1,
761 0, PATTERN_SRC, true); 661 0, PATTERN_SRC, true);
762 error_count += verify_result_add(info, result, total_tests, 662 error_count += dmatest_verify(thread->srcs, src_off,
763 src_off, dst_off, len, thread->srcs, 0, 663 src_off + len, src_off,
764 src_off, PATTERN_SRC | PATTERN_COPY, true); 664 PATTERN_SRC | PATTERN_COPY, true);
765 error_count += verify_result_add(info, result, total_tests, 665 error_count += dmatest_verify(thread->srcs, src_off + len,
766 src_off, dst_off, len, thread->srcs, 1, 666 params->buf_size, src_off + len,
767 src_off + len, PATTERN_SRC, true); 667 PATTERN_SRC, true);
768 668
769 pr_debug("%s: verifying dest buffer...\n", thread_name); 669 pr_debug("%s: verifying dest buffer...\n", current->comm);
770 error_count += verify_result_add(info, result, total_tests, 670 error_count += dmatest_verify(thread->dsts, 0, dst_off,
771 src_off, dst_off, len, thread->dsts, -1,
772 0, PATTERN_DST, false); 671 0, PATTERN_DST, false);
773 error_count += verify_result_add(info, result, total_tests, 672 error_count += dmatest_verify(thread->dsts, dst_off,
774 src_off, dst_off, len, thread->dsts, 0, 673 dst_off + len, src_off,
775 src_off, PATTERN_SRC | PATTERN_COPY, false); 674 PATTERN_SRC | PATTERN_COPY, false);
776 error_count += verify_result_add(info, result, total_tests, 675 error_count += dmatest_verify(thread->dsts, dst_off + len,
777 src_off, dst_off, len, thread->dsts, 1, 676 params->buf_size, dst_off + len,
778 dst_off + len, PATTERN_DST, false); 677 PATTERN_DST, false);
779 678
780 if (error_count) { 679 if (error_count) {
781 thread_result_add(info, result, DMATEST_ET_VERIFY, 680 result("data error", total_tests, src_off, dst_off,
782 total_tests, src_off, dst_off, 681 len, error_count);
783 len, error_count);
784 failed_tests++; 682 failed_tests++;
785 } else { 683 } else {
786 thread_result_add(info, result, DMATEST_ET_OK, 684 verbose_result("test passed", total_tests, src_off,
787 total_tests, src_off, dst_off, 685 dst_off, len, 0);
788 len, 0);
789 } 686 }
790 } 687 }
688 runtime = ktime_us_delta(ktime_get(), ktime);
791 689
792 ret = 0; 690 ret = 0;
793 for (i = 0; thread->dsts[i]; i++) 691 for (i = 0; thread->dsts[i]; i++)
@@ -802,20 +700,17 @@ err_srcbuf:
802err_srcs: 700err_srcs:
803 kfree(pq_coefs); 701 kfree(pq_coefs);
804err_thread_type: 702err_thread_type:
805 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", 703 pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
806 thread_name, total_tests, failed_tests, ret); 704 current->comm, total_tests, failed_tests,
705 dmatest_persec(runtime, total_tests),
706 dmatest_KBs(runtime, total_len), ret);
807 707
808 /* terminate all transfers on specified channels */ 708 /* terminate all transfers on specified channels */
809 if (ret) 709 if (ret)
810 dmaengine_terminate_all(chan); 710 dmaengine_terminate_all(chan);
811 711
812 thread->done = true; 712 thread->done = true;
813 713 wake_up(&thread_wait);
814 if (params->iterations > 0)
815 while (!kthread_should_stop()) {
816 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit);
817 interruptible_sleep_on(&wait_dmatest_exit);
818 }
819 714
820 return ret; 715 return ret;
821} 716}
@@ -828,9 +723,10 @@ static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
828 723
829 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 724 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
830 ret = kthread_stop(thread->task); 725 ret = kthread_stop(thread->task);
831 pr_debug("dmatest: thread %s exited with status %d\n", 726 pr_debug("thread %s exited with status %d\n",
832 thread->task->comm, ret); 727 thread->task->comm, ret);
833 list_del(&thread->node); 728 list_del(&thread->node);
729 put_task_struct(thread->task);
834 kfree(thread); 730 kfree(thread);
835 } 731 }
836 732
@@ -861,27 +757,27 @@ static int dmatest_add_threads(struct dmatest_info *info,
861 for (i = 0; i < params->threads_per_chan; i++) { 757 for (i = 0; i < params->threads_per_chan; i++) {
862 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 758 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
863 if (!thread) { 759 if (!thread) {
864 pr_warning("dmatest: No memory for %s-%s%u\n", 760 pr_warn("No memory for %s-%s%u\n",
865 dma_chan_name(chan), op, i); 761 dma_chan_name(chan), op, i);
866
867 break; 762 break;
868 } 763 }
869 thread->info = info; 764 thread->info = info;
870 thread->chan = dtc->chan; 765 thread->chan = dtc->chan;
871 thread->type = type; 766 thread->type = type;
872 smp_wmb(); 767 smp_wmb();
873 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", 768 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
874 dma_chan_name(chan), op, i); 769 dma_chan_name(chan), op, i);
875 if (IS_ERR(thread->task)) { 770 if (IS_ERR(thread->task)) {
876 pr_warning("dmatest: Failed to run thread %s-%s%u\n", 771 pr_warn("Failed to create thread %s-%s%u\n",
877 dma_chan_name(chan), op, i); 772 dma_chan_name(chan), op, i);
878 kfree(thread); 773 kfree(thread);
879 break; 774 break;
880 } 775 }
881 776
882 /* srcbuf and dstbuf are allocated by the thread itself */ 777 /* srcbuf and dstbuf are allocated by the thread itself */
883 778 get_task_struct(thread->task);
884 list_add_tail(&thread->node, &dtc->threads); 779 list_add_tail(&thread->node, &dtc->threads);
780 wake_up_process(thread->task);
885 } 781 }
886 782
887 return i; 783 return i;
@@ -897,7 +793,7 @@ static int dmatest_add_channel(struct dmatest_info *info,
897 793
898 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 794 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
899 if (!dtc) { 795 if (!dtc) {
900 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan)); 796 pr_warn("No memory for %s\n", dma_chan_name(chan));
901 return -ENOMEM; 797 return -ENOMEM;
902 } 798 }
903 799
@@ -917,7 +813,7 @@ static int dmatest_add_channel(struct dmatest_info *info,
917 thread_count += cnt > 0 ? cnt : 0; 813 thread_count += cnt > 0 ? cnt : 0;
918 } 814 }
919 815
920 pr_info("dmatest: Started %u threads using %s\n", 816 pr_info("Started %u threads using %s\n",
921 thread_count, dma_chan_name(chan)); 817 thread_count, dma_chan_name(chan));
922 818
923 list_add_tail(&dtc->node, &info->channels); 819 list_add_tail(&dtc->node, &info->channels);
@@ -937,20 +833,20 @@ static bool filter(struct dma_chan *chan, void *param)
937 return true; 833 return true;
938} 834}
939 835
940static int __run_threaded_test(struct dmatest_info *info) 836static void request_channels(struct dmatest_info *info,
837 enum dma_transaction_type type)
941{ 838{
942 dma_cap_mask_t mask; 839 dma_cap_mask_t mask;
943 struct dma_chan *chan;
944 struct dmatest_params *params = &info->params;
945 int err = 0;
946 840
947 dma_cap_zero(mask); 841 dma_cap_zero(mask);
948 dma_cap_set(DMA_MEMCPY, mask); 842 dma_cap_set(type, mask);
949 for (;;) { 843 for (;;) {
844 struct dmatest_params *params = &info->params;
845 struct dma_chan *chan;
846
950 chan = dma_request_channel(mask, filter, params); 847 chan = dma_request_channel(mask, filter, params);
951 if (chan) { 848 if (chan) {
952 err = dmatest_add_channel(info, chan); 849 if (dmatest_add_channel(info, chan)) {
953 if (err) {
954 dma_release_channel(chan); 850 dma_release_channel(chan);
955 break; /* add_channel failed, punt */ 851 break; /* add_channel failed, punt */
956 } 852 }
@@ -960,22 +856,30 @@ static int __run_threaded_test(struct dmatest_info *info)
960 info->nr_channels >= params->max_channels) 856 info->nr_channels >= params->max_channels)
961 break; /* we have all we need */ 857 break; /* we have all we need */
962 } 858 }
963 return err;
964} 859}
965 860
966#ifndef MODULE 861static void run_threaded_test(struct dmatest_info *info)
967static int run_threaded_test(struct dmatest_info *info)
968{ 862{
969 int ret; 863 struct dmatest_params *params = &info->params;
970 864
971 mutex_lock(&info->lock); 865 /* Copy test parameters */
972 ret = __run_threaded_test(info); 866 params->buf_size = test_buf_size;
973 mutex_unlock(&info->lock); 867 strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
974 return ret; 868 strlcpy(params->device, strim(test_device), sizeof(params->device));
869 params->threads_per_chan = threads_per_chan;
870 params->max_channels = max_channels;
871 params->iterations = iterations;
872 params->xor_sources = xor_sources;
873 params->pq_sources = pq_sources;
874 params->timeout = timeout;
875 params->noverify = noverify;
876
877 request_channels(info, DMA_MEMCPY);
878 request_channels(info, DMA_XOR);
879 request_channels(info, DMA_PQ);
975} 880}
976#endif
977 881
978static void __stop_threaded_test(struct dmatest_info *info) 882static void stop_threaded_test(struct dmatest_info *info)
979{ 883{
980 struct dmatest_chan *dtc, *_dtc; 884 struct dmatest_chan *dtc, *_dtc;
981 struct dma_chan *chan; 885 struct dma_chan *chan;
@@ -984,203 +888,86 @@ static void __stop_threaded_test(struct dmatest_info *info)
984 list_del(&dtc->node); 888 list_del(&dtc->node);
985 chan = dtc->chan; 889 chan = dtc->chan;
986 dmatest_cleanup_channel(dtc); 890 dmatest_cleanup_channel(dtc);
987 pr_debug("dmatest: dropped channel %s\n", dma_chan_name(chan)); 891 pr_debug("dropped channel %s\n", dma_chan_name(chan));
988 dma_release_channel(chan); 892 dma_release_channel(chan);
989 } 893 }
990 894
991 info->nr_channels = 0; 895 info->nr_channels = 0;
992} 896}
993 897
994static void stop_threaded_test(struct dmatest_info *info) 898static void restart_threaded_test(struct dmatest_info *info, bool run)
995{ 899{
996 mutex_lock(&info->lock); 900 /* we might be called early to set run=, defer running until all
997 __stop_threaded_test(info); 901 * parameters have been evaluated
998 mutex_unlock(&info->lock); 902 */
999} 903 if (!info->did_init)
1000 904 return;
1001static int __restart_threaded_test(struct dmatest_info *info, bool run)
1002{
1003 struct dmatest_params *params = &info->params;
1004 905
1005 /* Stop any running test first */ 906 /* Stop any running test first */
1006 __stop_threaded_test(info); 907 stop_threaded_test(info);
1007
1008 if (run == false)
1009 return 0;
1010
1011 /* Clear results from previous run */
1012 result_free(info, NULL);
1013
1014 /* Copy test parameters */
1015 params->buf_size = test_buf_size;
1016 strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
1017 strlcpy(params->device, strim(test_device), sizeof(params->device));
1018 params->threads_per_chan = threads_per_chan;
1019 params->max_channels = max_channels;
1020 params->iterations = iterations;
1021 params->xor_sources = xor_sources;
1022 params->pq_sources = pq_sources;
1023 params->timeout = timeout;
1024 908
1025 /* Run test with new parameters */ 909 /* Run test with new parameters */
1026 return __run_threaded_test(info); 910 run_threaded_test(info);
1027}
1028
1029static bool __is_threaded_test_run(struct dmatest_info *info)
1030{
1031 struct dmatest_chan *dtc;
1032
1033 list_for_each_entry(dtc, &info->channels, node) {
1034 struct dmatest_thread *thread;
1035
1036 list_for_each_entry(thread, &dtc->threads, node) {
1037 if (!thread->done)
1038 return true;
1039 }
1040 }
1041
1042 return false;
1043} 911}
1044 912
1045static ssize_t dtf_read_run(struct file *file, char __user *user_buf, 913static int dmatest_run_get(char *val, const struct kernel_param *kp)
1046 size_t count, loff_t *ppos)
1047{ 914{
1048 struct dmatest_info *info = file->private_data; 915 struct dmatest_info *info = &test_info;
1049 char buf[3];
1050 916
1051 mutex_lock(&info->lock); 917 mutex_lock(&info->lock);
1052 918 if (is_threaded_test_run(info)) {
1053 if (__is_threaded_test_run(info)) { 919 dmatest_run = true;
1054 buf[0] = 'Y';
1055 } else { 920 } else {
1056 __stop_threaded_test(info); 921 stop_threaded_test(info);
1057 buf[0] = 'N'; 922 dmatest_run = false;
1058 } 923 }
1059
1060 mutex_unlock(&info->lock); 924 mutex_unlock(&info->lock);
1061 buf[1] = '\n';
1062 buf[2] = 0x00;
1063 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1064}
1065
1066static ssize_t dtf_write_run(struct file *file, const char __user *user_buf,
1067 size_t count, loff_t *ppos)
1068{
1069 struct dmatest_info *info = file->private_data;
1070 char buf[16];
1071 bool bv;
1072 int ret = 0;
1073 925
1074 if (copy_from_user(buf, user_buf, min(count, (sizeof(buf) - 1)))) 926 return param_get_bool(val, kp);
1075 return -EFAULT;
1076
1077 if (strtobool(buf, &bv) == 0) {
1078 mutex_lock(&info->lock);
1079
1080 if (__is_threaded_test_run(info))
1081 ret = -EBUSY;
1082 else
1083 ret = __restart_threaded_test(info, bv);
1084
1085 mutex_unlock(&info->lock);
1086 }
1087
1088 return ret ? ret : count;
1089} 927}
1090 928
1091static const struct file_operations dtf_run_fops = { 929static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1092 .read = dtf_read_run,
1093 .write = dtf_write_run,
1094 .open = simple_open,
1095 .llseek = default_llseek,
1096};
1097
1098static int dtf_results_show(struct seq_file *sf, void *data)
1099{ 930{
1100 struct dmatest_info *info = sf->private; 931 struct dmatest_info *info = &test_info;
1101 struct dmatest_result *result; 932 int ret;
1102 struct dmatest_thread_result *tr;
1103 unsigned int i;
1104 933
1105 mutex_lock(&info->results_lock); 934 mutex_lock(&info->lock);
1106 list_for_each_entry(result, &info->results, node) { 935 ret = param_set_bool(val, kp);
1107 list_for_each_entry(tr, &result->results, node) { 936 if (ret) {
1108 seq_printf(sf, "%s\n", 937 mutex_unlock(&info->lock);
1109 thread_result_get(result->name, tr)); 938 return ret;
1110 if (tr->type == DMATEST_ET_VERIFY_BUF) {
1111 for (i = 0; i < tr->vr->error_count; i++) {
1112 seq_printf(sf, "\t%s\n",
1113 verify_result_get_one(tr->vr, i));
1114 }
1115 }
1116 }
1117 } 939 }
1118 940
1119 mutex_unlock(&info->results_lock); 941 if (is_threaded_test_run(info))
1120 return 0; 942 ret = -EBUSY;
1121} 943 else if (dmatest_run)
1122 944 restart_threaded_test(info, dmatest_run);
1123static int dtf_results_open(struct inode *inode, struct file *file)
1124{
1125 return single_open(file, dtf_results_show, inode->i_private);
1126}
1127
1128static const struct file_operations dtf_results_fops = {
1129 .open = dtf_results_open,
1130 .read = seq_read,
1131 .llseek = seq_lseek,
1132 .release = single_release,
1133};
1134
1135static int dmatest_register_dbgfs(struct dmatest_info *info)
1136{
1137 struct dentry *d;
1138
1139 d = debugfs_create_dir("dmatest", NULL);
1140 if (IS_ERR(d))
1141 return PTR_ERR(d);
1142 if (!d)
1143 goto err_root;
1144 945
1145 info->root = d; 946 mutex_unlock(&info->lock);
1146
1147 /* Run or stop threaded test */
1148 debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root, info,
1149 &dtf_run_fops);
1150
1151 /* Results of test in progress */
1152 debugfs_create_file("results", S_IRUGO, info->root, info,
1153 &dtf_results_fops);
1154
1155 return 0;
1156 947
1157err_root: 948 return ret;
1158 pr_err("dmatest: Failed to initialize debugfs\n");
1159 return -ENOMEM;
1160} 949}
1161 950
1162static int __init dmatest_init(void) 951static int __init dmatest_init(void)
1163{ 952{
1164 struct dmatest_info *info = &test_info; 953 struct dmatest_info *info = &test_info;
1165 int ret; 954 struct dmatest_params *params = &info->params;
1166
1167 memset(info, 0, sizeof(*info));
1168 955
1169 mutex_init(&info->lock); 956 if (dmatest_run) {
1170 INIT_LIST_HEAD(&info->channels); 957 mutex_lock(&info->lock);
958 run_threaded_test(info);
959 mutex_unlock(&info->lock);
960 }
1171 961
1172 mutex_init(&info->results_lock); 962 if (params->iterations && wait)
1173 INIT_LIST_HEAD(&info->results); 963 wait_event(thread_wait, !is_threaded_test_run(info));
1174 964
1175 ret = dmatest_register_dbgfs(info); 965 /* module parameters are stable, inittime tests are started,
1176 if (ret) 966 * let userspace take over 'run' control
1177 return ret; 967 */
968 info->did_init = true;
1178 969
1179#ifdef MODULE
1180 return 0; 970 return 0;
1181#else
1182 return run_threaded_test(info);
1183#endif
1184} 971}
1185/* when compiled-in wait for drivers to load first */ 972/* when compiled-in wait for drivers to load first */
1186late_initcall(dmatest_init); 973late_initcall(dmatest_init);
@@ -1189,9 +976,9 @@ static void __exit dmatest_exit(void)
1189{ 976{
1190 struct dmatest_info *info = &test_info; 977 struct dmatest_info *info = &test_info;
1191 978
1192 debugfs_remove_recursive(info->root); 979 mutex_lock(&info->lock);
1193 stop_threaded_test(info); 980 stop_threaded_test(info);
1194 result_free(info, NULL); 981 mutex_unlock(&info->lock);
1195} 982}
1196module_exit(dmatest_exit); 983module_exit(dmatest_exit);
1197 984
diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c
index 2c29331571e4..7516be4677cf 100644
--- a/drivers/dma/dw/core.c
+++ b/drivers/dma/dw/core.c
@@ -85,10 +85,6 @@ static struct device *chan2dev(struct dma_chan *chan)
85{ 85{
86 return &chan->dev->device; 86 return &chan->dev->device;
87} 87}
88static struct device *chan2parent(struct dma_chan *chan)
89{
90 return chan->dev->device.parent;
91}
92 88
93static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc) 89static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
94{ 90{
@@ -311,26 +307,7 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
311 list_splice_init(&desc->tx_list, &dwc->free_list); 307 list_splice_init(&desc->tx_list, &dwc->free_list);
312 list_move(&desc->desc_node, &dwc->free_list); 308 list_move(&desc->desc_node, &dwc->free_list);
313 309
314 if (!is_slave_direction(dwc->direction)) { 310 dma_descriptor_unmap(txd);
315 struct device *parent = chan2parent(&dwc->chan);
316 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
317 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
318 dma_unmap_single(parent, desc->lli.dar,
319 desc->total_len, DMA_FROM_DEVICE);
320 else
321 dma_unmap_page(parent, desc->lli.dar,
322 desc->total_len, DMA_FROM_DEVICE);
323 }
324 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
325 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
326 dma_unmap_single(parent, desc->lli.sar,
327 desc->total_len, DMA_TO_DEVICE);
328 else
329 dma_unmap_page(parent, desc->lli.sar,
330 desc->total_len, DMA_TO_DEVICE);
331 }
332 }
333
334 spin_unlock_irqrestore(&dwc->lock, flags); 311 spin_unlock_irqrestore(&dwc->lock, flags);
335 312
336 if (callback) 313 if (callback)
diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c
index 591cd8c63abb..cb4bf682a708 100644
--- a/drivers/dma/ep93xx_dma.c
+++ b/drivers/dma/ep93xx_dma.c
@@ -733,28 +733,6 @@ static void ep93xx_dma_advance_work(struct ep93xx_dma_chan *edmac)
733 spin_unlock_irqrestore(&edmac->lock, flags); 733 spin_unlock_irqrestore(&edmac->lock, flags);
734} 734}
735 735
736static void ep93xx_dma_unmap_buffers(struct ep93xx_dma_desc *desc)
737{
738 struct device *dev = desc->txd.chan->device->dev;
739
740 if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
741 if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
742 dma_unmap_single(dev, desc->src_addr, desc->size,
743 DMA_TO_DEVICE);
744 else
745 dma_unmap_page(dev, desc->src_addr, desc->size,
746 DMA_TO_DEVICE);
747 }
748 if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
749 if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
750 dma_unmap_single(dev, desc->dst_addr, desc->size,
751 DMA_FROM_DEVICE);
752 else
753 dma_unmap_page(dev, desc->dst_addr, desc->size,
754 DMA_FROM_DEVICE);
755 }
756}
757
758static void ep93xx_dma_tasklet(unsigned long data) 736static void ep93xx_dma_tasklet(unsigned long data)
759{ 737{
760 struct ep93xx_dma_chan *edmac = (struct ep93xx_dma_chan *)data; 738 struct ep93xx_dma_chan *edmac = (struct ep93xx_dma_chan *)data;
@@ -787,13 +765,7 @@ static void ep93xx_dma_tasklet(unsigned long data)
787 765
788 /* Now we can release all the chained descriptors */ 766 /* Now we can release all the chained descriptors */
789 list_for_each_entry_safe(desc, d, &list, node) { 767 list_for_each_entry_safe(desc, d, &list, node) {
790 /* 768 dma_descriptor_unmap(&desc->txd);
791 * For the memcpy channels the API requires us to unmap the
792 * buffers unless requested otherwise.
793 */
794 if (!edmac->chan.private)
795 ep93xx_dma_unmap_buffers(desc);
796
797 ep93xx_dma_desc_put(edmac, desc); 769 ep93xx_dma_desc_put(edmac, desc);
798 } 770 }
799 771
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index 4e898121707b..bd7f888b7601 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -868,22 +868,7 @@ static void fsldma_cleanup_descriptor(struct fsldma_chan *chan,
868 /* Run any dependencies */ 868 /* Run any dependencies */
869 dma_run_dependencies(txd); 869 dma_run_dependencies(txd);
870 870
871 /* Unmap the dst buffer, if requested */ 871 dma_descriptor_unmap(txd);
872 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
873 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
874 dma_unmap_single(dev, dst, len, DMA_FROM_DEVICE);
875 else
876 dma_unmap_page(dev, dst, len, DMA_FROM_DEVICE);
877 }
878
879 /* Unmap the src buffer, if requested */
880 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
881 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
882 dma_unmap_single(dev, src, len, DMA_TO_DEVICE);
883 else
884 dma_unmap_page(dev, src, len, DMA_TO_DEVICE);
885 }
886
887#ifdef FSL_DMA_LD_DEBUG 872#ifdef FSL_DMA_LD_DEBUG
888 chan_dbg(chan, "LD %p free\n", desc); 873 chan_dbg(chan, "LD %p free\n", desc);
889#endif 874#endif
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index a0f0fce5a84e..1a49c777607c 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -531,21 +531,6 @@ static void ioat1_cleanup_event(unsigned long data)
531 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); 531 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
532} 532}
533 533
534void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
535 size_t len, struct ioat_dma_descriptor *hw)
536{
537 struct pci_dev *pdev = chan->device->pdev;
538 size_t offset = len - hw->size;
539
540 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
541 ioat_unmap(pdev, hw->dst_addr - offset, len,
542 PCI_DMA_FROMDEVICE, flags, 1);
543
544 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP))
545 ioat_unmap(pdev, hw->src_addr - offset, len,
546 PCI_DMA_TODEVICE, flags, 0);
547}
548
549dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) 534dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan)
550{ 535{
551 dma_addr_t phys_complete; 536 dma_addr_t phys_complete;
@@ -602,7 +587,7 @@ static void __cleanup(struct ioat_dma_chan *ioat, dma_addr_t phys_complete)
602 dump_desc_dbg(ioat, desc); 587 dump_desc_dbg(ioat, desc);
603 if (tx->cookie) { 588 if (tx->cookie) {
604 dma_cookie_complete(tx); 589 dma_cookie_complete(tx);
605 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw); 590 dma_descriptor_unmap(tx);
606 ioat->active -= desc->hw->tx_cnt; 591 ioat->active -= desc->hw->tx_cnt;
607 if (tx->callback) { 592 if (tx->callback) {
608 tx->callback(tx->callback_param); 593 tx->callback(tx->callback_param);
@@ -833,8 +818,7 @@ int ioat_dma_self_test(struct ioatdma_device *device)
833 818
834 dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); 819 dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
835 dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); 820 dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
836 flags = DMA_COMPL_SKIP_SRC_UNMAP | DMA_COMPL_SKIP_DEST_UNMAP | 821 flags = DMA_PREP_INTERRUPT;
837 DMA_PREP_INTERRUPT;
838 tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src, 822 tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
839 IOAT_TEST_SIZE, flags); 823 IOAT_TEST_SIZE, flags);
840 if (!tx) { 824 if (!tx) {
@@ -885,8 +869,7 @@ static char ioat_interrupt_style[32] = "msix";
885module_param_string(ioat_interrupt_style, ioat_interrupt_style, 869module_param_string(ioat_interrupt_style, ioat_interrupt_style,
886 sizeof(ioat_interrupt_style), 0644); 870 sizeof(ioat_interrupt_style), 0644);
887MODULE_PARM_DESC(ioat_interrupt_style, 871MODULE_PARM_DESC(ioat_interrupt_style,
888 "set ioat interrupt style: msix (default), " 872 "set ioat interrupt style: msix (default), msi, intx");
889 "msix-single-vector, msi, intx)");
890 873
891/** 874/**
892 * ioat_dma_setup_interrupts - setup interrupt handler 875 * ioat_dma_setup_interrupts - setup interrupt handler
@@ -904,8 +887,6 @@ int ioat_dma_setup_interrupts(struct ioatdma_device *device)
904 887
905 if (!strcmp(ioat_interrupt_style, "msix")) 888 if (!strcmp(ioat_interrupt_style, "msix"))
906 goto msix; 889 goto msix;
907 if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
908 goto msix_single_vector;
909 if (!strcmp(ioat_interrupt_style, "msi")) 890 if (!strcmp(ioat_interrupt_style, "msi"))
910 goto msi; 891 goto msi;
911 if (!strcmp(ioat_interrupt_style, "intx")) 892 if (!strcmp(ioat_interrupt_style, "intx"))
@@ -920,10 +901,8 @@ msix:
920 device->msix_entries[i].entry = i; 901 device->msix_entries[i].entry = i;
921 902
922 err = pci_enable_msix(pdev, device->msix_entries, msixcnt); 903 err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
923 if (err < 0) 904 if (err)
924 goto msi; 905 goto msi;
925 if (err > 0)
926 goto msix_single_vector;
927 906
928 for (i = 0; i < msixcnt; i++) { 907 for (i = 0; i < msixcnt; i++) {
929 msix = &device->msix_entries[i]; 908 msix = &device->msix_entries[i];
@@ -937,29 +916,13 @@ msix:
937 chan = ioat_chan_by_index(device, j); 916 chan = ioat_chan_by_index(device, j);
938 devm_free_irq(dev, msix->vector, chan); 917 devm_free_irq(dev, msix->vector, chan);
939 } 918 }
940 goto msix_single_vector; 919 goto msi;
941 } 920 }
942 } 921 }
943 intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL; 922 intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
944 device->irq_mode = IOAT_MSIX; 923 device->irq_mode = IOAT_MSIX;
945 goto done; 924 goto done;
946 925
947msix_single_vector:
948 msix = &device->msix_entries[0];
949 msix->entry = 0;
950 err = pci_enable_msix(pdev, device->msix_entries, 1);
951 if (err)
952 goto msi;
953
954 err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
955 "ioat-msix", device);
956 if (err) {
957 pci_disable_msix(pdev);
958 goto msi;
959 }
960 device->irq_mode = IOAT_MSIX_SINGLE;
961 goto done;
962
963msi: 926msi:
964 err = pci_enable_msi(pdev); 927 err = pci_enable_msi(pdev);
965 if (err) 928 if (err)
@@ -971,7 +934,7 @@ msi:
971 pci_disable_msi(pdev); 934 pci_disable_msi(pdev);
972 goto intx; 935 goto intx;
973 } 936 }
974 device->irq_mode = IOAT_MSIX; 937 device->irq_mode = IOAT_MSI;
975 goto done; 938 goto done;
976 939
977intx: 940intx:
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 54fb7b9ff9aa..11fb877ddca9 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -52,7 +52,6 @@
52enum ioat_irq_mode { 52enum ioat_irq_mode {
53 IOAT_NOIRQ = 0, 53 IOAT_NOIRQ = 0,
54 IOAT_MSIX, 54 IOAT_MSIX,
55 IOAT_MSIX_SINGLE,
56 IOAT_MSI, 55 IOAT_MSI,
57 IOAT_INTX 56 IOAT_INTX
58}; 57};
@@ -83,7 +82,6 @@ struct ioatdma_device {
83 struct pci_pool *completion_pool; 82 struct pci_pool *completion_pool;
84#define MAX_SED_POOLS 5 83#define MAX_SED_POOLS 5
85 struct dma_pool *sed_hw_pool[MAX_SED_POOLS]; 84 struct dma_pool *sed_hw_pool[MAX_SED_POOLS];
86 struct kmem_cache *sed_pool;
87 struct dma_device common; 85 struct dma_device common;
88 u8 version; 86 u8 version;
89 struct msix_entry msix_entries[4]; 87 struct msix_entry msix_entries[4];
@@ -342,16 +340,6 @@ static inline bool is_ioat_bug(unsigned long err)
342 return !!err; 340 return !!err;
343} 341}
344 342
345static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
346 int direction, enum dma_ctrl_flags flags, bool dst)
347{
348 if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
349 (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
350 pci_unmap_single(pdev, addr, len, direction);
351 else
352 pci_unmap_page(pdev, addr, len, direction);
353}
354
355int ioat_probe(struct ioatdma_device *device); 343int ioat_probe(struct ioatdma_device *device);
356int ioat_register(struct ioatdma_device *device); 344int ioat_register(struct ioatdma_device *device);
357int ioat1_dma_probe(struct ioatdma_device *dev, int dca); 345int ioat1_dma_probe(struct ioatdma_device *dev, int dca);
@@ -363,8 +351,6 @@ void ioat_init_channel(struct ioatdma_device *device,
363 struct ioat_chan_common *chan, int idx); 351 struct ioat_chan_common *chan, int idx);
364enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, 352enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
365 struct dma_tx_state *txstate); 353 struct dma_tx_state *txstate);
366void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
367 size_t len, struct ioat_dma_descriptor *hw);
368bool ioat_cleanup_preamble(struct ioat_chan_common *chan, 354bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
369 dma_addr_t *phys_complete); 355 dma_addr_t *phys_complete);
370void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type); 356void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type);
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
index b925e1b1d139..5d3affe7e976 100644
--- a/drivers/dma/ioat/dma_v2.c
+++ b/drivers/dma/ioat/dma_v2.c
@@ -148,7 +148,7 @@ static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)
148 tx = &desc->txd; 148 tx = &desc->txd;
149 dump_desc_dbg(ioat, desc); 149 dump_desc_dbg(ioat, desc);
150 if (tx->cookie) { 150 if (tx->cookie) {
151 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw); 151 dma_descriptor_unmap(tx);
152 dma_cookie_complete(tx); 152 dma_cookie_complete(tx);
153 if (tx->callback) { 153 if (tx->callback) {
154 tx->callback(tx->callback_param); 154 tx->callback(tx->callback_param);
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
index 212d584fe427..470292767e68 100644
--- a/drivers/dma/ioat/dma_v2.h
+++ b/drivers/dma/ioat/dma_v2.h
@@ -157,7 +157,6 @@ static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
157 157
158int ioat2_dma_probe(struct ioatdma_device *dev, int dca); 158int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
159int ioat3_dma_probe(struct ioatdma_device *dev, int dca); 159int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
160void ioat3_dma_remove(struct ioatdma_device *dev);
161struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); 160struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
162struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); 161struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
163int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs); 162int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs);
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c
index 806b4ce5e38c..820817e97e62 100644
--- a/drivers/dma/ioat/dma_v3.c
+++ b/drivers/dma/ioat/dma_v3.c
@@ -67,6 +67,8 @@
67#include "dma.h" 67#include "dma.h"
68#include "dma_v2.h" 68#include "dma_v2.h"
69 69
70extern struct kmem_cache *ioat3_sed_cache;
71
70/* ioat hardware assumes at least two sources for raid operations */ 72/* ioat hardware assumes at least two sources for raid operations */
71#define src_cnt_to_sw(x) ((x) + 2) 73#define src_cnt_to_sw(x) ((x) + 2)
72#define src_cnt_to_hw(x) ((x) - 2) 74#define src_cnt_to_hw(x) ((x) - 2)
@@ -87,22 +89,8 @@ static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 };
87static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7, 89static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7,
88 0, 1, 2, 3, 4, 5, 6 }; 90 0, 1, 2, 3, 4, 5, 6 };
89 91
90/*
91 * technically sources 1 and 2 do not require SED, but the op will have
92 * at least 9 descriptors so that's irrelevant.
93 */
94static const u8 pq16_idx_to_sed[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0,
95 1, 1, 1, 1, 1, 1, 1 };
96
97static void ioat3_eh(struct ioat2_dma_chan *ioat); 92static void ioat3_eh(struct ioat2_dma_chan *ioat);
98 93
99static dma_addr_t xor_get_src(struct ioat_raw_descriptor *descs[2], int idx)
100{
101 struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
102
103 return raw->field[xor_idx_to_field[idx]];
104}
105
106static void xor_set_src(struct ioat_raw_descriptor *descs[2], 94static void xor_set_src(struct ioat_raw_descriptor *descs[2],
107 dma_addr_t addr, u32 offset, int idx) 95 dma_addr_t addr, u32 offset, int idx)
108{ 96{
@@ -135,12 +123,6 @@ static void pq_set_src(struct ioat_raw_descriptor *descs[2],
135 pq->coef[idx] = coef; 123 pq->coef[idx] = coef;
136} 124}
137 125
138static int sed_get_pq16_pool_idx(int src_cnt)
139{
140
141 return pq16_idx_to_sed[src_cnt];
142}
143
144static bool is_jf_ioat(struct pci_dev *pdev) 126static bool is_jf_ioat(struct pci_dev *pdev)
145{ 127{
146 switch (pdev->device) { 128 switch (pdev->device) {
@@ -272,7 +254,7 @@ ioat3_alloc_sed(struct ioatdma_device *device, unsigned int hw_pool)
272 struct ioat_sed_ent *sed; 254 struct ioat_sed_ent *sed;
273 gfp_t flags = __GFP_ZERO | GFP_ATOMIC; 255 gfp_t flags = __GFP_ZERO | GFP_ATOMIC;
274 256
275 sed = kmem_cache_alloc(device->sed_pool, flags); 257 sed = kmem_cache_alloc(ioat3_sed_cache, flags);
276 if (!sed) 258 if (!sed)
277 return NULL; 259 return NULL;
278 260
@@ -280,7 +262,7 @@ ioat3_alloc_sed(struct ioatdma_device *device, unsigned int hw_pool)
280 sed->hw = dma_pool_alloc(device->sed_hw_pool[hw_pool], 262 sed->hw = dma_pool_alloc(device->sed_hw_pool[hw_pool],
281 flags, &sed->dma); 263 flags, &sed->dma);
282 if (!sed->hw) { 264 if (!sed->hw) {
283 kmem_cache_free(device->sed_pool, sed); 265 kmem_cache_free(ioat3_sed_cache, sed);
284 return NULL; 266 return NULL;
285 } 267 }
286 268
@@ -293,165 +275,7 @@ static void ioat3_free_sed(struct ioatdma_device *device, struct ioat_sed_ent *s
293 return; 275 return;
294 276
295 dma_pool_free(device->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma); 277 dma_pool_free(device->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma);
296 kmem_cache_free(device->sed_pool, sed); 278 kmem_cache_free(ioat3_sed_cache, sed);
297}
298
299static void ioat3_dma_unmap(struct ioat2_dma_chan *ioat,
300 struct ioat_ring_ent *desc, int idx)
301{
302 struct ioat_chan_common *chan = &ioat->base;
303 struct pci_dev *pdev = chan->device->pdev;
304 size_t len = desc->len;
305 size_t offset = len - desc->hw->size;
306 struct dma_async_tx_descriptor *tx = &desc->txd;
307 enum dma_ctrl_flags flags = tx->flags;
308
309 switch (desc->hw->ctl_f.op) {
310 case IOAT_OP_COPY:
311 if (!desc->hw->ctl_f.null) /* skip 'interrupt' ops */
312 ioat_dma_unmap(chan, flags, len, desc->hw);
313 break;
314 case IOAT_OP_XOR_VAL:
315 case IOAT_OP_XOR: {
316 struct ioat_xor_descriptor *xor = desc->xor;
317 struct ioat_ring_ent *ext;
318 struct ioat_xor_ext_descriptor *xor_ex = NULL;
319 int src_cnt = src_cnt_to_sw(xor->ctl_f.src_cnt);
320 struct ioat_raw_descriptor *descs[2];
321 int i;
322
323 if (src_cnt > 5) {
324 ext = ioat2_get_ring_ent(ioat, idx + 1);
325 xor_ex = ext->xor_ex;
326 }
327
328 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
329 descs[0] = (struct ioat_raw_descriptor *) xor;
330 descs[1] = (struct ioat_raw_descriptor *) xor_ex;
331 for (i = 0; i < src_cnt; i++) {
332 dma_addr_t src = xor_get_src(descs, i);
333
334 ioat_unmap(pdev, src - offset, len,
335 PCI_DMA_TODEVICE, flags, 0);
336 }
337
338 /* dest is a source in xor validate operations */
339 if (xor->ctl_f.op == IOAT_OP_XOR_VAL) {
340 ioat_unmap(pdev, xor->dst_addr - offset, len,
341 PCI_DMA_TODEVICE, flags, 1);
342 break;
343 }
344 }
345
346 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
347 ioat_unmap(pdev, xor->dst_addr - offset, len,
348 PCI_DMA_FROMDEVICE, flags, 1);
349 break;
350 }
351 case IOAT_OP_PQ_VAL:
352 case IOAT_OP_PQ: {
353 struct ioat_pq_descriptor *pq = desc->pq;
354 struct ioat_ring_ent *ext;
355 struct ioat_pq_ext_descriptor *pq_ex = NULL;
356 int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
357 struct ioat_raw_descriptor *descs[2];
358 int i;
359
360 if (src_cnt > 3) {
361 ext = ioat2_get_ring_ent(ioat, idx + 1);
362 pq_ex = ext->pq_ex;
363 }
364
365 /* in the 'continue' case don't unmap the dests as sources */
366 if (dmaf_p_disabled_continue(flags))
367 src_cnt--;
368 else if (dmaf_continue(flags))
369 src_cnt -= 3;
370
371 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
372 descs[0] = (struct ioat_raw_descriptor *) pq;
373 descs[1] = (struct ioat_raw_descriptor *) pq_ex;
374 for (i = 0; i < src_cnt; i++) {
375 dma_addr_t src = pq_get_src(descs, i);
376
377 ioat_unmap(pdev, src - offset, len,
378 PCI_DMA_TODEVICE, flags, 0);
379 }
380
381 /* the dests are sources in pq validate operations */
382 if (pq->ctl_f.op == IOAT_OP_XOR_VAL) {
383 if (!(flags & DMA_PREP_PQ_DISABLE_P))
384 ioat_unmap(pdev, pq->p_addr - offset,
385 len, PCI_DMA_TODEVICE, flags, 0);
386 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
387 ioat_unmap(pdev, pq->q_addr - offset,
388 len, PCI_DMA_TODEVICE, flags, 0);
389 break;
390 }
391 }
392
393 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
394 if (!(flags & DMA_PREP_PQ_DISABLE_P))
395 ioat_unmap(pdev, pq->p_addr - offset, len,
396 PCI_DMA_BIDIRECTIONAL, flags, 1);
397 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
398 ioat_unmap(pdev, pq->q_addr - offset, len,
399 PCI_DMA_BIDIRECTIONAL, flags, 1);
400 }
401 break;
402 }
403 case IOAT_OP_PQ_16S:
404 case IOAT_OP_PQ_VAL_16S: {
405 struct ioat_pq_descriptor *pq = desc->pq;
406 int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt);
407 struct ioat_raw_descriptor *descs[4];
408 int i;
409
410 /* in the 'continue' case don't unmap the dests as sources */
411 if (dmaf_p_disabled_continue(flags))
412 src_cnt--;
413 else if (dmaf_continue(flags))
414 src_cnt -= 3;
415
416 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
417 descs[0] = (struct ioat_raw_descriptor *)pq;
418 descs[1] = (struct ioat_raw_descriptor *)(desc->sed->hw);
419 descs[2] = (struct ioat_raw_descriptor *)(&desc->sed->hw->b[0]);
420 for (i = 0; i < src_cnt; i++) {
421 dma_addr_t src = pq16_get_src(descs, i);
422
423 ioat_unmap(pdev, src - offset, len,
424 PCI_DMA_TODEVICE, flags, 0);
425 }
426
427 /* the dests are sources in pq validate operations */
428 if (pq->ctl_f.op == IOAT_OP_XOR_VAL) {
429 if (!(flags & DMA_PREP_PQ_DISABLE_P))
430 ioat_unmap(pdev, pq->p_addr - offset,
431 len, PCI_DMA_TODEVICE,
432 flags, 0);
433 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
434 ioat_unmap(pdev, pq->q_addr - offset,
435 len, PCI_DMA_TODEVICE,
436 flags, 0);
437 break;
438 }
439 }
440
441 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
442 if (!(flags & DMA_PREP_PQ_DISABLE_P))
443 ioat_unmap(pdev, pq->p_addr - offset, len,
444 PCI_DMA_BIDIRECTIONAL, flags, 1);
445 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
446 ioat_unmap(pdev, pq->q_addr - offset, len,
447 PCI_DMA_BIDIRECTIONAL, flags, 1);
448 }
449 break;
450 }
451 default:
452 dev_err(&pdev->dev, "%s: unknown op type: %#x\n",
453 __func__, desc->hw->ctl_f.op);
454 }
455} 279}
456 280
457static bool desc_has_ext(struct ioat_ring_ent *desc) 281static bool desc_has_ext(struct ioat_ring_ent *desc)
@@ -577,7 +401,7 @@ static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)
577 tx = &desc->txd; 401 tx = &desc->txd;
578 if (tx->cookie) { 402 if (tx->cookie) {
579 dma_cookie_complete(tx); 403 dma_cookie_complete(tx);
580 ioat3_dma_unmap(ioat, desc, idx + i); 404 dma_descriptor_unmap(tx);
581 if (tx->callback) { 405 if (tx->callback) {
582 tx->callback(tx->callback_param); 406 tx->callback(tx->callback_param);
583 tx->callback = NULL; 407 tx->callback = NULL;
@@ -1129,9 +953,6 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
1129 u8 op; 953 u8 op;
1130 int i, s, idx, num_descs; 954 int i, s, idx, num_descs;
1131 955
1132 /* this function only handles src_cnt 9 - 16 */
1133 BUG_ON(src_cnt < 9);
1134
1135 /* this function is only called with 9-16 sources */ 956 /* this function is only called with 9-16 sources */
1136 op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S; 957 op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S;
1137 958
@@ -1159,8 +980,7 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
1159 980
1160 descs[0] = (struct ioat_raw_descriptor *) pq; 981 descs[0] = (struct ioat_raw_descriptor *) pq;
1161 982
1162 desc->sed = ioat3_alloc_sed(device, 983 desc->sed = ioat3_alloc_sed(device, (src_cnt-2) >> 3);
1163 sed_get_pq16_pool_idx(src_cnt));
1164 if (!desc->sed) { 984 if (!desc->sed) {
1165 dev_err(to_dev(chan), 985 dev_err(to_dev(chan),
1166 "%s: no free sed entries\n", __func__); 986 "%s: no free sed entries\n", __func__);
@@ -1218,13 +1038,21 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
1218 return &desc->txd; 1038 return &desc->txd;
1219} 1039}
1220 1040
1041static int src_cnt_flags(unsigned int src_cnt, unsigned long flags)
1042{
1043 if (dmaf_p_disabled_continue(flags))
1044 return src_cnt + 1;
1045 else if (dmaf_continue(flags))
1046 return src_cnt + 3;
1047 else
1048 return src_cnt;
1049}
1050
1221static struct dma_async_tx_descriptor * 1051static struct dma_async_tx_descriptor *
1222ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 1052ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
1223 unsigned int src_cnt, const unsigned char *scf, size_t len, 1053 unsigned int src_cnt, const unsigned char *scf, size_t len,
1224 unsigned long flags) 1054 unsigned long flags)
1225{ 1055{
1226 struct dma_device *dma = chan->device;
1227
1228 /* specify valid address for disabled result */ 1056 /* specify valid address for disabled result */
1229 if (flags & DMA_PREP_PQ_DISABLE_P) 1057 if (flags & DMA_PREP_PQ_DISABLE_P)
1230 dst[0] = dst[1]; 1058 dst[0] = dst[1];
@@ -1244,7 +1072,7 @@ ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
1244 single_source_coef[0] = scf[0]; 1072 single_source_coef[0] = scf[0];
1245 single_source_coef[1] = 0; 1073 single_source_coef[1] = 0;
1246 1074
1247 return (src_cnt > 8) && (dma->max_pq > 8) ? 1075 return src_cnt_flags(src_cnt, flags) > 8 ?
1248 __ioat3_prep_pq16_lock(chan, NULL, dst, single_source, 1076 __ioat3_prep_pq16_lock(chan, NULL, dst, single_source,
1249 2, single_source_coef, len, 1077 2, single_source_coef, len,
1250 flags) : 1078 flags) :
@@ -1252,7 +1080,7 @@ ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
1252 single_source_coef, len, flags); 1080 single_source_coef, len, flags);
1253 1081
1254 } else { 1082 } else {
1255 return (src_cnt > 8) && (dma->max_pq > 8) ? 1083 return src_cnt_flags(src_cnt, flags) > 8 ?
1256 __ioat3_prep_pq16_lock(chan, NULL, dst, src, src_cnt, 1084 __ioat3_prep_pq16_lock(chan, NULL, dst, src, src_cnt,
1257 scf, len, flags) : 1085 scf, len, flags) :
1258 __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt, 1086 __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt,
@@ -1265,8 +1093,6 @@ ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
1265 unsigned int src_cnt, const unsigned char *scf, size_t len, 1093 unsigned int src_cnt, const unsigned char *scf, size_t len,
1266 enum sum_check_flags *pqres, unsigned long flags) 1094 enum sum_check_flags *pqres, unsigned long flags)
1267{ 1095{
1268 struct dma_device *dma = chan->device;
1269
1270 /* specify valid address for disabled result */ 1096 /* specify valid address for disabled result */
1271 if (flags & DMA_PREP_PQ_DISABLE_P) 1097 if (flags & DMA_PREP_PQ_DISABLE_P)
1272 pq[0] = pq[1]; 1098 pq[0] = pq[1];
@@ -1278,7 +1104,7 @@ ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
1278 */ 1104 */
1279 *pqres = 0; 1105 *pqres = 0;
1280 1106
1281 return (src_cnt > 8) && (dma->max_pq > 8) ? 1107 return src_cnt_flags(src_cnt, flags) > 8 ?
1282 __ioat3_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len, 1108 __ioat3_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len,
1283 flags) : 1109 flags) :
1284 __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len, 1110 __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
@@ -1289,7 +1115,6 @@ static struct dma_async_tx_descriptor *
1289ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, 1115ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
1290 unsigned int src_cnt, size_t len, unsigned long flags) 1116 unsigned int src_cnt, size_t len, unsigned long flags)
1291{ 1117{
1292 struct dma_device *dma = chan->device;
1293 unsigned char scf[src_cnt]; 1118 unsigned char scf[src_cnt];
1294 dma_addr_t pq[2]; 1119 dma_addr_t pq[2];
1295 1120
@@ -1298,7 +1123,7 @@ ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
1298 flags |= DMA_PREP_PQ_DISABLE_Q; 1123 flags |= DMA_PREP_PQ_DISABLE_Q;
1299 pq[1] = dst; /* specify valid address for disabled result */ 1124 pq[1] = dst; /* specify valid address for disabled result */
1300 1125
1301 return (src_cnt > 8) && (dma->max_pq > 8) ? 1126 return src_cnt_flags(src_cnt, flags) > 8 ?
1302 __ioat3_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len, 1127 __ioat3_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len,
1303 flags) : 1128 flags) :
1304 __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len, 1129 __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
@@ -1310,7 +1135,6 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
1310 unsigned int src_cnt, size_t len, 1135 unsigned int src_cnt, size_t len,
1311 enum sum_check_flags *result, unsigned long flags) 1136 enum sum_check_flags *result, unsigned long flags)
1312{ 1137{
1313 struct dma_device *dma = chan->device;
1314 unsigned char scf[src_cnt]; 1138 unsigned char scf[src_cnt];
1315 dma_addr_t pq[2]; 1139 dma_addr_t pq[2];
1316 1140
@@ -1324,8 +1148,7 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
1324 flags |= DMA_PREP_PQ_DISABLE_Q; 1148 flags |= DMA_PREP_PQ_DISABLE_Q;
1325 pq[1] = pq[0]; /* specify valid address for disabled result */ 1149 pq[1] = pq[0]; /* specify valid address for disabled result */
1326 1150
1327 1151 return src_cnt_flags(src_cnt, flags) > 8 ?
1328 return (src_cnt > 8) && (dma->max_pq > 8) ?
1329 __ioat3_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1, 1152 __ioat3_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1,
1330 scf, len, flags) : 1153 scf, len, flags) :
1331 __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, 1154 __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1,
@@ -1444,9 +1267,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device)
1444 DMA_TO_DEVICE); 1267 DMA_TO_DEVICE);
1445 tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs, 1268 tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
1446 IOAT_NUM_SRC_TEST, PAGE_SIZE, 1269 IOAT_NUM_SRC_TEST, PAGE_SIZE,
1447 DMA_PREP_INTERRUPT | 1270 DMA_PREP_INTERRUPT);
1448 DMA_COMPL_SKIP_SRC_UNMAP |
1449 DMA_COMPL_SKIP_DEST_UNMAP);
1450 1271
1451 if (!tx) { 1272 if (!tx) {
1452 dev_err(dev, "Self-test xor prep failed\n"); 1273 dev_err(dev, "Self-test xor prep failed\n");
@@ -1507,9 +1328,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device)
1507 DMA_TO_DEVICE); 1328 DMA_TO_DEVICE);
1508 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, 1329 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
1509 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, 1330 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
1510 &xor_val_result, DMA_PREP_INTERRUPT | 1331 &xor_val_result, DMA_PREP_INTERRUPT);
1511 DMA_COMPL_SKIP_SRC_UNMAP |
1512 DMA_COMPL_SKIP_DEST_UNMAP);
1513 if (!tx) { 1332 if (!tx) {
1514 dev_err(dev, "Self-test zero prep failed\n"); 1333 dev_err(dev, "Self-test zero prep failed\n");
1515 err = -ENODEV; 1334 err = -ENODEV;
@@ -1545,6 +1364,8 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device)
1545 goto free_resources; 1364 goto free_resources;
1546 } 1365 }
1547 1366
1367 memset(page_address(dest), 0, PAGE_SIZE);
1368
1548 /* test for non-zero parity sum */ 1369 /* test for non-zero parity sum */
1549 op = IOAT_OP_XOR_VAL; 1370 op = IOAT_OP_XOR_VAL;
1550 1371
@@ -1554,9 +1375,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device)
1554 DMA_TO_DEVICE); 1375 DMA_TO_DEVICE);
1555 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, 1376 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
1556 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, 1377 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
1557 &xor_val_result, DMA_PREP_INTERRUPT | 1378 &xor_val_result, DMA_PREP_INTERRUPT);
1558 DMA_COMPL_SKIP_SRC_UNMAP |
1559 DMA_COMPL_SKIP_DEST_UNMAP);
1560 if (!tx) { 1379 if (!tx) {
1561 dev_err(dev, "Self-test 2nd zero prep failed\n"); 1380 dev_err(dev, "Self-test 2nd zero prep failed\n");
1562 err = -ENODEV; 1381 err = -ENODEV;
@@ -1630,52 +1449,36 @@ static int ioat3_dma_self_test(struct ioatdma_device *device)
1630 1449
1631static int ioat3_irq_reinit(struct ioatdma_device *device) 1450static int ioat3_irq_reinit(struct ioatdma_device *device)
1632{ 1451{
1633 int msixcnt = device->common.chancnt;
1634 struct pci_dev *pdev = device->pdev; 1452 struct pci_dev *pdev = device->pdev;
1635 int i; 1453 int irq = pdev->irq, i;
1636 struct msix_entry *msix; 1454
1637 struct ioat_chan_common *chan; 1455 if (!is_bwd_ioat(pdev))
1638 int err = 0; 1456 return 0;
1639 1457
1640 switch (device->irq_mode) { 1458 switch (device->irq_mode) {
1641 case IOAT_MSIX: 1459 case IOAT_MSIX:
1460 for (i = 0; i < device->common.chancnt; i++) {
1461 struct msix_entry *msix = &device->msix_entries[i];
1462 struct ioat_chan_common *chan;
1642 1463
1643 for (i = 0; i < msixcnt; i++) {
1644 msix = &device->msix_entries[i];
1645 chan = ioat_chan_by_index(device, i); 1464 chan = ioat_chan_by_index(device, i);
1646 devm_free_irq(&pdev->dev, msix->vector, chan); 1465 devm_free_irq(&pdev->dev, msix->vector, chan);
1647 } 1466 }
1648 1467
1649 pci_disable_msix(pdev); 1468 pci_disable_msix(pdev);
1650 break; 1469 break;
1651
1652 case IOAT_MSIX_SINGLE:
1653 msix = &device->msix_entries[0];
1654 chan = ioat_chan_by_index(device, 0);
1655 devm_free_irq(&pdev->dev, msix->vector, chan);
1656 pci_disable_msix(pdev);
1657 break;
1658
1659 case IOAT_MSI: 1470 case IOAT_MSI:
1660 chan = ioat_chan_by_index(device, 0);
1661 devm_free_irq(&pdev->dev, pdev->irq, chan);
1662 pci_disable_msi(pdev); 1471 pci_disable_msi(pdev);
1663 break; 1472 /* fall through */
1664
1665 case IOAT_INTX: 1473 case IOAT_INTX:
1666 chan = ioat_chan_by_index(device, 0); 1474 devm_free_irq(&pdev->dev, irq, device);
1667 devm_free_irq(&pdev->dev, pdev->irq, chan);
1668 break; 1475 break;
1669
1670 default: 1476 default:
1671 return 0; 1477 return 0;
1672 } 1478 }
1673
1674 device->irq_mode = IOAT_NOIRQ; 1479 device->irq_mode = IOAT_NOIRQ;
1675 1480
1676 err = ioat_dma_setup_interrupts(device); 1481 return ioat_dma_setup_interrupts(device);
1677
1678 return err;
1679} 1482}
1680 1483
1681static int ioat3_reset_hw(struct ioat_chan_common *chan) 1484static int ioat3_reset_hw(struct ioat_chan_common *chan)
@@ -1718,14 +1521,12 @@ static int ioat3_reset_hw(struct ioat_chan_common *chan)
1718 } 1521 }
1719 1522
1720 err = ioat2_reset_sync(chan, msecs_to_jiffies(200)); 1523 err = ioat2_reset_sync(chan, msecs_to_jiffies(200));
1721 if (err) { 1524 if (!err)
1722 dev_err(&pdev->dev, "Failed to reset!\n");
1723 return err;
1724 }
1725
1726 if (device->irq_mode != IOAT_NOIRQ && is_bwd_ioat(pdev))
1727 err = ioat3_irq_reinit(device); 1525 err = ioat3_irq_reinit(device);
1728 1526
1527 if (err)
1528 dev_err(&pdev->dev, "Failed to reset: %d\n", err);
1529
1729 return err; 1530 return err;
1730} 1531}
1731 1532
@@ -1835,21 +1636,15 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca)
1835 char pool_name[14]; 1636 char pool_name[14];
1836 int i; 1637 int i;
1837 1638
1838 /* allocate sw descriptor pool for SED */
1839 device->sed_pool = kmem_cache_create("ioat_sed",
1840 sizeof(struct ioat_sed_ent), 0, 0, NULL);
1841 if (!device->sed_pool)
1842 return -ENOMEM;
1843
1844 for (i = 0; i < MAX_SED_POOLS; i++) { 1639 for (i = 0; i < MAX_SED_POOLS; i++) {
1845 snprintf(pool_name, 14, "ioat_hw%d_sed", i); 1640 snprintf(pool_name, 14, "ioat_hw%d_sed", i);
1846 1641
1847 /* allocate SED DMA pool */ 1642 /* allocate SED DMA pool */
1848 device->sed_hw_pool[i] = dma_pool_create(pool_name, 1643 device->sed_hw_pool[i] = dmam_pool_create(pool_name,
1849 &pdev->dev, 1644 &pdev->dev,
1850 SED_SIZE * (i + 1), 64, 0); 1645 SED_SIZE * (i + 1), 64, 0);
1851 if (!device->sed_hw_pool[i]) 1646 if (!device->sed_hw_pool[i])
1852 goto sed_pool_cleanup; 1647 return -ENOMEM;
1853 1648
1854 } 1649 }
1855 } 1650 }
@@ -1875,28 +1670,4 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca)
1875 device->dca = ioat3_dca_init(pdev, device->reg_base); 1670 device->dca = ioat3_dca_init(pdev, device->reg_base);
1876 1671
1877 return 0; 1672 return 0;
1878
1879sed_pool_cleanup:
1880 if (device->sed_pool) {
1881 int i;
1882 kmem_cache_destroy(device->sed_pool);
1883
1884 for (i = 0; i < MAX_SED_POOLS; i++)
1885 if (device->sed_hw_pool[i])
1886 dma_pool_destroy(device->sed_hw_pool[i]);
1887 }
1888
1889 return -ENOMEM;
1890}
1891
1892void ioat3_dma_remove(struct ioatdma_device *device)
1893{
1894 if (device->sed_pool) {
1895 int i;
1896 kmem_cache_destroy(device->sed_pool);
1897
1898 for (i = 0; i < MAX_SED_POOLS; i++)
1899 if (device->sed_hw_pool[i])
1900 dma_pool_destroy(device->sed_hw_pool[i]);
1901 }
1902} 1673}
diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c
index 2c8d560e6334..1d051cd045db 100644
--- a/drivers/dma/ioat/pci.c
+++ b/drivers/dma/ioat/pci.c
@@ -123,6 +123,7 @@ module_param(ioat_dca_enabled, int, 0644);
123MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); 123MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
124 124
125struct kmem_cache *ioat2_cache; 125struct kmem_cache *ioat2_cache;
126struct kmem_cache *ioat3_sed_cache;
126 127
127#define DRV_NAME "ioatdma" 128#define DRV_NAME "ioatdma"
128 129
@@ -207,9 +208,6 @@ static void ioat_remove(struct pci_dev *pdev)
207 if (!device) 208 if (!device)
208 return; 209 return;
209 210
210 if (device->version >= IOAT_VER_3_0)
211 ioat3_dma_remove(device);
212
213 dev_err(&pdev->dev, "Removing dma and dca services\n"); 211 dev_err(&pdev->dev, "Removing dma and dca services\n");
214 if (device->dca) { 212 if (device->dca) {
215 unregister_dca_provider(device->dca, &pdev->dev); 213 unregister_dca_provider(device->dca, &pdev->dev);
@@ -221,7 +219,7 @@ static void ioat_remove(struct pci_dev *pdev)
221 219
222static int __init ioat_init_module(void) 220static int __init ioat_init_module(void)
223{ 221{
224 int err; 222 int err = -ENOMEM;
225 223
226 pr_info("%s: Intel(R) QuickData Technology Driver %s\n", 224 pr_info("%s: Intel(R) QuickData Technology Driver %s\n",
227 DRV_NAME, IOAT_DMA_VERSION); 225 DRV_NAME, IOAT_DMA_VERSION);
@@ -231,9 +229,21 @@ static int __init ioat_init_module(void)
231 if (!ioat2_cache) 229 if (!ioat2_cache)
232 return -ENOMEM; 230 return -ENOMEM;
233 231
232 ioat3_sed_cache = KMEM_CACHE(ioat_sed_ent, 0);
233 if (!ioat3_sed_cache)
234 goto err_ioat2_cache;
235
234 err = pci_register_driver(&ioat_pci_driver); 236 err = pci_register_driver(&ioat_pci_driver);
235 if (err) 237 if (err)
236 kmem_cache_destroy(ioat2_cache); 238 goto err_ioat3_cache;
239
240 return 0;
241
242 err_ioat3_cache:
243 kmem_cache_destroy(ioat3_sed_cache);
244
245 err_ioat2_cache:
246 kmem_cache_destroy(ioat2_cache);
237 247
238 return err; 248 return err;
239} 249}
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index 408fe6be15f4..c56137bc3868 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -61,80 +61,6 @@ static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
61 } 61 }
62} 62}
63 63
64static void
65iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
66{
67 struct dma_async_tx_descriptor *tx = &desc->async_tx;
68 struct iop_adma_desc_slot *unmap = desc->group_head;
69 struct device *dev = &iop_chan->device->pdev->dev;
70 u32 len = unmap->unmap_len;
71 enum dma_ctrl_flags flags = tx->flags;
72 u32 src_cnt;
73 dma_addr_t addr;
74 dma_addr_t dest;
75
76 src_cnt = unmap->unmap_src_cnt;
77 dest = iop_desc_get_dest_addr(unmap, iop_chan);
78 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
79 enum dma_data_direction dir;
80
81 if (src_cnt > 1) /* is xor? */
82 dir = DMA_BIDIRECTIONAL;
83 else
84 dir = DMA_FROM_DEVICE;
85
86 dma_unmap_page(dev, dest, len, dir);
87 }
88
89 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
90 while (src_cnt--) {
91 addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt);
92 if (addr == dest)
93 continue;
94 dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
95 }
96 }
97 desc->group_head = NULL;
98}
99
100static void
101iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc)
102{
103 struct dma_async_tx_descriptor *tx = &desc->async_tx;
104 struct iop_adma_desc_slot *unmap = desc->group_head;
105 struct device *dev = &iop_chan->device->pdev->dev;
106 u32 len = unmap->unmap_len;
107 enum dma_ctrl_flags flags = tx->flags;
108 u32 src_cnt = unmap->unmap_src_cnt;
109 dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan);
110 dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan);
111 int i;
112
113 if (tx->flags & DMA_PREP_CONTINUE)
114 src_cnt -= 3;
115
116 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) {
117 dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL);
118 dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL);
119 }
120
121 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
122 dma_addr_t addr;
123
124 for (i = 0; i < src_cnt; i++) {
125 addr = iop_desc_get_src_addr(unmap, iop_chan, i);
126 dma_unmap_page(dev, addr, len, DMA_TO_DEVICE);
127 }
128 if (desc->pq_check_result) {
129 dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE);
130 dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE);
131 }
132 }
133
134 desc->group_head = NULL;
135}
136
137
138static dma_cookie_t 64static dma_cookie_t
139iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, 65iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
140 struct iop_adma_chan *iop_chan, dma_cookie_t cookie) 66 struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
@@ -152,15 +78,9 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
152 if (tx->callback) 78 if (tx->callback)
153 tx->callback(tx->callback_param); 79 tx->callback(tx->callback_param);
154 80
155 /* unmap dma addresses 81 dma_descriptor_unmap(tx);
156 * (unmap_single vs unmap_page?) 82 if (desc->group_head)
157 */ 83 desc->group_head = NULL;
158 if (desc->group_head && desc->unmap_len) {
159 if (iop_desc_is_pq(desc))
160 iop_desc_unmap_pq(iop_chan, desc);
161 else
162 iop_desc_unmap(iop_chan, desc);
163 }
164 } 84 }
165 85
166 /* run dependent operations */ 86 /* run dependent operations */
@@ -591,7 +511,6 @@ iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
591 if (sw_desc) { 511 if (sw_desc) {
592 grp_start = sw_desc->group_head; 512 grp_start = sw_desc->group_head;
593 iop_desc_init_interrupt(grp_start, iop_chan); 513 iop_desc_init_interrupt(grp_start, iop_chan);
594 grp_start->unmap_len = 0;
595 sw_desc->async_tx.flags = flags; 514 sw_desc->async_tx.flags = flags;
596 } 515 }
597 spin_unlock_bh(&iop_chan->lock); 516 spin_unlock_bh(&iop_chan->lock);
@@ -623,8 +542,6 @@ iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
623 iop_desc_set_byte_count(grp_start, iop_chan, len); 542 iop_desc_set_byte_count(grp_start, iop_chan, len);
624 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 543 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
625 iop_desc_set_memcpy_src_addr(grp_start, dma_src); 544 iop_desc_set_memcpy_src_addr(grp_start, dma_src);
626 sw_desc->unmap_src_cnt = 1;
627 sw_desc->unmap_len = len;
628 sw_desc->async_tx.flags = flags; 545 sw_desc->async_tx.flags = flags;
629 } 546 }
630 spin_unlock_bh(&iop_chan->lock); 547 spin_unlock_bh(&iop_chan->lock);
@@ -657,8 +574,6 @@ iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
657 iop_desc_init_xor(grp_start, src_cnt, flags); 574 iop_desc_init_xor(grp_start, src_cnt, flags);
658 iop_desc_set_byte_count(grp_start, iop_chan, len); 575 iop_desc_set_byte_count(grp_start, iop_chan, len);
659 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); 576 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
660 sw_desc->unmap_src_cnt = src_cnt;
661 sw_desc->unmap_len = len;
662 sw_desc->async_tx.flags = flags; 577 sw_desc->async_tx.flags = flags;
663 while (src_cnt--) 578 while (src_cnt--)
664 iop_desc_set_xor_src_addr(grp_start, src_cnt, 579 iop_desc_set_xor_src_addr(grp_start, src_cnt,
@@ -694,8 +609,6 @@ iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
694 grp_start->xor_check_result = result; 609 grp_start->xor_check_result = result;
695 pr_debug("\t%s: grp_start->xor_check_result: %p\n", 610 pr_debug("\t%s: grp_start->xor_check_result: %p\n",
696 __func__, grp_start->xor_check_result); 611 __func__, grp_start->xor_check_result);
697 sw_desc->unmap_src_cnt = src_cnt;
698 sw_desc->unmap_len = len;
699 sw_desc->async_tx.flags = flags; 612 sw_desc->async_tx.flags = flags;
700 while (src_cnt--) 613 while (src_cnt--)
701 iop_desc_set_zero_sum_src_addr(grp_start, src_cnt, 614 iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
@@ -748,8 +661,6 @@ iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
748 dst[0] = dst[1] & 0x7; 661 dst[0] = dst[1] & 0x7;
749 662
750 iop_desc_set_pq_addr(g, dst); 663 iop_desc_set_pq_addr(g, dst);
751 sw_desc->unmap_src_cnt = src_cnt;
752 sw_desc->unmap_len = len;
753 sw_desc->async_tx.flags = flags; 664 sw_desc->async_tx.flags = flags;
754 for (i = 0; i < src_cnt; i++) 665 for (i = 0; i < src_cnt; i++)
755 iop_desc_set_pq_src_addr(g, i, src[i], scf[i]); 666 iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
@@ -804,8 +715,6 @@ iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
804 g->pq_check_result = pqres; 715 g->pq_check_result = pqres;
805 pr_debug("\t%s: g->pq_check_result: %p\n", 716 pr_debug("\t%s: g->pq_check_result: %p\n",
806 __func__, g->pq_check_result); 717 __func__, g->pq_check_result);
807 sw_desc->unmap_src_cnt = src_cnt+2;
808 sw_desc->unmap_len = len;
809 sw_desc->async_tx.flags = flags; 718 sw_desc->async_tx.flags = flags;
810 while (src_cnt--) 719 while (src_cnt--)
811 iop_desc_set_pq_zero_sum_src_addr(g, src_cnt, 720 iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index 8d5bce9e867e..7807f0ef4e20 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -60,14 +60,6 @@ static u32 mv_desc_get_dest_addr(struct mv_xor_desc_slot *desc)
60 return hw_desc->phy_dest_addr; 60 return hw_desc->phy_dest_addr;
61} 61}
62 62
63static u32 mv_desc_get_src_addr(struct mv_xor_desc_slot *desc,
64 int src_idx)
65{
66 struct mv_xor_desc *hw_desc = desc->hw_desc;
67 return hw_desc->phy_src_addr[mv_phy_src_idx(src_idx)];
68}
69
70
71static void mv_desc_set_byte_count(struct mv_xor_desc_slot *desc, 63static void mv_desc_set_byte_count(struct mv_xor_desc_slot *desc,
72 u32 byte_count) 64 u32 byte_count)
73{ 65{
@@ -278,42 +270,9 @@ mv_xor_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
278 desc->async_tx.callback( 270 desc->async_tx.callback(
279 desc->async_tx.callback_param); 271 desc->async_tx.callback_param);
280 272
281 /* unmap dma addresses 273 dma_descriptor_unmap(&desc->async_tx);
282 * (unmap_single vs unmap_page?) 274 if (desc->group_head)
283 */
284 if (desc->group_head && desc->unmap_len) {
285 struct mv_xor_desc_slot *unmap = desc->group_head;
286 struct device *dev = mv_chan_to_devp(mv_chan);
287 u32 len = unmap->unmap_len;
288 enum dma_ctrl_flags flags = desc->async_tx.flags;
289 u32 src_cnt;
290 dma_addr_t addr;
291 dma_addr_t dest;
292
293 src_cnt = unmap->unmap_src_cnt;
294 dest = mv_desc_get_dest_addr(unmap);
295 if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
296 enum dma_data_direction dir;
297
298 if (src_cnt > 1) /* is xor ? */
299 dir = DMA_BIDIRECTIONAL;
300 else
301 dir = DMA_FROM_DEVICE;
302 dma_unmap_page(dev, dest, len, dir);
303 }
304
305 if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
306 while (src_cnt--) {
307 addr = mv_desc_get_src_addr(unmap,
308 src_cnt);
309 if (addr == dest)
310 continue;
311 dma_unmap_page(dev, addr, len,
312 DMA_TO_DEVICE);
313 }
314 }
315 desc->group_head = NULL; 275 desc->group_head = NULL;
316 }
317 } 276 }
318 277
319 /* run dependent operations */ 278 /* run dependent operations */
@@ -1076,10 +1035,7 @@ mv_xor_channel_add(struct mv_xor_device *xordev,
1076 } 1035 }
1077 1036
1078 mv_chan->mmr_base = xordev->xor_base; 1037 mv_chan->mmr_base = xordev->xor_base;
1079 if (!mv_chan->mmr_base) { 1038 mv_chan->mmr_high_base = xordev->xor_high_base;
1080 ret = -ENOMEM;
1081 goto err_free_dma;
1082 }
1083 tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long) 1039 tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long)
1084 mv_chan); 1040 mv_chan);
1085 1041
@@ -1138,7 +1094,7 @@ static void
1138mv_xor_conf_mbus_windows(struct mv_xor_device *xordev, 1094mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
1139 const struct mbus_dram_target_info *dram) 1095 const struct mbus_dram_target_info *dram)
1140{ 1096{
1141 void __iomem *base = xordev->xor_base; 1097 void __iomem *base = xordev->xor_high_base;
1142 u32 win_enable = 0; 1098 u32 win_enable = 0;
1143 int i; 1099 int i;
1144 1100
diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h
index 06b067f24c9b..d0749229c875 100644
--- a/drivers/dma/mv_xor.h
+++ b/drivers/dma/mv_xor.h
@@ -34,13 +34,13 @@
34#define XOR_OPERATION_MODE_MEMCPY 2 34#define XOR_OPERATION_MODE_MEMCPY 2
35#define XOR_DESCRIPTOR_SWAP BIT(14) 35#define XOR_DESCRIPTOR_SWAP BIT(14)
36 36
37#define XOR_CURR_DESC(chan) (chan->mmr_base + 0x210 + (chan->idx * 4)) 37#define XOR_CURR_DESC(chan) (chan->mmr_high_base + 0x10 + (chan->idx * 4))
38#define XOR_NEXT_DESC(chan) (chan->mmr_base + 0x200 + (chan->idx * 4)) 38#define XOR_NEXT_DESC(chan) (chan->mmr_high_base + 0x00 + (chan->idx * 4))
39#define XOR_BYTE_COUNT(chan) (chan->mmr_base + 0x220 + (chan->idx * 4)) 39#define XOR_BYTE_COUNT(chan) (chan->mmr_high_base + 0x20 + (chan->idx * 4))
40#define XOR_DEST_POINTER(chan) (chan->mmr_base + 0x2B0 + (chan->idx * 4)) 40#define XOR_DEST_POINTER(chan) (chan->mmr_high_base + 0xB0 + (chan->idx * 4))
41#define XOR_BLOCK_SIZE(chan) (chan->mmr_base + 0x2C0 + (chan->idx * 4)) 41#define XOR_BLOCK_SIZE(chan) (chan->mmr_high_base + 0xC0 + (chan->idx * 4))
42#define XOR_INIT_VALUE_LOW(chan) (chan->mmr_base + 0x2E0) 42#define XOR_INIT_VALUE_LOW(chan) (chan->mmr_high_base + 0xE0)
43#define XOR_INIT_VALUE_HIGH(chan) (chan->mmr_base + 0x2E4) 43#define XOR_INIT_VALUE_HIGH(chan) (chan->mmr_high_base + 0xE4)
44 44
45#define XOR_CONFIG(chan) (chan->mmr_base + 0x10 + (chan->idx * 4)) 45#define XOR_CONFIG(chan) (chan->mmr_base + 0x10 + (chan->idx * 4))
46#define XOR_ACTIVATION(chan) (chan->mmr_base + 0x20 + (chan->idx * 4)) 46#define XOR_ACTIVATION(chan) (chan->mmr_base + 0x20 + (chan->idx * 4))
@@ -50,11 +50,11 @@
50#define XOR_ERROR_ADDR(chan) (chan->mmr_base + 0x60) 50#define XOR_ERROR_ADDR(chan) (chan->mmr_base + 0x60)
51#define XOR_INTR_MASK_VALUE 0x3F5 51#define XOR_INTR_MASK_VALUE 0x3F5
52 52
53#define WINDOW_BASE(w) (0x250 + ((w) << 2)) 53#define WINDOW_BASE(w) (0x50 + ((w) << 2))
54#define WINDOW_SIZE(w) (0x270 + ((w) << 2)) 54#define WINDOW_SIZE(w) (0x70 + ((w) << 2))
55#define WINDOW_REMAP_HIGH(w) (0x290 + ((w) << 2)) 55#define WINDOW_REMAP_HIGH(w) (0x90 + ((w) << 2))
56#define WINDOW_BAR_ENABLE(chan) (0x240 + ((chan) << 2)) 56#define WINDOW_BAR_ENABLE(chan) (0x40 + ((chan) << 2))
57#define WINDOW_OVERRIDE_CTRL(chan) (0x2A0 + ((chan) << 2)) 57#define WINDOW_OVERRIDE_CTRL(chan) (0xA0 + ((chan) << 2))
58 58
59struct mv_xor_device { 59struct mv_xor_device {
60 void __iomem *xor_base; 60 void __iomem *xor_base;
@@ -82,6 +82,7 @@ struct mv_xor_chan {
82 int pending; 82 int pending;
83 spinlock_t lock; /* protects the descriptor slot pool */ 83 spinlock_t lock; /* protects the descriptor slot pool */
84 void __iomem *mmr_base; 84 void __iomem *mmr_base;
85 void __iomem *mmr_high_base;
85 unsigned int idx; 86 unsigned int idx;
86 int irq; 87 int irq;
87 enum dma_transaction_type current_type; 88 enum dma_transaction_type current_type;
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index 0ee2c547b304..877d7bb767d9 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -2268,6 +2268,8 @@ static void pl330_tasklet(unsigned long data)
2268 list_move_tail(&desc->node, &pch->dmac->desc_pool); 2268 list_move_tail(&desc->node, &pch->dmac->desc_pool);
2269 } 2269 }
2270 2270
2271 dma_descriptor_unmap(&desc->txd);
2272
2271 if (callback) { 2273 if (callback) {
2272 spin_unlock_irqrestore(&pch->lock, flags); 2274 spin_unlock_irqrestore(&pch->lock, flags);
2273 callback(callback_param); 2275 callback(callback_param);
diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c
index 60e02ae38b04..3005938ba47a 100644
--- a/drivers/dma/ppc4xx/adma.c
+++ b/drivers/dma/ppc4xx/adma.c
@@ -802,218 +802,6 @@ static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
802} 802}
803 803
804/** 804/**
805 * ppc440spe_desc_get_src_addr - extract the source address from the descriptor
806 */
807static u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc,
808 struct ppc440spe_adma_chan *chan, int src_idx)
809{
810 struct dma_cdb *dma_hw_desc;
811 struct xor_cb *xor_hw_desc;
812
813 switch (chan->device->id) {
814 case PPC440SPE_DMA0_ID:
815 case PPC440SPE_DMA1_ID:
816 dma_hw_desc = desc->hw_desc;
817 /* May have 0, 1, 2, or 3 sources */
818 switch (dma_hw_desc->opc) {
819 case DMA_CDB_OPC_NO_OP:
820 case DMA_CDB_OPC_DFILL128:
821 return 0;
822 case DMA_CDB_OPC_DCHECK128:
823 if (unlikely(src_idx)) {
824 printk(KERN_ERR "%s: try to get %d source for"
825 " DCHECK128\n", __func__, src_idx);
826 BUG();
827 }
828 return le32_to_cpu(dma_hw_desc->sg1l);
829 case DMA_CDB_OPC_MULTICAST:
830 case DMA_CDB_OPC_MV_SG1_SG2:
831 if (unlikely(src_idx > 2)) {
832 printk(KERN_ERR "%s: try to get %d source from"
833 " DMA descr\n", __func__, src_idx);
834 BUG();
835 }
836 if (src_idx) {
837 if (le32_to_cpu(dma_hw_desc->sg1u) &
838 DMA_CUED_XOR_WIN_MSK) {
839 u8 region;
840
841 if (src_idx == 1)
842 return le32_to_cpu(
843 dma_hw_desc->sg1l) +
844 desc->unmap_len;
845
846 region = (le32_to_cpu(
847 dma_hw_desc->sg1u)) >>
848 DMA_CUED_REGION_OFF;
849
850 region &= DMA_CUED_REGION_MSK;
851 switch (region) {
852 case DMA_RXOR123:
853 return le32_to_cpu(
854 dma_hw_desc->sg1l) +
855 (desc->unmap_len << 1);
856 case DMA_RXOR124:
857 return le32_to_cpu(
858 dma_hw_desc->sg1l) +
859 (desc->unmap_len * 3);
860 case DMA_RXOR125:
861 return le32_to_cpu(
862 dma_hw_desc->sg1l) +
863 (desc->unmap_len << 2);
864 default:
865 printk(KERN_ERR
866 "%s: try to"
867 " get src3 for region %02x"
868 "PPC440SPE_DESC_RXOR12?\n",
869 __func__, region);
870 BUG();
871 }
872 } else {
873 printk(KERN_ERR
874 "%s: try to get %d"
875 " source for non-cued descr\n",
876 __func__, src_idx);
877 BUG();
878 }
879 }
880 return le32_to_cpu(dma_hw_desc->sg1l);
881 default:
882 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
883 __func__, dma_hw_desc->opc);
884 BUG();
885 }
886 return le32_to_cpu(dma_hw_desc->sg1l);
887 case PPC440SPE_XOR_ID:
888 /* May have up to 16 sources */
889 xor_hw_desc = desc->hw_desc;
890 return xor_hw_desc->ops[src_idx].l;
891 }
892 return 0;
893}
894
895/**
896 * ppc440spe_desc_get_dest_addr - extract the destination address from the
897 * descriptor
898 */
899static u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc,
900 struct ppc440spe_adma_chan *chan, int idx)
901{
902 struct dma_cdb *dma_hw_desc;
903 struct xor_cb *xor_hw_desc;
904
905 switch (chan->device->id) {
906 case PPC440SPE_DMA0_ID:
907 case PPC440SPE_DMA1_ID:
908 dma_hw_desc = desc->hw_desc;
909
910 if (likely(!idx))
911 return le32_to_cpu(dma_hw_desc->sg2l);
912 return le32_to_cpu(dma_hw_desc->sg3l);
913 case PPC440SPE_XOR_ID:
914 xor_hw_desc = desc->hw_desc;
915 return xor_hw_desc->cbtal;
916 }
917 return 0;
918}
919
920/**
921 * ppc440spe_desc_get_src_num - extract the number of source addresses from
922 * the descriptor
923 */
924static u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc,
925 struct ppc440spe_adma_chan *chan)
926{
927 struct dma_cdb *dma_hw_desc;
928 struct xor_cb *xor_hw_desc;
929
930 switch (chan->device->id) {
931 case PPC440SPE_DMA0_ID:
932 case PPC440SPE_DMA1_ID:
933 dma_hw_desc = desc->hw_desc;
934
935 switch (dma_hw_desc->opc) {
936 case DMA_CDB_OPC_NO_OP:
937 case DMA_CDB_OPC_DFILL128:
938 return 0;
939 case DMA_CDB_OPC_DCHECK128:
940 return 1;
941 case DMA_CDB_OPC_MV_SG1_SG2:
942 case DMA_CDB_OPC_MULTICAST:
943 /*
944 * Only for RXOR operations we have more than
945 * one source
946 */
947 if (le32_to_cpu(dma_hw_desc->sg1u) &
948 DMA_CUED_XOR_WIN_MSK) {
949 /* RXOR op, there are 2 or 3 sources */
950 if (((le32_to_cpu(dma_hw_desc->sg1u) >>
951 DMA_CUED_REGION_OFF) &
952 DMA_CUED_REGION_MSK) == DMA_RXOR12) {
953 /* RXOR 1-2 */
954 return 2;
955 } else {
956 /* RXOR 1-2-3/1-2-4/1-2-5 */
957 return 3;
958 }
959 }
960 return 1;
961 default:
962 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
963 __func__, dma_hw_desc->opc);
964 BUG();
965 }
966 case PPC440SPE_XOR_ID:
967 /* up to 16 sources */
968 xor_hw_desc = desc->hw_desc;
969 return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK;
970 default:
971 BUG();
972 }
973 return 0;
974}
975
976/**
977 * ppc440spe_desc_get_dst_num - get the number of destination addresses in
978 * this descriptor
979 */
980static u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc,
981 struct ppc440spe_adma_chan *chan)
982{
983 struct dma_cdb *dma_hw_desc;
984
985 switch (chan->device->id) {
986 case PPC440SPE_DMA0_ID:
987 case PPC440SPE_DMA1_ID:
988 /* May be 1 or 2 destinations */
989 dma_hw_desc = desc->hw_desc;
990 switch (dma_hw_desc->opc) {
991 case DMA_CDB_OPC_NO_OP:
992 case DMA_CDB_OPC_DCHECK128:
993 return 0;
994 case DMA_CDB_OPC_MV_SG1_SG2:
995 case DMA_CDB_OPC_DFILL128:
996 return 1;
997 case DMA_CDB_OPC_MULTICAST:
998 if (desc->dst_cnt == 2)
999 return 2;
1000 else
1001 return 1;
1002 default:
1003 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
1004 __func__, dma_hw_desc->opc);
1005 BUG();
1006 }
1007 case PPC440SPE_XOR_ID:
1008 /* Always only 1 destination */
1009 return 1;
1010 default:
1011 BUG();
1012 }
1013 return 0;
1014}
1015
1016/**
1017 * ppc440spe_desc_get_link - get the address of the descriptor that 805 * ppc440spe_desc_get_link - get the address of the descriptor that
1018 * follows this one 806 * follows this one
1019 */ 807 */
@@ -1705,43 +1493,6 @@ static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1705 } 1493 }
1706} 1494}
1707 1495
1708static void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan,
1709 struct ppc440spe_adma_desc_slot *desc)
1710{
1711 u32 src_cnt, dst_cnt;
1712 dma_addr_t addr;
1713
1714 /*
1715 * get the number of sources & destination
1716 * included in this descriptor and unmap
1717 * them all
1718 */
1719 src_cnt = ppc440spe_desc_get_src_num(desc, chan);
1720 dst_cnt = ppc440spe_desc_get_dst_num(desc, chan);
1721
1722 /* unmap destinations */
1723 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1724 while (dst_cnt--) {
1725 addr = ppc440spe_desc_get_dest_addr(
1726 desc, chan, dst_cnt);
1727 dma_unmap_page(chan->device->dev,
1728 addr, desc->unmap_len,
1729 DMA_FROM_DEVICE);
1730 }
1731 }
1732
1733 /* unmap sources */
1734 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1735 while (src_cnt--) {
1736 addr = ppc440spe_desc_get_src_addr(
1737 desc, chan, src_cnt);
1738 dma_unmap_page(chan->device->dev,
1739 addr, desc->unmap_len,
1740 DMA_TO_DEVICE);
1741 }
1742 }
1743}
1744
1745/** 1496/**
1746 * ppc440spe_adma_run_tx_complete_actions - call functions to be called 1497 * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1747 * upon completion 1498 * upon completion
@@ -1765,26 +1516,7 @@ static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1765 desc->async_tx.callback( 1516 desc->async_tx.callback(
1766 desc->async_tx.callback_param); 1517 desc->async_tx.callback_param);
1767 1518
1768 /* unmap dma addresses 1519 dma_descriptor_unmap(&desc->async_tx);
1769 * (unmap_single vs unmap_page?)
1770 *
1771 * actually, ppc's dma_unmap_page() functions are empty, so
1772 * the following code is just for the sake of completeness
1773 */
1774 if (chan && chan->needs_unmap && desc->group_head &&
1775 desc->unmap_len) {
1776 struct ppc440spe_adma_desc_slot *unmap =
1777 desc->group_head;
1778 /* assume 1 slot per op always */
1779 u32 slot_count = unmap->slot_cnt;
1780
1781 /* Run through the group list and unmap addresses */
1782 for (i = 0; i < slot_count; i++) {
1783 BUG_ON(!unmap);
1784 ppc440spe_adma_unmap(chan, unmap);
1785 unmap = unmap->hw_next;
1786 }
1787 }
1788 } 1520 }
1789 1521
1790 /* run dependent operations */ 1522 /* run dependent operations */
diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c
index 28af214fce04..4506a7b4f972 100644
--- a/drivers/dma/timb_dma.c
+++ b/drivers/dma/timb_dma.c
@@ -154,38 +154,6 @@ static bool __td_dma_done_ack(struct timb_dma_chan *td_chan)
154 return done; 154 return done;
155} 155}
156 156
157static void __td_unmap_desc(struct timb_dma_chan *td_chan, const u8 *dma_desc,
158 bool single)
159{
160 dma_addr_t addr;
161 int len;
162
163 addr = (dma_desc[7] << 24) | (dma_desc[6] << 16) | (dma_desc[5] << 8) |
164 dma_desc[4];
165
166 len = (dma_desc[3] << 8) | dma_desc[2];
167
168 if (single)
169 dma_unmap_single(chan2dev(&td_chan->chan), addr, len,
170 DMA_TO_DEVICE);
171 else
172 dma_unmap_page(chan2dev(&td_chan->chan), addr, len,
173 DMA_TO_DEVICE);
174}
175
176static void __td_unmap_descs(struct timb_dma_desc *td_desc, bool single)
177{
178 struct timb_dma_chan *td_chan = container_of(td_desc->txd.chan,
179 struct timb_dma_chan, chan);
180 u8 *descs;
181
182 for (descs = td_desc->desc_list; ; descs += TIMB_DMA_DESC_SIZE) {
183 __td_unmap_desc(td_chan, descs, single);
184 if (descs[0] & 0x02)
185 break;
186 }
187}
188
189static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc, 157static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc,
190 struct scatterlist *sg, bool last) 158 struct scatterlist *sg, bool last)
191{ 159{
@@ -293,10 +261,7 @@ static void __td_finish(struct timb_dma_chan *td_chan)
293 261
294 list_move(&td_desc->desc_node, &td_chan->free_list); 262 list_move(&td_desc->desc_node, &td_chan->free_list);
295 263
296 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) 264 dma_descriptor_unmap(txd);
297 __td_unmap_descs(td_desc,
298 txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE);
299
300 /* 265 /*
301 * The API requires that no submissions are done from a 266 * The API requires that no submissions are done from a
302 * callback, so we don't need to drop the lock here 267 * callback, so we don't need to drop the lock here
diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c
index c2829b481bf2..bae6c29f5502 100644
--- a/drivers/dma/txx9dmac.c
+++ b/drivers/dma/txx9dmac.c
@@ -419,30 +419,7 @@ txx9dmac_descriptor_complete(struct txx9dmac_chan *dc,
419 list_splice_init(&desc->tx_list, &dc->free_list); 419 list_splice_init(&desc->tx_list, &dc->free_list);
420 list_move(&desc->desc_node, &dc->free_list); 420 list_move(&desc->desc_node, &dc->free_list);
421 421
422 if (!ds) { 422 dma_descriptor_unmap(txd);
423 dma_addr_t dmaaddr;
424 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
425 dmaaddr = is_dmac64(dc) ?
426 desc->hwdesc.DAR : desc->hwdesc32.DAR;
427 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
428 dma_unmap_single(chan2parent(&dc->chan),
429 dmaaddr, desc->len, DMA_FROM_DEVICE);
430 else
431 dma_unmap_page(chan2parent(&dc->chan),
432 dmaaddr, desc->len, DMA_FROM_DEVICE);
433 }
434 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
435 dmaaddr = is_dmac64(dc) ?
436 desc->hwdesc.SAR : desc->hwdesc32.SAR;
437 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
438 dma_unmap_single(chan2parent(&dc->chan),
439 dmaaddr, desc->len, DMA_TO_DEVICE);
440 else
441 dma_unmap_page(chan2parent(&dc->chan),
442 dmaaddr, desc->len, DMA_TO_DEVICE);
443 }
444 }
445
446 /* 423 /*
447 * The API requires that no submissions are done from a 424 * The API requires that no submissions are done from a
448 * callback, so we don't need to drop the lock here 425 * callback, so we don't need to drop the lock here
diff --git a/drivers/media/platform/m2m-deinterlace.c b/drivers/media/platform/m2m-deinterlace.c
index 540516ca872c..879ea6fdd1be 100644
--- a/drivers/media/platform/m2m-deinterlace.c
+++ b/drivers/media/platform/m2m-deinterlace.c
@@ -341,8 +341,7 @@ static void deinterlace_issue_dma(struct deinterlace_ctx *ctx, int op,
341 ctx->xt->dir = DMA_MEM_TO_MEM; 341 ctx->xt->dir = DMA_MEM_TO_MEM;
342 ctx->xt->src_sgl = false; 342 ctx->xt->src_sgl = false;
343 ctx->xt->dst_sgl = true; 343 ctx->xt->dst_sgl = true;
344 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT | 344 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
345 DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SKIP_SRC_UNMAP;
346 345
347 tx = dmadev->device_prep_interleaved_dma(chan, ctx->xt, flags); 346 tx = dmadev->device_prep_interleaved_dma(chan, ctx->xt, flags);
348 if (tx == NULL) { 347 if (tx == NULL) {
diff --git a/drivers/media/platform/timblogiw.c b/drivers/media/platform/timblogiw.c
index b557caf5b1a4..59a95e3ab0e3 100644
--- a/drivers/media/platform/timblogiw.c
+++ b/drivers/media/platform/timblogiw.c
@@ -565,7 +565,7 @@ static void buffer_queue(struct videobuf_queue *vq, struct videobuf_buffer *vb)
565 565
566 desc = dmaengine_prep_slave_sg(fh->chan, 566 desc = dmaengine_prep_slave_sg(fh->chan,
567 buf->sg, sg_elems, DMA_DEV_TO_MEM, 567 buf->sg, sg_elems, DMA_DEV_TO_MEM,
568 DMA_PREP_INTERRUPT | DMA_COMPL_SKIP_SRC_UNMAP); 568 DMA_PREP_INTERRUPT);
569 if (!desc) { 569 if (!desc) {
570 spin_lock_irq(&fh->queue_lock); 570 spin_lock_irq(&fh->queue_lock);
571 list_del_init(&vb->queue); 571 list_del_init(&vb->queue);
diff --git a/drivers/misc/carma/carma-fpga.c b/drivers/misc/carma/carma-fpga.c
index 7b56563f8b74..5335104e7c84 100644
--- a/drivers/misc/carma/carma-fpga.c
+++ b/drivers/misc/carma/carma-fpga.c
@@ -631,8 +631,7 @@ static int data_submit_dma(struct fpga_device *priv, struct data_buf *buf)
631 struct dma_async_tx_descriptor *tx; 631 struct dma_async_tx_descriptor *tx;
632 dma_cookie_t cookie; 632 dma_cookie_t cookie;
633 dma_addr_t dst, src; 633 dma_addr_t dst, src;
634 unsigned long dma_flags = DMA_COMPL_SKIP_DEST_UNMAP | 634 unsigned long dma_flags = 0;
635 DMA_COMPL_SKIP_SRC_UNMAP;
636 635
637 dst_sg = buf->vb.sglist; 636 dst_sg = buf->vb.sglist;
638 dst_nents = buf->vb.sglen; 637 dst_nents = buf->vb.sglen;
diff --git a/drivers/mtd/nand/atmel_nand.c b/drivers/mtd/nand/atmel_nand.c
index 060feeaf6b3e..2a837cb425d7 100644
--- a/drivers/mtd/nand/atmel_nand.c
+++ b/drivers/mtd/nand/atmel_nand.c
@@ -375,8 +375,7 @@ static int atmel_nand_dma_op(struct mtd_info *mtd, void *buf, int len,
375 375
376 dma_dev = host->dma_chan->device; 376 dma_dev = host->dma_chan->device;
377 377
378 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT | DMA_COMPL_SKIP_SRC_UNMAP | 378 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
379 DMA_COMPL_SKIP_DEST_UNMAP;
380 379
381 phys_addr = dma_map_single(dma_dev->dev, p, len, dir); 380 phys_addr = dma_map_single(dma_dev->dev, p, len, dir);
382 if (dma_mapping_error(dma_dev->dev, phys_addr)) { 381 if (dma_mapping_error(dma_dev->dev, phys_addr)) {
diff --git a/drivers/mtd/nand/fsmc_nand.c b/drivers/mtd/nand/fsmc_nand.c
index 3dc1a7564d87..8b2752263db9 100644
--- a/drivers/mtd/nand/fsmc_nand.c
+++ b/drivers/mtd/nand/fsmc_nand.c
@@ -573,8 +573,6 @@ static int dma_xfer(struct fsmc_nand_data *host, void *buffer, int len,
573 dma_dev = chan->device; 573 dma_dev = chan->device;
574 dma_addr = dma_map_single(dma_dev->dev, buffer, len, direction); 574 dma_addr = dma_map_single(dma_dev->dev, buffer, len, direction);
575 575
576 flags |= DMA_COMPL_SKIP_SRC_UNMAP | DMA_COMPL_SKIP_DEST_UNMAP;
577
578 if (direction == DMA_TO_DEVICE) { 576 if (direction == DMA_TO_DEVICE) {
579 dma_src = dma_addr; 577 dma_src = dma_addr;
580 dma_dst = host->data_pa; 578 dma_dst = host->data_pa;
diff --git a/drivers/net/ethernet/micrel/ks8842.c b/drivers/net/ethernet/micrel/ks8842.c
index 0951f7aca1ef..822616e3c375 100644
--- a/drivers/net/ethernet/micrel/ks8842.c
+++ b/drivers/net/ethernet/micrel/ks8842.c
@@ -459,8 +459,7 @@ static int ks8842_tx_frame_dma(struct sk_buff *skb, struct net_device *netdev)
459 sg_dma_len(&ctl->sg) += 4 - sg_dma_len(&ctl->sg) % 4; 459 sg_dma_len(&ctl->sg) += 4 - sg_dma_len(&ctl->sg) % 4;
460 460
461 ctl->adesc = dmaengine_prep_slave_sg(ctl->chan, 461 ctl->adesc = dmaengine_prep_slave_sg(ctl->chan,
462 &ctl->sg, 1, DMA_MEM_TO_DEV, 462 &ctl->sg, 1, DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT);
463 DMA_PREP_INTERRUPT | DMA_COMPL_SKIP_SRC_UNMAP);
464 if (!ctl->adesc) 463 if (!ctl->adesc)
465 return NETDEV_TX_BUSY; 464 return NETDEV_TX_BUSY;
466 465
@@ -571,8 +570,7 @@ static int __ks8842_start_new_rx_dma(struct net_device *netdev)
571 sg_dma_len(sg) = DMA_BUFFER_SIZE; 570 sg_dma_len(sg) = DMA_BUFFER_SIZE;
572 571
573 ctl->adesc = dmaengine_prep_slave_sg(ctl->chan, 572 ctl->adesc = dmaengine_prep_slave_sg(ctl->chan,
574 sg, 1, DMA_DEV_TO_MEM, 573 sg, 1, DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT);
575 DMA_PREP_INTERRUPT | DMA_COMPL_SKIP_SRC_UNMAP);
576 574
577 if (!ctl->adesc) 575 if (!ctl->adesc)
578 goto out; 576 goto out;
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 12a9e83c008b..d0222f13d154 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -1034,10 +1034,9 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset,
1034 struct dma_chan *chan = qp->dma_chan; 1034 struct dma_chan *chan = qp->dma_chan;
1035 struct dma_device *device; 1035 struct dma_device *device;
1036 size_t pay_off, buff_off; 1036 size_t pay_off, buff_off;
1037 dma_addr_t src, dest; 1037 struct dmaengine_unmap_data *unmap;
1038 dma_cookie_t cookie; 1038 dma_cookie_t cookie;
1039 void *buf = entry->buf; 1039 void *buf = entry->buf;
1040 unsigned long flags;
1041 1040
1042 entry->len = len; 1041 entry->len = len;
1043 1042
@@ -1045,35 +1044,49 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset,
1045 goto err; 1044 goto err;
1046 1045
1047 if (len < copy_bytes) 1046 if (len < copy_bytes)
1048 goto err1; 1047 goto err_wait;
1049 1048
1050 device = chan->device; 1049 device = chan->device;
1051 pay_off = (size_t) offset & ~PAGE_MASK; 1050 pay_off = (size_t) offset & ~PAGE_MASK;
1052 buff_off = (size_t) buf & ~PAGE_MASK; 1051 buff_off = (size_t) buf & ~PAGE_MASK;
1053 1052
1054 if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) 1053 if (!is_dma_copy_aligned(device, pay_off, buff_off, len))
1055 goto err1; 1054 goto err_wait;
1056 1055
1057 dest = dma_map_single(device->dev, buf, len, DMA_FROM_DEVICE); 1056 unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT);
1058 if (dma_mapping_error(device->dev, dest)) 1057 if (!unmap)
1059 goto err1; 1058 goto err_wait;
1060 1059
1061 src = dma_map_single(device->dev, offset, len, DMA_TO_DEVICE); 1060 unmap->len = len;
1062 if (dma_mapping_error(device->dev, src)) 1061 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(offset),
1063 goto err2; 1062 pay_off, len, DMA_TO_DEVICE);
1063 if (dma_mapping_error(device->dev, unmap->addr[0]))
1064 goto err_get_unmap;
1065
1066 unmap->to_cnt = 1;
1064 1067
1065 flags = DMA_COMPL_DEST_UNMAP_SINGLE | DMA_COMPL_SRC_UNMAP_SINGLE | 1068 unmap->addr[1] = dma_map_page(device->dev, virt_to_page(buf),
1066 DMA_PREP_INTERRUPT; 1069 buff_off, len, DMA_FROM_DEVICE);
1067 txd = device->device_prep_dma_memcpy(chan, dest, src, len, flags); 1070 if (dma_mapping_error(device->dev, unmap->addr[1]))
1071 goto err_get_unmap;
1072
1073 unmap->from_cnt = 1;
1074
1075 txd = device->device_prep_dma_memcpy(chan, unmap->addr[1],
1076 unmap->addr[0], len,
1077 DMA_PREP_INTERRUPT);
1068 if (!txd) 1078 if (!txd)
1069 goto err3; 1079 goto err_get_unmap;
1070 1080
1071 txd->callback = ntb_rx_copy_callback; 1081 txd->callback = ntb_rx_copy_callback;
1072 txd->callback_param = entry; 1082 txd->callback_param = entry;
1083 dma_set_unmap(txd, unmap);
1073 1084
1074 cookie = dmaengine_submit(txd); 1085 cookie = dmaengine_submit(txd);
1075 if (dma_submit_error(cookie)) 1086 if (dma_submit_error(cookie))
1076 goto err3; 1087 goto err_set_unmap;
1088
1089 dmaengine_unmap_put(unmap);
1077 1090
1078 qp->last_cookie = cookie; 1091 qp->last_cookie = cookie;
1079 1092
@@ -1081,11 +1094,11 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset,
1081 1094
1082 return; 1095 return;
1083 1096
1084err3: 1097err_set_unmap:
1085 dma_unmap_single(device->dev, src, len, DMA_TO_DEVICE); 1098 dmaengine_unmap_put(unmap);
1086err2: 1099err_get_unmap:
1087 dma_unmap_single(device->dev, dest, len, DMA_FROM_DEVICE); 1100 dmaengine_unmap_put(unmap);
1088err1: 1101err_wait:
1089 /* If the callbacks come out of order, the writing of the index to the 1102 /* If the callbacks come out of order, the writing of the index to the
1090 * last completed will be out of order. This may result in the 1103 * last completed will be out of order. This may result in the
1091 * receive stalling forever. 1104 * receive stalling forever.
@@ -1245,12 +1258,12 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
1245 struct dma_chan *chan = qp->dma_chan; 1258 struct dma_chan *chan = qp->dma_chan;
1246 struct dma_device *device; 1259 struct dma_device *device;
1247 size_t dest_off, buff_off; 1260 size_t dest_off, buff_off;
1248 dma_addr_t src, dest; 1261 struct dmaengine_unmap_data *unmap;
1262 dma_addr_t dest;
1249 dma_cookie_t cookie; 1263 dma_cookie_t cookie;
1250 void __iomem *offset; 1264 void __iomem *offset;
1251 size_t len = entry->len; 1265 size_t len = entry->len;
1252 void *buf = entry->buf; 1266 void *buf = entry->buf;
1253 unsigned long flags;
1254 1267
1255 offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index; 1268 offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index;
1256 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); 1269 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);
@@ -1273,28 +1286,41 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
1273 if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) 1286 if (!is_dma_copy_aligned(device, buff_off, dest_off, len))
1274 goto err; 1287 goto err;
1275 1288
1276 src = dma_map_single(device->dev, buf, len, DMA_TO_DEVICE); 1289 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT);
1277 if (dma_mapping_error(device->dev, src)) 1290 if (!unmap)
1278 goto err; 1291 goto err;
1279 1292
1280 flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_PREP_INTERRUPT; 1293 unmap->len = len;
1281 txd = device->device_prep_dma_memcpy(chan, dest, src, len, flags); 1294 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf),
1295 buff_off, len, DMA_TO_DEVICE);
1296 if (dma_mapping_error(device->dev, unmap->addr[0]))
1297 goto err_get_unmap;
1298
1299 unmap->to_cnt = 1;
1300
1301 txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len,
1302 DMA_PREP_INTERRUPT);
1282 if (!txd) 1303 if (!txd)
1283 goto err1; 1304 goto err_get_unmap;
1284 1305
1285 txd->callback = ntb_tx_copy_callback; 1306 txd->callback = ntb_tx_copy_callback;
1286 txd->callback_param = entry; 1307 txd->callback_param = entry;
1308 dma_set_unmap(txd, unmap);
1287 1309
1288 cookie = dmaengine_submit(txd); 1310 cookie = dmaengine_submit(txd);
1289 if (dma_submit_error(cookie)) 1311 if (dma_submit_error(cookie))
1290 goto err1; 1312 goto err_set_unmap;
1313
1314 dmaengine_unmap_put(unmap);
1291 1315
1292 dma_async_issue_pending(chan); 1316 dma_async_issue_pending(chan);
1293 qp->tx_async++; 1317 qp->tx_async++;
1294 1318
1295 return; 1319 return;
1296err1: 1320err_set_unmap:
1297 dma_unmap_single(device->dev, src, len, DMA_TO_DEVICE); 1321 dmaengine_unmap_put(unmap);
1322err_get_unmap:
1323 dmaengine_unmap_put(unmap);
1298err: 1324err:
1299 ntb_memcpy_tx(entry, offset); 1325 ntb_memcpy_tx(entry, offset);
1300 qp->tx_memcpy++; 1326 qp->tx_memcpy++;
diff --git a/drivers/spi/spi-dw-mid.c b/drivers/spi/spi-dw-mid.c
index b9f0192758d6..6d207afec8cb 100644
--- a/drivers/spi/spi-dw-mid.c
+++ b/drivers/spi/spi-dw-mid.c
@@ -150,7 +150,7 @@ static int mid_spi_dma_transfer(struct dw_spi *dws, int cs_change)
150 &dws->tx_sgl, 150 &dws->tx_sgl,
151 1, 151 1,
152 DMA_MEM_TO_DEV, 152 DMA_MEM_TO_DEV,
153 DMA_PREP_INTERRUPT | DMA_COMPL_SKIP_DEST_UNMAP); 153 DMA_PREP_INTERRUPT);
154 txdesc->callback = dw_spi_dma_done; 154 txdesc->callback = dw_spi_dma_done;
155 txdesc->callback_param = dws; 155 txdesc->callback_param = dws;
156 156
@@ -173,7 +173,7 @@ static int mid_spi_dma_transfer(struct dw_spi *dws, int cs_change)
173 &dws->rx_sgl, 173 &dws->rx_sgl,
174 1, 174 1,
175 DMA_DEV_TO_MEM, 175 DMA_DEV_TO_MEM,
176 DMA_PREP_INTERRUPT | DMA_COMPL_SKIP_DEST_UNMAP); 176 DMA_PREP_INTERRUPT);
177 rxdesc->callback = dw_spi_dma_done; 177 rxdesc->callback = dw_spi_dma_done;
178 rxdesc->callback_param = dws; 178 rxdesc->callback_param = dws;
179 179
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index 4b460a683968..41cf0c399288 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -171,12 +171,6 @@ struct dma_interleaved_template {
171 * @DMA_CTRL_ACK - if clear, the descriptor cannot be reused until the client 171 * @DMA_CTRL_ACK - if clear, the descriptor cannot be reused until the client
172 * acknowledges receipt, i.e. has has a chance to establish any dependency 172 * acknowledges receipt, i.e. has has a chance to establish any dependency
173 * chains 173 * chains
174 * @DMA_COMPL_SKIP_SRC_UNMAP - set to disable dma-unmapping the source buffer(s)
175 * @DMA_COMPL_SKIP_DEST_UNMAP - set to disable dma-unmapping the destination(s)
176 * @DMA_COMPL_SRC_UNMAP_SINGLE - set to do the source dma-unmapping as single
177 * (if not set, do the source dma-unmapping as page)
178 * @DMA_COMPL_DEST_UNMAP_SINGLE - set to do the destination dma-unmapping as single
179 * (if not set, do the destination dma-unmapping as page)
180 * @DMA_PREP_PQ_DISABLE_P - prevent generation of P while generating Q 174 * @DMA_PREP_PQ_DISABLE_P - prevent generation of P while generating Q
181 * @DMA_PREP_PQ_DISABLE_Q - prevent generation of Q while generating P 175 * @DMA_PREP_PQ_DISABLE_Q - prevent generation of Q while generating P
182 * @DMA_PREP_CONTINUE - indicate to a driver that it is reusing buffers as 176 * @DMA_PREP_CONTINUE - indicate to a driver that it is reusing buffers as
@@ -188,14 +182,10 @@ struct dma_interleaved_template {
188enum dma_ctrl_flags { 182enum dma_ctrl_flags {
189 DMA_PREP_INTERRUPT = (1 << 0), 183 DMA_PREP_INTERRUPT = (1 << 0),
190 DMA_CTRL_ACK = (1 << 1), 184 DMA_CTRL_ACK = (1 << 1),
191 DMA_COMPL_SKIP_SRC_UNMAP = (1 << 2), 185 DMA_PREP_PQ_DISABLE_P = (1 << 2),
192 DMA_COMPL_SKIP_DEST_UNMAP = (1 << 3), 186 DMA_PREP_PQ_DISABLE_Q = (1 << 3),
193 DMA_COMPL_SRC_UNMAP_SINGLE = (1 << 4), 187 DMA_PREP_CONTINUE = (1 << 4),
194 DMA_COMPL_DEST_UNMAP_SINGLE = (1 << 5), 188 DMA_PREP_FENCE = (1 << 5),
195 DMA_PREP_PQ_DISABLE_P = (1 << 6),
196 DMA_PREP_PQ_DISABLE_Q = (1 << 7),
197 DMA_PREP_CONTINUE = (1 << 8),
198 DMA_PREP_FENCE = (1 << 9),
199}; 189};
200 190
201/** 191/**
@@ -413,6 +403,17 @@ void dma_chan_cleanup(struct kref *kref);
413typedef bool (*dma_filter_fn)(struct dma_chan *chan, void *filter_param); 403typedef bool (*dma_filter_fn)(struct dma_chan *chan, void *filter_param);
414 404
415typedef void (*dma_async_tx_callback)(void *dma_async_param); 405typedef void (*dma_async_tx_callback)(void *dma_async_param);
406
407struct dmaengine_unmap_data {
408 u8 to_cnt;
409 u8 from_cnt;
410 u8 bidi_cnt;
411 struct device *dev;
412 struct kref kref;
413 size_t len;
414 dma_addr_t addr[0];
415};
416
416/** 417/**
417 * struct dma_async_tx_descriptor - async transaction descriptor 418 * struct dma_async_tx_descriptor - async transaction descriptor
418 * ---dma generic offload fields--- 419 * ---dma generic offload fields---
@@ -438,6 +439,7 @@ struct dma_async_tx_descriptor {
438 dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx); 439 dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx);
439 dma_async_tx_callback callback; 440 dma_async_tx_callback callback;
440 void *callback_param; 441 void *callback_param;
442 struct dmaengine_unmap_data *unmap;
441#ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH 443#ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
442 struct dma_async_tx_descriptor *next; 444 struct dma_async_tx_descriptor *next;
443 struct dma_async_tx_descriptor *parent; 445 struct dma_async_tx_descriptor *parent;
@@ -445,6 +447,40 @@ struct dma_async_tx_descriptor {
445#endif 447#endif
446}; 448};
447 449
450#ifdef CONFIG_DMA_ENGINE
451static inline void dma_set_unmap(struct dma_async_tx_descriptor *tx,
452 struct dmaengine_unmap_data *unmap)
453{
454 kref_get(&unmap->kref);
455 tx->unmap = unmap;
456}
457
458struct dmaengine_unmap_data *
459dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags);
460void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap);
461#else
462static inline void dma_set_unmap(struct dma_async_tx_descriptor *tx,
463 struct dmaengine_unmap_data *unmap)
464{
465}
466static inline struct dmaengine_unmap_data *
467dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)
468{
469 return NULL;
470}
471static inline void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)
472{
473}
474#endif
475
476static inline void dma_descriptor_unmap(struct dma_async_tx_descriptor *tx)
477{
478 if (tx->unmap) {
479 dmaengine_unmap_put(tx->unmap);
480 tx->unmap = NULL;
481 }
482}
483
448#ifndef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH 484#ifndef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
449static inline void txd_lock(struct dma_async_tx_descriptor *txd) 485static inline void txd_lock(struct dma_async_tx_descriptor *txd)
450{ 486{
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-12 05:46:39 -0500