aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2009-07-30 19:46:31 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2009-07-30 19:46:31 -0400
commitdb06816cb9ceb21a2bf24122407b7b4fe419c3fc (patch)
tree25a96cbc392edfe35ea6dcfaa4b0b2a7601d779d
parent784b1d6b21cb25791b68276af27a7c2321d5a86f (diff)
parent808347f6a31792079e345ec865e9cfcb6e8ae6b2 (diff)
Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx
* 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx: dmaengine: at_hdmac: add DMA slave transfers dmaengine: at_hdmac: new driver for the Atmel AHB DMA Controller dmaengine: dmatest: correct thread_count while using multiple thread per channel dmaengine: dmatest: add a maximum number of test iterations drivers/dma: Remove unnecessary semicolons drivers/dma/fsldma.c: Remove unnecessary semicolons dmaengine: move HIGHMEM64G restriction to ASYNC_TX_DMA fsldma: do not clear bandwidth control bits on the 83xx controller fsldma: enable external start for the 83xx controller fsldma: use PCI Read Multiple command
Diffstat
-rw-r--r--arch/arm/mach-at91/include/mach/at_hdmac.h102
-rw-r--r--crypto/async_tx/async_xor.c2
-rw-r--r--drivers/dma/Kconfig12
-rw-r--r--drivers/dma/Makefile1
-rw-r--r--drivers/dma/at_hdmac.c1213
-rw-r--r--drivers/dma/at_hdmac_regs.h353
-rw-r--r--drivers/dma/dmatest.c21
-rw-r--r--drivers/dma/fsldma.c17
-rw-r--r--drivers/dma/fsldma.h1
-rw-r--r--drivers/dma/mv_xor.c2
10 files changed, 1711 insertions, 13 deletions
diff --git a/arch/arm/mach-at91/include/mach/at_hdmac.h b/arch/arm/mach-at91/include/mach/at_hdmac.h
new file mode 100644
index 000000000000..187cb58345c0
--- /dev/null
+++ b/arch/arm/mach-at91/include/mach/at_hdmac.h
@@ -0,0 +1,102 @@
1/*
2 * Header file for the Atmel AHB DMA Controller driver
3 *
4 * Copyright (C) 2008 Atmel Corporation
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 */
11#ifndef AT_HDMAC_H
12#define AT_HDMAC_H
13
14#include <linux/dmaengine.h>
15
16/**
17 * struct at_dma_platform_data - Controller configuration parameters
18 * @nr_channels: Number of channels supported by hardware (max 8)
19 * @cap_mask: dma_capability flags supported by the platform
20 */
21struct at_dma_platform_data {
22 unsigned int nr_channels;
23 dma_cap_mask_t cap_mask;
24};
25
26/**
27 * enum at_dma_slave_width - DMA slave register access width.
28 * @AT_DMA_SLAVE_WIDTH_8BIT: Do 8-bit slave register accesses
29 * @AT_DMA_SLAVE_WIDTH_16BIT: Do 16-bit slave register accesses
30 * @AT_DMA_SLAVE_WIDTH_32BIT: Do 32-bit slave register accesses
31 */
32enum at_dma_slave_width {
33 AT_DMA_SLAVE_WIDTH_8BIT = 0,
34 AT_DMA_SLAVE_WIDTH_16BIT,
35 AT_DMA_SLAVE_WIDTH_32BIT,
36};
37
38/**
39 * struct at_dma_slave - Controller-specific information about a slave
40 * @dma_dev: required DMA master device
41 * @tx_reg: physical address of data register used for
42 * memory-to-peripheral transfers
43 * @rx_reg: physical address of data register used for
44 * peripheral-to-memory transfers
45 * @reg_width: peripheral register width
46 * @cfg: Platform-specific initializer for the CFG register
47 * @ctrla: Platform-specific initializer for the CTRLA register
48 */
49struct at_dma_slave {
50 struct device *dma_dev;
51 dma_addr_t tx_reg;
52 dma_addr_t rx_reg;
53 enum at_dma_slave_width reg_width;
54 u32 cfg;
55 u32 ctrla;
56};
57
58
59/* Platform-configurable bits in CFG */
60#define ATC_SRC_PER(h) (0xFU & (h)) /* Channel src rq associated with periph handshaking ifc h */
61#define ATC_DST_PER(h) ((0xFU & (h)) << 4) /* Channel dst rq associated with periph handshaking ifc h */
62#define ATC_SRC_REP (0x1 << 8) /* Source Replay Mod */
63#define ATC_SRC_H2SEL (0x1 << 9) /* Source Handshaking Mod */
64#define ATC_SRC_H2SEL_SW (0x0 << 9)
65#define ATC_SRC_H2SEL_HW (0x1 << 9)
66#define ATC_DST_REP (0x1 << 12) /* Destination Replay Mod */
67#define ATC_DST_H2SEL (0x1 << 13) /* Destination Handshaking Mod */
68#define ATC_DST_H2SEL_SW (0x0 << 13)
69#define ATC_DST_H2SEL_HW (0x1 << 13)
70#define ATC_SOD (0x1 << 16) /* Stop On Done */
71#define ATC_LOCK_IF (0x1 << 20) /* Interface Lock */
72#define ATC_LOCK_B (0x1 << 21) /* AHB Bus Lock */
73#define ATC_LOCK_IF_L (0x1 << 22) /* Master Interface Arbiter Lock */
74#define ATC_LOCK_IF_L_CHUNK (0x0 << 22)
75#define ATC_LOCK_IF_L_BUFFER (0x1 << 22)
76#define ATC_AHB_PROT_MASK (0x7 << 24) /* AHB Protection */
77#define ATC_FIFOCFG_MASK (0x3 << 28) /* FIFO Request Configuration */
78#define ATC_FIFOCFG_LARGESTBURST (0x0 << 28)
79#define ATC_FIFOCFG_HALFFIFO (0x1 << 28)
80#define ATC_FIFOCFG_ENOUGHSPACE (0x2 << 28)
81
82/* Platform-configurable bits in CTRLA */
83#define ATC_SCSIZE_MASK (0x7 << 16) /* Source Chunk Transfer Size */
84#define ATC_SCSIZE_1 (0x0 << 16)
85#define ATC_SCSIZE_4 (0x1 << 16)
86#define ATC_SCSIZE_8 (0x2 << 16)
87#define ATC_SCSIZE_16 (0x3 << 16)
88#define ATC_SCSIZE_32 (0x4 << 16)
89#define ATC_SCSIZE_64 (0x5 << 16)
90#define ATC_SCSIZE_128 (0x6 << 16)
91#define ATC_SCSIZE_256 (0x7 << 16)
92#define ATC_DCSIZE_MASK (0x7 << 20) /* Destination Chunk Transfer Size */
93#define ATC_DCSIZE_1 (0x0 << 20)
94#define ATC_DCSIZE_4 (0x1 << 20)
95#define ATC_DCSIZE_8 (0x2 << 20)
96#define ATC_DCSIZE_16 (0x3 << 20)
97#define ATC_DCSIZE_32 (0x4 << 20)
98#define ATC_DCSIZE_64 (0x5 << 20)
99#define ATC_DCSIZE_128 (0x6 << 20)
100#define ATC_DCSIZE_256 (0x7 << 20)
101
102#endif /* AT_HDMAC_H */
diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c
index 95fe2c8d6c51..90dd3f8bd283 100644
--- a/crypto/async_tx/async_xor.c
+++ b/crypto/async_tx/async_xor.c
@@ -300,7 +300,7 @@ EXPORT_SYMBOL_GPL(async_xor_zero_sum);
300 300
301static int __init async_xor_init(void) 301static int __init async_xor_init(void)
302{ 302{
303 #ifdef CONFIG_DMA_ENGINE 303 #ifdef CONFIG_ASYNC_TX_DMA
304 /* To conserve stack space the input src_list (array of page pointers) 304 /* To conserve stack space the input src_list (array of page pointers)
305 * is reused to hold the array of dma addresses passed to the driver. 305 * is reused to hold the array of dma addresses passed to the driver.
306 * This conversion is only possible when dma_addr_t is less than the 306 * This conversion is only possible when dma_addr_t is less than the
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 070357aaedbc..81e1020fb514 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -4,7 +4,7 @@
4 4
5menuconfig DMADEVICES 5menuconfig DMADEVICES
6 bool "DMA Engine support" 6 bool "DMA Engine support"
7 depends on !HIGHMEM64G && HAS_DMA 7 depends on HAS_DMA
8 help 8 help
9 DMA engines can do asynchronous data transfers without 9 DMA engines can do asynchronous data transfers without
10 involving the host CPU. Currently, this framework can be 10 involving the host CPU. Currently, this framework can be
@@ -46,6 +46,14 @@ config DW_DMAC
46 Support the Synopsys DesignWare AHB DMA controller. This 46 Support the Synopsys DesignWare AHB DMA controller. This
47 can be integrated in chips such as the Atmel AT32ap7000. 47 can be integrated in chips such as the Atmel AT32ap7000.
48 48
49config AT_HDMAC
50 tristate "Atmel AHB DMA support"
51 depends on ARCH_AT91SAM9RL
52 select DMA_ENGINE
53 help
54 Support the Atmel AHB DMA controller. This can be integrated in
55 chips such as the Atmel AT91SAM9RL.
56
49config FSL_DMA 57config FSL_DMA
50 tristate "Freescale Elo and Elo Plus DMA support" 58 tristate "Freescale Elo and Elo Plus DMA support"
51 depends on FSL_SOC 59 depends on FSL_SOC
@@ -108,7 +116,7 @@ config NET_DMA
108 116
109config ASYNC_TX_DMA 117config ASYNC_TX_DMA
110 bool "Async_tx: Offload support for the async_tx api" 118 bool "Async_tx: Offload support for the async_tx api"
111 depends on DMA_ENGINE 119 depends on DMA_ENGINE && !HIGHMEM64G
112 help 120 help
113 This allows the async_tx api to take advantage of offload engines for 121 This allows the async_tx api to take advantage of offload engines for
114 memcpy, memset, xor, and raid6 p+q operations. If your platform has 122 memcpy, memset, xor, and raid6 p+q operations. If your platform has
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index a0b6564800c4..40e1e0083571 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -7,5 +7,6 @@ obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
7obj-$(CONFIG_FSL_DMA) += fsldma.o 7obj-$(CONFIG_FSL_DMA) += fsldma.o
8obj-$(CONFIG_MV_XOR) += mv_xor.o 8obj-$(CONFIG_MV_XOR) += mv_xor.o
9obj-$(CONFIG_DW_DMAC) += dw_dmac.o 9obj-$(CONFIG_DW_DMAC) += dw_dmac.o
10obj-$(CONFIG_AT_HDMAC) += at_hdmac.o
10obj-$(CONFIG_MX3_IPU) += ipu/ 11obj-$(CONFIG_MX3_IPU) += ipu/
11obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o 12obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
new file mode 100644
index 000000000000..9a1e5fb412ed
--- /dev/null
+++ b/drivers/dma/at_hdmac.c
@@ -0,0 +1,1213 @@
1/*
2 * Driver for the Atmel AHB DMA Controller (aka HDMA or DMAC on AT91 systems)
3 *
4 * Copyright (C) 2008 Atmel Corporation
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 *
12 * This supports the Atmel AHB DMA Controller,
13 *
14 * The driver has currently been tested with the Atmel AT91SAM9RL
15 * and AT91SAM9G45 series.
16 */
17
18#include <linux/clk.h>
19#include <linux/dmaengine.h>
20#include <linux/dma-mapping.h>
21#include <linux/dmapool.h>
22#include <linux/interrupt.h>
23#include <linux/module.h>
24#include <linux/platform_device.h>
25
26#include "at_hdmac_regs.h"
27
28/*
29 * Glossary
30 * --------
31 *
32 * at_hdmac : Name of the ATmel AHB DMA Controller
33 * at_dma_ / atdma : ATmel DMA controller entity related
34 * atc_ / atchan : ATmel DMA Channel entity related
35 */
36
37#define ATC_DEFAULT_CFG (ATC_FIFOCFG_HALFFIFO)
38#define ATC_DEFAULT_CTRLA (0)
39#define ATC_DEFAULT_CTRLB (ATC_SIF(0) \
40 |ATC_DIF(1))
41
42/*
43 * Initial number of descriptors to allocate for each channel. This could
44 * be increased during dma usage.
45 */
46static unsigned int init_nr_desc_per_channel = 64;
47module_param(init_nr_desc_per_channel, uint, 0644);
48MODULE_PARM_DESC(init_nr_desc_per_channel,
49 "initial descriptors per channel (default: 64)");
50
51
52/* prototypes */
53static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx);
54
55
56/*----------------------------------------------------------------------*/
57
58static struct at_desc *atc_first_active(struct at_dma_chan *atchan)
59{
60 return list_first_entry(&atchan->active_list,
61 struct at_desc, desc_node);
62}
63
64static struct at_desc *atc_first_queued(struct at_dma_chan *atchan)
65{
66 return list_first_entry(&atchan->queue,
67 struct at_desc, desc_node);
68}
69
70/**
71 * atc_alloc_descriptor - allocate and return an initilized descriptor
72 * @chan: the channel to allocate descriptors for
73 * @gfp_flags: GFP allocation flags
74 *
75 * Note: The ack-bit is positioned in the descriptor flag at creation time
76 * to make initial allocation more convenient. This bit will be cleared
77 * and control will be given to client at usage time (during
78 * preparation functions).
79 */
80static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,
81 gfp_t gfp_flags)
82{
83 struct at_desc *desc = NULL;
84 struct at_dma *atdma = to_at_dma(chan->device);
85 dma_addr_t phys;
86
87 desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);
88 if (desc) {
89 memset(desc, 0, sizeof(struct at_desc));
90 dma_async_tx_descriptor_init(&desc->txd, chan);
91 /* txd.flags will be overwritten in prep functions */
92 desc->txd.flags = DMA_CTRL_ACK;
93 desc->txd.tx_submit = atc_tx_submit;
94 desc->txd.phys = phys;
95 }
96
97 return desc;
98}
99
100/**
101 * atc_desc_get - get a unsused descriptor from free_list
102 * @atchan: channel we want a new descriptor for
103 */
104static struct at_desc *atc_desc_get(struct at_dma_chan *atchan)
105{
106 struct at_desc *desc, *_desc;
107 struct at_desc *ret = NULL;
108 unsigned int i = 0;
109 LIST_HEAD(tmp_list);
110
111 spin_lock_bh(&atchan->lock);
112 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
113 i++;
114 if (async_tx_test_ack(&desc->txd)) {
115 list_del(&desc->desc_node);
116 ret = desc;
117 break;
118 }
119 dev_dbg(chan2dev(&atchan->chan_common),
120 "desc %p not ACKed\n", desc);
121 }
122 spin_unlock_bh(&atchan->lock);
123 dev_vdbg(chan2dev(&atchan->chan_common),
124 "scanned %u descriptors on freelist\n", i);
125
126 /* no more descriptor available in initial pool: create one more */
127 if (!ret) {
128 ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC);
129 if (ret) {
130 spin_lock_bh(&atchan->lock);
131 atchan->descs_allocated++;
132 spin_unlock_bh(&atchan->lock);
133 } else {
134 dev_err(chan2dev(&atchan->chan_common),
135 "not enough descriptors available\n");
136 }
137 }
138
139 return ret;
140}
141
142/**
143 * atc_desc_put - move a descriptor, including any children, to the free list
144 * @atchan: channel we work on
145 * @desc: descriptor, at the head of a chain, to move to free list
146 */
147static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc)
148{
149 if (desc) {
150 struct at_desc *child;
151
152 spin_lock_bh(&atchan->lock);
153 list_for_each_entry(child, &desc->txd.tx_list, desc_node)
154 dev_vdbg(chan2dev(&atchan->chan_common),
155 "moving child desc %p to freelist\n",
156 child);
157 list_splice_init(&desc->txd.tx_list, &atchan->free_list);
158 dev_vdbg(chan2dev(&atchan->chan_common),
159 "moving desc %p to freelist\n", desc);
160 list_add(&desc->desc_node, &atchan->free_list);
161 spin_unlock_bh(&atchan->lock);
162 }
163}
164
165/**
166 * atc_assign_cookie - compute and assign new cookie
167 * @atchan: channel we work on
168 * @desc: descriptor to asign cookie for
169 *
170 * Called with atchan->lock held and bh disabled
171 */
172static dma_cookie_t
173atc_assign_cookie(struct at_dma_chan *atchan, struct at_desc *desc)
174{
175 dma_cookie_t cookie = atchan->chan_common.cookie;
176
177 if (++cookie < 0)
178 cookie = 1;
179
180 atchan->chan_common.cookie = cookie;
181 desc->txd.cookie = cookie;
182
183 return cookie;
184}
185
186/**
187 * atc_dostart - starts the DMA engine for real
188 * @atchan: the channel we want to start
189 * @first: first descriptor in the list we want to begin with
190 *
191 * Called with atchan->lock held and bh disabled
192 */
193static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first)
194{
195 struct at_dma *atdma = to_at_dma(atchan->chan_common.device);
196
197 /* ASSERT: channel is idle */
198 if (atc_chan_is_enabled(atchan)) {
199 dev_err(chan2dev(&atchan->chan_common),
200 "BUG: Attempted to start non-idle channel\n");
201 dev_err(chan2dev(&atchan->chan_common),
202 " channel: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n",
203 channel_readl(atchan, SADDR),
204 channel_readl(atchan, DADDR),
205 channel_readl(atchan, CTRLA),
206 channel_readl(atchan, CTRLB),
207 channel_readl(atchan, DSCR));
208
209 /* The tasklet will hopefully advance the queue... */
210 return;
211 }
212
213 vdbg_dump_regs(atchan);
214
215 /* clear any pending interrupt */
216 while (dma_readl(atdma, EBCISR))
217 cpu_relax();
218
219 channel_writel(atchan, SADDR, 0);
220 channel_writel(atchan, DADDR, 0);
221 channel_writel(atchan, CTRLA, 0);
222 channel_writel(atchan, CTRLB, 0);
223 channel_writel(atchan, DSCR, first->txd.phys);
224 dma_writel(atdma, CHER, atchan->mask);
225
226 vdbg_dump_regs(atchan);
227}
228
229/**
230 * atc_chain_complete - finish work for one transaction chain
231 * @atchan: channel we work on
232 * @desc: descriptor at the head of the chain we want do complete
233 *
234 * Called with atchan->lock held and bh disabled */
235static void
236atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
237{
238 dma_async_tx_callback callback;
239 void *param;
240 struct dma_async_tx_descriptor *txd = &desc->txd;
241
242 dev_vdbg(chan2dev(&atchan->chan_common),
243 "descriptor %u complete\n", txd->cookie);
244
245 atchan->completed_cookie = txd->cookie;
246 callback = txd->callback;
247 param = txd->callback_param;
248
249 /* move children to free_list */
250 list_splice_init(&txd->tx_list, &atchan->free_list);
251 /* move myself to free_list */
252 list_move(&desc->desc_node, &atchan->free_list);
253
254 /* unmap dma addresses */
255 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
256 if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
257 dma_unmap_single(chan2parent(&atchan->chan_common),
258 desc->lli.daddr,
259 desc->len, DMA_FROM_DEVICE);
260 else
261 dma_unmap_page(chan2parent(&atchan->chan_common),
262 desc->lli.daddr,
263 desc->len, DMA_FROM_DEVICE);
264 }
265 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
266 if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
267 dma_unmap_single(chan2parent(&atchan->chan_common),
268 desc->lli.saddr,
269 desc->len, DMA_TO_DEVICE);
270 else
271 dma_unmap_page(chan2parent(&atchan->chan_common),
272 desc->lli.saddr,
273 desc->len, DMA_TO_DEVICE);
274 }
275
276 /*
277 * The API requires that no submissions are done from a
278 * callback, so we don't need to drop the lock here
279 */
280 if (callback)
281 callback(param);
282
283 dma_run_dependencies(txd);
284}
285
286/**
287 * atc_complete_all - finish work for all transactions
288 * @atchan: channel to complete transactions for
289 *
290 * Eventually submit queued descriptors if any
291 *
292 * Assume channel is idle while calling this function
293 * Called with atchan->lock held and bh disabled
294 */
295static void atc_complete_all(struct at_dma_chan *atchan)
296{
297 struct at_desc *desc, *_desc;
298 LIST_HEAD(list);
299
300 dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n");
301
302 BUG_ON(atc_chan_is_enabled(atchan));
303
304 /*
305 * Submit queued descriptors ASAP, i.e. before we go through
306 * the completed ones.
307 */
308 if (!list_empty(&atchan->queue))
309 atc_dostart(atchan, atc_first_queued(atchan));
310 /* empty active_list now it is completed */
311 list_splice_init(&atchan->active_list, &list);
312 /* empty queue list by moving descriptors (if any) to active_list */
313 list_splice_init(&atchan->queue, &atchan->active_list);
314
315 list_for_each_entry_safe(desc, _desc, &list, desc_node)
316 atc_chain_complete(atchan, desc);
317}
318
319/**
320 * atc_cleanup_descriptors - cleanup up finished descriptors in active_list
321 * @atchan: channel to be cleaned up
322 *
323 * Called with atchan->lock held and bh disabled
324 */
325static void atc_cleanup_descriptors(struct at_dma_chan *atchan)
326{
327 struct at_desc *desc, *_desc;
328 struct at_desc *child;
329
330 dev_vdbg(chan2dev(&atchan->chan_common), "cleanup descriptors\n");
331
332 list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) {
333 if (!(desc->lli.ctrla & ATC_DONE))
334 /* This one is currently in progress */
335 return;
336
337 list_for_each_entry(child, &desc->txd.tx_list, desc_node)
338 if (!(child->lli.ctrla & ATC_DONE))
339 /* Currently in progress */
340 return;
341
342 /*
343 * No descriptors so far seem to be in progress, i.e.
344 * this chain must be done.
345 */
346 atc_chain_complete(atchan, desc);
347 }
348}
349
350/**
351 * atc_advance_work - at the end of a transaction, move forward
352 * @atchan: channel where the transaction ended
353 *
354 * Called with atchan->lock held and bh disabled
355 */
356static void atc_advance_work(struct at_dma_chan *atchan)
357{
358 dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n");
359
360 if (list_empty(&atchan->active_list) ||
361 list_is_singular(&atchan->active_list)) {
362 atc_complete_all(atchan);
363 } else {
364 atc_chain_complete(atchan, atc_first_active(atchan));
365 /* advance work */
366 atc_dostart(atchan, atc_first_active(atchan));
367 }
368}
369
370
371/**
372 * atc_handle_error - handle errors reported by DMA controller
373 * @atchan: channel where error occurs
374 *
375 * Called with atchan->lock held and bh disabled
376 */
377static void atc_handle_error(struct at_dma_chan *atchan)
378{
379 struct at_desc *bad_desc;
380 struct at_desc *child;
381
382 /*
383 * The descriptor currently at the head of the active list is
384 * broked. Since we don't have any way to report errors, we'll
385 * just have to scream loudly and try to carry on.
386 */
387 bad_desc = atc_first_active(atchan);
388 list_del_init(&bad_desc->desc_node);
389
390 /* As we are stopped, take advantage to push queued descriptors
391 * in active_list */
392 list_splice_init(&atchan->queue, atchan->active_list.prev);
393
394 /* Try to restart the controller */
395 if (!list_empty(&atchan->active_list))
396 atc_dostart(atchan, atc_first_active(atchan));
397
398 /*
399 * KERN_CRITICAL may seem harsh, but since this only happens
400 * when someone submits a bad physical address in a
401 * descriptor, we should consider ourselves lucky that the
402 * controller flagged an error instead of scribbling over
403 * random memory locations.
404 */
405 dev_crit(chan2dev(&atchan->chan_common),
406 "Bad descriptor submitted for DMA!\n");
407 dev_crit(chan2dev(&atchan->chan_common),
408 " cookie: %d\n", bad_desc->txd.cookie);
409 atc_dump_lli(atchan, &bad_desc->lli);
410 list_for_each_entry(child, &bad_desc->txd.tx_list, desc_node)
411 atc_dump_lli(atchan, &child->lli);
412
413 /* Pretend the descriptor completed successfully */
414 atc_chain_complete(atchan, bad_desc);
415}
416
417
418/*-- IRQ & Tasklet ---------------------------------------------------*/
419
420static void atc_tasklet(unsigned long data)
421{
422 struct at_dma_chan *atchan = (struct at_dma_chan *)data;
423
424 /* Channel cannot be enabled here */
425 if (atc_chan_is_enabled(atchan)) {
426 dev_err(chan2dev(&atchan->chan_common),
427 "BUG: channel enabled in tasklet\n");
428 return;
429 }
430
431 spin_lock(&atchan->lock);
432 if (test_and_clear_bit(0, &atchan->error_status))
433 atc_handle_error(atchan);
434 else
435 atc_advance_work(atchan);
436
437 spin_unlock(&atchan->lock);
438}
439
440static irqreturn_t at_dma_interrupt(int irq, void *dev_id)
441{
442 struct at_dma *atdma = (struct at_dma *)dev_id;
443 struct at_dma_chan *atchan;
444 int i;
445 u32 status, pending, imr;
446 int ret = IRQ_NONE;
447
448 do {
449 imr = dma_readl(atdma, EBCIMR);
450 status = dma_readl(atdma, EBCISR);
451 pending = status & imr;
452
453 if (!pending)
454 break;
455
456 dev_vdbg(atdma->dma_common.dev,
457 "interrupt: status = 0x%08x, 0x%08x, 0x%08x\n",
458 status, imr, pending);
459
460 for (i = 0; i < atdma->dma_common.chancnt; i++) {
461 atchan = &atdma->chan[i];
462 if (pending & (AT_DMA_CBTC(i) | AT_DMA_ERR(i))) {
463 if (pending & AT_DMA_ERR(i)) {
464 /* Disable channel on AHB error */
465 dma_writel(atdma, CHDR, atchan->mask);
466 /* Give information to tasklet */
467 set_bit(0, &atchan->error_status);
468 }
469 tasklet_schedule(&atchan->tasklet);
470 ret = IRQ_HANDLED;
471 }
472 }
473
474 } while (pending);
475
476 return ret;
477}
478
479
480/*-- DMA Engine API --------------------------------------------------*/
481
482/**
483 * atc_tx_submit - set the prepared descriptor(s) to be executed by the engine
484 * @desc: descriptor at the head of the transaction chain
485 *
486 * Queue chain if DMA engine is working already
487 *
488 * Cookie increment and adding to active_list or queue must be atomic
489 */
490static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx)
491{
492 struct at_desc *desc = txd_to_at_desc(tx);
493 struct at_dma_chan *atchan = to_at_dma_chan(tx->chan);
494 dma_cookie_t cookie;
495
496 spin_lock_bh(&atchan->lock);
497 cookie = atc_assign_cookie(atchan, desc);
498
499 if (list_empty(&atchan->active_list)) {
500 dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",
501 desc->txd.cookie);
502 atc_dostart(atchan, desc);
503 list_add_tail(&desc->desc_node, &atchan->active_list);
504 } else {
505 dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",
506 desc->txd.cookie);
507 list_add_tail(&desc->desc_node, &atchan->queue);
508 }
509
510 spin_unlock_bh(&atchan->lock);
511
512 return cookie;
513}
514
515/**
516 * atc_prep_dma_memcpy - prepare a memcpy operation
517 * @chan: the channel to prepare operation on
518 * @dest: operation virtual destination address
519 * @src: operation virtual source address
520 * @len: operation length
521 * @flags: tx descriptor status flags
522 */
523static struct dma_async_tx_descriptor *
524atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
525 size_t len, unsigned long flags)
526{
527 struct at_dma_chan *atchan = to_at_dma_chan(chan);
528 struct at_desc *desc = NULL;
529 struct at_desc *first = NULL;
530 struct at_desc *prev = NULL;
531 size_t xfer_count;
532 size_t offset;
533 unsigned int src_width;
534 unsigned int dst_width;
535 u32 ctrla;
536 u32 ctrlb;
537
538 dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d0x%x s0x%x l0x%zx f0x%lx\n",
539 dest, src, len, flags);
540
541 if (unlikely(!len)) {
542 dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
543 return NULL;
544 }
545
546 ctrla = ATC_DEFAULT_CTRLA;
547 ctrlb = ATC_DEFAULT_CTRLB
548 | ATC_SRC_ADDR_MODE_INCR
549 | ATC_DST_ADDR_MODE_INCR
550 | ATC_FC_MEM2MEM;
551
552 /*
553 * We can be a lot more clever here, but this should take care
554 * of the most common optimization.
555 */
556 if (!((src | dest | len) & 3)) {
557 ctrla |= ATC_SRC_WIDTH_WORD | ATC_DST_WIDTH_WORD;
558 src_width = dst_width = 2;
559 } else if (!((src | dest | len) & 1)) {
560 ctrla |= ATC_SRC_WIDTH_HALFWORD | ATC_DST_WIDTH_HALFWORD;
561 src_width = dst_width = 1;
562 } else {
563 ctrla |= ATC_SRC_WIDTH_BYTE | ATC_DST_WIDTH_BYTE;
564 src_width = dst_width = 0;
565 }
566
567 for (offset = 0; offset < len; offset += xfer_count << src_width) {
568 xfer_count = min_t(size_t, (len - offset) >> src_width,
569 ATC_BTSIZE_MAX);
570
571 desc = atc_desc_get(atchan);
572 if (!desc)
573 goto err_desc_get;
574
575 desc->lli.saddr = src + offset;
576 desc->lli.daddr = dest + offset;
577 desc->lli.ctrla = ctrla | xfer_count;
578 desc->lli.ctrlb = ctrlb;
579
580 desc->txd.cookie = 0;
581 async_tx_ack(&desc->txd);
582
583 if (!first) {
584 first = desc;
585 } else {
586 /* inform the HW lli about chaining */
587 prev->lli.dscr = desc->txd.phys;
588 /* insert the link descriptor to the LD ring */
589 list_add_tail(&desc->desc_node,
590 &first->txd.tx_list);
591 }
592 prev = desc;
593 }
594
595 /* First descriptor of the chain embedds additional information */
596 first->txd.cookie = -EBUSY;
597 first->len = len;
598
599 /* set end-of-link to the last link descriptor of list*/
600 set_desc_eol(desc);
601
602 desc->txd.flags = flags; /* client is in control of this ack */
603
604 return &first->txd;
605
606err_desc_get:
607 atc_desc_put(atchan, first);
608 return NULL;
609}
610
611
612/**
613 * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
614 * @chan: DMA channel
615 * @sgl: scatterlist to transfer to/from
616 * @sg_len: number of entries in @scatterlist
617 * @direction: DMA direction
618 * @flags: tx descriptor status flags
619 */
620static struct dma_async_tx_descriptor *
621atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
622 unsigned int sg_len, enum dma_data_direction direction,
623 unsigned long flags)
624{
625 struct at_dma_chan *atchan = to_at_dma_chan(chan);
626 struct at_dma_slave *atslave = chan->private;
627 struct at_desc *first = NULL;
628 struct at_desc *prev = NULL;
629 u32 ctrla;
630 u32 ctrlb;
631 dma_addr_t reg;
632 unsigned int reg_width;
633 unsigned int mem_width;
634 unsigned int i;
635 struct scatterlist *sg;
636 size_t total_len = 0;
637
638 dev_vdbg(chan2dev(chan), "prep_slave_sg: %s f0x%lx\n",
639 direction == DMA_TO_DEVICE ? "TO DEVICE" : "FROM DEVICE",
640 flags);
641
642 if (unlikely(!atslave || !sg_len)) {
643 dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
644 return NULL;
645 }
646
647 reg_width = atslave->reg_width;
648
649 sg_len = dma_map_sg(chan2parent(chan), sgl, sg_len, direction);
650
651 ctrla = ATC_DEFAULT_CTRLA | atslave->ctrla;
652 ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN;
653
654 switch (direction) {
655 case DMA_TO_DEVICE:
656 ctrla |= ATC_DST_WIDTH(reg_width);
657 ctrlb |= ATC_DST_ADDR_MODE_FIXED
658 | ATC_SRC_ADDR_MODE_INCR
659 | ATC_FC_MEM2PER;
660 reg = atslave->tx_reg;
661 for_each_sg(sgl, sg, sg_len, i) {
662 struct at_desc *desc;
663 u32 len;
664 u32 mem;
665
666 desc = atc_desc_get(atchan);
667 if (!desc)
668 goto err_desc_get;
669
670 mem = sg_phys(sg);
671 len = sg_dma_len(sg);
672 mem_width = 2;
673 if (unlikely(mem & 3 || len & 3))
674 mem_width = 0;
675
676 desc->lli.saddr = mem;
677 desc->lli.daddr = reg;
678 desc->lli.ctrla = ctrla
679 | ATC_SRC_WIDTH(mem_width)
680 | len >> mem_width;
681 desc->lli.ctrlb = ctrlb;
682
683 if (!first) {
684 first = desc;
685 } else {
686 /* inform the HW lli about chaining */
687 prev->lli.dscr = desc->txd.phys;
688 /* insert the link descriptor to the LD ring */
689 list_add_tail(&desc->desc_node,
690 &first->txd.tx_list);
691 }
692 prev = desc;
693 total_len += len;
694 }
695 break;
696 case DMA_FROM_DEVICE:
697 ctrla |= ATC_SRC_WIDTH(reg_width);
698 ctrlb |= ATC_DST_ADDR_MODE_INCR
699 | ATC_SRC_ADDR_MODE_FIXED
700 | ATC_FC_PER2MEM;
701
702 reg = atslave->rx_reg;
703 for_each_sg(sgl, sg, sg_len, i) {
704 struct at_desc *desc;
705 u32 len;
706 u32 mem;
707
708 desc = atc_desc_get(atchan);
709 if (!desc)
710 goto err_desc_get;
711
712 mem = sg_phys(sg);
713 len = sg_dma_len(sg);
714 mem_width = 2;
715 if (unlikely(mem & 3 || len & 3))
716 mem_width = 0;
717
718 desc->lli.saddr = reg;
719 desc->lli.daddr = mem;
720 desc->lli.ctrla = ctrla
721 | ATC_DST_WIDTH(mem_width)
722 | len >> mem_width;
723 desc->lli.ctrlb = ctrlb;
724
725 if (!first) {
726 first = desc;
727 } else {
728 /* inform the HW lli about chaining */
729 prev->lli.dscr = desc->txd.phys;
730 /* insert the link descriptor to the LD ring */
731 list_add_tail(&desc->desc_node,
732 &first->txd.tx_list);
733 }
734 prev = desc;
735 total_len += len;
736 }
737 break;
738 default:
739 return NULL;
740 }
741
742 /* set end-of-link to the last link descriptor of list*/
743 set_desc_eol(prev);
744
745 /* First descriptor of the chain embedds additional information */
746 first->txd.cookie = -EBUSY;
747 first->len = total_len;
748
749 /* last link descriptor of list is responsible of flags */
750 prev->txd.flags = flags; /* client is in control of this ack */
751
752 return &first->txd;
753
754err_desc_get:
755 dev_err(chan2dev(chan), "not enough descriptors available\n");
756 atc_desc_put(atchan, first);
757 return NULL;
758}
759
760static void atc_terminate_all(struct dma_chan *chan)
761{
762 struct at_dma_chan *atchan = to_at_dma_chan(chan);
763 struct at_dma *atdma = to_at_dma(chan->device);
764 struct at_desc *desc, *_desc;
765 LIST_HEAD(list);
766
767 /*
768 * This is only called when something went wrong elsewhere, so
769 * we don't really care about the data. Just disable the
770 * channel. We still have to poll the channel enable bit due
771 * to AHB/HSB limitations.
772 */
773 spin_lock_bh(&atchan->lock);
774
775 dma_writel(atdma, CHDR, atchan->mask);
776
777 /* confirm that this channel is disabled */
778 while (dma_readl(atdma, CHSR) & atchan->mask)
779 cpu_relax();
780
781 /* active_list entries will end up before queued entries */
782 list_splice_init(&atchan->queue, &list);
783 list_splice_init(&atchan->active_list, &list);
784
785 spin_unlock_bh(&atchan->lock);
786
787 /* Flush all pending and queued descriptors */
788 list_for_each_entry_safe(desc, _desc, &list, desc_node)
789 atc_chain_complete(atchan, desc);
790}
791
792/**
793 * atc_is_tx_complete - poll for transaction completion
794 * @chan: DMA channel
795 * @cookie: transaction identifier to check status of
796 * @done: if not %NULL, updated with last completed transaction
797 * @used: if not %NULL, updated with last used transaction
798 *
799 * If @done and @used are passed in, upon return they reflect the driver
800 * internal state and can be used with dma_async_is_complete() to check
801 * the status of multiple cookies without re-checking hardware state.
802 */
803static enum dma_status
804atc_is_tx_complete(struct dma_chan *chan,
805 dma_cookie_t cookie,
806 dma_cookie_t *done, dma_cookie_t *used)
807{
808 struct at_dma_chan *atchan = to_at_dma_chan(chan);
809 dma_cookie_t last_used;
810 dma_cookie_t last_complete;
811 enum dma_status ret;
812
813 dev_vdbg(chan2dev(chan), "is_tx_complete: %d (d%d, u%d)\n",
814 cookie, done ? *done : 0, used ? *used : 0);
815
816 spin_lock_bh(atchan->lock);
817
818 last_complete = atchan->completed_cookie;
819 last_used = chan->cookie;
820
821 ret = dma_async_is_complete(cookie, last_complete, last_used);
822 if (ret != DMA_SUCCESS) {
823 atc_cleanup_descriptors(atchan);
824
825 last_complete = atchan->completed_cookie;
826 last_used = chan->cookie;
827
828 ret = dma_async_is_complete(cookie, last_complete, last_used);
829 }
830
831 spin_unlock_bh(atchan->lock);
832
833 if (done)
834 *done = last_complete;
835 if (used)
836 *used = last_used;
837
838 return ret;
839}
840
841/**
842 * atc_issue_pending - try to finish work
843 * @chan: target DMA channel
844 */
845static void atc_issue_pending(struct dma_chan *chan)
846{
847 struct at_dma_chan *atchan = to_at_dma_chan(chan);
848
849 dev_vdbg(chan2dev(chan), "issue_pending\n");
850
851 if (!atc_chan_is_enabled(atchan)) {
852 spin_lock_bh(&atchan->lock);
853 atc_advance_work(atchan);
854 spin_unlock_bh(&atchan->lock);
855 }
856}
857
858/**
859 * atc_alloc_chan_resources - allocate resources for DMA channel
860 * @chan: allocate descriptor resources for this channel
861 * @client: current client requesting the channel be ready for requests
862 *
863 * return - the number of allocated descriptors
864 */
865static int atc_alloc_chan_resources(struct dma_chan *chan)
866{
867 struct at_dma_chan *atchan = to_at_dma_chan(chan);
868 struct at_dma *atdma = to_at_dma(chan->device);
869 struct at_desc *desc;
870 struct at_dma_slave *atslave;
871 int i;
872 u32 cfg;
873 LIST_HEAD(tmp_list);
874
875 dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
876
877 /* ASSERT: channel is idle */
878 if (atc_chan_is_enabled(atchan)) {
879 dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");
880 return -EIO;
881 }
882
883 cfg = ATC_DEFAULT_CFG;
884
885 atslave = chan->private;
886 if (atslave) {
887 /*
888 * We need controller-specific data to set up slave
889 * transfers.
890 */
891 BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev);
892
893 /* if cfg configuration specified take it instad of default */
894 if (atslave->cfg)
895 cfg = atslave->cfg;
896 }
897
898 /* have we already been set up?
899 * reconfigure channel but no need to reallocate descriptors */
900 if (!list_empty(&atchan->free_list))
901 return atchan->descs_allocated;
902
903 /* Allocate initial pool of descriptors */
904 for (i = 0; i < init_nr_desc_per_channel; i++) {
905 desc = atc_alloc_descriptor(chan, GFP_KERNEL);
906 if (!desc) {
907 dev_err(atdma->dma_common.dev,
908 "Only %d initial descriptors\n", i);
909 break;
910 }
911 list_add_tail(&desc->desc_node, &tmp_list);
912 }
913
914 spin_lock_bh(&atchan->lock);
915 atchan->descs_allocated = i;
916 list_splice(&tmp_list, &atchan->free_list);
917 atchan->completed_cookie = chan->cookie = 1;
918 spin_unlock_bh(&atchan->lock);
919
920 /* channel parameters */
921 channel_writel(atchan, CFG, cfg);
922
923 dev_dbg(chan2dev(chan),
924 "alloc_chan_resources: allocated %d descriptors\n",
925 atchan->descs_allocated);
926
927 return atchan->descs_allocated;
928}
929
930/**
931 * atc_free_chan_resources - free all channel resources
932 * @chan: DMA channel
933 */
934static void atc_free_chan_resources(struct dma_chan *chan)
935{
936 struct at_dma_chan *atchan = to_at_dma_chan(chan);
937 struct at_dma *atdma = to_at_dma(chan->device);
938 struct at_desc *desc, *_desc;
939 LIST_HEAD(list);
940
941 dev_dbg(chan2dev(chan), "free_chan_resources: (descs allocated=%u)\n",
942 atchan->descs_allocated);
943
944 /* ASSERT: channel is idle */
945 BUG_ON(!list_empty(&atchan->active_list));
946 BUG_ON(!list_empty(&atchan->queue));
947 BUG_ON(atc_chan_is_enabled(atchan));
948
949 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
950 dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc);
951 list_del(&desc->desc_node);
952 /* free link descriptor */
953 dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys);
954 }
955 list_splice_init(&atchan->free_list, &list);
956 atchan->descs_allocated = 0;
957
958 dev_vdbg(chan2dev(chan), "free_chan_resources: done\n");
959}
960
961
962/*-- Module Management -----------------------------------------------*/
963
964/**
965 * at_dma_off - disable DMA controller
966 * @atdma: the Atmel HDAMC device
967 */
968static void at_dma_off(struct at_dma *atdma)
969{
970 dma_writel(atdma, EN, 0);
971
972 /* disable all interrupts */
973 dma_writel(atdma, EBCIDR, -1L);
974
975 /* confirm that all channels are disabled */
976 while (dma_readl(atdma, CHSR) & atdma->all_chan_mask)
977 cpu_relax();
978}
979
980static int __init at_dma_probe(struct platform_device *pdev)
981{
982 struct at_dma_platform_data *pdata;
983 struct resource *io;
984 struct at_dma *atdma;
985 size_t size;
986 int irq;
987 int err;
988 int i;
989
990 /* get DMA Controller parameters from platform */
991 pdata = pdev->dev.platform_data;
992 if (!pdata || pdata->nr_channels > AT_DMA_MAX_NR_CHANNELS)
993 return -EINVAL;
994
995 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
996 if (!io)
997 return -EINVAL;
998
999 irq = platform_get_irq(pdev, 0);
1000 if (irq < 0)
1001 return irq;
1002
1003 size = sizeof(struct at_dma);
1004 size += pdata->nr_channels * sizeof(struct at_dma_chan);
1005 atdma = kzalloc(size, GFP_KERNEL);
1006 if (!atdma)
1007 return -ENOMEM;
1008
1009 /* discover transaction capabilites from the platform data */
1010 atdma->dma_common.cap_mask = pdata->cap_mask;
1011 atdma->all_chan_mask = (1 << pdata->nr_channels) - 1;
1012
1013 size = io->end - io->start + 1;
1014 if (!request_mem_region(io->start, size, pdev->dev.driver->name)) {
1015 err = -EBUSY;
1016 goto err_kfree;
1017 }
1018
1019 atdma->regs = ioremap(io->start, size);
1020 if (!atdma->regs) {
1021 err = -ENOMEM;
1022 goto err_release_r;
1023 }
1024
1025 atdma->clk = clk_get(&pdev->dev, "dma_clk");
1026 if (IS_ERR(atdma->clk)) {
1027 err = PTR_ERR(atdma->clk);
1028 goto err_clk;
1029 }
1030 clk_enable(atdma->clk);
1031
1032 /* force dma off, just in case */
1033 at_dma_off(atdma);
1034
1035 err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma);
1036 if (err)
1037 goto err_irq;
1038
1039 platform_set_drvdata(pdev, atdma);
1040
1041 /* create a pool of consistent memory blocks for hardware descriptors */
1042 atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool",
1043 &pdev->dev, sizeof(struct at_desc),
1044 4 /* word alignment */, 0);
1045 if (!atdma->dma_desc_pool) {
1046 dev_err(&pdev->dev, "No memory for descriptors dma pool\n");
1047 err = -ENOMEM;
1048 goto err_pool_create;
1049 }
1050
1051 /* clear any pending interrupt */
1052 while (dma_readl(atdma, EBCISR))
1053 cpu_relax();
1054
1055 /* initialize channels related values */
1056 INIT_LIST_HEAD(&atdma->dma_common.channels);
1057 for (i = 0; i < pdata->nr_channels; i++, atdma->dma_common.chancnt++) {
1058 struct at_dma_chan *atchan = &atdma->chan[i];
1059
1060 atchan->chan_common.device = &atdma->dma_common;
1061 atchan->chan_common.cookie = atchan->completed_cookie = 1;
1062 atchan->chan_common.chan_id = i;
1063 list_add_tail(&atchan->chan_common.device_node,
1064 &atdma->dma_common.channels);
1065
1066 atchan->ch_regs = atdma->regs + ch_regs(i);
1067 spin_lock_init(&atchan->lock);
1068 atchan->mask = 1 << i;
1069
1070 INIT_LIST_HEAD(&atchan->active_list);
1071 INIT_LIST_HEAD(&atchan->queue);
1072 INIT_LIST_HEAD(&atchan->free_list);
1073
1074 tasklet_init(&atchan->tasklet, atc_tasklet,
1075 (unsigned long)atchan);
1076 atc_enable_irq(atchan);
1077 }
1078
1079 /* set base routines */
1080 atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources;
1081 atdma->dma_common.device_free_chan_resources = atc_free_chan_resources;
1082 atdma->dma_common.device_is_tx_complete = atc_is_tx_complete;
1083 atdma->dma_common.device_issue_pending = atc_issue_pending;
1084 atdma->dma_common.dev = &pdev->dev;
1085
1086 /* set prep routines based on capability */
1087 if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))
1088 atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;
1089
1090 if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {
1091 atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;
1092 atdma->dma_common.device_terminate_all = atc_terminate_all;
1093 }
1094
1095 dma_writel(atdma, EN, AT_DMA_ENABLE);
1096
1097 dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s), %d channels\n",
1098 dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",
1099 dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "",
1100 atdma->dma_common.chancnt);
1101
1102 dma_async_device_register(&atdma->dma_common);
1103
1104 return 0;
1105
1106err_pool_create:
1107 platform_set_drvdata(pdev, NULL);
1108 free_irq(platform_get_irq(pdev, 0), atdma);
1109err_irq:
1110 clk_disable(atdma->clk);
1111 clk_put(atdma->clk);
1112err_clk:
1113 iounmap(atdma->regs);
1114 atdma->regs = NULL;
1115err_release_r:
1116 release_mem_region(io->start, size);
1117err_kfree:
1118 kfree(atdma);
1119 return err;
1120}
1121
1122static int __exit at_dma_remove(struct platform_device *pdev)
1123{
1124 struct at_dma *atdma = platform_get_drvdata(pdev);
1125 struct dma_chan *chan, *_chan;
1126 struct resource *io;
1127
1128 at_dma_off(atdma);
1129 dma_async_device_unregister(&atdma->dma_common);
1130
1131 dma_pool_destroy(atdma->dma_desc_pool);
1132 platform_set_drvdata(pdev, NULL);
1133 free_irq(platform_get_irq(pdev, 0), atdma);
1134
1135 list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
1136 device_node) {
1137 struct at_dma_chan *atchan = to_at_dma_chan(chan);
1138
1139 /* Disable interrupts */
1140 atc_disable_irq(atchan);
1141 tasklet_disable(&atchan->tasklet);
1142
1143 tasklet_kill(&atchan->tasklet);
1144 list_del(&chan->device_node);
1145 }
1146
1147 clk_disable(atdma->clk);
1148 clk_put(atdma->clk);
1149
1150 iounmap(atdma->regs);
1151 atdma->regs = NULL;
1152
1153 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1154 release_mem_region(io->start, io->end - io->start + 1);
1155
1156 kfree(atdma);
1157
1158 return 0;
1159}
1160
1161static void at_dma_shutdown(struct platform_device *pdev)
1162{
1163 struct at_dma *atdma = platform_get_drvdata(pdev);
1164
1165 at_dma_off(platform_get_drvdata(pdev));
1166 clk_disable(atdma->clk);
1167}
1168
1169static int at_dma_suspend_late(struct platform_device *pdev, pm_message_t mesg)
1170{
1171 struct at_dma *atdma = platform_get_drvdata(pdev);
1172
1173 at_dma_off(platform_get_drvdata(pdev));
1174 clk_disable(atdma->clk);
1175 return 0;
1176}
1177
1178static int at_dma_resume_early(struct platform_device *pdev)
1179{
1180 struct at_dma *atdma = platform_get_drvdata(pdev);
1181
1182 clk_enable(atdma->clk);
1183 dma_writel(atdma, EN, AT_DMA_ENABLE);
1184 return 0;
1185
1186}
1187
1188static struct platform_driver at_dma_driver = {
1189 .remove = __exit_p(at_dma_remove),
1190 .shutdown = at_dma_shutdown,
1191 .suspend_late = at_dma_suspend_late,
1192 .resume_early = at_dma_resume_early,
1193 .driver = {
1194 .name = "at_hdmac",
1195 },
1196};
1197
1198static int __init at_dma_init(void)
1199{
1200 return platform_driver_probe(&at_dma_driver, at_dma_probe);
1201}
1202module_init(at_dma_init);
1203
1204static void __exit at_dma_exit(void)
1205{
1206 platform_driver_unregister(&at_dma_driver);
1207}
1208module_exit(at_dma_exit);
1209
1210MODULE_DESCRIPTION("Atmel AHB DMA Controller driver");
1211MODULE_AUTHOR("Nicolas Ferre <nicolas.ferre@atmel.com>");
1212MODULE_LICENSE("GPL");
1213MODULE_ALIAS("platform:at_hdmac");
diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h
new file mode 100644
index 000000000000..4c972afc49ec
--- /dev/null
+++ b/drivers/dma/at_hdmac_regs.h
@@ -0,0 +1,353 @@
1/*
2 * Header file for the Atmel AHB DMA Controller driver
3 *
4 * Copyright (C) 2008 Atmel Corporation
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 */
11#ifndef AT_HDMAC_REGS_H
12#define AT_HDMAC_REGS_H
13
14#include <mach/at_hdmac.h>
15
16#define AT_DMA_MAX_NR_CHANNELS 8
17
18
19#define AT_DMA_GCFG 0x00 /* Global Configuration Register */
20#define AT_DMA_IF_BIGEND(i) (0x1 << (i)) /* AHB-Lite Interface i in Big-endian mode */
21#define AT_DMA_ARB_CFG (0x1 << 4) /* Arbiter mode. */
22#define AT_DMA_ARB_CFG_FIXED (0x0 << 4)
23#define AT_DMA_ARB_CFG_ROUND_ROBIN (0x1 << 4)
24
25#define AT_DMA_EN 0x04 /* Controller Enable Register */
26#define AT_DMA_ENABLE (0x1 << 0)
27
28#define AT_DMA_SREQ 0x08 /* Software Single Request Register */
29#define AT_DMA_SSREQ(x) (0x1 << ((x) << 1)) /* Request a source single transfer on channel x */
30#define AT_DMA_DSREQ(x) (0x1 << (1 + ((x) << 1))) /* Request a destination single transfer on channel x */
31
32#define AT_DMA_CREQ 0x0C /* Software Chunk Transfer Request Register */
33#define AT_DMA_SCREQ(x) (0x1 << ((x) << 1)) /* Request a source chunk transfer on channel x */
34#define AT_DMA_DCREQ(x) (0x1 << (1 + ((x) << 1))) /* Request a destination chunk transfer on channel x */
35
36#define AT_DMA_LAST 0x10 /* Software Last Transfer Flag Register */
37#define AT_DMA_SLAST(x) (0x1 << ((x) << 1)) /* This src rq is last tx of buffer on channel x */
38#define AT_DMA_DLAST(x) (0x1 << (1 + ((x) << 1))) /* This dst rq is last tx of buffer on channel x */
39
40#define AT_DMA_SYNC 0x14 /* Request Synchronization Register */
41#define AT_DMA_SYR(h) (0x1 << (h)) /* Synchronize handshake line h */
42
43/* Error, Chained Buffer transfer completed and Buffer transfer completed Interrupt registers */
44#define AT_DMA_EBCIER 0x18 /* Enable register */
45#define AT_DMA_EBCIDR 0x1C /* Disable register */
46#define AT_DMA_EBCIMR 0x20 /* Mask Register */
47#define AT_DMA_EBCISR 0x24 /* Status Register */
48#define AT_DMA_CBTC_OFFSET 8
49#define AT_DMA_ERR_OFFSET 16
50#define AT_DMA_BTC(x) (0x1 << (x))
51#define AT_DMA_CBTC(x) (0x1 << (AT_DMA_CBTC_OFFSET + (x)))
52#define AT_DMA_ERR(x) (0x1 << (AT_DMA_ERR_OFFSET + (x)))
53
54#define AT_DMA_CHER 0x28 /* Channel Handler Enable Register */
55#define AT_DMA_ENA(x) (0x1 << (x))
56#define AT_DMA_SUSP(x) (0x1 << ( 8 + (x)))
57#define AT_DMA_KEEP(x) (0x1 << (24 + (x)))
58
59#define AT_DMA_CHDR 0x2C /* Channel Handler Disable Register */
60#define AT_DMA_DIS(x) (0x1 << (x))
61#define AT_DMA_RES(x) (0x1 << ( 8 + (x)))
62
63#define AT_DMA_CHSR 0x30 /* Channel Handler Status Register */
64#define AT_DMA_EMPT(x) (0x1 << (16 + (x)))
65#define AT_DMA_STAL(x) (0x1 << (24 + (x)))
66
67
68#define AT_DMA_CH_REGS_BASE 0x3C /* Channel registers base address */
69#define ch_regs(x) (AT_DMA_CH_REGS_BASE + (x) * 0x28) /* Channel x base addr */
70
71/* Hardware register offset for each channel */
72#define ATC_SADDR_OFFSET 0x00 /* Source Address Register */
73#define ATC_DADDR_OFFSET 0x04 /* Destination Address Register */
74#define ATC_DSCR_OFFSET 0x08 /* Descriptor Address Register */
75#define ATC_CTRLA_OFFSET 0x0C /* Control A Register */
76#define ATC_CTRLB_OFFSET 0x10 /* Control B Register */
77#define ATC_CFG_OFFSET 0x14 /* Configuration Register */
78#define ATC_SPIP_OFFSET 0x18 /* Src PIP Configuration Register */
79#define ATC_DPIP_OFFSET 0x1C /* Dst PIP Configuration Register */
80
81
82/* Bitfield definitions */
83
84/* Bitfields in DSCR */
85#define ATC_DSCR_IF(i) (0x3 & (i)) /* Dsc feched via AHB-Lite Interface i */
86
87/* Bitfields in CTRLA */
88#define ATC_BTSIZE_MAX 0xFFFFUL /* Maximum Buffer Transfer Size */
89#define ATC_BTSIZE(x) (ATC_BTSIZE_MAX & (x)) /* Buffer Transfer Size */
90/* Chunck Tranfer size definitions are in at_hdmac.h */
91#define ATC_SRC_WIDTH_MASK (0x3 << 24) /* Source Single Transfer Size */
92#define ATC_SRC_WIDTH(x) ((x) << 24)
93#define ATC_SRC_WIDTH_BYTE (0x0 << 24)
94#define ATC_SRC_WIDTH_HALFWORD (0x1 << 24)
95#define ATC_SRC_WIDTH_WORD (0x2 << 24)
96#define ATC_DST_WIDTH_MASK (0x3 << 28) /* Destination Single Transfer Size */
97#define ATC_DST_WIDTH(x) ((x) << 28)
98#define ATC_DST_WIDTH_BYTE (0x0 << 28)
99#define ATC_DST_WIDTH_HALFWORD (0x1 << 28)
100#define ATC_DST_WIDTH_WORD (0x2 << 28)
101#define ATC_DONE (0x1 << 31) /* Tx Done (only written back in descriptor) */
102
103/* Bitfields in CTRLB */
104#define ATC_SIF(i) (0x3 & (i)) /* Src tx done via AHB-Lite Interface i */
105#define ATC_DIF(i) ((0x3 & (i)) << 4) /* Dst tx done via AHB-Lite Interface i */
106#define ATC_SRC_PIP (0x1 << 8) /* Source Picture-in-Picture enabled */
107#define ATC_DST_PIP (0x1 << 12) /* Destination Picture-in-Picture enabled */
108#define ATC_SRC_DSCR_DIS (0x1 << 16) /* Src Descriptor fetch disable */
109#define ATC_DST_DSCR_DIS (0x1 << 20) /* Dst Descriptor fetch disable */
110#define ATC_FC_MASK (0x7 << 21) /* Choose Flow Controller */
111#define ATC_FC_MEM2MEM (0x0 << 21) /* Mem-to-Mem (DMA) */
112#define ATC_FC_MEM2PER (0x1 << 21) /* Mem-to-Periph (DMA) */
113#define ATC_FC_PER2MEM (0x2 << 21) /* Periph-to-Mem (DMA) */
114#define ATC_FC_PER2PER (0x3 << 21) /* Periph-to-Periph (DMA) */
115#define ATC_FC_PER2MEM_PER (0x4 << 21) /* Periph-to-Mem (Peripheral) */
116#define ATC_FC_MEM2PER_PER (0x5 << 21) /* Mem-to-Periph (Peripheral) */
117#define ATC_FC_PER2PER_SRCPER (0x6 << 21) /* Periph-to-Periph (Src Peripheral) */
118#define ATC_FC_PER2PER_DSTPER (0x7 << 21) /* Periph-to-Periph (Dst Peripheral) */
119#define ATC_SRC_ADDR_MODE_MASK (0x3 << 24)
120#define ATC_SRC_ADDR_MODE_INCR (0x0 << 24) /* Incrementing Mode */
121#define ATC_SRC_ADDR_MODE_DECR (0x1 << 24) /* Decrementing Mode */
122#define ATC_SRC_ADDR_MODE_FIXED (0x2 << 24) /* Fixed Mode */
123#define ATC_DST_ADDR_MODE_MASK (0x3 << 28)
124#define ATC_DST_ADDR_MODE_INCR (0x0 << 28) /* Incrementing Mode */
125#define ATC_DST_ADDR_MODE_DECR (0x1 << 28) /* Decrementing Mode */
126#define ATC_DST_ADDR_MODE_FIXED (0x2 << 28) /* Fixed Mode */
127#define ATC_IEN (0x1 << 30) /* BTC interrupt enable (active low) */
128#define ATC_AUTO (0x1 << 31) /* Auto multiple buffer tx enable */
129
130/* Bitfields in CFG */
131/* are in at_hdmac.h */
132
133/* Bitfields in SPIP */
134#define ATC_SPIP_HOLE(x) (0xFFFFU & (x))
135#define ATC_SPIP_BOUNDARY(x) ((0x3FF & (x)) << 16)
136
137/* Bitfields in DPIP */
138#define ATC_DPIP_HOLE(x) (0xFFFFU & (x))
139#define ATC_DPIP_BOUNDARY(x) ((0x3FF & (x)) << 16)
140
141
142/*-- descriptors -----------------------------------------------------*/
143
144/* LLI == Linked List Item; aka DMA buffer descriptor */
145struct at_lli {
146 /* values that are not changed by hardware */
147 dma_addr_t saddr;
148 dma_addr_t daddr;
149 /* value that may get written back: */
150 u32 ctrla;
151 /* more values that are not changed by hardware */
152 u32 ctrlb;
153 dma_addr_t dscr; /* chain to next lli */
154};
155
156/**
157 * struct at_desc - software descriptor
158 * @at_lli: hardware lli structure
159 * @txd: support for the async_tx api
160 * @desc_node: node on the channed descriptors list
161 * @len: total transaction bytecount
162 */
163struct at_desc {
164 /* FIRST values the hardware uses */
165 struct at_lli lli;
166
167 /* THEN values for driver housekeeping */
168 struct dma_async_tx_descriptor txd;
169 struct list_head desc_node;
170 size_t len;
171};
172
173static inline struct at_desc *
174txd_to_at_desc(struct dma_async_tx_descriptor *txd)
175{
176 return container_of(txd, struct at_desc, txd);
177}
178
179
180/*-- Channels --------------------------------------------------------*/
181
182/**
183 * struct at_dma_chan - internal representation of an Atmel HDMAC channel
184 * @chan_common: common dmaengine channel object members
185 * @device: parent device
186 * @ch_regs: memory mapped register base
187 * @mask: channel index in a mask
188 * @error_status: transmit error status information from irq handler
189 * to tasklet (use atomic operations)
190 * @tasklet: bottom half to finish transaction work
191 * @lock: serializes enqueue/dequeue operations to descriptors lists
192 * @completed_cookie: identifier for the most recently completed operation
193 * @active_list: list of descriptors dmaengine is being running on
194 * @queue: list of descriptors ready to be submitted to engine
195 * @free_list: list of descriptors usable by the channel
196 * @descs_allocated: records the actual size of the descriptor pool
197 */
198struct at_dma_chan {
199 struct dma_chan chan_common;
200 struct at_dma *device;
201 void __iomem *ch_regs;
202 u8 mask;
203 unsigned long error_status;
204 struct tasklet_struct tasklet;
205
206 spinlock_t lock;
207
208 /* these other elements are all protected by lock */
209 dma_cookie_t completed_cookie;
210 struct list_head active_list;
211 struct list_head queue;
212 struct list_head free_list;
213 unsigned int descs_allocated;
214};
215
216#define channel_readl(atchan, name) \
217 __raw_readl((atchan)->ch_regs + ATC_##name##_OFFSET)
218
219#define channel_writel(atchan, name, val) \
220 __raw_writel((val), (atchan)->ch_regs + ATC_##name##_OFFSET)
221
222static inline struct at_dma_chan *to_at_dma_chan(struct dma_chan *dchan)
223{
224 return container_of(dchan, struct at_dma_chan, chan_common);
225}
226
227
228/*-- Controller ------------------------------------------------------*/
229
230/**
231 * struct at_dma - internal representation of an Atmel HDMA Controller
232 * @chan_common: common dmaengine dma_device object members
233 * @ch_regs: memory mapped register base
234 * @clk: dma controller clock
235 * @all_chan_mask: all channels availlable in a mask
236 * @dma_desc_pool: base of DMA descriptor region (DMA address)
237 * @chan: channels table to store at_dma_chan structures
238 */
239struct at_dma {
240 struct dma_device dma_common;
241 void __iomem *regs;
242 struct clk *clk;
243
244 u8 all_chan_mask;
245
246 struct dma_pool *dma_desc_pool;
247 /* AT THE END channels table */
248 struct at_dma_chan chan[0];
249};
250
251#define dma_readl(atdma, name) \
252 __raw_readl((atdma)->regs + AT_DMA_##name)
253#define dma_writel(atdma, name, val) \
254 __raw_writel((val), (atdma)->regs + AT_DMA_##name)
255
256static inline struct at_dma *to_at_dma(struct dma_device *ddev)
257{
258 return container_of(ddev, struct at_dma, dma_common);
259}
260
261
262/*-- Helper functions ------------------------------------------------*/
263
264static struct device *chan2dev(struct dma_chan *chan)
265{
266 return &chan->dev->device;
267}
268static struct device *chan2parent(struct dma_chan *chan)
269{
270 return chan->dev->device.parent;
271}
272
273#if defined(VERBOSE_DEBUG)
274static void vdbg_dump_regs(struct at_dma_chan *atchan)
275{
276 struct at_dma *atdma = to_at_dma(atchan->chan_common.device);
277
278 dev_err(chan2dev(&atchan->chan_common),
279 " channel %d : imr = 0x%x, chsr = 0x%x\n",
280 atchan->chan_common.chan_id,
281 dma_readl(atdma, EBCIMR),
282 dma_readl(atdma, CHSR));
283
284 dev_err(chan2dev(&atchan->chan_common),
285 " channel: s0x%x d0x%x ctrl0x%x:0x%x cfg0x%x l0x%x\n",
286 channel_readl(atchan, SADDR),
287 channel_readl(atchan, DADDR),
288 channel_readl(atchan, CTRLA),
289 channel_readl(atchan, CTRLB),
290 channel_readl(atchan, CFG),
291 channel_readl(atchan, DSCR));
292}
293#else
294static void vdbg_dump_regs(struct at_dma_chan *atchan) {}
295#endif
296
297static void atc_dump_lli(struct at_dma_chan *atchan, struct at_lli *lli)
298{
299 dev_printk(KERN_CRIT, chan2dev(&atchan->chan_common),
300 " desc: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n",
301 lli->saddr, lli->daddr,
302 lli->ctrla, lli->ctrlb, lli->dscr);
303}
304
305
306static void atc_setup_irq(struct at_dma_chan *atchan, int on)
307{
308 struct at_dma *atdma = to_at_dma(atchan->chan_common.device);
309 u32 ebci;
310
311 /* enable interrupts on buffer chain completion & error */
312 ebci = AT_DMA_CBTC(atchan->chan_common.chan_id)
313 | AT_DMA_ERR(atchan->chan_common.chan_id);
314 if (on)
315 dma_writel(atdma, EBCIER, ebci);
316 else
317 dma_writel(atdma, EBCIDR, ebci);
318}
319
320static inline void atc_enable_irq(struct at_dma_chan *atchan)
321{
322 atc_setup_irq(atchan, 1);
323}
324
325static inline void atc_disable_irq(struct at_dma_chan *atchan)
326{
327 atc_setup_irq(atchan, 0);
328}
329
330
331/**
332 * atc_chan_is_enabled - test if given channel is enabled
333 * @atchan: channel we want to test status
334 */
335static inline int atc_chan_is_enabled(struct at_dma_chan *atchan)
336{
337 struct at_dma *atdma = to_at_dma(atchan->chan_common.device);
338
339 return !!(dma_readl(atdma, CHSR) & atchan->mask);
340}
341
342
343/**
344 * set_desc_eol - set end-of-link to descriptor so it will end transfer
345 * @desc: descriptor, signle or at the end of a chain, to end chain on
346 */
347static void set_desc_eol(struct at_desc *desc)
348{
349 desc->lli.ctrlb |= ATC_SRC_DSCR_DIS | ATC_DST_DSCR_DIS;
350 desc->lli.dscr = 0;
351}
352
353#endif /* AT_HDMAC_REGS_H */
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index fb7da5141e96..d93017fc7872 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -38,6 +38,11 @@ module_param(max_channels, uint, S_IRUGO);
38MODULE_PARM_DESC(max_channels, 38MODULE_PARM_DESC(max_channels,
39 "Maximum number of channels to use (default: all)"); 39 "Maximum number of channels to use (default: all)");
40 40
41static unsigned int iterations;
42module_param(iterations, uint, S_IRUGO);
43MODULE_PARM_DESC(iterations,
44 "Iterations before stopping test (default: infinite)");
45
41static unsigned int xor_sources = 3; 46static unsigned int xor_sources = 3;
42module_param(xor_sources, uint, S_IRUGO); 47module_param(xor_sources, uint, S_IRUGO);
43MODULE_PARM_DESC(xor_sources, 48MODULE_PARM_DESC(xor_sources,
@@ -114,7 +119,7 @@ static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len)
114 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 119 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
115 for ( ; i < start + len; i++) 120 for ( ; i < start + len; i++)
116 buf[i] = PATTERN_SRC | PATTERN_COPY 121 buf[i] = PATTERN_SRC | PATTERN_COPY
117 | (~i & PATTERN_COUNT_MASK);; 122 | (~i & PATTERN_COUNT_MASK);
118 for ( ; i < test_buf_size; i++) 123 for ( ; i < test_buf_size; i++)
119 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 124 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
120 buf++; 125 buf++;
@@ -270,7 +275,8 @@ static int dmatest_func(void *data)
270 275
271 flags = DMA_CTRL_ACK | DMA_COMPL_SKIP_DEST_UNMAP | DMA_PREP_INTERRUPT; 276 flags = DMA_CTRL_ACK | DMA_COMPL_SKIP_DEST_UNMAP | DMA_PREP_INTERRUPT;
272 277
273 while (!kthread_should_stop()) { 278 while (!kthread_should_stop()
279 && !(iterations && total_tests >= iterations)) {
274 struct dma_device *dev = chan->device; 280 struct dma_device *dev = chan->device;
275 struct dma_async_tx_descriptor *tx = NULL; 281 struct dma_async_tx_descriptor *tx = NULL;
276 dma_addr_t dma_srcs[src_cnt]; 282 dma_addr_t dma_srcs[src_cnt];
@@ -416,6 +422,13 @@ err_srcbuf:
416err_srcs: 422err_srcs:
417 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", 423 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
418 thread_name, total_tests, failed_tests, ret); 424 thread_name, total_tests, failed_tests, ret);
425
426 if (iterations > 0)
427 while (!kthread_should_stop()) {
428 DECLARE_WAIT_QUEUE_HEAD(wait_dmatest_exit);
429 interruptible_sleep_on(&wait_dmatest_exit);
430 }
431
419 return ret; 432 return ret;
420} 433}
421 434
@@ -495,11 +508,11 @@ static int dmatest_add_channel(struct dma_chan *chan)
495 508
496 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 509 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
497 cnt = dmatest_add_threads(dtc, DMA_MEMCPY); 510 cnt = dmatest_add_threads(dtc, DMA_MEMCPY);
498 thread_count += cnt > 0 ?: 0; 511 thread_count += cnt > 0 ? cnt : 0;
499 } 512 }
500 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 513 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
501 cnt = dmatest_add_threads(dtc, DMA_XOR); 514 cnt = dmatest_add_threads(dtc, DMA_XOR);
502 thread_count += cnt > 0 ?: 0; 515 thread_count += cnt > 0 ? cnt : 0;
503 } 516 }
504 517
505 pr_info("dmatest: Started %u threads using %s\n", 518 pr_info("dmatest: Started %u threads using %s\n",
diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c
index f18d1bde0439..ef87a8984145 100644
--- a/drivers/dma/fsldma.c
+++ b/drivers/dma/fsldma.c
@@ -12,6 +12,11 @@
12 * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc. 12 * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
13 * The support for MPC8349 DMA contorller is also added. 13 * The support for MPC8349 DMA contorller is also added.
14 * 14 *
15 * This driver instructs the DMA controller to issue the PCI Read Multiple
16 * command for PCI read operations, instead of using the default PCI Read Line
17 * command. Please be aware that this setting may result in read pre-fetching
18 * on some platforms.
19 *
15 * This is free software; you can redistribute it and/or modify 20 * This is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by 21 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or 22 * the Free Software Foundation; either version 2 of the License, or
@@ -49,9 +54,10 @@ static void dma_init(struct fsl_dma_chan *fsl_chan)
49 case FSL_DMA_IP_83XX: 54 case FSL_DMA_IP_83XX:
50 /* Set the channel to below modes: 55 /* Set the channel to below modes:
51 * EOTIE - End-of-transfer interrupt enable 56 * EOTIE - End-of-transfer interrupt enable
57 * PRC_RM - PCI read multiple
52 */ 58 */
53 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EOTIE, 59 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EOTIE
54 32); 60 | FSL_DMA_MR_PRC_RM, 32);
55 break; 61 break;
56 } 62 }
57 63
@@ -136,15 +142,16 @@ static int dma_is_idle(struct fsl_dma_chan *fsl_chan)
136 142
137static void dma_start(struct fsl_dma_chan *fsl_chan) 143static void dma_start(struct fsl_dma_chan *fsl_chan)
138{ 144{
139 u32 mr_set = 0;; 145 u32 mr_set = 0;
140 146
141 if (fsl_chan->feature & FSL_DMA_CHAN_PAUSE_EXT) { 147 if (fsl_chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {
142 DMA_OUT(fsl_chan, &fsl_chan->reg_base->bcr, 0, 32); 148 DMA_OUT(fsl_chan, &fsl_chan->reg_base->bcr, 0, 32);
143 mr_set |= FSL_DMA_MR_EMP_EN; 149 mr_set |= FSL_DMA_MR_EMP_EN;
144 } else 150 } else if ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
145 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 151 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
146 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) 152 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
147 & ~FSL_DMA_MR_EMP_EN, 32); 153 & ~FSL_DMA_MR_EMP_EN, 32);
154 }
148 155
149 if (fsl_chan->feature & FSL_DMA_CHAN_START_EXT) 156 if (fsl_chan->feature & FSL_DMA_CHAN_START_EXT)
150 mr_set |= FSL_DMA_MR_EMS_EN; 157 mr_set |= FSL_DMA_MR_EMS_EN;
@@ -871,9 +878,9 @@ static int __devinit fsl_dma_chan_probe(struct fsl_dma_device *fdev,
871 878
872 switch (new_fsl_chan->feature & FSL_DMA_IP_MASK) { 879 switch (new_fsl_chan->feature & FSL_DMA_IP_MASK) {
873 case FSL_DMA_IP_85XX: 880 case FSL_DMA_IP_85XX:
874 new_fsl_chan->toggle_ext_start = fsl_chan_toggle_ext_start;
875 new_fsl_chan->toggle_ext_pause = fsl_chan_toggle_ext_pause; 881 new_fsl_chan->toggle_ext_pause = fsl_chan_toggle_ext_pause;
876 case FSL_DMA_IP_83XX: 882 case FSL_DMA_IP_83XX:
883 new_fsl_chan->toggle_ext_start = fsl_chan_toggle_ext_start;
877 new_fsl_chan->set_src_loop_size = fsl_chan_set_src_loop_size; 884 new_fsl_chan->set_src_loop_size = fsl_chan_set_src_loop_size;
878 new_fsl_chan->set_dest_loop_size = fsl_chan_set_dest_loop_size; 885 new_fsl_chan->set_dest_loop_size = fsl_chan_set_dest_loop_size;
879 } 886 }
diff --git a/drivers/dma/fsldma.h b/drivers/dma/fsldma.h
index 4f21a512d848..dc7f26865797 100644
--- a/drivers/dma/fsldma.h
+++ b/drivers/dma/fsldma.h
@@ -38,6 +38,7 @@
38 38
39/* Special MR definition for MPC8349 */ 39/* Special MR definition for MPC8349 */
40#define FSL_DMA_MR_EOTIE 0x00000080 40#define FSL_DMA_MR_EOTIE 0x00000080
41#define FSL_DMA_MR_PRC_RM 0x00000800
41 42
42#define FSL_DMA_SR_CH 0x00000020 43#define FSL_DMA_SR_CH 0x00000020
43#define FSL_DMA_SR_PE 0x00000010 44#define FSL_DMA_SR_PE 0x00000010
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index ddab94f51224..3f23eabe09f2 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -1176,7 +1176,7 @@ static int __devinit mv_xor_probe(struct platform_device *pdev)
1176 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) 1176 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask))
1177 dma_dev->device_prep_dma_memset = mv_xor_prep_dma_memset; 1177 dma_dev->device_prep_dma_memset = mv_xor_prep_dma_memset;
1178 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1178 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1179 dma_dev->max_xor = 8; ; 1179 dma_dev->max_xor = 8;
1180 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor; 1180 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
1181 } 1181 }
1182 1182
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-16 06:02:51 -0500