aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm/plat-nomadik/include/plat/ste_dma40.h8
-rw-r--r--drivers/dma/amba-pl08x.c1040
-rw-r--r--drivers/dma/ste_dma40.c191
-rw-r--r--drivers/dma/ste_dma40_ll.c246
-rw-r--r--drivers/dma/ste_dma40_ll.h36
-rw-r--r--include/linux/amba/pl08x.h56
6 files changed, 871 insertions, 706 deletions
diff --git a/arch/arm/plat-nomadik/include/plat/ste_dma40.h b/arch/arm/plat-nomadik/include/plat/ste_dma40.h
index 74b62f10d07f..4d6dd4c39b75 100644
--- a/arch/arm/plat-nomadik/include/plat/ste_dma40.h
+++ b/arch/arm/plat-nomadik/include/plat/ste_dma40.h
@@ -13,6 +13,14 @@
13#include <linux/workqueue.h> 13#include <linux/workqueue.h>
14#include <linux/interrupt.h> 14#include <linux/interrupt.h>
15 15
16/*
17 * Maxium size for a single dma descriptor
18 * Size is limited to 16 bits.
19 * Size is in the units of addr-widths (1,2,4,8 bytes)
20 * Larger transfers will be split up to multiple linked desc
21 */
22#define STEDMA40_MAX_SEG_SIZE 0xFFFF
23
16/* dev types for memcpy */ 24/* dev types for memcpy */
17#define STEDMA40_DEV_DST_MEMORY (-1) 25#define STEDMA40_DEV_DST_MEMORY (-1)
18#define STEDMA40_DEV_SRC_MEMORY (-1) 26#define STEDMA40_DEV_SRC_MEMORY (-1)
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index b605cc9ac3a2..bebc678ed4fc 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -19,7 +19,7 @@
19 * this program; if not, write to the Free Software Foundation, Inc., 59 19 * this program; if not, write to the Free Software Foundation, Inc., 59
20 * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 20 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
21 * 21 *
22 * The full GNU General Public License is iin this distribution in the 22 * The full GNU General Public License is in this distribution in the
23 * file called COPYING. 23 * file called COPYING.
24 * 24 *
25 * Documentation: ARM DDI 0196G == PL080 25 * Documentation: ARM DDI 0196G == PL080
@@ -53,7 +53,23 @@
53 * 53 *
54 * ASSUMES default (little) endianness for DMA transfers 54 * ASSUMES default (little) endianness for DMA transfers
55 * 55 *
56 * Only DMAC flow control is implemented 56 * The PL08x has two flow control settings:
57 * - DMAC flow control: the transfer size defines the number of transfers
58 * which occur for the current LLI entry, and the DMAC raises TC at the
59 * end of every LLI entry. Observed behaviour shows the DMAC listening
60 * to both the BREQ and SREQ signals (contrary to documented),
61 * transferring data if either is active. The LBREQ and LSREQ signals
62 * are ignored.
63 *
64 * - Peripheral flow control: the transfer size is ignored (and should be
65 * zero). The data is transferred from the current LLI entry, until
66 * after the final transfer signalled by LBREQ or LSREQ. The DMAC
67 * will then move to the next LLI entry.
68 *
69 * Only the former works sanely with scatter lists, so we only implement
70 * the DMAC flow control method. However, peripherals which use the LBREQ
71 * and LSREQ signals (eg, MMCI) are unable to use this mode, which through
72 * these hardware restrictions prevents them from using scatter DMA.
57 * 73 *
58 * Global TODO: 74 * Global TODO:
59 * - Break out common code from arch/arm/mach-s3c64xx and share 75 * - Break out common code from arch/arm/mach-s3c64xx and share
@@ -61,50 +77,41 @@
61#include <linux/device.h> 77#include <linux/device.h>
62#include <linux/init.h> 78#include <linux/init.h>
63#include <linux/module.h> 79#include <linux/module.h>
64#include <linux/pci.h>
65#include <linux/interrupt.h> 80#include <linux/interrupt.h>
66#include <linux/slab.h> 81#include <linux/slab.h>
67#include <linux/dmapool.h> 82#include <linux/dmapool.h>
68#include <linux/amba/bus.h>
69#include <linux/dmaengine.h> 83#include <linux/dmaengine.h>
84#include <linux/amba/bus.h>
70#include <linux/amba/pl08x.h> 85#include <linux/amba/pl08x.h>
71#include <linux/debugfs.h> 86#include <linux/debugfs.h>
72#include <linux/seq_file.h> 87#include <linux/seq_file.h>
73 88
74#include <asm/hardware/pl080.h> 89#include <asm/hardware/pl080.h>
75#include <asm/dma.h>
76#include <asm/mach/dma.h>
77#include <asm/atomic.h>
78#include <asm/processor.h>
79#include <asm/cacheflush.h>
80 90
81#define DRIVER_NAME "pl08xdmac" 91#define DRIVER_NAME "pl08xdmac"
82 92
83/** 93/**
84 * struct vendor_data - vendor-specific config parameters 94 * struct vendor_data - vendor-specific config parameters
85 * for PL08x derivates 95 * for PL08x derivatives
86 * @name: the name of this specific variant
87 * @channels: the number of channels available in this variant 96 * @channels: the number of channels available in this variant
88 * @dualmaster: whether this version supports dual AHB masters 97 * @dualmaster: whether this version supports dual AHB masters
89 * or not. 98 * or not.
90 */ 99 */
91struct vendor_data { 100struct vendor_data {
92 char *name;
93 u8 channels; 101 u8 channels;
94 bool dualmaster; 102 bool dualmaster;
95}; 103};
96 104
97/* 105/*
98 * PL08X private data structures 106 * PL08X private data structures
99 * An LLI struct - see pl08x TRM 107 * An LLI struct - see PL08x TRM. Note that next uses bit[0] as a bus bit,
100 * Note that next uses bit[0] as a bus bit, 108 * start & end do not - their bus bit info is in cctl. Also note that these
101 * start & end do not - their bus bit info 109 * are fixed 32-bit quantities.
102 * is in cctl
103 */ 110 */
104struct lli { 111struct pl08x_lli {
105 dma_addr_t src; 112 u32 src;
106 dma_addr_t dst; 113 u32 dst;
107 dma_addr_t next; 114 u32 lli;
108 u32 cctl; 115 u32 cctl;
109}; 116};
110 117
@@ -119,6 +126,8 @@ struct lli {
119 * @phy_chans: array of data for the physical channels 126 * @phy_chans: array of data for the physical channels
120 * @pool: a pool for the LLI descriptors 127 * @pool: a pool for the LLI descriptors
121 * @pool_ctr: counter of LLIs in the pool 128 * @pool_ctr: counter of LLIs in the pool
129 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI fetches
130 * @mem_buses: set to indicate memory transfers on AHB2.
122 * @lock: a spinlock for this struct 131 * @lock: a spinlock for this struct
123 */ 132 */
124struct pl08x_driver_data { 133struct pl08x_driver_data {
@@ -126,11 +135,13 @@ struct pl08x_driver_data {
126 struct dma_device memcpy; 135 struct dma_device memcpy;
127 void __iomem *base; 136 void __iomem *base;
128 struct amba_device *adev; 137 struct amba_device *adev;
129 struct vendor_data *vd; 138 const struct vendor_data *vd;
130 struct pl08x_platform_data *pd; 139 struct pl08x_platform_data *pd;
131 struct pl08x_phy_chan *phy_chans; 140 struct pl08x_phy_chan *phy_chans;
132 struct dma_pool *pool; 141 struct dma_pool *pool;
133 int pool_ctr; 142 int pool_ctr;
143 u8 lli_buses;
144 u8 mem_buses;
134 spinlock_t lock; 145 spinlock_t lock;
135}; 146};
136 147
@@ -152,9 +163,9 @@ struct pl08x_driver_data {
152/* Size (bytes) of each LLI buffer allocated for one transfer */ 163/* Size (bytes) of each LLI buffer allocated for one transfer */
153# define PL08X_LLI_TSFR_SIZE 0x2000 164# define PL08X_LLI_TSFR_SIZE 0x2000
154 165
155/* Maximimum times we call dma_pool_alloc on this pool without freeing */ 166/* Maximum times we call dma_pool_alloc on this pool without freeing */
156#define PL08X_MAX_ALLOCS 0x40 167#define PL08X_MAX_ALLOCS 0x40
157#define MAX_NUM_TSFR_LLIS (PL08X_LLI_TSFR_SIZE/sizeof(struct lli)) 168#define MAX_NUM_TSFR_LLIS (PL08X_LLI_TSFR_SIZE/sizeof(struct pl08x_lli))
158#define PL08X_ALIGN 8 169#define PL08X_ALIGN 8
159 170
160static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan) 171static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
@@ -162,6 +173,11 @@ static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
162 return container_of(chan, struct pl08x_dma_chan, chan); 173 return container_of(chan, struct pl08x_dma_chan, chan);
163} 174}
164 175
176static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
177{
178 return container_of(tx, struct pl08x_txd, tx);
179}
180
165/* 181/*
166 * Physical channel handling 182 * Physical channel handling
167 */ 183 */
@@ -177,88 +193,47 @@ static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
177 193
178/* 194/*
179 * Set the initial DMA register values i.e. those for the first LLI 195 * Set the initial DMA register values i.e. those for the first LLI
180 * The next lli pointer and the configuration interrupt bit have 196 * The next LLI pointer and the configuration interrupt bit have
181 * been set when the LLIs were constructed 197 * been set when the LLIs were constructed. Poke them into the hardware
198 * and start the transfer.
182 */ 199 */
183static void pl08x_set_cregs(struct pl08x_driver_data *pl08x, 200static void pl08x_start_txd(struct pl08x_dma_chan *plchan,
184 struct pl08x_phy_chan *ch) 201 struct pl08x_txd *txd)
185{
186 /* Wait for channel inactive */
187 while (pl08x_phy_channel_busy(ch))
188 ;
189
190 dev_vdbg(&pl08x->adev->dev,
191 "WRITE channel %d: csrc=%08x, cdst=%08x, "
192 "cctl=%08x, clli=%08x, ccfg=%08x\n",
193 ch->id,
194 ch->csrc,
195 ch->cdst,
196 ch->cctl,
197 ch->clli,
198 ch->ccfg);
199
200 writel(ch->csrc, ch->base + PL080_CH_SRC_ADDR);
201 writel(ch->cdst, ch->base + PL080_CH_DST_ADDR);
202 writel(ch->clli, ch->base + PL080_CH_LLI);
203 writel(ch->cctl, ch->base + PL080_CH_CONTROL);
204 writel(ch->ccfg, ch->base + PL080_CH_CONFIG);
205}
206
207static inline void pl08x_config_phychan_for_txd(struct pl08x_dma_chan *plchan)
208{ 202{
209 struct pl08x_channel_data *cd = plchan->cd; 203 struct pl08x_driver_data *pl08x = plchan->host;
210 struct pl08x_phy_chan *phychan = plchan->phychan; 204 struct pl08x_phy_chan *phychan = plchan->phychan;
211 struct pl08x_txd *txd = plchan->at; 205 struct pl08x_lli *lli = &txd->llis_va[0];
212
213 /* Copy the basic control register calculated at transfer config */
214 phychan->csrc = txd->csrc;
215 phychan->cdst = txd->cdst;
216 phychan->clli = txd->clli;
217 phychan->cctl = txd->cctl;
218
219 /* Assign the signal to the proper control registers */
220 phychan->ccfg = cd->ccfg;
221 phychan->ccfg &= ~PL080_CONFIG_SRC_SEL_MASK;
222 phychan->ccfg &= ~PL080_CONFIG_DST_SEL_MASK;
223 /* If it wasn't set from AMBA, ignore it */
224 if (txd->direction == DMA_TO_DEVICE)
225 /* Select signal as destination */
226 phychan->ccfg |=
227 (phychan->signal << PL080_CONFIG_DST_SEL_SHIFT);
228 else if (txd->direction == DMA_FROM_DEVICE)
229 /* Select signal as source */
230 phychan->ccfg |=
231 (phychan->signal << PL080_CONFIG_SRC_SEL_SHIFT);
232 /* Always enable error interrupts */
233 phychan->ccfg |= PL080_CONFIG_ERR_IRQ_MASK;
234 /* Always enable terminal interrupts */
235 phychan->ccfg |= PL080_CONFIG_TC_IRQ_MASK;
236}
237
238/*
239 * Enable the DMA channel
240 * Assumes all other configuration bits have been set
241 * as desired before this code is called
242 */
243static void pl08x_enable_phy_chan(struct pl08x_driver_data *pl08x,
244 struct pl08x_phy_chan *ch)
245{
246 u32 val; 206 u32 val;
247 207
248 /* 208 plchan->at = txd;
249 * Do not access config register until channel shows as disabled
250 */
251 while (readl(pl08x->base + PL080_EN_CHAN) & (1 << ch->id))
252 ;
253 209
254 /* 210 /* Wait for channel inactive */
255 * Do not access config register until channel shows as inactive 211 while (pl08x_phy_channel_busy(phychan))
256 */ 212 cpu_relax();
257 val = readl(ch->base + PL080_CH_CONFIG); 213
214 dev_vdbg(&pl08x->adev->dev,
215 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
216 "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
217 phychan->id, lli->src, lli->dst, lli->lli, lli->cctl,
218 txd->ccfg);
219
220 writel(lli->src, phychan->base + PL080_CH_SRC_ADDR);
221 writel(lli->dst, phychan->base + PL080_CH_DST_ADDR);
222 writel(lli->lli, phychan->base + PL080_CH_LLI);
223 writel(lli->cctl, phychan->base + PL080_CH_CONTROL);
224 writel(txd->ccfg, phychan->base + PL080_CH_CONFIG);
225
226 /* Enable the DMA channel */
227 /* Do not access config register until channel shows as disabled */
228 while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id))
229 cpu_relax();
230
231 /* Do not access config register until channel shows as inactive */
232 val = readl(phychan->base + PL080_CH_CONFIG);
258 while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE)) 233 while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE))
259 val = readl(ch->base + PL080_CH_CONFIG); 234 val = readl(phychan->base + PL080_CH_CONFIG);
260 235
261 writel(val | PL080_CONFIG_ENABLE, ch->base + PL080_CH_CONFIG); 236 writel(val | PL080_CONFIG_ENABLE, phychan->base + PL080_CH_CONFIG);
262} 237}
263 238
264/* 239/*
@@ -282,7 +257,7 @@ static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
282 257
283 /* Wait for channel inactive */ 258 /* Wait for channel inactive */
284 while (pl08x_phy_channel_busy(ch)) 259 while (pl08x_phy_channel_busy(ch))
285 ; 260 cpu_relax();
286} 261}
287 262
288static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch) 263static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
@@ -333,54 +308,56 @@ static inline u32 get_bytes_in_cctl(u32 cctl)
333static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan) 308static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
334{ 309{
335 struct pl08x_phy_chan *ch; 310 struct pl08x_phy_chan *ch;
336 struct pl08x_txd *txdi = NULL;
337 struct pl08x_txd *txd; 311 struct pl08x_txd *txd;
338 unsigned long flags; 312 unsigned long flags;
339 u32 bytes = 0; 313 size_t bytes = 0;
340 314
341 spin_lock_irqsave(&plchan->lock, flags); 315 spin_lock_irqsave(&plchan->lock, flags);
342
343 ch = plchan->phychan; 316 ch = plchan->phychan;
344 txd = plchan->at; 317 txd = plchan->at;
345 318
346 /* 319 /*
347 * Next follow the LLIs to get the number of pending bytes in the 320 * Follow the LLIs to get the number of remaining
348 * currently active transaction. 321 * bytes in the currently active transaction.
349 */ 322 */
350 if (ch && txd) { 323 if (ch && txd) {
351 struct lli *llis_va = txd->llis_va; 324 u32 clli = readl(ch->base + PL080_CH_LLI) & ~PL080_LLI_LM_AHB2;
352 struct lli *llis_bus = (struct lli *) txd->llis_bus;
353 u32 clli = readl(ch->base + PL080_CH_LLI);
354 325
355 /* First get the bytes in the current active LLI */ 326 /* First get the remaining bytes in the active transfer */
356 bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL)); 327 bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL));
357 328
358 if (clli) { 329 if (clli) {
359 int i = 0; 330 struct pl08x_lli *llis_va = txd->llis_va;
331 dma_addr_t llis_bus = txd->llis_bus;
332 int index;
333
334 BUG_ON(clli < llis_bus || clli >= llis_bus +
335 sizeof(struct pl08x_lli) * MAX_NUM_TSFR_LLIS);
336
337 /*
338 * Locate the next LLI - as this is an array,
339 * it's simple maths to find.
340 */
341 index = (clli - llis_bus) / sizeof(struct pl08x_lli);
360 342
361 /* Forward to the LLI pointed to by clli */ 343 for (; index < MAX_NUM_TSFR_LLIS; index++) {
362 while ((clli != (u32) &(llis_bus[i])) && 344 bytes += get_bytes_in_cctl(llis_va[index].cctl);
363 (i < MAX_NUM_TSFR_LLIS))
364 i++;
365 345
366 while (clli) {
367 bytes += get_bytes_in_cctl(llis_va[i].cctl);
368 /* 346 /*
369 * A clli of 0x00000000 will terminate the 347 * A LLI pointer of 0 terminates the LLI list
370 * LLI list
371 */ 348 */
372 clli = llis_va[i].next; 349 if (!llis_va[index].lli)
373 i++; 350 break;
374 } 351 }
375 } 352 }
376 } 353 }
377 354
378 /* Sum up all queued transactions */ 355 /* Sum up all queued transactions */
379 if (!list_empty(&plchan->desc_list)) { 356 if (!list_empty(&plchan->pend_list)) {
380 list_for_each_entry(txdi, &plchan->desc_list, node) { 357 struct pl08x_txd *txdi;
358 list_for_each_entry(txdi, &plchan->pend_list, node) {
381 bytes += txdi->len; 359 bytes += txdi->len;
382 } 360 }
383
384 } 361 }
385 362
386 spin_unlock_irqrestore(&plchan->lock, flags); 363 spin_unlock_irqrestore(&plchan->lock, flags);
@@ -465,11 +442,11 @@ static inline unsigned int pl08x_get_bytes_for_cctl(unsigned int coded)
465} 442}
466 443
467static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth, 444static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth,
468 u32 tsize) 445 size_t tsize)
469{ 446{
470 u32 retbits = cctl; 447 u32 retbits = cctl;
471 448
472 /* Remove all src, dst and transfersize bits */ 449 /* Remove all src, dst and transfer size bits */
473 retbits &= ~PL080_CONTROL_DWIDTH_MASK; 450 retbits &= ~PL080_CONTROL_DWIDTH_MASK;
474 retbits &= ~PL080_CONTROL_SWIDTH_MASK; 451 retbits &= ~PL080_CONTROL_SWIDTH_MASK;
475 retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK; 452 retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;
@@ -509,38 +486,45 @@ static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth,
509 return retbits; 486 return retbits;
510} 487}
511 488
489struct pl08x_lli_build_data {
490 struct pl08x_txd *txd;
491 struct pl08x_driver_data *pl08x;
492 struct pl08x_bus_data srcbus;
493 struct pl08x_bus_data dstbus;
494 size_t remainder;
495};
496
512/* 497/*
513 * Autoselect a master bus to use for the transfer 498 * Autoselect a master bus to use for the transfer
514 * this prefers the destination bus if both available 499 * this prefers the destination bus if both available
515 * if fixed address on one bus the other will be chosen 500 * if fixed address on one bus the other will be chosen
516 */ 501 */
517void pl08x_choose_master_bus(struct pl08x_bus_data *src_bus, 502static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd,
518 struct pl08x_bus_data *dst_bus, struct pl08x_bus_data **mbus, 503 struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl)
519 struct pl08x_bus_data **sbus, u32 cctl)
520{ 504{
521 if (!(cctl & PL080_CONTROL_DST_INCR)) { 505 if (!(cctl & PL080_CONTROL_DST_INCR)) {
522 *mbus = src_bus; 506 *mbus = &bd->srcbus;
523 *sbus = dst_bus; 507 *sbus = &bd->dstbus;
524 } else if (!(cctl & PL080_CONTROL_SRC_INCR)) { 508 } else if (!(cctl & PL080_CONTROL_SRC_INCR)) {
525 *mbus = dst_bus; 509 *mbus = &bd->dstbus;
526 *sbus = src_bus; 510 *sbus = &bd->srcbus;
527 } else { 511 } else {
528 if (dst_bus->buswidth == 4) { 512 if (bd->dstbus.buswidth == 4) {
529 *mbus = dst_bus; 513 *mbus = &bd->dstbus;
530 *sbus = src_bus; 514 *sbus = &bd->srcbus;
531 } else if (src_bus->buswidth == 4) { 515 } else if (bd->srcbus.buswidth == 4) {
532 *mbus = src_bus; 516 *mbus = &bd->srcbus;
533 *sbus = dst_bus; 517 *sbus = &bd->dstbus;
534 } else if (dst_bus->buswidth == 2) { 518 } else if (bd->dstbus.buswidth == 2) {
535 *mbus = dst_bus; 519 *mbus = &bd->dstbus;
536 *sbus = src_bus; 520 *sbus = &bd->srcbus;
537 } else if (src_bus->buswidth == 2) { 521 } else if (bd->srcbus.buswidth == 2) {
538 *mbus = src_bus; 522 *mbus = &bd->srcbus;
539 *sbus = dst_bus; 523 *sbus = &bd->dstbus;
540 } else { 524 } else {
541 /* src_bus->buswidth == 1 */ 525 /* bd->srcbus.buswidth == 1 */
542 *mbus = dst_bus; 526 *mbus = &bd->dstbus;
543 *sbus = src_bus; 527 *sbus = &bd->srcbus;
544 } 528 }
545 } 529 }
546} 530}
@@ -549,55 +533,41 @@ void pl08x_choose_master_bus(struct pl08x_bus_data *src_bus,
549 * Fills in one LLI for a certain transfer descriptor 533 * Fills in one LLI for a certain transfer descriptor
550 * and advance the counter 534 * and advance the counter
551 */ 535 */
552int pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x, 536static void pl08x_fill_lli_for_desc(struct pl08x_lli_build_data *bd,
553 struct pl08x_txd *txd, int num_llis, int len, 537 int num_llis, int len, u32 cctl)
554 u32 cctl, u32 *remainder)
555{ 538{
556 struct lli *llis_va = txd->llis_va; 539 struct pl08x_lli *llis_va = bd->txd->llis_va;
557 struct lli *llis_bus = (struct lli *) txd->llis_bus; 540 dma_addr_t llis_bus = bd->txd->llis_bus;
558 541
559 BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS); 542 BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);
560 543
561 llis_va[num_llis].cctl = cctl; 544 llis_va[num_llis].cctl = cctl;
562 llis_va[num_llis].src = txd->srcbus.addr; 545 llis_va[num_llis].src = bd->srcbus.addr;
563 llis_va[num_llis].dst = txd->dstbus.addr; 546 llis_va[num_llis].dst = bd->dstbus.addr;
564 547 llis_va[num_llis].lli = llis_bus + (num_llis + 1) * sizeof(struct pl08x_lli);
565 /* 548 if (bd->pl08x->lli_buses & PL08X_AHB2)
566 * On versions with dual masters, you can optionally AND on 549 llis_va[num_llis].lli |= PL080_LLI_LM_AHB2;
567 * PL080_LLI_LM_AHB2 to the LLI to tell the hardware to read
568 * in new LLIs with that controller, but we always try to
569 * choose AHB1 to point into memory. The idea is to have AHB2
570 * fixed on the peripheral and AHB1 messing around in the
571 * memory. So we don't manipulate this bit currently.
572 */
573
574 llis_va[num_llis].next =
575 (dma_addr_t)((u32) &(llis_bus[num_llis + 1]));
576 550
577 if (cctl & PL080_CONTROL_SRC_INCR) 551 if (cctl & PL080_CONTROL_SRC_INCR)
578 txd->srcbus.addr += len; 552 bd->srcbus.addr += len;
579 if (cctl & PL080_CONTROL_DST_INCR) 553 if (cctl & PL080_CONTROL_DST_INCR)
580 txd->dstbus.addr += len; 554 bd->dstbus.addr += len;
581 555
582 *remainder -= len; 556 BUG_ON(bd->remainder < len);
583 557
584 return num_llis + 1; 558 bd->remainder -= len;
585} 559}
586 560
587/* 561/*
588 * Return number of bytes to fill to boundary, or len 562 * Return number of bytes to fill to boundary, or len.
563 * This calculation works for any value of addr.
589 */ 564 */
590static inline u32 pl08x_pre_boundary(u32 addr, u32 len) 565static inline size_t pl08x_pre_boundary(u32 addr, size_t len)
591{ 566{
592 u32 boundary; 567 size_t boundary_len = PL08X_BOUNDARY_SIZE -
568 (addr & (PL08X_BOUNDARY_SIZE - 1));
593 569
594 boundary = ((addr >> PL08X_BOUNDARY_SHIFT) + 1) 570 return min(boundary_len, len);
595 << PL08X_BOUNDARY_SHIFT;
596
597 if (boundary < addr + len)
598 return boundary - addr;
599 else
600 return len;
601} 571}
602 572
603/* 573/*
@@ -608,20 +578,13 @@ static inline u32 pl08x_pre_boundary(u32 addr, u32 len)
608static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, 578static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
609 struct pl08x_txd *txd) 579 struct pl08x_txd *txd)
610{ 580{
611 struct pl08x_channel_data *cd = txd->cd;
612 struct pl08x_bus_data *mbus, *sbus; 581 struct pl08x_bus_data *mbus, *sbus;
613 u32 remainder; 582 struct pl08x_lli_build_data bd;
614 int num_llis = 0; 583 int num_llis = 0;
615 u32 cctl; 584 u32 cctl;
616 int max_bytes_per_lli; 585 size_t max_bytes_per_lli;
617 int total_bytes = 0; 586 size_t total_bytes = 0;
618 struct lli *llis_va; 587 struct pl08x_lli *llis_va;
619 struct lli *llis_bus;
620
621 if (!txd) {
622 dev_err(&pl08x->adev->dev, "%s no descriptor\n", __func__);
623 return 0;
624 }
625 588
626 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, 589 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT,
627 &txd->llis_bus); 590 &txd->llis_bus);
@@ -632,101 +595,67 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
632 595
633 pl08x->pool_ctr++; 596 pl08x->pool_ctr++;
634 597
635 /* 598 /* Get the default CCTL */
636 * Initialize bus values for this transfer 599 cctl = txd->cctl;
637 * from the passed optimal values
638 */
639 if (!cd) {
640 dev_err(&pl08x->adev->dev, "%s no channel data\n", __func__);
641 return 0;
642 }
643 600
644 /* Get the default CCTL from the platform data */ 601 bd.txd = txd;
645 cctl = cd->cctl; 602 bd.pl08x = pl08x;
646 603 bd.srcbus.addr = txd->src_addr;
647 /* 604 bd.dstbus.addr = txd->dst_addr;
648 * On the PL080 we have two bus masters and we
649 * should select one for source and one for
650 * destination. We try to use AHB2 for the
651 * bus which does not increment (typically the
652 * peripheral) else we just choose something.
653 */
654 cctl &= ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2);
655 if (pl08x->vd->dualmaster) {
656 if (cctl & PL080_CONTROL_SRC_INCR)
657 /* Source increments, use AHB2 for destination */
658 cctl |= PL080_CONTROL_DST_AHB2;
659 else if (cctl & PL080_CONTROL_DST_INCR)
660 /* Destination increments, use AHB2 for source */
661 cctl |= PL080_CONTROL_SRC_AHB2;
662 else
663 /* Just pick something, source AHB1 dest AHB2 */
664 cctl |= PL080_CONTROL_DST_AHB2;
665 }
666 605
667 /* Find maximum width of the source bus */ 606 /* Find maximum width of the source bus */
668 txd->srcbus.maxwidth = 607 bd.srcbus.maxwidth =
669 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >> 608 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >>
670 PL080_CONTROL_SWIDTH_SHIFT); 609 PL080_CONTROL_SWIDTH_SHIFT);
671 610
672 /* Find maximum width of the destination bus */ 611 /* Find maximum width of the destination bus */
673 txd->dstbus.maxwidth = 612 bd.dstbus.maxwidth =
674 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >> 613 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >>
675 PL080_CONTROL_DWIDTH_SHIFT); 614 PL080_CONTROL_DWIDTH_SHIFT);
676 615
677 /* Set up the bus widths to the maximum */ 616 /* Set up the bus widths to the maximum */
678 txd->srcbus.buswidth = txd->srcbus.maxwidth; 617 bd.srcbus.buswidth = bd.srcbus.maxwidth;
679 txd->dstbus.buswidth = txd->dstbus.maxwidth; 618 bd.dstbus.buswidth = bd.dstbus.maxwidth;
680 dev_vdbg(&pl08x->adev->dev, 619 dev_vdbg(&pl08x->adev->dev,
681 "%s source bus is %d bytes wide, dest bus is %d bytes wide\n", 620 "%s source bus is %d bytes wide, dest bus is %d bytes wide\n",
682 __func__, txd->srcbus.buswidth, txd->dstbus.buswidth); 621 __func__, bd.srcbus.buswidth, bd.dstbus.buswidth);
683 622
684 623
685 /* 624 /*
686 * Bytes transferred == tsize * MIN(buswidths), not max(buswidths) 625 * Bytes transferred == tsize * MIN(buswidths), not max(buswidths)
687 */ 626 */
688 max_bytes_per_lli = min(txd->srcbus.buswidth, txd->dstbus.buswidth) * 627 max_bytes_per_lli = min(bd.srcbus.buswidth, bd.dstbus.buswidth) *
689 PL080_CONTROL_TRANSFER_SIZE_MASK; 628 PL080_CONTROL_TRANSFER_SIZE_MASK;
690 dev_vdbg(&pl08x->adev->dev, 629 dev_vdbg(&pl08x->adev->dev,
691 "%s max bytes per lli = %d\n", 630 "%s max bytes per lli = %zu\n",
692 __func__, max_bytes_per_lli); 631 __func__, max_bytes_per_lli);
693 632
694 /* We need to count this down to zero */ 633 /* We need to count this down to zero */
695 remainder = txd->len; 634 bd.remainder = txd->len;
696 dev_vdbg(&pl08x->adev->dev, 635 dev_vdbg(&pl08x->adev->dev,
697 "%s remainder = %d\n", 636 "%s remainder = %zu\n",
698 __func__, remainder); 637 __func__, bd.remainder);
699 638
700 /* 639 /*
701 * Choose bus to align to 640 * Choose bus to align to
702 * - prefers destination bus if both available 641 * - prefers destination bus if both available
703 * - if fixed address on one bus chooses other 642 * - if fixed address on one bus chooses other
704 * - modifies cctl to choose an apropriate master 643 * - modifies cctl to choose an appropriate master
705 */
706 pl08x_choose_master_bus(&txd->srcbus, &txd->dstbus,
707 &mbus, &sbus, cctl);
708
709
710 /*
711 * The lowest bit of the LLI register
712 * is also used to indicate which master to
713 * use for reading the LLIs.
714 */ 644 */
645 pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl);
715 646
716 if (txd->len < mbus->buswidth) { 647 if (txd->len < mbus->buswidth) {
717 /* 648 /*
718 * Less than a bus width available 649 * Less than a bus width available
719 * - send as single bytes 650 * - send as single bytes
720 */ 651 */
721 while (remainder) { 652 while (bd.remainder) {
722 dev_vdbg(&pl08x->adev->dev, 653 dev_vdbg(&pl08x->adev->dev,
723 "%s single byte LLIs for a transfer of " 654 "%s single byte LLIs for a transfer of "
724 "less than a bus width (remain %08x)\n", 655 "less than a bus width (remain 0x%08x)\n",
725 __func__, remainder); 656 __func__, bd.remainder);
726 cctl = pl08x_cctl_bits(cctl, 1, 1, 1); 657 cctl = pl08x_cctl_bits(cctl, 1, 1, 1);
727 num_llis = 658 pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl);
728 pl08x_fill_lli_for_desc(pl08x, txd, num_llis, 1,
729 cctl, &remainder);
730 total_bytes++; 659 total_bytes++;
731 } 660 }
732 } else { 661 } else {
@@ -737,11 +666,10 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
737 while ((mbus->addr) % (mbus->buswidth)) { 666 while ((mbus->addr) % (mbus->buswidth)) {
738 dev_vdbg(&pl08x->adev->dev, 667 dev_vdbg(&pl08x->adev->dev,
739 "%s adjustment lli for less than bus width " 668 "%s adjustment lli for less than bus width "
740 "(remain %08x)\n", 669 "(remain 0x%08x)\n",
741 __func__, remainder); 670 __func__, bd.remainder);
742 cctl = pl08x_cctl_bits(cctl, 1, 1, 1); 671 cctl = pl08x_cctl_bits(cctl, 1, 1, 1);
743 num_llis = pl08x_fill_lli_for_desc 672 pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl);
744 (pl08x, txd, num_llis, 1, cctl, &remainder);
745 total_bytes++; 673 total_bytes++;
746 } 674 }
747 675
@@ -761,53 +689,43 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
761 * Make largest possible LLIs until less than one bus 689 * Make largest possible LLIs until less than one bus
762 * width left 690 * width left
763 */ 691 */
764 while (remainder > (mbus->buswidth - 1)) { 692 while (bd.remainder > (mbus->buswidth - 1)) {
765 int lli_len, target_len; 693 size_t lli_len, target_len, tsize, odd_bytes;
766 int tsize;
767 int odd_bytes;
768 694
769 /* 695 /*
770 * If enough left try to send max possible, 696 * If enough left try to send max possible,
771 * otherwise try to send the remainder 697 * otherwise try to send the remainder
772 */ 698 */
773 target_len = remainder; 699 target_len = min(bd.remainder, max_bytes_per_lli);
774 if (remainder > max_bytes_per_lli)
775 target_len = max_bytes_per_lli;
776 700
777 /* 701 /*
778 * Set bus lengths for incrementing busses 702 * Set bus lengths for incrementing buses to the
779 * to number of bytes which fill to next memory 703 * number of bytes which fill to next memory boundary,
780 * boundary 704 * limiting on the target length calculated above.
781 */ 705 */
782 if (cctl & PL080_CONTROL_SRC_INCR) 706 if (cctl & PL080_CONTROL_SRC_INCR)
783 txd->srcbus.fill_bytes = 707 bd.srcbus.fill_bytes =
784 pl08x_pre_boundary( 708 pl08x_pre_boundary(bd.srcbus.addr,
785 txd->srcbus.addr, 709 target_len);
786 remainder);
787 else 710 else
788 txd->srcbus.fill_bytes = 711 bd.srcbus.fill_bytes = target_len;
789 max_bytes_per_lli;
790 712
791 if (cctl & PL080_CONTROL_DST_INCR) 713 if (cctl & PL080_CONTROL_DST_INCR)
792 txd->dstbus.fill_bytes = 714 bd.dstbus.fill_bytes =
793 pl08x_pre_boundary( 715 pl08x_pre_boundary(bd.dstbus.addr,
794 txd->dstbus.addr, 716 target_len);
795 remainder);
796 else 717 else
797 txd->dstbus.fill_bytes = 718 bd.dstbus.fill_bytes = target_len;
798 max_bytes_per_lli;
799 719
800 /* 720 /* Find the nearest */
801 * Find the nearest 721 lli_len = min(bd.srcbus.fill_bytes,
802 */ 722 bd.dstbus.fill_bytes);
803 lli_len = min(txd->srcbus.fill_bytes,
804 txd->dstbus.fill_bytes);
805 723
806 BUG_ON(lli_len > remainder); 724 BUG_ON(lli_len > bd.remainder);
807 725
808 if (lli_len <= 0) { 726 if (lli_len <= 0) {
809 dev_err(&pl08x->adev->dev, 727 dev_err(&pl08x->adev->dev,
810 "%s lli_len is %d, <= 0\n", 728 "%s lli_len is %zu, <= 0\n",
811 __func__, lli_len); 729 __func__, lli_len);
812 return 0; 730 return 0;
813 } 731 }
@@ -826,7 +744,7 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
826 /* 744 /*
827 * So now we know how many bytes to transfer 745 * So now we know how many bytes to transfer
828 * to get to the nearest boundary 746 * to get to the nearest boundary
829 * The next lli will past the boundary 747 * The next LLI will past the boundary
830 * - however we may be working to a boundary 748 * - however we may be working to a boundary
831 * on the slave bus 749 * on the slave bus
832 * We need to ensure the master stays aligned 750 * We need to ensure the master stays aligned
@@ -855,21 +773,20 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
855 773
856 if (target_len != lli_len) { 774 if (target_len != lli_len) {
857 dev_vdbg(&pl08x->adev->dev, 775 dev_vdbg(&pl08x->adev->dev,
858 "%s can't send what we want. Desired %08x, lli of %08x bytes in txd of %08x\n", 776 "%s can't send what we want. Desired 0x%08zx, lli of 0x%08zx bytes in txd of 0x%08zx\n",
859 __func__, target_len, lli_len, txd->len); 777 __func__, target_len, lli_len, txd->len);
860 } 778 }
861 779
862 cctl = pl08x_cctl_bits(cctl, 780 cctl = pl08x_cctl_bits(cctl,
863 txd->srcbus.buswidth, 781 bd.srcbus.buswidth,
864 txd->dstbus.buswidth, 782 bd.dstbus.buswidth,
865 tsize); 783 tsize);
866 784
867 dev_vdbg(&pl08x->adev->dev, 785 dev_vdbg(&pl08x->adev->dev,
868 "%s fill lli with single lli chunk of size %08x (remainder %08x)\n", 786 "%s fill lli with single lli chunk of size 0x%08zx (remainder 0x%08zx)\n",
869 __func__, lli_len, remainder); 787 __func__, lli_len, bd.remainder);
870 num_llis = pl08x_fill_lli_for_desc(pl08x, txd, 788 pl08x_fill_lli_for_desc(&bd, num_llis++,
871 num_llis, lli_len, cctl, 789 lli_len, cctl);
872 &remainder);
873 total_bytes += lli_len; 790 total_bytes += lli_len;
874 } 791 }
875 792
@@ -881,15 +798,13 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
881 */ 798 */
882 int j; 799 int j;
883 for (j = 0; (j < mbus->buswidth) 800 for (j = 0; (j < mbus->buswidth)
884 && (remainder); j++) { 801 && (bd.remainder); j++) {
885 cctl = pl08x_cctl_bits(cctl, 1, 1, 1); 802 cctl = pl08x_cctl_bits(cctl, 1, 1, 1);
886 dev_vdbg(&pl08x->adev->dev, 803 dev_vdbg(&pl08x->adev->dev,
887 "%s align with boundardy, single byte (remain %08x)\n", 804 "%s align with boundary, single byte (remain 0x%08zx)\n",
888 __func__, remainder); 805 __func__, bd.remainder);
889 num_llis = 806 pl08x_fill_lli_for_desc(&bd,
890 pl08x_fill_lli_for_desc(pl08x, 807 num_llis++, 1, cctl);
891 txd, num_llis, 1,
892 cctl, &remainder);
893 total_bytes++; 808 total_bytes++;
894 } 809 }
895 } 810 }
@@ -898,25 +813,18 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
898 /* 813 /*
899 * Send any odd bytes 814 * Send any odd bytes
900 */ 815 */
901 if (remainder < 0) { 816 while (bd.remainder) {
902 dev_err(&pl08x->adev->dev, "%s remainder not fitted 0x%08x bytes\n",
903 __func__, remainder);
904 return 0;
905 }
906
907 while (remainder) {
908 cctl = pl08x_cctl_bits(cctl, 1, 1, 1); 817 cctl = pl08x_cctl_bits(cctl, 1, 1, 1);
909 dev_vdbg(&pl08x->adev->dev, 818 dev_vdbg(&pl08x->adev->dev,
910 "%s align with boundardy, single odd byte (remain %d)\n", 819 "%s align with boundary, single odd byte (remain %zu)\n",
911 __func__, remainder); 820 __func__, bd.remainder);
912 num_llis = pl08x_fill_lli_for_desc(pl08x, txd, num_llis, 821 pl08x_fill_lli_for_desc(&bd, num_llis++, 1, cctl);
913 1, cctl, &remainder);
914 total_bytes++; 822 total_bytes++;
915 } 823 }
916 } 824 }
917 if (total_bytes != txd->len) { 825 if (total_bytes != txd->len) {
918 dev_err(&pl08x->adev->dev, 826 dev_err(&pl08x->adev->dev,
919 "%s size of encoded lli:s don't match total txd, transferred 0x%08x from size 0x%08x\n", 827 "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
920 __func__, total_bytes, txd->len); 828 __func__, total_bytes, txd->len);
921 return 0; 829 return 0;
922 } 830 }
@@ -927,41 +835,16 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
927 __func__, (u32) MAX_NUM_TSFR_LLIS); 835 __func__, (u32) MAX_NUM_TSFR_LLIS);
928 return 0; 836 return 0;
929 } 837 }
838
839 llis_va = txd->llis_va;
930 /* 840 /*
931 * Decide whether this is a loop or a terminated transfer 841 * The final LLI terminates the LLI.
932 */ 842 */
933 llis_va = txd->llis_va; 843 llis_va[num_llis - 1].lli = 0;
934 llis_bus = (struct lli *) txd->llis_bus; 844 /*
935 845 * The final LLI element shall also fire an interrupt
936 if (cd->circular_buffer) { 846 */
937 /* 847 llis_va[num_llis - 1].cctl |= PL080_CONTROL_TC_IRQ_EN;
938 * Loop the circular buffer so that the next element
939 * points back to the beginning of the LLI.
940 */
941 llis_va[num_llis - 1].next =
942 (dma_addr_t)((unsigned int)&(llis_bus[0]));
943 } else {
944 /*
945 * On non-circular buffers, the final LLI terminates
946 * the LLI.
947 */
948 llis_va[num_llis - 1].next = 0;
949 /*
950 * The final LLI element shall also fire an interrupt
951 */
952 llis_va[num_llis - 1].cctl |= PL080_CONTROL_TC_IRQ_EN;
953 }
954
955 /* Now store the channel register values */
956 txd->csrc = llis_va[0].src;
957 txd->cdst = llis_va[0].dst;
958 if (num_llis > 1)
959 txd->clli = llis_va[0].next;
960 else
961 txd->clli = 0;
962
963 txd->cctl = llis_va[0].cctl;
964 /* ccfg will be set at physical channel allocation time */
965 848
966#ifdef VERBOSE_DEBUG 849#ifdef VERBOSE_DEBUG
967 { 850 {
@@ -969,13 +852,13 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
969 852
970 for (i = 0; i < num_llis; i++) { 853 for (i = 0; i < num_llis; i++) {
971 dev_vdbg(&pl08x->adev->dev, 854 dev_vdbg(&pl08x->adev->dev,
972 "lli %d @%p: csrc=%08x, cdst=%08x, cctl=%08x, clli=%08x\n", 855 "lli %d @%p: csrc=0x%08x, cdst=0x%08x, cctl=0x%08x, clli=0x%08x\n",
973 i, 856 i,
974 &llis_va[i], 857 &llis_va[i],
975 llis_va[i].src, 858 llis_va[i].src,
976 llis_va[i].dst, 859 llis_va[i].dst,
977 llis_va[i].cctl, 860 llis_va[i].cctl,
978 llis_va[i].next 861 llis_va[i].lli
979 ); 862 );
980 } 863 }
981 } 864 }
@@ -988,14 +871,8 @@ static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
988static void pl08x_free_txd(struct pl08x_driver_data *pl08x, 871static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
989 struct pl08x_txd *txd) 872 struct pl08x_txd *txd)
990{ 873{
991 if (!txd)
992 dev_err(&pl08x->adev->dev,
993 "%s no descriptor to free\n",
994 __func__);
995
996 /* Free the LLI */ 874 /* Free the LLI */
997 dma_pool_free(pl08x->pool, txd->llis_va, 875 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);
998 txd->llis_bus);
999 876
1000 pl08x->pool_ctr--; 877 pl08x->pool_ctr--;
1001 878
@@ -1008,9 +885,9 @@ static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
1008 struct pl08x_txd *txdi = NULL; 885 struct pl08x_txd *txdi = NULL;
1009 struct pl08x_txd *next; 886 struct pl08x_txd *next;
1010 887
1011 if (!list_empty(&plchan->desc_list)) { 888 if (!list_empty(&plchan->pend_list)) {
1012 list_for_each_entry_safe(txdi, 889 list_for_each_entry_safe(txdi,
1013 next, &plchan->desc_list, node) { 890 next, &plchan->pend_list, node) {
1014 list_del(&txdi->node); 891 list_del(&txdi->node);
1015 pl08x_free_txd(pl08x, txdi); 892 pl08x_free_txd(pl08x, txdi);
1016 } 893 }
@@ -1069,6 +946,12 @@ static int prep_phy_channel(struct pl08x_dma_chan *plchan,
1069 return -EBUSY; 946 return -EBUSY;
1070 } 947 }
1071 ch->signal = ret; 948 ch->signal = ret;
949
950 /* Assign the flow control signal to this channel */
951 if (txd->direction == DMA_TO_DEVICE)
952 txd->ccfg |= ch->signal << PL080_CONFIG_DST_SEL_SHIFT;
953 else if (txd->direction == DMA_FROM_DEVICE)
954 txd->ccfg |= ch->signal << PL080_CONFIG_SRC_SEL_SHIFT;
1072 } 955 }
1073 956
1074 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d and signal %d for xfer on %s\n", 957 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d and signal %d for xfer on %s\n",
@@ -1076,19 +959,54 @@ static int prep_phy_channel(struct pl08x_dma_chan *plchan,
1076 ch->signal, 959 ch->signal,
1077 plchan->name); 960 plchan->name);
1078 961
962 plchan->phychan_hold++;
1079 plchan->phychan = ch; 963 plchan->phychan = ch;
1080 964
1081 return 0; 965 return 0;
1082} 966}
1083 967
968static void release_phy_channel(struct pl08x_dma_chan *plchan)
969{
970 struct pl08x_driver_data *pl08x = plchan->host;
971
972 if ((plchan->phychan->signal >= 0) && pl08x->pd->put_signal) {
973 pl08x->pd->put_signal(plchan);
974 plchan->phychan->signal = -1;
975 }
976 pl08x_put_phy_channel(pl08x, plchan->phychan);
977 plchan->phychan = NULL;
978}
979
1084static dma_cookie_t pl08x_tx_submit(struct dma_async_tx_descriptor *tx) 980static dma_cookie_t pl08x_tx_submit(struct dma_async_tx_descriptor *tx)
1085{ 981{
1086 struct pl08x_dma_chan *plchan = to_pl08x_chan(tx->chan); 982 struct pl08x_dma_chan *plchan = to_pl08x_chan(tx->chan);
983 struct pl08x_txd *txd = to_pl08x_txd(tx);
984 unsigned long flags;
985
986 spin_lock_irqsave(&plchan->lock, flags);
987
988 plchan->chan.cookie += 1;
989 if (plchan->chan.cookie < 0)
990 plchan->chan.cookie = 1;
991 tx->cookie = plchan->chan.cookie;
992
993 /* Put this onto the pending list */
994 list_add_tail(&txd->node, &plchan->pend_list);
1087 995
1088 atomic_inc(&plchan->last_issued); 996 /*
1089 tx->cookie = atomic_read(&plchan->last_issued); 997 * If there was no physical channel available for this memcpy,
1090 /* This unlock follows the lock in the prep() function */ 998 * stack the request up and indicate that the channel is waiting
1091 spin_unlock_irqrestore(&plchan->lock, plchan->lockflags); 999 * for a free physical channel.
1000 */
1001 if (!plchan->slave && !plchan->phychan) {
1002 /* Do this memcpy whenever there is a channel ready */
1003 plchan->state = PL08X_CHAN_WAITING;
1004 plchan->waiting = txd;
1005 } else {
1006 plchan->phychan_hold--;
1007 }
1008
1009 spin_unlock_irqrestore(&plchan->lock, flags);
1092 1010
1093 return tx->cookie; 1011 return tx->cookie;
1094} 1012}
@@ -1118,7 +1036,7 @@ pl08x_dma_tx_status(struct dma_chan *chan,
1118 enum dma_status ret; 1036 enum dma_status ret;
1119 u32 bytesleft = 0; 1037 u32 bytesleft = 0;
1120 1038
1121 last_used = atomic_read(&plchan->last_issued); 1039 last_used = plchan->chan.cookie;
1122 last_complete = plchan->lc; 1040 last_complete = plchan->lc;
1123 1041
1124 ret = dma_async_is_complete(cookie, last_complete, last_used); 1042 ret = dma_async_is_complete(cookie, last_complete, last_used);
@@ -1134,7 +1052,7 @@ pl08x_dma_tx_status(struct dma_chan *chan,
1134 /* 1052 /*
1135 * This cookie not complete yet 1053 * This cookie not complete yet
1136 */ 1054 */
1137 last_used = atomic_read(&plchan->last_issued); 1055 last_used = plchan->chan.cookie;
1138 last_complete = plchan->lc; 1056 last_complete = plchan->lc;
1139 1057
1140 /* Get number of bytes left in the active transactions and queue */ 1058 /* Get number of bytes left in the active transactions and queue */
@@ -1199,37 +1117,35 @@ static const struct burst_table burst_sizes[] = {
1199 }, 1117 },
1200}; 1118};
1201 1119
1202static void dma_set_runtime_config(struct dma_chan *chan, 1120static int dma_set_runtime_config(struct dma_chan *chan,
1203 struct dma_slave_config *config) 1121 struct dma_slave_config *config)
1204{ 1122{
1205 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1123 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1206 struct pl08x_driver_data *pl08x = plchan->host; 1124 struct pl08x_driver_data *pl08x = plchan->host;
1207 struct pl08x_channel_data *cd = plchan->cd; 1125 struct pl08x_channel_data *cd = plchan->cd;
1208 enum dma_slave_buswidth addr_width; 1126 enum dma_slave_buswidth addr_width;
1127 dma_addr_t addr;
1209 u32 maxburst; 1128 u32 maxburst;
1210 u32 cctl = 0; 1129 u32 cctl = 0;
1211 /* Mask out all except src and dst channel */ 1130 int i;
1212 u32 ccfg = cd->ccfg & 0x000003DEU; 1131
1213 int i = 0; 1132 if (!plchan->slave)
1133 return -EINVAL;
1214 1134
1215 /* Transfer direction */ 1135 /* Transfer direction */
1216 plchan->runtime_direction = config->direction; 1136 plchan->runtime_direction = config->direction;
1217 if (config->direction == DMA_TO_DEVICE) { 1137 if (config->direction == DMA_TO_DEVICE) {
1218 plchan->runtime_addr = config->dst_addr; 1138 addr = config->dst_addr;
1219 cctl |= PL080_CONTROL_SRC_INCR;
1220 ccfg |= PL080_FLOW_MEM2PER << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1221 addr_width = config->dst_addr_width; 1139 addr_width = config->dst_addr_width;
1222 maxburst = config->dst_maxburst; 1140 maxburst = config->dst_maxburst;
1223 } else if (config->direction == DMA_FROM_DEVICE) { 1141 } else if (config->direction == DMA_FROM_DEVICE) {
1224 plchan->runtime_addr = config->src_addr; 1142 addr = config->src_addr;
1225 cctl |= PL080_CONTROL_DST_INCR;
1226 ccfg |= PL080_FLOW_PER2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1227 addr_width = config->src_addr_width; 1143 addr_width = config->src_addr_width;
1228 maxburst = config->src_maxburst; 1144 maxburst = config->src_maxburst;
1229 } else { 1145 } else {
1230 dev_err(&pl08x->adev->dev, 1146 dev_err(&pl08x->adev->dev,
1231 "bad runtime_config: alien transfer direction\n"); 1147 "bad runtime_config: alien transfer direction\n");
1232 return; 1148 return -EINVAL;
1233 } 1149 }
1234 1150
1235 switch (addr_width) { 1151 switch (addr_width) {
@@ -1248,42 +1164,40 @@ static void dma_set_runtime_config(struct dma_chan *chan,
1248 default: 1164 default:
1249 dev_err(&pl08x->adev->dev, 1165 dev_err(&pl08x->adev->dev,
1250 "bad runtime_config: alien address width\n"); 1166 "bad runtime_config: alien address width\n");
1251 return; 1167 return -EINVAL;
1252 } 1168 }
1253 1169
1254 /* 1170 /*
1255 * Now decide on a maxburst: 1171 * Now decide on a maxburst:
1256 * If this channel will only request single transfers, set 1172 * If this channel will only request single transfers, set this
1257 * this down to ONE element. 1173 * down to ONE element. Also select one element if no maxburst
1174 * is specified.
1258 */ 1175 */
1259 if (plchan->cd->single) { 1176 if (plchan->cd->single || maxburst == 0) {
1260 cctl |= (PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT) | 1177 cctl |= (PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT) |
1261 (PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT); 1178 (PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT);
1262 } else { 1179 } else {
1263 while (i < ARRAY_SIZE(burst_sizes)) { 1180 for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
1264 if (burst_sizes[i].burstwords <= maxburst) 1181 if (burst_sizes[i].burstwords <= maxburst)
1265 break; 1182 break;
1266 i++;
1267 }
1268 cctl |= burst_sizes[i].reg; 1183 cctl |= burst_sizes[i].reg;
1269 } 1184 }
1270 1185
1271 /* Access the cell in privileged mode, non-bufferable, non-cacheable */ 1186 plchan->runtime_addr = addr;
1272 cctl &= ~PL080_CONTROL_PROT_MASK;
1273 cctl |= PL080_CONTROL_PROT_SYS;
1274 1187
1275 /* Modify the default channel data to fit PrimeCell request */ 1188 /* Modify the default channel data to fit PrimeCell request */
1276 cd->cctl = cctl; 1189 cd->cctl = cctl;
1277 cd->ccfg = ccfg;
1278 1190
1279 dev_dbg(&pl08x->adev->dev, 1191 dev_dbg(&pl08x->adev->dev,
1280 "configured channel %s (%s) for %s, data width %d, " 1192 "configured channel %s (%s) for %s, data width %d, "
1281 "maxburst %d words, LE, CCTL=%08x, CCFG=%08x\n", 1193 "maxburst %d words, LE, CCTL=0x%08x\n",
1282 dma_chan_name(chan), plchan->name, 1194 dma_chan_name(chan), plchan->name,
1283 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX", 1195 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
1284 addr_width, 1196 addr_width,
1285 maxburst, 1197 maxburst,
1286 cctl, ccfg); 1198 cctl);
1199
1200 return 0;
1287} 1201}
1288 1202
1289/* 1203/*
@@ -1293,35 +1207,26 @@ static void dma_set_runtime_config(struct dma_chan *chan,
1293static void pl08x_issue_pending(struct dma_chan *chan) 1207static void pl08x_issue_pending(struct dma_chan *chan)
1294{ 1208{
1295 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1209 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1296 struct pl08x_driver_data *pl08x = plchan->host;
1297 unsigned long flags; 1210 unsigned long flags;
1298 1211
1299 spin_lock_irqsave(&plchan->lock, flags); 1212 spin_lock_irqsave(&plchan->lock, flags);
1300 /* Something is already active */ 1213 /* Something is already active, or we're waiting for a channel... */
1301 if (plchan->at) { 1214 if (plchan->at || plchan->state == PL08X_CHAN_WAITING) {
1302 spin_unlock_irqrestore(&plchan->lock, flags); 1215 spin_unlock_irqrestore(&plchan->lock, flags);
1303 return;
1304 }
1305
1306 /* Didn't get a physical channel so waiting for it ... */
1307 if (plchan->state == PL08X_CHAN_WAITING)
1308 return; 1216 return;
1217 }
1309 1218
1310 /* Take the first element in the queue and execute it */ 1219 /* Take the first element in the queue and execute it */
1311 if (!list_empty(&plchan->desc_list)) { 1220 if (!list_empty(&plchan->pend_list)) {
1312 struct pl08x_txd *next; 1221 struct pl08x_txd *next;
1313 1222
1314 next = list_first_entry(&plchan->desc_list, 1223 next = list_first_entry(&plchan->pend_list,
1315 struct pl08x_txd, 1224 struct pl08x_txd,
1316 node); 1225 node);
1317 list_del(&next->node); 1226 list_del(&next->node);
1318 plchan->at = next;
1319 plchan->state = PL08X_CHAN_RUNNING; 1227 plchan->state = PL08X_CHAN_RUNNING;
1320 1228
1321 /* Configure the physical channel for the active txd */ 1229 pl08x_start_txd(plchan, next);
1322 pl08x_config_phychan_for_txd(plchan);
1323 pl08x_set_cregs(pl08x, plchan->phychan);
1324 pl08x_enable_phy_chan(pl08x, plchan->phychan);
1325 } 1230 }
1326 1231
1327 spin_unlock_irqrestore(&plchan->lock, flags); 1232 spin_unlock_irqrestore(&plchan->lock, flags);
@@ -1330,30 +1235,17 @@ static void pl08x_issue_pending(struct dma_chan *chan)
1330static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan, 1235static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan,
1331 struct pl08x_txd *txd) 1236 struct pl08x_txd *txd)
1332{ 1237{
1333 int num_llis;
1334 struct pl08x_driver_data *pl08x = plchan->host; 1238 struct pl08x_driver_data *pl08x = plchan->host;
1335 int ret; 1239 unsigned long flags;
1240 int num_llis, ret;
1336 1241
1337 num_llis = pl08x_fill_llis_for_desc(pl08x, txd); 1242 num_llis = pl08x_fill_llis_for_desc(pl08x, txd);
1338 1243 if (!num_llis) {
1339 if (!num_llis) 1244 kfree(txd);
1340 return -EINVAL; 1245 return -EINVAL;
1246 }
1341 1247
1342 spin_lock_irqsave(&plchan->lock, plchan->lockflags); 1248 spin_lock_irqsave(&plchan->lock, flags);
1343
1344 /*
1345 * If this device is not using a circular buffer then
1346 * queue this new descriptor for transfer.
1347 * The descriptor for a circular buffer continues
1348 * to be used until the channel is freed.
1349 */
1350 if (txd->cd->circular_buffer)
1351 dev_err(&pl08x->adev->dev,
1352 "%s attempting to queue a circular buffer\n",
1353 __func__);
1354 else
1355 list_add_tail(&txd->node,
1356 &plchan->desc_list);
1357 1249
1358 /* 1250 /*
1359 * See if we already have a physical channel allocated, 1251 * See if we already have a physical channel allocated,
@@ -1362,24 +1254,23 @@ static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan,
1362 ret = prep_phy_channel(plchan, txd); 1254 ret = prep_phy_channel(plchan, txd);
1363 if (ret) { 1255 if (ret) {
1364 /* 1256 /*
1365 * No physical channel available, we will 1257 * No physical channel was available.
1366 * stack up the memcpy channels until there is a channel 1258 *
1367 * available to handle it whereas slave transfers may 1259 * memcpy transfers can be sorted out at submission time.
1368 * have been denied due to platform channel muxing restrictions 1260 *
1369 * and since there is no guarantee that this will ever be 1261 * Slave transfers may have been denied due to platform
1370 * resolved, and since the signal must be aquired AFTER 1262 * channel muxing restrictions. Since there is no guarantee
1371 * aquiring the physical channel, we will let them be NACK:ed 1263 * that this will ever be resolved, and the signal must be
1372 * with -EBUSY here. The drivers can alway retry the prep() 1264 * acquired AFTER acquiring the physical channel, we will let
1373 * call if they are eager on doing this using DMA. 1265 * them be NACK:ed with -EBUSY here. The drivers can retry
1266 * the prep() call if they are eager on doing this using DMA.
1374 */ 1267 */
1375 if (plchan->slave) { 1268 if (plchan->slave) {
1376 pl08x_free_txd_list(pl08x, plchan); 1269 pl08x_free_txd_list(pl08x, plchan);
1377 spin_unlock_irqrestore(&plchan->lock, plchan->lockflags); 1270 pl08x_free_txd(pl08x, txd);
1271 spin_unlock_irqrestore(&plchan->lock, flags);
1378 return -EBUSY; 1272 return -EBUSY;
1379 } 1273 }
1380 /* Do this memcpy whenever there is a channel ready */
1381 plchan->state = PL08X_CHAN_WAITING;
1382 plchan->waiting = txd;
1383 } else 1274 } else
1384 /* 1275 /*
1385 * Else we're all set, paused and ready to roll, 1276 * Else we're all set, paused and ready to roll,
@@ -1391,16 +1282,47 @@ static int pl08x_prep_channel_resources(struct pl08x_dma_chan *plchan,
1391 if (plchan->state == PL08X_CHAN_IDLE) 1282 if (plchan->state == PL08X_CHAN_IDLE)
1392 plchan->state = PL08X_CHAN_PAUSED; 1283 plchan->state = PL08X_CHAN_PAUSED;
1393 1284
1394 /* 1285 spin_unlock_irqrestore(&plchan->lock, flags);
1395 * Notice that we leave plchan->lock locked on purpose:
1396 * it will be unlocked in the subsequent tx_submit()
1397 * call. This is a consequence of the current API.
1398 */
1399 1286
1400 return 0; 1287 return 0;
1401} 1288}
1402 1289
1403/* 1290/*
1291 * Given the source and destination available bus masks, select which
1292 * will be routed to each port. We try to have source and destination
1293 * on separate ports, but always respect the allowable settings.
1294 */
1295static u32 pl08x_select_bus(struct pl08x_driver_data *pl08x, u8 src, u8 dst)
1296{
1297 u32 cctl = 0;
1298
1299 if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
1300 cctl |= PL080_CONTROL_DST_AHB2;
1301 if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
1302 cctl |= PL080_CONTROL_SRC_AHB2;
1303
1304 return cctl;
1305}
1306
1307static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan,
1308 unsigned long flags)
1309{
1310 struct pl08x_txd *txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT);
1311
1312 if (txd) {
1313 dma_async_tx_descriptor_init(&txd->tx, &plchan->chan);
1314 txd->tx.flags = flags;
1315 txd->tx.tx_submit = pl08x_tx_submit;
1316 INIT_LIST_HEAD(&txd->node);
1317
1318 /* Always enable error and terminal interrupts */
1319 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
1320 PL080_CONFIG_TC_IRQ_MASK;
1321 }
1322 return txd;
1323}
1324
1325/*
1404 * Initialize a descriptor to be used by memcpy submit 1326 * Initialize a descriptor to be used by memcpy submit
1405 */ 1327 */
1406static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( 1328static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
@@ -1412,40 +1334,38 @@ static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
1412 struct pl08x_txd *txd; 1334 struct pl08x_txd *txd;
1413 int ret; 1335 int ret;
1414 1336
1415 txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT); 1337 txd = pl08x_get_txd(plchan, flags);
1416 if (!txd) { 1338 if (!txd) {
1417 dev_err(&pl08x->adev->dev, 1339 dev_err(&pl08x->adev->dev,
1418 "%s no memory for descriptor\n", __func__); 1340 "%s no memory for descriptor\n", __func__);
1419 return NULL; 1341 return NULL;
1420 } 1342 }
1421 1343
1422 dma_async_tx_descriptor_init(&txd->tx, chan);
1423 txd->direction = DMA_NONE; 1344 txd->direction = DMA_NONE;
1424 txd->srcbus.addr = src; 1345 txd->src_addr = src;
1425 txd->dstbus.addr = dest; 1346 txd->dst_addr = dest;
1347 txd->len = len;
1426 1348
1427 /* Set platform data for m2m */ 1349 /* Set platform data for m2m */
1428 txd->cd = &pl08x->pd->memcpy_channel; 1350 txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1351 txd->cctl = pl08x->pd->memcpy_channel.cctl &
1352 ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2);
1353
1429 /* Both to be incremented or the code will break */ 1354 /* Both to be incremented or the code will break */
1430 txd->cd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR; 1355 txd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
1431 txd->tx.tx_submit = pl08x_tx_submit; 1356
1432 txd->tx.callback = NULL; 1357 if (pl08x->vd->dualmaster)
1433 txd->tx.callback_param = NULL; 1358 txd->cctl |= pl08x_select_bus(pl08x,
1434 txd->len = len; 1359 pl08x->mem_buses, pl08x->mem_buses);
1435 1360
1436 INIT_LIST_HEAD(&txd->node);
1437 ret = pl08x_prep_channel_resources(plchan, txd); 1361 ret = pl08x_prep_channel_resources(plchan, txd);
1438 if (ret) 1362 if (ret)
1439 return NULL; 1363 return NULL;
1440 /*
1441 * NB: the channel lock is held at this point so tx_submit()
1442 * must be called in direct succession.
1443 */
1444 1364
1445 return &txd->tx; 1365 return &txd->tx;
1446} 1366}
1447 1367
1448struct dma_async_tx_descriptor *pl08x_prep_slave_sg( 1368static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
1449 struct dma_chan *chan, struct scatterlist *sgl, 1369 struct dma_chan *chan, struct scatterlist *sgl,
1450 unsigned int sg_len, enum dma_data_direction direction, 1370 unsigned int sg_len, enum dma_data_direction direction,
1451 unsigned long flags) 1371 unsigned long flags)
@@ -1453,6 +1373,7 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
1453 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1373 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1454 struct pl08x_driver_data *pl08x = plchan->host; 1374 struct pl08x_driver_data *pl08x = plchan->host;
1455 struct pl08x_txd *txd; 1375 struct pl08x_txd *txd;
1376 u8 src_buses, dst_buses;
1456 int ret; 1377 int ret;
1457 1378
1458 /* 1379 /*
@@ -1467,14 +1388,12 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
1467 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", 1388 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
1468 __func__, sgl->length, plchan->name); 1389 __func__, sgl->length, plchan->name);
1469 1390
1470 txd = kzalloc(sizeof(struct pl08x_txd), GFP_NOWAIT); 1391 txd = pl08x_get_txd(plchan, flags);
1471 if (!txd) { 1392 if (!txd) {
1472 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__); 1393 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
1473 return NULL; 1394 return NULL;
1474 } 1395 }
1475 1396
1476 dma_async_tx_descriptor_init(&txd->tx, chan);
1477
1478 if (direction != plchan->runtime_direction) 1397 if (direction != plchan->runtime_direction)
1479 dev_err(&pl08x->adev->dev, "%s DMA setup does not match " 1398 dev_err(&pl08x->adev->dev, "%s DMA setup does not match "
1480 "the direction configured for the PrimeCell\n", 1399 "the direction configured for the PrimeCell\n",
@@ -1486,37 +1405,47 @@ struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
1486 * channel target address dynamically at runtime. 1405 * channel target address dynamically at runtime.
1487 */ 1406 */
1488 txd->direction = direction; 1407 txd->direction = direction;
1408 txd->len = sgl->length;
1409
1410 txd->cctl = plchan->cd->cctl &
1411 ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
1412 PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
1413 PL080_CONTROL_PROT_MASK);
1414
1415 /* Access the cell in privileged mode, non-bufferable, non-cacheable */
1416 txd->cctl |= PL080_CONTROL_PROT_SYS;
1417
1489 if (direction == DMA_TO_DEVICE) { 1418 if (direction == DMA_TO_DEVICE) {
1490 txd->srcbus.addr = sgl->dma_address; 1419 txd->ccfg |= PL080_FLOW_MEM2PER << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1420 txd->cctl |= PL080_CONTROL_SRC_INCR;
1421 txd->src_addr = sgl->dma_address;
1491 if (plchan->runtime_addr) 1422 if (plchan->runtime_addr)
1492 txd->dstbus.addr = plchan->runtime_addr; 1423 txd->dst_addr = plchan->runtime_addr;
1493 else 1424 else
1494 txd->dstbus.addr = plchan->cd->addr; 1425 txd->dst_addr = plchan->cd->addr;
1426 src_buses = pl08x->mem_buses;
1427 dst_buses = plchan->cd->periph_buses;
1495 } else if (direction == DMA_FROM_DEVICE) { 1428 } else if (direction == DMA_FROM_DEVICE) {
1429 txd->ccfg |= PL080_FLOW_PER2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1430 txd->cctl |= PL080_CONTROL_DST_INCR;
1496 if (plchan->runtime_addr) 1431 if (plchan->runtime_addr)
1497 txd->srcbus.addr = plchan->runtime_addr; 1432 txd->src_addr = plchan->runtime_addr;
1498 else 1433 else
1499 txd->srcbus.addr = plchan->cd->addr; 1434 txd->src_addr = plchan->cd->addr;
1500 txd->dstbus.addr = sgl->dma_address; 1435 txd->dst_addr = sgl->dma_address;
1436 src_buses = plchan->cd->periph_buses;
1437 dst_buses = pl08x->mem_buses;
1501 } else { 1438 } else {
1502 dev_err(&pl08x->adev->dev, 1439 dev_err(&pl08x->adev->dev,
1503 "%s direction unsupported\n", __func__); 1440 "%s direction unsupported\n", __func__);
1504 return NULL; 1441 return NULL;
1505 } 1442 }
1506 txd->cd = plchan->cd; 1443
1507 txd->tx.tx_submit = pl08x_tx_submit; 1444 txd->cctl |= pl08x_select_bus(pl08x, src_buses, dst_buses);
1508 txd->tx.callback = NULL;
1509 txd->tx.callback_param = NULL;
1510 txd->len = sgl->length;
1511 INIT_LIST_HEAD(&txd->node);
1512 1445
1513 ret = pl08x_prep_channel_resources(plchan, txd); 1446 ret = pl08x_prep_channel_resources(plchan, txd);
1514 if (ret) 1447 if (ret)
1515 return NULL; 1448 return NULL;
1516 /*
1517 * NB: the channel lock is held at this point so tx_submit()
1518 * must be called in direct succession.
1519 */
1520 1449
1521 return &txd->tx; 1450 return &txd->tx;
1522} 1451}
@@ -1531,10 +1460,8 @@ static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1531 1460
1532 /* Controls applicable to inactive channels */ 1461 /* Controls applicable to inactive channels */
1533 if (cmd == DMA_SLAVE_CONFIG) { 1462 if (cmd == DMA_SLAVE_CONFIG) {
1534 dma_set_runtime_config(chan, 1463 return dma_set_runtime_config(chan,
1535 (struct dma_slave_config *) 1464 (struct dma_slave_config *)arg);
1536 arg);
1537 return 0;
1538 } 1465 }
1539 1466
1540 /* 1467 /*
@@ -1558,16 +1485,8 @@ static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1558 * Mark physical channel as free and free any slave 1485 * Mark physical channel as free and free any slave
1559 * signal 1486 * signal
1560 */ 1487 */
1561 if ((plchan->phychan->signal >= 0) && 1488 release_phy_channel(plchan);
1562 pl08x->pd->put_signal) {
1563 pl08x->pd->put_signal(plchan);
1564 plchan->phychan->signal = -1;
1565 }
1566 pl08x_put_phy_channel(pl08x, plchan->phychan);
1567 plchan->phychan = NULL;
1568 } 1489 }
1569 /* Stop any pending tasklet */
1570 tasklet_disable(&plchan->tasklet);
1571 /* Dequeue jobs and free LLIs */ 1490 /* Dequeue jobs and free LLIs */
1572 if (plchan->at) { 1491 if (plchan->at) {
1573 pl08x_free_txd(pl08x, plchan->at); 1492 pl08x_free_txd(pl08x, plchan->at);
@@ -1620,78 +1539,71 @@ static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
1620 1539
1621 val = readl(pl08x->base + PL080_CONFIG); 1540 val = readl(pl08x->base + PL080_CONFIG);
1622 val &= ~(PL080_CONFIG_M2_BE | PL080_CONFIG_M1_BE | PL080_CONFIG_ENABLE); 1541 val &= ~(PL080_CONFIG_M2_BE | PL080_CONFIG_M1_BE | PL080_CONFIG_ENABLE);
1623 /* We implictly clear bit 1 and that means little-endian mode */ 1542 /* We implicitly clear bit 1 and that means little-endian mode */
1624 val |= PL080_CONFIG_ENABLE; 1543 val |= PL080_CONFIG_ENABLE;
1625 writel(val, pl08x->base + PL080_CONFIG); 1544 writel(val, pl08x->base + PL080_CONFIG);
1626} 1545}
1627 1546
1547static void pl08x_unmap_buffers(struct pl08x_txd *txd)
1548{
1549 struct device *dev = txd->tx.chan->device->dev;
1550
1551 if (!(txd->tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1552 if (txd->tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
1553 dma_unmap_single(dev, txd->src_addr, txd->len,
1554 DMA_TO_DEVICE);
1555 else
1556 dma_unmap_page(dev, txd->src_addr, txd->len,
1557 DMA_TO_DEVICE);
1558 }
1559 if (!(txd->tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1560 if (txd->tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
1561 dma_unmap_single(dev, txd->dst_addr, txd->len,
1562 DMA_FROM_DEVICE);
1563 else
1564 dma_unmap_page(dev, txd->dst_addr, txd->len,
1565 DMA_FROM_DEVICE);
1566 }
1567}
1568
1628static void pl08x_tasklet(unsigned long data) 1569static void pl08x_tasklet(unsigned long data)
1629{ 1570{
1630 struct pl08x_dma_chan *plchan = (struct pl08x_dma_chan *) data; 1571 struct pl08x_dma_chan *plchan = (struct pl08x_dma_chan *) data;
1631 struct pl08x_phy_chan *phychan = plchan->phychan;
1632 struct pl08x_driver_data *pl08x = plchan->host; 1572 struct pl08x_driver_data *pl08x = plchan->host;
1573 struct pl08x_txd *txd;
1574 unsigned long flags;
1633 1575
1634 if (!plchan) 1576 spin_lock_irqsave(&plchan->lock, flags);
1635 BUG();
1636
1637 spin_lock(&plchan->lock);
1638 1577
1639 if (plchan->at) { 1578 txd = plchan->at;
1640 dma_async_tx_callback callback = 1579 plchan->at = NULL;
1641 plchan->at->tx.callback;
1642 void *callback_param =
1643 plchan->at->tx.callback_param;
1644 1580
1581 if (txd) {
1645 /* 1582 /*
1646 * Update last completed 1583 * Update last completed
1647 */ 1584 */
1648 plchan->lc = 1585 plchan->lc = txd->tx.cookie;
1649 (plchan->at->tx.cookie);
1650
1651 /*
1652 * Callback to signal completion
1653 */
1654 if (callback)
1655 callback(callback_param);
1656
1657 /*
1658 * Device callbacks should NOT clear
1659 * the current transaction on the channel
1660 * Linus: sometimes they should?
1661 */
1662 if (!plchan->at)
1663 BUG();
1664
1665 /*
1666 * Free the descriptor if it's not for a device
1667 * using a circular buffer
1668 */
1669 if (!plchan->at->cd->circular_buffer) {
1670 pl08x_free_txd(pl08x, plchan->at);
1671 plchan->at = NULL;
1672 }
1673 /*
1674 * else descriptor for circular
1675 * buffers only freed when
1676 * client has disabled dma
1677 */
1678 } 1586 }
1587
1679 /* 1588 /*
1680 * If a new descriptor is queued, set it up 1589 * If a new descriptor is queued, set it up
1681 * plchan->at is NULL here 1590 * plchan->at is NULL here
1682 */ 1591 */
1683 if (!list_empty(&plchan->desc_list)) { 1592 if (!list_empty(&plchan->pend_list)) {
1684 struct pl08x_txd *next; 1593 struct pl08x_txd *next;
1685 1594
1686 next = list_first_entry(&plchan->desc_list, 1595 next = list_first_entry(&plchan->pend_list,
1687 struct pl08x_txd, 1596 struct pl08x_txd,
1688 node); 1597 node);
1689 list_del(&next->node); 1598 list_del(&next->node);
1690 plchan->at = next; 1599
1691 /* Configure the physical channel for the next txd */ 1600 pl08x_start_txd(plchan, next);
1692 pl08x_config_phychan_for_txd(plchan); 1601 } else if (plchan->phychan_hold) {
1693 pl08x_set_cregs(pl08x, plchan->phychan); 1602 /*
1694 pl08x_enable_phy_chan(pl08x, plchan->phychan); 1603 * This channel is still in use - we have a new txd being
1604 * prepared and will soon be queued. Don't give up the
1605 * physical channel.
1606 */
1695 } else { 1607 } else {
1696 struct pl08x_dma_chan *waiting = NULL; 1608 struct pl08x_dma_chan *waiting = NULL;
1697 1609
@@ -1699,12 +1611,7 @@ static void pl08x_tasklet(unsigned long data)
1699 * No more jobs, so free up the physical channel 1611 * No more jobs, so free up the physical channel
1700 * Free any allocated signal on slave transfers too 1612 * Free any allocated signal on slave transfers too
1701 */ 1613 */
1702 if ((phychan->signal >= 0) && pl08x->pd->put_signal) { 1614 release_phy_channel(plchan);
1703 pl08x->pd->put_signal(plchan);
1704 phychan->signal = -1;
1705 }
1706 pl08x_put_phy_channel(pl08x, phychan);
1707 plchan->phychan = NULL;
1708 plchan->state = PL08X_CHAN_IDLE; 1615 plchan->state = PL08X_CHAN_IDLE;
1709 1616
1710 /* 1617 /*
@@ -1724,6 +1631,7 @@ static void pl08x_tasklet(unsigned long data)
1724 ret = prep_phy_channel(waiting, 1631 ret = prep_phy_channel(waiting,
1725 waiting->waiting); 1632 waiting->waiting);
1726 BUG_ON(ret); 1633 BUG_ON(ret);
1634 waiting->phychan_hold--;
1727 waiting->state = PL08X_CHAN_RUNNING; 1635 waiting->state = PL08X_CHAN_RUNNING;
1728 waiting->waiting = NULL; 1636 waiting->waiting = NULL;
1729 pl08x_issue_pending(&waiting->chan); 1637 pl08x_issue_pending(&waiting->chan);
@@ -1732,7 +1640,25 @@ static void pl08x_tasklet(unsigned long data)
1732 } 1640 }
1733 } 1641 }
1734 1642
1735 spin_unlock(&plchan->lock); 1643 spin_unlock_irqrestore(&plchan->lock, flags);
1644
1645 if (txd) {
1646 dma_async_tx_callback callback = txd->tx.callback;
1647 void *callback_param = txd->tx.callback_param;
1648
1649 /* Don't try to unmap buffers on slave channels */
1650 if (!plchan->slave)
1651 pl08x_unmap_buffers(txd);
1652
1653 /* Free the descriptor */
1654 spin_lock_irqsave(&plchan->lock, flags);
1655 pl08x_free_txd(pl08x, txd);
1656 spin_unlock_irqrestore(&plchan->lock, flags);
1657
1658 /* Callback to signal completion */
1659 if (callback)
1660 callback(callback_param);
1661 }
1736} 1662}
1737 1663
1738static irqreturn_t pl08x_irq(int irq, void *dev) 1664static irqreturn_t pl08x_irq(int irq, void *dev)
@@ -1819,16 +1745,23 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
1819 return -ENOMEM; 1745 return -ENOMEM;
1820 } 1746 }
1821 } 1747 }
1748 if (chan->cd->circular_buffer) {
1749 dev_err(&pl08x->adev->dev,
1750 "channel %s: circular buffers not supported\n",
1751 chan->name);
1752 kfree(chan);
1753 continue;
1754 }
1822 dev_info(&pl08x->adev->dev, 1755 dev_info(&pl08x->adev->dev,
1823 "initialize virtual channel \"%s\"\n", 1756 "initialize virtual channel \"%s\"\n",
1824 chan->name); 1757 chan->name);
1825 1758
1826 chan->chan.device = dmadev; 1759 chan->chan.device = dmadev;
1827 atomic_set(&chan->last_issued, 0); 1760 chan->chan.cookie = 0;
1828 chan->lc = atomic_read(&chan->last_issued); 1761 chan->lc = 0;
1829 1762
1830 spin_lock_init(&chan->lock); 1763 spin_lock_init(&chan->lock);
1831 INIT_LIST_HEAD(&chan->desc_list); 1764 INIT_LIST_HEAD(&chan->pend_list);
1832 tasklet_init(&chan->tasklet, pl08x_tasklet, 1765 tasklet_init(&chan->tasklet, pl08x_tasklet,
1833 (unsigned long) chan); 1766 (unsigned long) chan);
1834 1767
@@ -1898,7 +1831,7 @@ static int pl08x_debugfs_show(struct seq_file *s, void *data)
1898 seq_printf(s, "CHANNEL:\tSTATE:\n"); 1831 seq_printf(s, "CHANNEL:\tSTATE:\n");
1899 seq_printf(s, "--------\t------\n"); 1832 seq_printf(s, "--------\t------\n");
1900 list_for_each_entry(chan, &pl08x->memcpy.channels, chan.device_node) { 1833 list_for_each_entry(chan, &pl08x->memcpy.channels, chan.device_node) {
1901 seq_printf(s, "%s\t\t\%s\n", chan->name, 1834 seq_printf(s, "%s\t\t%s\n", chan->name,
1902 pl08x_state_str(chan->state)); 1835 pl08x_state_str(chan->state));
1903 } 1836 }
1904 1837
@@ -1906,7 +1839,7 @@ static int pl08x_debugfs_show(struct seq_file *s, void *data)
1906 seq_printf(s, "CHANNEL:\tSTATE:\n"); 1839 seq_printf(s, "CHANNEL:\tSTATE:\n");
1907 seq_printf(s, "--------\t------\n"); 1840 seq_printf(s, "--------\t------\n");
1908 list_for_each_entry(chan, &pl08x->slave.channels, chan.device_node) { 1841 list_for_each_entry(chan, &pl08x->slave.channels, chan.device_node) {
1909 seq_printf(s, "%s\t\t\%s\n", chan->name, 1842 seq_printf(s, "%s\t\t%s\n", chan->name,
1910 pl08x_state_str(chan->state)); 1843 pl08x_state_str(chan->state));
1911 } 1844 }
1912 1845
@@ -1942,7 +1875,7 @@ static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
1942static int pl08x_probe(struct amba_device *adev, struct amba_id *id) 1875static int pl08x_probe(struct amba_device *adev, struct amba_id *id)
1943{ 1876{
1944 struct pl08x_driver_data *pl08x; 1877 struct pl08x_driver_data *pl08x;
1945 struct vendor_data *vd = id->data; 1878 const struct vendor_data *vd = id->data;
1946 int ret = 0; 1879 int ret = 0;
1947 int i; 1880 int i;
1948 1881
@@ -1990,6 +1923,14 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id)
1990 pl08x->adev = adev; 1923 pl08x->adev = adev;
1991 pl08x->vd = vd; 1924 pl08x->vd = vd;
1992 1925
1926 /* By default, AHB1 only. If dualmaster, from platform */
1927 pl08x->lli_buses = PL08X_AHB1;
1928 pl08x->mem_buses = PL08X_AHB1;
1929 if (pl08x->vd->dualmaster) {
1930 pl08x->lli_buses = pl08x->pd->lli_buses;
1931 pl08x->mem_buses = pl08x->pd->mem_buses;
1932 }
1933
1993 /* A DMA memory pool for LLIs, align on 1-byte boundary */ 1934 /* A DMA memory pool for LLIs, align on 1-byte boundary */
1994 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev, 1935 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,
1995 PL08X_LLI_TSFR_SIZE, PL08X_ALIGN, 0); 1936 PL08X_LLI_TSFR_SIZE, PL08X_ALIGN, 0);
@@ -2016,7 +1957,7 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id)
2016 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); 1957 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);
2017 1958
2018 ret = request_irq(adev->irq[0], pl08x_irq, IRQF_DISABLED, 1959 ret = request_irq(adev->irq[0], pl08x_irq, IRQF_DISABLED,
2019 vd->name, pl08x); 1960 DRIVER_NAME, pl08x);
2020 if (ret) { 1961 if (ret) {
2021 dev_err(&adev->dev, "%s failed to request interrupt %d\n", 1962 dev_err(&adev->dev, "%s failed to request interrupt %d\n",
2022 __func__, adev->irq[0]); 1963 __func__, adev->irq[0]);
@@ -2087,8 +2028,9 @@ static int pl08x_probe(struct amba_device *adev, struct amba_id *id)
2087 2028
2088 amba_set_drvdata(adev, pl08x); 2029 amba_set_drvdata(adev, pl08x);
2089 init_pl08x_debugfs(pl08x); 2030 init_pl08x_debugfs(pl08x);
2090 dev_info(&pl08x->adev->dev, "ARM(R) %s DMA block initialized @%08x\n", 2031 dev_info(&pl08x->adev->dev, "DMA: PL%03x rev%u at 0x%08llx irq %d\n",
2091 vd->name, adev->res.start); 2032 amba_part(adev), amba_rev(adev),
2033 (unsigned long long)adev->res.start, adev->irq[0]);
2092 return 0; 2034 return 0;
2093 2035
2094out_no_slave_reg: 2036out_no_slave_reg:
@@ -2115,13 +2057,11 @@ out_no_pl08x:
2115 2057
2116/* PL080 has 8 channels and the PL080 have just 2 */ 2058/* PL080 has 8 channels and the PL080 have just 2 */
2117static struct vendor_data vendor_pl080 = { 2059static struct vendor_data vendor_pl080 = {
2118 .name = "PL080",
2119 .channels = 8, 2060 .channels = 8,
2120 .dualmaster = true, 2061 .dualmaster = true,
2121}; 2062};
2122 2063
2123static struct vendor_data vendor_pl081 = { 2064static struct vendor_data vendor_pl081 = {
2124 .name = "PL081",
2125 .channels = 2, 2065 .channels = 2,
2126 .dualmaster = false, 2066 .dualmaster = false,
2127}; 2067};
@@ -2160,7 +2100,7 @@ static int __init pl08x_init(void)
2160 retval = amba_driver_register(&pl08x_amba_driver); 2100 retval = amba_driver_register(&pl08x_amba_driver);
2161 if (retval) 2101 if (retval)
2162 printk(KERN_WARNING DRIVER_NAME 2102 printk(KERN_WARNING DRIVER_NAME
2163 "failed to register as an amba device (%d)\n", 2103 "failed to register as an AMBA device (%d)\n",
2164 retval); 2104 retval);
2165 return retval; 2105 return retval;
2166} 2106}
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index fab68a553205..6e1d46a65d0e 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -1,5 +1,6 @@
1/* 1/*
2 * Copyright (C) ST-Ericsson SA 2007-2010 2 * Copyright (C) Ericsson AB 2007-2008
3 * Copyright (C) ST-Ericsson SA 2008-2010
3 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson 4 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
4 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson 5 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
5 * License terms: GNU General Public License (GPL) version 2 6 * License terms: GNU General Public License (GPL) version 2
@@ -554,8 +555,66 @@ static struct d40_desc *d40_last_queued(struct d40_chan *d40c)
554 return d; 555 return d;
555} 556}
556 557
557/* Support functions for logical channels */ 558static int d40_psize_2_burst_size(bool is_log, int psize)
559{
560 if (is_log) {
561 if (psize == STEDMA40_PSIZE_LOG_1)
562 return 1;
563 } else {
564 if (psize == STEDMA40_PSIZE_PHY_1)
565 return 1;
566 }
567
568 return 2 << psize;
569}
570
571/*
572 * The dma only supports transmitting packages up to
573 * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
574 * dma elements required to send the entire sg list
575 */
576static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
577{
578 int dmalen;
579 u32 max_w = max(data_width1, data_width2);
580 u32 min_w = min(data_width1, data_width2);
581 u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
582
583 if (seg_max > STEDMA40_MAX_SEG_SIZE)
584 seg_max -= (1 << max_w);
585
586 if (!IS_ALIGNED(size, 1 << max_w))
587 return -EINVAL;
588
589 if (size <= seg_max)
590 dmalen = 1;
591 else {
592 dmalen = size / seg_max;
593 if (dmalen * seg_max < size)
594 dmalen++;
595 }
596 return dmalen;
597}
598
599static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
600 u32 data_width1, u32 data_width2)
601{
602 struct scatterlist *sg;
603 int i;
604 int len = 0;
605 int ret;
606
607 for_each_sg(sgl, sg, sg_len, i) {
608 ret = d40_size_2_dmalen(sg_dma_len(sg),
609 data_width1, data_width2);
610 if (ret < 0)
611 return ret;
612 len += ret;
613 }
614 return len;
615}
558 616
617/* Support functions for logical channels */
559 618
560static int d40_channel_execute_command(struct d40_chan *d40c, 619static int d40_channel_execute_command(struct d40_chan *d40c,
561 enum d40_command command) 620 enum d40_command command)
@@ -1241,6 +1300,21 @@ static int d40_validate_conf(struct d40_chan *d40c,
1241 res = -EINVAL; 1300 res = -EINVAL;
1242 } 1301 }
1243 1302
1303 if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1304 (1 << conf->src_info.data_width) !=
1305 d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1306 (1 << conf->dst_info.data_width)) {
1307 /*
1308 * The DMAC hardware only supports
1309 * src (burst x width) == dst (burst x width)
1310 */
1311
1312 dev_err(&d40c->chan.dev->device,
1313 "[%s] src (burst x width) != dst (burst x width)\n",
1314 __func__);
1315 res = -EINVAL;
1316 }
1317
1244 return res; 1318 return res;
1245} 1319}
1246 1320
@@ -1638,13 +1712,21 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1638 if (d40d == NULL) 1712 if (d40d == NULL)
1639 goto err; 1713 goto err;
1640 1714
1641 d40d->lli_len = sgl_len; 1715 d40d->lli_len = d40_sg_2_dmalen(sgl_dst, sgl_len,
1716 d40c->dma_cfg.src_info.data_width,
1717 d40c->dma_cfg.dst_info.data_width);
1718 if (d40d->lli_len < 0) {
1719 dev_err(&d40c->chan.dev->device,
1720 "[%s] Unaligned size\n", __func__);
1721 goto err;
1722 }
1723
1642 d40d->lli_current = 0; 1724 d40d->lli_current = 0;
1643 d40d->txd.flags = dma_flags; 1725 d40d->txd.flags = dma_flags;
1644 1726
1645 if (d40c->log_num != D40_PHY_CHAN) { 1727 if (d40c->log_num != D40_PHY_CHAN) {
1646 1728
1647 if (d40_pool_lli_alloc(d40d, sgl_len, true) < 0) { 1729 if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
1648 dev_err(&d40c->chan.dev->device, 1730 dev_err(&d40c->chan.dev->device,
1649 "[%s] Out of memory\n", __func__); 1731 "[%s] Out of memory\n", __func__);
1650 goto err; 1732 goto err;
@@ -1654,15 +1736,17 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1654 sgl_len, 1736 sgl_len,
1655 d40d->lli_log.src, 1737 d40d->lli_log.src,
1656 d40c->log_def.lcsp1, 1738 d40c->log_def.lcsp1,
1657 d40c->dma_cfg.src_info.data_width); 1739 d40c->dma_cfg.src_info.data_width,
1740 d40c->dma_cfg.dst_info.data_width);
1658 1741
1659 (void) d40_log_sg_to_lli(sgl_dst, 1742 (void) d40_log_sg_to_lli(sgl_dst,
1660 sgl_len, 1743 sgl_len,
1661 d40d->lli_log.dst, 1744 d40d->lli_log.dst,
1662 d40c->log_def.lcsp3, 1745 d40c->log_def.lcsp3,
1663 d40c->dma_cfg.dst_info.data_width); 1746 d40c->dma_cfg.dst_info.data_width,
1747 d40c->dma_cfg.src_info.data_width);
1664 } else { 1748 } else {
1665 if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) { 1749 if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
1666 dev_err(&d40c->chan.dev->device, 1750 dev_err(&d40c->chan.dev->device,
1667 "[%s] Out of memory\n", __func__); 1751 "[%s] Out of memory\n", __func__);
1668 goto err; 1752 goto err;
@@ -1675,6 +1759,7 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1675 virt_to_phys(d40d->lli_phy.src), 1759 virt_to_phys(d40d->lli_phy.src),
1676 d40c->src_def_cfg, 1760 d40c->src_def_cfg,
1677 d40c->dma_cfg.src_info.data_width, 1761 d40c->dma_cfg.src_info.data_width,
1762 d40c->dma_cfg.dst_info.data_width,
1678 d40c->dma_cfg.src_info.psize); 1763 d40c->dma_cfg.src_info.psize);
1679 1764
1680 if (res < 0) 1765 if (res < 0)
@@ -1687,6 +1772,7 @@ struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1687 virt_to_phys(d40d->lli_phy.dst), 1772 virt_to_phys(d40d->lli_phy.dst),
1688 d40c->dst_def_cfg, 1773 d40c->dst_def_cfg,
1689 d40c->dma_cfg.dst_info.data_width, 1774 d40c->dma_cfg.dst_info.data_width,
1775 d40c->dma_cfg.src_info.data_width,
1690 d40c->dma_cfg.dst_info.psize); 1776 d40c->dma_cfg.dst_info.psize);
1691 1777
1692 if (res < 0) 1778 if (res < 0)
@@ -1826,7 +1912,6 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1826 struct d40_chan *d40c = container_of(chan, struct d40_chan, 1912 struct d40_chan *d40c = container_of(chan, struct d40_chan,
1827 chan); 1913 chan);
1828 unsigned long flags; 1914 unsigned long flags;
1829 int err = 0;
1830 1915
1831 if (d40c->phy_chan == NULL) { 1916 if (d40c->phy_chan == NULL) {
1832 dev_err(&d40c->chan.dev->device, 1917 dev_err(&d40c->chan.dev->device,
@@ -1844,6 +1929,15 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1844 } 1929 }
1845 1930
1846 d40d->txd.flags = dma_flags; 1931 d40d->txd.flags = dma_flags;
1932 d40d->lli_len = d40_size_2_dmalen(size,
1933 d40c->dma_cfg.src_info.data_width,
1934 d40c->dma_cfg.dst_info.data_width);
1935 if (d40d->lli_len < 0) {
1936 dev_err(&d40c->chan.dev->device,
1937 "[%s] Unaligned size\n", __func__);
1938 goto err;
1939 }
1940
1847 1941
1848 dma_async_tx_descriptor_init(&d40d->txd, chan); 1942 dma_async_tx_descriptor_init(&d40d->txd, chan);
1849 1943
@@ -1851,37 +1945,40 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1851 1945
1852 if (d40c->log_num != D40_PHY_CHAN) { 1946 if (d40c->log_num != D40_PHY_CHAN) {
1853 1947
1854 if (d40_pool_lli_alloc(d40d, 1, true) < 0) { 1948 if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
1855 dev_err(&d40c->chan.dev->device, 1949 dev_err(&d40c->chan.dev->device,
1856 "[%s] Out of memory\n", __func__); 1950 "[%s] Out of memory\n", __func__);
1857 goto err; 1951 goto err;
1858 } 1952 }
1859 d40d->lli_len = 1;
1860 d40d->lli_current = 0; 1953 d40d->lli_current = 0;
1861 1954
1862 d40_log_fill_lli(d40d->lli_log.src, 1955 if (d40_log_buf_to_lli(d40d->lli_log.src,
1863 src, 1956 src,
1864 size, 1957 size,
1865 d40c->log_def.lcsp1, 1958 d40c->log_def.lcsp1,
1866 d40c->dma_cfg.src_info.data_width, 1959 d40c->dma_cfg.src_info.data_width,
1867 true); 1960 d40c->dma_cfg.dst_info.data_width,
1961 true) == NULL)
1962 goto err;
1868 1963
1869 d40_log_fill_lli(d40d->lli_log.dst, 1964 if (d40_log_buf_to_lli(d40d->lli_log.dst,
1870 dst, 1965 dst,
1871 size, 1966 size,
1872 d40c->log_def.lcsp3, 1967 d40c->log_def.lcsp3,
1873 d40c->dma_cfg.dst_info.data_width, 1968 d40c->dma_cfg.dst_info.data_width,
1874 true); 1969 d40c->dma_cfg.src_info.data_width,
1970 true) == NULL)
1971 goto err;
1875 1972
1876 } else { 1973 } else {
1877 1974
1878 if (d40_pool_lli_alloc(d40d, 1, false) < 0) { 1975 if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
1879 dev_err(&d40c->chan.dev->device, 1976 dev_err(&d40c->chan.dev->device,
1880 "[%s] Out of memory\n", __func__); 1977 "[%s] Out of memory\n", __func__);
1881 goto err; 1978 goto err;
1882 } 1979 }
1883 1980
1884 err = d40_phy_fill_lli(d40d->lli_phy.src, 1981 if (d40_phy_buf_to_lli(d40d->lli_phy.src,
1885 src, 1982 src,
1886 size, 1983 size,
1887 d40c->dma_cfg.src_info.psize, 1984 d40c->dma_cfg.src_info.psize,
@@ -1889,11 +1986,11 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1889 d40c->src_def_cfg, 1986 d40c->src_def_cfg,
1890 true, 1987 true,
1891 d40c->dma_cfg.src_info.data_width, 1988 d40c->dma_cfg.src_info.data_width,
1892 false); 1989 d40c->dma_cfg.dst_info.data_width,
1893 if (err) 1990 false) == NULL)
1894 goto err_fill_lli; 1991 goto err;
1895 1992
1896 err = d40_phy_fill_lli(d40d->lli_phy.dst, 1993 if (d40_phy_buf_to_lli(d40d->lli_phy.dst,
1897 dst, 1994 dst,
1898 size, 1995 size,
1899 d40c->dma_cfg.dst_info.psize, 1996 d40c->dma_cfg.dst_info.psize,
@@ -1901,10 +1998,9 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1901 d40c->dst_def_cfg, 1998 d40c->dst_def_cfg,
1902 true, 1999 true,
1903 d40c->dma_cfg.dst_info.data_width, 2000 d40c->dma_cfg.dst_info.data_width,
1904 false); 2001 d40c->dma_cfg.src_info.data_width,
1905 2002 false) == NULL)
1906 if (err) 2003 goto err;
1907 goto err_fill_lli;
1908 2004
1909 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src, 2005 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
1910 d40d->lli_pool.size, DMA_TO_DEVICE); 2006 d40d->lli_pool.size, DMA_TO_DEVICE);
@@ -1913,9 +2009,6 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1913 spin_unlock_irqrestore(&d40c->lock, flags); 2009 spin_unlock_irqrestore(&d40c->lock, flags);
1914 return &d40d->txd; 2010 return &d40d->txd;
1915 2011
1916err_fill_lli:
1917 dev_err(&d40c->chan.dev->device,
1918 "[%s] Failed filling in PHY LLI\n", __func__);
1919err: 2012err:
1920 if (d40d) 2013 if (d40d)
1921 d40_desc_free(d40c, d40d); 2014 d40_desc_free(d40c, d40d);
@@ -1945,13 +2038,21 @@ static int d40_prep_slave_sg_log(struct d40_desc *d40d,
1945 dma_addr_t dev_addr = 0; 2038 dma_addr_t dev_addr = 0;
1946 int total_size; 2039 int total_size;
1947 2040
1948 if (d40_pool_lli_alloc(d40d, sg_len, true) < 0) { 2041 d40d->lli_len = d40_sg_2_dmalen(sgl, sg_len,
2042 d40c->dma_cfg.src_info.data_width,
2043 d40c->dma_cfg.dst_info.data_width);
2044 if (d40d->lli_len < 0) {
2045 dev_err(&d40c->chan.dev->device,
2046 "[%s] Unaligned size\n", __func__);
2047 return -EINVAL;
2048 }
2049
2050 if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
1949 dev_err(&d40c->chan.dev->device, 2051 dev_err(&d40c->chan.dev->device,
1950 "[%s] Out of memory\n", __func__); 2052 "[%s] Out of memory\n", __func__);
1951 return -ENOMEM; 2053 return -ENOMEM;
1952 } 2054 }
1953 2055
1954 d40d->lli_len = sg_len;
1955 d40d->lli_current = 0; 2056 d40d->lli_current = 0;
1956 2057
1957 if (direction == DMA_FROM_DEVICE) 2058 if (direction == DMA_FROM_DEVICE)
@@ -1993,13 +2094,21 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
1993 dma_addr_t dst_dev_addr; 2094 dma_addr_t dst_dev_addr;
1994 int res; 2095 int res;
1995 2096
1996 if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) { 2097 d40d->lli_len = d40_sg_2_dmalen(sgl, sgl_len,
2098 d40c->dma_cfg.src_info.data_width,
2099 d40c->dma_cfg.dst_info.data_width);
2100 if (d40d->lli_len < 0) {
2101 dev_err(&d40c->chan.dev->device,
2102 "[%s] Unaligned size\n", __func__);
2103 return -EINVAL;
2104 }
2105
2106 if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
1997 dev_err(&d40c->chan.dev->device, 2107 dev_err(&d40c->chan.dev->device,
1998 "[%s] Out of memory\n", __func__); 2108 "[%s] Out of memory\n", __func__);
1999 return -ENOMEM; 2109 return -ENOMEM;
2000 } 2110 }
2001 2111
2002 d40d->lli_len = sgl_len;
2003 d40d->lli_current = 0; 2112 d40d->lli_current = 0;
2004 2113
2005 if (direction == DMA_FROM_DEVICE) { 2114 if (direction == DMA_FROM_DEVICE) {
@@ -2024,6 +2133,7 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
2024 virt_to_phys(d40d->lli_phy.src), 2133 virt_to_phys(d40d->lli_phy.src),
2025 d40c->src_def_cfg, 2134 d40c->src_def_cfg,
2026 d40c->dma_cfg.src_info.data_width, 2135 d40c->dma_cfg.src_info.data_width,
2136 d40c->dma_cfg.dst_info.data_width,
2027 d40c->dma_cfg.src_info.psize); 2137 d40c->dma_cfg.src_info.psize);
2028 if (res < 0) 2138 if (res < 0)
2029 return res; 2139 return res;
@@ -2035,6 +2145,7 @@ static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
2035 virt_to_phys(d40d->lli_phy.dst), 2145 virt_to_phys(d40d->lli_phy.dst),
2036 d40c->dst_def_cfg, 2146 d40c->dst_def_cfg,
2037 d40c->dma_cfg.dst_info.data_width, 2147 d40c->dma_cfg.dst_info.data_width,
2148 d40c->dma_cfg.src_info.data_width,
2038 d40c->dma_cfg.dst_info.psize); 2149 d40c->dma_cfg.dst_info.psize);
2039 if (res < 0) 2150 if (res < 0)
2040 return res; 2151 return res;
@@ -2244,6 +2355,8 @@ static void d40_set_runtime_config(struct dma_chan *chan,
2244 psize = STEDMA40_PSIZE_PHY_8; 2355 psize = STEDMA40_PSIZE_PHY_8;
2245 else if (config_maxburst >= 4) 2356 else if (config_maxburst >= 4)
2246 psize = STEDMA40_PSIZE_PHY_4; 2357 psize = STEDMA40_PSIZE_PHY_4;
2358 else if (config_maxburst >= 2)
2359 psize = STEDMA40_PSIZE_PHY_2;
2247 else 2360 else
2248 psize = STEDMA40_PSIZE_PHY_1; 2361 psize = STEDMA40_PSIZE_PHY_1;
2249 } 2362 }
diff --git a/drivers/dma/ste_dma40_ll.c b/drivers/dma/ste_dma40_ll.c
index 8557cb88b255..0b096a38322d 100644
--- a/drivers/dma/ste_dma40_ll.c
+++ b/drivers/dma/ste_dma40_ll.c
@@ -1,6 +1,6 @@
1/* 1/*
2 * Copyright (C) ST-Ericsson SA 2007-2010 2 * Copyright (C) ST-Ericsson SA 2007-2010
3 * Author: Per Friden <per.friden@stericsson.com> for ST-Ericsson 3 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
4 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson 4 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
5 * License terms: GNU General Public License (GPL) version 2 5 * License terms: GNU General Public License (GPL) version 2
6 */ 6 */
@@ -122,15 +122,15 @@ void d40_phy_cfg(struct stedma40_chan_cfg *cfg,
122 *dst_cfg = dst; 122 *dst_cfg = dst;
123} 123}
124 124
125int d40_phy_fill_lli(struct d40_phy_lli *lli, 125static int d40_phy_fill_lli(struct d40_phy_lli *lli,
126 dma_addr_t data, 126 dma_addr_t data,
127 u32 data_size, 127 u32 data_size,
128 int psize, 128 int psize,
129 dma_addr_t next_lli, 129 dma_addr_t next_lli,
130 u32 reg_cfg, 130 u32 reg_cfg,
131 bool term_int, 131 bool term_int,
132 u32 data_width, 132 u32 data_width,
133 bool is_device) 133 bool is_device)
134{ 134{
135 int num_elems; 135 int num_elems;
136 136
@@ -139,13 +139,6 @@ int d40_phy_fill_lli(struct d40_phy_lli *lli,
139 else 139 else
140 num_elems = 2 << psize; 140 num_elems = 2 << psize;
141 141
142 /*
143 * Size is 16bit. data_width is 8, 16, 32 or 64 bit
144 * Block large than 64 KiB must be split.
145 */
146 if (data_size > (0xffff << data_width))
147 return -EINVAL;
148
149 /* Must be aligned */ 142 /* Must be aligned */
150 if (!IS_ALIGNED(data, 0x1 << data_width)) 143 if (!IS_ALIGNED(data, 0x1 << data_width))
151 return -EINVAL; 144 return -EINVAL;
@@ -187,55 +180,118 @@ int d40_phy_fill_lli(struct d40_phy_lli *lli,
187 return 0; 180 return 0;
188} 181}
189 182
183static int d40_seg_size(int size, int data_width1, int data_width2)
184{
185 u32 max_w = max(data_width1, data_width2);
186 u32 min_w = min(data_width1, data_width2);
187 u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
188
189 if (seg_max > STEDMA40_MAX_SEG_SIZE)
190 seg_max -= (1 << max_w);
191
192 if (size <= seg_max)
193 return size;
194
195 if (size <= 2 * seg_max)
196 return ALIGN(size / 2, 1 << max_w);
197
198 return seg_max;
199}
200
201struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli,
202 dma_addr_t addr,
203 u32 size,
204 int psize,
205 dma_addr_t lli_phys,
206 u32 reg_cfg,
207 bool term_int,
208 u32 data_width1,
209 u32 data_width2,
210 bool is_device)
211{
212 int err;
213 dma_addr_t next = lli_phys;
214 int size_rest = size;
215 int size_seg = 0;
216
217 do {
218 size_seg = d40_seg_size(size_rest, data_width1, data_width2);
219 size_rest -= size_seg;
220
221 if (term_int && size_rest == 0)
222 next = 0;
223 else
224 next = ALIGN(next + sizeof(struct d40_phy_lli),
225 D40_LLI_ALIGN);
226
227 err = d40_phy_fill_lli(lli,
228 addr,
229 size_seg,
230 psize,
231 next,
232 reg_cfg,
233 !next,
234 data_width1,
235 is_device);
236
237 if (err)
238 goto err;
239
240 lli++;
241 if (!is_device)
242 addr += size_seg;
243 } while (size_rest);
244
245 return lli;
246
247 err:
248 return NULL;
249}
250
190int d40_phy_sg_to_lli(struct scatterlist *sg, 251int d40_phy_sg_to_lli(struct scatterlist *sg,
191 int sg_len, 252 int sg_len,
192 dma_addr_t target, 253 dma_addr_t target,
193 struct d40_phy_lli *lli, 254 struct d40_phy_lli *lli_sg,
194 dma_addr_t lli_phys, 255 dma_addr_t lli_phys,
195 u32 reg_cfg, 256 u32 reg_cfg,
196 u32 data_width, 257 u32 data_width1,
258 u32 data_width2,
197 int psize) 259 int psize)
198{ 260{
199 int total_size = 0; 261 int total_size = 0;
200 int i; 262 int i;
201 struct scatterlist *current_sg = sg; 263 struct scatterlist *current_sg = sg;
202 dma_addr_t next_lli_phys;
203 dma_addr_t dst; 264 dma_addr_t dst;
204 int err = 0; 265 struct d40_phy_lli *lli = lli_sg;
266 dma_addr_t l_phys = lli_phys;
205 267
206 for_each_sg(sg, current_sg, sg_len, i) { 268 for_each_sg(sg, current_sg, sg_len, i) {
207 269
208 total_size += sg_dma_len(current_sg); 270 total_size += sg_dma_len(current_sg);
209 271
210 /* If this scatter list entry is the last one, no next link */
211 if (sg_len - 1 == i)
212 next_lli_phys = 0;
213 else
214 next_lli_phys = ALIGN(lli_phys + (i + 1) *
215 sizeof(struct d40_phy_lli),
216 D40_LLI_ALIGN);
217
218 if (target) 272 if (target)
219 dst = target; 273 dst = target;
220 else 274 else
221 dst = sg_phys(current_sg); 275 dst = sg_phys(current_sg);
222 276
223 err = d40_phy_fill_lli(&lli[i], 277 l_phys = ALIGN(lli_phys + (lli - lli_sg) *
224 dst, 278 sizeof(struct d40_phy_lli), D40_LLI_ALIGN);
225 sg_dma_len(current_sg), 279
226 psize, 280 lli = d40_phy_buf_to_lli(lli,
227 next_lli_phys, 281 dst,
228 reg_cfg, 282 sg_dma_len(current_sg),
229 !next_lli_phys, 283 psize,
230 data_width, 284 l_phys,
231 target == dst); 285 reg_cfg,
232 if (err) 286 sg_len - 1 == i,
233 goto err; 287 data_width1,
288 data_width2,
289 target == dst);
290 if (lli == NULL)
291 return -EINVAL;
234 } 292 }
235 293
236 return total_size; 294 return total_size;
237err:
238 return err;
239} 295}
240 296
241 297
@@ -315,17 +371,20 @@ void d40_log_lli_lcla_write(struct d40_log_lli *lcla,
315 writel(lli_dst->lcsp13, &lcla[1].lcsp13); 371 writel(lli_dst->lcsp13, &lcla[1].lcsp13);
316} 372}
317 373
318void d40_log_fill_lli(struct d40_log_lli *lli, 374static void d40_log_fill_lli(struct d40_log_lli *lli,
319 dma_addr_t data, u32 data_size, 375 dma_addr_t data, u32 data_size,
320 u32 reg_cfg, 376 u32 reg_cfg,
321 u32 data_width, 377 u32 data_width,
322 bool addr_inc) 378 bool addr_inc)
323{ 379{
324 lli->lcsp13 = reg_cfg; 380 lli->lcsp13 = reg_cfg;
325 381
326 /* The number of elements to transfer */ 382 /* The number of elements to transfer */
327 lli->lcsp02 = ((data_size >> data_width) << 383 lli->lcsp02 = ((data_size >> data_width) <<
328 D40_MEM_LCSP0_ECNT_POS) & D40_MEM_LCSP0_ECNT_MASK; 384 D40_MEM_LCSP0_ECNT_POS) & D40_MEM_LCSP0_ECNT_MASK;
385
386 BUG_ON((data_size >> data_width) > STEDMA40_MAX_SEG_SIZE);
387
329 /* 16 LSBs address of the current element */ 388 /* 16 LSBs address of the current element */
330 lli->lcsp02 |= data & D40_MEM_LCSP0_SPTR_MASK; 389 lli->lcsp02 |= data & D40_MEM_LCSP0_SPTR_MASK;
331 /* 16 MSBs address of the current element */ 390 /* 16 MSBs address of the current element */
@@ -348,55 +407,94 @@ int d40_log_sg_to_dev(struct scatterlist *sg,
348 int total_size = 0; 407 int total_size = 0;
349 struct scatterlist *current_sg = sg; 408 struct scatterlist *current_sg = sg;
350 int i; 409 int i;
410 struct d40_log_lli *lli_src = lli->src;
411 struct d40_log_lli *lli_dst = lli->dst;
351 412
352 for_each_sg(sg, current_sg, sg_len, i) { 413 for_each_sg(sg, current_sg, sg_len, i) {
353 total_size += sg_dma_len(current_sg); 414 total_size += sg_dma_len(current_sg);
354 415
355 if (direction == DMA_TO_DEVICE) { 416 if (direction == DMA_TO_DEVICE) {
356 d40_log_fill_lli(&lli->src[i], 417 lli_src =
357 sg_phys(current_sg), 418 d40_log_buf_to_lli(lli_src,
358 sg_dma_len(current_sg), 419 sg_phys(current_sg),
359 lcsp->lcsp1, src_data_width, 420 sg_dma_len(current_sg),
360 true); 421 lcsp->lcsp1, src_data_width,
361 d40_log_fill_lli(&lli->dst[i], 422 dst_data_width,
362 dev_addr, 423 true);
363 sg_dma_len(current_sg), 424 lli_dst =
364 lcsp->lcsp3, dst_data_width, 425 d40_log_buf_to_lli(lli_dst,
365 false); 426 dev_addr,
427 sg_dma_len(current_sg),
428 lcsp->lcsp3, dst_data_width,
429 src_data_width,
430 false);
366 } else { 431 } else {
367 d40_log_fill_lli(&lli->dst[i], 432 lli_dst =
368 sg_phys(current_sg), 433 d40_log_buf_to_lli(lli_dst,
369 sg_dma_len(current_sg), 434 sg_phys(current_sg),
370 lcsp->lcsp3, dst_data_width, 435 sg_dma_len(current_sg),
371 true); 436 lcsp->lcsp3, dst_data_width,
372 d40_log_fill_lli(&lli->src[i], 437 src_data_width,
373 dev_addr, 438 true);
374 sg_dma_len(current_sg), 439 lli_src =
375 lcsp->lcsp1, src_data_width, 440 d40_log_buf_to_lli(lli_src,
376 false); 441 dev_addr,
442 sg_dma_len(current_sg),
443 lcsp->lcsp1, src_data_width,
444 dst_data_width,
445 false);
377 } 446 }
378 } 447 }
379 return total_size; 448 return total_size;
380} 449}
381 450
451struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
452 dma_addr_t addr,
453 int size,
454 u32 lcsp13, /* src or dst*/
455 u32 data_width1,
456 u32 data_width2,
457 bool addr_inc)
458{
459 struct d40_log_lli *lli = lli_sg;
460 int size_rest = size;
461 int size_seg = 0;
462
463 do {
464 size_seg = d40_seg_size(size_rest, data_width1, data_width2);
465 size_rest -= size_seg;
466
467 d40_log_fill_lli(lli,
468 addr,
469 size_seg,
470 lcsp13, data_width1,
471 addr_inc);
472 if (addr_inc)
473 addr += size_seg;
474 lli++;
475 } while (size_rest);
476
477 return lli;
478}
479
382int d40_log_sg_to_lli(struct scatterlist *sg, 480int d40_log_sg_to_lli(struct scatterlist *sg,
383 int sg_len, 481 int sg_len,
384 struct d40_log_lli *lli_sg, 482 struct d40_log_lli *lli_sg,
385 u32 lcsp13, /* src or dst*/ 483 u32 lcsp13, /* src or dst*/
386 u32 data_width) 484 u32 data_width1, u32 data_width2)
387{ 485{
388 int total_size = 0; 486 int total_size = 0;
389 struct scatterlist *current_sg = sg; 487 struct scatterlist *current_sg = sg;
390 int i; 488 int i;
489 struct d40_log_lli *lli = lli_sg;
391 490
392 for_each_sg(sg, current_sg, sg_len, i) { 491 for_each_sg(sg, current_sg, sg_len, i) {
393 total_size += sg_dma_len(current_sg); 492 total_size += sg_dma_len(current_sg);
394 493 lli = d40_log_buf_to_lli(lli,
395 d40_log_fill_lli(&lli_sg[i], 494 sg_phys(current_sg),
396 sg_phys(current_sg), 495 sg_dma_len(current_sg),
397 sg_dma_len(current_sg), 496 lcsp13,
398 lcsp13, data_width, 497 data_width1, data_width2, true);
399 true);
400 } 498 }
401 return total_size; 499 return total_size;
402} 500}
diff --git a/drivers/dma/ste_dma40_ll.h b/drivers/dma/ste_dma40_ll.h
index 9e419b907544..9cc43495bea2 100644
--- a/drivers/dma/ste_dma40_ll.h
+++ b/drivers/dma/ste_dma40_ll.h
@@ -292,18 +292,20 @@ int d40_phy_sg_to_lli(struct scatterlist *sg,
292 struct d40_phy_lli *lli, 292 struct d40_phy_lli *lli,
293 dma_addr_t lli_phys, 293 dma_addr_t lli_phys,
294 u32 reg_cfg, 294 u32 reg_cfg,
295 u32 data_width, 295 u32 data_width1,
296 u32 data_width2,
296 int psize); 297 int psize);
297 298
298int d40_phy_fill_lli(struct d40_phy_lli *lli, 299struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli,
299 dma_addr_t data, 300 dma_addr_t data,
300 u32 data_size, 301 u32 data_size,
301 int psize, 302 int psize,
302 dma_addr_t next_lli, 303 dma_addr_t next_lli,
303 u32 reg_cfg, 304 u32 reg_cfg,
304 bool term_int, 305 bool term_int,
305 u32 data_width, 306 u32 data_width1,
306 bool is_device); 307 u32 data_width2,
308 bool is_device);
307 309
308void d40_phy_lli_write(void __iomem *virtbase, 310void d40_phy_lli_write(void __iomem *virtbase,
309 u32 phy_chan_num, 311 u32 phy_chan_num,
@@ -312,12 +314,12 @@ void d40_phy_lli_write(void __iomem *virtbase,
312 314
313/* Logical channels */ 315/* Logical channels */
314 316
315void d40_log_fill_lli(struct d40_log_lli *lli, 317struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
316 dma_addr_t data, 318 dma_addr_t addr,
317 u32 data_size, 319 int size,
318 u32 reg_cfg, 320 u32 lcsp13, /* src or dst*/
319 u32 data_width, 321 u32 data_width1, u32 data_width2,
320 bool addr_inc); 322 bool addr_inc);
321 323
322int d40_log_sg_to_dev(struct scatterlist *sg, 324int d40_log_sg_to_dev(struct scatterlist *sg,
323 int sg_len, 325 int sg_len,
@@ -332,7 +334,7 @@ int d40_log_sg_to_lli(struct scatterlist *sg,
332 int sg_len, 334 int sg_len,
333 struct d40_log_lli *lli_sg, 335 struct d40_log_lli *lli_sg,
334 u32 lcsp13, /* src or dst*/ 336 u32 lcsp13, /* src or dst*/
335 u32 data_width); 337 u32 data_width1, u32 data_width2);
336 338
337void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa, 339void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa,
338 struct d40_log_lli *lli_dst, 340 struct d40_log_lli *lli_dst,
diff --git a/include/linux/amba/pl08x.h b/include/linux/amba/pl08x.h
index 521a0f8974ac..933b4ed12be5 100644
--- a/include/linux/amba/pl08x.h
+++ b/include/linux/amba/pl08x.h
@@ -22,6 +22,15 @@
22#include <linux/dmaengine.h> 22#include <linux/dmaengine.h>
23#include <linux/interrupt.h> 23#include <linux/interrupt.h>
24 24
25struct pl08x_lli;
26struct pl08x_driver_data;
27
28/* Bitmasks for selecting AHB ports for DMA transfers */
29enum {
30 PL08X_AHB1 = (1 << 0),
31 PL08X_AHB2 = (1 << 1)
32};
33
25/** 34/**
26 * struct pl08x_channel_data - data structure to pass info between 35 * struct pl08x_channel_data - data structure to pass info between
27 * platform and PL08x driver regarding channel configuration 36 * platform and PL08x driver regarding channel configuration
@@ -48,6 +57,8 @@
48 * round round round) 57 * round round round)
49 * @single: the device connected to this channel will request single 58 * @single: the device connected to this channel will request single
50 * DMA transfers, not bursts. (Bursts are default.) 59 * DMA transfers, not bursts. (Bursts are default.)
60 * @periph_buses: the device connected to this channel is accessible via
61 * these buses (use PL08X_AHB1 | PL08X_AHB2).
51 */ 62 */
52struct pl08x_channel_data { 63struct pl08x_channel_data {
53 char *bus_id; 64 char *bus_id;
@@ -55,10 +66,10 @@ struct pl08x_channel_data {
55 int max_signal; 66 int max_signal;
56 u32 muxval; 67 u32 muxval;
57 u32 cctl; 68 u32 cctl;
58 u32 ccfg;
59 dma_addr_t addr; 69 dma_addr_t addr;
60 bool circular_buffer; 70 bool circular_buffer;
61 bool single; 71 bool single;
72 u8 periph_buses;
62}; 73};
63 74
64/** 75/**
@@ -74,7 +85,7 @@ struct pl08x_bus_data {
74 dma_addr_t addr; 85 dma_addr_t addr;
75 u8 maxwidth; 86 u8 maxwidth;
76 u8 buswidth; 87 u8 buswidth;
77 u32 fill_bytes; 88 size_t fill_bytes;
78}; 89};
79 90
80/** 91/**
@@ -92,11 +103,6 @@ struct pl08x_phy_chan {
92 spinlock_t lock; 103 spinlock_t lock;
93 int signal; 104 int signal;
94 struct pl08x_dma_chan *serving; 105 struct pl08x_dma_chan *serving;
95 u32 csrc;
96 u32 cdst;
97 u32 clli;
98 u32 cctl;
99 u32 ccfg;
100}; 106};
101 107
102/** 108/**
@@ -108,21 +114,19 @@ struct pl08x_txd {
108 struct dma_async_tx_descriptor tx; 114 struct dma_async_tx_descriptor tx;
109 struct list_head node; 115 struct list_head node;
110 enum dma_data_direction direction; 116 enum dma_data_direction direction;
111 struct pl08x_bus_data srcbus; 117 dma_addr_t src_addr;
112 struct pl08x_bus_data dstbus; 118 dma_addr_t dst_addr;
113 int len; 119 size_t len;
114 dma_addr_t llis_bus; 120 dma_addr_t llis_bus;
115 void *llis_va; 121 void *llis_va;
116 struct pl08x_channel_data *cd;
117 bool active; 122 bool active;
123 /* Default cctl value for LLIs */
124 u32 cctl;
118 /* 125 /*
119 * Settings to be put into the physical channel when we 126 * Settings to be put into the physical channel when we
120 * trigger this txd 127 * trigger this txd. Other registers are in llis_va[0].
121 */ 128 */
122 u32 csrc; 129 u32 ccfg;
123 u32 cdst;
124 u32 clli;
125 u32 cctl;
126}; 130};
127 131
128/** 132/**
@@ -147,6 +151,8 @@ enum pl08x_dma_chan_state {
147 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel 151 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
148 * @chan: wrappped abstract channel 152 * @chan: wrappped abstract channel
149 * @phychan: the physical channel utilized by this channel, if there is one 153 * @phychan: the physical channel utilized by this channel, if there is one
154 * @phychan_hold: if non-zero, hold on to the physical channel even if we
155 * have no pending entries
150 * @tasklet: tasklet scheduled by the IRQ to handle actual work etc 156 * @tasklet: tasklet scheduled by the IRQ to handle actual work etc
151 * @name: name of channel 157 * @name: name of channel
152 * @cd: channel platform data 158 * @cd: channel platform data
@@ -154,11 +160,8 @@ enum pl08x_dma_chan_state {
154 * @runtime_direction: current direction of this channel according to 160 * @runtime_direction: current direction of this channel according to
155 * runtime config 161 * runtime config
156 * @lc: last completed transaction on this channel 162 * @lc: last completed transaction on this channel
157 * @desc_list: queued transactions pending on this channel 163 * @pend_list: queued transactions pending on this channel
158 * @at: active transaction on this channel 164 * @at: active transaction on this channel
159 * @lockflags: sometimes we let a lock last between two function calls,
160 * especially prep/submit, and then we need to store the IRQ flags
161 * in the channel state, here
162 * @lock: a lock for this channel data 165 * @lock: a lock for this channel data
163 * @host: a pointer to the host (internal use) 166 * @host: a pointer to the host (internal use)
164 * @state: whether the channel is idle, paused, running etc 167 * @state: whether the channel is idle, paused, running etc
@@ -169,18 +172,17 @@ enum pl08x_dma_chan_state {
169struct pl08x_dma_chan { 172struct pl08x_dma_chan {
170 struct dma_chan chan; 173 struct dma_chan chan;
171 struct pl08x_phy_chan *phychan; 174 struct pl08x_phy_chan *phychan;
175 int phychan_hold;
172 struct tasklet_struct tasklet; 176 struct tasklet_struct tasklet;
173 char *name; 177 char *name;
174 struct pl08x_channel_data *cd; 178 struct pl08x_channel_data *cd;
175 dma_addr_t runtime_addr; 179 dma_addr_t runtime_addr;
176 enum dma_data_direction runtime_direction; 180 enum dma_data_direction runtime_direction;
177 atomic_t last_issued;
178 dma_cookie_t lc; 181 dma_cookie_t lc;
179 struct list_head desc_list; 182 struct list_head pend_list;
180 struct pl08x_txd *at; 183 struct pl08x_txd *at;
181 unsigned long lockflags;
182 spinlock_t lock; 184 spinlock_t lock;
183 void *host; 185 struct pl08x_driver_data *host;
184 enum pl08x_dma_chan_state state; 186 enum pl08x_dma_chan_state state;
185 bool slave; 187 bool slave;
186 struct pl08x_txd *waiting; 188 struct pl08x_txd *waiting;
@@ -199,8 +201,8 @@ struct pl08x_dma_chan {
199 * less than zero, else it returns the allocated signal number 201 * less than zero, else it returns the allocated signal number
200 * @put_signal: indicate to the platform that this physical signal is not 202 * @put_signal: indicate to the platform that this physical signal is not
201 * running any DMA transfer and multiplexing can be recycled 203 * running any DMA transfer and multiplexing can be recycled
202 * @bus_bit_lli: Bit[0] of the address indicated which AHB bus master the 204 * @lli_buses: buses which LLIs can be fetched from: PL08X_AHB1 | PL08X_AHB2
203 * LLI addresses are on 0/1 Master 1/2. 205 * @mem_buses: buses which memory can be accessed from: PL08X_AHB1 | PL08X_AHB2
204 */ 206 */
205struct pl08x_platform_data { 207struct pl08x_platform_data {
206 struct pl08x_channel_data *slave_channels; 208 struct pl08x_channel_data *slave_channels;
@@ -208,6 +210,8 @@ struct pl08x_platform_data {
208 struct pl08x_channel_data memcpy_channel; 210 struct pl08x_channel_data memcpy_channel;
209 int (*get_signal)(struct pl08x_dma_chan *); 211 int (*get_signal)(struct pl08x_dma_chan *);
210 void (*put_signal)(struct pl08x_dma_chan *); 212 void (*put_signal)(struct pl08x_dma_chan *);
213 u8 lli_buses;
214 u8 mem_buses;
211}; 215};
212 216
213#ifdef CONFIG_AMBA_PL08X 217#ifdef CONFIG_AMBA_PL08X
generated by cgit v1.2.2 (git 2.25.0) at 2025-05-31 06:14:14 -0400