aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDan Williams <dan.j.williams@intel.com>2011-02-14 06:36:44 -0500
committerDan Williams <dan.j.williams@intel.com>2011-02-14 06:36:44 -0500
commit80cc07af0f6692a7d8fdc5087594d1988a701266 (patch)
treea21ae71038bbfd75306b81a85c0c65121cd84f4f
parente19d1d4988f8020c25bf1758f9a898e1374cef35 (diff)
parent0c842b551063c5f7382ac9b457992f3b34972801 (diff)
Merge branch 'dma40' into dmaengine
Diffstat
-rw-r--r--arch/arm/plat-nomadik/include/plat/ste_dma40.h22
-rw-r--r--drivers/dma/ste_dma40.c1402
-rw-r--r--drivers/dma/ste_dma40_ll.c218
-rw-r--r--drivers/dma/ste_dma40_ll.h66
4 files changed, 761 insertions, 947 deletions
diff --git a/arch/arm/plat-nomadik/include/plat/ste_dma40.h b/arch/arm/plat-nomadik/include/plat/ste_dma40.h
index 4d6dd4c39b75..c44886062f8e 100644
--- a/arch/arm/plat-nomadik/include/plat/ste_dma40.h
+++ b/arch/arm/plat-nomadik/include/plat/ste_dma40.h
@@ -104,6 +104,8 @@ struct stedma40_half_channel_info {
104 * 104 *
105 * @dir: MEM 2 MEM, PERIPH 2 MEM , MEM 2 PERIPH, PERIPH 2 PERIPH 105 * @dir: MEM 2 MEM, PERIPH 2 MEM , MEM 2 PERIPH, PERIPH 2 PERIPH
106 * @high_priority: true if high-priority 106 * @high_priority: true if high-priority
107 * @realtime: true if realtime mode is to be enabled. Only available on DMA40
108 * version 3+, i.e DB8500v2+
107 * @mode: channel mode: physical, logical, or operation 109 * @mode: channel mode: physical, logical, or operation
108 * @mode_opt: options for the chosen channel mode 110 * @mode_opt: options for the chosen channel mode
109 * @src_dev_type: Src device type 111 * @src_dev_type: Src device type
@@ -119,6 +121,7 @@ struct stedma40_half_channel_info {
119struct stedma40_chan_cfg { 121struct stedma40_chan_cfg {
120 enum stedma40_xfer_dir dir; 122 enum stedma40_xfer_dir dir;
121 bool high_priority; 123 bool high_priority;
124 bool realtime;
122 enum stedma40_mode mode; 125 enum stedma40_mode mode;
123 enum stedma40_mode_opt mode_opt; 126 enum stedma40_mode_opt mode_opt;
124 int src_dev_type; 127 int src_dev_type;
@@ -169,25 +172,6 @@ struct stedma40_platform_data {
169bool stedma40_filter(struct dma_chan *chan, void *data); 172bool stedma40_filter(struct dma_chan *chan, void *data);
170 173
171/** 174/**
172 * stedma40_memcpy_sg() - extension of the dma framework, memcpy to/from
173 * scattergatter lists.
174 *
175 * @chan: dmaengine handle
176 * @sgl_dst: Destination scatter list
177 * @sgl_src: Source scatter list
178 * @sgl_len: The length of each scatterlist. Both lists must be of equal length
179 * and each element must match the corresponding element in the other scatter
180 * list.
181 * @flags: is actually enum dma_ctrl_flags. See dmaengine.h
182 */
183
184struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
185 struct scatterlist *sgl_dst,
186 struct scatterlist *sgl_src,
187 unsigned int sgl_len,
188 unsigned long flags);
189
190/**
191 * stedma40_slave_mem() - Transfers a raw data buffer to or from a slave 175 * stedma40_slave_mem() - Transfers a raw data buffer to or from a slave
192 * (=device) 176 * (=device)
193 * 177 *
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index 6e1d46a65d0e..af955de035f4 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -68,6 +68,7 @@ enum d40_command {
68 * @base: Pointer to memory area when the pre_alloc_lli's are not large 68 * @base: Pointer to memory area when the pre_alloc_lli's are not large
69 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if 69 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
70 * pre_alloc_lli is used. 70 * pre_alloc_lli is used.
71 * @dma_addr: DMA address, if mapped
71 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli. 72 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
72 * @pre_alloc_lli: Pre allocated area for the most common case of transfers, 73 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
73 * one buffer to one buffer. 74 * one buffer to one buffer.
@@ -75,6 +76,7 @@ enum d40_command {
75struct d40_lli_pool { 76struct d40_lli_pool {
76 void *base; 77 void *base;
77 int size; 78 int size;
79 dma_addr_t dma_addr;
78 /* Space for dst and src, plus an extra for padding */ 80 /* Space for dst and src, plus an extra for padding */
79 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)]; 81 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
80}; 82};
@@ -94,7 +96,6 @@ struct d40_lli_pool {
94 * during a transfer. 96 * during a transfer.
95 * @node: List entry. 97 * @node: List entry.
96 * @is_in_client_list: true if the client owns this descriptor. 98 * @is_in_client_list: true if the client owns this descriptor.
97 * @is_hw_linked: true if this job will automatically be continued for
98 * the previous one. 99 * the previous one.
99 * 100 *
100 * This descriptor is used for both logical and physical transfers. 101 * This descriptor is used for both logical and physical transfers.
@@ -114,7 +115,7 @@ struct d40_desc {
114 struct list_head node; 115 struct list_head node;
115 116
116 bool is_in_client_list; 117 bool is_in_client_list;
117 bool is_hw_linked; 118 bool cyclic;
118}; 119};
119 120
120/** 121/**
@@ -130,6 +131,7 @@ struct d40_desc {
130 */ 131 */
131struct d40_lcla_pool { 132struct d40_lcla_pool {
132 void *base; 133 void *base;
134 dma_addr_t dma_addr;
133 void *base_unaligned; 135 void *base_unaligned;
134 int pages; 136 int pages;
135 spinlock_t lock; 137 spinlock_t lock;
@@ -303,9 +305,37 @@ struct d40_reg_val {
303 unsigned int val; 305 unsigned int val;
304}; 306};
305 307
306static int d40_pool_lli_alloc(struct d40_desc *d40d, 308static struct device *chan2dev(struct d40_chan *d40c)
307 int lli_len, bool is_log)
308{ 309{
310 return &d40c->chan.dev->device;
311}
312
313static bool chan_is_physical(struct d40_chan *chan)
314{
315 return chan->log_num == D40_PHY_CHAN;
316}
317
318static bool chan_is_logical(struct d40_chan *chan)
319{
320 return !chan_is_physical(chan);
321}
322
323static void __iomem *chan_base(struct d40_chan *chan)
324{
325 return chan->base->virtbase + D40_DREG_PCBASE +
326 chan->phy_chan->num * D40_DREG_PCDELTA;
327}
328
329#define d40_err(dev, format, arg...) \
330 dev_err(dev, "[%s] " format, __func__, ## arg)
331
332#define chan_err(d40c, format, arg...) \
333 d40_err(chan2dev(d40c), format, ## arg)
334
335static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
336 int lli_len)
337{
338 bool is_log = chan_is_logical(d40c);
309 u32 align; 339 u32 align;
310 void *base; 340 void *base;
311 341
@@ -319,7 +349,7 @@ static int d40_pool_lli_alloc(struct d40_desc *d40d,
319 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli); 349 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
320 d40d->lli_pool.base = NULL; 350 d40d->lli_pool.base = NULL;
321 } else { 351 } else {
322 d40d->lli_pool.size = ALIGN(lli_len * 2 * align, align); 352 d40d->lli_pool.size = lli_len * 2 * align;
323 353
324 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT); 354 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
325 d40d->lli_pool.base = base; 355 d40d->lli_pool.base = base;
@@ -329,22 +359,37 @@ static int d40_pool_lli_alloc(struct d40_desc *d40d,
329 } 359 }
330 360
331 if (is_log) { 361 if (is_log) {
332 d40d->lli_log.src = PTR_ALIGN((struct d40_log_lli *) base, 362 d40d->lli_log.src = PTR_ALIGN(base, align);
333 align); 363 d40d->lli_log.dst = d40d->lli_log.src + lli_len;
334 d40d->lli_log.dst = PTR_ALIGN(d40d->lli_log.src + lli_len, 364
335 align); 365 d40d->lli_pool.dma_addr = 0;
336 } else { 366 } else {
337 d40d->lli_phy.src = PTR_ALIGN((struct d40_phy_lli *)base, 367 d40d->lli_phy.src = PTR_ALIGN(base, align);
338 align); 368 d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
339 d40d->lli_phy.dst = PTR_ALIGN(d40d->lli_phy.src + lli_len, 369
340 align); 370 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
371 d40d->lli_phy.src,
372 d40d->lli_pool.size,
373 DMA_TO_DEVICE);
374
375 if (dma_mapping_error(d40c->base->dev,
376 d40d->lli_pool.dma_addr)) {
377 kfree(d40d->lli_pool.base);
378 d40d->lli_pool.base = NULL;
379 d40d->lli_pool.dma_addr = 0;
380 return -ENOMEM;
381 }
341 } 382 }
342 383
343 return 0; 384 return 0;
344} 385}
345 386
346static void d40_pool_lli_free(struct d40_desc *d40d) 387static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
347{ 388{
389 if (d40d->lli_pool.dma_addr)
390 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
391 d40d->lli_pool.size, DMA_TO_DEVICE);
392
348 kfree(d40d->lli_pool.base); 393 kfree(d40d->lli_pool.base);
349 d40d->lli_pool.base = NULL; 394 d40d->lli_pool.base = NULL;
350 d40d->lli_pool.size = 0; 395 d40d->lli_pool.size = 0;
@@ -391,7 +436,7 @@ static int d40_lcla_free_all(struct d40_chan *d40c,
391 int i; 436 int i;
392 int ret = -EINVAL; 437 int ret = -EINVAL;
393 438
394 if (d40c->log_num == D40_PHY_CHAN) 439 if (chan_is_physical(d40c))
395 return 0; 440 return 0;
396 441
397 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags); 442 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
@@ -430,7 +475,7 @@ static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
430 475
431 list_for_each_entry_safe(d, _d, &d40c->client, node) 476 list_for_each_entry_safe(d, _d, &d40c->client, node)
432 if (async_tx_test_ack(&d->txd)) { 477 if (async_tx_test_ack(&d->txd)) {
433 d40_pool_lli_free(d); 478 d40_pool_lli_free(d40c, d);
434 d40_desc_remove(d); 479 d40_desc_remove(d);
435 desc = d; 480 desc = d;
436 memset(desc, 0, sizeof(*desc)); 481 memset(desc, 0, sizeof(*desc));
@@ -450,6 +495,7 @@ static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
450static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d) 495static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
451{ 496{
452 497
498 d40_pool_lli_free(d40c, d40d);
453 d40_lcla_free_all(d40c, d40d); 499 d40_lcla_free_all(d40c, d40d);
454 kmem_cache_free(d40c->base->desc_slab, d40d); 500 kmem_cache_free(d40c->base->desc_slab, d40d);
455} 501}
@@ -459,57 +505,128 @@ static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
459 list_add_tail(&desc->node, &d40c->active); 505 list_add_tail(&desc->node, &d40c->active);
460} 506}
461 507
462static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d) 508static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
463{ 509{
464 int curr_lcla = -EINVAL, next_lcla; 510 struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
511 struct d40_phy_lli *lli_src = desc->lli_phy.src;
512 void __iomem *base = chan_base(chan);
513
514 writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);
515 writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);
516 writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);
517 writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);
518
519 writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);
520 writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);
521 writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);
522 writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);
523}
465 524
466 if (d40c->log_num == D40_PHY_CHAN) { 525static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
467 d40_phy_lli_write(d40c->base->virtbase, 526{
468 d40c->phy_chan->num, 527 struct d40_lcla_pool *pool = &chan->base->lcla_pool;
469 d40d->lli_phy.dst, 528 struct d40_log_lli_bidir *lli = &desc->lli_log;
470 d40d->lli_phy.src); 529 int lli_current = desc->lli_current;
471 d40d->lli_current = d40d->lli_len; 530 int lli_len = desc->lli_len;
472 } else { 531 bool cyclic = desc->cyclic;
532 int curr_lcla = -EINVAL;
533 int first_lcla = 0;
534 bool linkback;
473 535
474 if ((d40d->lli_len - d40d->lli_current) > 1) 536 /*
475 curr_lcla = d40_lcla_alloc_one(d40c, d40d); 537 * We may have partially running cyclic transfers, in case we did't get
538 * enough LCLA entries.
539 */
540 linkback = cyclic && lli_current == 0;
476 541
477 d40_log_lli_lcpa_write(d40c->lcpa, 542 /*
478 &d40d->lli_log.dst[d40d->lli_current], 543 * For linkback, we need one LCLA even with only one link, because we
479 &d40d->lli_log.src[d40d->lli_current], 544 * can't link back to the one in LCPA space
480 curr_lcla); 545 */
546 if (linkback || (lli_len - lli_current > 1)) {
547 curr_lcla = d40_lcla_alloc_one(chan, desc);
548 first_lcla = curr_lcla;
549 }
481 550
482 d40d->lli_current++; 551 /*
483 for (; d40d->lli_current < d40d->lli_len; d40d->lli_current++) { 552 * For linkback, we normally load the LCPA in the loop since we need to
484 struct d40_log_lli *lcla; 553 * link it to the second LCLA and not the first. However, if we
554 * couldn't even get a first LCLA, then we have to run in LCPA and
555 * reload manually.
556 */
557 if (!linkback || curr_lcla == -EINVAL) {
558 unsigned int flags = 0;
485 559
486 if (d40d->lli_current + 1 < d40d->lli_len) 560 if (curr_lcla == -EINVAL)
487 next_lcla = d40_lcla_alloc_one(d40c, d40d); 561 flags |= LLI_TERM_INT;
488 else
489 next_lcla = -EINVAL;
490 562
491 lcla = d40c->base->lcla_pool.base + 563 d40_log_lli_lcpa_write(chan->lcpa,
492 d40c->phy_chan->num * 1024 + 564 &lli->dst[lli_current],
493 8 * curr_lcla * 2; 565 &lli->src[lli_current],
566 curr_lcla,
567 flags);
568 lli_current++;
569 }
494 570
495 d40_log_lli_lcla_write(lcla, 571 if (curr_lcla < 0)
496 &d40d->lli_log.dst[d40d->lli_current], 572 goto out;
497 &d40d->lli_log.src[d40d->lli_current],
498 next_lcla);
499 573
500 (void) dma_map_single(d40c->base->dev, lcla, 574 for (; lli_current < lli_len; lli_current++) {
501 2 * sizeof(struct d40_log_lli), 575 unsigned int lcla_offset = chan->phy_chan->num * 1024 +
502 DMA_TO_DEVICE); 576 8 * curr_lcla * 2;
577 struct d40_log_lli *lcla = pool->base + lcla_offset;
578 unsigned int flags = 0;
579 int next_lcla;
503 580
504 curr_lcla = next_lcla; 581 if (lli_current + 1 < lli_len)
582 next_lcla = d40_lcla_alloc_one(chan, desc);
583 else
584 next_lcla = linkback ? first_lcla : -EINVAL;
505 585
506 if (curr_lcla == -EINVAL) { 586 if (cyclic || next_lcla == -EINVAL)
507 d40d->lli_current++; 587 flags |= LLI_TERM_INT;
508 break; 588
509 } 589 if (linkback && curr_lcla == first_lcla) {
590 /* First link goes in both LCPA and LCLA */
591 d40_log_lli_lcpa_write(chan->lcpa,
592 &lli->dst[lli_current],
593 &lli->src[lli_current],
594 next_lcla, flags);
595 }
596
597 /*
598 * One unused LCLA in the cyclic case if the very first
599 * next_lcla fails...
600 */
601 d40_log_lli_lcla_write(lcla,
602 &lli->dst[lli_current],
603 &lli->src[lli_current],
604 next_lcla, flags);
605
606 dma_sync_single_range_for_device(chan->base->dev,
607 pool->dma_addr, lcla_offset,
608 2 * sizeof(struct d40_log_lli),
609 DMA_TO_DEVICE);
510 610
611 curr_lcla = next_lcla;
612
613 if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {
614 lli_current++;
615 break;
511 } 616 }
512 } 617 }
618
619out:
620 desc->lli_current = lli_current;
621}
622
623static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
624{
625 if (chan_is_physical(d40c)) {
626 d40_phy_lli_load(d40c, d40d);
627 d40d->lli_current = d40d->lli_len;
628 } else
629 d40_log_lli_to_lcxa(d40c, d40d);
513} 630}
514 631
515static struct d40_desc *d40_first_active_get(struct d40_chan *d40c) 632static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
@@ -543,18 +660,6 @@ static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
543 return d; 660 return d;
544} 661}
545 662
546static struct d40_desc *d40_last_queued(struct d40_chan *d40c)
547{
548 struct d40_desc *d;
549
550 if (list_empty(&d40c->queue))
551 return NULL;
552 list_for_each_entry(d, &d40c->queue, node)
553 if (list_is_last(&d->node, &d40c->queue))
554 break;
555 return d;
556}
557
558static int d40_psize_2_burst_size(bool is_log, int psize) 663static int d40_psize_2_burst_size(bool is_log, int psize)
559{ 664{
560 if (is_log) { 665 if (is_log) {
@@ -666,9 +771,9 @@ static int d40_channel_execute_command(struct d40_chan *d40c,
666 } 771 }
667 772
668 if (i == D40_SUSPEND_MAX_IT) { 773 if (i == D40_SUSPEND_MAX_IT) {
669 dev_err(&d40c->chan.dev->device, 774 chan_err(d40c,
670 "[%s]: unable to suspend the chl %d (log: %d) status %x\n", 775 "unable to suspend the chl %d (log: %d) status %x\n",
671 __func__, d40c->phy_chan->num, d40c->log_num, 776 d40c->phy_chan->num, d40c->log_num,
672 status); 777 status);
673 dump_stack(); 778 dump_stack();
674 ret = -EBUSY; 779 ret = -EBUSY;
@@ -701,17 +806,45 @@ static void d40_term_all(struct d40_chan *d40c)
701 d40c->busy = false; 806 d40c->busy = false;
702} 807}
703 808
809static void __d40_config_set_event(struct d40_chan *d40c, bool enable,
810 u32 event, int reg)
811{
812 void __iomem *addr = chan_base(d40c) + reg;
813 int tries;
814
815 if (!enable) {
816 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
817 | ~D40_EVENTLINE_MASK(event), addr);
818 return;
819 }
820
821 /*
822 * The hardware sometimes doesn't register the enable when src and dst
823 * event lines are active on the same logical channel. Retry to ensure
824 * it does. Usually only one retry is sufficient.
825 */
826 tries = 100;
827 while (--tries) {
828 writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
829 | ~D40_EVENTLINE_MASK(event), addr);
830
831 if (readl(addr) & D40_EVENTLINE_MASK(event))
832 break;
833 }
834
835 if (tries != 99)
836 dev_dbg(chan2dev(d40c),
837 "[%s] workaround enable S%cLNK (%d tries)\n",
838 __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
839 100 - tries);
840
841 WARN_ON(!tries);
842}
843
704static void d40_config_set_event(struct d40_chan *d40c, bool do_enable) 844static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
705{ 845{
706 u32 val;
707 unsigned long flags; 846 unsigned long flags;
708 847
709 /* Notice, that disable requires the physical channel to be stopped */
710 if (do_enable)
711 val = D40_ACTIVATE_EVENTLINE;
712 else
713 val = D40_DEACTIVATE_EVENTLINE;
714
715 spin_lock_irqsave(&d40c->phy_chan->lock, flags); 848 spin_lock_irqsave(&d40c->phy_chan->lock, flags);
716 849
717 /* Enable event line connected to device (or memcpy) */ 850 /* Enable event line connected to device (or memcpy) */
@@ -719,20 +852,15 @@ static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
719 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) { 852 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
720 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); 853 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
721 854
722 writel((val << D40_EVENTLINE_POS(event)) | 855 __d40_config_set_event(d40c, do_enable, event,
723 ~D40_EVENTLINE_MASK(event), 856 D40_CHAN_REG_SSLNK);
724 d40c->base->virtbase + D40_DREG_PCBASE +
725 d40c->phy_chan->num * D40_DREG_PCDELTA +
726 D40_CHAN_REG_SSLNK);
727 } 857 }
858
728 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) { 859 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) {
729 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); 860 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
730 861
731 writel((val << D40_EVENTLINE_POS(event)) | 862 __d40_config_set_event(d40c, do_enable, event,
732 ~D40_EVENTLINE_MASK(event), 863 D40_CHAN_REG_SDLNK);
733 d40c->base->virtbase + D40_DREG_PCBASE +
734 d40c->phy_chan->num * D40_DREG_PCDELTA +
735 D40_CHAN_REG_SDLNK);
736 } 864 }
737 865
738 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags); 866 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
@@ -740,15 +868,12 @@ static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
740 868
741static u32 d40_chan_has_events(struct d40_chan *d40c) 869static u32 d40_chan_has_events(struct d40_chan *d40c)
742{ 870{
871 void __iomem *chanbase = chan_base(d40c);
743 u32 val; 872 u32 val;
744 873
745 val = readl(d40c->base->virtbase + D40_DREG_PCBASE + 874 val = readl(chanbase + D40_CHAN_REG_SSLNK);
746 d40c->phy_chan->num * D40_DREG_PCDELTA + 875 val |= readl(chanbase + D40_CHAN_REG_SDLNK);
747 D40_CHAN_REG_SSLNK);
748 876
749 val |= readl(d40c->base->virtbase + D40_DREG_PCBASE +
750 d40c->phy_chan->num * D40_DREG_PCDELTA +
751 D40_CHAN_REG_SDLNK);
752 return val; 877 return val;
753} 878}
754 879
@@ -771,7 +896,7 @@ static u32 d40_get_prmo(struct d40_chan *d40c)
771 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG, 896 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
772 }; 897 };
773 898
774 if (d40c->log_num == D40_PHY_CHAN) 899 if (chan_is_physical(d40c))
775 return phy_map[d40c->dma_cfg.mode_opt]; 900 return phy_map[d40c->dma_cfg.mode_opt];
776 else 901 else
777 return log_map[d40c->dma_cfg.mode_opt]; 902 return log_map[d40c->dma_cfg.mode_opt];
@@ -785,7 +910,7 @@ static void d40_config_write(struct d40_chan *d40c)
785 /* Odd addresses are even addresses + 4 */ 910 /* Odd addresses are even addresses + 4 */
786 addr_base = (d40c->phy_chan->num % 2) * 4; 911 addr_base = (d40c->phy_chan->num % 2) * 4;
787 /* Setup channel mode to logical or physical */ 912 /* Setup channel mode to logical or physical */
788 var = ((u32)(d40c->log_num != D40_PHY_CHAN) + 1) << 913 var = ((u32)(chan_is_logical(d40c)) + 1) <<
789 D40_CHAN_POS(d40c->phy_chan->num); 914 D40_CHAN_POS(d40c->phy_chan->num);
790 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base); 915 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
791 916
@@ -794,30 +919,18 @@ static void d40_config_write(struct d40_chan *d40c)
794 919
795 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base); 920 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
796 921
797 if (d40c->log_num != D40_PHY_CHAN) { 922 if (chan_is_logical(d40c)) {
923 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
924 & D40_SREG_ELEM_LOG_LIDX_MASK;
925 void __iomem *chanbase = chan_base(d40c);
926
798 /* Set default config for CFG reg */ 927 /* Set default config for CFG reg */
799 writel(d40c->src_def_cfg, 928 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
800 d40c->base->virtbase + D40_DREG_PCBASE + 929 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
801 d40c->phy_chan->num * D40_DREG_PCDELTA +
802 D40_CHAN_REG_SSCFG);
803 writel(d40c->dst_def_cfg,
804 d40c->base->virtbase + D40_DREG_PCBASE +
805 d40c->phy_chan->num * D40_DREG_PCDELTA +
806 D40_CHAN_REG_SDCFG);
807 930
808 /* Set LIDX for lcla */ 931 /* Set LIDX for lcla */
809 writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) & 932 writel(lidx, chanbase + D40_CHAN_REG_SSELT);
810 D40_SREG_ELEM_LOG_LIDX_MASK, 933 writel(lidx, chanbase + D40_CHAN_REG_SDELT);
811 d40c->base->virtbase + D40_DREG_PCBASE +
812 d40c->phy_chan->num * D40_DREG_PCDELTA +
813 D40_CHAN_REG_SDELT);
814
815 writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
816 D40_SREG_ELEM_LOG_LIDX_MASK,
817 d40c->base->virtbase + D40_DREG_PCBASE +
818 d40c->phy_chan->num * D40_DREG_PCDELTA +
819 D40_CHAN_REG_SSELT);
820
821 } 934 }
822} 935}
823 936
@@ -825,15 +938,15 @@ static u32 d40_residue(struct d40_chan *d40c)
825{ 938{
826 u32 num_elt; 939 u32 num_elt;
827 940
828 if (d40c->log_num != D40_PHY_CHAN) 941 if (chan_is_logical(d40c))
829 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK) 942 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
830 >> D40_MEM_LCSP2_ECNT_POS; 943 >> D40_MEM_LCSP2_ECNT_POS;
831 else 944 else {
832 num_elt = (readl(d40c->base->virtbase + D40_DREG_PCBASE + 945 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
833 d40c->phy_chan->num * D40_DREG_PCDELTA + 946 num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
834 D40_CHAN_REG_SDELT) & 947 >> D40_SREG_ELEM_PHY_ECNT_POS;
835 D40_SREG_ELEM_PHY_ECNT_MASK) >> 948 }
836 D40_SREG_ELEM_PHY_ECNT_POS; 949
837 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width); 950 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
838} 951}
839 952
@@ -841,20 +954,17 @@ static bool d40_tx_is_linked(struct d40_chan *d40c)
841{ 954{
842 bool is_link; 955 bool is_link;
843 956
844 if (d40c->log_num != D40_PHY_CHAN) 957 if (chan_is_logical(d40c))
845 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK; 958 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
846 else 959 else
847 is_link = readl(d40c->base->virtbase + D40_DREG_PCBASE + 960 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
848 d40c->phy_chan->num * D40_DREG_PCDELTA + 961 & D40_SREG_LNK_PHYS_LNK_MASK;
849 D40_CHAN_REG_SDLNK) & 962
850 D40_SREG_LNK_PHYS_LNK_MASK;
851 return is_link; 963 return is_link;
852} 964}
853 965
854static int d40_pause(struct dma_chan *chan) 966static int d40_pause(struct d40_chan *d40c)
855{ 967{
856 struct d40_chan *d40c =
857 container_of(chan, struct d40_chan, chan);
858 int res = 0; 968 int res = 0;
859 unsigned long flags; 969 unsigned long flags;
860 970
@@ -865,7 +975,7 @@ static int d40_pause(struct dma_chan *chan)
865 975
866 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); 976 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
867 if (res == 0) { 977 if (res == 0) {
868 if (d40c->log_num != D40_PHY_CHAN) { 978 if (chan_is_logical(d40c)) {
869 d40_config_set_event(d40c, false); 979 d40_config_set_event(d40c, false);
870 /* Resume the other logical channels if any */ 980 /* Resume the other logical channels if any */
871 if (d40_chan_has_events(d40c)) 981 if (d40_chan_has_events(d40c))
@@ -878,10 +988,8 @@ static int d40_pause(struct dma_chan *chan)
878 return res; 988 return res;
879} 989}
880 990
881static int d40_resume(struct dma_chan *chan) 991static int d40_resume(struct d40_chan *d40c)
882{ 992{
883 struct d40_chan *d40c =
884 container_of(chan, struct d40_chan, chan);
885 int res = 0; 993 int res = 0;
886 unsigned long flags; 994 unsigned long flags;
887 995
@@ -891,7 +999,7 @@ static int d40_resume(struct dma_chan *chan)
891 spin_lock_irqsave(&d40c->lock, flags); 999 spin_lock_irqsave(&d40c->lock, flags);
892 1000
893 if (d40c->base->rev == 0) 1001 if (d40c->base->rev == 0)
894 if (d40c->log_num != D40_PHY_CHAN) { 1002 if (chan_is_logical(d40c)) {
895 res = d40_channel_execute_command(d40c, 1003 res = d40_channel_execute_command(d40c,
896 D40_DMA_SUSPEND_REQ); 1004 D40_DMA_SUSPEND_REQ);
897 goto no_suspend; 1005 goto no_suspend;
@@ -900,7 +1008,7 @@ static int d40_resume(struct dma_chan *chan)
900 /* If bytes left to transfer or linked tx resume job */ 1008 /* If bytes left to transfer or linked tx resume job */
901 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) { 1009 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
902 1010
903 if (d40c->log_num != D40_PHY_CHAN) 1011 if (chan_is_logical(d40c))
904 d40_config_set_event(d40c, true); 1012 d40_config_set_event(d40c, true);
905 1013
906 res = d40_channel_execute_command(d40c, D40_DMA_RUN); 1014 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
@@ -911,75 +1019,20 @@ no_suspend:
911 return res; 1019 return res;
912} 1020}
913 1021
914static void d40_tx_submit_log(struct d40_chan *d40c, struct d40_desc *d40d) 1022static int d40_terminate_all(struct d40_chan *chan)
915{ 1023{
916 /* TODO: Write */ 1024 unsigned long flags;
917} 1025 int ret = 0;
918
919static void d40_tx_submit_phy(struct d40_chan *d40c, struct d40_desc *d40d)
920{
921 struct d40_desc *d40d_prev = NULL;
922 int i;
923 u32 val;
924
925 if (!list_empty(&d40c->queue))
926 d40d_prev = d40_last_queued(d40c);
927 else if (!list_empty(&d40c->active))
928 d40d_prev = d40_first_active_get(d40c);
929
930 if (!d40d_prev)
931 return;
932
933 /* Here we try to join this job with previous jobs */
934 val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
935 d40c->phy_chan->num * D40_DREG_PCDELTA +
936 D40_CHAN_REG_SSLNK);
937
938 /* Figure out which link we're currently transmitting */
939 for (i = 0; i < d40d_prev->lli_len; i++)
940 if (val == d40d_prev->lli_phy.src[i].reg_lnk)
941 break;
942
943 val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
944 d40c->phy_chan->num * D40_DREG_PCDELTA +
945 D40_CHAN_REG_SSELT) >> D40_SREG_ELEM_LOG_ECNT_POS;
946
947 if (i == (d40d_prev->lli_len - 1) && val > 0) {
948 /* Change the current one */
949 writel(virt_to_phys(d40d->lli_phy.src),
950 d40c->base->virtbase + D40_DREG_PCBASE +
951 d40c->phy_chan->num * D40_DREG_PCDELTA +
952 D40_CHAN_REG_SSLNK);
953 writel(virt_to_phys(d40d->lli_phy.dst),
954 d40c->base->virtbase + D40_DREG_PCBASE +
955 d40c->phy_chan->num * D40_DREG_PCDELTA +
956 D40_CHAN_REG_SDLNK);
957
958 d40d->is_hw_linked = true;
959
960 } else if (i < d40d_prev->lli_len) {
961 (void) dma_unmap_single(d40c->base->dev,
962 virt_to_phys(d40d_prev->lli_phy.src),
963 d40d_prev->lli_pool.size,
964 DMA_TO_DEVICE);
965 1026
966 /* Keep the settings */ 1027 ret = d40_pause(chan);
967 val = d40d_prev->lli_phy.src[d40d_prev->lli_len - 1].reg_lnk & 1028 if (!ret && chan_is_physical(chan))
968 ~D40_SREG_LNK_PHYS_LNK_MASK; 1029 ret = d40_channel_execute_command(chan, D40_DMA_STOP);
969 d40d_prev->lli_phy.src[d40d_prev->lli_len - 1].reg_lnk =
970 val | virt_to_phys(d40d->lli_phy.src);
971 1030
972 val = d40d_prev->lli_phy.dst[d40d_prev->lli_len - 1].reg_lnk & 1031 spin_lock_irqsave(&chan->lock, flags);
973 ~D40_SREG_LNK_PHYS_LNK_MASK; 1032 d40_term_all(chan);
974 d40d_prev->lli_phy.dst[d40d_prev->lli_len - 1].reg_lnk = 1033 spin_unlock_irqrestore(&chan->lock, flags);
975 val | virt_to_phys(d40d->lli_phy.dst);
976 1034
977 (void) dma_map_single(d40c->base->dev, 1035 return ret;
978 d40d_prev->lli_phy.src,
979 d40d_prev->lli_pool.size,
980 DMA_TO_DEVICE);
981 d40d->is_hw_linked = true;
982 }
983} 1036}
984 1037
985static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx) 1038static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
@@ -990,8 +1043,6 @@ static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
990 struct d40_desc *d40d = container_of(tx, struct d40_desc, txd); 1043 struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
991 unsigned long flags; 1044 unsigned long flags;
992 1045
993 (void) d40_pause(&d40c->chan);
994
995 spin_lock_irqsave(&d40c->lock, flags); 1046 spin_lock_irqsave(&d40c->lock, flags);
996 1047
997 d40c->chan.cookie++; 1048 d40c->chan.cookie++;
@@ -1001,17 +1052,10 @@ static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
1001 1052
1002 d40d->txd.cookie = d40c->chan.cookie; 1053 d40d->txd.cookie = d40c->chan.cookie;
1003 1054
1004 if (d40c->log_num == D40_PHY_CHAN)
1005 d40_tx_submit_phy(d40c, d40d);
1006 else
1007 d40_tx_submit_log(d40c, d40d);
1008
1009 d40_desc_queue(d40c, d40d); 1055 d40_desc_queue(d40c, d40d);
1010 1056
1011 spin_unlock_irqrestore(&d40c->lock, flags); 1057 spin_unlock_irqrestore(&d40c->lock, flags);
1012 1058
1013 (void) d40_resume(&d40c->chan);
1014
1015 return tx->cookie; 1059 return tx->cookie;
1016} 1060}
1017 1061
@@ -1020,7 +1064,7 @@ static int d40_start(struct d40_chan *d40c)
1020 if (d40c->base->rev == 0) { 1064 if (d40c->base->rev == 0) {
1021 int err; 1065 int err;
1022 1066
1023 if (d40c->log_num != D40_PHY_CHAN) { 1067 if (chan_is_logical(d40c)) {
1024 err = d40_channel_execute_command(d40c, 1068 err = d40_channel_execute_command(d40c,
1025 D40_DMA_SUSPEND_REQ); 1069 D40_DMA_SUSPEND_REQ);
1026 if (err) 1070 if (err)
@@ -1028,7 +1072,7 @@ static int d40_start(struct d40_chan *d40c)
1028 } 1072 }
1029 } 1073 }
1030 1074
1031 if (d40c->log_num != D40_PHY_CHAN) 1075 if (chan_is_logical(d40c))
1032 d40_config_set_event(d40c, true); 1076 d40_config_set_event(d40c, true);
1033 1077
1034 return d40_channel_execute_command(d40c, D40_DMA_RUN); 1078 return d40_channel_execute_command(d40c, D40_DMA_RUN);
@@ -1051,21 +1095,14 @@ static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
1051 /* Add to active queue */ 1095 /* Add to active queue */
1052 d40_desc_submit(d40c, d40d); 1096 d40_desc_submit(d40c, d40d);
1053 1097
1054 /* 1098 /* Initiate DMA job */
1055 * If this job is already linked in hw, 1099 d40_desc_load(d40c, d40d);
1056 * do not submit it.
1057 */
1058
1059 if (!d40d->is_hw_linked) {
1060 /* Initiate DMA job */
1061 d40_desc_load(d40c, d40d);
1062 1100
1063 /* Start dma job */ 1101 /* Start dma job */
1064 err = d40_start(d40c); 1102 err = d40_start(d40c);
1065 1103
1066 if (err) 1104 if (err)
1067 return NULL; 1105 return NULL;
1068 }
1069 } 1106 }
1070 1107
1071 return d40d; 1108 return d40d;
@@ -1082,17 +1119,36 @@ static void dma_tc_handle(struct d40_chan *d40c)
1082 if (d40d == NULL) 1119 if (d40d == NULL)
1083 return; 1120 return;
1084 1121
1085 d40_lcla_free_all(d40c, d40d); 1122 if (d40d->cyclic) {
1123 /*
1124 * If this was a paritially loaded list, we need to reloaded
1125 * it, and only when the list is completed. We need to check
1126 * for done because the interrupt will hit for every link, and
1127 * not just the last one.
1128 */
1129 if (d40d->lli_current < d40d->lli_len
1130 && !d40_tx_is_linked(d40c)
1131 && !d40_residue(d40c)) {
1132 d40_lcla_free_all(d40c, d40d);
1133 d40_desc_load(d40c, d40d);
1134 (void) d40_start(d40c);
1086 1135
1087 if (d40d->lli_current < d40d->lli_len) { 1136 if (d40d->lli_current == d40d->lli_len)
1088 d40_desc_load(d40c, d40d); 1137 d40d->lli_current = 0;
1089 /* Start dma job */ 1138 }
1090 (void) d40_start(d40c); 1139 } else {
1091 return; 1140 d40_lcla_free_all(d40c, d40d);
1092 }
1093 1141
1094 if (d40_queue_start(d40c) == NULL) 1142 if (d40d->lli_current < d40d->lli_len) {
1095 d40c->busy = false; 1143 d40_desc_load(d40c, d40d);
1144 /* Start dma job */
1145 (void) d40_start(d40c);
1146 return;
1147 }
1148
1149 if (d40_queue_start(d40c) == NULL)
1150 d40c->busy = false;
1151 }
1096 1152
1097 d40c->pending_tx++; 1153 d40c->pending_tx++;
1098 tasklet_schedule(&d40c->tasklet); 1154 tasklet_schedule(&d40c->tasklet);
@@ -1111,11 +1167,11 @@ static void dma_tasklet(unsigned long data)
1111 1167
1112 /* Get first active entry from list */ 1168 /* Get first active entry from list */
1113 d40d = d40_first_active_get(d40c); 1169 d40d = d40_first_active_get(d40c);
1114
1115 if (d40d == NULL) 1170 if (d40d == NULL)
1116 goto err; 1171 goto err;
1117 1172
1118 d40c->completed = d40d->txd.cookie; 1173 if (!d40d->cyclic)
1174 d40c->completed = d40d->txd.cookie;
1119 1175
1120 /* 1176 /*
1121 * If terminating a channel pending_tx is set to zero. 1177 * If terminating a channel pending_tx is set to zero.
@@ -1130,16 +1186,18 @@ static void dma_tasklet(unsigned long data)
1130 callback = d40d->txd.callback; 1186 callback = d40d->txd.callback;
1131 callback_param = d40d->txd.callback_param; 1187 callback_param = d40d->txd.callback_param;
1132 1188
1133 if (async_tx_test_ack(&d40d->txd)) { 1189 if (!d40d->cyclic) {
1134 d40_pool_lli_free(d40d); 1190 if (async_tx_test_ack(&d40d->txd)) {
1135 d40_desc_remove(d40d); 1191 d40_pool_lli_free(d40c, d40d);
1136 d40_desc_free(d40c, d40d);
1137 } else {
1138 if (!d40d->is_in_client_list) {
1139 d40_desc_remove(d40d); 1192 d40_desc_remove(d40d);
1140 d40_lcla_free_all(d40c, d40d); 1193 d40_desc_free(d40c, d40d);
1141 list_add_tail(&d40d->node, &d40c->client); 1194 } else {
1142 d40d->is_in_client_list = true; 1195 if (!d40d->is_in_client_list) {
1196 d40_desc_remove(d40d);
1197 d40_lcla_free_all(d40c, d40d);
1198 list_add_tail(&d40d->node, &d40c->client);
1199 d40d->is_in_client_list = true;
1200 }
1143 } 1201 }
1144 } 1202 }
1145 1203
@@ -1216,9 +1274,8 @@ static irqreturn_t d40_handle_interrupt(int irq, void *data)
1216 if (!il[row].is_error) 1274 if (!il[row].is_error)
1217 dma_tc_handle(d40c); 1275 dma_tc_handle(d40c);
1218 else 1276 else
1219 dev_err(base->dev, 1277 d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
1220 "[%s] IRQ chan: %ld offset %d idx %d\n", 1278 chan, il[row].offset, idx);
1221 __func__, chan, il[row].offset, idx);
1222 1279
1223 spin_unlock(&d40c->lock); 1280 spin_unlock(&d40c->lock);
1224 } 1281 }
@@ -1237,8 +1294,7 @@ static int d40_validate_conf(struct d40_chan *d40c,
1237 bool is_log = conf->mode == STEDMA40_MODE_LOGICAL; 1294 bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
1238 1295
1239 if (!conf->dir) { 1296 if (!conf->dir) {
1240 dev_err(&d40c->chan.dev->device, "[%s] Invalid direction.\n", 1297 chan_err(d40c, "Invalid direction.\n");
1241 __func__);
1242 res = -EINVAL; 1298 res = -EINVAL;
1243 } 1299 }
1244 1300
@@ -1246,46 +1302,40 @@ static int d40_validate_conf(struct d40_chan *d40c,
1246 d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 && 1302 d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
1247 d40c->runtime_addr == 0) { 1303 d40c->runtime_addr == 0) {
1248 1304
1249 dev_err(&d40c->chan.dev->device, 1305 chan_err(d40c, "Invalid TX channel address (%d)\n",
1250 "[%s] Invalid TX channel address (%d)\n", 1306 conf->dst_dev_type);
1251 __func__, conf->dst_dev_type);
1252 res = -EINVAL; 1307 res = -EINVAL;
1253 } 1308 }
1254 1309
1255 if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY && 1310 if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1256 d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 && 1311 d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1257 d40c->runtime_addr == 0) { 1312 d40c->runtime_addr == 0) {
1258 dev_err(&d40c->chan.dev->device, 1313 chan_err(d40c, "Invalid RX channel address (%d)\n",
1259 "[%s] Invalid RX channel address (%d)\n", 1314 conf->src_dev_type);
1260 __func__, conf->src_dev_type);
1261 res = -EINVAL; 1315 res = -EINVAL;
1262 } 1316 }
1263 1317
1264 if (conf->dir == STEDMA40_MEM_TO_PERIPH && 1318 if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
1265 dst_event_group == STEDMA40_DEV_DST_MEMORY) { 1319 dst_event_group == STEDMA40_DEV_DST_MEMORY) {
1266 dev_err(&d40c->chan.dev->device, "[%s] Invalid dst\n", 1320 chan_err(d40c, "Invalid dst\n");
1267 __func__);
1268 res = -EINVAL; 1321 res = -EINVAL;
1269 } 1322 }
1270 1323
1271 if (conf->dir == STEDMA40_PERIPH_TO_MEM && 1324 if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
1272 src_event_group == STEDMA40_DEV_SRC_MEMORY) { 1325 src_event_group == STEDMA40_DEV_SRC_MEMORY) {
1273 dev_err(&d40c->chan.dev->device, "[%s] Invalid src\n", 1326 chan_err(d40c, "Invalid src\n");
1274 __func__);
1275 res = -EINVAL; 1327 res = -EINVAL;
1276 } 1328 }
1277 1329
1278 if (src_event_group == STEDMA40_DEV_SRC_MEMORY && 1330 if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1279 dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) { 1331 dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
1280 dev_err(&d40c->chan.dev->device, 1332 chan_err(d40c, "No event line\n");
1281 "[%s] No event line\n", __func__);
1282 res = -EINVAL; 1333 res = -EINVAL;
1283 } 1334 }
1284 1335
1285 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH && 1336 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1286 (src_event_group != dst_event_group)) { 1337 (src_event_group != dst_event_group)) {
1287 dev_err(&d40c->chan.dev->device, 1338 chan_err(d40c, "Invalid event group\n");
1288 "[%s] Invalid event group\n", __func__);
1289 res = -EINVAL; 1339 res = -EINVAL;
1290 } 1340 }
1291 1341
@@ -1294,9 +1344,7 @@ static int d40_validate_conf(struct d40_chan *d40c,
1294 * DMAC HW supports it. Will be added to this driver, 1344 * DMAC HW supports it. Will be added to this driver,
1295 * in case any dma client requires it. 1345 * in case any dma client requires it.
1296 */ 1346 */
1297 dev_err(&d40c->chan.dev->device, 1347 chan_err(d40c, "periph to periph not supported\n");
1298 "[%s] periph to periph not supported\n",
1299 __func__);
1300 res = -EINVAL; 1348 res = -EINVAL;
1301 } 1349 }
1302 1350
@@ -1309,9 +1357,7 @@ static int d40_validate_conf(struct d40_chan *d40c,
1309 * src (burst x width) == dst (burst x width) 1357 * src (burst x width) == dst (burst x width)
1310 */ 1358 */
1311 1359
1312 dev_err(&d40c->chan.dev->device, 1360 chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
1313 "[%s] src (burst x width) != dst (burst x width)\n",
1314 __func__);
1315 res = -EINVAL; 1361 res = -EINVAL;
1316 } 1362 }
1317 1363
@@ -1514,8 +1560,7 @@ static int d40_config_memcpy(struct d40_chan *d40c)
1514 dma_has_cap(DMA_SLAVE, cap)) { 1560 dma_has_cap(DMA_SLAVE, cap)) {
1515 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy; 1561 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1516 } else { 1562 } else {
1517 dev_err(&d40c->chan.dev->device, "[%s] No memcpy\n", 1563 chan_err(d40c, "No memcpy\n");
1518 __func__);
1519 return -EINVAL; 1564 return -EINVAL;
1520 } 1565 }
1521 1566
@@ -1540,21 +1585,19 @@ static int d40_free_dma(struct d40_chan *d40c)
1540 /* Release client owned descriptors */ 1585 /* Release client owned descriptors */
1541 if (!list_empty(&d40c->client)) 1586 if (!list_empty(&d40c->client))
1542 list_for_each_entry_safe(d, _d, &d40c->client, node) { 1587 list_for_each_entry_safe(d, _d, &d40c->client, node) {
1543 d40_pool_lli_free(d); 1588 d40_pool_lli_free(d40c, d);
1544 d40_desc_remove(d); 1589 d40_desc_remove(d);
1545 d40_desc_free(d40c, d); 1590 d40_desc_free(d40c, d);
1546 } 1591 }
1547 1592
1548 if (phy == NULL) { 1593 if (phy == NULL) {
1549 dev_err(&d40c->chan.dev->device, "[%s] phy == null\n", 1594 chan_err(d40c, "phy == null\n");
1550 __func__);
1551 return -EINVAL; 1595 return -EINVAL;
1552 } 1596 }
1553 1597
1554 if (phy->allocated_src == D40_ALLOC_FREE && 1598 if (phy->allocated_src == D40_ALLOC_FREE &&
1555 phy->allocated_dst == D40_ALLOC_FREE) { 1599 phy->allocated_dst == D40_ALLOC_FREE) {
1556 dev_err(&d40c->chan.dev->device, "[%s] channel already free\n", 1600 chan_err(d40c, "channel already free\n");
1557 __func__);
1558 return -EINVAL; 1601 return -EINVAL;
1559 } 1602 }
1560 1603
@@ -1566,19 +1609,17 @@ static int d40_free_dma(struct d40_chan *d40c)
1566 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); 1609 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1567 is_src = true; 1610 is_src = true;
1568 } else { 1611 } else {
1569 dev_err(&d40c->chan.dev->device, 1612 chan_err(d40c, "Unknown direction\n");
1570 "[%s] Unknown direction\n", __func__);
1571 return -EINVAL; 1613 return -EINVAL;
1572 } 1614 }
1573 1615
1574 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); 1616 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1575 if (res) { 1617 if (res) {
1576 dev_err(&d40c->chan.dev->device, "[%s] suspend failed\n", 1618 chan_err(d40c, "suspend failed\n");
1577 __func__);
1578 return res; 1619 return res;
1579 } 1620 }
1580 1621
1581 if (d40c->log_num != D40_PHY_CHAN) { 1622 if (chan_is_logical(d40c)) {
1582 /* Release logical channel, deactivate the event line */ 1623 /* Release logical channel, deactivate the event line */
1583 1624
1584 d40_config_set_event(d40c, false); 1625 d40_config_set_event(d40c, false);
@@ -1594,9 +1635,8 @@ static int d40_free_dma(struct d40_chan *d40c)
1594 res = d40_channel_execute_command(d40c, 1635 res = d40_channel_execute_command(d40c,
1595 D40_DMA_RUN); 1636 D40_DMA_RUN);
1596 if (res) { 1637 if (res) {
1597 dev_err(&d40c->chan.dev->device, 1638 chan_err(d40c,
1598 "[%s] Executing RUN command\n", 1639 "Executing RUN command\n");
1599 __func__);
1600 return res; 1640 return res;
1601 } 1641 }
1602 } 1642 }
@@ -1609,8 +1649,7 @@ static int d40_free_dma(struct d40_chan *d40c)
1609 /* Release physical channel */ 1649 /* Release physical channel */
1610 res = d40_channel_execute_command(d40c, D40_DMA_STOP); 1650 res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1611 if (res) { 1651 if (res) {
1612 dev_err(&d40c->chan.dev->device, 1652 chan_err(d40c, "Failed to stop channel\n");
1613 "[%s] Failed to stop channel\n", __func__);
1614 return res; 1653 return res;
1615 } 1654 }
1616 d40c->phy_chan = NULL; 1655 d40c->phy_chan = NULL;
@@ -1622,6 +1661,7 @@ static int d40_free_dma(struct d40_chan *d40c)
1622 1661
1623static bool d40_is_paused(struct d40_chan *d40c) 1662static bool d40_is_paused(struct d40_chan *d40c)
1624{ 1663{
1664 void __iomem *chanbase = chan_base(d40c);
1625 bool is_paused = false; 1665 bool is_paused = false;
1626 unsigned long flags; 1666 unsigned long flags;
1627 void __iomem *active_reg; 1667 void __iomem *active_reg;
@@ -1630,7 +1670,7 @@ static bool d40_is_paused(struct d40_chan *d40c)
1630 1670
1631 spin_lock_irqsave(&d40c->lock, flags); 1671 spin_lock_irqsave(&d40c->lock, flags);
1632 1672
1633 if (d40c->log_num == D40_PHY_CHAN) { 1673 if (chan_is_physical(d40c)) {
1634 if (d40c->phy_chan->num % 2 == 0) 1674 if (d40c->phy_chan->num % 2 == 0)
1635 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; 1675 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1636 else 1676 else
@@ -1648,17 +1688,12 @@ static bool d40_is_paused(struct d40_chan *d40c)
1648 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || 1688 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1649 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { 1689 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1650 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); 1690 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1651 status = readl(d40c->base->virtbase + D40_DREG_PCBASE + 1691 status = readl(chanbase + D40_CHAN_REG_SDLNK);
1652 d40c->phy_chan->num * D40_DREG_PCDELTA +
1653 D40_CHAN_REG_SDLNK);
1654 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { 1692 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1655 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); 1693 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1656 status = readl(d40c->base->virtbase + D40_DREG_PCBASE + 1694 status = readl(chanbase + D40_CHAN_REG_SSLNK);
1657 d40c->phy_chan->num * D40_DREG_PCDELTA +
1658 D40_CHAN_REG_SSLNK);
1659 } else { 1695 } else {
1660 dev_err(&d40c->chan.dev->device, 1696 chan_err(d40c, "Unknown direction\n");
1661 "[%s] Unknown direction\n", __func__);
1662 goto _exit; 1697 goto _exit;
1663 } 1698 }
1664 1699
@@ -1688,114 +1723,184 @@ static u32 stedma40_residue(struct dma_chan *chan)
1688 return bytes_left; 1723 return bytes_left;
1689} 1724}
1690 1725
1691struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan, 1726static int
1692 struct scatterlist *sgl_dst, 1727d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
1693 struct scatterlist *sgl_src, 1728 struct scatterlist *sg_src, struct scatterlist *sg_dst,
1694 unsigned int sgl_len, 1729 unsigned int sg_len, dma_addr_t src_dev_addr,
1695 unsigned long dma_flags) 1730 dma_addr_t dst_dev_addr)
1696{ 1731{
1697 int res; 1732 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1698 struct d40_desc *d40d; 1733 struct stedma40_half_channel_info *src_info = &cfg->src_info;
1699 struct d40_chan *d40c = container_of(chan, struct d40_chan, 1734 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
1700 chan); 1735 int ret;
1701 unsigned long flags;
1702 1736
1703 if (d40c->phy_chan == NULL) { 1737 ret = d40_log_sg_to_lli(sg_src, sg_len,
1704 dev_err(&d40c->chan.dev->device, 1738 src_dev_addr,
1705 "[%s] Unallocated channel.\n", __func__); 1739 desc->lli_log.src,
1706 return ERR_PTR(-EINVAL); 1740 chan->log_def.lcsp1,
1707 } 1741 src_info->data_width,
1742 dst_info->data_width);
1708 1743
1709 spin_lock_irqsave(&d40c->lock, flags); 1744 ret = d40_log_sg_to_lli(sg_dst, sg_len,
1710 d40d = d40_desc_get(d40c); 1745 dst_dev_addr,
1746 desc->lli_log.dst,
1747 chan->log_def.lcsp3,
1748 dst_info->data_width,
1749 src_info->data_width);
1711 1750
1712 if (d40d == NULL) 1751 return ret < 0 ? ret : 0;
1752}
1753
1754static int
1755d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
1756 struct scatterlist *sg_src, struct scatterlist *sg_dst,
1757 unsigned int sg_len, dma_addr_t src_dev_addr,
1758 dma_addr_t dst_dev_addr)
1759{
1760 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1761 struct stedma40_half_channel_info *src_info = &cfg->src_info;
1762 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
1763 unsigned long flags = 0;
1764 int ret;
1765
1766 if (desc->cyclic)
1767 flags |= LLI_CYCLIC | LLI_TERM_INT;
1768
1769 ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
1770 desc->lli_phy.src,
1771 virt_to_phys(desc->lli_phy.src),
1772 chan->src_def_cfg,
1773 src_info, dst_info, flags);
1774
1775 ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
1776 desc->lli_phy.dst,
1777 virt_to_phys(desc->lli_phy.dst),
1778 chan->dst_def_cfg,
1779 dst_info, src_info, flags);
1780
1781 dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
1782 desc->lli_pool.size, DMA_TO_DEVICE);
1783
1784 return ret < 0 ? ret : 0;
1785}
1786
1787
1788static struct d40_desc *
1789d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
1790 unsigned int sg_len, unsigned long dma_flags)
1791{
1792 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1793 struct d40_desc *desc;
1794 int ret;
1795
1796 desc = d40_desc_get(chan);
1797 if (!desc)
1798 return NULL;
1799
1800 desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
1801 cfg->dst_info.data_width);
1802 if (desc->lli_len < 0) {
1803 chan_err(chan, "Unaligned size\n");
1713 goto err; 1804 goto err;
1805 }
1714 1806
1715 d40d->lli_len = d40_sg_2_dmalen(sgl_dst, sgl_len, 1807 ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
1716 d40c->dma_cfg.src_info.data_width, 1808 if (ret < 0) {
1717 d40c->dma_cfg.dst_info.data_width); 1809 chan_err(chan, "Could not allocate lli\n");
1718 if (d40d->lli_len < 0) {
1719 dev_err(&d40c->chan.dev->device,
1720 "[%s] Unaligned size\n", __func__);
1721 goto err; 1810 goto err;
1722 } 1811 }
1723 1812
1724 d40d->lli_current = 0;
1725 d40d->txd.flags = dma_flags;
1726 1813
1727 if (d40c->log_num != D40_PHY_CHAN) { 1814 desc->lli_current = 0;
1815 desc->txd.flags = dma_flags;
1816 desc->txd.tx_submit = d40_tx_submit;
1728 1817
1729 if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) { 1818 dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
1730 dev_err(&d40c->chan.dev->device,
1731 "[%s] Out of memory\n", __func__);
1732 goto err;
1733 }
1734 1819
1735 (void) d40_log_sg_to_lli(sgl_src, 1820 return desc;
1736 sgl_len, 1821
1737 d40d->lli_log.src, 1822err:
1738 d40c->log_def.lcsp1, 1823 d40_desc_free(chan, desc);
1739 d40c->dma_cfg.src_info.data_width, 1824 return NULL;
1740 d40c->dma_cfg.dst_info.data_width); 1825}
1741 1826
1742 (void) d40_log_sg_to_lli(sgl_dst, 1827static dma_addr_t
1743 sgl_len, 1828d40_get_dev_addr(struct d40_chan *chan, enum dma_data_direction direction)
1744 d40d->lli_log.dst, 1829{
1745 d40c->log_def.lcsp3, 1830 struct stedma40_platform_data *plat = chan->base->plat_data;
1746 d40c->dma_cfg.dst_info.data_width, 1831 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1747 d40c->dma_cfg.src_info.data_width); 1832 dma_addr_t addr;
1748 } else {
1749 if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
1750 dev_err(&d40c->chan.dev->device,
1751 "[%s] Out of memory\n", __func__);
1752 goto err;
1753 }
1754 1833
1755 res = d40_phy_sg_to_lli(sgl_src, 1834 if (chan->runtime_addr)
1756 sgl_len, 1835 return chan->runtime_addr;
1757 0,
1758 d40d->lli_phy.src,
1759 virt_to_phys(d40d->lli_phy.src),
1760 d40c->src_def_cfg,
1761 d40c->dma_cfg.src_info.data_width,
1762 d40c->dma_cfg.dst_info.data_width,
1763 d40c->dma_cfg.src_info.psize);
1764 1836
1765 if (res < 0) 1837 if (direction == DMA_FROM_DEVICE)
1766 goto err; 1838 addr = plat->dev_rx[cfg->src_dev_type];
1839 else if (direction == DMA_TO_DEVICE)
1840 addr = plat->dev_tx[cfg->dst_dev_type];
1767 1841
1768 res = d40_phy_sg_to_lli(sgl_dst, 1842 return addr;
1769 sgl_len, 1843}
1770 0,
1771 d40d->lli_phy.dst,
1772 virt_to_phys(d40d->lli_phy.dst),
1773 d40c->dst_def_cfg,
1774 d40c->dma_cfg.dst_info.data_width,
1775 d40c->dma_cfg.src_info.data_width,
1776 d40c->dma_cfg.dst_info.psize);
1777 1844
1778 if (res < 0) 1845static struct dma_async_tx_descriptor *
1779 goto err; 1846d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
1847 struct scatterlist *sg_dst, unsigned int sg_len,
1848 enum dma_data_direction direction, unsigned long dma_flags)
1849{
1850 struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
1851 dma_addr_t src_dev_addr = 0;
1852 dma_addr_t dst_dev_addr = 0;
1853 struct d40_desc *desc;
1854 unsigned long flags;
1855 int ret;
1780 1856
1781 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src, 1857 if (!chan->phy_chan) {
1782 d40d->lli_pool.size, DMA_TO_DEVICE); 1858 chan_err(chan, "Cannot prepare unallocated channel\n");
1859 return NULL;
1783 } 1860 }
1784 1861
1785 dma_async_tx_descriptor_init(&d40d->txd, chan);
1786 1862
1787 d40d->txd.tx_submit = d40_tx_submit; 1863 spin_lock_irqsave(&chan->lock, flags);
1788 1864
1789 spin_unlock_irqrestore(&d40c->lock, flags); 1865 desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
1866 if (desc == NULL)
1867 goto err;
1868
1869 if (sg_next(&sg_src[sg_len - 1]) == sg_src)
1870 desc->cyclic = true;
1871
1872 if (direction != DMA_NONE) {
1873 dma_addr_t dev_addr = d40_get_dev_addr(chan, direction);
1874
1875 if (direction == DMA_FROM_DEVICE)
1876 src_dev_addr = dev_addr;
1877 else if (direction == DMA_TO_DEVICE)
1878 dst_dev_addr = dev_addr;
1879 }
1880
1881 if (chan_is_logical(chan))
1882 ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
1883 sg_len, src_dev_addr, dst_dev_addr);
1884 else
1885 ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
1886 sg_len, src_dev_addr, dst_dev_addr);
1887
1888 if (ret) {
1889 chan_err(chan, "Failed to prepare %s sg job: %d\n",
1890 chan_is_logical(chan) ? "log" : "phy", ret);
1891 goto err;
1892 }
1893
1894 spin_unlock_irqrestore(&chan->lock, flags);
1895
1896 return &desc->txd;
1790 1897
1791 return &d40d->txd;
1792err: 1898err:
1793 if (d40d) 1899 if (desc)
1794 d40_desc_free(d40c, d40d); 1900 d40_desc_free(chan, desc);
1795 spin_unlock_irqrestore(&d40c->lock, flags); 1901 spin_unlock_irqrestore(&chan->lock, flags);
1796 return NULL; 1902 return NULL;
1797} 1903}
1798EXPORT_SYMBOL(stedma40_memcpy_sg);
1799 1904
1800bool stedma40_filter(struct dma_chan *chan, void *data) 1905bool stedma40_filter(struct dma_chan *chan, void *data)
1801{ 1906{
@@ -1818,6 +1923,38 @@ bool stedma40_filter(struct dma_chan *chan, void *data)
1818} 1923}
1819EXPORT_SYMBOL(stedma40_filter); 1924EXPORT_SYMBOL(stedma40_filter);
1820 1925
1926static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
1927{
1928 bool realtime = d40c->dma_cfg.realtime;
1929 bool highprio = d40c->dma_cfg.high_priority;
1930 u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1;
1931 u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1;
1932 u32 event = D40_TYPE_TO_EVENT(dev_type);
1933 u32 group = D40_TYPE_TO_GROUP(dev_type);
1934 u32 bit = 1 << event;
1935
1936 /* Destination event lines are stored in the upper halfword */
1937 if (!src)
1938 bit <<= 16;
1939
1940 writel(bit, d40c->base->virtbase + prioreg + group * 4);
1941 writel(bit, d40c->base->virtbase + rtreg + group * 4);
1942}
1943
1944static void d40_set_prio_realtime(struct d40_chan *d40c)
1945{
1946 if (d40c->base->rev < 3)
1947 return;
1948
1949 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
1950 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1951 __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true);
1952
1953 if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) ||
1954 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1955 __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false);
1956}
1957
1821/* DMA ENGINE functions */ 1958/* DMA ENGINE functions */
1822static int d40_alloc_chan_resources(struct dma_chan *chan) 1959static int d40_alloc_chan_resources(struct dma_chan *chan)
1823{ 1960{
@@ -1834,9 +1971,7 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
1834 if (!d40c->configured) { 1971 if (!d40c->configured) {
1835 err = d40_config_memcpy(d40c); 1972 err = d40_config_memcpy(d40c);
1836 if (err) { 1973 if (err) {
1837 dev_err(&d40c->chan.dev->device, 1974 chan_err(d40c, "Failed to configure memcpy channel\n");
1838 "[%s] Failed to configure memcpy channel\n",
1839 __func__);
1840 goto fail; 1975 goto fail;
1841 } 1976 }
1842 } 1977 }
@@ -1844,16 +1979,17 @@ static int d40_alloc_chan_resources(struct dma_chan *chan)
1844 1979
1845 err = d40_allocate_channel(d40c); 1980 err = d40_allocate_channel(d40c);
1846 if (err) { 1981 if (err) {
1847 dev_err(&d40c->chan.dev->device, 1982 chan_err(d40c, "Failed to allocate channel\n");
1848 "[%s] Failed to allocate channel\n", __func__);
1849 goto fail; 1983 goto fail;
1850 } 1984 }
1851 1985
1852 /* Fill in basic CFG register values */ 1986 /* Fill in basic CFG register values */
1853 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg, 1987 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1854 &d40c->dst_def_cfg, d40c->log_num != D40_PHY_CHAN); 1988 &d40c->dst_def_cfg, chan_is_logical(d40c));
1855 1989
1856 if (d40c->log_num != D40_PHY_CHAN) { 1990 d40_set_prio_realtime(d40c);
1991
1992 if (chan_is_logical(d40c)) {
1857 d40_log_cfg(&d40c->dma_cfg, 1993 d40_log_cfg(&d40c->dma_cfg,
1858 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); 1994 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1859 1995
@@ -1886,8 +2022,7 @@ static void d40_free_chan_resources(struct dma_chan *chan)
1886 unsigned long flags; 2022 unsigned long flags;
1887 2023
1888 if (d40c->phy_chan == NULL) { 2024 if (d40c->phy_chan == NULL) {
1889 dev_err(&d40c->chan.dev->device, 2025 chan_err(d40c, "Cannot free unallocated channel\n");
1890 "[%s] Cannot free unallocated channel\n", __func__);
1891 return; 2026 return;
1892 } 2027 }
1893 2028
@@ -1897,8 +2032,7 @@ static void d40_free_chan_resources(struct dma_chan *chan)
1897 err = d40_free_dma(d40c); 2032 err = d40_free_dma(d40c);
1898 2033
1899 if (err) 2034 if (err)
1900 dev_err(&d40c->chan.dev->device, 2035 chan_err(d40c, "Failed to free channel\n");
1901 "[%s] Failed to free channel\n", __func__);
1902 spin_unlock_irqrestore(&d40c->lock, flags); 2036 spin_unlock_irqrestore(&d40c->lock, flags);
1903} 2037}
1904 2038
@@ -1908,251 +2042,31 @@ static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1908 size_t size, 2042 size_t size,
1909 unsigned long dma_flags) 2043 unsigned long dma_flags)
1910{ 2044{
1911 struct d40_desc *d40d; 2045 struct scatterlist dst_sg;
1912 struct d40_chan *d40c = container_of(chan, struct d40_chan, 2046 struct scatterlist src_sg;
1913 chan);
1914 unsigned long flags;
1915
1916 if (d40c->phy_chan == NULL) {
1917 dev_err(&d40c->chan.dev->device,
1918 "[%s] Channel is not allocated.\n", __func__);
1919 return ERR_PTR(-EINVAL);
1920 }
1921
1922 spin_lock_irqsave(&d40c->lock, flags);
1923 d40d = d40_desc_get(d40c);
1924
1925 if (d40d == NULL) {
1926 dev_err(&d40c->chan.dev->device,
1927 "[%s] Descriptor is NULL\n", __func__);
1928 goto err;
1929 }
1930 2047
1931 d40d->txd.flags = dma_flags; 2048 sg_init_table(&dst_sg, 1);
1932 d40d->lli_len = d40_size_2_dmalen(size, 2049 sg_init_table(&src_sg, 1);
1933 d40c->dma_cfg.src_info.data_width,
1934 d40c->dma_cfg.dst_info.data_width);
1935 if (d40d->lli_len < 0) {
1936 dev_err(&d40c->chan.dev->device,
1937 "[%s] Unaligned size\n", __func__);
1938 goto err;
1939 }
1940 2050
2051 sg_dma_address(&dst_sg) = dst;
2052 sg_dma_address(&src_sg) = src;
1941 2053
1942 dma_async_tx_descriptor_init(&d40d->txd, chan); 2054 sg_dma_len(&dst_sg) = size;
2055 sg_dma_len(&src_sg) = size;
1943 2056
1944 d40d->txd.tx_submit = d40_tx_submit; 2057 return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags);
1945
1946 if (d40c->log_num != D40_PHY_CHAN) {
1947
1948 if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
1949 dev_err(&d40c->chan.dev->device,
1950 "[%s] Out of memory\n", __func__);
1951 goto err;
1952 }
1953 d40d->lli_current = 0;
1954
1955 if (d40_log_buf_to_lli(d40d->lli_log.src,
1956 src,
1957 size,
1958 d40c->log_def.lcsp1,
1959 d40c->dma_cfg.src_info.data_width,
1960 d40c->dma_cfg.dst_info.data_width,
1961 true) == NULL)
1962 goto err;
1963
1964 if (d40_log_buf_to_lli(d40d->lli_log.dst,
1965 dst,
1966 size,
1967 d40c->log_def.lcsp3,
1968 d40c->dma_cfg.dst_info.data_width,
1969 d40c->dma_cfg.src_info.data_width,
1970 true) == NULL)
1971 goto err;
1972
1973 } else {
1974
1975 if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
1976 dev_err(&d40c->chan.dev->device,
1977 "[%s] Out of memory\n", __func__);
1978 goto err;
1979 }
1980
1981 if (d40_phy_buf_to_lli(d40d->lli_phy.src,
1982 src,
1983 size,
1984 d40c->dma_cfg.src_info.psize,
1985 0,
1986 d40c->src_def_cfg,
1987 true,
1988 d40c->dma_cfg.src_info.data_width,
1989 d40c->dma_cfg.dst_info.data_width,
1990 false) == NULL)
1991 goto err;
1992
1993 if (d40_phy_buf_to_lli(d40d->lli_phy.dst,
1994 dst,
1995 size,
1996 d40c->dma_cfg.dst_info.psize,
1997 0,
1998 d40c->dst_def_cfg,
1999 true,
2000 d40c->dma_cfg.dst_info.data_width,
2001 d40c->dma_cfg.src_info.data_width,
2002 false) == NULL)
2003 goto err;
2004
2005 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
2006 d40d->lli_pool.size, DMA_TO_DEVICE);
2007 }
2008
2009 spin_unlock_irqrestore(&d40c->lock, flags);
2010 return &d40d->txd;
2011
2012err:
2013 if (d40d)
2014 d40_desc_free(d40c, d40d);
2015 spin_unlock_irqrestore(&d40c->lock, flags);
2016 return NULL;
2017} 2058}
2018 2059
2019static struct dma_async_tx_descriptor * 2060static struct dma_async_tx_descriptor *
2020d40_prep_sg(struct dma_chan *chan, 2061d40_prep_memcpy_sg(struct dma_chan *chan,
2021 struct scatterlist *dst_sg, unsigned int dst_nents, 2062 struct scatterlist *dst_sg, unsigned int dst_nents,
2022 struct scatterlist *src_sg, unsigned int src_nents, 2063 struct scatterlist *src_sg, unsigned int src_nents,
2023 unsigned long dma_flags) 2064 unsigned long dma_flags)
2024{ 2065{
2025 if (dst_nents != src_nents) 2066 if (dst_nents != src_nents)
2026 return NULL; 2067 return NULL;
2027 2068
2028 return stedma40_memcpy_sg(chan, dst_sg, src_sg, dst_nents, dma_flags); 2069 return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags);
2029}
2030
2031static int d40_prep_slave_sg_log(struct d40_desc *d40d,
2032 struct d40_chan *d40c,
2033 struct scatterlist *sgl,
2034 unsigned int sg_len,
2035 enum dma_data_direction direction,
2036 unsigned long dma_flags)
2037{
2038 dma_addr_t dev_addr = 0;
2039 int total_size;
2040
2041 d40d->lli_len = d40_sg_2_dmalen(sgl, sg_len,
2042 d40c->dma_cfg.src_info.data_width,
2043 d40c->dma_cfg.dst_info.data_width);
2044 if (d40d->lli_len < 0) {
2045 dev_err(&d40c->chan.dev->device,
2046 "[%s] Unaligned size\n", __func__);
2047 return -EINVAL;
2048 }
2049
2050 if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
2051 dev_err(&d40c->chan.dev->device,
2052 "[%s] Out of memory\n", __func__);
2053 return -ENOMEM;
2054 }
2055
2056 d40d->lli_current = 0;
2057
2058 if (direction == DMA_FROM_DEVICE)
2059 if (d40c->runtime_addr)
2060 dev_addr = d40c->runtime_addr;
2061 else
2062 dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
2063 else if (direction == DMA_TO_DEVICE)
2064 if (d40c->runtime_addr)
2065 dev_addr = d40c->runtime_addr;
2066 else
2067 dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
2068
2069 else
2070 return -EINVAL;
2071
2072 total_size = d40_log_sg_to_dev(sgl, sg_len,
2073 &d40d->lli_log,
2074 &d40c->log_def,
2075 d40c->dma_cfg.src_info.data_width,
2076 d40c->dma_cfg.dst_info.data_width,
2077 direction,
2078 dev_addr);
2079
2080 if (total_size < 0)
2081 return -EINVAL;
2082
2083 return 0;
2084}
2085
2086static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
2087 struct d40_chan *d40c,
2088 struct scatterlist *sgl,
2089 unsigned int sgl_len,
2090 enum dma_data_direction direction,
2091 unsigned long dma_flags)
2092{
2093 dma_addr_t src_dev_addr;
2094 dma_addr_t dst_dev_addr;
2095 int res;
2096
2097 d40d->lli_len = d40_sg_2_dmalen(sgl, sgl_len,
2098 d40c->dma_cfg.src_info.data_width,
2099 d40c->dma_cfg.dst_info.data_width);
2100 if (d40d->lli_len < 0) {
2101 dev_err(&d40c->chan.dev->device,
2102 "[%s] Unaligned size\n", __func__);
2103 return -EINVAL;
2104 }
2105
2106 if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
2107 dev_err(&d40c->chan.dev->device,
2108 "[%s] Out of memory\n", __func__);
2109 return -ENOMEM;
2110 }
2111
2112 d40d->lli_current = 0;
2113
2114 if (direction == DMA_FROM_DEVICE) {
2115 dst_dev_addr = 0;
2116 if (d40c->runtime_addr)
2117 src_dev_addr = d40c->runtime_addr;
2118 else
2119 src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
2120 } else if (direction == DMA_TO_DEVICE) {
2121 if (d40c->runtime_addr)
2122 dst_dev_addr = d40c->runtime_addr;
2123 else
2124 dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
2125 src_dev_addr = 0;
2126 } else
2127 return -EINVAL;
2128
2129 res = d40_phy_sg_to_lli(sgl,
2130 sgl_len,
2131 src_dev_addr,
2132 d40d->lli_phy.src,
2133 virt_to_phys(d40d->lli_phy.src),
2134 d40c->src_def_cfg,
2135 d40c->dma_cfg.src_info.data_width,
2136 d40c->dma_cfg.dst_info.data_width,
2137 d40c->dma_cfg.src_info.psize);
2138 if (res < 0)
2139 return res;
2140
2141 res = d40_phy_sg_to_lli(sgl,
2142 sgl_len,
2143 dst_dev_addr,
2144 d40d->lli_phy.dst,
2145 virt_to_phys(d40d->lli_phy.dst),
2146 d40c->dst_def_cfg,
2147 d40c->dma_cfg.dst_info.data_width,
2148 d40c->dma_cfg.src_info.data_width,
2149 d40c->dma_cfg.dst_info.psize);
2150 if (res < 0)
2151 return res;
2152
2153 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
2154 d40d->lli_pool.size, DMA_TO_DEVICE);
2155 return 0;
2156} 2070}
2157 2071
2158static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan, 2072static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
@@ -2161,52 +2075,40 @@ static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
2161 enum dma_data_direction direction, 2075 enum dma_data_direction direction,
2162 unsigned long dma_flags) 2076 unsigned long dma_flags)
2163{ 2077{
2164 struct d40_desc *d40d; 2078 if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE)
2165 struct d40_chan *d40c = container_of(chan, struct d40_chan, 2079 return NULL;
2166 chan);
2167 unsigned long flags;
2168 int err;
2169
2170 if (d40c->phy_chan == NULL) {
2171 dev_err(&d40c->chan.dev->device,
2172 "[%s] Cannot prepare unallocated channel\n", __func__);
2173 return ERR_PTR(-EINVAL);
2174 }
2175 2080
2176 spin_lock_irqsave(&d40c->lock, flags); 2081 return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);
2177 d40d = d40_desc_get(d40c); 2082}
2178 2083
2179 if (d40d == NULL) 2084static struct dma_async_tx_descriptor *
2180 goto err; 2085dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
2086 size_t buf_len, size_t period_len,
2087 enum dma_data_direction direction)
2088{
2089 unsigned int periods = buf_len / period_len;
2090 struct dma_async_tx_descriptor *txd;
2091 struct scatterlist *sg;
2092 int i;
2181 2093
2182 if (d40c->log_num != D40_PHY_CHAN) 2094 sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_KERNEL);
2183 err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len, 2095 for (i = 0; i < periods; i++) {
2184 direction, dma_flags); 2096 sg_dma_address(&sg[i]) = dma_addr;
2185 else 2097 sg_dma_len(&sg[i]) = period_len;
2186 err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len, 2098 dma_addr += period_len;
2187 direction, dma_flags);
2188 if (err) {
2189 dev_err(&d40c->chan.dev->device,
2190 "[%s] Failed to prepare %s slave sg job: %d\n",
2191 __func__,
2192 d40c->log_num != D40_PHY_CHAN ? "log" : "phy", err);
2193 goto err;
2194 } 2099 }
2195 2100
2196 d40d->txd.flags = dma_flags; 2101 sg[periods].offset = 0;
2102 sg[periods].length = 0;
2103 sg[periods].page_link =
2104 ((unsigned long)sg | 0x01) & ~0x02;
2197 2105
2198 dma_async_tx_descriptor_init(&d40d->txd, chan); 2106 txd = d40_prep_sg(chan, sg, sg, periods, direction,
2107 DMA_PREP_INTERRUPT);
2199 2108
2200 d40d->txd.tx_submit = d40_tx_submit; 2109 kfree(sg);
2201 2110
2202 spin_unlock_irqrestore(&d40c->lock, flags); 2111 return txd;
2203 return &d40d->txd;
2204
2205err:
2206 if (d40d)
2207 d40_desc_free(d40c, d40d);
2208 spin_unlock_irqrestore(&d40c->lock, flags);
2209 return NULL;
2210} 2112}
2211 2113
2212static enum dma_status d40_tx_status(struct dma_chan *chan, 2114static enum dma_status d40_tx_status(struct dma_chan *chan,
@@ -2219,9 +2121,7 @@ static enum dma_status d40_tx_status(struct dma_chan *chan,
2219 int ret; 2121 int ret;
2220 2122
2221 if (d40c->phy_chan == NULL) { 2123 if (d40c->phy_chan == NULL) {
2222 dev_err(&d40c->chan.dev->device, 2124 chan_err(d40c, "Cannot read status of unallocated channel\n");
2223 "[%s] Cannot read status of unallocated channel\n",
2224 __func__);
2225 return -EINVAL; 2125 return -EINVAL;
2226 } 2126 }
2227 2127
@@ -2245,8 +2145,7 @@ static void d40_issue_pending(struct dma_chan *chan)
2245 unsigned long flags; 2145 unsigned long flags;
2246 2146
2247 if (d40c->phy_chan == NULL) { 2147 if (d40c->phy_chan == NULL) {
2248 dev_err(&d40c->chan.dev->device, 2148 chan_err(d40c, "Channel is not allocated!\n");
2249 "[%s] Channel is not allocated!\n", __func__);
2250 return; 2149 return;
2251 } 2150 }
2252 2151
@@ -2339,7 +2238,7 @@ static void d40_set_runtime_config(struct dma_chan *chan,
2339 return; 2238 return;
2340 } 2239 }
2341 2240
2342 if (d40c->log_num != D40_PHY_CHAN) { 2241 if (chan_is_logical(d40c)) {
2343 if (config_maxburst >= 16) 2242 if (config_maxburst >= 16)
2344 psize = STEDMA40_PSIZE_LOG_16; 2243 psize = STEDMA40_PSIZE_LOG_16;
2345 else if (config_maxburst >= 8) 2244 else if (config_maxburst >= 8)
@@ -2372,7 +2271,7 @@ static void d40_set_runtime_config(struct dma_chan *chan,
2372 cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL; 2271 cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2373 2272
2374 /* Fill in register values */ 2273 /* Fill in register values */
2375 if (d40c->log_num != D40_PHY_CHAN) 2274 if (chan_is_logical(d40c))
2376 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); 2275 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2377 else 2276 else
2378 d40_phy_cfg(cfg, &d40c->src_def_cfg, 2277 d40_phy_cfg(cfg, &d40c->src_def_cfg,
@@ -2393,25 +2292,20 @@ static void d40_set_runtime_config(struct dma_chan *chan,
2393static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, 2292static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2394 unsigned long arg) 2293 unsigned long arg)
2395{ 2294{
2396 unsigned long flags;
2397 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); 2295 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2398 2296
2399 if (d40c->phy_chan == NULL) { 2297 if (d40c->phy_chan == NULL) {
2400 dev_err(&d40c->chan.dev->device, 2298 chan_err(d40c, "Channel is not allocated!\n");
2401 "[%s] Channel is not allocated!\n", __func__);
2402 return -EINVAL; 2299 return -EINVAL;
2403 } 2300 }
2404 2301
2405 switch (cmd) { 2302 switch (cmd) {
2406 case DMA_TERMINATE_ALL: 2303 case DMA_TERMINATE_ALL:
2407 spin_lock_irqsave(&d40c->lock, flags); 2304 return d40_terminate_all(d40c);
2408 d40_term_all(d40c);
2409 spin_unlock_irqrestore(&d40c->lock, flags);
2410 return 0;
2411 case DMA_PAUSE: 2305 case DMA_PAUSE:
2412 return d40_pause(chan); 2306 return d40_pause(d40c);
2413 case DMA_RESUME: 2307 case DMA_RESUME:
2414 return d40_resume(chan); 2308 return d40_resume(d40c);
2415 case DMA_SLAVE_CONFIG: 2309 case DMA_SLAVE_CONFIG:
2416 d40_set_runtime_config(chan, 2310 d40_set_runtime_config(chan,
2417 (struct dma_slave_config *) arg); 2311 (struct dma_slave_config *) arg);
@@ -2456,6 +2350,35 @@ static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2456 } 2350 }
2457} 2351}
2458 2352
2353static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
2354{
2355 if (dma_has_cap(DMA_SLAVE, dev->cap_mask))
2356 dev->device_prep_slave_sg = d40_prep_slave_sg;
2357
2358 if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
2359 dev->device_prep_dma_memcpy = d40_prep_memcpy;
2360
2361 /*
2362 * This controller can only access address at even
2363 * 32bit boundaries, i.e. 2^2
2364 */
2365 dev->copy_align = 2;
2366 }
2367
2368 if (dma_has_cap(DMA_SG, dev->cap_mask))
2369 dev->device_prep_dma_sg = d40_prep_memcpy_sg;
2370
2371 if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
2372 dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
2373
2374 dev->device_alloc_chan_resources = d40_alloc_chan_resources;
2375 dev->device_free_chan_resources = d40_free_chan_resources;
2376 dev->device_issue_pending = d40_issue_pending;
2377 dev->device_tx_status = d40_tx_status;
2378 dev->device_control = d40_control;
2379 dev->dev = base->dev;
2380}
2381
2459static int __init d40_dmaengine_init(struct d40_base *base, 2382static int __init d40_dmaengine_init(struct d40_base *base,
2460 int num_reserved_chans) 2383 int num_reserved_chans)
2461{ 2384{
@@ -2466,23 +2389,14 @@ static int __init d40_dmaengine_init(struct d40_base *base,
2466 2389
2467 dma_cap_zero(base->dma_slave.cap_mask); 2390 dma_cap_zero(base->dma_slave.cap_mask);
2468 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask); 2391 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2392 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
2469 2393
2470 base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources; 2394 d40_ops_init(base, &base->dma_slave);
2471 base->dma_slave.device_free_chan_resources = d40_free_chan_resources;
2472 base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy;
2473 base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2474 base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg;
2475 base->dma_slave.device_tx_status = d40_tx_status;
2476 base->dma_slave.device_issue_pending = d40_issue_pending;
2477 base->dma_slave.device_control = d40_control;
2478 base->dma_slave.dev = base->dev;
2479 2395
2480 err = dma_async_device_register(&base->dma_slave); 2396 err = dma_async_device_register(&base->dma_slave);
2481 2397
2482 if (err) { 2398 if (err) {
2483 dev_err(base->dev, 2399 d40_err(base->dev, "Failed to register slave channels\n");
2484 "[%s] Failed to register slave channels\n",
2485 __func__);
2486 goto failure1; 2400 goto failure1;
2487 } 2401 }
2488 2402
@@ -2491,29 +2405,15 @@ static int __init d40_dmaengine_init(struct d40_base *base,
2491 2405
2492 dma_cap_zero(base->dma_memcpy.cap_mask); 2406 dma_cap_zero(base->dma_memcpy.cap_mask);
2493 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask); 2407 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2494 dma_cap_set(DMA_SG, base->dma_slave.cap_mask); 2408 dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask);
2495 2409
2496 base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources; 2410 d40_ops_init(base, &base->dma_memcpy);
2497 base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources;
2498 base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy;
2499 base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2500 base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg;
2501 base->dma_memcpy.device_tx_status = d40_tx_status;
2502 base->dma_memcpy.device_issue_pending = d40_issue_pending;
2503 base->dma_memcpy.device_control = d40_control;
2504 base->dma_memcpy.dev = base->dev;
2505 /*
2506 * This controller can only access address at even
2507 * 32bit boundaries, i.e. 2^2
2508 */
2509 base->dma_memcpy.copy_align = 2;
2510 2411
2511 err = dma_async_device_register(&base->dma_memcpy); 2412 err = dma_async_device_register(&base->dma_memcpy);
2512 2413
2513 if (err) { 2414 if (err) {
2514 dev_err(base->dev, 2415 d40_err(base->dev,
2515 "[%s] Failed to regsiter memcpy only channels\n", 2416 "Failed to regsiter memcpy only channels\n");
2516 __func__);
2517 goto failure2; 2417 goto failure2;
2518 } 2418 }
2519 2419
@@ -2523,24 +2423,15 @@ static int __init d40_dmaengine_init(struct d40_base *base,
2523 dma_cap_zero(base->dma_both.cap_mask); 2423 dma_cap_zero(base->dma_both.cap_mask);
2524 dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask); 2424 dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2525 dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask); 2425 dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2526 dma_cap_set(DMA_SG, base->dma_slave.cap_mask); 2426 dma_cap_set(DMA_SG, base->dma_both.cap_mask);
2527 2427 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
2528 base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources; 2428
2529 base->dma_both.device_free_chan_resources = d40_free_chan_resources; 2429 d40_ops_init(base, &base->dma_both);
2530 base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy;
2531 base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2532 base->dma_both.device_prep_slave_sg = d40_prep_slave_sg;
2533 base->dma_both.device_tx_status = d40_tx_status;
2534 base->dma_both.device_issue_pending = d40_issue_pending;
2535 base->dma_both.device_control = d40_control;
2536 base->dma_both.dev = base->dev;
2537 base->dma_both.copy_align = 2;
2538 err = dma_async_device_register(&base->dma_both); 2430 err = dma_async_device_register(&base->dma_both);
2539 2431
2540 if (err) { 2432 if (err) {
2541 dev_err(base->dev, 2433 d40_err(base->dev,
2542 "[%s] Failed to register logical and physical capable channels\n", 2434 "Failed to register logical and physical capable channels\n");
2543 __func__);
2544 goto failure3; 2435 goto failure3;
2545 } 2436 }
2546 return 0; 2437 return 0;
@@ -2616,9 +2507,10 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2616 { .reg = D40_DREG_PERIPHID1, .val = 0x0000}, 2507 { .reg = D40_DREG_PERIPHID1, .val = 0x0000},
2617 /* 2508 /*
2618 * D40_DREG_PERIPHID2 Depends on HW revision: 2509 * D40_DREG_PERIPHID2 Depends on HW revision:
2619 * MOP500/HREF ED has 0x0008, 2510 * DB8500ed has 0x0008,
2620 * ? has 0x0018, 2511 * ? has 0x0018,
2621 * HREF V1 has 0x0028 2512 * DB8500v1 has 0x0028
2513 * DB8500v2 has 0x0038
2622 */ 2514 */
2623 { .reg = D40_DREG_PERIPHID3, .val = 0x0000}, 2515 { .reg = D40_DREG_PERIPHID3, .val = 0x0000},
2624 2516
@@ -2642,8 +2534,7 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2642 clk = clk_get(&pdev->dev, NULL); 2534 clk = clk_get(&pdev->dev, NULL);
2643 2535
2644 if (IS_ERR(clk)) { 2536 if (IS_ERR(clk)) {
2645 dev_err(&pdev->dev, "[%s] No matching clock found\n", 2537 d40_err(&pdev->dev, "No matching clock found\n");
2646 __func__);
2647 goto failure; 2538 goto failure;
2648 } 2539 }
2649 2540
@@ -2666,9 +2557,8 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2666 for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) { 2557 for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
2667 if (dma_id_regs[i].val != 2558 if (dma_id_regs[i].val !=
2668 readl(virtbase + dma_id_regs[i].reg)) { 2559 readl(virtbase + dma_id_regs[i].reg)) {
2669 dev_err(&pdev->dev, 2560 d40_err(&pdev->dev,
2670 "[%s] Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n", 2561 "Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
2671 __func__,
2672 dma_id_regs[i].val, 2562 dma_id_regs[i].val,
2673 dma_id_regs[i].reg, 2563 dma_id_regs[i].reg,
2674 readl(virtbase + dma_id_regs[i].reg)); 2564 readl(virtbase + dma_id_regs[i].reg));
@@ -2681,9 +2571,8 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2681 2571
2682 if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) != 2572 if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) !=
2683 D40_HW_DESIGNER) { 2573 D40_HW_DESIGNER) {
2684 dev_err(&pdev->dev, 2574 d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
2685 "[%s] Unknown designer! Got %x wanted %x\n", 2575 val & D40_DREG_PERIPHID2_DESIGNER_MASK,
2686 __func__, val & D40_DREG_PERIPHID2_DESIGNER_MASK,
2687 D40_HW_DESIGNER); 2576 D40_HW_DESIGNER);
2688 goto failure; 2577 goto failure;
2689 } 2578 }
@@ -2713,7 +2602,7 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2713 sizeof(struct d40_chan), GFP_KERNEL); 2602 sizeof(struct d40_chan), GFP_KERNEL);
2714 2603
2715 if (base == NULL) { 2604 if (base == NULL) {
2716 dev_err(&pdev->dev, "[%s] Out of memory\n", __func__); 2605 d40_err(&pdev->dev, "Out of memory\n");
2717 goto failure; 2606 goto failure;
2718 } 2607 }
2719 2608
@@ -2860,6 +2749,7 @@ static void __init d40_hw_init(struct d40_base *base)
2860 2749
2861static int __init d40_lcla_allocate(struct d40_base *base) 2750static int __init d40_lcla_allocate(struct d40_base *base)
2862{ 2751{
2752 struct d40_lcla_pool *pool = &base->lcla_pool;
2863 unsigned long *page_list; 2753 unsigned long *page_list;
2864 int i, j; 2754 int i, j;
2865 int ret = 0; 2755 int ret = 0;
@@ -2885,9 +2775,8 @@ static int __init d40_lcla_allocate(struct d40_base *base)
2885 base->lcla_pool.pages); 2775 base->lcla_pool.pages);
2886 if (!page_list[i]) { 2776 if (!page_list[i]) {
2887 2777
2888 dev_err(base->dev, 2778 d40_err(base->dev, "Failed to allocate %d pages.\n",
2889 "[%s] Failed to allocate %d pages.\n", 2779 base->lcla_pool.pages);
2890 __func__, base->lcla_pool.pages);
2891 2780
2892 for (j = 0; j < i; j++) 2781 for (j = 0; j < i; j++)
2893 free_pages(page_list[j], base->lcla_pool.pages); 2782 free_pages(page_list[j], base->lcla_pool.pages);
@@ -2925,6 +2814,15 @@ static int __init d40_lcla_allocate(struct d40_base *base)
2925 LCLA_ALIGNMENT); 2814 LCLA_ALIGNMENT);
2926 } 2815 }
2927 2816
2817 pool->dma_addr = dma_map_single(base->dev, pool->base,
2818 SZ_1K * base->num_phy_chans,
2819 DMA_TO_DEVICE);
2820 if (dma_mapping_error(base->dev, pool->dma_addr)) {
2821 pool->dma_addr = 0;
2822 ret = -ENOMEM;
2823 goto failure;
2824 }
2825
2928 writel(virt_to_phys(base->lcla_pool.base), 2826 writel(virt_to_phys(base->lcla_pool.base),
2929 base->virtbase + D40_DREG_LCLA); 2827 base->virtbase + D40_DREG_LCLA);
2930failure: 2828failure:
@@ -2957,9 +2855,7 @@ static int __init d40_probe(struct platform_device *pdev)
2957 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa"); 2855 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
2958 if (!res) { 2856 if (!res) {
2959 ret = -ENOENT; 2857 ret = -ENOENT;
2960 dev_err(&pdev->dev, 2858 d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
2961 "[%s] No \"lcpa\" memory resource\n",
2962 __func__);
2963 goto failure; 2859 goto failure;
2964 } 2860 }
2965 base->lcpa_size = resource_size(res); 2861 base->lcpa_size = resource_size(res);
@@ -2968,9 +2864,9 @@ static int __init d40_probe(struct platform_device *pdev)
2968 if (request_mem_region(res->start, resource_size(res), 2864 if (request_mem_region(res->start, resource_size(res),
2969 D40_NAME " I/O lcpa") == NULL) { 2865 D40_NAME " I/O lcpa") == NULL) {
2970 ret = -EBUSY; 2866 ret = -EBUSY;
2971 dev_err(&pdev->dev, 2867 d40_err(&pdev->dev,
2972 "[%s] Failed to request LCPA region 0x%x-0x%x\n", 2868 "Failed to request LCPA region 0x%x-0x%x\n",
2973 __func__, res->start, res->end); 2869 res->start, res->end);
2974 goto failure; 2870 goto failure;
2975 } 2871 }
2976 2872
@@ -2986,16 +2882,13 @@ static int __init d40_probe(struct platform_device *pdev)
2986 base->lcpa_base = ioremap(res->start, resource_size(res)); 2882 base->lcpa_base = ioremap(res->start, resource_size(res));
2987 if (!base->lcpa_base) { 2883 if (!base->lcpa_base) {
2988 ret = -ENOMEM; 2884 ret = -ENOMEM;
2989 dev_err(&pdev->dev, 2885 d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
2990 "[%s] Failed to ioremap LCPA region\n",
2991 __func__);
2992 goto failure; 2886 goto failure;
2993 } 2887 }
2994 2888
2995 ret = d40_lcla_allocate(base); 2889 ret = d40_lcla_allocate(base);
2996 if (ret) { 2890 if (ret) {
2997 dev_err(&pdev->dev, "[%s] Failed to allocate LCLA area\n", 2891 d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
2998 __func__);
2999 goto failure; 2892 goto failure;
3000 } 2893 }
3001 2894
@@ -3004,9 +2897,8 @@ static int __init d40_probe(struct platform_device *pdev)
3004 base->irq = platform_get_irq(pdev, 0); 2897 base->irq = platform_get_irq(pdev, 0);
3005 2898
3006 ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base); 2899 ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
3007
3008 if (ret) { 2900 if (ret) {
3009 dev_err(&pdev->dev, "[%s] No IRQ defined\n", __func__); 2901 d40_err(&pdev->dev, "No IRQ defined\n");
3010 goto failure; 2902 goto failure;
3011 } 2903 }
3012 2904
@@ -3025,6 +2917,12 @@ failure:
3025 kmem_cache_destroy(base->desc_slab); 2917 kmem_cache_destroy(base->desc_slab);
3026 if (base->virtbase) 2918 if (base->virtbase)
3027 iounmap(base->virtbase); 2919 iounmap(base->virtbase);
2920
2921 if (base->lcla_pool.dma_addr)
2922 dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
2923 SZ_1K * base->num_phy_chans,
2924 DMA_TO_DEVICE);
2925
3028 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base) 2926 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
3029 free_pages((unsigned long)base->lcla_pool.base, 2927 free_pages((unsigned long)base->lcla_pool.base,
3030 base->lcla_pool.pages); 2928 base->lcla_pool.pages);
@@ -3049,7 +2947,7 @@ failure:
3049 kfree(base); 2947 kfree(base);
3050 } 2948 }
3051 2949
3052 dev_err(&pdev->dev, "[%s] probe failed\n", __func__); 2950 d40_err(&pdev->dev, "probe failed\n");
3053 return ret; 2951 return ret;
3054} 2952}
3055 2953
@@ -3060,7 +2958,7 @@ static struct platform_driver d40_driver = {
3060 }, 2958 },
3061}; 2959};
3062 2960
3063int __init stedma40_init(void) 2961static int __init stedma40_init(void)
3064{ 2962{
3065 return platform_driver_probe(&d40_driver, d40_probe); 2963 return platform_driver_probe(&d40_driver, d40_probe);
3066} 2964}
diff --git a/drivers/dma/ste_dma40_ll.c b/drivers/dma/ste_dma40_ll.c
index 0b096a38322d..cad9e1daedff 100644
--- a/drivers/dma/ste_dma40_ll.c
+++ b/drivers/dma/ste_dma40_ll.c
@@ -125,13 +125,15 @@ void d40_phy_cfg(struct stedma40_chan_cfg *cfg,
125static int d40_phy_fill_lli(struct d40_phy_lli *lli, 125static int d40_phy_fill_lli(struct d40_phy_lli *lli,
126 dma_addr_t data, 126 dma_addr_t data,
127 u32 data_size, 127 u32 data_size,
128 int psize,
129 dma_addr_t next_lli, 128 dma_addr_t next_lli,
130 u32 reg_cfg, 129 u32 reg_cfg,
131 bool term_int, 130 struct stedma40_half_channel_info *info,
132 u32 data_width, 131 unsigned int flags)
133 bool is_device)
134{ 132{
133 bool addr_inc = flags & LLI_ADDR_INC;
134 bool term_int = flags & LLI_TERM_INT;
135 unsigned int data_width = info->data_width;
136 int psize = info->psize;
135 int num_elems; 137 int num_elems;
136 138
137 if (psize == STEDMA40_PSIZE_PHY_1) 139 if (psize == STEDMA40_PSIZE_PHY_1)
@@ -154,7 +156,7 @@ static int d40_phy_fill_lli(struct d40_phy_lli *lli,
154 * Distance to next element sized entry. 156 * Distance to next element sized entry.
155 * Usually the size of the element unless you want gaps. 157 * Usually the size of the element unless you want gaps.
156 */ 158 */
157 if (!is_device) 159 if (addr_inc)
158 lli->reg_elt |= (0x1 << data_width) << 160 lli->reg_elt |= (0x1 << data_width) <<
159 D40_SREG_ELEM_PHY_EIDX_POS; 161 D40_SREG_ELEM_PHY_EIDX_POS;
160 162
@@ -198,47 +200,51 @@ static int d40_seg_size(int size, int data_width1, int data_width2)
198 return seg_max; 200 return seg_max;
199} 201}
200 202
201struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli, 203static struct d40_phy_lli *
202 dma_addr_t addr, 204d40_phy_buf_to_lli(struct d40_phy_lli *lli, dma_addr_t addr, u32 size,
203 u32 size, 205 dma_addr_t lli_phys, dma_addr_t first_phys, u32 reg_cfg,
204 int psize, 206 struct stedma40_half_channel_info *info,
205 dma_addr_t lli_phys, 207 struct stedma40_half_channel_info *otherinfo,
206 u32 reg_cfg, 208 unsigned long flags)
207 bool term_int,
208 u32 data_width1,
209 u32 data_width2,
210 bool is_device)
211{ 209{
210 bool lastlink = flags & LLI_LAST_LINK;
211 bool addr_inc = flags & LLI_ADDR_INC;
212 bool term_int = flags & LLI_TERM_INT;
213 bool cyclic = flags & LLI_CYCLIC;
212 int err; 214 int err;
213 dma_addr_t next = lli_phys; 215 dma_addr_t next = lli_phys;
214 int size_rest = size; 216 int size_rest = size;
215 int size_seg = 0; 217 int size_seg = 0;
216 218
219 /*
220 * This piece may be split up based on d40_seg_size(); we only want the
221 * term int on the last part.
222 */
223 if (term_int)
224 flags &= ~LLI_TERM_INT;
225
217 do { 226 do {
218 size_seg = d40_seg_size(size_rest, data_width1, data_width2); 227 size_seg = d40_seg_size(size_rest, info->data_width,
228 otherinfo->data_width);
219 size_rest -= size_seg; 229 size_rest -= size_seg;
220 230
221 if (term_int && size_rest == 0) 231 if (size_rest == 0 && term_int)
222 next = 0; 232 flags |= LLI_TERM_INT;
233
234 if (size_rest == 0 && lastlink)
235 next = cyclic ? first_phys : 0;
223 else 236 else
224 next = ALIGN(next + sizeof(struct d40_phy_lli), 237 next = ALIGN(next + sizeof(struct d40_phy_lli),
225 D40_LLI_ALIGN); 238 D40_LLI_ALIGN);
226 239
227 err = d40_phy_fill_lli(lli, 240 err = d40_phy_fill_lli(lli, addr, size_seg, next,
228 addr, 241 reg_cfg, info, flags);
229 size_seg,
230 psize,
231 next,
232 reg_cfg,
233 !next,
234 data_width1,
235 is_device);
236 242
237 if (err) 243 if (err)
238 goto err; 244 goto err;
239 245
240 lli++; 246 lli++;
241 if (!is_device) 247 if (addr_inc)
242 addr += size_seg; 248 addr += size_seg;
243 } while (size_rest); 249 } while (size_rest);
244 250
@@ -254,39 +260,35 @@ int d40_phy_sg_to_lli(struct scatterlist *sg,
254 struct d40_phy_lli *lli_sg, 260 struct d40_phy_lli *lli_sg,
255 dma_addr_t lli_phys, 261 dma_addr_t lli_phys,
256 u32 reg_cfg, 262 u32 reg_cfg,
257 u32 data_width1, 263 struct stedma40_half_channel_info *info,
258 u32 data_width2, 264 struct stedma40_half_channel_info *otherinfo,
259 int psize) 265 unsigned long flags)
260{ 266{
261 int total_size = 0; 267 int total_size = 0;
262 int i; 268 int i;
263 struct scatterlist *current_sg = sg; 269 struct scatterlist *current_sg = sg;
264 dma_addr_t dst;
265 struct d40_phy_lli *lli = lli_sg; 270 struct d40_phy_lli *lli = lli_sg;
266 dma_addr_t l_phys = lli_phys; 271 dma_addr_t l_phys = lli_phys;
267 272
273 if (!target)
274 flags |= LLI_ADDR_INC;
275
268 for_each_sg(sg, current_sg, sg_len, i) { 276 for_each_sg(sg, current_sg, sg_len, i) {
277 dma_addr_t sg_addr = sg_dma_address(current_sg);
278 unsigned int len = sg_dma_len(current_sg);
279 dma_addr_t dst = target ?: sg_addr;
269 280
270 total_size += sg_dma_len(current_sg); 281 total_size += sg_dma_len(current_sg);
271 282
272 if (target) 283 if (i == sg_len - 1)
273 dst = target; 284 flags |= LLI_TERM_INT | LLI_LAST_LINK;
274 else
275 dst = sg_phys(current_sg);
276 285
277 l_phys = ALIGN(lli_phys + (lli - lli_sg) * 286 l_phys = ALIGN(lli_phys + (lli - lli_sg) *
278 sizeof(struct d40_phy_lli), D40_LLI_ALIGN); 287 sizeof(struct d40_phy_lli), D40_LLI_ALIGN);
279 288
280 lli = d40_phy_buf_to_lli(lli, 289 lli = d40_phy_buf_to_lli(lli, dst, len, l_phys, lli_phys,
281 dst, 290 reg_cfg, info, otherinfo, flags);
282 sg_dma_len(current_sg), 291
283 psize,
284 l_phys,
285 reg_cfg,
286 sg_len - 1 == i,
287 data_width1,
288 data_width2,
289 target == dst);
290 if (lli == NULL) 292 if (lli == NULL)
291 return -EINVAL; 293 return -EINVAL;
292 } 294 }
@@ -295,45 +297,22 @@ int d40_phy_sg_to_lli(struct scatterlist *sg,
295} 297}
296 298
297 299
298void d40_phy_lli_write(void __iomem *virtbase,
299 u32 phy_chan_num,
300 struct d40_phy_lli *lli_dst,
301 struct d40_phy_lli *lli_src)
302{
303
304 writel(lli_src->reg_cfg, virtbase + D40_DREG_PCBASE +
305 phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSCFG);
306 writel(lli_src->reg_elt, virtbase + D40_DREG_PCBASE +
307 phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSELT);
308 writel(lli_src->reg_ptr, virtbase + D40_DREG_PCBASE +
309 phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSPTR);
310 writel(lli_src->reg_lnk, virtbase + D40_DREG_PCBASE +
311 phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SSLNK);
312
313 writel(lli_dst->reg_cfg, virtbase + D40_DREG_PCBASE +
314 phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDCFG);
315 writel(lli_dst->reg_elt, virtbase + D40_DREG_PCBASE +
316 phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDELT);
317 writel(lli_dst->reg_ptr, virtbase + D40_DREG_PCBASE +
318 phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDPTR);
319 writel(lli_dst->reg_lnk, virtbase + D40_DREG_PCBASE +
320 phy_chan_num * D40_DREG_PCDELTA + D40_CHAN_REG_SDLNK);
321
322}
323
324/* DMA logical lli operations */ 300/* DMA logical lli operations */
325 301
326static void d40_log_lli_link(struct d40_log_lli *lli_dst, 302static void d40_log_lli_link(struct d40_log_lli *lli_dst,
327 struct d40_log_lli *lli_src, 303 struct d40_log_lli *lli_src,
328 int next) 304 int next, unsigned int flags)
329{ 305{
306 bool interrupt = flags & LLI_TERM_INT;
330 u32 slos = 0; 307 u32 slos = 0;
331 u32 dlos = 0; 308 u32 dlos = 0;
332 309
333 if (next != -EINVAL) { 310 if (next != -EINVAL) {
334 slos = next * 2; 311 slos = next * 2;
335 dlos = next * 2 + 1; 312 dlos = next * 2 + 1;
336 } else { 313 }
314
315 if (interrupt) {
337 lli_dst->lcsp13 |= D40_MEM_LCSP1_SCFG_TIM_MASK; 316 lli_dst->lcsp13 |= D40_MEM_LCSP1_SCFG_TIM_MASK;
338 lli_dst->lcsp13 |= D40_MEM_LCSP3_DTCP_MASK; 317 lli_dst->lcsp13 |= D40_MEM_LCSP3_DTCP_MASK;
339 } 318 }
@@ -348,9 +327,9 @@ static void d40_log_lli_link(struct d40_log_lli *lli_dst,
348void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa, 327void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa,
349 struct d40_log_lli *lli_dst, 328 struct d40_log_lli *lli_dst,
350 struct d40_log_lli *lli_src, 329 struct d40_log_lli *lli_src,
351 int next) 330 int next, unsigned int flags)
352{ 331{
353 d40_log_lli_link(lli_dst, lli_src, next); 332 d40_log_lli_link(lli_dst, lli_src, next, flags);
354 333
355 writel(lli_src->lcsp02, &lcpa[0].lcsp0); 334 writel(lli_src->lcsp02, &lcpa[0].lcsp0);
356 writel(lli_src->lcsp13, &lcpa[0].lcsp1); 335 writel(lli_src->lcsp13, &lcpa[0].lcsp1);
@@ -361,9 +340,9 @@ void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa,
361void d40_log_lli_lcla_write(struct d40_log_lli *lcla, 340void d40_log_lli_lcla_write(struct d40_log_lli *lcla,
362 struct d40_log_lli *lli_dst, 341 struct d40_log_lli *lli_dst,
363 struct d40_log_lli *lli_src, 342 struct d40_log_lli *lli_src,
364 int next) 343 int next, unsigned int flags)
365{ 344{
366 d40_log_lli_link(lli_dst, lli_src, next); 345 d40_log_lli_link(lli_dst, lli_src, next, flags);
367 346
368 writel(lli_src->lcsp02, &lcla[0].lcsp02); 347 writel(lli_src->lcsp02, &lcla[0].lcsp02);
369 writel(lli_src->lcsp13, &lcla[0].lcsp13); 348 writel(lli_src->lcsp13, &lcla[0].lcsp13);
@@ -375,8 +354,10 @@ static void d40_log_fill_lli(struct d40_log_lli *lli,
375 dma_addr_t data, u32 data_size, 354 dma_addr_t data, u32 data_size,
376 u32 reg_cfg, 355 u32 reg_cfg,
377 u32 data_width, 356 u32 data_width,
378 bool addr_inc) 357 unsigned int flags)
379{ 358{
359 bool addr_inc = flags & LLI_ADDR_INC;
360
380 lli->lcsp13 = reg_cfg; 361 lli->lcsp13 = reg_cfg;
381 362
382 /* The number of elements to transfer */ 363 /* The number of elements to transfer */
@@ -395,67 +376,15 @@ static void d40_log_fill_lli(struct d40_log_lli *lli,
395 376
396} 377}
397 378
398int d40_log_sg_to_dev(struct scatterlist *sg, 379static struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
399 int sg_len,
400 struct d40_log_lli_bidir *lli,
401 struct d40_def_lcsp *lcsp,
402 u32 src_data_width,
403 u32 dst_data_width,
404 enum dma_data_direction direction,
405 dma_addr_t dev_addr)
406{
407 int total_size = 0;
408 struct scatterlist *current_sg = sg;
409 int i;
410 struct d40_log_lli *lli_src = lli->src;
411 struct d40_log_lli *lli_dst = lli->dst;
412
413 for_each_sg(sg, current_sg, sg_len, i) {
414 total_size += sg_dma_len(current_sg);
415
416 if (direction == DMA_TO_DEVICE) {
417 lli_src =
418 d40_log_buf_to_lli(lli_src,
419 sg_phys(current_sg),
420 sg_dma_len(current_sg),
421 lcsp->lcsp1, src_data_width,
422 dst_data_width,
423 true);
424 lli_dst =
425 d40_log_buf_to_lli(lli_dst,
426 dev_addr,
427 sg_dma_len(current_sg),
428 lcsp->lcsp3, dst_data_width,
429 src_data_width,
430 false);
431 } else {
432 lli_dst =
433 d40_log_buf_to_lli(lli_dst,
434 sg_phys(current_sg),
435 sg_dma_len(current_sg),
436 lcsp->lcsp3, dst_data_width,
437 src_data_width,
438 true);
439 lli_src =
440 d40_log_buf_to_lli(lli_src,
441 dev_addr,
442 sg_dma_len(current_sg),
443 lcsp->lcsp1, src_data_width,
444 dst_data_width,
445 false);
446 }
447 }
448 return total_size;
449}
450
451struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
452 dma_addr_t addr, 380 dma_addr_t addr,
453 int size, 381 int size,
454 u32 lcsp13, /* src or dst*/ 382 u32 lcsp13, /* src or dst*/
455 u32 data_width1, 383 u32 data_width1,
456 u32 data_width2, 384 u32 data_width2,
457 bool addr_inc) 385 unsigned int flags)
458{ 386{
387 bool addr_inc = flags & LLI_ADDR_INC;
459 struct d40_log_lli *lli = lli_sg; 388 struct d40_log_lli *lli = lli_sg;
460 int size_rest = size; 389 int size_rest = size;
461 int size_seg = 0; 390 int size_seg = 0;
@@ -468,7 +397,7 @@ struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
468 addr, 397 addr,
469 size_seg, 398 size_seg,
470 lcsp13, data_width1, 399 lcsp13, data_width1,
471 addr_inc); 400 flags);
472 if (addr_inc) 401 if (addr_inc)
473 addr += size_seg; 402 addr += size_seg;
474 lli++; 403 lli++;
@@ -479,6 +408,7 @@ struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
479 408
480int d40_log_sg_to_lli(struct scatterlist *sg, 409int d40_log_sg_to_lli(struct scatterlist *sg,
481 int sg_len, 410 int sg_len,
411 dma_addr_t dev_addr,
482 struct d40_log_lli *lli_sg, 412 struct d40_log_lli *lli_sg,
483 u32 lcsp13, /* src or dst*/ 413 u32 lcsp13, /* src or dst*/
484 u32 data_width1, u32 data_width2) 414 u32 data_width1, u32 data_width2)
@@ -487,14 +417,24 @@ int d40_log_sg_to_lli(struct scatterlist *sg,
487 struct scatterlist *current_sg = sg; 417 struct scatterlist *current_sg = sg;
488 int i; 418 int i;
489 struct d40_log_lli *lli = lli_sg; 419 struct d40_log_lli *lli = lli_sg;
420 unsigned long flags = 0;
421
422 if (!dev_addr)
423 flags |= LLI_ADDR_INC;
490 424
491 for_each_sg(sg, current_sg, sg_len, i) { 425 for_each_sg(sg, current_sg, sg_len, i) {
426 dma_addr_t sg_addr = sg_dma_address(current_sg);
427 unsigned int len = sg_dma_len(current_sg);
428 dma_addr_t addr = dev_addr ?: sg_addr;
429
492 total_size += sg_dma_len(current_sg); 430 total_size += sg_dma_len(current_sg);
493 lli = d40_log_buf_to_lli(lli, 431
494 sg_phys(current_sg), 432 lli = d40_log_buf_to_lli(lli, addr, len,
495 sg_dma_len(current_sg),
496 lcsp13, 433 lcsp13,
497 data_width1, data_width2, true); 434 data_width1,
435 data_width2,
436 flags);
498 } 437 }
438
499 return total_size; 439 return total_size;
500} 440}
diff --git a/drivers/dma/ste_dma40_ll.h b/drivers/dma/ste_dma40_ll.h
index 9cc43495bea2..195ee65ee7f3 100644
--- a/drivers/dma/ste_dma40_ll.h
+++ b/drivers/dma/ste_dma40_ll.h
@@ -163,6 +163,22 @@
163#define D40_DREG_LCEIS1 0x0B4 163#define D40_DREG_LCEIS1 0x0B4
164#define D40_DREG_LCEIS2 0x0B8 164#define D40_DREG_LCEIS2 0x0B8
165#define D40_DREG_LCEIS3 0x0BC 165#define D40_DREG_LCEIS3 0x0BC
166#define D40_DREG_PSEG1 0x110
167#define D40_DREG_PSEG2 0x114
168#define D40_DREG_PSEG3 0x118
169#define D40_DREG_PSEG4 0x11C
170#define D40_DREG_PCEG1 0x120
171#define D40_DREG_PCEG2 0x124
172#define D40_DREG_PCEG3 0x128
173#define D40_DREG_PCEG4 0x12C
174#define D40_DREG_RSEG1 0x130
175#define D40_DREG_RSEG2 0x134
176#define D40_DREG_RSEG3 0x138
177#define D40_DREG_RSEG4 0x13C
178#define D40_DREG_RCEG1 0x140
179#define D40_DREG_RCEG2 0x144
180#define D40_DREG_RCEG3 0x148
181#define D40_DREG_RCEG4 0x14C
166#define D40_DREG_STFU 0xFC8 182#define D40_DREG_STFU 0xFC8
167#define D40_DREG_ICFG 0xFCC 183#define D40_DREG_ICFG 0xFCC
168#define D40_DREG_PERIPHID0 0xFE0 184#define D40_DREG_PERIPHID0 0xFE0
@@ -277,6 +293,13 @@ struct d40_def_lcsp {
277 293
278/* Physical channels */ 294/* Physical channels */
279 295
296enum d40_lli_flags {
297 LLI_ADDR_INC = 1 << 0,
298 LLI_TERM_INT = 1 << 1,
299 LLI_CYCLIC = 1 << 2,
300 LLI_LAST_LINK = 1 << 3,
301};
302
280void d40_phy_cfg(struct stedma40_chan_cfg *cfg, 303void d40_phy_cfg(struct stedma40_chan_cfg *cfg,
281 u32 *src_cfg, 304 u32 *src_cfg,
282 u32 *dst_cfg, 305 u32 *dst_cfg,
@@ -292,46 +315,15 @@ int d40_phy_sg_to_lli(struct scatterlist *sg,
292 struct d40_phy_lli *lli, 315 struct d40_phy_lli *lli,
293 dma_addr_t lli_phys, 316 dma_addr_t lli_phys,
294 u32 reg_cfg, 317 u32 reg_cfg,
295 u32 data_width1, 318 struct stedma40_half_channel_info *info,
296 u32 data_width2, 319 struct stedma40_half_channel_info *otherinfo,
297 int psize); 320 unsigned long flags);
298
299struct d40_phy_lli *d40_phy_buf_to_lli(struct d40_phy_lli *lli,
300 dma_addr_t data,
301 u32 data_size,
302 int psize,
303 dma_addr_t next_lli,
304 u32 reg_cfg,
305 bool term_int,
306 u32 data_width1,
307 u32 data_width2,
308 bool is_device);
309
310void d40_phy_lli_write(void __iomem *virtbase,
311 u32 phy_chan_num,
312 struct d40_phy_lli *lli_dst,
313 struct d40_phy_lli *lli_src);
314 321
315/* Logical channels */ 322/* Logical channels */
316 323
317struct d40_log_lli *d40_log_buf_to_lli(struct d40_log_lli *lli_sg,
318 dma_addr_t addr,
319 int size,
320 u32 lcsp13, /* src or dst*/
321 u32 data_width1, u32 data_width2,
322 bool addr_inc);
323
324int d40_log_sg_to_dev(struct scatterlist *sg,
325 int sg_len,
326 struct d40_log_lli_bidir *lli,
327 struct d40_def_lcsp *lcsp,
328 u32 src_data_width,
329 u32 dst_data_width,
330 enum dma_data_direction direction,
331 dma_addr_t dev_addr);
332
333int d40_log_sg_to_lli(struct scatterlist *sg, 324int d40_log_sg_to_lli(struct scatterlist *sg,
334 int sg_len, 325 int sg_len,
326 dma_addr_t dev_addr,
335 struct d40_log_lli *lli_sg, 327 struct d40_log_lli *lli_sg,
336 u32 lcsp13, /* src or dst*/ 328 u32 lcsp13, /* src or dst*/
337 u32 data_width1, u32 data_width2); 329 u32 data_width1, u32 data_width2);
@@ -339,11 +331,11 @@ int d40_log_sg_to_lli(struct scatterlist *sg,
339void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa, 331void d40_log_lli_lcpa_write(struct d40_log_lli_full *lcpa,
340 struct d40_log_lli *lli_dst, 332 struct d40_log_lli *lli_dst,
341 struct d40_log_lli *lli_src, 333 struct d40_log_lli *lli_src,
342 int next); 334 int next, unsigned int flags);
343 335
344void d40_log_lli_lcla_write(struct d40_log_lli *lcla, 336void d40_log_lli_lcla_write(struct d40_log_lli *lcla,
345 struct d40_log_lli *lli_dst, 337 struct d40_log_lli *lli_dst,
346 struct d40_log_lli *lli_src, 338 struct d40_log_lli *lli_src,
347 int next); 339 int next, unsigned int flags);
348 340
349#endif /* STE_DMA40_LLI_H */ 341#endif /* STE_DMA40_LLI_H */
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-30 21:25:59 -0400