aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2009-12-16 13:28:56 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2009-12-16 13:28:56 -0500
commit7949456b1b96924c2d9ae5aea5fa7d4c81c946ed (patch)
tree819e64dcd686c8b53c698c164aea96a002e8b5f8 /drivers/dma
parent60d9aa758c00f20ade0cb1951f6a934f628dd2d7 (diff)
parent12458ea06efd7b44281e68fe59c950ec7d59c649 (diff)
Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx
* 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx: ppc440spe-adma: adds updated ppc440spe adma driver iop-adma.c: use resource_size() dmaengine: clarify the meaning of the DMA_CTRL_ACK flag sh: stylistic improvements for the DMA driver dmaengine: fix dmatest to verify minimum transfer length and test buffer size sh: DMA driver has to specify its alignment requirements Add COH 901 318 DMA block driver v5
Diffstat (limited to 'drivers/dma')
-rw-r--r--drivers/dma/Kconfig18
-rw-r--r--drivers/dma/Makefile2
-rw-r--r--drivers/dma/coh901318.c1325
-rw-r--r--drivers/dma/coh901318_lli.c318
-rw-r--r--drivers/dma/coh901318_lli.h124
-rw-r--r--drivers/dma/dmatest.c16
-rw-r--r--drivers/dma/iop-adma.c4
-rw-r--r--drivers/dma/ppc4xx/Makefile1
-rw-r--r--drivers/dma/ppc4xx/adma.c5027
-rw-r--r--drivers/dma/ppc4xx/adma.h195
-rw-r--r--drivers/dma/ppc4xx/dma.h223
-rw-r--r--drivers/dma/ppc4xx/xor.h110
-rw-r--r--drivers/dma/shdma.c36
-rw-r--r--drivers/dma/shdma.h14
14 files changed, 7383 insertions, 30 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index eb140ff38c2..e02d74b1e89 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -111,6 +111,24 @@ config SH_DMAE
111 help 111 help
112 Enable support for the Renesas SuperH DMA controllers. 112 Enable support for the Renesas SuperH DMA controllers.
113 113
114config COH901318
115 bool "ST-Ericsson COH901318 DMA support"
116 select DMA_ENGINE
117 depends on ARCH_U300
118 help
119 Enable support for ST-Ericsson COH 901 318 DMA.
120
121config AMCC_PPC440SPE_ADMA
122 tristate "AMCC PPC440SPe ADMA support"
123 depends on 440SPe || 440SP
124 select DMA_ENGINE
125 select ARCH_HAS_ASYNC_TX_FIND_CHANNEL
126 help
127 Enable support for the AMCC PPC440SPe RAID engines.
128
129config ARCH_HAS_ASYNC_TX_FIND_CHANNEL
130 bool
131
114config DMA_ENGINE 132config DMA_ENGINE
115 bool 133 bool
116 134
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index eca71ba78ae..807053d4823 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -10,3 +10,5 @@ obj-$(CONFIG_AT_HDMAC) += at_hdmac.o
10obj-$(CONFIG_MX3_IPU) += ipu/ 10obj-$(CONFIG_MX3_IPU) += ipu/
11obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o 11obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
12obj-$(CONFIG_SH_DMAE) += shdma.o 12obj-$(CONFIG_SH_DMAE) += shdma.o
13obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o
14obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/
diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c
new file mode 100644
index 00000000000..4a99cd94536
--- /dev/null
+++ b/drivers/dma/coh901318.c
@@ -0,0 +1,1325 @@
1/*
2 * driver/dma/coh901318.c
3 *
4 * Copyright (C) 2007-2009 ST-Ericsson
5 * License terms: GNU General Public License (GPL) version 2
6 * DMA driver for COH 901 318
7 * Author: Per Friden <per.friden@stericsson.com>
8 */
9
10#include <linux/init.h>
11#include <linux/module.h>
12#include <linux/kernel.h> /* printk() */
13#include <linux/fs.h> /* everything... */
14#include <linux/slab.h> /* kmalloc() */
15#include <linux/dmaengine.h>
16#include <linux/platform_device.h>
17#include <linux/device.h>
18#include <linux/irqreturn.h>
19#include <linux/interrupt.h>
20#include <linux/io.h>
21#include <linux/uaccess.h>
22#include <linux/debugfs.h>
23#include <mach/coh901318.h>
24
25#include "coh901318_lli.h"
26
27#define COHC_2_DEV(cohc) (&cohc->chan.dev->device)
28
29#ifdef VERBOSE_DEBUG
30#define COH_DBG(x) ({ if (1) x; 0; })
31#else
32#define COH_DBG(x) ({ if (0) x; 0; })
33#endif
34
35struct coh901318_desc {
36 struct dma_async_tx_descriptor desc;
37 struct list_head node;
38 struct scatterlist *sg;
39 unsigned int sg_len;
40 struct coh901318_lli *data;
41 enum dma_data_direction dir;
42 int pending_irqs;
43 unsigned long flags;
44};
45
46struct coh901318_base {
47 struct device *dev;
48 void __iomem *virtbase;
49 struct coh901318_pool pool;
50 struct powersave pm;
51 struct dma_device dma_slave;
52 struct dma_device dma_memcpy;
53 struct coh901318_chan *chans;
54 struct coh901318_platform *platform;
55};
56
57struct coh901318_chan {
58 spinlock_t lock;
59 int allocated;
60 int completed;
61 int id;
62 int stopped;
63
64 struct work_struct free_work;
65 struct dma_chan chan;
66
67 struct tasklet_struct tasklet;
68
69 struct list_head active;
70 struct list_head queue;
71 struct list_head free;
72
73 unsigned long nbr_active_done;
74 unsigned long busy;
75 int pending_irqs;
76
77 struct coh901318_base *base;
78};
79
80static void coh901318_list_print(struct coh901318_chan *cohc,
81 struct coh901318_lli *lli)
82{
83 struct coh901318_lli *l;
84 dma_addr_t addr = virt_to_phys(lli);
85 int i = 0;
86
87 while (addr) {
88 l = phys_to_virt(addr);
89 dev_vdbg(COHC_2_DEV(cohc), "i %d, lli %p, ctrl 0x%x, src 0x%x"
90 ", dst 0x%x, link 0x%x link_virt 0x%p\n",
91 i, l, l->control, l->src_addr, l->dst_addr,
92 l->link_addr, phys_to_virt(l->link_addr));
93 i++;
94 addr = l->link_addr;
95 }
96}
97
98#ifdef CONFIG_DEBUG_FS
99
100#define COH901318_DEBUGFS_ASSIGN(x, y) (x = y)
101
102static struct coh901318_base *debugfs_dma_base;
103static struct dentry *dma_dentry;
104
105static int coh901318_debugfs_open(struct inode *inode, struct file *file)
106{
107
108 file->private_data = inode->i_private;
109 return 0;
110}
111
112static int coh901318_debugfs_read(struct file *file, char __user *buf,
113 size_t count, loff_t *f_pos)
114{
115 u64 started_channels = debugfs_dma_base->pm.started_channels;
116 int pool_count = debugfs_dma_base->pool.debugfs_pool_counter;
117 int i;
118 int ret = 0;
119 char *dev_buf;
120 char *tmp;
121 int dev_size;
122
123 dev_buf = kmalloc(4*1024, GFP_KERNEL);
124 if (dev_buf == NULL)
125 goto err_kmalloc;
126 tmp = dev_buf;
127
128 tmp += sprintf(tmp, "DMA -- enable dma channels\n");
129
130 for (i = 0; i < debugfs_dma_base->platform->max_channels; i++)
131 if (started_channels & (1 << i))
132 tmp += sprintf(tmp, "channel %d\n", i);
133
134 tmp += sprintf(tmp, "Pool alloc nbr %d\n", pool_count);
135 dev_size = tmp - dev_buf;
136
137 /* No more to read if offset != 0 */
138 if (*f_pos > dev_size)
139 goto out;
140
141 if (count > dev_size - *f_pos)
142 count = dev_size - *f_pos;
143
144 if (copy_to_user(buf, dev_buf + *f_pos, count))
145 ret = -EINVAL;
146 ret = count;
147 *f_pos += count;
148
149 out:
150 kfree(dev_buf);
151 return ret;
152
153 err_kmalloc:
154 return 0;
155}
156
157static const struct file_operations coh901318_debugfs_status_operations = {
158 .owner = THIS_MODULE,
159 .open = coh901318_debugfs_open,
160 .read = coh901318_debugfs_read,
161};
162
163
164static int __init init_coh901318_debugfs(void)
165{
166
167 dma_dentry = debugfs_create_dir("dma", NULL);
168
169 (void) debugfs_create_file("status",
170 S_IFREG | S_IRUGO,
171 dma_dentry, NULL,
172 &coh901318_debugfs_status_operations);
173 return 0;
174}
175
176static void __exit exit_coh901318_debugfs(void)
177{
178 debugfs_remove_recursive(dma_dentry);
179}
180
181module_init(init_coh901318_debugfs);
182module_exit(exit_coh901318_debugfs);
183#else
184
185#define COH901318_DEBUGFS_ASSIGN(x, y)
186
187#endif /* CONFIG_DEBUG_FS */
188
189static inline struct coh901318_chan *to_coh901318_chan(struct dma_chan *chan)
190{
191 return container_of(chan, struct coh901318_chan, chan);
192}
193
194static inline dma_addr_t
195cohc_dev_addr(struct coh901318_chan *cohc)
196{
197 return cohc->base->platform->chan_conf[cohc->id].dev_addr;
198}
199
200static inline const struct coh901318_params *
201cohc_chan_param(struct coh901318_chan *cohc)
202{
203 return &cohc->base->platform->chan_conf[cohc->id].param;
204}
205
206static inline const struct coh_dma_channel *
207cohc_chan_conf(struct coh901318_chan *cohc)
208{
209 return &cohc->base->platform->chan_conf[cohc->id];
210}
211
212static void enable_powersave(struct coh901318_chan *cohc)
213{
214 unsigned long flags;
215 struct powersave *pm = &cohc->base->pm;
216
217 spin_lock_irqsave(&pm->lock, flags);
218
219 pm->started_channels &= ~(1ULL << cohc->id);
220
221 if (!pm->started_channels) {
222 /* DMA no longer intends to access memory */
223 cohc->base->platform->access_memory_state(cohc->base->dev,
224 false);
225 }
226
227 spin_unlock_irqrestore(&pm->lock, flags);
228}
229static void disable_powersave(struct coh901318_chan *cohc)
230{
231 unsigned long flags;
232 struct powersave *pm = &cohc->base->pm;
233
234 spin_lock_irqsave(&pm->lock, flags);
235
236 if (!pm->started_channels) {
237 /* DMA intends to access memory */
238 cohc->base->platform->access_memory_state(cohc->base->dev,
239 true);
240 }
241
242 pm->started_channels |= (1ULL << cohc->id);
243
244 spin_unlock_irqrestore(&pm->lock, flags);
245}
246
247static inline int coh901318_set_ctrl(struct coh901318_chan *cohc, u32 control)
248{
249 int channel = cohc->id;
250 void __iomem *virtbase = cohc->base->virtbase;
251
252 writel(control,
253 virtbase + COH901318_CX_CTRL +
254 COH901318_CX_CTRL_SPACING * channel);
255 return 0;
256}
257
258static inline int coh901318_set_conf(struct coh901318_chan *cohc, u32 conf)
259{
260 int channel = cohc->id;
261 void __iomem *virtbase = cohc->base->virtbase;
262
263 writel(conf,
264 virtbase + COH901318_CX_CFG +
265 COH901318_CX_CFG_SPACING*channel);
266 return 0;
267}
268
269
270static int coh901318_start(struct coh901318_chan *cohc)
271{
272 u32 val;
273 int channel = cohc->id;
274 void __iomem *virtbase = cohc->base->virtbase;
275
276 disable_powersave(cohc);
277
278 val = readl(virtbase + COH901318_CX_CFG +
279 COH901318_CX_CFG_SPACING * channel);
280
281 /* Enable channel */
282 val |= COH901318_CX_CFG_CH_ENABLE;
283 writel(val, virtbase + COH901318_CX_CFG +
284 COH901318_CX_CFG_SPACING * channel);
285
286 return 0;
287}
288
289static int coh901318_prep_linked_list(struct coh901318_chan *cohc,
290 struct coh901318_lli *data)
291{
292 int channel = cohc->id;
293 void __iomem *virtbase = cohc->base->virtbase;
294
295 BUG_ON(readl(virtbase + COH901318_CX_STAT +
296 COH901318_CX_STAT_SPACING*channel) &
297 COH901318_CX_STAT_ACTIVE);
298
299 writel(data->src_addr,
300 virtbase + COH901318_CX_SRC_ADDR +
301 COH901318_CX_SRC_ADDR_SPACING * channel);
302
303 writel(data->dst_addr, virtbase +
304 COH901318_CX_DST_ADDR +
305 COH901318_CX_DST_ADDR_SPACING * channel);
306
307 writel(data->link_addr, virtbase + COH901318_CX_LNK_ADDR +
308 COH901318_CX_LNK_ADDR_SPACING * channel);
309
310 writel(data->control, virtbase + COH901318_CX_CTRL +
311 COH901318_CX_CTRL_SPACING * channel);
312
313 return 0;
314}
315static dma_cookie_t
316coh901318_assign_cookie(struct coh901318_chan *cohc,
317 struct coh901318_desc *cohd)
318{
319 dma_cookie_t cookie = cohc->chan.cookie;
320
321 if (++cookie < 0)
322 cookie = 1;
323
324 cohc->chan.cookie = cookie;
325 cohd->desc.cookie = cookie;
326
327 return cookie;
328}
329
330static struct coh901318_desc *
331coh901318_desc_get(struct coh901318_chan *cohc)
332{
333 struct coh901318_desc *desc;
334
335 if (list_empty(&cohc->free)) {
336 /* alloc new desc because we're out of used ones
337 * TODO: alloc a pile of descs instead of just one,
338 * avoid many small allocations.
339 */
340 desc = kmalloc(sizeof(struct coh901318_desc), GFP_NOWAIT);
341 if (desc == NULL)
342 goto out;
343 INIT_LIST_HEAD(&desc->node);
344 } else {
345 /* Reuse an old desc. */
346 desc = list_first_entry(&cohc->free,
347 struct coh901318_desc,
348 node);
349 list_del(&desc->node);
350 }
351
352 out:
353 return desc;
354}
355
356static void
357coh901318_desc_free(struct coh901318_chan *cohc, struct coh901318_desc *cohd)
358{
359 list_add_tail(&cohd->node, &cohc->free);
360}
361
362/* call with irq lock held */
363static void
364coh901318_desc_submit(struct coh901318_chan *cohc, struct coh901318_desc *desc)
365{
366 list_add_tail(&desc->node, &cohc->active);
367
368 BUG_ON(cohc->pending_irqs != 0);
369
370 cohc->pending_irqs = desc->pending_irqs;
371}
372
373static struct coh901318_desc *
374coh901318_first_active_get(struct coh901318_chan *cohc)
375{
376 struct coh901318_desc *d;
377
378 if (list_empty(&cohc->active))
379 return NULL;
380
381 d = list_first_entry(&cohc->active,
382 struct coh901318_desc,
383 node);
384 return d;
385}
386
387static void
388coh901318_desc_remove(struct coh901318_desc *cohd)
389{
390 list_del(&cohd->node);
391}
392
393static void
394coh901318_desc_queue(struct coh901318_chan *cohc, struct coh901318_desc *desc)
395{
396 list_add_tail(&desc->node, &cohc->queue);
397}
398
399static struct coh901318_desc *
400coh901318_first_queued(struct coh901318_chan *cohc)
401{
402 struct coh901318_desc *d;
403
404 if (list_empty(&cohc->queue))
405 return NULL;
406
407 d = list_first_entry(&cohc->queue,
408 struct coh901318_desc,
409 node);
410 return d;
411}
412
413/*
414 * DMA start/stop controls
415 */
416u32 coh901318_get_bytes_left(struct dma_chan *chan)
417{
418 unsigned long flags;
419 u32 ret;
420 struct coh901318_chan *cohc = to_coh901318_chan(chan);
421
422 spin_lock_irqsave(&cohc->lock, flags);
423
424 /* Read transfer count value */
425 ret = readl(cohc->base->virtbase +
426 COH901318_CX_CTRL+COH901318_CX_CTRL_SPACING *
427 cohc->id) & COH901318_CX_CTRL_TC_VALUE_MASK;
428
429 spin_unlock_irqrestore(&cohc->lock, flags);
430
431 return ret;
432}
433EXPORT_SYMBOL(coh901318_get_bytes_left);
434
435
436/* Stops a transfer without losing data. Enables power save.
437 Use this function in conjunction with coh901318_continue(..)
438*/
439void coh901318_stop(struct dma_chan *chan)
440{
441 u32 val;
442 unsigned long flags;
443 struct coh901318_chan *cohc = to_coh901318_chan(chan);
444 int channel = cohc->id;
445 void __iomem *virtbase = cohc->base->virtbase;
446
447 spin_lock_irqsave(&cohc->lock, flags);
448
449 /* Disable channel in HW */
450 val = readl(virtbase + COH901318_CX_CFG +
451 COH901318_CX_CFG_SPACING * channel);
452
453 /* Stopping infinit transfer */
454 if ((val & COH901318_CX_CTRL_TC_ENABLE) == 0 &&
455 (val & COH901318_CX_CFG_CH_ENABLE))
456 cohc->stopped = 1;
457
458
459 val &= ~COH901318_CX_CFG_CH_ENABLE;
460 /* Enable twice, HW bug work around */
461 writel(val, virtbase + COH901318_CX_CFG +
462 COH901318_CX_CFG_SPACING * channel);
463 writel(val, virtbase + COH901318_CX_CFG +
464 COH901318_CX_CFG_SPACING * channel);
465
466 /* Spin-wait for it to actually go inactive */
467 while (readl(virtbase + COH901318_CX_STAT+COH901318_CX_STAT_SPACING *
468 channel) & COH901318_CX_STAT_ACTIVE)
469 cpu_relax();
470
471 /* Check if we stopped an active job */
472 if ((readl(virtbase + COH901318_CX_CTRL+COH901318_CX_CTRL_SPACING *
473 channel) & COH901318_CX_CTRL_TC_VALUE_MASK) > 0)
474 cohc->stopped = 1;
475
476 enable_powersave(cohc);
477
478 spin_unlock_irqrestore(&cohc->lock, flags);
479}
480EXPORT_SYMBOL(coh901318_stop);
481
482/* Continues a transfer that has been stopped via 300_dma_stop(..).
483 Power save is handled.
484*/
485void coh901318_continue(struct dma_chan *chan)
486{
487 u32 val;
488 unsigned long flags;
489 struct coh901318_chan *cohc = to_coh901318_chan(chan);
490 int channel = cohc->id;
491
492 spin_lock_irqsave(&cohc->lock, flags);
493
494 disable_powersave(cohc);
495
496 if (cohc->stopped) {
497 /* Enable channel in HW */
498 val = readl(cohc->base->virtbase + COH901318_CX_CFG +
499 COH901318_CX_CFG_SPACING * channel);
500
501 val |= COH901318_CX_CFG_CH_ENABLE;
502
503 writel(val, cohc->base->virtbase + COH901318_CX_CFG +
504 COH901318_CX_CFG_SPACING*channel);
505
506 cohc->stopped = 0;
507 }
508
509 spin_unlock_irqrestore(&cohc->lock, flags);
510}
511EXPORT_SYMBOL(coh901318_continue);
512
513bool coh901318_filter_id(struct dma_chan *chan, void *chan_id)
514{
515 unsigned int ch_nr = (unsigned int) chan_id;
516
517 if (ch_nr == to_coh901318_chan(chan)->id)
518 return true;
519
520 return false;
521}
522EXPORT_SYMBOL(coh901318_filter_id);
523
524/*
525 * DMA channel allocation
526 */
527static int coh901318_config(struct coh901318_chan *cohc,
528 struct coh901318_params *param)
529{
530 unsigned long flags;
531 const struct coh901318_params *p;
532 int channel = cohc->id;
533 void __iomem *virtbase = cohc->base->virtbase;
534
535 spin_lock_irqsave(&cohc->lock, flags);
536
537 if (param)
538 p = param;
539 else
540 p = &cohc->base->platform->chan_conf[channel].param;
541
542 /* Clear any pending BE or TC interrupt */
543 if (channel < 32) {
544 writel(1 << channel, virtbase + COH901318_BE_INT_CLEAR1);
545 writel(1 << channel, virtbase + COH901318_TC_INT_CLEAR1);
546 } else {
547 writel(1 << (channel - 32), virtbase +
548 COH901318_BE_INT_CLEAR2);
549 writel(1 << (channel - 32), virtbase +
550 COH901318_TC_INT_CLEAR2);
551 }
552
553 coh901318_set_conf(cohc, p->config);
554 coh901318_set_ctrl(cohc, p->ctrl_lli_last);
555
556 spin_unlock_irqrestore(&cohc->lock, flags);
557
558 return 0;
559}
560
561/* must lock when calling this function
562 * start queued jobs, if any
563 * TODO: start all queued jobs in one go
564 *
565 * Returns descriptor if queued job is started otherwise NULL.
566 * If the queue is empty NULL is returned.
567 */
568static struct coh901318_desc *coh901318_queue_start(struct coh901318_chan *cohc)
569{
570 struct coh901318_desc *cohd_que;
571
572 /* start queued jobs, if any
573 * TODO: transmit all queued jobs in one go
574 */
575 cohd_que = coh901318_first_queued(cohc);
576
577 if (cohd_que != NULL) {
578 /* Remove from queue */
579 coh901318_desc_remove(cohd_que);
580 /* initiate DMA job */
581 cohc->busy = 1;
582
583 coh901318_desc_submit(cohc, cohd_que);
584
585 coh901318_prep_linked_list(cohc, cohd_que->data);
586
587 /* start dma job */
588 coh901318_start(cohc);
589
590 }
591
592 return cohd_que;
593}
594
595static void dma_tasklet(unsigned long data)
596{
597 struct coh901318_chan *cohc = (struct coh901318_chan *) data;
598 struct coh901318_desc *cohd_fin;
599 unsigned long flags;
600 dma_async_tx_callback callback;
601 void *callback_param;
602
603 spin_lock_irqsave(&cohc->lock, flags);
604
605 /* get first active entry from list */
606 cohd_fin = coh901318_first_active_get(cohc);
607
608 BUG_ON(cohd_fin->pending_irqs == 0);
609
610 if (cohd_fin == NULL)
611 goto err;
612
613 cohd_fin->pending_irqs--;
614 cohc->completed = cohd_fin->desc.cookie;
615
616 BUG_ON(cohc->nbr_active_done && cohd_fin == NULL);
617
618 if (cohc->nbr_active_done == 0)
619 return;
620
621 if (!cohd_fin->pending_irqs) {
622 /* release the lli allocation*/
623 coh901318_lli_free(&cohc->base->pool, &cohd_fin->data);
624 }
625
626 dev_vdbg(COHC_2_DEV(cohc), "[%s] chan_id %d pending_irqs %d"
627 " nbr_active_done %ld\n", __func__,
628 cohc->id, cohc->pending_irqs, cohc->nbr_active_done);
629
630 /* callback to client */
631 callback = cohd_fin->desc.callback;
632 callback_param = cohd_fin->desc.callback_param;
633
634 if (!cohd_fin->pending_irqs) {
635 coh901318_desc_remove(cohd_fin);
636
637 /* return desc to free-list */
638 coh901318_desc_free(cohc, cohd_fin);
639 }
640
641 if (cohc->nbr_active_done)
642 cohc->nbr_active_done--;
643
644 if (cohc->nbr_active_done) {
645 if (cohc_chan_conf(cohc)->priority_high)
646 tasklet_hi_schedule(&cohc->tasklet);
647 else
648 tasklet_schedule(&cohc->tasklet);
649 }
650 spin_unlock_irqrestore(&cohc->lock, flags);
651
652 if (callback)
653 callback(callback_param);
654
655 return;
656
657 err:
658 spin_unlock_irqrestore(&cohc->lock, flags);
659 dev_err(COHC_2_DEV(cohc), "[%s] No active dma desc\n", __func__);
660}
661
662
663/* called from interrupt context */
664static void dma_tc_handle(struct coh901318_chan *cohc)
665{
666 BUG_ON(!cohc->allocated && (list_empty(&cohc->active) ||
667 list_empty(&cohc->queue)));
668
669 if (!cohc->allocated)
670 return;
671
672 BUG_ON(cohc->pending_irqs == 0);
673
674 cohc->pending_irqs--;
675 cohc->nbr_active_done++;
676
677 if (cohc->pending_irqs == 0 && coh901318_queue_start(cohc) == NULL)
678 cohc->busy = 0;
679
680 BUG_ON(list_empty(&cohc->active));
681
682 if (cohc_chan_conf(cohc)->priority_high)
683 tasklet_hi_schedule(&cohc->tasklet);
684 else
685 tasklet_schedule(&cohc->tasklet);
686}
687
688
689static irqreturn_t dma_irq_handler(int irq, void *dev_id)
690{
691 u32 status1;
692 u32 status2;
693 int i;
694 int ch;
695 struct coh901318_base *base = dev_id;
696 struct coh901318_chan *cohc;
697 void __iomem *virtbase = base->virtbase;
698
699 status1 = readl(virtbase + COH901318_INT_STATUS1);
700 status2 = readl(virtbase + COH901318_INT_STATUS2);
701
702 if (unlikely(status1 == 0 && status2 == 0)) {
703 dev_warn(base->dev, "spurious DMA IRQ from no channel!\n");
704 return IRQ_HANDLED;
705 }
706
707 /* TODO: consider handle IRQ in tasklet here to
708 * minimize interrupt latency */
709
710 /* Check the first 32 DMA channels for IRQ */
711 while (status1) {
712 /* Find first bit set, return as a number. */
713 i = ffs(status1) - 1;
714 ch = i;
715
716 cohc = &base->chans[ch];
717 spin_lock(&cohc->lock);
718
719 /* Mask off this bit */
720 status1 &= ~(1 << i);
721 /* Check the individual channel bits */
722 if (test_bit(i, virtbase + COH901318_BE_INT_STATUS1)) {
723 dev_crit(COHC_2_DEV(cohc),
724 "DMA bus error on channel %d!\n", ch);
725 BUG_ON(1);
726 /* Clear BE interrupt */
727 __set_bit(i, virtbase + COH901318_BE_INT_CLEAR1);
728 } else {
729 /* Caused by TC, really? */
730 if (unlikely(!test_bit(i, virtbase +
731 COH901318_TC_INT_STATUS1))) {
732 dev_warn(COHC_2_DEV(cohc),
733 "ignoring interrupt not caused by terminal count on channel %d\n", ch);
734 /* Clear TC interrupt */
735 BUG_ON(1);
736 __set_bit(i, virtbase + COH901318_TC_INT_CLEAR1);
737 } else {
738 /* Enable powersave if transfer has finished */
739 if (!(readl(virtbase + COH901318_CX_STAT +
740 COH901318_CX_STAT_SPACING*ch) &
741 COH901318_CX_STAT_ENABLED)) {
742 enable_powersave(cohc);
743 }
744
745 /* Must clear TC interrupt before calling
746 * dma_tc_handle
747 * in case tc_handle initate a new dma job
748 */
749 __set_bit(i, virtbase + COH901318_TC_INT_CLEAR1);
750
751 dma_tc_handle(cohc);
752 }
753 }
754 spin_unlock(&cohc->lock);
755 }
756
757 /* Check the remaining 32 DMA channels for IRQ */
758 while (status2) {
759 /* Find first bit set, return as a number. */
760 i = ffs(status2) - 1;
761 ch = i + 32;
762 cohc = &base->chans[ch];
763 spin_lock(&cohc->lock);
764
765 /* Mask off this bit */
766 status2 &= ~(1 << i);
767 /* Check the individual channel bits */
768 if (test_bit(i, virtbase + COH901318_BE_INT_STATUS2)) {
769 dev_crit(COHC_2_DEV(cohc),
770 "DMA bus error on channel %d!\n", ch);
771 /* Clear BE interrupt */
772 BUG_ON(1);
773 __set_bit(i, virtbase + COH901318_BE_INT_CLEAR2);
774 } else {
775 /* Caused by TC, really? */
776 if (unlikely(!test_bit(i, virtbase +
777 COH901318_TC_INT_STATUS2))) {
778 dev_warn(COHC_2_DEV(cohc),
779 "ignoring interrupt not caused by terminal count on channel %d\n", ch);
780 /* Clear TC interrupt */
781 __set_bit(i, virtbase + COH901318_TC_INT_CLEAR2);
782 BUG_ON(1);
783 } else {
784 /* Enable powersave if transfer has finished */
785 if (!(readl(virtbase + COH901318_CX_STAT +
786 COH901318_CX_STAT_SPACING*ch) &
787 COH901318_CX_STAT_ENABLED)) {
788 enable_powersave(cohc);
789 }
790 /* Must clear TC interrupt before calling
791 * dma_tc_handle
792 * in case tc_handle initate a new dma job
793 */
794 __set_bit(i, virtbase + COH901318_TC_INT_CLEAR2);
795
796 dma_tc_handle(cohc);
797 }
798 }
799 spin_unlock(&cohc->lock);
800 }
801
802 return IRQ_HANDLED;
803}
804
805static int coh901318_alloc_chan_resources(struct dma_chan *chan)
806{
807 struct coh901318_chan *cohc = to_coh901318_chan(chan);
808
809 dev_vdbg(COHC_2_DEV(cohc), "[%s] DMA channel %d\n",
810 __func__, cohc->id);
811
812 if (chan->client_count > 1)
813 return -EBUSY;
814
815 coh901318_config(cohc, NULL);
816
817 cohc->allocated = 1;
818 cohc->completed = chan->cookie = 1;
819
820 return 1;
821}
822
823static void
824coh901318_free_chan_resources(struct dma_chan *chan)
825{
826 struct coh901318_chan *cohc = to_coh901318_chan(chan);
827 int channel = cohc->id;
828 unsigned long flags;
829
830 spin_lock_irqsave(&cohc->lock, flags);
831
832 /* Disable HW */
833 writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CFG +
834 COH901318_CX_CFG_SPACING*channel);
835 writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CTRL +
836 COH901318_CX_CTRL_SPACING*channel);
837
838 cohc->allocated = 0;
839
840 spin_unlock_irqrestore(&cohc->lock, flags);
841
842 chan->device->device_terminate_all(chan);
843}
844
845
846static dma_cookie_t
847coh901318_tx_submit(struct dma_async_tx_descriptor *tx)
848{
849 struct coh901318_desc *cohd = container_of(tx, struct coh901318_desc,
850 desc);
851 struct coh901318_chan *cohc = to_coh901318_chan(tx->chan);
852 unsigned long flags;
853
854 spin_lock_irqsave(&cohc->lock, flags);
855
856 tx->cookie = coh901318_assign_cookie(cohc, cohd);
857
858 coh901318_desc_queue(cohc, cohd);
859
860 spin_unlock_irqrestore(&cohc->lock, flags);
861
862 return tx->cookie;
863}
864
865static struct dma_async_tx_descriptor *
866coh901318_prep_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
867 size_t size, unsigned long flags)
868{
869 struct coh901318_lli *data;
870 struct coh901318_desc *cohd;
871 unsigned long flg;
872 struct coh901318_chan *cohc = to_coh901318_chan(chan);
873 int lli_len;
874 u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last;
875
876 spin_lock_irqsave(&cohc->lock, flg);
877
878 dev_vdbg(COHC_2_DEV(cohc),
879 "[%s] channel %d src 0x%x dest 0x%x size %d\n",
880 __func__, cohc->id, src, dest, size);
881
882 if (flags & DMA_PREP_INTERRUPT)
883 /* Trigger interrupt after last lli */
884 ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE;
885
886 lli_len = size >> MAX_DMA_PACKET_SIZE_SHIFT;
887 if ((lli_len << MAX_DMA_PACKET_SIZE_SHIFT) < size)
888 lli_len++;
889
890 data = coh901318_lli_alloc(&cohc->base->pool, lli_len);
891
892 if (data == NULL)
893 goto err;
894
895 cohd = coh901318_desc_get(cohc);
896 cohd->sg = NULL;
897 cohd->sg_len = 0;
898 cohd->data = data;
899
900 cohd->pending_irqs =
901 coh901318_lli_fill_memcpy(
902 &cohc->base->pool, data, src, size, dest,
903 cohc_chan_param(cohc)->ctrl_lli_chained,
904 ctrl_last);
905 cohd->flags = flags;
906
907 COH_DBG(coh901318_list_print(cohc, data));
908
909 dma_async_tx_descriptor_init(&cohd->desc, chan);
910
911 cohd->desc.tx_submit = coh901318_tx_submit;
912
913 spin_unlock_irqrestore(&cohc->lock, flg);
914
915 return &cohd->desc;
916 err:
917 spin_unlock_irqrestore(&cohc->lock, flg);
918 return NULL;
919}
920
921static struct dma_async_tx_descriptor *
922coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
923 unsigned int sg_len, enum dma_data_direction direction,
924 unsigned long flags)
925{
926 struct coh901318_chan *cohc = to_coh901318_chan(chan);
927 struct coh901318_lli *data;
928 struct coh901318_desc *cohd;
929 struct scatterlist *sg;
930 int len = 0;
931 int size;
932 int i;
933 u32 ctrl_chained = cohc_chan_param(cohc)->ctrl_lli_chained;
934 u32 ctrl = cohc_chan_param(cohc)->ctrl_lli;
935 u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last;
936 unsigned long flg;
937
938 if (!sgl)
939 goto out;
940 if (sgl->length == 0)
941 goto out;
942
943 spin_lock_irqsave(&cohc->lock, flg);
944
945 dev_vdbg(COHC_2_DEV(cohc), "[%s] sg_len %d dir %d\n",
946 __func__, sg_len, direction);
947
948 if (flags & DMA_PREP_INTERRUPT)
949 /* Trigger interrupt after last lli */
950 ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE;
951
952 cohd = coh901318_desc_get(cohc);
953 cohd->sg = NULL;
954 cohd->sg_len = 0;
955 cohd->dir = direction;
956
957 if (direction == DMA_TO_DEVICE) {
958 u32 tx_flags = COH901318_CX_CTRL_PRDD_SOURCE |
959 COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE;
960
961 ctrl_chained |= tx_flags;
962 ctrl_last |= tx_flags;
963 ctrl |= tx_flags;
964 } else if (direction == DMA_FROM_DEVICE) {
965 u32 rx_flags = COH901318_CX_CTRL_PRDD_DEST |
966 COH901318_CX_CTRL_DST_ADDR_INC_ENABLE;
967
968 ctrl_chained |= rx_flags;
969 ctrl_last |= rx_flags;
970 ctrl |= rx_flags;
971 } else
972 goto err_direction;
973
974 dma_async_tx_descriptor_init(&cohd->desc, chan);
975
976 cohd->desc.tx_submit = coh901318_tx_submit;
977
978
979 /* The dma only supports transmitting packages up to
980 * MAX_DMA_PACKET_SIZE. Calculate to total number of
981 * dma elemts required to send the entire sg list
982 */
983 for_each_sg(sgl, sg, sg_len, i) {
984 unsigned int factor;
985 size = sg_dma_len(sg);
986
987 if (size <= MAX_DMA_PACKET_SIZE) {
988 len++;
989 continue;
990 }
991
992 factor = size >> MAX_DMA_PACKET_SIZE_SHIFT;
993 if ((factor << MAX_DMA_PACKET_SIZE_SHIFT) < size)
994 factor++;
995
996 len += factor;
997 }
998
999 data = coh901318_lli_alloc(&cohc->base->pool, len);
1000
1001 if (data == NULL)
1002 goto err_dma_alloc;
1003
1004 /* initiate allocated data list */
1005 cohd->pending_irqs =
1006 coh901318_lli_fill_sg(&cohc->base->pool, data, sgl, sg_len,
1007 cohc_dev_addr(cohc),
1008 ctrl_chained,
1009 ctrl,
1010 ctrl_last,
1011 direction, COH901318_CX_CTRL_TC_IRQ_ENABLE);
1012 cohd->data = data;
1013
1014 cohd->flags = flags;
1015
1016 COH_DBG(coh901318_list_print(cohc, data));
1017
1018 spin_unlock_irqrestore(&cohc->lock, flg);
1019
1020 return &cohd->desc;
1021 err_dma_alloc:
1022 err_direction:
1023 coh901318_desc_remove(cohd);
1024 coh901318_desc_free(cohc, cohd);
1025 spin_unlock_irqrestore(&cohc->lock, flg);
1026 out:
1027 return NULL;
1028}
1029
1030static enum dma_status
1031coh901318_is_tx_complete(struct dma_chan *chan,
1032 dma_cookie_t cookie, dma_cookie_t *done,
1033 dma_cookie_t *used)
1034{
1035 struct coh901318_chan *cohc = to_coh901318_chan(chan);
1036 dma_cookie_t last_used;
1037 dma_cookie_t last_complete;
1038 int ret;
1039
1040 last_complete = cohc->completed;
1041 last_used = chan->cookie;
1042
1043 ret = dma_async_is_complete(cookie, last_complete, last_used);
1044
1045 if (done)
1046 *done = last_complete;
1047 if (used)
1048 *used = last_used;
1049
1050 return ret;
1051}
1052
1053static void
1054coh901318_issue_pending(struct dma_chan *chan)
1055{
1056 struct coh901318_chan *cohc = to_coh901318_chan(chan);
1057 unsigned long flags;
1058
1059 spin_lock_irqsave(&cohc->lock, flags);
1060
1061 /* Busy means that pending jobs are already being processed */
1062 if (!cohc->busy)
1063 coh901318_queue_start(cohc);
1064
1065 spin_unlock_irqrestore(&cohc->lock, flags);
1066}
1067
1068static void
1069coh901318_terminate_all(struct dma_chan *chan)
1070{
1071 unsigned long flags;
1072 struct coh901318_chan *cohc = to_coh901318_chan(chan);
1073 struct coh901318_desc *cohd;
1074 void __iomem *virtbase = cohc->base->virtbase;
1075
1076 coh901318_stop(chan);
1077
1078 spin_lock_irqsave(&cohc->lock, flags);
1079
1080 /* Clear any pending BE or TC interrupt */
1081 if (cohc->id < 32) {
1082 writel(1 << cohc->id, virtbase + COH901318_BE_INT_CLEAR1);
1083 writel(1 << cohc->id, virtbase + COH901318_TC_INT_CLEAR1);
1084 } else {
1085 writel(1 << (cohc->id - 32), virtbase +
1086 COH901318_BE_INT_CLEAR2);
1087 writel(1 << (cohc->id - 32), virtbase +
1088 COH901318_TC_INT_CLEAR2);
1089 }
1090
1091 enable_powersave(cohc);
1092
1093 while ((cohd = coh901318_first_active_get(cohc))) {
1094 /* release the lli allocation*/
1095 coh901318_lli_free(&cohc->base->pool, &cohd->data);
1096
1097 coh901318_desc_remove(cohd);
1098
1099 /* return desc to free-list */
1100 coh901318_desc_free(cohc, cohd);
1101 }
1102
1103 while ((cohd = coh901318_first_queued(cohc))) {
1104 /* release the lli allocation*/
1105 coh901318_lli_free(&cohc->base->pool, &cohd->data);
1106
1107 coh901318_desc_remove(cohd);
1108
1109 /* return desc to free-list */
1110 coh901318_desc_free(cohc, cohd);
1111 }
1112
1113
1114 cohc->nbr_active_done = 0;
1115 cohc->busy = 0;
1116 cohc->pending_irqs = 0;
1117
1118 spin_unlock_irqrestore(&cohc->lock, flags);
1119}
1120void coh901318_base_init(struct dma_device *dma, const int *pick_chans,
1121 struct coh901318_base *base)
1122{
1123 int chans_i;
1124 int i = 0;
1125 struct coh901318_chan *cohc;
1126
1127 INIT_LIST_HEAD(&dma->channels);
1128
1129 for (chans_i = 0; pick_chans[chans_i] != -1; chans_i += 2) {
1130 for (i = pick_chans[chans_i]; i <= pick_chans[chans_i+1]; i++) {
1131 cohc = &base->chans[i];
1132
1133 cohc->base = base;
1134 cohc->chan.device = dma;
1135 cohc->id = i;
1136
1137 /* TODO: do we really need this lock if only one
1138 * client is connected to each channel?
1139 */
1140
1141 spin_lock_init(&cohc->lock);
1142
1143 cohc->pending_irqs = 0;
1144 cohc->nbr_active_done = 0;
1145 cohc->busy = 0;
1146 INIT_LIST_HEAD(&cohc->free);
1147 INIT_LIST_HEAD(&cohc->active);
1148 INIT_LIST_HEAD(&cohc->queue);
1149
1150 tasklet_init(&cohc->tasklet, dma_tasklet,
1151 (unsigned long) cohc);
1152
1153 list_add_tail(&cohc->chan.device_node,
1154 &dma->channels);
1155 }
1156 }
1157}
1158
1159static int __init coh901318_probe(struct platform_device *pdev)
1160{
1161 int err = 0;
1162 struct coh901318_platform *pdata;
1163 struct coh901318_base *base;
1164 int irq;
1165 struct resource *io;
1166
1167 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1168 if (!io)
1169 goto err_get_resource;
1170
1171 /* Map DMA controller registers to virtual memory */
1172 if (request_mem_region(io->start,
1173 resource_size(io),
1174 pdev->dev.driver->name) == NULL) {
1175 err = -EBUSY;
1176 goto err_request_mem;
1177 }
1178
1179 pdata = pdev->dev.platform_data;
1180 if (!pdata)
1181 goto err_no_platformdata;
1182
1183 base = kmalloc(ALIGN(sizeof(struct coh901318_base), 4) +
1184 pdata->max_channels *
1185 sizeof(struct coh901318_chan),
1186 GFP_KERNEL);
1187 if (!base)
1188 goto err_alloc_coh_dma_channels;
1189
1190 base->chans = ((void *)base) + ALIGN(sizeof(struct coh901318_base), 4);
1191
1192 base->virtbase = ioremap(io->start, resource_size(io));
1193 if (!base->virtbase) {
1194 err = -ENOMEM;
1195 goto err_no_ioremap;
1196 }
1197
1198 base->dev = &pdev->dev;
1199 base->platform = pdata;
1200 spin_lock_init(&base->pm.lock);
1201 base->pm.started_channels = 0;
1202
1203 COH901318_DEBUGFS_ASSIGN(debugfs_dma_base, base);
1204
1205 platform_set_drvdata(pdev, base);
1206
1207 irq = platform_get_irq(pdev, 0);
1208 if (irq < 0)
1209 goto err_no_irq;
1210
1211 err = request_irq(irq, dma_irq_handler, IRQF_DISABLED,
1212 "coh901318", base);
1213 if (err) {
1214 dev_crit(&pdev->dev,
1215 "Cannot allocate IRQ for DMA controller!\n");
1216 goto err_request_irq;
1217 }
1218
1219 err = coh901318_pool_create(&base->pool, &pdev->dev,
1220 sizeof(struct coh901318_lli),
1221 32);
1222 if (err)
1223 goto err_pool_create;
1224
1225 /* init channels for device transfers */
1226 coh901318_base_init(&base->dma_slave, base->platform->chans_slave,
1227 base);
1228
1229 dma_cap_zero(base->dma_slave.cap_mask);
1230 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
1231
1232 base->dma_slave.device_alloc_chan_resources = coh901318_alloc_chan_resources;
1233 base->dma_slave.device_free_chan_resources = coh901318_free_chan_resources;
1234 base->dma_slave.device_prep_slave_sg = coh901318_prep_slave_sg;
1235 base->dma_slave.device_is_tx_complete = coh901318_is_tx_complete;
1236 base->dma_slave.device_issue_pending = coh901318_issue_pending;
1237 base->dma_slave.device_terminate_all = coh901318_terminate_all;
1238 base->dma_slave.dev = &pdev->dev;
1239
1240 err = dma_async_device_register(&base->dma_slave);
1241
1242 if (err)
1243 goto err_register_slave;
1244
1245 /* init channels for memcpy */
1246 coh901318_base_init(&base->dma_memcpy, base->platform->chans_memcpy,
1247 base);
1248
1249 dma_cap_zero(base->dma_memcpy.cap_mask);
1250 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
1251
1252 base->dma_memcpy.device_alloc_chan_resources = coh901318_alloc_chan_resources;
1253 base->dma_memcpy.device_free_chan_resources = coh901318_free_chan_resources;
1254 base->dma_memcpy.device_prep_dma_memcpy = coh901318_prep_memcpy;
1255 base->dma_memcpy.device_is_tx_complete = coh901318_is_tx_complete;
1256 base->dma_memcpy.device_issue_pending = coh901318_issue_pending;
1257 base->dma_memcpy.device_terminate_all = coh901318_terminate_all;
1258 base->dma_memcpy.dev = &pdev->dev;
1259 err = dma_async_device_register(&base->dma_memcpy);
1260
1261 if (err)
1262 goto err_register_memcpy;
1263
1264 dev_dbg(&pdev->dev, "Initialized COH901318 DMA on virtual base 0x%08x\n",
1265 (u32) base->virtbase);
1266
1267 return err;
1268
1269 err_register_memcpy:
1270 dma_async_device_unregister(&base->dma_slave);
1271 err_register_slave:
1272 coh901318_pool_destroy(&base->pool);
1273 err_pool_create:
1274 free_irq(platform_get_irq(pdev, 0), base);
1275 err_request_irq:
1276 err_no_irq:
1277 iounmap(base->virtbase);
1278 err_no_ioremap:
1279 kfree(base);
1280 err_alloc_coh_dma_channels:
1281 err_no_platformdata:
1282 release_mem_region(pdev->resource->start,
1283 resource_size(pdev->resource));
1284 err_request_mem:
1285 err_get_resource:
1286 return err;
1287}
1288
1289static int __exit coh901318_remove(struct platform_device *pdev)
1290{
1291 struct coh901318_base *base = platform_get_drvdata(pdev);
1292
1293 dma_async_device_unregister(&base->dma_memcpy);
1294 dma_async_device_unregister(&base->dma_slave);
1295 coh901318_pool_destroy(&base->pool);
1296 free_irq(platform_get_irq(pdev, 0), base);
1297 kfree(base);
1298 iounmap(base->virtbase);
1299 release_mem_region(pdev->resource->start,
1300 resource_size(pdev->resource));
1301 return 0;
1302}
1303
1304
1305static struct platform_driver coh901318_driver = {
1306 .remove = __exit_p(coh901318_remove),
1307 .driver = {
1308 .name = "coh901318",
1309 },
1310};
1311
1312int __init coh901318_init(void)
1313{
1314 return platform_driver_probe(&coh901318_driver, coh901318_probe);
1315}
1316subsys_initcall(coh901318_init);
1317
1318void __exit coh901318_exit(void)
1319{
1320 platform_driver_unregister(&coh901318_driver);
1321}
1322module_exit(coh901318_exit);
1323
1324MODULE_LICENSE("GPL");
1325MODULE_AUTHOR("Per Friden");
diff --git a/drivers/dma/coh901318_lli.c b/drivers/dma/coh901318_lli.c
new file mode 100644
index 00000000000..f5120f238a4
--- /dev/null
+++ b/drivers/dma/coh901318_lli.c
@@ -0,0 +1,318 @@
1/*
2 * driver/dma/coh901318_lli.c
3 *
4 * Copyright (C) 2007-2009 ST-Ericsson
5 * License terms: GNU General Public License (GPL) version 2
6 * Support functions for handling lli for dma
7 * Author: Per Friden <per.friden@stericsson.com>
8 */
9
10#include <linux/dma-mapping.h>
11#include <linux/spinlock.h>
12#include <linux/dmapool.h>
13#include <linux/memory.h>
14#include <mach/coh901318.h>
15
16#include "coh901318_lli.h"
17
18#if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_U300_DEBUG))
19#define DEBUGFS_POOL_COUNTER_RESET(pool) (pool->debugfs_pool_counter = 0)
20#define DEBUGFS_POOL_COUNTER_ADD(pool, add) (pool->debugfs_pool_counter += add)
21#else
22#define DEBUGFS_POOL_COUNTER_RESET(pool)
23#define DEBUGFS_POOL_COUNTER_ADD(pool, add)
24#endif
25
26static struct coh901318_lli *
27coh901318_lli_next(struct coh901318_lli *data)
28{
29 if (data == NULL || data->link_addr == 0)
30 return NULL;
31
32 return (struct coh901318_lli *) data->virt_link_addr;
33}
34
35int coh901318_pool_create(struct coh901318_pool *pool,
36 struct device *dev,
37 size_t size, size_t align)
38{
39 spin_lock_init(&pool->lock);
40 pool->dev = dev;
41 pool->dmapool = dma_pool_create("lli_pool", dev, size, align, 0);
42
43 DEBUGFS_POOL_COUNTER_RESET(pool);
44 return 0;
45}
46
47int coh901318_pool_destroy(struct coh901318_pool *pool)
48{
49
50 dma_pool_destroy(pool->dmapool);
51 return 0;
52}
53
54struct coh901318_lli *
55coh901318_lli_alloc(struct coh901318_pool *pool, unsigned int len)
56{
57 int i;
58 struct coh901318_lli *head;
59 struct coh901318_lli *lli;
60 struct coh901318_lli *lli_prev;
61 dma_addr_t phy;
62
63 if (len == 0)
64 goto err;
65
66 spin_lock(&pool->lock);
67
68 head = dma_pool_alloc(pool->dmapool, GFP_NOWAIT, &phy);
69
70 if (head == NULL)
71 goto err;
72
73 DEBUGFS_POOL_COUNTER_ADD(pool, 1);
74
75 lli = head;
76 lli->phy_this = phy;
77
78 for (i = 1; i < len; i++) {
79 lli_prev = lli;
80
81 lli = dma_pool_alloc(pool->dmapool, GFP_NOWAIT, &phy);
82
83 if (lli == NULL)
84 goto err_clean_up;
85
86 DEBUGFS_POOL_COUNTER_ADD(pool, 1);
87 lli->phy_this = phy;
88
89 lli_prev->link_addr = phy;
90 lli_prev->virt_link_addr = lli;
91 }
92
93 lli->link_addr = 0x00000000U;
94
95 spin_unlock(&pool->lock);
96
97 return head;
98
99 err:
100 spin_unlock(&pool->lock);
101 return NULL;
102
103 err_clean_up:
104 lli_prev->link_addr = 0x00000000U;
105 spin_unlock(&pool->lock);
106 coh901318_lli_free(pool, &head);
107 return NULL;
108}
109
110void coh901318_lli_free(struct coh901318_pool *pool,
111 struct coh901318_lli **lli)
112{
113 struct coh901318_lli *l;
114 struct coh901318_lli *next;
115
116 if (lli == NULL)
117 return;
118
119 l = *lli;
120
121 if (l == NULL)
122 return;
123
124 spin_lock(&pool->lock);
125
126 while (l->link_addr) {
127 next = l->virt_link_addr;
128 dma_pool_free(pool->dmapool, l, l->phy_this);
129 DEBUGFS_POOL_COUNTER_ADD(pool, -1);
130 l = next;
131 }
132 dma_pool_free(pool->dmapool, l, l->phy_this);
133 DEBUGFS_POOL_COUNTER_ADD(pool, -1);
134
135 spin_unlock(&pool->lock);
136 *lli = NULL;
137}
138
139int
140coh901318_lli_fill_memcpy(struct coh901318_pool *pool,
141 struct coh901318_lli *lli,
142 dma_addr_t source, unsigned int size,
143 dma_addr_t destination, u32 ctrl_chained,
144 u32 ctrl_eom)
145{
146 int s = size;
147 dma_addr_t src = source;
148 dma_addr_t dst = destination;
149
150 lli->src_addr = src;
151 lli->dst_addr = dst;
152
153 while (lli->link_addr) {
154 lli->control = ctrl_chained | MAX_DMA_PACKET_SIZE;
155 lli->src_addr = src;
156 lli->dst_addr = dst;
157
158 s -= MAX_DMA_PACKET_SIZE;
159 lli = coh901318_lli_next(lli);
160
161 src += MAX_DMA_PACKET_SIZE;
162 dst += MAX_DMA_PACKET_SIZE;
163 }
164
165 lli->control = ctrl_eom | s;
166 lli->src_addr = src;
167 lli->dst_addr = dst;
168
169 /* One irq per single transfer */
170 return 1;
171}
172
173int
174coh901318_lli_fill_single(struct coh901318_pool *pool,
175 struct coh901318_lli *lli,
176 dma_addr_t buf, unsigned int size,
177 dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl_eom,
178 enum dma_data_direction dir)
179{
180 int s = size;
181 dma_addr_t src;
182 dma_addr_t dst;
183
184
185 if (dir == DMA_TO_DEVICE) {
186 src = buf;
187 dst = dev_addr;
188
189 } else if (dir == DMA_FROM_DEVICE) {
190
191 src = dev_addr;
192 dst = buf;
193 } else {
194 return -EINVAL;
195 }
196
197 while (lli->link_addr) {
198 size_t block_size = MAX_DMA_PACKET_SIZE;
199 lli->control = ctrl_chained | MAX_DMA_PACKET_SIZE;
200
201 /* If we are on the next-to-final block and there will
202 * be less than half a DMA packet left for the last
203 * block, then we want to make this block a little
204 * smaller to balance the sizes. This is meant to
205 * avoid too small transfers if the buffer size is
206 * (MAX_DMA_PACKET_SIZE*N + 1) */
207 if (s < (MAX_DMA_PACKET_SIZE + MAX_DMA_PACKET_SIZE/2))
208 block_size = MAX_DMA_PACKET_SIZE/2;
209
210 s -= block_size;
211 lli->src_addr = src;
212 lli->dst_addr = dst;
213
214 lli = coh901318_lli_next(lli);
215
216 if (dir == DMA_TO_DEVICE)
217 src += block_size;
218 else if (dir == DMA_FROM_DEVICE)
219 dst += block_size;
220 }
221
222 lli->control = ctrl_eom | s;
223 lli->src_addr = src;
224 lli->dst_addr = dst;
225
226 /* One irq per single transfer */
227 return 1;
228}
229
230int
231coh901318_lli_fill_sg(struct coh901318_pool *pool,
232 struct coh901318_lli *lli,
233 struct scatterlist *sgl, unsigned int nents,
234 dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl,
235 u32 ctrl_last,
236 enum dma_data_direction dir, u32 ctrl_irq_mask)
237{
238 int i;
239 struct scatterlist *sg;
240 u32 ctrl_sg;
241 dma_addr_t src = 0;
242 dma_addr_t dst = 0;
243 int nbr_of_irq = 0;
244 u32 bytes_to_transfer;
245 u32 elem_size;
246
247 if (lli == NULL)
248 goto err;
249
250 spin_lock(&pool->lock);
251
252 if (dir == DMA_TO_DEVICE)
253 dst = dev_addr;
254 else if (dir == DMA_FROM_DEVICE)
255 src = dev_addr;
256 else
257 goto err;
258
259 for_each_sg(sgl, sg, nents, i) {
260 if (sg_is_chain(sg)) {
261 /* sg continues to the next sg-element don't
262 * send ctrl_finish until the last
263 * sg-element in the chain
264 */
265 ctrl_sg = ctrl_chained;
266 } else if (i == nents - 1)
267 ctrl_sg = ctrl_last;
268 else
269 ctrl_sg = ctrl ? ctrl : ctrl_last;
270
271
272 if ((ctrl_sg & ctrl_irq_mask))
273 nbr_of_irq++;
274
275 if (dir == DMA_TO_DEVICE)
276 /* increment source address */
277 src = sg_dma_address(sg);
278 else
279 /* increment destination address */
280 dst = sg_dma_address(sg);
281
282 bytes_to_transfer = sg_dma_len(sg);
283
284 while (bytes_to_transfer) {
285 u32 val;
286
287 if (bytes_to_transfer > MAX_DMA_PACKET_SIZE) {
288 elem_size = MAX_DMA_PACKET_SIZE;
289 val = ctrl_chained;
290 } else {
291 elem_size = bytes_to_transfer;
292 val = ctrl_sg;
293 }
294
295 lli->control = val | elem_size;
296 lli->src_addr = src;
297 lli->dst_addr = dst;
298
299 if (dir == DMA_FROM_DEVICE)
300 dst += elem_size;
301 else
302 src += elem_size;
303
304 BUG_ON(lli->link_addr & 3);
305
306 bytes_to_transfer -= elem_size;
307 lli = coh901318_lli_next(lli);
308 }
309
310 }
311 spin_unlock(&pool->lock);
312
313 /* There can be many IRQs per sg transfer */
314 return nbr_of_irq;
315 err:
316 spin_unlock(&pool->lock);
317 return -EINVAL;
318}
diff --git a/drivers/dma/coh901318_lli.h b/drivers/dma/coh901318_lli.h
new file mode 100644
index 00000000000..7bf713b79c6
--- /dev/null
+++ b/drivers/dma/coh901318_lli.h
@@ -0,0 +1,124 @@
1/*
2 * driver/dma/coh901318_lli.h
3 *
4 * Copyright (C) 2007-2009 ST-Ericsson
5 * License terms: GNU General Public License (GPL) version 2
6 * Support functions for handling lli for coh901318
7 * Author: Per Friden <per.friden@stericsson.com>
8 */
9
10#ifndef COH901318_LLI_H
11#define COH901318_LLI_H
12
13#include <mach/coh901318.h>
14
15struct device;
16
17struct coh901318_pool {
18 spinlock_t lock;
19 struct dma_pool *dmapool;
20 struct device *dev;
21
22#ifdef CONFIG_DEBUG_FS
23 int debugfs_pool_counter;
24#endif
25};
26
27struct device;
28/**
29 * coh901318_pool_create() - Creates an dma pool for lli:s
30 * @pool: pool handle
31 * @dev: dma device
32 * @lli_nbr: number of lli:s in the pool
33 * @algin: adress alignemtn of lli:s
34 * returns 0 on success otherwise none zero
35 */
36int coh901318_pool_create(struct coh901318_pool *pool,
37 struct device *dev,
38 size_t lli_nbr, size_t align);
39
40/**
41 * coh901318_pool_destroy() - Destroys the dma pool
42 * @pool: pool handle
43 * returns 0 on success otherwise none zero
44 */
45int coh901318_pool_destroy(struct coh901318_pool *pool);
46
47/**
48 * coh901318_lli_alloc() - Allocates a linked list
49 *
50 * @pool: pool handle
51 * @len: length to list
52 * return: none NULL if success otherwise NULL
53 */
54struct coh901318_lli *
55coh901318_lli_alloc(struct coh901318_pool *pool,
56 unsigned int len);
57
58/**
59 * coh901318_lli_free() - Returns the linked list items to the pool
60 * @pool: pool handle
61 * @lli: reference to lli pointer to be freed
62 */
63void coh901318_lli_free(struct coh901318_pool *pool,
64 struct coh901318_lli **lli);
65
66/**
67 * coh901318_lli_fill_memcpy() - Prepares the lli:s for dma memcpy
68 * @pool: pool handle
69 * @lli: allocated lli
70 * @src: src address
71 * @size: transfer size
72 * @dst: destination address
73 * @ctrl_chained: ctrl for chained lli
74 * @ctrl_last: ctrl for the last lli
75 * returns number of CPU interrupts for the lli, negative on error.
76 */
77int
78coh901318_lli_fill_memcpy(struct coh901318_pool *pool,
79 struct coh901318_lli *lli,
80 dma_addr_t src, unsigned int size,
81 dma_addr_t dst, u32 ctrl_chained, u32 ctrl_last);
82
83/**
84 * coh901318_lli_fill_single() - Prepares the lli:s for dma single transfer
85 * @pool: pool handle
86 * @lli: allocated lli
87 * @buf: transfer buffer
88 * @size: transfer size
89 * @dev_addr: address of periphal
90 * @ctrl_chained: ctrl for chained lli
91 * @ctrl_last: ctrl for the last lli
92 * @dir: direction of transfer (to or from device)
93 * returns number of CPU interrupts for the lli, negative on error.
94 */
95int
96coh901318_lli_fill_single(struct coh901318_pool *pool,
97 struct coh901318_lli *lli,
98 dma_addr_t buf, unsigned int size,
99 dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl_last,
100 enum dma_data_direction dir);
101
102/**
103 * coh901318_lli_fill_single() - Prepares the lli:s for dma scatter list transfer
104 * @pool: pool handle
105 * @lli: allocated lli
106 * @sg: scatter gather list
107 * @nents: number of entries in sg
108 * @dev_addr: address of periphal
109 * @ctrl_chained: ctrl for chained lli
110 * @ctrl: ctrl of middle lli
111 * @ctrl_last: ctrl for the last lli
112 * @dir: direction of transfer (to or from device)
113 * @ctrl_irq_mask: ctrl mask for CPU interrupt
114 * returns number of CPU interrupts for the lli, negative on error.
115 */
116int
117coh901318_lli_fill_sg(struct coh901318_pool *pool,
118 struct coh901318_lli *lli,
119 struct scatterlist *sg, unsigned int nents,
120 dma_addr_t dev_addr, u32 ctrl_chained,
121 u32 ctrl, u32 ctrl_last,
122 enum dma_data_direction dir, u32 ctrl_irq_mask);
123
124#endif /* COH901318_LLI_H */
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index a32a4cf7b1e..8b905161fbf 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -298,10 +298,6 @@ static int dmatest_func(void *data)
298 298
299 total_tests++; 299 total_tests++;
300 300
301 len = dmatest_random() % test_buf_size + 1;
302 src_off = dmatest_random() % (test_buf_size - len + 1);
303 dst_off = dmatest_random() % (test_buf_size - len + 1);
304
305 /* honor alignment restrictions */ 301 /* honor alignment restrictions */
306 if (thread->type == DMA_MEMCPY) 302 if (thread->type == DMA_MEMCPY)
307 align = dev->copy_align; 303 align = dev->copy_align;
@@ -310,7 +306,19 @@ static int dmatest_func(void *data)
310 else if (thread->type == DMA_PQ) 306 else if (thread->type == DMA_PQ)
311 align = dev->pq_align; 307 align = dev->pq_align;
312 308
309 if (1 << align > test_buf_size) {
310 pr_err("%u-byte buffer too small for %d-byte alignment\n",
311 test_buf_size, 1 << align);
312 break;
313 }
314
315 len = dmatest_random() % test_buf_size + 1;
313 len = (len >> align) << align; 316 len = (len >> align) << align;
317 if (!len)
318 len = 1 << align;
319 src_off = dmatest_random() % (test_buf_size - len + 1);
320 dst_off = dmatest_random() % (test_buf_size - len + 1);
321
314 src_off = (src_off >> align) << align; 322 src_off = (src_off >> align) << align;
315 dst_off = (dst_off >> align) << align; 323 dst_off = (dst_off >> align) << align;
316 324
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index 645ca8d54ec..ca6e6a0cb79 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -1470,7 +1470,7 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
1470 return -ENODEV; 1470 return -ENODEV;
1471 1471
1472 if (!devm_request_mem_region(&pdev->dev, res->start, 1472 if (!devm_request_mem_region(&pdev->dev, res->start,
1473 res->end - res->start, pdev->name)) 1473 resource_size(res), pdev->name))
1474 return -EBUSY; 1474 return -EBUSY;
1475 1475
1476 adev = kzalloc(sizeof(*adev), GFP_KERNEL); 1476 adev = kzalloc(sizeof(*adev), GFP_KERNEL);
@@ -1542,7 +1542,7 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
1542 iop_chan->device = adev; 1542 iop_chan->device = adev;
1543 1543
1544 iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start, 1544 iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1545 res->end - res->start); 1545 resource_size(res));
1546 if (!iop_chan->mmr_base) { 1546 if (!iop_chan->mmr_base) {
1547 ret = -ENOMEM; 1547 ret = -ENOMEM;
1548 goto err_free_iop_chan; 1548 goto err_free_iop_chan;
diff --git a/drivers/dma/ppc4xx/Makefile b/drivers/dma/ppc4xx/Makefile
new file mode 100644
index 00000000000..b3d259b3e52
--- /dev/null
+++ b/drivers/dma/ppc4xx/Makefile
@@ -0,0 +1 @@
obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += adma.o
diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c
new file mode 100644
index 00000000000..0a3478e910f
--- /dev/null
+++ b/drivers/dma/ppc4xx/adma.c
@@ -0,0 +1,5027 @@
1/*
2 * Copyright (C) 2006-2009 DENX Software Engineering.
3 *
4 * Author: Yuri Tikhonov <yur@emcraft.com>
5 *
6 * Further porting to arch/powerpc by
7 * Anatolij Gustschin <agust@denx.de>
8 *
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU General Public License as published by the Free
11 * Software Foundation; either version 2 of the License, or (at your option)
12 * any later version.
13 *
14 * This program is distributed in the hope that it will be useful, but WITHOUT
15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 * more details.
18 *
19 * You should have received a copy of the GNU General Public License along with
20 * this program; if not, write to the Free Software Foundation, Inc., 59
21 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 *
23 * The full GNU General Public License is included in this distribution in the
24 * file called COPYING.
25 */
26
27/*
28 * This driver supports the asynchrounous DMA copy and RAID engines available
29 * on the AMCC PPC440SPe Processors.
30 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
31 * ADMA driver written by D.Williams.
32 */
33
34#include <linux/init.h>
35#include <linux/module.h>
36#include <linux/async_tx.h>
37#include <linux/delay.h>
38#include <linux/dma-mapping.h>
39#include <linux/spinlock.h>
40#include <linux/interrupt.h>
41#include <linux/uaccess.h>
42#include <linux/proc_fs.h>
43#include <linux/of.h>
44#include <linux/of_platform.h>
45#include <asm/dcr.h>
46#include <asm/dcr-regs.h>
47#include "adma.h"
48
49enum ppc_adma_init_code {
50 PPC_ADMA_INIT_OK = 0,
51 PPC_ADMA_INIT_MEMRES,
52 PPC_ADMA_INIT_MEMREG,
53 PPC_ADMA_INIT_ALLOC,
54 PPC_ADMA_INIT_COHERENT,
55 PPC_ADMA_INIT_CHANNEL,
56 PPC_ADMA_INIT_IRQ1,
57 PPC_ADMA_INIT_IRQ2,
58 PPC_ADMA_INIT_REGISTER
59};
60
61static char *ppc_adma_errors[] = {
62 [PPC_ADMA_INIT_OK] = "ok",
63 [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
64 [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
65 [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
66 "structure",
67 [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
68 "hardware descriptors",
69 [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
70 [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
71 [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
72 [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
73};
74
75static enum ppc_adma_init_code
76ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
77
78struct ppc_dma_chan_ref {
79 struct dma_chan *chan;
80 struct list_head node;
81};
82
83/* The list of channels exported by ppc440spe ADMA */
84struct list_head
85ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
86
87/* This flag is set when want to refetch the xor chain in the interrupt
88 * handler
89 */
90static u32 do_xor_refetch;
91
92/* Pointer to DMA0, DMA1 CP/CS FIFO */
93static void *ppc440spe_dma_fifo_buf;
94
95/* Pointers to last submitted to DMA0, DMA1 CDBs */
96static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
97static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
98
99/* Pointer to last linked and submitted xor CB */
100static struct ppc440spe_adma_desc_slot *xor_last_linked;
101static struct ppc440spe_adma_desc_slot *xor_last_submit;
102
103/* This array is used in data-check operations for storing a pattern */
104static char ppc440spe_qword[16];
105
106static atomic_t ppc440spe_adma_err_irq_ref;
107static dcr_host_t ppc440spe_mq_dcr_host;
108static unsigned int ppc440spe_mq_dcr_len;
109
110/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
111 * the block size in transactions, then we do not allow to activate more than
112 * only one RXOR transactions simultaneously. So use this var to store
113 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
114 * set) or not (PPC440SPE_RXOR_RUN is clear).
115 */
116static unsigned long ppc440spe_rxor_state;
117
118/* These are used in enable & check routines
119 */
120static u32 ppc440spe_r6_enabled;
121static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
122static struct completion ppc440spe_r6_test_comp;
123
124static int ppc440spe_adma_dma2rxor_prep_src(
125 struct ppc440spe_adma_desc_slot *desc,
126 struct ppc440spe_rxor *cursor, int index,
127 int src_cnt, u32 addr);
128static void ppc440spe_adma_dma2rxor_set_src(
129 struct ppc440spe_adma_desc_slot *desc,
130 int index, dma_addr_t addr);
131static void ppc440spe_adma_dma2rxor_set_mult(
132 struct ppc440spe_adma_desc_slot *desc,
133 int index, u8 mult);
134
135#ifdef ADMA_LL_DEBUG
136#define ADMA_LL_DBG(x) ({ if (1) x; 0; })
137#else
138#define ADMA_LL_DBG(x) ({ if (0) x; 0; })
139#endif
140
141static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
142{
143 struct dma_cdb *cdb;
144 struct xor_cb *cb;
145 int i;
146
147 switch (chan->device->id) {
148 case 0:
149 case 1:
150 cdb = block;
151
152 pr_debug("CDB at %p [%d]:\n"
153 "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
154 "\t sg1u 0x%08x sg1l 0x%08x\n"
155 "\t sg2u 0x%08x sg2l 0x%08x\n"
156 "\t sg3u 0x%08x sg3l 0x%08x\n",
157 cdb, chan->device->id,
158 cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
159 le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
160 le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
161 le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
162 );
163 break;
164 case 2:
165 cb = block;
166
167 pr_debug("CB at %p [%d]:\n"
168 "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
169 "\t cbtah 0x%08x cbtal 0x%08x\n"
170 "\t cblah 0x%08x cblal 0x%08x\n",
171 cb, chan->device->id,
172 cb->cbc, cb->cbbc, cb->cbs,
173 cb->cbtah, cb->cbtal,
174 cb->cblah, cb->cblal);
175 for (i = 0; i < 16; i++) {
176 if (i && !cb->ops[i].h && !cb->ops[i].l)
177 continue;
178 pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
179 i, cb->ops[i].h, cb->ops[i].l);
180 }
181 break;
182 }
183}
184
185static void print_cb_list(struct ppc440spe_adma_chan *chan,
186 struct ppc440spe_adma_desc_slot *iter)
187{
188 for (; iter; iter = iter->hw_next)
189 print_cb(chan, iter->hw_desc);
190}
191
192static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
193 unsigned int src_cnt)
194{
195 int i;
196
197 pr_debug("\n%s(%d):\nsrc: ", __func__, id);
198 for (i = 0; i < src_cnt; i++)
199 pr_debug("\t0x%016llx ", src[i]);
200 pr_debug("dst:\n\t0x%016llx\n", dst);
201}
202
203static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
204 unsigned int src_cnt)
205{
206 int i;
207
208 pr_debug("\n%s(%d):\nsrc: ", __func__, id);
209 for (i = 0; i < src_cnt; i++)
210 pr_debug("\t0x%016llx ", src[i]);
211 pr_debug("dst: ");
212 for (i = 0; i < 2; i++)
213 pr_debug("\t0x%016llx ", dst[i]);
214}
215
216static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
217 unsigned int src_cnt,
218 const unsigned char *scf)
219{
220 int i;
221
222 pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
223 if (scf) {
224 for (i = 0; i < src_cnt; i++)
225 pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
226 } else {
227 for (i = 0; i < src_cnt; i++)
228 pr_debug("\t0x%016llx(no) ", src[i]);
229 }
230
231 pr_debug("dst: ");
232 for (i = 0; i < 2; i++)
233 pr_debug("\t0x%016llx ", src[src_cnt + i]);
234}
235
236/******************************************************************************
237 * Command (Descriptor) Blocks low-level routines
238 ******************************************************************************/
239/**
240 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
241 * pseudo operation
242 */
243static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
244 struct ppc440spe_adma_chan *chan)
245{
246 struct xor_cb *p;
247
248 switch (chan->device->id) {
249 case PPC440SPE_XOR_ID:
250 p = desc->hw_desc;
251 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
252 /* NOP with Command Block Complete Enable */
253 p->cbc = XOR_CBCR_CBCE_BIT;
254 break;
255 case PPC440SPE_DMA0_ID:
256 case PPC440SPE_DMA1_ID:
257 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
258 /* NOP with interrupt */
259 set_bit(PPC440SPE_DESC_INT, &desc->flags);
260 break;
261 default:
262 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
263 __func__);
264 break;
265 }
266}
267
268/**
269 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
270 * pseudo operation
271 */
272static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
273{
274 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
275 desc->hw_next = NULL;
276 desc->src_cnt = 0;
277 desc->dst_cnt = 1;
278}
279
280/**
281 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
282 */
283static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
284 int src_cnt, unsigned long flags)
285{
286 struct xor_cb *hw_desc = desc->hw_desc;
287
288 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
289 desc->hw_next = NULL;
290 desc->src_cnt = src_cnt;
291 desc->dst_cnt = 1;
292
293 hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
294 if (flags & DMA_PREP_INTERRUPT)
295 /* Enable interrupt on completion */
296 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
297}
298
299/**
300 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
301 * operation in DMA2 controller
302 */
303static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
304 int dst_cnt, int src_cnt, unsigned long flags)
305{
306 struct xor_cb *hw_desc = desc->hw_desc;
307
308 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
309 desc->hw_next = NULL;
310 desc->src_cnt = src_cnt;
311 desc->dst_cnt = dst_cnt;
312 memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
313 desc->descs_per_op = 0;
314
315 hw_desc->cbc = XOR_CBCR_TGT_BIT;
316 if (flags & DMA_PREP_INTERRUPT)
317 /* Enable interrupt on completion */
318 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
319}
320
321#define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE
322#define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1)
323#define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1)
324
325/**
326 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
327 * with DMA0/1
328 */
329static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
330 int dst_cnt, int src_cnt, unsigned long flags,
331 unsigned long op)
332{
333 struct dma_cdb *hw_desc;
334 struct ppc440spe_adma_desc_slot *iter;
335 u8 dopc;
336
337 /* Common initialization of a PQ descriptors chain */
338 set_bits(op, &desc->flags);
339 desc->src_cnt = src_cnt;
340 desc->dst_cnt = dst_cnt;
341
342 /* WXOR MULTICAST if both P and Q are being computed
343 * MV_SG1_SG2 if Q only
344 */
345 dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
346 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
347
348 list_for_each_entry(iter, &desc->group_list, chain_node) {
349 hw_desc = iter->hw_desc;
350 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
351
352 if (likely(!list_is_last(&iter->chain_node,
353 &desc->group_list))) {
354 /* set 'next' pointer */
355 iter->hw_next = list_entry(iter->chain_node.next,
356 struct ppc440spe_adma_desc_slot, chain_node);
357 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
358 } else {
359 /* this is the last descriptor.
360 * this slot will be pasted from ADMA level
361 * each time it wants to configure parameters
362 * of the transaction (src, dst, ...)
363 */
364 iter->hw_next = NULL;
365 if (flags & DMA_PREP_INTERRUPT)
366 set_bit(PPC440SPE_DESC_INT, &iter->flags);
367 else
368 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
369 }
370 }
371
372 /* Set OPS depending on WXOR/RXOR type of operation */
373 if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
374 /* This is a WXOR only chain:
375 * - first descriptors are for zeroing destinations
376 * if PPC440SPE_ZERO_P/Q set;
377 * - descriptors remained are for GF-XOR operations.
378 */
379 iter = list_first_entry(&desc->group_list,
380 struct ppc440spe_adma_desc_slot,
381 chain_node);
382
383 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
384 hw_desc = iter->hw_desc;
385 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
386 iter = list_first_entry(&iter->chain_node,
387 struct ppc440spe_adma_desc_slot,
388 chain_node);
389 }
390
391 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
392 hw_desc = iter->hw_desc;
393 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
394 iter = list_first_entry(&iter->chain_node,
395 struct ppc440spe_adma_desc_slot,
396 chain_node);
397 }
398
399 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
400 hw_desc = iter->hw_desc;
401 hw_desc->opc = dopc;
402 }
403 } else {
404 /* This is either RXOR-only or mixed RXOR/WXOR */
405
406 /* The first 1 or 2 slots in chain are always RXOR,
407 * if need to calculate P & Q, then there are two
408 * RXOR slots; if only P or only Q, then there is one
409 */
410 iter = list_first_entry(&desc->group_list,
411 struct ppc440spe_adma_desc_slot,
412 chain_node);
413 hw_desc = iter->hw_desc;
414 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
415
416 if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
417 iter = list_first_entry(&iter->chain_node,
418 struct ppc440spe_adma_desc_slot,
419 chain_node);
420 hw_desc = iter->hw_desc;
421 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
422 }
423
424 /* The remaining descs (if any) are WXORs */
425 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
426 iter = list_first_entry(&iter->chain_node,
427 struct ppc440spe_adma_desc_slot,
428 chain_node);
429 list_for_each_entry_from(iter, &desc->group_list,
430 chain_node) {
431 hw_desc = iter->hw_desc;
432 hw_desc->opc = dopc;
433 }
434 }
435 }
436}
437
438/**
439 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
440 * for PQ_ZERO_SUM operation
441 */
442static void ppc440spe_desc_init_dma01pqzero_sum(
443 struct ppc440spe_adma_desc_slot *desc,
444 int dst_cnt, int src_cnt)
445{
446 struct dma_cdb *hw_desc;
447 struct ppc440spe_adma_desc_slot *iter;
448 int i = 0;
449 u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
450 DMA_CDB_OPC_MV_SG1_SG2;
451 /*
452 * Initialize starting from 2nd or 3rd descriptor dependent
453 * on dst_cnt. First one or two slots are for cloning P
454 * and/or Q to chan->pdest and/or chan->qdest as we have
455 * to preserve original P/Q.
456 */
457 iter = list_first_entry(&desc->group_list,
458 struct ppc440spe_adma_desc_slot, chain_node);
459 iter = list_entry(iter->chain_node.next,
460 struct ppc440spe_adma_desc_slot, chain_node);
461
462 if (dst_cnt > 1) {
463 iter = list_entry(iter->chain_node.next,
464 struct ppc440spe_adma_desc_slot, chain_node);
465 }
466 /* initialize each source descriptor in chain */
467 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
468 hw_desc = iter->hw_desc;
469 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
470 iter->src_cnt = 0;
471 iter->dst_cnt = 0;
472
473 /* This is a ZERO_SUM operation:
474 * - <src_cnt> descriptors starting from 2nd or 3rd
475 * descriptor are for GF-XOR operations;
476 * - remaining <dst_cnt> descriptors are for checking the result
477 */
478 if (i++ < src_cnt)
479 /* MV_SG1_SG2 if only Q is being verified
480 * MULTICAST if both P and Q are being verified
481 */
482 hw_desc->opc = dopc;
483 else
484 /* DMA_CDB_OPC_DCHECK128 operation */
485 hw_desc->opc = DMA_CDB_OPC_DCHECK128;
486
487 if (likely(!list_is_last(&iter->chain_node,
488 &desc->group_list))) {
489 /* set 'next' pointer */
490 iter->hw_next = list_entry(iter->chain_node.next,
491 struct ppc440spe_adma_desc_slot,
492 chain_node);
493 } else {
494 /* this is the last descriptor.
495 * this slot will be pasted from ADMA level
496 * each time it wants to configure parameters
497 * of the transaction (src, dst, ...)
498 */
499 iter->hw_next = NULL;
500 /* always enable interrupt generation since we get
501 * the status of pqzero from the handler
502 */
503 set_bit(PPC440SPE_DESC_INT, &iter->flags);
504 }
505 }
506 desc->src_cnt = src_cnt;
507 desc->dst_cnt = dst_cnt;
508}
509
510/**
511 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
512 */
513static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
514 unsigned long flags)
515{
516 struct dma_cdb *hw_desc = desc->hw_desc;
517
518 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
519 desc->hw_next = NULL;
520 desc->src_cnt = 1;
521 desc->dst_cnt = 1;
522
523 if (flags & DMA_PREP_INTERRUPT)
524 set_bit(PPC440SPE_DESC_INT, &desc->flags);
525 else
526 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
527
528 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
529}
530
531/**
532 * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation
533 */
534static void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc,
535 int value, unsigned long flags)
536{
537 struct dma_cdb *hw_desc = desc->hw_desc;
538
539 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
540 desc->hw_next = NULL;
541 desc->src_cnt = 1;
542 desc->dst_cnt = 1;
543
544 if (flags & DMA_PREP_INTERRUPT)
545 set_bit(PPC440SPE_DESC_INT, &desc->flags);
546 else
547 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
548
549 hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value);
550 hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value);
551 hw_desc->opc = DMA_CDB_OPC_DFILL128;
552}
553
554/**
555 * ppc440spe_desc_set_src_addr - set source address into the descriptor
556 */
557static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
558 struct ppc440spe_adma_chan *chan,
559 int src_idx, dma_addr_t addrh,
560 dma_addr_t addrl)
561{
562 struct dma_cdb *dma_hw_desc;
563 struct xor_cb *xor_hw_desc;
564 phys_addr_t addr64, tmplow, tmphi;
565
566 switch (chan->device->id) {
567 case PPC440SPE_DMA0_ID:
568 case PPC440SPE_DMA1_ID:
569 if (!addrh) {
570 addr64 = addrl;
571 tmphi = (addr64 >> 32);
572 tmplow = (addr64 & 0xFFFFFFFF);
573 } else {
574 tmphi = addrh;
575 tmplow = addrl;
576 }
577 dma_hw_desc = desc->hw_desc;
578 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
579 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
580 break;
581 case PPC440SPE_XOR_ID:
582 xor_hw_desc = desc->hw_desc;
583 xor_hw_desc->ops[src_idx].l = addrl;
584 xor_hw_desc->ops[src_idx].h |= addrh;
585 break;
586 }
587}
588
589/**
590 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
591 */
592static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
593 struct ppc440spe_adma_chan *chan, u32 mult_index,
594 int sg_index, unsigned char mult_value)
595{
596 struct dma_cdb *dma_hw_desc;
597 struct xor_cb *xor_hw_desc;
598 u32 *psgu;
599
600 switch (chan->device->id) {
601 case PPC440SPE_DMA0_ID:
602 case PPC440SPE_DMA1_ID:
603 dma_hw_desc = desc->hw_desc;
604
605 switch (sg_index) {
606 /* for RXOR operations set multiplier
607 * into source cued address
608 */
609 case DMA_CDB_SG_SRC:
610 psgu = &dma_hw_desc->sg1u;
611 break;
612 /* for WXOR operations set multiplier
613 * into destination cued address(es)
614 */
615 case DMA_CDB_SG_DST1:
616 psgu = &dma_hw_desc->sg2u;
617 break;
618 case DMA_CDB_SG_DST2:
619 psgu = &dma_hw_desc->sg3u;
620 break;
621 default:
622 BUG();
623 }
624
625 *psgu |= cpu_to_le32(mult_value << mult_index);
626 break;
627 case PPC440SPE_XOR_ID:
628 xor_hw_desc = desc->hw_desc;
629 break;
630 default:
631 BUG();
632 }
633}
634
635/**
636 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
637 */
638static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
639 struct ppc440spe_adma_chan *chan,
640 dma_addr_t addrh, dma_addr_t addrl,
641 u32 dst_idx)
642{
643 struct dma_cdb *dma_hw_desc;
644 struct xor_cb *xor_hw_desc;
645 phys_addr_t addr64, tmphi, tmplow;
646 u32 *psgu, *psgl;
647
648 switch (chan->device->id) {
649 case PPC440SPE_DMA0_ID:
650 case PPC440SPE_DMA1_ID:
651 if (!addrh) {
652 addr64 = addrl;
653 tmphi = (addr64 >> 32);
654 tmplow = (addr64 & 0xFFFFFFFF);
655 } else {
656 tmphi = addrh;
657 tmplow = addrl;
658 }
659 dma_hw_desc = desc->hw_desc;
660
661 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
662 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
663
664 *psgl = cpu_to_le32((u32)tmplow);
665 *psgu |= cpu_to_le32((u32)tmphi);
666 break;
667 case PPC440SPE_XOR_ID:
668 xor_hw_desc = desc->hw_desc;
669 xor_hw_desc->cbtal = addrl;
670 xor_hw_desc->cbtah |= addrh;
671 break;
672 }
673}
674
675/**
676 * ppc440spe_desc_set_byte_count - set number of data bytes involved
677 * into the operation
678 */
679static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
680 struct ppc440spe_adma_chan *chan,
681 u32 byte_count)
682{
683 struct dma_cdb *dma_hw_desc;
684 struct xor_cb *xor_hw_desc;
685
686 switch (chan->device->id) {
687 case PPC440SPE_DMA0_ID:
688 case PPC440SPE_DMA1_ID:
689 dma_hw_desc = desc->hw_desc;
690 dma_hw_desc->cnt = cpu_to_le32(byte_count);
691 break;
692 case PPC440SPE_XOR_ID:
693 xor_hw_desc = desc->hw_desc;
694 xor_hw_desc->cbbc = byte_count;
695 break;
696 }
697}
698
699/**
700 * ppc440spe_desc_set_rxor_block_size - set RXOR block size
701 */
702static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
703{
704 /* assume that byte_count is aligned on the 512-boundary;
705 * thus write it directly to the register (bits 23:31 are
706 * reserved there).
707 */
708 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
709}
710
711/**
712 * ppc440spe_desc_set_dcheck - set CHECK pattern
713 */
714static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
715 struct ppc440spe_adma_chan *chan, u8 *qword)
716{
717 struct dma_cdb *dma_hw_desc;
718
719 switch (chan->device->id) {
720 case PPC440SPE_DMA0_ID:
721 case PPC440SPE_DMA1_ID:
722 dma_hw_desc = desc->hw_desc;
723 iowrite32(qword[0], &dma_hw_desc->sg3l);
724 iowrite32(qword[4], &dma_hw_desc->sg3u);
725 iowrite32(qword[8], &dma_hw_desc->sg2l);
726 iowrite32(qword[12], &dma_hw_desc->sg2u);
727 break;
728 default:
729 BUG();
730 }
731}
732
733/**
734 * ppc440spe_xor_set_link - set link address in xor CB
735 */
736static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
737 struct ppc440spe_adma_desc_slot *next_desc)
738{
739 struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
740
741 if (unlikely(!next_desc || !(next_desc->phys))) {
742 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
743 __func__, next_desc,
744 next_desc ? next_desc->phys : 0);
745 BUG();
746 }
747
748 xor_hw_desc->cbs = 0;
749 xor_hw_desc->cblal = next_desc->phys;
750 xor_hw_desc->cblah = 0;
751 xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
752}
753
754/**
755 * ppc440spe_desc_set_link - set the address of descriptor following this
756 * descriptor in chain
757 */
758static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
759 struct ppc440spe_adma_desc_slot *prev_desc,
760 struct ppc440spe_adma_desc_slot *next_desc)
761{
762 unsigned long flags;
763 struct ppc440spe_adma_desc_slot *tail = next_desc;
764
765 if (unlikely(!prev_desc || !next_desc ||
766 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
767 /* If previous next is overwritten something is wrong.
768 * though we may refetch from append to initiate list
769 * processing; in this case - it's ok.
770 */
771 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
772 "prev->hw_next=0x%p\n", __func__, prev_desc,
773 next_desc, prev_desc ? prev_desc->hw_next : 0);
774 BUG();
775 }
776
777 local_irq_save(flags);
778
779 /* do s/w chaining both for DMA and XOR descriptors */
780 prev_desc->hw_next = next_desc;
781
782 switch (chan->device->id) {
783 case PPC440SPE_DMA0_ID:
784 case PPC440SPE_DMA1_ID:
785 break;
786 case PPC440SPE_XOR_ID:
787 /* bind descriptor to the chain */
788 while (tail->hw_next)
789 tail = tail->hw_next;
790 xor_last_linked = tail;
791
792 if (prev_desc == xor_last_submit)
793 /* do not link to the last submitted CB */
794 break;
795 ppc440spe_xor_set_link(prev_desc, next_desc);
796 break;
797 }
798
799 local_irq_restore(flags);
800}
801
802/**
803 * ppc440spe_desc_get_src_addr - extract the source address from the descriptor
804 */
805static u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc,
806 struct ppc440spe_adma_chan *chan, int src_idx)
807{
808 struct dma_cdb *dma_hw_desc;
809 struct xor_cb *xor_hw_desc;
810
811 switch (chan->device->id) {
812 case PPC440SPE_DMA0_ID:
813 case PPC440SPE_DMA1_ID:
814 dma_hw_desc = desc->hw_desc;
815 /* May have 0, 1, 2, or 3 sources */
816 switch (dma_hw_desc->opc) {
817 case DMA_CDB_OPC_NO_OP:
818 case DMA_CDB_OPC_DFILL128:
819 return 0;
820 case DMA_CDB_OPC_DCHECK128:
821 if (unlikely(src_idx)) {
822 printk(KERN_ERR "%s: try to get %d source for"
823 " DCHECK128\n", __func__, src_idx);
824 BUG();
825 }
826 return le32_to_cpu(dma_hw_desc->sg1l);
827 case DMA_CDB_OPC_MULTICAST:
828 case DMA_CDB_OPC_MV_SG1_SG2:
829 if (unlikely(src_idx > 2)) {
830 printk(KERN_ERR "%s: try to get %d source from"
831 " DMA descr\n", __func__, src_idx);
832 BUG();
833 }
834 if (src_idx) {
835 if (le32_to_cpu(dma_hw_desc->sg1u) &
836 DMA_CUED_XOR_WIN_MSK) {
837 u8 region;
838
839 if (src_idx == 1)
840 return le32_to_cpu(
841 dma_hw_desc->sg1l) +
842 desc->unmap_len;
843
844 region = (le32_to_cpu(
845 dma_hw_desc->sg1u)) >>
846 DMA_CUED_REGION_OFF;
847
848 region &= DMA_CUED_REGION_MSK;
849 switch (region) {
850 case DMA_RXOR123:
851 return le32_to_cpu(
852 dma_hw_desc->sg1l) +
853 (desc->unmap_len << 1);
854 case DMA_RXOR124:
855 return le32_to_cpu(
856 dma_hw_desc->sg1l) +
857 (desc->unmap_len * 3);
858 case DMA_RXOR125:
859 return le32_to_cpu(
860 dma_hw_desc->sg1l) +
861 (desc->unmap_len << 2);
862 default:
863 printk(KERN_ERR
864 "%s: try to"
865 " get src3 for region %02x"
866 "PPC440SPE_DESC_RXOR12?\n",
867 __func__, region);
868 BUG();
869 }
870 } else {
871 printk(KERN_ERR
872 "%s: try to get %d"
873 " source for non-cued descr\n",
874 __func__, src_idx);
875 BUG();
876 }
877 }
878 return le32_to_cpu(dma_hw_desc->sg1l);
879 default:
880 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
881 __func__, dma_hw_desc->opc);
882 BUG();
883 }
884 return le32_to_cpu(dma_hw_desc->sg1l);
885 case PPC440SPE_XOR_ID:
886 /* May have up to 16 sources */
887 xor_hw_desc = desc->hw_desc;
888 return xor_hw_desc->ops[src_idx].l;
889 }
890 return 0;
891}
892
893/**
894 * ppc440spe_desc_get_dest_addr - extract the destination address from the
895 * descriptor
896 */
897static u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc,
898 struct ppc440spe_adma_chan *chan, int idx)
899{
900 struct dma_cdb *dma_hw_desc;
901 struct xor_cb *xor_hw_desc;
902
903 switch (chan->device->id) {
904 case PPC440SPE_DMA0_ID:
905 case PPC440SPE_DMA1_ID:
906 dma_hw_desc = desc->hw_desc;
907
908 if (likely(!idx))
909 return le32_to_cpu(dma_hw_desc->sg2l);
910 return le32_to_cpu(dma_hw_desc->sg3l);
911 case PPC440SPE_XOR_ID:
912 xor_hw_desc = desc->hw_desc;
913 return xor_hw_desc->cbtal;
914 }
915 return 0;
916}
917
918/**
919 * ppc440spe_desc_get_src_num - extract the number of source addresses from
920 * the descriptor
921 */
922static u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc,
923 struct ppc440spe_adma_chan *chan)
924{
925 struct dma_cdb *dma_hw_desc;
926 struct xor_cb *xor_hw_desc;
927
928 switch (chan->device->id) {
929 case PPC440SPE_DMA0_ID:
930 case PPC440SPE_DMA1_ID:
931 dma_hw_desc = desc->hw_desc;
932
933 switch (dma_hw_desc->opc) {
934 case DMA_CDB_OPC_NO_OP:
935 case DMA_CDB_OPC_DFILL128:
936 return 0;
937 case DMA_CDB_OPC_DCHECK128:
938 return 1;
939 case DMA_CDB_OPC_MV_SG1_SG2:
940 case DMA_CDB_OPC_MULTICAST:
941 /*
942 * Only for RXOR operations we have more than
943 * one source
944 */
945 if (le32_to_cpu(dma_hw_desc->sg1u) &
946 DMA_CUED_XOR_WIN_MSK) {
947 /* RXOR op, there are 2 or 3 sources */
948 if (((le32_to_cpu(dma_hw_desc->sg1u) >>
949 DMA_CUED_REGION_OFF) &
950 DMA_CUED_REGION_MSK) == DMA_RXOR12) {
951 /* RXOR 1-2 */
952 return 2;
953 } else {
954 /* RXOR 1-2-3/1-2-4/1-2-5 */
955 return 3;
956 }
957 }
958 return 1;
959 default:
960 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
961 __func__, dma_hw_desc->opc);
962 BUG();
963 }
964 case PPC440SPE_XOR_ID:
965 /* up to 16 sources */
966 xor_hw_desc = desc->hw_desc;
967 return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK;
968 default:
969 BUG();
970 }
971 return 0;
972}
973
974/**
975 * ppc440spe_desc_get_dst_num - get the number of destination addresses in
976 * this descriptor
977 */
978static u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc,
979 struct ppc440spe_adma_chan *chan)
980{
981 struct dma_cdb *dma_hw_desc;
982
983 switch (chan->device->id) {
984 case PPC440SPE_DMA0_ID:
985 case PPC440SPE_DMA1_ID:
986 /* May be 1 or 2 destinations */
987 dma_hw_desc = desc->hw_desc;
988 switch (dma_hw_desc->opc) {
989 case DMA_CDB_OPC_NO_OP:
990 case DMA_CDB_OPC_DCHECK128:
991 return 0;
992 case DMA_CDB_OPC_MV_SG1_SG2:
993 case DMA_CDB_OPC_DFILL128:
994 return 1;
995 case DMA_CDB_OPC_MULTICAST:
996 if (desc->dst_cnt == 2)
997 return 2;
998 else
999 return 1;
1000 default:
1001 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
1002 __func__, dma_hw_desc->opc);
1003 BUG();
1004 }
1005 case PPC440SPE_XOR_ID:
1006 /* Always only 1 destination */
1007 return 1;
1008 default:
1009 BUG();
1010 }
1011 return 0;
1012}
1013
1014/**
1015 * ppc440spe_desc_get_link - get the address of the descriptor that
1016 * follows this one
1017 */
1018static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
1019 struct ppc440spe_adma_chan *chan)
1020{
1021 if (!desc->hw_next)
1022 return 0;
1023
1024 return desc->hw_next->phys;
1025}
1026
1027/**
1028 * ppc440spe_desc_is_aligned - check alignment
1029 */
1030static inline int ppc440spe_desc_is_aligned(
1031 struct ppc440spe_adma_desc_slot *desc, int num_slots)
1032{
1033 return (desc->idx & (num_slots - 1)) ? 0 : 1;
1034}
1035
1036/**
1037 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
1038 * XOR operation
1039 */
1040static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
1041 int *slots_per_op)
1042{
1043 int slot_cnt;
1044
1045 /* each XOR descriptor provides up to 16 source operands */
1046 slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
1047
1048 if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
1049 return slot_cnt;
1050
1051 printk(KERN_ERR "%s: len %d > max %d !!\n",
1052 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
1053 BUG();
1054 return slot_cnt;
1055}
1056
1057/**
1058 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
1059 * DMA2 PQ operation
1060 */
1061static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
1062 int src_cnt, size_t len)
1063{
1064 signed long long order = 0;
1065 int state = 0;
1066 int addr_count = 0;
1067 int i;
1068 for (i = 1; i < src_cnt; i++) {
1069 dma_addr_t cur_addr = srcs[i];
1070 dma_addr_t old_addr = srcs[i-1];
1071 switch (state) {
1072 case 0:
1073 if (cur_addr == old_addr + len) {
1074 /* direct RXOR */
1075 order = 1;
1076 state = 1;
1077 if (i == src_cnt-1)
1078 addr_count++;
1079 } else if (old_addr == cur_addr + len) {
1080 /* reverse RXOR */
1081 order = -1;
1082 state = 1;
1083 if (i == src_cnt-1)
1084 addr_count++;
1085 } else {
1086 state = 3;
1087 }
1088 break;
1089 case 1:
1090 if (i == src_cnt-2 || (order == -1
1091 && cur_addr != old_addr - len)) {
1092 order = 0;
1093 state = 0;
1094 addr_count++;
1095 } else if (cur_addr == old_addr + len*order) {
1096 state = 2;
1097 if (i == src_cnt-1)
1098 addr_count++;
1099 } else if (cur_addr == old_addr + 2*len) {
1100 state = 2;
1101 if (i == src_cnt-1)
1102 addr_count++;
1103 } else if (cur_addr == old_addr + 3*len) {
1104 state = 2;
1105 if (i == src_cnt-1)
1106 addr_count++;
1107 } else {
1108 order = 0;
1109 state = 0;
1110 addr_count++;
1111 }
1112 break;
1113 case 2:
1114 order = 0;
1115 state = 0;
1116 addr_count++;
1117 break;
1118 }
1119 if (state == 3)
1120 break;
1121 }
1122 if (src_cnt <= 1 || (state != 1 && state != 2)) {
1123 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
1124 __func__, src_cnt, state, addr_count, order);
1125 for (i = 0; i < src_cnt; i++)
1126 pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
1127 BUG();
1128 }
1129
1130 return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
1131}
1132
1133
1134/******************************************************************************
1135 * ADMA channel low-level routines
1136 ******************************************************************************/
1137
1138static u32
1139ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
1140static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
1141
1142/**
1143 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
1144 */
1145static void ppc440spe_adma_device_clear_eot_status(
1146 struct ppc440spe_adma_chan *chan)
1147{
1148 struct dma_regs *dma_reg;
1149 struct xor_regs *xor_reg;
1150 u8 *p = chan->device->dma_desc_pool_virt;
1151 struct dma_cdb *cdb;
1152 u32 rv, i;
1153
1154 switch (chan->device->id) {
1155 case PPC440SPE_DMA0_ID:
1156 case PPC440SPE_DMA1_ID:
1157 /* read FIFO to ack */
1158 dma_reg = chan->device->dma_reg;
1159 while ((rv = ioread32(&dma_reg->csfpl))) {
1160 i = rv & DMA_CDB_ADDR_MSK;
1161 cdb = (struct dma_cdb *)&p[i -
1162 (u32)chan->device->dma_desc_pool];
1163
1164 /* Clear opcode to ack. This is necessary for
1165 * ZeroSum operations only
1166 */
1167 cdb->opc = 0;
1168
1169 if (test_bit(PPC440SPE_RXOR_RUN,
1170 &ppc440spe_rxor_state)) {
1171 /* probably this is a completed RXOR op,
1172 * get pointer to CDB using the fact that
1173 * physical and virtual addresses of CDB
1174 * in pools have the same offsets
1175 */
1176 if (le32_to_cpu(cdb->sg1u) &
1177 DMA_CUED_XOR_BASE) {
1178 /* this is a RXOR */
1179 clear_bit(PPC440SPE_RXOR_RUN,
1180 &ppc440spe_rxor_state);
1181 }
1182 }
1183
1184 if (rv & DMA_CDB_STATUS_MSK) {
1185 /* ZeroSum check failed
1186 */
1187 struct ppc440spe_adma_desc_slot *iter;
1188 dma_addr_t phys = rv & ~DMA_CDB_MSK;
1189
1190 /*
1191 * Update the status of corresponding
1192 * descriptor.
1193 */
1194 list_for_each_entry(iter, &chan->chain,
1195 chain_node) {
1196 if (iter->phys == phys)
1197 break;
1198 }
1199 /*
1200 * if cannot find the corresponding
1201 * slot it's a bug
1202 */
1203 BUG_ON(&iter->chain_node == &chan->chain);
1204
1205 if (iter->xor_check_result) {
1206 if (test_bit(PPC440SPE_DESC_PCHECK,
1207 &iter->flags)) {
1208 *iter->xor_check_result |=
1209 SUM_CHECK_P_RESULT;
1210 } else
1211 if (test_bit(PPC440SPE_DESC_QCHECK,
1212 &iter->flags)) {
1213 *iter->xor_check_result |=
1214 SUM_CHECK_Q_RESULT;
1215 } else
1216 BUG();
1217 }
1218 }
1219 }
1220
1221 rv = ioread32(&dma_reg->dsts);
1222 if (rv) {
1223 pr_err("DMA%d err status: 0x%x\n",
1224 chan->device->id, rv);
1225 /* write back to clear */
1226 iowrite32(rv, &dma_reg->dsts);
1227 }
1228 break;
1229 case PPC440SPE_XOR_ID:
1230 /* reset status bits to ack */
1231 xor_reg = chan->device->xor_reg;
1232 rv = ioread32be(&xor_reg->sr);
1233 iowrite32be(rv, &xor_reg->sr);
1234
1235 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
1236 if (rv & XOR_IE_RPTIE_BIT) {
1237 /* Read PLB Timeout Error.
1238 * Try to resubmit the CB
1239 */
1240 u32 val = ioread32be(&xor_reg->ccbalr);
1241
1242 iowrite32be(val, &xor_reg->cblalr);
1243
1244 val = ioread32be(&xor_reg->crsr);
1245 iowrite32be(val | XOR_CRSR_XAE_BIT,
1246 &xor_reg->crsr);
1247 } else
1248 pr_err("XOR ERR 0x%x status\n", rv);
1249 break;
1250 }
1251
1252 /* if the XORcore is idle, but there are unprocessed CBs
1253 * then refetch the s/w chain here
1254 */
1255 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1256 do_xor_refetch)
1257 ppc440spe_chan_append(chan);
1258 break;
1259 }
1260}
1261
1262/**
1263 * ppc440spe_chan_is_busy - get the channel status
1264 */
1265static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1266{
1267 struct dma_regs *dma_reg;
1268 struct xor_regs *xor_reg;
1269 int busy = 0;
1270
1271 switch (chan->device->id) {
1272 case PPC440SPE_DMA0_ID:
1273 case PPC440SPE_DMA1_ID:
1274 dma_reg = chan->device->dma_reg;
1275 /* if command FIFO's head and tail pointers are equal and
1276 * status tail is the same as command, then channel is free
1277 */
1278 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1279 ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1280 busy = 1;
1281 break;
1282 case PPC440SPE_XOR_ID:
1283 /* use the special status bit for the XORcore
1284 */
1285 xor_reg = chan->device->xor_reg;
1286 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1287 break;
1288 }
1289
1290 return busy;
1291}
1292
1293/**
1294 * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain
1295 */
1296static void ppc440spe_chan_set_first_xor_descriptor(
1297 struct ppc440spe_adma_chan *chan,
1298 struct ppc440spe_adma_desc_slot *next_desc)
1299{
1300 struct xor_regs *xor_reg = chan->device->xor_reg;
1301
1302 if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1303 printk(KERN_INFO "%s: Warn: XORcore is running "
1304 "when try to set the first CDB!\n",
1305 __func__);
1306
1307 xor_last_submit = xor_last_linked = next_desc;
1308
1309 iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1310
1311 iowrite32be(next_desc->phys, &xor_reg->cblalr);
1312 iowrite32be(0, &xor_reg->cblahr);
1313 iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1314 &xor_reg->cbcr);
1315
1316 chan->hw_chain_inited = 1;
1317}
1318
1319/**
1320 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1321 * called with irqs disabled
1322 */
1323static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1324 struct ppc440spe_adma_desc_slot *desc)
1325{
1326 u32 pcdb;
1327 struct dma_regs *dma_reg = chan->device->dma_reg;
1328
1329 pcdb = desc->phys;
1330 if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1331 pcdb |= DMA_CDB_NO_INT;
1332
1333 chan_last_sub[chan->device->id] = desc;
1334
1335 ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1336
1337 iowrite32(pcdb, &dma_reg->cpfpl);
1338}
1339
1340/**
1341 * ppc440spe_chan_append - update the h/w chain in the channel
1342 */
1343static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1344{
1345 struct xor_regs *xor_reg;
1346 struct ppc440spe_adma_desc_slot *iter;
1347 struct xor_cb *xcb;
1348 u32 cur_desc;
1349 unsigned long flags;
1350
1351 local_irq_save(flags);
1352
1353 switch (chan->device->id) {
1354 case PPC440SPE_DMA0_ID:
1355 case PPC440SPE_DMA1_ID:
1356 cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1357
1358 if (likely(cur_desc)) {
1359 iter = chan_last_sub[chan->device->id];
1360 BUG_ON(!iter);
1361 } else {
1362 /* first peer */
1363 iter = chan_first_cdb[chan->device->id];
1364 BUG_ON(!iter);
1365 ppc440spe_dma_put_desc(chan, iter);
1366 chan->hw_chain_inited = 1;
1367 }
1368
1369 /* is there something new to append */
1370 if (!iter->hw_next)
1371 break;
1372
1373 /* flush descriptors from the s/w queue to fifo */
1374 list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1375 ppc440spe_dma_put_desc(chan, iter);
1376 if (!iter->hw_next)
1377 break;
1378 }
1379 break;
1380 case PPC440SPE_XOR_ID:
1381 /* update h/w links and refetch */
1382 if (!xor_last_submit->hw_next)
1383 break;
1384
1385 xor_reg = chan->device->xor_reg;
1386 /* the last linked CDB has to generate an interrupt
1387 * that we'd be able to append the next lists to h/w
1388 * regardless of the XOR engine state at the moment of
1389 * appending of these next lists
1390 */
1391 xcb = xor_last_linked->hw_desc;
1392 xcb->cbc |= XOR_CBCR_CBCE_BIT;
1393
1394 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1395 /* XORcore is idle. Refetch now */
1396 do_xor_refetch = 0;
1397 ppc440spe_xor_set_link(xor_last_submit,
1398 xor_last_submit->hw_next);
1399
1400 ADMA_LL_DBG(print_cb_list(chan,
1401 xor_last_submit->hw_next));
1402
1403 xor_last_submit = xor_last_linked;
1404 iowrite32be(ioread32be(&xor_reg->crsr) |
1405 XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1406 &xor_reg->crsr);
1407 } else {
1408 /* XORcore is running. Refetch later in the handler */
1409 do_xor_refetch = 1;
1410 }
1411
1412 break;
1413 }
1414
1415 local_irq_restore(flags);
1416}
1417
1418/**
1419 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1420 */
1421static u32
1422ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1423{
1424 struct dma_regs *dma_reg;
1425 struct xor_regs *xor_reg;
1426
1427 if (unlikely(!chan->hw_chain_inited))
1428 /* h/w descriptor chain is not initialized yet */
1429 return 0;
1430
1431 switch (chan->device->id) {
1432 case PPC440SPE_DMA0_ID:
1433 case PPC440SPE_DMA1_ID:
1434 dma_reg = chan->device->dma_reg;
1435 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1436 case PPC440SPE_XOR_ID:
1437 xor_reg = chan->device->xor_reg;
1438 return ioread32be(&xor_reg->ccbalr);
1439 }
1440 return 0;
1441}
1442
1443/**
1444 * ppc440spe_chan_run - enable the channel
1445 */
1446static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1447{
1448 struct xor_regs *xor_reg;
1449
1450 switch (chan->device->id) {
1451 case PPC440SPE_DMA0_ID:
1452 case PPC440SPE_DMA1_ID:
1453 /* DMAs are always enabled, do nothing */
1454 break;
1455 case PPC440SPE_XOR_ID:
1456 /* drain write buffer */
1457 xor_reg = chan->device->xor_reg;
1458
1459 /* fetch descriptor pointed to in <link> */
1460 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1461 &xor_reg->crsr);
1462 break;
1463 }
1464}
1465
1466/******************************************************************************
1467 * ADMA device level
1468 ******************************************************************************/
1469
1470static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1471static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1472
1473static dma_cookie_t
1474ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1475
1476static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1477 dma_addr_t addr, int index);
1478static void
1479ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1480 dma_addr_t addr, int index);
1481
1482static void
1483ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1484 dma_addr_t *paddr, unsigned long flags);
1485static void
1486ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1487 dma_addr_t addr, int index);
1488static void
1489ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1490 unsigned char mult, int index, int dst_pos);
1491static void
1492ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1493 dma_addr_t paddr, dma_addr_t qaddr);
1494
1495static struct page *ppc440spe_rxor_srcs[32];
1496
1497/**
1498 * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1499 */
1500static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1501{
1502 int i, order = 0, state = 0;
1503 int idx = 0;
1504
1505 if (unlikely(!(src_cnt > 1)))
1506 return 0;
1507
1508 BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1509
1510 /* Skip holes in the source list before checking */
1511 for (i = 0; i < src_cnt; i++) {
1512 if (!srcs[i])
1513 continue;
1514 ppc440spe_rxor_srcs[idx++] = srcs[i];
1515 }
1516 src_cnt = idx;
1517
1518 for (i = 1; i < src_cnt; i++) {
1519 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1520 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1521
1522 switch (state) {
1523 case 0:
1524 if (cur_addr == old_addr + len) {
1525 /* direct RXOR */
1526 order = 1;
1527 state = 1;
1528 } else if (old_addr == cur_addr + len) {
1529 /* reverse RXOR */
1530 order = -1;
1531 state = 1;
1532 } else
1533 goto out;
1534 break;
1535 case 1:
1536 if ((i == src_cnt - 2) ||
1537 (order == -1 && cur_addr != old_addr - len)) {
1538 order = 0;
1539 state = 0;
1540 } else if ((cur_addr == old_addr + len * order) ||
1541 (cur_addr == old_addr + 2 * len) ||
1542 (cur_addr == old_addr + 3 * len)) {
1543 state = 2;
1544 } else {
1545 order = 0;
1546 state = 0;
1547 }
1548 break;
1549 case 2:
1550 order = 0;
1551 state = 0;
1552 break;
1553 }
1554 }
1555
1556out:
1557 if (state == 1 || state == 2)
1558 return 1;
1559
1560 return 0;
1561}
1562
1563/**
1564 * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1565 * the operation given on this channel. It's assumed that 'chan' is
1566 * capable to process 'cap' type of operation.
1567 * @chan: channel to use
1568 * @cap: type of transaction
1569 * @dst_lst: array of destination pointers
1570 * @dst_cnt: number of destination operands
1571 * @src_lst: array of source pointers
1572 * @src_cnt: number of source operands
1573 * @src_sz: size of each source operand
1574 */
1575static int ppc440spe_adma_estimate(struct dma_chan *chan,
1576 enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1577 struct page **src_lst, int src_cnt, size_t src_sz)
1578{
1579 int ef = 1;
1580
1581 if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1582 /* If RAID-6 capabilities were not activated don't try
1583 * to use them
1584 */
1585 if (unlikely(!ppc440spe_r6_enabled))
1586 return -1;
1587 }
1588 /* In the current implementation of ppc440spe ADMA driver it
1589 * makes sense to pick out only pq case, because it may be
1590 * processed:
1591 * (1) either using Biskup method on DMA2;
1592 * (2) or on DMA0/1.
1593 * Thus we give a favour to (1) if the sources are suitable;
1594 * else let it be processed on one of the DMA0/1 engines.
1595 * In the sum_product case where destination is also the
1596 * source process it on DMA0/1 only.
1597 */
1598 if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1599
1600 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1601 ef = 0; /* sum_product case, process on DMA0/1 */
1602 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1603 ef = 3; /* override (DMA0/1 + idle) */
1604 else
1605 ef = 0; /* can't process on DMA2 if !rxor */
1606 }
1607
1608 /* channel idleness increases the priority */
1609 if (likely(ef) &&
1610 !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1611 ef++;
1612
1613 return ef;
1614}
1615
1616struct dma_chan *
1617ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1618 struct page **dst_lst, int dst_cnt, struct page **src_lst,
1619 int src_cnt, size_t src_sz)
1620{
1621 struct dma_chan *best_chan = NULL;
1622 struct ppc_dma_chan_ref *ref;
1623 int best_rank = -1;
1624
1625 if (unlikely(!src_sz))
1626 return NULL;
1627 if (src_sz > PAGE_SIZE) {
1628 /*
1629 * should a user of the api ever pass > PAGE_SIZE requests
1630 * we sort out cases where temporary page-sized buffers
1631 * are used.
1632 */
1633 switch (cap) {
1634 case DMA_PQ:
1635 if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1636 return NULL;
1637 if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1638 return NULL;
1639 break;
1640 case DMA_PQ_VAL:
1641 case DMA_XOR_VAL:
1642 return NULL;
1643 default:
1644 break;
1645 }
1646 }
1647
1648 list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1649 if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1650 int rank;
1651
1652 rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1653 dst_cnt, src_lst, src_cnt, src_sz);
1654 if (rank > best_rank) {
1655 best_rank = rank;
1656 best_chan = ref->chan;
1657 }
1658 }
1659 }
1660
1661 return best_chan;
1662}
1663EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1664
1665/**
1666 * ppc440spe_get_group_entry - get group entry with index idx
1667 * @tdesc: is the last allocated slot in the group.
1668 */
1669static struct ppc440spe_adma_desc_slot *
1670ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1671{
1672 struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1673 int i = 0;
1674
1675 if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1676 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1677 __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1678 BUG();
1679 }
1680
1681 list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1682 if (i++ == entry_idx)
1683 break;
1684 }
1685 return iter;
1686}
1687
1688/**
1689 * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1690 * @slot: Slot to free
1691 * Caller must hold &ppc440spe_chan->lock while calling this function
1692 */
1693static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1694 struct ppc440spe_adma_chan *chan)
1695{
1696 int stride = slot->slots_per_op;
1697
1698 while (stride--) {
1699 slot->slots_per_op = 0;
1700 slot = list_entry(slot->slot_node.next,
1701 struct ppc440spe_adma_desc_slot,
1702 slot_node);
1703 }
1704}
1705
1706static void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan,
1707 struct ppc440spe_adma_desc_slot *desc)
1708{
1709 u32 src_cnt, dst_cnt;
1710 dma_addr_t addr;
1711
1712 /*
1713 * get the number of sources & destination
1714 * included in this descriptor and unmap
1715 * them all
1716 */
1717 src_cnt = ppc440spe_desc_get_src_num(desc, chan);
1718 dst_cnt = ppc440spe_desc_get_dst_num(desc, chan);
1719
1720 /* unmap destinations */
1721 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1722 while (dst_cnt--) {
1723 addr = ppc440spe_desc_get_dest_addr(
1724 desc, chan, dst_cnt);
1725 dma_unmap_page(chan->device->dev,
1726 addr, desc->unmap_len,
1727 DMA_FROM_DEVICE);
1728 }
1729 }
1730
1731 /* unmap sources */
1732 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1733 while (src_cnt--) {
1734 addr = ppc440spe_desc_get_src_addr(
1735 desc, chan, src_cnt);
1736 dma_unmap_page(chan->device->dev,
1737 addr, desc->unmap_len,
1738 DMA_TO_DEVICE);
1739 }
1740 }
1741}
1742
1743/**
1744 * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1745 * upon completion
1746 */
1747static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1748 struct ppc440spe_adma_desc_slot *desc,
1749 struct ppc440spe_adma_chan *chan,
1750 dma_cookie_t cookie)
1751{
1752 int i;
1753
1754 BUG_ON(desc->async_tx.cookie < 0);
1755 if (desc->async_tx.cookie > 0) {
1756 cookie = desc->async_tx.cookie;
1757 desc->async_tx.cookie = 0;
1758
1759 /* call the callback (must not sleep or submit new
1760 * operations to this channel)
1761 */
1762 if (desc->async_tx.callback)
1763 desc->async_tx.callback(
1764 desc->async_tx.callback_param);
1765
1766 /* unmap dma addresses
1767 * (unmap_single vs unmap_page?)
1768 *
1769 * actually, ppc's dma_unmap_page() functions are empty, so
1770 * the following code is just for the sake of completeness
1771 */
1772 if (chan && chan->needs_unmap && desc->group_head &&
1773 desc->unmap_len) {
1774 struct ppc440spe_adma_desc_slot *unmap =
1775 desc->group_head;
1776 /* assume 1 slot per op always */
1777 u32 slot_count = unmap->slot_cnt;
1778
1779 /* Run through the group list and unmap addresses */
1780 for (i = 0; i < slot_count; i++) {
1781 BUG_ON(!unmap);
1782 ppc440spe_adma_unmap(chan, unmap);
1783 unmap = unmap->hw_next;
1784 }
1785 }
1786 }
1787
1788 /* run dependent operations */
1789 dma_run_dependencies(&desc->async_tx);
1790
1791 return cookie;
1792}
1793
1794/**
1795 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1796 */
1797static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1798 struct ppc440spe_adma_chan *chan)
1799{
1800 /* the client is allowed to attach dependent operations
1801 * until 'ack' is set
1802 */
1803 if (!async_tx_test_ack(&desc->async_tx))
1804 return 0;
1805
1806 /* leave the last descriptor in the chain
1807 * so we can append to it
1808 */
1809 if (list_is_last(&desc->chain_node, &chan->chain) ||
1810 desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1811 return 1;
1812
1813 if (chan->device->id != PPC440SPE_XOR_ID) {
1814 /* our DMA interrupt handler clears opc field of
1815 * each processed descriptor. For all types of
1816 * operations except for ZeroSum we do not actually
1817 * need ack from the interrupt handler. ZeroSum is a
1818 * special case since the result of this operation
1819 * is available from the handler only, so if we see
1820 * such type of descriptor (which is unprocessed yet)
1821 * then leave it in chain.
1822 */
1823 struct dma_cdb *cdb = desc->hw_desc;
1824 if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1825 return 1;
1826 }
1827
1828 dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1829 desc->phys, desc->idx, desc->slots_per_op);
1830
1831 list_del(&desc->chain_node);
1832 ppc440spe_adma_free_slots(desc, chan);
1833 return 0;
1834}
1835
1836/**
1837 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1838 * which runs through the channel CDBs list until reach the descriptor
1839 * currently processed. When routine determines that all CDBs of group
1840 * are completed then corresponding callbacks (if any) are called and slots
1841 * are freed.
1842 */
1843static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1844{
1845 struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1846 dma_cookie_t cookie = 0;
1847 u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1848 int busy = ppc440spe_chan_is_busy(chan);
1849 int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1850
1851 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1852 chan->device->id, __func__);
1853
1854 if (!current_desc) {
1855 /* There were no transactions yet, so
1856 * nothing to clean
1857 */
1858 return;
1859 }
1860
1861 /* free completed slots from the chain starting with
1862 * the oldest descriptor
1863 */
1864 list_for_each_entry_safe(iter, _iter, &chan->chain,
1865 chain_node) {
1866 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1867 "busy: %d this_desc: %#llx next_desc: %#x "
1868 "cur: %#x ack: %d\n",
1869 iter->async_tx.cookie, iter->idx, busy, iter->phys,
1870 ppc440spe_desc_get_link(iter, chan), current_desc,
1871 async_tx_test_ack(&iter->async_tx));
1872 prefetch(_iter);
1873 prefetch(&_iter->async_tx);
1874
1875 /* do not advance past the current descriptor loaded into the
1876 * hardware channel,subsequent descriptors are either in process
1877 * or have not been submitted
1878 */
1879 if (seen_current)
1880 break;
1881
1882 /* stop the search if we reach the current descriptor and the
1883 * channel is busy, or if it appears that the current descriptor
1884 * needs to be re-read (i.e. has been appended to)
1885 */
1886 if (iter->phys == current_desc) {
1887 BUG_ON(seen_current++);
1888 if (busy || ppc440spe_desc_get_link(iter, chan)) {
1889 /* not all descriptors of the group have
1890 * been completed; exit.
1891 */
1892 break;
1893 }
1894 }
1895
1896 /* detect the start of a group transaction */
1897 if (!slot_cnt && !slots_per_op) {
1898 slot_cnt = iter->slot_cnt;
1899 slots_per_op = iter->slots_per_op;
1900 if (slot_cnt <= slots_per_op) {
1901 slot_cnt = 0;
1902 slots_per_op = 0;
1903 }
1904 }
1905
1906 if (slot_cnt) {
1907 if (!group_start)
1908 group_start = iter;
1909 slot_cnt -= slots_per_op;
1910 }
1911
1912 /* all the members of a group are complete */
1913 if (slots_per_op != 0 && slot_cnt == 0) {
1914 struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1915 int end_of_chain = 0;
1916
1917 /* clean up the group */
1918 slot_cnt = group_start->slot_cnt;
1919 grp_iter = group_start;
1920 list_for_each_entry_safe_from(grp_iter, _grp_iter,
1921 &chan->chain, chain_node) {
1922
1923 cookie = ppc440spe_adma_run_tx_complete_actions(
1924 grp_iter, chan, cookie);
1925
1926 slot_cnt -= slots_per_op;
1927 end_of_chain = ppc440spe_adma_clean_slot(
1928 grp_iter, chan);
1929 if (end_of_chain && slot_cnt) {
1930 /* Should wait for ZeroSum completion */
1931 if (cookie > 0)
1932 chan->completed_cookie = cookie;
1933 return;
1934 }
1935
1936 if (slot_cnt == 0 || end_of_chain)
1937 break;
1938 }
1939
1940 /* the group should be complete at this point */
1941 BUG_ON(slot_cnt);
1942
1943 slots_per_op = 0;
1944 group_start = NULL;
1945 if (end_of_chain)
1946 break;
1947 else
1948 continue;
1949 } else if (slots_per_op) /* wait for group completion */
1950 continue;
1951
1952 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1953 cookie);
1954
1955 if (ppc440spe_adma_clean_slot(iter, chan))
1956 break;
1957 }
1958
1959 BUG_ON(!seen_current);
1960
1961 if (cookie > 0) {
1962 chan->completed_cookie = cookie;
1963 pr_debug("\tcompleted cookie %d\n", cookie);
1964 }
1965
1966}
1967
1968/**
1969 * ppc440spe_adma_tasklet - clean up watch-dog initiator
1970 */
1971static void ppc440spe_adma_tasklet(unsigned long data)
1972{
1973 struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data;
1974
1975 spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1976 __ppc440spe_adma_slot_cleanup(chan);
1977 spin_unlock(&chan->lock);
1978}
1979
1980/**
1981 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1982 */
1983static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1984{
1985 spin_lock_bh(&chan->lock);
1986 __ppc440spe_adma_slot_cleanup(chan);
1987 spin_unlock_bh(&chan->lock);
1988}
1989
1990/**
1991 * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1992 */
1993static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1994 struct ppc440spe_adma_chan *chan, int num_slots,
1995 int slots_per_op)
1996{
1997 struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1998 struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1999 struct list_head chain = LIST_HEAD_INIT(chain);
2000 int slots_found, retry = 0;
2001
2002
2003 BUG_ON(!num_slots || !slots_per_op);
2004 /* start search from the last allocated descrtiptor
2005 * if a contiguous allocation can not be found start searching
2006 * from the beginning of the list
2007 */
2008retry:
2009 slots_found = 0;
2010 if (retry == 0)
2011 iter = chan->last_used;
2012 else
2013 iter = list_entry(&chan->all_slots,
2014 struct ppc440spe_adma_desc_slot,
2015 slot_node);
2016 list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
2017 slot_node) {
2018 prefetch(_iter);
2019 prefetch(&_iter->async_tx);
2020 if (iter->slots_per_op) {
2021 slots_found = 0;
2022 continue;
2023 }
2024
2025 /* start the allocation if the slot is correctly aligned */
2026 if (!slots_found++)
2027 alloc_start = iter;
2028
2029 if (slots_found == num_slots) {
2030 struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
2031 struct ppc440spe_adma_desc_slot *last_used = NULL;
2032
2033 iter = alloc_start;
2034 while (num_slots) {
2035 int i;
2036 /* pre-ack all but the last descriptor */
2037 if (num_slots != slots_per_op)
2038 async_tx_ack(&iter->async_tx);
2039
2040 list_add_tail(&iter->chain_node, &chain);
2041 alloc_tail = iter;
2042 iter->async_tx.cookie = 0;
2043 iter->hw_next = NULL;
2044 iter->flags = 0;
2045 iter->slot_cnt = num_slots;
2046 iter->xor_check_result = NULL;
2047 for (i = 0; i < slots_per_op; i++) {
2048 iter->slots_per_op = slots_per_op - i;
2049 last_used = iter;
2050 iter = list_entry(iter->slot_node.next,
2051 struct ppc440spe_adma_desc_slot,
2052 slot_node);
2053 }
2054 num_slots -= slots_per_op;
2055 }
2056 alloc_tail->group_head = alloc_start;
2057 alloc_tail->async_tx.cookie = -EBUSY;
2058 list_splice(&chain, &alloc_tail->group_list);
2059 chan->last_used = last_used;
2060 return alloc_tail;
2061 }
2062 }
2063 if (!retry++)
2064 goto retry;
2065
2066 /* try to free some slots if the allocation fails */
2067 tasklet_schedule(&chan->irq_tasklet);
2068 return NULL;
2069}
2070
2071/**
2072 * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots
2073 */
2074static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
2075{
2076 struct ppc440spe_adma_chan *ppc440spe_chan;
2077 struct ppc440spe_adma_desc_slot *slot = NULL;
2078 char *hw_desc;
2079 int i, db_sz;
2080 int init;
2081
2082 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2083 init = ppc440spe_chan->slots_allocated ? 0 : 1;
2084 chan->chan_id = ppc440spe_chan->device->id;
2085
2086 /* Allocate descriptor slots */
2087 i = ppc440spe_chan->slots_allocated;
2088 if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
2089 db_sz = sizeof(struct dma_cdb);
2090 else
2091 db_sz = sizeof(struct xor_cb);
2092
2093 for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
2094 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
2095 GFP_KERNEL);
2096 if (!slot) {
2097 printk(KERN_INFO "SPE ADMA Channel only initialized"
2098 " %d descriptor slots", i--);
2099 break;
2100 }
2101
2102 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
2103 slot->hw_desc = (void *) &hw_desc[i * db_sz];
2104 dma_async_tx_descriptor_init(&slot->async_tx, chan);
2105 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
2106 INIT_LIST_HEAD(&slot->chain_node);
2107 INIT_LIST_HEAD(&slot->slot_node);
2108 INIT_LIST_HEAD(&slot->group_list);
2109 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
2110 slot->idx = i;
2111
2112 spin_lock_bh(&ppc440spe_chan->lock);
2113 ppc440spe_chan->slots_allocated++;
2114 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
2115 spin_unlock_bh(&ppc440spe_chan->lock);
2116 }
2117
2118 if (i && !ppc440spe_chan->last_used) {
2119 ppc440spe_chan->last_used =
2120 list_entry(ppc440spe_chan->all_slots.next,
2121 struct ppc440spe_adma_desc_slot,
2122 slot_node);
2123 }
2124
2125 dev_dbg(ppc440spe_chan->device->common.dev,
2126 "ppc440spe adma%d: allocated %d descriptor slots\n",
2127 ppc440spe_chan->device->id, i);
2128
2129 /* initialize the channel and the chain with a null operation */
2130 if (init) {
2131 switch (ppc440spe_chan->device->id) {
2132 case PPC440SPE_DMA0_ID:
2133 case PPC440SPE_DMA1_ID:
2134 ppc440spe_chan->hw_chain_inited = 0;
2135 /* Use WXOR for self-testing */
2136 if (!ppc440spe_r6_tchan)
2137 ppc440spe_r6_tchan = ppc440spe_chan;
2138 break;
2139 case PPC440SPE_XOR_ID:
2140 ppc440spe_chan_start_null_xor(ppc440spe_chan);
2141 break;
2142 default:
2143 BUG();
2144 }
2145 ppc440spe_chan->needs_unmap = 1;
2146 }
2147
2148 return (i > 0) ? i : -ENOMEM;
2149}
2150
2151/**
2152 * ppc440spe_desc_assign_cookie - assign a cookie
2153 */
2154static dma_cookie_t ppc440spe_desc_assign_cookie(
2155 struct ppc440spe_adma_chan *chan,
2156 struct ppc440spe_adma_desc_slot *desc)
2157{
2158 dma_cookie_t cookie = chan->common.cookie;
2159
2160 cookie++;
2161 if (cookie < 0)
2162 cookie = 1;
2163 chan->common.cookie = desc->async_tx.cookie = cookie;
2164 return cookie;
2165}
2166
2167/**
2168 * ppc440spe_rxor_set_region_data -
2169 */
2170static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
2171 u8 xor_arg_no, u32 mask)
2172{
2173 struct xor_cb *xcb = desc->hw_desc;
2174
2175 xcb->ops[xor_arg_no].h |= mask;
2176}
2177
2178/**
2179 * ppc440spe_rxor_set_src -
2180 */
2181static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
2182 u8 xor_arg_no, dma_addr_t addr)
2183{
2184 struct xor_cb *xcb = desc->hw_desc;
2185
2186 xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
2187 xcb->ops[xor_arg_no].l = addr;
2188}
2189
2190/**
2191 * ppc440spe_rxor_set_mult -
2192 */
2193static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
2194 u8 xor_arg_no, u8 idx, u8 mult)
2195{
2196 struct xor_cb *xcb = desc->hw_desc;
2197
2198 xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
2199}
2200
2201/**
2202 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
2203 * has been achieved
2204 */
2205static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
2206{
2207 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
2208 chan->device->id, chan->pending);
2209
2210 if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
2211 chan->pending = 0;
2212 ppc440spe_chan_append(chan);
2213 }
2214}
2215
2216/**
2217 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
2218 * (it's not necessary that descriptors will be submitted to the h/w
2219 * chains too right now)
2220 */
2221static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
2222{
2223 struct ppc440spe_adma_desc_slot *sw_desc;
2224 struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
2225 struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
2226 int slot_cnt;
2227 int slots_per_op;
2228 dma_cookie_t cookie;
2229
2230 sw_desc = tx_to_ppc440spe_adma_slot(tx);
2231
2232 group_start = sw_desc->group_head;
2233 slot_cnt = group_start->slot_cnt;
2234 slots_per_op = group_start->slots_per_op;
2235
2236 spin_lock_bh(&chan->lock);
2237
2238 cookie = ppc440spe_desc_assign_cookie(chan, sw_desc);
2239
2240 if (unlikely(list_empty(&chan->chain))) {
2241 /* first peer */
2242 list_splice_init(&sw_desc->group_list, &chan->chain);
2243 chan_first_cdb[chan->device->id] = group_start;
2244 } else {
2245 /* isn't first peer, bind CDBs to chain */
2246 old_chain_tail = list_entry(chan->chain.prev,
2247 struct ppc440spe_adma_desc_slot,
2248 chain_node);
2249 list_splice_init(&sw_desc->group_list,
2250 &old_chain_tail->chain_node);
2251 /* fix up the hardware chain */
2252 ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
2253 }
2254
2255 /* increment the pending count by the number of operations */
2256 chan->pending += slot_cnt / slots_per_op;
2257 ppc440spe_adma_check_threshold(chan);
2258 spin_unlock_bh(&chan->lock);
2259
2260 dev_dbg(chan->device->common.dev,
2261 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
2262 chan->device->id, __func__,
2263 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
2264
2265 return cookie;
2266}
2267
2268/**
2269 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
2270 */
2271static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
2272 struct dma_chan *chan, unsigned long flags)
2273{
2274 struct ppc440spe_adma_chan *ppc440spe_chan;
2275 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2276 int slot_cnt, slots_per_op;
2277
2278 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2279
2280 dev_dbg(ppc440spe_chan->device->common.dev,
2281 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
2282 __func__);
2283
2284 spin_lock_bh(&ppc440spe_chan->lock);
2285 slot_cnt = slots_per_op = 1;
2286 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2287 slots_per_op);
2288 if (sw_desc) {
2289 group_start = sw_desc->group_head;
2290 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
2291 group_start->unmap_len = 0;
2292 sw_desc->async_tx.flags = flags;
2293 }
2294 spin_unlock_bh(&ppc440spe_chan->lock);
2295
2296 return sw_desc ? &sw_desc->async_tx : NULL;
2297}
2298
2299/**
2300 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
2301 */
2302static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
2303 struct dma_chan *chan, dma_addr_t dma_dest,
2304 dma_addr_t dma_src, size_t len, unsigned long flags)
2305{
2306 struct ppc440spe_adma_chan *ppc440spe_chan;
2307 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2308 int slot_cnt, slots_per_op;
2309
2310 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2311
2312 if (unlikely(!len))
2313 return NULL;
2314
2315 BUG_ON(unlikely(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT));
2316
2317 spin_lock_bh(&ppc440spe_chan->lock);
2318
2319 dev_dbg(ppc440spe_chan->device->common.dev,
2320 "ppc440spe adma%d: %s len: %u int_en %d\n",
2321 ppc440spe_chan->device->id, __func__, len,
2322 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2323 slot_cnt = slots_per_op = 1;
2324 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2325 slots_per_op);
2326 if (sw_desc) {
2327 group_start = sw_desc->group_head;
2328 ppc440spe_desc_init_memcpy(group_start, flags);
2329 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2330 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2331 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2332 sw_desc->unmap_len = len;
2333 sw_desc->async_tx.flags = flags;
2334 }
2335 spin_unlock_bh(&ppc440spe_chan->lock);
2336
2337 return sw_desc ? &sw_desc->async_tx : NULL;
2338}
2339
2340/**
2341 * ppc440spe_adma_prep_dma_memset - prepare CDB for a MEMSET operation
2342 */
2343static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memset(
2344 struct dma_chan *chan, dma_addr_t dma_dest, int value,
2345 size_t len, unsigned long flags)
2346{
2347 struct ppc440spe_adma_chan *ppc440spe_chan;
2348 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2349 int slot_cnt, slots_per_op;
2350
2351 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2352
2353 if (unlikely(!len))
2354 return NULL;
2355
2356 BUG_ON(unlikely(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT));
2357
2358 spin_lock_bh(&ppc440spe_chan->lock);
2359
2360 dev_dbg(ppc440spe_chan->device->common.dev,
2361 "ppc440spe adma%d: %s cal: %u len: %u int_en %d\n",
2362 ppc440spe_chan->device->id, __func__, value, len,
2363 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2364
2365 slot_cnt = slots_per_op = 1;
2366 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2367 slots_per_op);
2368 if (sw_desc) {
2369 group_start = sw_desc->group_head;
2370 ppc440spe_desc_init_memset(group_start, value, flags);
2371 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2372 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2373 sw_desc->unmap_len = len;
2374 sw_desc->async_tx.flags = flags;
2375 }
2376 spin_unlock_bh(&ppc440spe_chan->lock);
2377
2378 return sw_desc ? &sw_desc->async_tx : NULL;
2379}
2380
2381/**
2382 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2383 */
2384static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2385 struct dma_chan *chan, dma_addr_t dma_dest,
2386 dma_addr_t *dma_src, u32 src_cnt, size_t len,
2387 unsigned long flags)
2388{
2389 struct ppc440spe_adma_chan *ppc440spe_chan;
2390 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2391 int slot_cnt, slots_per_op;
2392
2393 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2394
2395 ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2396 dma_dest, dma_src, src_cnt));
2397 if (unlikely(!len))
2398 return NULL;
2399 BUG_ON(unlikely(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT));
2400
2401 dev_dbg(ppc440spe_chan->device->common.dev,
2402 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2403 ppc440spe_chan->device->id, __func__, src_cnt, len,
2404 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2405
2406 spin_lock_bh(&ppc440spe_chan->lock);
2407 slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2408 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2409 slots_per_op);
2410 if (sw_desc) {
2411 group_start = sw_desc->group_head;
2412 ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2413 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2414 while (src_cnt--)
2415 ppc440spe_adma_memcpy_xor_set_src(group_start,
2416 dma_src[src_cnt], src_cnt);
2417 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2418 sw_desc->unmap_len = len;
2419 sw_desc->async_tx.flags = flags;
2420 }
2421 spin_unlock_bh(&ppc440spe_chan->lock);
2422
2423 return sw_desc ? &sw_desc->async_tx : NULL;
2424}
2425
2426static inline void
2427ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2428 int src_cnt);
2429static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2430
2431/**
2432 * ppc440spe_adma_init_dma2rxor_slot -
2433 */
2434static void ppc440spe_adma_init_dma2rxor_slot(
2435 struct ppc440spe_adma_desc_slot *desc,
2436 dma_addr_t *src, int src_cnt)
2437{
2438 int i;
2439
2440 /* initialize CDB */
2441 for (i = 0; i < src_cnt; i++) {
2442 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2443 desc->src_cnt, (u32)src[i]);
2444 }
2445}
2446
2447/**
2448 * ppc440spe_dma01_prep_mult -
2449 * for Q operation where destination is also the source
2450 */
2451static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2452 struct ppc440spe_adma_chan *ppc440spe_chan,
2453 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2454 const unsigned char *scf, size_t len, unsigned long flags)
2455{
2456 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2457 unsigned long op = 0;
2458 int slot_cnt;
2459
2460 set_bit(PPC440SPE_DESC_WXOR, &op);
2461 slot_cnt = 2;
2462
2463 spin_lock_bh(&ppc440spe_chan->lock);
2464
2465 /* use WXOR, each descriptor occupies one slot */
2466 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2467 if (sw_desc) {
2468 struct ppc440spe_adma_chan *chan;
2469 struct ppc440spe_adma_desc_slot *iter;
2470 struct dma_cdb *hw_desc;
2471
2472 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2473 set_bits(op, &sw_desc->flags);
2474 sw_desc->src_cnt = src_cnt;
2475 sw_desc->dst_cnt = dst_cnt;
2476 /* First descriptor, zero data in the destination and copy it
2477 * to q page using MULTICAST transfer.
2478 */
2479 iter = list_first_entry(&sw_desc->group_list,
2480 struct ppc440spe_adma_desc_slot,
2481 chain_node);
2482 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2483 /* set 'next' pointer */
2484 iter->hw_next = list_entry(iter->chain_node.next,
2485 struct ppc440spe_adma_desc_slot,
2486 chain_node);
2487 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2488 hw_desc = iter->hw_desc;
2489 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2490
2491 ppc440spe_desc_set_dest_addr(iter, chan,
2492 DMA_CUED_XOR_BASE, dst[0], 0);
2493 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2494 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2495 src[0]);
2496 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2497 iter->unmap_len = len;
2498
2499 /*
2500 * Second descriptor, multiply data from the q page
2501 * and store the result in real destination.
2502 */
2503 iter = list_first_entry(&iter->chain_node,
2504 struct ppc440spe_adma_desc_slot,
2505 chain_node);
2506 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2507 iter->hw_next = NULL;
2508 if (flags & DMA_PREP_INTERRUPT)
2509 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2510 else
2511 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2512
2513 hw_desc = iter->hw_desc;
2514 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2515 ppc440spe_desc_set_src_addr(iter, chan, 0,
2516 DMA_CUED_XOR_HB, dst[1]);
2517 ppc440spe_desc_set_dest_addr(iter, chan,
2518 DMA_CUED_XOR_BASE, dst[0], 0);
2519
2520 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2521 DMA_CDB_SG_DST1, scf[0]);
2522 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2523 iter->unmap_len = len;
2524 sw_desc->async_tx.flags = flags;
2525 }
2526
2527 spin_unlock_bh(&ppc440spe_chan->lock);
2528
2529 return sw_desc;
2530}
2531
2532/**
2533 * ppc440spe_dma01_prep_sum_product -
2534 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2535 * the source.
2536 */
2537static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2538 struct ppc440spe_adma_chan *ppc440spe_chan,
2539 dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2540 const unsigned char *scf, size_t len, unsigned long flags)
2541{
2542 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2543 unsigned long op = 0;
2544 int slot_cnt;
2545
2546 set_bit(PPC440SPE_DESC_WXOR, &op);
2547 slot_cnt = 3;
2548
2549 spin_lock_bh(&ppc440spe_chan->lock);
2550
2551 /* WXOR, each descriptor occupies one slot */
2552 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2553 if (sw_desc) {
2554 struct ppc440spe_adma_chan *chan;
2555 struct ppc440spe_adma_desc_slot *iter;
2556 struct dma_cdb *hw_desc;
2557
2558 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2559 set_bits(op, &sw_desc->flags);
2560 sw_desc->src_cnt = src_cnt;
2561 sw_desc->dst_cnt = 1;
2562 /* 1st descriptor, src[1] data to q page and zero destination */
2563 iter = list_first_entry(&sw_desc->group_list,
2564 struct ppc440spe_adma_desc_slot,
2565 chain_node);
2566 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2567 iter->hw_next = list_entry(iter->chain_node.next,
2568 struct ppc440spe_adma_desc_slot,
2569 chain_node);
2570 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2571 hw_desc = iter->hw_desc;
2572 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2573
2574 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2575 *dst, 0);
2576 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2577 ppc440spe_chan->qdest, 1);
2578 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2579 src[1]);
2580 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2581 iter->unmap_len = len;
2582
2583 /* 2nd descriptor, multiply src[1] data and store the
2584 * result in destination */
2585 iter = list_first_entry(&iter->chain_node,
2586 struct ppc440spe_adma_desc_slot,
2587 chain_node);
2588 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2589 /* set 'next' pointer */
2590 iter->hw_next = list_entry(iter->chain_node.next,
2591 struct ppc440spe_adma_desc_slot,
2592 chain_node);
2593 if (flags & DMA_PREP_INTERRUPT)
2594 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2595 else
2596 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2597
2598 hw_desc = iter->hw_desc;
2599 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2600 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2601 ppc440spe_chan->qdest);
2602 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2603 *dst, 0);
2604 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2605 DMA_CDB_SG_DST1, scf[1]);
2606 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2607 iter->unmap_len = len;
2608
2609 /*
2610 * 3rd descriptor, multiply src[0] data and xor it
2611 * with destination
2612 */
2613 iter = list_first_entry(&iter->chain_node,
2614 struct ppc440spe_adma_desc_slot,
2615 chain_node);
2616 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2617 iter->hw_next = NULL;
2618 if (flags & DMA_PREP_INTERRUPT)
2619 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2620 else
2621 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2622
2623 hw_desc = iter->hw_desc;
2624 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2625 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2626 src[0]);
2627 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2628 *dst, 0);
2629 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2630 DMA_CDB_SG_DST1, scf[0]);
2631 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2632 iter->unmap_len = len;
2633 sw_desc->async_tx.flags = flags;
2634 }
2635
2636 spin_unlock_bh(&ppc440spe_chan->lock);
2637
2638 return sw_desc;
2639}
2640
2641static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2642 struct ppc440spe_adma_chan *ppc440spe_chan,
2643 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2644 const unsigned char *scf, size_t len, unsigned long flags)
2645{
2646 int slot_cnt;
2647 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2648 unsigned long op = 0;
2649 unsigned char mult = 1;
2650
2651 pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2652 __func__, dst_cnt, src_cnt, len);
2653 /* select operations WXOR/RXOR depending on the
2654 * source addresses of operators and the number
2655 * of destinations (RXOR support only Q-parity calculations)
2656 */
2657 set_bit(PPC440SPE_DESC_WXOR, &op);
2658 if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2659 /* no active RXOR;
2660 * do RXOR if:
2661 * - there are more than 1 source,
2662 * - len is aligned on 512-byte boundary,
2663 * - source addresses fit to one of 4 possible regions.
2664 */
2665 if (src_cnt > 1 &&
2666 !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2667 (src[0] + len) == src[1]) {
2668 /* may do RXOR R1 R2 */
2669 set_bit(PPC440SPE_DESC_RXOR, &op);
2670 if (src_cnt != 2) {
2671 /* may try to enhance region of RXOR */
2672 if ((src[1] + len) == src[2]) {
2673 /* do RXOR R1 R2 R3 */
2674 set_bit(PPC440SPE_DESC_RXOR123,
2675 &op);
2676 } else if ((src[1] + len * 2) == src[2]) {
2677 /* do RXOR R1 R2 R4 */
2678 set_bit(PPC440SPE_DESC_RXOR124, &op);
2679 } else if ((src[1] + len * 3) == src[2]) {
2680 /* do RXOR R1 R2 R5 */
2681 set_bit(PPC440SPE_DESC_RXOR125,
2682 &op);
2683 } else {
2684 /* do RXOR R1 R2 */
2685 set_bit(PPC440SPE_DESC_RXOR12,
2686 &op);
2687 }
2688 } else {
2689 /* do RXOR R1 R2 */
2690 set_bit(PPC440SPE_DESC_RXOR12, &op);
2691 }
2692 }
2693
2694 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2695 /* can not do this operation with RXOR */
2696 clear_bit(PPC440SPE_RXOR_RUN,
2697 &ppc440spe_rxor_state);
2698 } else {
2699 /* can do; set block size right now */
2700 ppc440spe_desc_set_rxor_block_size(len);
2701 }
2702 }
2703
2704 /* Number of necessary slots depends on operation type selected */
2705 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2706 /* This is a WXOR only chain. Need descriptors for each
2707 * source to GF-XOR them with WXOR, and need descriptors
2708 * for each destination to zero them with WXOR
2709 */
2710 slot_cnt = src_cnt;
2711
2712 if (flags & DMA_PREP_ZERO_P) {
2713 slot_cnt++;
2714 set_bit(PPC440SPE_ZERO_P, &op);
2715 }
2716 if (flags & DMA_PREP_ZERO_Q) {
2717 slot_cnt++;
2718 set_bit(PPC440SPE_ZERO_Q, &op);
2719 }
2720 } else {
2721 /* Need 1/2 descriptor for RXOR operation, and
2722 * need (src_cnt - (2 or 3)) for WXOR of sources
2723 * remained (if any)
2724 */
2725 slot_cnt = dst_cnt;
2726
2727 if (flags & DMA_PREP_ZERO_P)
2728 set_bit(PPC440SPE_ZERO_P, &op);
2729 if (flags & DMA_PREP_ZERO_Q)
2730 set_bit(PPC440SPE_ZERO_Q, &op);
2731
2732 if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2733 slot_cnt += src_cnt - 2;
2734 else
2735 slot_cnt += src_cnt - 3;
2736
2737 /* Thus we have either RXOR only chain or
2738 * mixed RXOR/WXOR
2739 */
2740 if (slot_cnt == dst_cnt)
2741 /* RXOR only chain */
2742 clear_bit(PPC440SPE_DESC_WXOR, &op);
2743 }
2744
2745 spin_lock_bh(&ppc440spe_chan->lock);
2746 /* for both RXOR/WXOR each descriptor occupies one slot */
2747 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2748 if (sw_desc) {
2749 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2750 flags, op);
2751
2752 /* setup dst/src/mult */
2753 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2754 __func__, dst[0], dst[1]);
2755 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2756 while (src_cnt--) {
2757 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2758 src_cnt);
2759
2760 /* NOTE: "Multi = 0 is equivalent to = 1" as it
2761 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2762 * doesn't work for RXOR with DMA0/1! Instead, multi=0
2763 * leads to zeroing source data after RXOR.
2764 * So, for P case set-up mult=1 explicitly.
2765 */
2766 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2767 mult = scf[src_cnt];
2768 ppc440spe_adma_pq_set_src_mult(sw_desc,
2769 mult, src_cnt, dst_cnt - 1);
2770 }
2771
2772 /* Setup byte count foreach slot just allocated */
2773 sw_desc->async_tx.flags = flags;
2774 list_for_each_entry(iter, &sw_desc->group_list,
2775 chain_node) {
2776 ppc440spe_desc_set_byte_count(iter,
2777 ppc440spe_chan, len);
2778 iter->unmap_len = len;
2779 }
2780 }
2781 spin_unlock_bh(&ppc440spe_chan->lock);
2782
2783 return sw_desc;
2784}
2785
2786static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2787 struct ppc440spe_adma_chan *ppc440spe_chan,
2788 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2789 const unsigned char *scf, size_t len, unsigned long flags)
2790{
2791 int slot_cnt, descs_per_op;
2792 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2793 unsigned long op = 0;
2794 unsigned char mult = 1;
2795
2796 BUG_ON(!dst_cnt);
2797 /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2798 __func__, dst_cnt, src_cnt, len);*/
2799
2800 spin_lock_bh(&ppc440spe_chan->lock);
2801 descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2802 if (descs_per_op < 0) {
2803 spin_unlock_bh(&ppc440spe_chan->lock);
2804 return NULL;
2805 }
2806
2807 /* depending on number of sources we have 1 or 2 RXOR chains */
2808 slot_cnt = descs_per_op * dst_cnt;
2809
2810 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2811 if (sw_desc) {
2812 op = slot_cnt;
2813 sw_desc->async_tx.flags = flags;
2814 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2815 ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2816 --op ? 0 : flags);
2817 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2818 len);
2819 iter->unmap_len = len;
2820
2821 ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2822 iter->rxor_cursor.len = len;
2823 iter->descs_per_op = descs_per_op;
2824 }
2825 op = 0;
2826 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2827 op++;
2828 if (op % descs_per_op == 0)
2829 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2830 src_cnt);
2831 if (likely(!list_is_last(&iter->chain_node,
2832 &sw_desc->group_list))) {
2833 /* set 'next' pointer */
2834 iter->hw_next =
2835 list_entry(iter->chain_node.next,
2836 struct ppc440spe_adma_desc_slot,
2837 chain_node);
2838 ppc440spe_xor_set_link(iter, iter->hw_next);
2839 } else {
2840 /* this is the last descriptor. */
2841 iter->hw_next = NULL;
2842 }
2843 }
2844
2845 /* fixup head descriptor */
2846 sw_desc->dst_cnt = dst_cnt;
2847 if (flags & DMA_PREP_ZERO_P)
2848 set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2849 if (flags & DMA_PREP_ZERO_Q)
2850 set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2851
2852 /* setup dst/src/mult */
2853 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2854
2855 while (src_cnt--) {
2856 /* handle descriptors (if dst_cnt == 2) inside
2857 * the ppc440spe_adma_pq_set_srcxxx() functions
2858 */
2859 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2860 src_cnt);
2861 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2862 mult = scf[src_cnt];
2863 ppc440spe_adma_pq_set_src_mult(sw_desc,
2864 mult, src_cnt, dst_cnt - 1);
2865 }
2866 }
2867 spin_unlock_bh(&ppc440spe_chan->lock);
2868 ppc440spe_desc_set_rxor_block_size(len);
2869 return sw_desc;
2870}
2871
2872/**
2873 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2874 */
2875static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2876 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2877 unsigned int src_cnt, const unsigned char *scf,
2878 size_t len, unsigned long flags)
2879{
2880 struct ppc440spe_adma_chan *ppc440spe_chan;
2881 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2882 int dst_cnt = 0;
2883
2884 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2885
2886 ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2887 dst, src, src_cnt));
2888 BUG_ON(!len);
2889 BUG_ON(unlikely(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT));
2890 BUG_ON(!src_cnt);
2891
2892 if (src_cnt == 1 && dst[1] == src[0]) {
2893 dma_addr_t dest[2];
2894
2895 /* dst[1] is real destination (Q) */
2896 dest[0] = dst[1];
2897 /* this is the page to multicast source data to */
2898 dest[1] = ppc440spe_chan->qdest;
2899 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2900 dest, 2, src, src_cnt, scf, len, flags);
2901 return sw_desc ? &sw_desc->async_tx : NULL;
2902 }
2903
2904 if (src_cnt == 2 && dst[1] == src[1]) {
2905 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2906 &dst[1], src, 2, scf, len, flags);
2907 return sw_desc ? &sw_desc->async_tx : NULL;
2908 }
2909
2910 if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2911 BUG_ON(!dst[0]);
2912 dst_cnt++;
2913 flags |= DMA_PREP_ZERO_P;
2914 }
2915
2916 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2917 BUG_ON(!dst[1]);
2918 dst_cnt++;
2919 flags |= DMA_PREP_ZERO_Q;
2920 }
2921
2922 BUG_ON(!dst_cnt);
2923
2924 dev_dbg(ppc440spe_chan->device->common.dev,
2925 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2926 ppc440spe_chan->device->id, __func__, src_cnt, len,
2927 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2928
2929 switch (ppc440spe_chan->device->id) {
2930 case PPC440SPE_DMA0_ID:
2931 case PPC440SPE_DMA1_ID:
2932 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2933 dst, dst_cnt, src, src_cnt, scf,
2934 len, flags);
2935 break;
2936
2937 case PPC440SPE_XOR_ID:
2938 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2939 dst, dst_cnt, src, src_cnt, scf,
2940 len, flags);
2941 break;
2942 }
2943
2944 return sw_desc ? &sw_desc->async_tx : NULL;
2945}
2946
2947/**
2948 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2949 * a PQ_ZERO_SUM operation
2950 */
2951static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2952 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2953 unsigned int src_cnt, const unsigned char *scf, size_t len,
2954 enum sum_check_flags *pqres, unsigned long flags)
2955{
2956 struct ppc440spe_adma_chan *ppc440spe_chan;
2957 struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2958 dma_addr_t pdest, qdest;
2959 int slot_cnt, slots_per_op, idst, dst_cnt;
2960
2961 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2962
2963 if (flags & DMA_PREP_PQ_DISABLE_P)
2964 pdest = 0;
2965 else
2966 pdest = pq[0];
2967
2968 if (flags & DMA_PREP_PQ_DISABLE_Q)
2969 qdest = 0;
2970 else
2971 qdest = pq[1];
2972
2973 ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2974 src, src_cnt, scf));
2975
2976 /* Always use WXOR for P/Q calculations (two destinations).
2977 * Need 1 or 2 extra slots to verify results are zero.
2978 */
2979 idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2980
2981 /* One additional slot per destination to clone P/Q
2982 * before calculation (we have to preserve destinations).
2983 */
2984 slot_cnt = src_cnt + dst_cnt * 2;
2985 slots_per_op = 1;
2986
2987 spin_lock_bh(&ppc440spe_chan->lock);
2988 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2989 slots_per_op);
2990 if (sw_desc) {
2991 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2992
2993 /* Setup byte count for each slot just allocated */
2994 sw_desc->async_tx.flags = flags;
2995 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2996 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2997 len);
2998 iter->unmap_len = len;
2999 }
3000
3001 if (pdest) {
3002 struct dma_cdb *hw_desc;
3003 struct ppc440spe_adma_chan *chan;
3004
3005 iter = sw_desc->group_head;
3006 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3007 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
3008 iter->hw_next = list_entry(iter->chain_node.next,
3009 struct ppc440spe_adma_desc_slot,
3010 chain_node);
3011 hw_desc = iter->hw_desc;
3012 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
3013 iter->src_cnt = 0;
3014 iter->dst_cnt = 0;
3015 ppc440spe_desc_set_dest_addr(iter, chan, 0,
3016 ppc440spe_chan->pdest, 0);
3017 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
3018 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
3019 len);
3020 iter->unmap_len = 0;
3021 /* override pdest to preserve original P */
3022 pdest = ppc440spe_chan->pdest;
3023 }
3024 if (qdest) {
3025 struct dma_cdb *hw_desc;
3026 struct ppc440spe_adma_chan *chan;
3027
3028 iter = list_first_entry(&sw_desc->group_list,
3029 struct ppc440spe_adma_desc_slot,
3030 chain_node);
3031 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3032
3033 if (pdest) {
3034 iter = list_entry(iter->chain_node.next,
3035 struct ppc440spe_adma_desc_slot,
3036 chain_node);
3037 }
3038
3039 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
3040 iter->hw_next = list_entry(iter->chain_node.next,
3041 struct ppc440spe_adma_desc_slot,
3042 chain_node);
3043 hw_desc = iter->hw_desc;
3044 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
3045 iter->src_cnt = 0;
3046 iter->dst_cnt = 0;
3047 ppc440spe_desc_set_dest_addr(iter, chan, 0,
3048 ppc440spe_chan->qdest, 0);
3049 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
3050 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
3051 len);
3052 iter->unmap_len = 0;
3053 /* override qdest to preserve original Q */
3054 qdest = ppc440spe_chan->qdest;
3055 }
3056
3057 /* Setup destinations for P/Q ops */
3058 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
3059
3060 /* Setup zero QWORDs into DCHECK CDBs */
3061 idst = dst_cnt;
3062 list_for_each_entry_reverse(iter, &sw_desc->group_list,
3063 chain_node) {
3064 /*
3065 * The last CDB corresponds to Q-parity check,
3066 * the one before last CDB corresponds
3067 * P-parity check
3068 */
3069 if (idst == DMA_DEST_MAX_NUM) {
3070 if (idst == dst_cnt) {
3071 set_bit(PPC440SPE_DESC_QCHECK,
3072 &iter->flags);
3073 } else {
3074 set_bit(PPC440SPE_DESC_PCHECK,
3075 &iter->flags);
3076 }
3077 } else {
3078 if (qdest) {
3079 set_bit(PPC440SPE_DESC_QCHECK,
3080 &iter->flags);
3081 } else {
3082 set_bit(PPC440SPE_DESC_PCHECK,
3083 &iter->flags);
3084 }
3085 }
3086 iter->xor_check_result = pqres;
3087
3088 /*
3089 * set it to zero, if check fail then result will
3090 * be updated
3091 */
3092 *iter->xor_check_result = 0;
3093 ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
3094 ppc440spe_qword);
3095
3096 if (!(--dst_cnt))
3097 break;
3098 }
3099
3100 /* Setup sources and mults for P/Q ops */
3101 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
3102 chain_node) {
3103 struct ppc440spe_adma_chan *chan;
3104 u32 mult_dst;
3105
3106 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3107 ppc440spe_desc_set_src_addr(iter, chan, 0,
3108 DMA_CUED_XOR_HB,
3109 src[src_cnt - 1]);
3110 if (qdest) {
3111 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
3112 DMA_CDB_SG_DST1;
3113 ppc440spe_desc_set_src_mult(iter, chan,
3114 DMA_CUED_MULT1_OFF,
3115 mult_dst,
3116 scf[src_cnt - 1]);
3117 }
3118 if (!(--src_cnt))
3119 break;
3120 }
3121 }
3122 spin_unlock_bh(&ppc440spe_chan->lock);
3123 return sw_desc ? &sw_desc->async_tx : NULL;
3124}
3125
3126/**
3127 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
3128 * XOR ZERO_SUM operation
3129 */
3130static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
3131 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
3132 size_t len, enum sum_check_flags *result, unsigned long flags)
3133{
3134 struct dma_async_tx_descriptor *tx;
3135 dma_addr_t pq[2];
3136
3137 /* validate P, disable Q */
3138 pq[0] = src[0];
3139 pq[1] = 0;
3140 flags |= DMA_PREP_PQ_DISABLE_Q;
3141
3142 tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
3143 src_cnt - 1, 0, len,
3144 result, flags);
3145 return tx;
3146}
3147
3148/**
3149 * ppc440spe_adma_set_dest - set destination address into descriptor
3150 */
3151static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
3152 dma_addr_t addr, int index)
3153{
3154 struct ppc440spe_adma_chan *chan;
3155
3156 BUG_ON(index >= sw_desc->dst_cnt);
3157
3158 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3159
3160 switch (chan->device->id) {
3161 case PPC440SPE_DMA0_ID:
3162 case PPC440SPE_DMA1_ID:
3163 /* to do: support transfers lengths >
3164 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
3165 */
3166 ppc440spe_desc_set_dest_addr(sw_desc->group_head,
3167 chan, 0, addr, index);
3168 break;
3169 case PPC440SPE_XOR_ID:
3170 sw_desc = ppc440spe_get_group_entry(sw_desc, index);
3171 ppc440spe_desc_set_dest_addr(sw_desc,
3172 chan, 0, addr, index);
3173 break;
3174 }
3175}
3176
3177static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
3178 struct ppc440spe_adma_chan *chan, dma_addr_t addr)
3179{
3180 /* To clear destinations update the descriptor
3181 * (P or Q depending on index) as follows:
3182 * addr is destination (0 corresponds to SG2):
3183 */
3184 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
3185
3186 /* ... and the addr is source: */
3187 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
3188
3189 /* addr is always SG2 then the mult is always DST1 */
3190 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
3191 DMA_CDB_SG_DST1, 1);
3192}
3193
3194/**
3195 * ppc440spe_adma_pq_set_dest - set destination address into descriptor
3196 * for the PQXOR operation
3197 */
3198static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
3199 dma_addr_t *addrs, unsigned long flags)
3200{
3201 struct ppc440spe_adma_desc_slot *iter;
3202 struct ppc440spe_adma_chan *chan;
3203 dma_addr_t paddr, qaddr;
3204 dma_addr_t addr = 0, ppath, qpath;
3205 int index = 0, i;
3206
3207 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3208
3209 if (flags & DMA_PREP_PQ_DISABLE_P)
3210 paddr = 0;
3211 else
3212 paddr = addrs[0];
3213
3214 if (flags & DMA_PREP_PQ_DISABLE_Q)
3215 qaddr = 0;
3216 else
3217 qaddr = addrs[1];
3218
3219 if (!paddr || !qaddr)
3220 addr = paddr ? paddr : qaddr;
3221
3222 switch (chan->device->id) {
3223 case PPC440SPE_DMA0_ID:
3224 case PPC440SPE_DMA1_ID:
3225 /* walk through the WXOR source list and set P/Q-destinations
3226 * for each slot:
3227 */
3228 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3229 /* This is WXOR-only chain; may have 1/2 zero descs */
3230 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3231 index++;
3232 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3233 index++;
3234
3235 iter = ppc440spe_get_group_entry(sw_desc, index);
3236 if (addr) {
3237 /* one destination */
3238 list_for_each_entry_from(iter,
3239 &sw_desc->group_list, chain_node)
3240 ppc440spe_desc_set_dest_addr(iter, chan,
3241 DMA_CUED_XOR_BASE, addr, 0);
3242 } else {
3243 /* two destinations */
3244 list_for_each_entry_from(iter,
3245 &sw_desc->group_list, chain_node) {
3246 ppc440spe_desc_set_dest_addr(iter, chan,
3247 DMA_CUED_XOR_BASE, paddr, 0);
3248 ppc440spe_desc_set_dest_addr(iter, chan,
3249 DMA_CUED_XOR_BASE, qaddr, 1);
3250 }
3251 }
3252
3253 if (index) {
3254 /* To clear destinations update the descriptor
3255 * (1st,2nd, or both depending on flags)
3256 */
3257 index = 0;
3258 if (test_bit(PPC440SPE_ZERO_P,
3259 &sw_desc->flags)) {
3260 iter = ppc440spe_get_group_entry(
3261 sw_desc, index++);
3262 ppc440spe_adma_pq_zero_op(iter, chan,
3263 paddr);
3264 }
3265
3266 if (test_bit(PPC440SPE_ZERO_Q,
3267 &sw_desc->flags)) {
3268 iter = ppc440spe_get_group_entry(
3269 sw_desc, index++);
3270 ppc440spe_adma_pq_zero_op(iter, chan,
3271 qaddr);
3272 }
3273
3274 return;
3275 }
3276 } else {
3277 /* This is RXOR-only or RXOR/WXOR mixed chain */
3278
3279 /* If we want to include destination into calculations,
3280 * then make dest addresses cued with mult=1 (XOR).
3281 */
3282 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3283 DMA_CUED_XOR_HB :
3284 DMA_CUED_XOR_BASE |
3285 (1 << DMA_CUED_MULT1_OFF);
3286 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3287 DMA_CUED_XOR_HB :
3288 DMA_CUED_XOR_BASE |
3289 (1 << DMA_CUED_MULT1_OFF);
3290
3291 /* Setup destination(s) in RXOR slot(s) */
3292 iter = ppc440spe_get_group_entry(sw_desc, index++);
3293 ppc440spe_desc_set_dest_addr(iter, chan,
3294 paddr ? ppath : qpath,
3295 paddr ? paddr : qaddr, 0);
3296 if (!addr) {
3297 /* two destinations */
3298 iter = ppc440spe_get_group_entry(sw_desc,
3299 index++);
3300 ppc440spe_desc_set_dest_addr(iter, chan,
3301 qpath, qaddr, 0);
3302 }
3303
3304 if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
3305 /* Setup destination(s) in remaining WXOR
3306 * slots
3307 */
3308 iter = ppc440spe_get_group_entry(sw_desc,
3309 index);
3310 if (addr) {
3311 /* one destination */
3312 list_for_each_entry_from(iter,
3313 &sw_desc->group_list,
3314 chain_node)
3315 ppc440spe_desc_set_dest_addr(
3316 iter, chan,
3317 DMA_CUED_XOR_BASE,
3318 addr, 0);
3319
3320 } else {
3321 /* two destinations */
3322 list_for_each_entry_from(iter,
3323 &sw_desc->group_list,
3324 chain_node) {
3325 ppc440spe_desc_set_dest_addr(
3326 iter, chan,
3327 DMA_CUED_XOR_BASE,
3328 paddr, 0);
3329 ppc440spe_desc_set_dest_addr(
3330 iter, chan,
3331 DMA_CUED_XOR_BASE,
3332 qaddr, 1);
3333 }
3334 }
3335 }
3336
3337 }
3338 break;
3339
3340 case PPC440SPE_XOR_ID:
3341 /* DMA2 descriptors have only 1 destination, so there are
3342 * two chains - one for each dest.
3343 * If we want to include destination into calculations,
3344 * then make dest addresses cued with mult=1 (XOR).
3345 */
3346 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3347 DMA_CUED_XOR_HB :
3348 DMA_CUED_XOR_BASE |
3349 (1 << DMA_CUED_MULT1_OFF);
3350
3351 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3352 DMA_CUED_XOR_HB :
3353 DMA_CUED_XOR_BASE |
3354 (1 << DMA_CUED_MULT1_OFF);
3355
3356 iter = ppc440spe_get_group_entry(sw_desc, 0);
3357 for (i = 0; i < sw_desc->descs_per_op; i++) {
3358 ppc440spe_desc_set_dest_addr(iter, chan,
3359 paddr ? ppath : qpath,
3360 paddr ? paddr : qaddr, 0);
3361 iter = list_entry(iter->chain_node.next,
3362 struct ppc440spe_adma_desc_slot,
3363 chain_node);
3364 }
3365
3366 if (!addr) {
3367 /* Two destinations; setup Q here */
3368 iter = ppc440spe_get_group_entry(sw_desc,
3369 sw_desc->descs_per_op);
3370 for (i = 0; i < sw_desc->descs_per_op; i++) {
3371 ppc440spe_desc_set_dest_addr(iter,
3372 chan, qpath, qaddr, 0);
3373 iter = list_entry(iter->chain_node.next,
3374 struct ppc440spe_adma_desc_slot,
3375 chain_node);
3376 }
3377 }
3378
3379 break;
3380 }
3381}
3382
3383/**
3384 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3385 * for the PQ_ZERO_SUM operation
3386 */
3387static void ppc440spe_adma_pqzero_sum_set_dest(
3388 struct ppc440spe_adma_desc_slot *sw_desc,
3389 dma_addr_t paddr, dma_addr_t qaddr)
3390{
3391 struct ppc440spe_adma_desc_slot *iter, *end;
3392 struct ppc440spe_adma_chan *chan;
3393 dma_addr_t addr = 0;
3394 int idx;
3395
3396 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3397
3398 /* walk through the WXOR source list and set P/Q-destinations
3399 * for each slot
3400 */
3401 idx = (paddr && qaddr) ? 2 : 1;
3402 /* set end */
3403 list_for_each_entry_reverse(end, &sw_desc->group_list,
3404 chain_node) {
3405 if (!(--idx))
3406 break;
3407 }
3408 /* set start */
3409 idx = (paddr && qaddr) ? 2 : 1;
3410 iter = ppc440spe_get_group_entry(sw_desc, idx);
3411
3412 if (paddr && qaddr) {
3413 /* two destinations */
3414 list_for_each_entry_from(iter, &sw_desc->group_list,
3415 chain_node) {
3416 if (unlikely(iter == end))
3417 break;
3418 ppc440spe_desc_set_dest_addr(iter, chan,
3419 DMA_CUED_XOR_BASE, paddr, 0);
3420 ppc440spe_desc_set_dest_addr(iter, chan,
3421 DMA_CUED_XOR_BASE, qaddr, 1);
3422 }
3423 } else {
3424 /* one destination */
3425 addr = paddr ? paddr : qaddr;
3426 list_for_each_entry_from(iter, &sw_desc->group_list,
3427 chain_node) {
3428 if (unlikely(iter == end))
3429 break;
3430 ppc440spe_desc_set_dest_addr(iter, chan,
3431 DMA_CUED_XOR_BASE, addr, 0);
3432 }
3433 }
3434
3435 /* The remaining descriptors are DATACHECK. These have no need in
3436 * destination. Actually, these destinations are used there
3437 * as sources for check operation. So, set addr as source.
3438 */
3439 ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3440
3441 if (!addr) {
3442 end = list_entry(end->chain_node.next,
3443 struct ppc440spe_adma_desc_slot, chain_node);
3444 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3445 }
3446}
3447
3448/**
3449 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3450 */
3451static inline void ppc440spe_desc_set_xor_src_cnt(
3452 struct ppc440spe_adma_desc_slot *desc,
3453 int src_cnt)
3454{
3455 struct xor_cb *hw_desc = desc->hw_desc;
3456
3457 hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3458 hw_desc->cbc |= src_cnt;
3459}
3460
3461/**
3462 * ppc440spe_adma_pq_set_src - set source address into descriptor
3463 */
3464static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3465 dma_addr_t addr, int index)
3466{
3467 struct ppc440spe_adma_chan *chan;
3468 dma_addr_t haddr = 0;
3469 struct ppc440spe_adma_desc_slot *iter = NULL;
3470
3471 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3472
3473 switch (chan->device->id) {
3474 case PPC440SPE_DMA0_ID:
3475 case PPC440SPE_DMA1_ID:
3476 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3477 */
3478 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3479 /* RXOR-only or RXOR/WXOR operation */
3480 int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3481 &sw_desc->flags) ? 2 : 3;
3482
3483 if (index == 0) {
3484 /* 1st slot (RXOR) */
3485 /* setup sources region (R1-2-3, R1-2-4,
3486 * or R1-2-5)
3487 */
3488 if (test_bit(PPC440SPE_DESC_RXOR12,
3489 &sw_desc->flags))
3490 haddr = DMA_RXOR12 <<
3491 DMA_CUED_REGION_OFF;
3492 else if (test_bit(PPC440SPE_DESC_RXOR123,
3493 &sw_desc->flags))
3494 haddr = DMA_RXOR123 <<
3495 DMA_CUED_REGION_OFF;
3496 else if (test_bit(PPC440SPE_DESC_RXOR124,
3497 &sw_desc->flags))
3498 haddr = DMA_RXOR124 <<
3499 DMA_CUED_REGION_OFF;
3500 else if (test_bit(PPC440SPE_DESC_RXOR125,
3501 &sw_desc->flags))
3502 haddr = DMA_RXOR125 <<
3503 DMA_CUED_REGION_OFF;
3504 else
3505 BUG();
3506 haddr |= DMA_CUED_XOR_BASE;
3507 iter = ppc440spe_get_group_entry(sw_desc, 0);
3508 } else if (index < iskip) {
3509 /* 1st slot (RXOR)
3510 * shall actually set source address only once
3511 * instead of first <iskip>
3512 */
3513 iter = NULL;
3514 } else {
3515 /* 2nd/3d and next slots (WXOR);
3516 * skip first slot with RXOR
3517 */
3518 haddr = DMA_CUED_XOR_HB;
3519 iter = ppc440spe_get_group_entry(sw_desc,
3520 index - iskip + sw_desc->dst_cnt);
3521 }
3522 } else {
3523 int znum = 0;
3524
3525 /* WXOR-only operation; skip first slots with
3526 * zeroing destinations
3527 */
3528 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3529 znum++;
3530 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3531 znum++;
3532
3533 haddr = DMA_CUED_XOR_HB;
3534 iter = ppc440spe_get_group_entry(sw_desc,
3535 index + znum);
3536 }
3537
3538 if (likely(iter)) {
3539 ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3540
3541 if (!index &&
3542 test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3543 sw_desc->dst_cnt == 2) {
3544 /* if we have two destinations for RXOR, then
3545 * setup source in the second descr too
3546 */
3547 iter = ppc440spe_get_group_entry(sw_desc, 1);
3548 ppc440spe_desc_set_src_addr(iter, chan, 0,
3549 haddr, addr);
3550 }
3551 }
3552 break;
3553
3554 case PPC440SPE_XOR_ID:
3555 /* DMA2 may do Biskup */
3556 iter = sw_desc->group_head;
3557 if (iter->dst_cnt == 2) {
3558 /* both P & Q calculations required; set P src here */
3559 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3560
3561 /* this is for Q */
3562 iter = ppc440spe_get_group_entry(sw_desc,
3563 sw_desc->descs_per_op);
3564 }
3565 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3566 break;
3567 }
3568}
3569
3570/**
3571 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3572 */
3573static void ppc440spe_adma_memcpy_xor_set_src(
3574 struct ppc440spe_adma_desc_slot *sw_desc,
3575 dma_addr_t addr, int index)
3576{
3577 struct ppc440spe_adma_chan *chan;
3578
3579 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3580 sw_desc = sw_desc->group_head;
3581
3582 if (likely(sw_desc))
3583 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3584}
3585
3586/**
3587 * ppc440spe_adma_dma2rxor_inc_addr -
3588 */
3589static void ppc440spe_adma_dma2rxor_inc_addr(
3590 struct ppc440spe_adma_desc_slot *desc,
3591 struct ppc440spe_rxor *cursor, int index, int src_cnt)
3592{
3593 cursor->addr_count++;
3594 if (index == src_cnt - 1) {
3595 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3596 } else if (cursor->addr_count == XOR_MAX_OPS) {
3597 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3598 cursor->addr_count = 0;
3599 cursor->desc_count++;
3600 }
3601}
3602
3603/**
3604 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3605 */
3606static int ppc440spe_adma_dma2rxor_prep_src(
3607 struct ppc440spe_adma_desc_slot *hdesc,
3608 struct ppc440spe_rxor *cursor, int index,
3609 int src_cnt, u32 addr)
3610{
3611 int rval = 0;
3612 u32 sign;
3613 struct ppc440spe_adma_desc_slot *desc = hdesc;
3614 int i;
3615
3616 for (i = 0; i < cursor->desc_count; i++) {
3617 desc = list_entry(hdesc->chain_node.next,
3618 struct ppc440spe_adma_desc_slot,
3619 chain_node);
3620 }
3621
3622 switch (cursor->state) {
3623 case 0:
3624 if (addr == cursor->addrl + cursor->len) {
3625 /* direct RXOR */
3626 cursor->state = 1;
3627 cursor->xor_count++;
3628 if (index == src_cnt-1) {
3629 ppc440spe_rxor_set_region(desc,
3630 cursor->addr_count,
3631 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3632 ppc440spe_adma_dma2rxor_inc_addr(
3633 desc, cursor, index, src_cnt);
3634 }
3635 } else if (cursor->addrl == addr + cursor->len) {
3636 /* reverse RXOR */
3637 cursor->state = 1;
3638 cursor->xor_count++;
3639 set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3640 if (index == src_cnt-1) {
3641 ppc440spe_rxor_set_region(desc,
3642 cursor->addr_count,
3643 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3644 ppc440spe_adma_dma2rxor_inc_addr(
3645 desc, cursor, index, src_cnt);
3646 }
3647 } else {
3648 printk(KERN_ERR "Cannot build "
3649 "DMA2 RXOR command block.\n");
3650 BUG();
3651 }
3652 break;
3653 case 1:
3654 sign = test_bit(cursor->addr_count,
3655 desc->reverse_flags)
3656 ? -1 : 1;
3657 if (index == src_cnt-2 || (sign == -1
3658 && addr != cursor->addrl - 2*cursor->len)) {
3659 cursor->state = 0;
3660 cursor->xor_count = 1;
3661 cursor->addrl = addr;
3662 ppc440spe_rxor_set_region(desc,
3663 cursor->addr_count,
3664 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3665 ppc440spe_adma_dma2rxor_inc_addr(
3666 desc, cursor, index, src_cnt);
3667 } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3668 cursor->state = 2;
3669 cursor->xor_count = 0;
3670 ppc440spe_rxor_set_region(desc,
3671 cursor->addr_count,
3672 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3673 if (index == src_cnt-1) {
3674 ppc440spe_adma_dma2rxor_inc_addr(
3675 desc, cursor, index, src_cnt);
3676 }
3677 } else if (addr == cursor->addrl + 3*cursor->len) {
3678 cursor->state = 2;
3679 cursor->xor_count = 0;
3680 ppc440spe_rxor_set_region(desc,
3681 cursor->addr_count,
3682 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3683 if (index == src_cnt-1) {
3684 ppc440spe_adma_dma2rxor_inc_addr(
3685 desc, cursor, index, src_cnt);
3686 }
3687 } else if (addr == cursor->addrl + 4*cursor->len) {
3688 cursor->state = 2;
3689 cursor->xor_count = 0;
3690 ppc440spe_rxor_set_region(desc,
3691 cursor->addr_count,
3692 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3693 if (index == src_cnt-1) {
3694 ppc440spe_adma_dma2rxor_inc_addr(
3695 desc, cursor, index, src_cnt);
3696 }
3697 } else {
3698 cursor->state = 0;
3699 cursor->xor_count = 1;
3700 cursor->addrl = addr;
3701 ppc440spe_rxor_set_region(desc,
3702 cursor->addr_count,
3703 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3704 ppc440spe_adma_dma2rxor_inc_addr(
3705 desc, cursor, index, src_cnt);
3706 }
3707 break;
3708 case 2:
3709 cursor->state = 0;
3710 cursor->addrl = addr;
3711 cursor->xor_count++;
3712 if (index) {
3713 ppc440spe_adma_dma2rxor_inc_addr(
3714 desc, cursor, index, src_cnt);
3715 }
3716 break;
3717 }
3718
3719 return rval;
3720}
3721
3722/**
3723 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3724 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3725 */
3726static void ppc440spe_adma_dma2rxor_set_src(
3727 struct ppc440spe_adma_desc_slot *desc,
3728 int index, dma_addr_t addr)
3729{
3730 struct xor_cb *xcb = desc->hw_desc;
3731 int k = 0, op = 0, lop = 0;
3732
3733 /* get the RXOR operand which corresponds to index addr */
3734 while (op <= index) {
3735 lop = op;
3736 if (k == XOR_MAX_OPS) {
3737 k = 0;
3738 desc = list_entry(desc->chain_node.next,
3739 struct ppc440spe_adma_desc_slot, chain_node);
3740 xcb = desc->hw_desc;
3741
3742 }
3743 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3744 (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3745 op += 2;
3746 else
3747 op += 3;
3748 }
3749
3750 BUG_ON(k < 1);
3751
3752 if (test_bit(k-1, desc->reverse_flags)) {
3753 /* reverse operand order; put last op in RXOR group */
3754 if (index == op - 1)
3755 ppc440spe_rxor_set_src(desc, k - 1, addr);
3756 } else {
3757 /* direct operand order; put first op in RXOR group */
3758 if (index == lop)
3759 ppc440spe_rxor_set_src(desc, k - 1, addr);
3760 }
3761}
3762
3763/**
3764 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3765 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3766 */
3767static void ppc440spe_adma_dma2rxor_set_mult(
3768 struct ppc440spe_adma_desc_slot *desc,
3769 int index, u8 mult)
3770{
3771 struct xor_cb *xcb = desc->hw_desc;
3772 int k = 0, op = 0, lop = 0;
3773
3774 /* get the RXOR operand which corresponds to index mult */
3775 while (op <= index) {
3776 lop = op;
3777 if (k == XOR_MAX_OPS) {
3778 k = 0;
3779 desc = list_entry(desc->chain_node.next,
3780 struct ppc440spe_adma_desc_slot,
3781 chain_node);
3782 xcb = desc->hw_desc;
3783
3784 }
3785 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3786 (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3787 op += 2;
3788 else
3789 op += 3;
3790 }
3791
3792 BUG_ON(k < 1);
3793 if (test_bit(k-1, desc->reverse_flags)) {
3794 /* reverse order */
3795 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3796 } else {
3797 /* direct order */
3798 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3799 }
3800}
3801
3802/**
3803 * ppc440spe_init_rxor_cursor -
3804 */
3805static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3806{
3807 memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3808 cursor->state = 2;
3809}
3810
3811/**
3812 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3813 * descriptor for the PQXOR operation
3814 */
3815static void ppc440spe_adma_pq_set_src_mult(
3816 struct ppc440spe_adma_desc_slot *sw_desc,
3817 unsigned char mult, int index, int dst_pos)
3818{
3819 struct ppc440spe_adma_chan *chan;
3820 u32 mult_idx, mult_dst;
3821 struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3822
3823 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3824
3825 switch (chan->device->id) {
3826 case PPC440SPE_DMA0_ID:
3827 case PPC440SPE_DMA1_ID:
3828 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3829 int region = test_bit(PPC440SPE_DESC_RXOR12,
3830 &sw_desc->flags) ? 2 : 3;
3831
3832 if (index < region) {
3833 /* RXOR multipliers */
3834 iter = ppc440spe_get_group_entry(sw_desc,
3835 sw_desc->dst_cnt - 1);
3836 if (sw_desc->dst_cnt == 2)
3837 iter1 = ppc440spe_get_group_entry(
3838 sw_desc, 0);
3839
3840 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3841 mult_dst = DMA_CDB_SG_SRC;
3842 } else {
3843 /* WXOR multiplier */
3844 iter = ppc440spe_get_group_entry(sw_desc,
3845 index - region +
3846 sw_desc->dst_cnt);
3847 mult_idx = DMA_CUED_MULT1_OFF;
3848 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3849 DMA_CDB_SG_DST1;
3850 }
3851 } else {
3852 int znum = 0;
3853
3854 /* WXOR-only;
3855 * skip first slots with destinations (if ZERO_DST has
3856 * place)
3857 */
3858 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3859 znum++;
3860 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3861 znum++;
3862
3863 iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3864 mult_idx = DMA_CUED_MULT1_OFF;
3865 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3866 }
3867
3868 if (likely(iter)) {
3869 ppc440spe_desc_set_src_mult(iter, chan,
3870 mult_idx, mult_dst, mult);
3871
3872 if (unlikely(iter1)) {
3873 /* if we have two destinations for RXOR, then
3874 * we've just set Q mult. Set-up P now.
3875 */
3876 ppc440spe_desc_set_src_mult(iter1, chan,
3877 mult_idx, mult_dst, 1);
3878 }
3879
3880 }
3881 break;
3882
3883 case PPC440SPE_XOR_ID:
3884 iter = sw_desc->group_head;
3885 if (sw_desc->dst_cnt == 2) {
3886 /* both P & Q calculations required; set P mult here */
3887 ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3888
3889 /* and then set Q mult */
3890 iter = ppc440spe_get_group_entry(sw_desc,
3891 sw_desc->descs_per_op);
3892 }
3893 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3894 break;
3895 }
3896}
3897
3898/**
3899 * ppc440spe_adma_free_chan_resources - free the resources allocated
3900 */
3901static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3902{
3903 struct ppc440spe_adma_chan *ppc440spe_chan;
3904 struct ppc440spe_adma_desc_slot *iter, *_iter;
3905 int in_use_descs = 0;
3906
3907 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3908 ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3909
3910 spin_lock_bh(&ppc440spe_chan->lock);
3911 list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3912 chain_node) {
3913 in_use_descs++;
3914 list_del(&iter->chain_node);
3915 }
3916 list_for_each_entry_safe_reverse(iter, _iter,
3917 &ppc440spe_chan->all_slots, slot_node) {
3918 list_del(&iter->slot_node);
3919 kfree(iter);
3920 ppc440spe_chan->slots_allocated--;
3921 }
3922 ppc440spe_chan->last_used = NULL;
3923
3924 dev_dbg(ppc440spe_chan->device->common.dev,
3925 "ppc440spe adma%d %s slots_allocated %d\n",
3926 ppc440spe_chan->device->id,
3927 __func__, ppc440spe_chan->slots_allocated);
3928 spin_unlock_bh(&ppc440spe_chan->lock);
3929
3930 /* one is ok since we left it on there on purpose */
3931 if (in_use_descs > 1)
3932 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3933 in_use_descs - 1);
3934}
3935
3936/**
3937 * ppc440spe_adma_is_complete - poll the status of an ADMA transaction
3938 * @chan: ADMA channel handle
3939 * @cookie: ADMA transaction identifier
3940 */
3941static enum dma_status ppc440spe_adma_is_complete(struct dma_chan *chan,
3942 dma_cookie_t cookie, dma_cookie_t *done, dma_cookie_t *used)
3943{
3944 struct ppc440spe_adma_chan *ppc440spe_chan;
3945 dma_cookie_t last_used;
3946 dma_cookie_t last_complete;
3947 enum dma_status ret;
3948
3949 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3950 last_used = chan->cookie;
3951 last_complete = ppc440spe_chan->completed_cookie;
3952
3953 if (done)
3954 *done = last_complete;
3955 if (used)
3956 *used = last_used;
3957
3958 ret = dma_async_is_complete(cookie, last_complete, last_used);
3959 if (ret == DMA_SUCCESS)
3960 return ret;
3961
3962 ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3963
3964 last_used = chan->cookie;
3965 last_complete = ppc440spe_chan->completed_cookie;
3966
3967 if (done)
3968 *done = last_complete;
3969 if (used)
3970 *used = last_used;
3971
3972 return dma_async_is_complete(cookie, last_complete, last_used);
3973}
3974
3975/**
3976 * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3977 */
3978static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3979{
3980 struct ppc440spe_adma_chan *chan = data;
3981
3982 dev_dbg(chan->device->common.dev,
3983 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3984
3985 tasklet_schedule(&chan->irq_tasklet);
3986 ppc440spe_adma_device_clear_eot_status(chan);
3987
3988 return IRQ_HANDLED;
3989}
3990
3991/**
3992 * ppc440spe_adma_err_handler - DMA error interrupt handler;
3993 * do the same things as a eot handler
3994 */
3995static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3996{
3997 struct ppc440spe_adma_chan *chan = data;
3998
3999 dev_dbg(chan->device->common.dev,
4000 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
4001
4002 tasklet_schedule(&chan->irq_tasklet);
4003 ppc440spe_adma_device_clear_eot_status(chan);
4004
4005 return IRQ_HANDLED;
4006}
4007
4008/**
4009 * ppc440spe_test_callback - called when test operation has been done
4010 */
4011static void ppc440spe_test_callback(void *unused)
4012{
4013 complete(&ppc440spe_r6_test_comp);
4014}
4015
4016/**
4017 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
4018 */
4019static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
4020{
4021 struct ppc440spe_adma_chan *ppc440spe_chan;
4022
4023 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4024 dev_dbg(ppc440spe_chan->device->common.dev,
4025 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
4026 __func__, ppc440spe_chan->pending);
4027
4028 if (ppc440spe_chan->pending) {
4029 ppc440spe_chan->pending = 0;
4030 ppc440spe_chan_append(ppc440spe_chan);
4031 }
4032}
4033
4034/**
4035 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
4036 * use FIFOs (as opposite to chains used in XOR) so this is a XOR
4037 * specific operation)
4038 */
4039static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
4040{
4041 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
4042 dma_cookie_t cookie;
4043 int slot_cnt, slots_per_op;
4044
4045 dev_dbg(chan->device->common.dev,
4046 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
4047
4048 spin_lock_bh(&chan->lock);
4049 slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
4050 sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
4051 if (sw_desc) {
4052 group_start = sw_desc->group_head;
4053 list_splice_init(&sw_desc->group_list, &chan->chain);
4054 async_tx_ack(&sw_desc->async_tx);
4055 ppc440spe_desc_init_null_xor(group_start);
4056
4057 cookie = chan->common.cookie;
4058 cookie++;
4059 if (cookie <= 1)
4060 cookie = 2;
4061
4062 /* initialize the completed cookie to be less than
4063 * the most recently used cookie
4064 */
4065 chan->completed_cookie = cookie - 1;
4066 chan->common.cookie = sw_desc->async_tx.cookie = cookie;
4067
4068 /* channel should not be busy */
4069 BUG_ON(ppc440spe_chan_is_busy(chan));
4070
4071 /* set the descriptor address */
4072 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
4073
4074 /* run the descriptor */
4075 ppc440spe_chan_run(chan);
4076 } else
4077 printk(KERN_ERR "ppc440spe adma%d"
4078 " failed to allocate null descriptor\n",
4079 chan->device->id);
4080 spin_unlock_bh(&chan->lock);
4081}
4082
4083/**
4084 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
4085 * For this we just perform one WXOR operation with the same source
4086 * and destination addresses, the GF-multiplier is 1; so if RAID-6
4087 * capabilities are enabled then we'll get src/dst filled with zero.
4088 */
4089static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
4090{
4091 struct ppc440spe_adma_desc_slot *sw_desc, *iter;
4092 struct page *pg;
4093 char *a;
4094 dma_addr_t dma_addr, addrs[2];
4095 unsigned long op = 0;
4096 int rval = 0;
4097
4098 set_bit(PPC440SPE_DESC_WXOR, &op);
4099
4100 pg = alloc_page(GFP_KERNEL);
4101 if (!pg)
4102 return -ENOMEM;
4103
4104 spin_lock_bh(&chan->lock);
4105 sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
4106 if (sw_desc) {
4107 /* 1 src, 1 dsr, int_ena, WXOR */
4108 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
4109 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
4110 ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
4111 iter->unmap_len = PAGE_SIZE;
4112 }
4113 } else {
4114 rval = -EFAULT;
4115 spin_unlock_bh(&chan->lock);
4116 goto exit;
4117 }
4118 spin_unlock_bh(&chan->lock);
4119
4120 /* Fill the test page with ones */
4121 memset(page_address(pg), 0xFF, PAGE_SIZE);
4122 dma_addr = dma_map_page(chan->device->dev, pg, 0,
4123 PAGE_SIZE, DMA_BIDIRECTIONAL);
4124
4125 /* Setup addresses */
4126 ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
4127 ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
4128 addrs[0] = dma_addr;
4129 addrs[1] = 0;
4130 ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
4131
4132 async_tx_ack(&sw_desc->async_tx);
4133 sw_desc->async_tx.callback = ppc440spe_test_callback;
4134 sw_desc->async_tx.callback_param = NULL;
4135
4136 init_completion(&ppc440spe_r6_test_comp);
4137
4138 ppc440spe_adma_tx_submit(&sw_desc->async_tx);
4139 ppc440spe_adma_issue_pending(&chan->common);
4140
4141 wait_for_completion(&ppc440spe_r6_test_comp);
4142
4143 /* Now check if the test page is zeroed */
4144 a = page_address(pg);
4145 if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
4146 /* page is zero - RAID-6 enabled */
4147 rval = 0;
4148 } else {
4149 /* RAID-6 was not enabled */
4150 rval = -EINVAL;
4151 }
4152exit:
4153 __free_page(pg);
4154 return rval;
4155}
4156
4157static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
4158{
4159 switch (adev->id) {
4160 case PPC440SPE_DMA0_ID:
4161 case PPC440SPE_DMA1_ID:
4162 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
4163 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
4164 dma_cap_set(DMA_MEMSET, adev->common.cap_mask);
4165 dma_cap_set(DMA_PQ, adev->common.cap_mask);
4166 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
4167 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
4168 break;
4169 case PPC440SPE_XOR_ID:
4170 dma_cap_set(DMA_XOR, adev->common.cap_mask);
4171 dma_cap_set(DMA_PQ, adev->common.cap_mask);
4172 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
4173 adev->common.cap_mask = adev->common.cap_mask;
4174 break;
4175 }
4176
4177 /* Set base routines */
4178 adev->common.device_alloc_chan_resources =
4179 ppc440spe_adma_alloc_chan_resources;
4180 adev->common.device_free_chan_resources =
4181 ppc440spe_adma_free_chan_resources;
4182 adev->common.device_is_tx_complete = ppc440spe_adma_is_complete;
4183 adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
4184
4185 /* Set prep routines based on capability */
4186 if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
4187 adev->common.device_prep_dma_memcpy =
4188 ppc440spe_adma_prep_dma_memcpy;
4189 }
4190 if (dma_has_cap(DMA_MEMSET, adev->common.cap_mask)) {
4191 adev->common.device_prep_dma_memset =
4192 ppc440spe_adma_prep_dma_memset;
4193 }
4194 if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
4195 adev->common.max_xor = XOR_MAX_OPS;
4196 adev->common.device_prep_dma_xor =
4197 ppc440spe_adma_prep_dma_xor;
4198 }
4199 if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
4200 switch (adev->id) {
4201 case PPC440SPE_DMA0_ID:
4202 dma_set_maxpq(&adev->common,
4203 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
4204 break;
4205 case PPC440SPE_DMA1_ID:
4206 dma_set_maxpq(&adev->common,
4207 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
4208 break;
4209 case PPC440SPE_XOR_ID:
4210 adev->common.max_pq = XOR_MAX_OPS * 3;
4211 break;
4212 }
4213 adev->common.device_prep_dma_pq =
4214 ppc440spe_adma_prep_dma_pq;
4215 }
4216 if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
4217 switch (adev->id) {
4218 case PPC440SPE_DMA0_ID:
4219 adev->common.max_pq = DMA0_FIFO_SIZE /
4220 sizeof(struct dma_cdb);
4221 break;
4222 case PPC440SPE_DMA1_ID:
4223 adev->common.max_pq = DMA1_FIFO_SIZE /
4224 sizeof(struct dma_cdb);
4225 break;
4226 }
4227 adev->common.device_prep_dma_pq_val =
4228 ppc440spe_adma_prep_dma_pqzero_sum;
4229 }
4230 if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
4231 switch (adev->id) {
4232 case PPC440SPE_DMA0_ID:
4233 adev->common.max_xor = DMA0_FIFO_SIZE /
4234 sizeof(struct dma_cdb);
4235 break;
4236 case PPC440SPE_DMA1_ID:
4237 adev->common.max_xor = DMA1_FIFO_SIZE /
4238 sizeof(struct dma_cdb);
4239 break;
4240 }
4241 adev->common.device_prep_dma_xor_val =
4242 ppc440spe_adma_prep_dma_xor_zero_sum;
4243 }
4244 if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
4245 adev->common.device_prep_dma_interrupt =
4246 ppc440spe_adma_prep_dma_interrupt;
4247 }
4248 pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
4249 "( %s%s%s%s%s%s%s)\n",
4250 dev_name(adev->dev),
4251 dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
4252 dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
4253 dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
4254 dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
4255 dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
4256 dma_has_cap(DMA_MEMSET, adev->common.cap_mask) ? "memset " : "",
4257 dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
4258}
4259
4260static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
4261 struct ppc440spe_adma_chan *chan,
4262 int *initcode)
4263{
4264 struct device_node *np;
4265 int ret;
4266
4267 np = container_of(adev->dev, struct of_device, dev)->node;
4268 if (adev->id != PPC440SPE_XOR_ID) {
4269 adev->err_irq = irq_of_parse_and_map(np, 1);
4270 if (adev->err_irq == NO_IRQ) {
4271 dev_warn(adev->dev, "no err irq resource?\n");
4272 *initcode = PPC_ADMA_INIT_IRQ2;
4273 adev->err_irq = -ENXIO;
4274 } else
4275 atomic_inc(&ppc440spe_adma_err_irq_ref);
4276 } else {
4277 adev->err_irq = -ENXIO;
4278 }
4279
4280 adev->irq = irq_of_parse_and_map(np, 0);
4281 if (adev->irq == NO_IRQ) {
4282 dev_err(adev->dev, "no irq resource\n");
4283 *initcode = PPC_ADMA_INIT_IRQ1;
4284 ret = -ENXIO;
4285 goto err_irq_map;
4286 }
4287 dev_dbg(adev->dev, "irq %d, err irq %d\n",
4288 adev->irq, adev->err_irq);
4289
4290 ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
4291 0, dev_driver_string(adev->dev), chan);
4292 if (ret) {
4293 dev_err(adev->dev, "can't request irq %d\n",
4294 adev->irq);
4295 *initcode = PPC_ADMA_INIT_IRQ1;
4296 ret = -EIO;
4297 goto err_req1;
4298 }
4299
4300 /* only DMA engines have a separate error IRQ
4301 * so it's Ok if err_irq < 0 in XOR engine case.
4302 */
4303 if (adev->err_irq > 0) {
4304 /* both DMA engines share common error IRQ */
4305 ret = request_irq(adev->err_irq,
4306 ppc440spe_adma_err_handler,
4307 IRQF_SHARED,
4308 dev_driver_string(adev->dev),
4309 chan);
4310 if (ret) {
4311 dev_err(adev->dev, "can't request irq %d\n",
4312 adev->err_irq);
4313 *initcode = PPC_ADMA_INIT_IRQ2;
4314 ret = -EIO;
4315 goto err_req2;
4316 }
4317 }
4318
4319 if (adev->id == PPC440SPE_XOR_ID) {
4320 /* enable XOR engine interrupts */
4321 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4322 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
4323 &adev->xor_reg->ier);
4324 } else {
4325 u32 mask, enable;
4326
4327 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4328 if (!np) {
4329 pr_err("%s: can't find I2O device tree node\n",
4330 __func__);
4331 ret = -ENODEV;
4332 goto err_req2;
4333 }
4334 adev->i2o_reg = of_iomap(np, 0);
4335 if (!adev->i2o_reg) {
4336 pr_err("%s: failed to map I2O registers\n", __func__);
4337 of_node_put(np);
4338 ret = -EINVAL;
4339 goto err_req2;
4340 }
4341 of_node_put(np);
4342 /* Unmask 'CS FIFO Attention' interrupts and
4343 * enable generating interrupts on errors
4344 */
4345 enable = (adev->id == PPC440SPE_DMA0_ID) ?
4346 ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4347 ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4348 mask = ioread32(&adev->i2o_reg->iopim) & enable;
4349 iowrite32(mask, &adev->i2o_reg->iopim);
4350 }
4351 return 0;
4352
4353err_req2:
4354 free_irq(adev->irq, chan);
4355err_req1:
4356 irq_dispose_mapping(adev->irq);
4357err_irq_map:
4358 if (adev->err_irq > 0) {
4359 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
4360 irq_dispose_mapping(adev->err_irq);
4361 }
4362 return ret;
4363}
4364
4365static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
4366 struct ppc440spe_adma_chan *chan)
4367{
4368 u32 mask, disable;
4369
4370 if (adev->id == PPC440SPE_XOR_ID) {
4371 /* disable XOR engine interrupts */
4372 mask = ioread32be(&adev->xor_reg->ier);
4373 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4374 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
4375 iowrite32be(mask, &adev->xor_reg->ier);
4376 } else {
4377 /* disable DMAx engine interrupts */
4378 disable = (adev->id == PPC440SPE_DMA0_ID) ?
4379 (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4380 (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4381 mask = ioread32(&adev->i2o_reg->iopim) | disable;
4382 iowrite32(mask, &adev->i2o_reg->iopim);
4383 }
4384 free_irq(adev->irq, chan);
4385 irq_dispose_mapping(adev->irq);
4386 if (adev->err_irq > 0) {
4387 free_irq(adev->err_irq, chan);
4388 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
4389 irq_dispose_mapping(adev->err_irq);
4390 iounmap(adev->i2o_reg);
4391 }
4392 }
4393}
4394
4395/**
4396 * ppc440spe_adma_probe - probe the asynch device
4397 */
4398static int __devinit ppc440spe_adma_probe(struct of_device *ofdev,
4399 const struct of_device_id *match)
4400{
4401 struct device_node *np = ofdev->node;
4402 struct resource res;
4403 struct ppc440spe_adma_device *adev;
4404 struct ppc440spe_adma_chan *chan;
4405 struct ppc_dma_chan_ref *ref, *_ref;
4406 int ret = 0, initcode = PPC_ADMA_INIT_OK;
4407 const u32 *idx;
4408 int len;
4409 void *regs;
4410 u32 id, pool_size;
4411
4412 if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4413 id = PPC440SPE_XOR_ID;
4414 /* As far as the XOR engine is concerned, it does not
4415 * use FIFOs but uses linked list. So there is no dependency
4416 * between pool size to allocate and the engine configuration.
4417 */
4418 pool_size = PAGE_SIZE << 1;
4419 } else {
4420 /* it is DMA0 or DMA1 */
4421 idx = of_get_property(np, "cell-index", &len);
4422 if (!idx || (len != sizeof(u32))) {
4423 dev_err(&ofdev->dev, "Device node %s has missing "
4424 "or invalid cell-index property\n",
4425 np->full_name);
4426 return -EINVAL;
4427 }
4428 id = *idx;
4429 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4430 * should allocate the pool accordingly to size of this
4431 * FIFO. Thus, the pool size depends on the FIFO depth:
4432 * how much CDBs pointers the FIFO may contain then so
4433 * much CDBs we should provide in the pool.
4434 * That is
4435 * CDB size = 32B;
4436 * CDBs number = (DMA0_FIFO_SIZE >> 3);
4437 * Pool size = CDBs number * CDB size =
4438 * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4439 */
4440 pool_size = (id == PPC440SPE_DMA0_ID) ?
4441 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4442 pool_size <<= 2;
4443 }
4444
4445 if (of_address_to_resource(np, 0, &res)) {
4446 dev_err(&ofdev->dev, "failed to get memory resource\n");
4447 initcode = PPC_ADMA_INIT_MEMRES;
4448 ret = -ENODEV;
4449 goto out;
4450 }
4451
4452 if (!request_mem_region(res.start, resource_size(&res),
4453 dev_driver_string(&ofdev->dev))) {
4454 dev_err(&ofdev->dev, "failed to request memory region "
4455 "(0x%016llx-0x%016llx)\n",
4456 (u64)res.start, (u64)res.end);
4457 initcode = PPC_ADMA_INIT_MEMREG;
4458 ret = -EBUSY;
4459 goto out;
4460 }
4461
4462 /* create a device */
4463 adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4464 if (!adev) {
4465 dev_err(&ofdev->dev, "failed to allocate device\n");
4466 initcode = PPC_ADMA_INIT_ALLOC;
4467 ret = -ENOMEM;
4468 goto err_adev_alloc;
4469 }
4470
4471 adev->id = id;
4472 adev->pool_size = pool_size;
4473 /* allocate coherent memory for hardware descriptors */
4474 adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4475 adev->pool_size, &adev->dma_desc_pool,
4476 GFP_KERNEL);
4477 if (adev->dma_desc_pool_virt == NULL) {
4478 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4479 "memory for hardware descriptors\n",
4480 adev->pool_size);
4481 initcode = PPC_ADMA_INIT_COHERENT;
4482 ret = -ENOMEM;
4483 goto err_dma_alloc;
4484 }
4485 dev_dbg(&ofdev->dev, "allocted descriptor pool virt 0x%p phys 0x%llx\n",
4486 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4487
4488 regs = ioremap(res.start, resource_size(&res));
4489 if (!regs) {
4490 dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4491 goto err_regs_alloc;
4492 }
4493
4494 if (adev->id == PPC440SPE_XOR_ID) {
4495 adev->xor_reg = regs;
4496 /* Reset XOR */
4497 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4498 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4499 } else {
4500 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4501 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4502 adev->dma_reg = regs;
4503 /* DMAx_FIFO_SIZE is defined in bytes,
4504 * <fsiz> - is defined in number of CDB pointers (8byte).
4505 * DMA FIFO Length = CSlength + CPlength, where
4506 * CSlength = CPlength = (fsiz + 1) * 8.
4507 */
4508 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4509 &adev->dma_reg->fsiz);
4510 /* Configure DMA engine */
4511 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4512 &adev->dma_reg->cfg);
4513 /* Clear Status */
4514 iowrite32(~0, &adev->dma_reg->dsts);
4515 }
4516
4517 adev->dev = &ofdev->dev;
4518 adev->common.dev = &ofdev->dev;
4519 INIT_LIST_HEAD(&adev->common.channels);
4520 dev_set_drvdata(&ofdev->dev, adev);
4521
4522 /* create a channel */
4523 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4524 if (!chan) {
4525 dev_err(&ofdev->dev, "can't allocate channel structure\n");
4526 initcode = PPC_ADMA_INIT_CHANNEL;
4527 ret = -ENOMEM;
4528 goto err_chan_alloc;
4529 }
4530
4531 spin_lock_init(&chan->lock);
4532 INIT_LIST_HEAD(&chan->chain);
4533 INIT_LIST_HEAD(&chan->all_slots);
4534 chan->device = adev;
4535 chan->common.device = &adev->common;
4536 list_add_tail(&chan->common.device_node, &adev->common.channels);
4537 tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet,
4538 (unsigned long)chan);
4539
4540 /* allocate and map helper pages for async validation or
4541 * async_mult/async_sum_product operations on DMA0/1.
4542 */
4543 if (adev->id != PPC440SPE_XOR_ID) {
4544 chan->pdest_page = alloc_page(GFP_KERNEL);
4545 chan->qdest_page = alloc_page(GFP_KERNEL);
4546 if (!chan->pdest_page ||
4547 !chan->qdest_page) {
4548 if (chan->pdest_page)
4549 __free_page(chan->pdest_page);
4550 if (chan->qdest_page)
4551 __free_page(chan->qdest_page);
4552 ret = -ENOMEM;
4553 goto err_page_alloc;
4554 }
4555 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4556 PAGE_SIZE, DMA_BIDIRECTIONAL);
4557 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4558 PAGE_SIZE, DMA_BIDIRECTIONAL);
4559 }
4560
4561 ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4562 if (ref) {
4563 ref->chan = &chan->common;
4564 INIT_LIST_HEAD(&ref->node);
4565 list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4566 } else {
4567 dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4568 ret = -ENOMEM;
4569 goto err_ref_alloc;
4570 }
4571
4572 ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4573 if (ret)
4574 goto err_irq;
4575
4576 ppc440spe_adma_init_capabilities(adev);
4577
4578 ret = dma_async_device_register(&adev->common);
4579 if (ret) {
4580 initcode = PPC_ADMA_INIT_REGISTER;
4581 dev_err(&ofdev->dev, "failed to register dma device\n");
4582 goto err_dev_reg;
4583 }
4584
4585 goto out;
4586
4587err_dev_reg:
4588 ppc440spe_adma_release_irqs(adev, chan);
4589err_irq:
4590 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4591 if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4592 list_del(&ref->node);
4593 kfree(ref);
4594 }
4595 }
4596err_ref_alloc:
4597 if (adev->id != PPC440SPE_XOR_ID) {
4598 dma_unmap_page(&ofdev->dev, chan->pdest,
4599 PAGE_SIZE, DMA_BIDIRECTIONAL);
4600 dma_unmap_page(&ofdev->dev, chan->qdest,
4601 PAGE_SIZE, DMA_BIDIRECTIONAL);
4602 __free_page(chan->pdest_page);
4603 __free_page(chan->qdest_page);
4604 }
4605err_page_alloc:
4606 kfree(chan);
4607err_chan_alloc:
4608 if (adev->id == PPC440SPE_XOR_ID)
4609 iounmap(adev->xor_reg);
4610 else
4611 iounmap(adev->dma_reg);
4612err_regs_alloc:
4613 dma_free_coherent(adev->dev, adev->pool_size,
4614 adev->dma_desc_pool_virt,
4615 adev->dma_desc_pool);
4616err_dma_alloc:
4617 kfree(adev);
4618err_adev_alloc:
4619 release_mem_region(res.start, resource_size(&res));
4620out:
4621 if (id < PPC440SPE_ADMA_ENGINES_NUM)
4622 ppc440spe_adma_devices[id] = initcode;
4623
4624 return ret;
4625}
4626
4627/**
4628 * ppc440spe_adma_remove - remove the asynch device
4629 */
4630static int __devexit ppc440spe_adma_remove(struct of_device *ofdev)
4631{
4632 struct ppc440spe_adma_device *adev = dev_get_drvdata(&ofdev->dev);
4633 struct device_node *np = ofdev->node;
4634 struct resource res;
4635 struct dma_chan *chan, *_chan;
4636 struct ppc_dma_chan_ref *ref, *_ref;
4637 struct ppc440spe_adma_chan *ppc440spe_chan;
4638
4639 dev_set_drvdata(&ofdev->dev, NULL);
4640 if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4641 ppc440spe_adma_devices[adev->id] = -1;
4642
4643 dma_async_device_unregister(&adev->common);
4644
4645 list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4646 device_node) {
4647 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4648 ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4649 tasklet_kill(&ppc440spe_chan->irq_tasklet);
4650 if (adev->id != PPC440SPE_XOR_ID) {
4651 dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4652 PAGE_SIZE, DMA_BIDIRECTIONAL);
4653 dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4654 PAGE_SIZE, DMA_BIDIRECTIONAL);
4655 __free_page(ppc440spe_chan->pdest_page);
4656 __free_page(ppc440spe_chan->qdest_page);
4657 }
4658 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4659 node) {
4660 if (ppc440spe_chan ==
4661 to_ppc440spe_adma_chan(ref->chan)) {
4662 list_del(&ref->node);
4663 kfree(ref);
4664 }
4665 }
4666 list_del(&chan->device_node);
4667 kfree(ppc440spe_chan);
4668 }
4669
4670 dma_free_coherent(adev->dev, adev->pool_size,
4671 adev->dma_desc_pool_virt, adev->dma_desc_pool);
4672 if (adev->id == PPC440SPE_XOR_ID)
4673 iounmap(adev->xor_reg);
4674 else
4675 iounmap(adev->dma_reg);
4676 of_address_to_resource(np, 0, &res);
4677 release_mem_region(res.start, resource_size(&res));
4678 kfree(adev);
4679 return 0;
4680}
4681
4682/*
4683 * /sys driver interface to enable h/w RAID-6 capabilities
4684 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4685 * directory are "devices", "enable" and "poly".
4686 * "devices" shows available engines.
4687 * "enable" is used to enable RAID-6 capabilities or to check
4688 * whether these has been activated.
4689 * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4690 */
4691
4692static ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf)
4693{
4694 ssize_t size = 0;
4695 int i;
4696
4697 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4698 if (ppc440spe_adma_devices[i] == -1)
4699 continue;
4700 size += snprintf(buf + size, PAGE_SIZE - size,
4701 "PPC440SP(E)-ADMA.%d: %s\n", i,
4702 ppc_adma_errors[ppc440spe_adma_devices[i]]);
4703 }
4704 return size;
4705}
4706
4707static ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf)
4708{
4709 return snprintf(buf, PAGE_SIZE,
4710 "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4711 ppc440spe_r6_enabled ? "EN" : "DIS");
4712}
4713
4714static ssize_t store_ppc440spe_r6enable(struct device_driver *dev,
4715 const char *buf, size_t count)
4716{
4717 unsigned long val;
4718
4719 if (!count || count > 11)
4720 return -EINVAL;
4721
4722 if (!ppc440spe_r6_tchan)
4723 return -EFAULT;
4724
4725 /* Write a key */
4726 sscanf(buf, "%lx", &val);
4727 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4728 isync();
4729
4730 /* Verify whether it really works now */
4731 if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4732 pr_info("PPC440SP(e) RAID-6 has been activated "
4733 "successfully\n");
4734 ppc440spe_r6_enabled = 1;
4735 } else {
4736 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4737 " Error key ?\n");
4738 ppc440spe_r6_enabled = 0;
4739 }
4740 return count;
4741}
4742
4743static ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf)
4744{
4745 ssize_t size = 0;
4746 u32 reg;
4747
4748#ifdef CONFIG_440SP
4749 /* 440SP has fixed polynomial */
4750 reg = 0x4d;
4751#else
4752 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4753 reg >>= MQ0_CFBHL_POLY;
4754 reg &= 0xFF;
4755#endif
4756
4757 size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4758 "uses 0x1%02x polynomial.\n", reg);
4759 return size;
4760}
4761
4762static ssize_t store_ppc440spe_r6poly(struct device_driver *dev,
4763 const char *buf, size_t count)
4764{
4765 unsigned long reg, val;
4766
4767#ifdef CONFIG_440SP
4768 /* 440SP uses default 0x14D polynomial only */
4769 return -EINVAL;
4770#endif
4771
4772 if (!count || count > 6)
4773 return -EINVAL;
4774
4775 /* e.g., 0x14D or 0x11D */
4776 sscanf(buf, "%lx", &val);
4777
4778 if (val & ~0x1FF)
4779 return -EINVAL;
4780
4781 val &= 0xFF;
4782 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4783 reg &= ~(0xFF << MQ0_CFBHL_POLY);
4784 reg |= val << MQ0_CFBHL_POLY;
4785 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4786
4787 return count;
4788}
4789
4790static DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL);
4791static DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable,
4792 store_ppc440spe_r6enable);
4793static DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly,
4794 store_ppc440spe_r6poly);
4795
4796/*
4797 * Common initialisation for RAID engines; allocate memory for
4798 * DMAx FIFOs, perform configuration common for all DMA engines.
4799 * Further DMA engine specific configuration is done at probe time.
4800 */
4801static int ppc440spe_configure_raid_devices(void)
4802{
4803 struct device_node *np;
4804 struct resource i2o_res;
4805 struct i2o_regs __iomem *i2o_reg;
4806 dcr_host_t i2o_dcr_host;
4807 unsigned int dcr_base, dcr_len;
4808 int i, ret;
4809
4810 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4811 if (!np) {
4812 pr_err("%s: can't find I2O device tree node\n",
4813 __func__);
4814 return -ENODEV;
4815 }
4816
4817 if (of_address_to_resource(np, 0, &i2o_res)) {
4818 of_node_put(np);
4819 return -EINVAL;
4820 }
4821
4822 i2o_reg = of_iomap(np, 0);
4823 if (!i2o_reg) {
4824 pr_err("%s: failed to map I2O registers\n", __func__);
4825 of_node_put(np);
4826 return -EINVAL;
4827 }
4828
4829 /* Get I2O DCRs base */
4830 dcr_base = dcr_resource_start(np, 0);
4831 dcr_len = dcr_resource_len(np, 0);
4832 if (!dcr_base && !dcr_len) {
4833 pr_err("%s: can't get DCR registers base/len!\n",
4834 np->full_name);
4835 of_node_put(np);
4836 iounmap(i2o_reg);
4837 return -ENODEV;
4838 }
4839
4840 i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4841 if (!DCR_MAP_OK(i2o_dcr_host)) {
4842 pr_err("%s: failed to map DCRs!\n", np->full_name);
4843 of_node_put(np);
4844 iounmap(i2o_reg);
4845 return -ENODEV;
4846 }
4847 of_node_put(np);
4848
4849 /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4850 * the base address of FIFO memory space.
4851 * Actually we need twice more physical memory than programmed in the
4852 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4853 */
4854 ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4855 GFP_KERNEL);
4856 if (!ppc440spe_dma_fifo_buf) {
4857 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4858 iounmap(i2o_reg);
4859 dcr_unmap(i2o_dcr_host, dcr_len);
4860 return -ENOMEM;
4861 }
4862
4863 /*
4864 * Configure h/w
4865 */
4866 /* Reset I2O/DMA */
4867 mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4868 mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4869
4870 /* Setup the base address of mmaped registers */
4871 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4872 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4873 I2O_REG_ENABLE);
4874 dcr_unmap(i2o_dcr_host, dcr_len);
4875
4876 /* Setup FIFO memory space base address */
4877 iowrite32(0, &i2o_reg->ifbah);
4878 iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4879
4880 /* set zero FIFO size for I2O, so the whole
4881 * ppc440spe_dma_fifo_buf is used by DMAs.
4882 * DMAx_FIFOs will be configured while probe.
4883 */
4884 iowrite32(0, &i2o_reg->ifsiz);
4885 iounmap(i2o_reg);
4886
4887 /* To prepare WXOR/RXOR functionality we need access to
4888 * Memory Queue Module DCRs (finally it will be enabled
4889 * via /sys interface of the ppc440spe ADMA driver).
4890 */
4891 np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4892 if (!np) {
4893 pr_err("%s: can't find MQ device tree node\n",
4894 __func__);
4895 ret = -ENODEV;
4896 goto out_free;
4897 }
4898
4899 /* Get MQ DCRs base */
4900 dcr_base = dcr_resource_start(np, 0);
4901 dcr_len = dcr_resource_len(np, 0);
4902 if (!dcr_base && !dcr_len) {
4903 pr_err("%s: can't get DCR registers base/len!\n",
4904 np->full_name);
4905 ret = -ENODEV;
4906 goto out_mq;
4907 }
4908
4909 ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4910 if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4911 pr_err("%s: failed to map DCRs!\n", np->full_name);
4912 ret = -ENODEV;
4913 goto out_mq;
4914 }
4915 of_node_put(np);
4916 ppc440spe_mq_dcr_len = dcr_len;
4917
4918 /* Set HB alias */
4919 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4920
4921 /* Set:
4922 * - LL transaction passing limit to 1;
4923 * - Memory controller cycle limit to 1;
4924 * - Galois Polynomial to 0x14d (default)
4925 */
4926 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4927 (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4928 (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4929
4930 atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4931 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4932 ppc440spe_adma_devices[i] = -1;
4933
4934 return 0;
4935
4936out_mq:
4937 of_node_put(np);
4938out_free:
4939 kfree(ppc440spe_dma_fifo_buf);
4940 return ret;
4941}
4942
4943static struct of_device_id __devinitdata ppc440spe_adma_of_match[] = {
4944 { .compatible = "ibm,dma-440spe", },
4945 { .compatible = "amcc,xor-accelerator", },
4946 {},
4947};
4948MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4949
4950static struct of_platform_driver ppc440spe_adma_driver = {
4951 .match_table = ppc440spe_adma_of_match,
4952 .probe = ppc440spe_adma_probe,
4953 .remove = __devexit_p(ppc440spe_adma_remove),
4954 .driver = {
4955 .name = "PPC440SP(E)-ADMA",
4956 .owner = THIS_MODULE,
4957 },
4958};
4959
4960static __init int ppc440spe_adma_init(void)
4961{
4962 int ret;
4963
4964 ret = ppc440spe_configure_raid_devices();
4965 if (ret)
4966 return ret;
4967
4968 ret = of_register_platform_driver(&ppc440spe_adma_driver);
4969 if (ret) {
4970 pr_err("%s: failed to register platform driver\n",
4971 __func__);
4972 goto out_reg;
4973 }
4974
4975 /* Initialization status */
4976 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4977 &driver_attr_devices);
4978 if (ret)
4979 goto out_dev;
4980
4981 /* RAID-6 h/w enable entry */
4982 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4983 &driver_attr_enable);
4984 if (ret)
4985 goto out_en;
4986
4987 /* GF polynomial to use */
4988 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4989 &driver_attr_poly);
4990 if (!ret)
4991 return ret;
4992
4993 driver_remove_file(&ppc440spe_adma_driver.driver,
4994 &driver_attr_enable);
4995out_en:
4996 driver_remove_file(&ppc440spe_adma_driver.driver,
4997 &driver_attr_devices);
4998out_dev:
4999 /* User will not be able to enable h/w RAID-6 */
5000 pr_err("%s: failed to create RAID-6 driver interface\n",
5001 __func__);
5002 of_unregister_platform_driver(&ppc440spe_adma_driver);
5003out_reg:
5004 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
5005 kfree(ppc440spe_dma_fifo_buf);
5006 return ret;
5007}
5008
5009static void __exit ppc440spe_adma_exit(void)
5010{
5011 driver_remove_file(&ppc440spe_adma_driver.driver,
5012 &driver_attr_poly);
5013 driver_remove_file(&ppc440spe_adma_driver.driver,
5014 &driver_attr_enable);
5015 driver_remove_file(&ppc440spe_adma_driver.driver,
5016 &driver_attr_devices);
5017 of_unregister_platform_driver(&ppc440spe_adma_driver);
5018 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
5019 kfree(ppc440spe_dma_fifo_buf);
5020}
5021
5022arch_initcall(ppc440spe_adma_init);
5023module_exit(ppc440spe_adma_exit);
5024
5025MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
5026MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
5027MODULE_LICENSE("GPL");
diff --git a/drivers/dma/ppc4xx/adma.h b/drivers/dma/ppc4xx/adma.h
new file mode 100644
index 00000000000..8ada5a812e3
--- /dev/null
+++ b/drivers/dma/ppc4xx/adma.h
@@ -0,0 +1,195 @@
1/*
2 * 2006-2009 (C) DENX Software Engineering.
3 *
4 * Author: Yuri Tikhonov <yur@emcraft.com>
5 *
6 * This file is licensed under the terms of the GNU General Public License
7 * version 2. This program is licensed "as is" without any warranty of
8 * any kind, whether express or implied.
9 */
10
11#ifndef _PPC440SPE_ADMA_H
12#define _PPC440SPE_ADMA_H
13
14#include <linux/types.h>
15#include "dma.h"
16#include "xor.h"
17
18#define to_ppc440spe_adma_chan(chan) \
19 container_of(chan, struct ppc440spe_adma_chan, common)
20#define to_ppc440spe_adma_device(dev) \
21 container_of(dev, struct ppc440spe_adma_device, common)
22#define tx_to_ppc440spe_adma_slot(tx) \
23 container_of(tx, struct ppc440spe_adma_desc_slot, async_tx)
24
25/* Default polynomial (for 440SP is only available) */
26#define PPC440SPE_DEFAULT_POLY 0x4d
27
28#define PPC440SPE_ADMA_ENGINES_NUM (XOR_ENGINES_NUM + DMA_ENGINES_NUM)
29
30#define PPC440SPE_ADMA_WATCHDOG_MSEC 3
31#define PPC440SPE_ADMA_THRESHOLD 1
32
33#define PPC440SPE_DMA0_ID 0
34#define PPC440SPE_DMA1_ID 1
35#define PPC440SPE_XOR_ID 2
36
37#define PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT 0xFFFFFFUL
38/* this is the XOR_CBBCR width */
39#define PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT (1 << 31)
40#define PPC440SPE_ADMA_ZERO_SUM_MAX_BYTE_COUNT PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT
41
42#define PPC440SPE_RXOR_RUN 0
43
44#define MQ0_CF2H_RXOR_BS_MASK 0x1FF
45
46#undef ADMA_LL_DEBUG
47
48/**
49 * struct ppc440spe_adma_device - internal representation of an ADMA device
50 * @dev: device
51 * @dma_reg: base for DMAx register access
52 * @xor_reg: base for XOR register access
53 * @i2o_reg: base for I2O register access
54 * @id: HW ADMA Device selector
55 * @dma_desc_pool_virt: base of DMA descriptor region (CPU address)
56 * @dma_desc_pool: base of DMA descriptor region (DMA address)
57 * @pool_size: size of the pool
58 * @irq: DMAx or XOR irq number
59 * @err_irq: DMAx error irq number
60 * @common: embedded struct dma_device
61 */
62struct ppc440spe_adma_device {
63 struct device *dev;
64 struct dma_regs __iomem *dma_reg;
65 struct xor_regs __iomem *xor_reg;
66 struct i2o_regs __iomem *i2o_reg;
67 int id;
68 void *dma_desc_pool_virt;
69 dma_addr_t dma_desc_pool;
70 size_t pool_size;
71 int irq;
72 int err_irq;
73 struct dma_device common;
74};
75
76/**
77 * struct ppc440spe_adma_chan - internal representation of an ADMA channel
78 * @lock: serializes enqueue/dequeue operations to the slot pool
79 * @device: parent device
80 * @chain: device chain view of the descriptors
81 * @common: common dmaengine channel object members
82 * @all_slots: complete domain of slots usable by the channel
83 * @pending: allows batching of hardware operations
84 * @completed_cookie: identifier for the most recently completed operation
85 * @slots_allocated: records the actual size of the descriptor slot pool
86 * @hw_chain_inited: h/w descriptor chain initialization flag
87 * @irq_tasklet: bottom half where ppc440spe_adma_slot_cleanup runs
88 * @needs_unmap: if buffers should not be unmapped upon final processing
89 * @pdest_page: P destination page for async validate operation
90 * @qdest_page: Q destination page for async validate operation
91 * @pdest: P dma addr for async validate operation
92 * @qdest: Q dma addr for async validate operation
93 */
94struct ppc440spe_adma_chan {
95 spinlock_t lock;
96 struct ppc440spe_adma_device *device;
97 struct list_head chain;
98 struct dma_chan common;
99 struct list_head all_slots;
100 struct ppc440spe_adma_desc_slot *last_used;
101 int pending;
102 dma_cookie_t completed_cookie;
103 int slots_allocated;
104 int hw_chain_inited;
105 struct tasklet_struct irq_tasklet;
106 u8 needs_unmap;
107 struct page *pdest_page;
108 struct page *qdest_page;
109 dma_addr_t pdest;
110 dma_addr_t qdest;
111};
112
113struct ppc440spe_rxor {
114 u32 addrl;
115 u32 addrh;
116 int len;
117 int xor_count;
118 int addr_count;
119 int desc_count;
120 int state;
121};
122
123/**
124 * struct ppc440spe_adma_desc_slot - PPC440SPE-ADMA software descriptor
125 * @phys: hardware address of the hardware descriptor chain
126 * @group_head: first operation in a transaction
127 * @hw_next: pointer to the next descriptor in chain
128 * @async_tx: support for the async_tx api
129 * @slot_node: node on the iop_adma_chan.all_slots list
130 * @chain_node: node on the op_adma_chan.chain list
131 * @group_list: list of slots that make up a multi-descriptor transaction
132 * for example transfer lengths larger than the supported hw max
133 * @unmap_len: transaction bytecount
134 * @hw_desc: virtual address of the hardware descriptor chain
135 * @stride: currently chained or not
136 * @idx: pool index
137 * @slot_cnt: total slots used in an transaction (group of operations)
138 * @src_cnt: number of sources set in this descriptor
139 * @dst_cnt: number of destinations set in the descriptor
140 * @slots_per_op: number of slots per operation
141 * @descs_per_op: number of slot per P/Q operation see comment
142 * for ppc440spe_prep_dma_pqxor function
143 * @flags: desc state/type
144 * @reverse_flags: 1 if a corresponding rxor address uses reversed address order
145 * @xor_check_result: result of zero sum
146 * @crc32_result: result crc calculation
147 */
148struct ppc440spe_adma_desc_slot {
149 dma_addr_t phys;
150 struct ppc440spe_adma_desc_slot *group_head;
151 struct ppc440spe_adma_desc_slot *hw_next;
152 struct dma_async_tx_descriptor async_tx;
153 struct list_head slot_node;
154 struct list_head chain_node; /* node in channel ops list */
155 struct list_head group_list; /* list */
156 unsigned int unmap_len;
157 void *hw_desc;
158 u16 stride;
159 u16 idx;
160 u16 slot_cnt;
161 u8 src_cnt;
162 u8 dst_cnt;
163 u8 slots_per_op;
164 u8 descs_per_op;
165 unsigned long flags;
166 unsigned long reverse_flags[8];
167
168#define PPC440SPE_DESC_INT 0 /* generate interrupt on complete */
169#define PPC440SPE_ZERO_P 1 /* clear P destionaion */
170#define PPC440SPE_ZERO_Q 2 /* clear Q destination */
171#define PPC440SPE_COHERENT 3 /* src/dst are coherent */
172
173#define PPC440SPE_DESC_WXOR 4 /* WXORs are in chain */
174#define PPC440SPE_DESC_RXOR 5 /* RXOR is in chain */
175
176#define PPC440SPE_DESC_RXOR123 8 /* CDB for RXOR123 operation */
177#define PPC440SPE_DESC_RXOR124 9 /* CDB for RXOR124 operation */
178#define PPC440SPE_DESC_RXOR125 10 /* CDB for RXOR125 operation */
179#define PPC440SPE_DESC_RXOR12 11 /* CDB for RXOR12 operation */
180#define PPC440SPE_DESC_RXOR_REV 12 /* CDB has srcs in reversed order */
181
182#define PPC440SPE_DESC_PCHECK 13
183#define PPC440SPE_DESC_QCHECK 14
184
185#define PPC440SPE_DESC_RXOR_MSK 0x3
186
187 struct ppc440spe_rxor rxor_cursor;
188
189 union {
190 u32 *xor_check_result;
191 u32 *crc32_result;
192 };
193};
194
195#endif /* _PPC440SPE_ADMA_H */
diff --git a/drivers/dma/ppc4xx/dma.h b/drivers/dma/ppc4xx/dma.h
new file mode 100644
index 00000000000..bcde2df2f37
--- /dev/null
+++ b/drivers/dma/ppc4xx/dma.h
@@ -0,0 +1,223 @@
1/*
2 * 440SPe's DMA engines support header file
3 *
4 * 2006-2009 (C) DENX Software Engineering.
5 *
6 * Author: Yuri Tikhonov <yur@emcraft.com>
7 *
8 * This file is licensed under the term of the GNU General Public License
9 * version 2. The program licensed "as is" without any warranty of any
10 * kind, whether express or implied.
11 */
12
13#ifndef _PPC440SPE_DMA_H
14#define _PPC440SPE_DMA_H
15
16#include <linux/types.h>
17
18/* Number of elements in the array with statical CDBs */
19#define MAX_STAT_DMA_CDBS 16
20/* Number of DMA engines available on the contoller */
21#define DMA_ENGINES_NUM 2
22
23/* Maximum h/w supported number of destinations */
24#define DMA_DEST_MAX_NUM 2
25
26/* FIFO's params */
27#define DMA0_FIFO_SIZE 0x1000
28#define DMA1_FIFO_SIZE 0x1000
29#define DMA_FIFO_ENABLE (1<<12)
30
31/* DMA Configuration Register. Data Transfer Engine PLB Priority: */
32#define DMA_CFG_DXEPR_LP (0<<26)
33#define DMA_CFG_DXEPR_HP (3<<26)
34#define DMA_CFG_DXEPR_HHP (2<<26)
35#define DMA_CFG_DXEPR_HHHP (1<<26)
36
37/* DMA Configuration Register. DMA FIFO Manager PLB Priority: */
38#define DMA_CFG_DFMPP_LP (0<<23)
39#define DMA_CFG_DFMPP_HP (3<<23)
40#define DMA_CFG_DFMPP_HHP (2<<23)
41#define DMA_CFG_DFMPP_HHHP (1<<23)
42
43/* DMA Configuration Register. Force 64-byte Alignment */
44#define DMA_CFG_FALGN (1 << 19)
45
46/*UIC0:*/
47#define D0CPF_INT (1<<12)
48#define D0CSF_INT (1<<11)
49#define D1CPF_INT (1<<10)
50#define D1CSF_INT (1<<9)
51/*UIC1:*/
52#define DMAE_INT (1<<9)
53
54/* I2O IOP Interrupt Mask Register */
55#define I2O_IOPIM_P0SNE (1<<3)
56#define I2O_IOPIM_P0EM (1<<5)
57#define I2O_IOPIM_P1SNE (1<<6)
58#define I2O_IOPIM_P1EM (1<<8)
59
60/* DMA CDB fields */
61#define DMA_CDB_MSK (0xF)
62#define DMA_CDB_64B_ADDR (1<<2)
63#define DMA_CDB_NO_INT (1<<3)
64#define DMA_CDB_STATUS_MSK (0x3)
65#define DMA_CDB_ADDR_MSK (0xFFFFFFF0)
66
67/* DMA CDB OpCodes */
68#define DMA_CDB_OPC_NO_OP (0x00)
69#define DMA_CDB_OPC_MV_SG1_SG2 (0x01)
70#define DMA_CDB_OPC_MULTICAST (0x05)
71#define DMA_CDB_OPC_DFILL128 (0x24)
72#define DMA_CDB_OPC_DCHECK128 (0x23)
73
74#define DMA_CUED_XOR_BASE (0x10000000)
75#define DMA_CUED_XOR_HB (0x00000008)
76
77#ifdef CONFIG_440SP
78#define DMA_CUED_MULT1_OFF 0
79#define DMA_CUED_MULT2_OFF 8
80#define DMA_CUED_MULT3_OFF 16
81#define DMA_CUED_REGION_OFF 24
82#define DMA_CUED_XOR_WIN_MSK (0xFC000000)
83#else
84#define DMA_CUED_MULT1_OFF 2
85#define DMA_CUED_MULT2_OFF 10
86#define DMA_CUED_MULT3_OFF 18
87#define DMA_CUED_REGION_OFF 26
88#define DMA_CUED_XOR_WIN_MSK (0xF0000000)
89#endif
90
91#define DMA_CUED_REGION_MSK 0x3
92#define DMA_RXOR123 0x0
93#define DMA_RXOR124 0x1
94#define DMA_RXOR125 0x2
95#define DMA_RXOR12 0x3
96
97/* S/G addresses */
98#define DMA_CDB_SG_SRC 1
99#define DMA_CDB_SG_DST1 2
100#define DMA_CDB_SG_DST2 3
101
102/*
103 * DMAx engines Command Descriptor Block Type
104 */
105struct dma_cdb {
106 /*
107 * Basic CDB structure (Table 20-17, p.499, 440spe_um_1_22.pdf)
108 */
109 u8 pad0[2]; /* reserved */
110 u8 attr; /* attributes */
111 u8 opc; /* opcode */
112 u32 sg1u; /* upper SG1 address */
113 u32 sg1l; /* lower SG1 address */
114 u32 cnt; /* SG count, 3B used */
115 u32 sg2u; /* upper SG2 address */
116 u32 sg2l; /* lower SG2 address */
117 u32 sg3u; /* upper SG3 address */
118 u32 sg3l; /* lower SG3 address */
119};
120
121/*
122 * DMAx hardware registers (p.515 in 440SPe UM 1.22)
123 */
124struct dma_regs {
125 u32 cpfpl;
126 u32 cpfph;
127 u32 csfpl;
128 u32 csfph;
129 u32 dsts;
130 u32 cfg;
131 u8 pad0[0x8];
132 u16 cpfhp;
133 u16 cpftp;
134 u16 csfhp;
135 u16 csftp;
136 u8 pad1[0x8];
137 u32 acpl;
138 u32 acph;
139 u32 s1bpl;
140 u32 s1bph;
141 u32 s2bpl;
142 u32 s2bph;
143 u32 s3bpl;
144 u32 s3bph;
145 u8 pad2[0x10];
146 u32 earl;
147 u32 earh;
148 u8 pad3[0x8];
149 u32 seat;
150 u32 sead;
151 u32 op;
152 u32 fsiz;
153};
154
155/*
156 * I2O hardware registers (p.528 in 440SPe UM 1.22)
157 */
158struct i2o_regs {
159 u32 ists;
160 u32 iseat;
161 u32 isead;
162 u8 pad0[0x14];
163 u32 idbel;
164 u8 pad1[0xc];
165 u32 ihis;
166 u32 ihim;
167 u8 pad2[0x8];
168 u32 ihiq;
169 u32 ihoq;
170 u8 pad3[0x8];
171 u32 iopis;
172 u32 iopim;
173 u32 iopiq;
174 u8 iopoq;
175 u8 pad4[3];
176 u16 iiflh;
177 u16 iiflt;
178 u16 iiplh;
179 u16 iiplt;
180 u16 ioflh;
181 u16 ioflt;
182 u16 ioplh;
183 u16 ioplt;
184 u32 iidc;
185 u32 ictl;
186 u32 ifcpp;
187 u8 pad5[0x4];
188 u16 mfac0;
189 u16 mfac1;
190 u16 mfac2;
191 u16 mfac3;
192 u16 mfac4;
193 u16 mfac5;
194 u16 mfac6;
195 u16 mfac7;
196 u16 ifcfh;
197 u16 ifcht;
198 u8 pad6[0x4];
199 u32 iifmc;
200 u32 iodb;
201 u32 iodbc;
202 u32 ifbal;
203 u32 ifbah;
204 u32 ifsiz;
205 u32 ispd0;
206 u32 ispd1;
207 u32 ispd2;
208 u32 ispd3;
209 u32 ihipl;
210 u32 ihiph;
211 u32 ihopl;
212 u32 ihoph;
213 u32 iiipl;
214 u32 iiiph;
215 u32 iiopl;
216 u32 iioph;
217 u32 ifcpl;
218 u32 ifcph;
219 u8 pad7[0x8];
220 u32 iopt;
221};
222
223#endif /* _PPC440SPE_DMA_H */
diff --git a/drivers/dma/ppc4xx/xor.h b/drivers/dma/ppc4xx/xor.h
new file mode 100644
index 00000000000..daed7384daa
--- /dev/null
+++ b/drivers/dma/ppc4xx/xor.h
@@ -0,0 +1,110 @@
1/*
2 * 440SPe's XOR engines support header file
3 *
4 * 2006-2009 (C) DENX Software Engineering.
5 *
6 * Author: Yuri Tikhonov <yur@emcraft.com>
7 *
8 * This file is licensed under the term of the GNU General Public License
9 * version 2. The program licensed "as is" without any warranty of any
10 * kind, whether express or implied.
11 */
12
13#ifndef _PPC440SPE_XOR_H
14#define _PPC440SPE_XOR_H
15
16#include <linux/types.h>
17
18/* Number of XOR engines available on the contoller */
19#define XOR_ENGINES_NUM 1
20
21/* Number of operands supported in the h/w */
22#define XOR_MAX_OPS 16
23
24/*
25 * XOR Command Block Control Register bits
26 */
27#define XOR_CBCR_LNK_BIT (1<<31) /* link present */
28#define XOR_CBCR_TGT_BIT (1<<30) /* target present */
29#define XOR_CBCR_CBCE_BIT (1<<29) /* command block compete enable */
30#define XOR_CBCR_RNZE_BIT (1<<28) /* result not zero enable */
31#define XOR_CBCR_XNOR_BIT (1<<15) /* XOR/XNOR */
32#define XOR_CDCR_OAC_MSK (0x7F) /* operand address count */
33
34/*
35 * XORCore Status Register bits
36 */
37#define XOR_SR_XCP_BIT (1<<31) /* core processing */
38#define XOR_SR_ICB_BIT (1<<17) /* invalid CB */
39#define XOR_SR_IC_BIT (1<<16) /* invalid command */
40#define XOR_SR_IPE_BIT (1<<15) /* internal parity error */
41#define XOR_SR_RNZ_BIT (1<<2) /* result not Zero */
42#define XOR_SR_CBC_BIT (1<<1) /* CB complete */
43#define XOR_SR_CBLC_BIT (1<<0) /* CB list complete */
44
45/*
46 * XORCore Control Set and Reset Register bits
47 */
48#define XOR_CRSR_XASR_BIT (1<<31) /* soft reset */
49#define XOR_CRSR_XAE_BIT (1<<30) /* enable */
50#define XOR_CRSR_RCBE_BIT (1<<29) /* refetch CB enable */
51#define XOR_CRSR_PAUS_BIT (1<<28) /* pause */
52#define XOR_CRSR_64BA_BIT (1<<27) /* 64/32 CB format */
53#define XOR_CRSR_CLP_BIT (1<<25) /* continue list processing */
54
55/*
56 * XORCore Interrupt Enable Register
57 */
58#define XOR_IE_ICBIE_BIT (1<<17) /* Invalid Command Block IRQ Enable */
59#define XOR_IE_ICIE_BIT (1<<16) /* Invalid Command IRQ Enable */
60#define XOR_IE_RPTIE_BIT (1<<14) /* Read PLB Timeout Error IRQ Enable */
61#define XOR_IE_CBCIE_BIT (1<<1) /* CB complete interrupt enable */
62#define XOR_IE_CBLCI_BIT (1<<0) /* CB list complete interrupt enable */
63
64/*
65 * XOR Accelerator engine Command Block Type
66 */
67struct xor_cb {
68 /*
69 * Basic 64-bit format XOR CB (Table 19-1, p.463, 440spe_um_1_22.pdf)
70 */
71 u32 cbc; /* control */
72 u32 cbbc; /* byte count */
73 u32 cbs; /* status */
74 u8 pad0[4]; /* reserved */
75 u32 cbtah; /* target address high */
76 u32 cbtal; /* target address low */
77 u32 cblah; /* link address high */
78 u32 cblal; /* link address low */
79 struct {
80 u32 h;
81 u32 l;
82 } __attribute__ ((packed)) ops[16];
83} __attribute__ ((packed));
84
85/*
86 * XOR hardware registers Table 19-3, UM 1.22
87 */
88struct xor_regs {
89 u32 op_ar[16][2]; /* operand address[0]-high,[1]-low registers */
90 u8 pad0[352]; /* reserved */
91 u32 cbcr; /* CB control register */
92 u32 cbbcr; /* CB byte count register */
93 u32 cbsr; /* CB status register */
94 u8 pad1[4]; /* reserved */
95 u32 cbtahr; /* operand target address high register */
96 u32 cbtalr; /* operand target address low register */
97 u32 cblahr; /* CB link address high register */
98 u32 cblalr; /* CB link address low register */
99 u32 crsr; /* control set register */
100 u32 crrr; /* control reset register */
101 u32 ccbahr; /* current CB address high register */
102 u32 ccbalr; /* current CB address low register */
103 u32 plbr; /* PLB configuration register */
104 u32 ier; /* interrupt enable register */
105 u32 pecr; /* parity error count register */
106 u32 sr; /* status register */
107 u32 revidr; /* revision ID register */
108};
109
110#endif /* _PPC440SPE_XOR_H */
diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c
index 034ecf0ace0..2e4a54c8afe 100644
--- a/drivers/dma/shdma.c
+++ b/drivers/dma/shdma.c
@@ -80,17 +80,17 @@ static int sh_dmae_rst(int id)
80 unsigned short dmaor; 80 unsigned short dmaor;
81 81
82 sh_dmae_ctl_stop(id); 82 sh_dmae_ctl_stop(id);
83 dmaor = (dmaor_read_reg(id)|DMAOR_INIT); 83 dmaor = dmaor_read_reg(id) | DMAOR_INIT;
84 84
85 dmaor_write_reg(id, dmaor); 85 dmaor_write_reg(id, dmaor);
86 if ((dmaor_read_reg(id) & (DMAOR_AE | DMAOR_NMIF))) { 86 if (dmaor_read_reg(id) & (DMAOR_AE | DMAOR_NMIF)) {
87 pr_warning(KERN_ERR "dma-sh: Can't initialize DMAOR.\n"); 87 pr_warning(KERN_ERR "dma-sh: Can't initialize DMAOR.\n");
88 return -EINVAL; 88 return -EINVAL;
89 } 89 }
90 return 0; 90 return 0;
91} 91}
92 92
93static int dmae_is_idle(struct sh_dmae_chan *sh_chan) 93static int dmae_is_busy(struct sh_dmae_chan *sh_chan)
94{ 94{
95 u32 chcr = sh_dmae_readl(sh_chan, CHCR); 95 u32 chcr = sh_dmae_readl(sh_chan, CHCR);
96 if (chcr & CHCR_DE) { 96 if (chcr & CHCR_DE) {
@@ -110,15 +110,14 @@ static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs hw)
110{ 110{
111 sh_dmae_writel(sh_chan, hw.sar, SAR); 111 sh_dmae_writel(sh_chan, hw.sar, SAR);
112 sh_dmae_writel(sh_chan, hw.dar, DAR); 112 sh_dmae_writel(sh_chan, hw.dar, DAR);
113 sh_dmae_writel(sh_chan, 113 sh_dmae_writel(sh_chan, hw.tcr >> calc_xmit_shift(sh_chan), TCR);
114 (hw.tcr >> calc_xmit_shift(sh_chan)), TCR);
115} 114}
116 115
117static void dmae_start(struct sh_dmae_chan *sh_chan) 116static void dmae_start(struct sh_dmae_chan *sh_chan)
118{ 117{
119 u32 chcr = sh_dmae_readl(sh_chan, CHCR); 118 u32 chcr = sh_dmae_readl(sh_chan, CHCR);
120 119
121 chcr |= (CHCR_DE|CHCR_IE); 120 chcr |= CHCR_DE | CHCR_IE;
122 sh_dmae_writel(sh_chan, chcr, CHCR); 121 sh_dmae_writel(sh_chan, chcr, CHCR);
123} 122}
124 123
@@ -132,7 +131,7 @@ static void dmae_halt(struct sh_dmae_chan *sh_chan)
132 131
133static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val) 132static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val)
134{ 133{
135 int ret = dmae_is_idle(sh_chan); 134 int ret = dmae_is_busy(sh_chan);
136 /* When DMA was working, can not set data to CHCR */ 135 /* When DMA was working, can not set data to CHCR */
137 if (ret) 136 if (ret)
138 return ret; 137 return ret;
@@ -149,7 +148,7 @@ static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val)
149{ 148{
150 u32 addr; 149 u32 addr;
151 int shift = 0; 150 int shift = 0;
152 int ret = dmae_is_idle(sh_chan); 151 int ret = dmae_is_busy(sh_chan);
153 if (ret) 152 if (ret)
154 return ret; 153 return ret;
155 154
@@ -307,7 +306,7 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy(
307 new = sh_dmae_get_desc(sh_chan); 306 new = sh_dmae_get_desc(sh_chan);
308 if (!new) { 307 if (!new) {
309 dev_err(sh_chan->dev, 308 dev_err(sh_chan->dev,
310 "No free memory for link descriptor\n"); 309 "No free memory for link descriptor\n");
311 goto err_get_desc; 310 goto err_get_desc;
312 } 311 }
313 312
@@ -388,7 +387,7 @@ static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan)
388 struct sh_dmae_regs hw; 387 struct sh_dmae_regs hw;
389 388
390 /* DMA work check */ 389 /* DMA work check */
391 if (dmae_is_idle(sh_chan)) 390 if (dmae_is_busy(sh_chan))
392 return; 391 return;
393 392
394 /* Find the first un-transfer desciptor */ 393 /* Find the first un-transfer desciptor */
@@ -497,8 +496,9 @@ static void dmae_do_tasklet(unsigned long data)
497 struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data; 496 struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
498 struct sh_desc *desc, *_desc, *cur_desc = NULL; 497 struct sh_desc *desc, *_desc, *cur_desc = NULL;
499 u32 sar_buf = sh_dmae_readl(sh_chan, SAR); 498 u32 sar_buf = sh_dmae_readl(sh_chan, SAR);
499
500 list_for_each_entry_safe(desc, _desc, 500 list_for_each_entry_safe(desc, _desc,
501 &sh_chan->ld_queue, node) { 501 &sh_chan->ld_queue, node) {
502 if ((desc->hw.sar + desc->hw.tcr) == sar_buf) { 502 if ((desc->hw.sar + desc->hw.tcr) == sar_buf) {
503 cur_desc = desc; 503 cur_desc = desc;
504 break; 504 break;
@@ -543,8 +543,8 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id)
543 /* alloc channel */ 543 /* alloc channel */
544 new_sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL); 544 new_sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL);
545 if (!new_sh_chan) { 545 if (!new_sh_chan) {
546 dev_err(shdev->common.dev, "No free memory for allocating " 546 dev_err(shdev->common.dev,
547 "dma channels!\n"); 547 "No free memory for allocating dma channels!\n");
548 return -ENOMEM; 548 return -ENOMEM;
549 } 549 }
550 550
@@ -586,8 +586,8 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id)
586 "sh-dmae%d", new_sh_chan->id); 586 "sh-dmae%d", new_sh_chan->id);
587 587
588 /* set up channel irq */ 588 /* set up channel irq */
589 err = request_irq(irq, &sh_dmae_interrupt, 589 err = request_irq(irq, &sh_dmae_interrupt, irqflags,
590 irqflags, new_sh_chan->dev_id, new_sh_chan); 590 new_sh_chan->dev_id, new_sh_chan);
591 if (err) { 591 if (err) {
592 dev_err(shdev->common.dev, "DMA channel %d request_irq error " 592 dev_err(shdev->common.dev, "DMA channel %d request_irq error "
593 "with return %d\n", id, err); 593 "with return %d\n", id, err);
@@ -676,6 +676,8 @@ static int __init sh_dmae_probe(struct platform_device *pdev)
676 shdev->common.device_is_tx_complete = sh_dmae_is_complete; 676 shdev->common.device_is_tx_complete = sh_dmae_is_complete;
677 shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending; 677 shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending;
678 shdev->common.dev = &pdev->dev; 678 shdev->common.dev = &pdev->dev;
679 /* Default transfer size of 32 bytes requires 32-byte alignment */
680 shdev->common.copy_align = 5;
679 681
680#if defined(CONFIG_CPU_SH4) 682#if defined(CONFIG_CPU_SH4)
681 /* Non Mix IRQ mode SH7722/SH7730 etc... */ 683 /* Non Mix IRQ mode SH7722/SH7730 etc... */
@@ -688,8 +690,8 @@ static int __init sh_dmae_probe(struct platform_device *pdev)
688 } 690 }
689 691
690 for (ecnt = 0 ; ecnt < ARRAY_SIZE(eirq); ecnt++) { 692 for (ecnt = 0 ; ecnt < ARRAY_SIZE(eirq); ecnt++) {
691 err = request_irq(eirq[ecnt], sh_dmae_err, 693 err = request_irq(eirq[ecnt], sh_dmae_err, irqflags,
692 irqflags, "DMAC Address Error", shdev); 694 "DMAC Address Error", shdev);
693 if (err) { 695 if (err) {
694 dev_err(&pdev->dev, "DMA device request_irq" 696 dev_err(&pdev->dev, "DMA device request_irq"
695 "error (irq %d) with return %d\n", 697 "error (irq %d) with return %d\n",
diff --git a/drivers/dma/shdma.h b/drivers/dma/shdma.h
index 2b4bc15a2c0..60b81e529b4 100644
--- a/drivers/dma/shdma.h
+++ b/drivers/dma/shdma.h
@@ -35,15 +35,15 @@ struct sh_desc {
35 35
36struct sh_dmae_chan { 36struct sh_dmae_chan {
37 dma_cookie_t completed_cookie; /* The maximum cookie completed */ 37 dma_cookie_t completed_cookie; /* The maximum cookie completed */
38 spinlock_t desc_lock; /* Descriptor operation lock */ 38 spinlock_t desc_lock; /* Descriptor operation lock */
39 struct list_head ld_queue; /* Link descriptors queue */ 39 struct list_head ld_queue; /* Link descriptors queue */
40 struct list_head ld_free; /* Link descriptors free */ 40 struct list_head ld_free; /* Link descriptors free */
41 struct dma_chan common; /* DMA common channel */ 41 struct dma_chan common; /* DMA common channel */
42 struct device *dev; /* Channel device */ 42 struct device *dev; /* Channel device */
43 struct tasklet_struct tasklet; /* Tasklet */ 43 struct tasklet_struct tasklet; /* Tasklet */
44 int descs_allocated; /* desc count */ 44 int descs_allocated; /* desc count */
45 int id; /* Raw id of this channel */ 45 int id; /* Raw id of this channel */
46 char dev_id[16]; /* unique name per DMAC of channel */ 46 char dev_id[16]; /* unique name per DMAC of channel */
47 47
48 /* Set chcr */ 48 /* Set chcr */
49 int (*set_chcr)(struct sh_dmae_chan *sh_chan, u32 regs); 49 int (*set_chcr)(struct sh_dmae_chan *sh_chan, u32 regs);
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-18 19:10:57 -0500