aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma/ppc4xx/adma.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma/ppc4xx/adma.c')
-rw-r--r--drivers/dma/ppc4xx/adma.c5027
1 files changed, 5027 insertions, 0 deletions
diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c
new file mode 100644
index 000000000000..0a3478e910f0
--- /dev/null
+++ b/drivers/dma/ppc4xx/adma.c
@@ -0,0 +1,5027 @@
1/*
2 * Copyright (C) 2006-2009 DENX Software Engineering.
3 *
4 * Author: Yuri Tikhonov <yur@emcraft.com>
5 *
6 * Further porting to arch/powerpc by
7 * Anatolij Gustschin <agust@denx.de>
8 *
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU General Public License as published by the Free
11 * Software Foundation; either version 2 of the License, or (at your option)
12 * any later version.
13 *
14 * This program is distributed in the hope that it will be useful, but WITHOUT
15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 * more details.
18 *
19 * You should have received a copy of the GNU General Public License along with
20 * this program; if not, write to the Free Software Foundation, Inc., 59
21 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 *
23 * The full GNU General Public License is included in this distribution in the
24 * file called COPYING.
25 */
26
27/*
28 * This driver supports the asynchrounous DMA copy and RAID engines available
29 * on the AMCC PPC440SPe Processors.
30 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
31 * ADMA driver written by D.Williams.
32 */
33
34#include <linux/init.h>
35#include <linux/module.h>
36#include <linux/async_tx.h>
37#include <linux/delay.h>
38#include <linux/dma-mapping.h>
39#include <linux/spinlock.h>
40#include <linux/interrupt.h>
41#include <linux/uaccess.h>
42#include <linux/proc_fs.h>
43#include <linux/of.h>
44#include <linux/of_platform.h>
45#include <asm/dcr.h>
46#include <asm/dcr-regs.h>
47#include "adma.h"
48
49enum ppc_adma_init_code {
50 PPC_ADMA_INIT_OK = 0,
51 PPC_ADMA_INIT_MEMRES,
52 PPC_ADMA_INIT_MEMREG,
53 PPC_ADMA_INIT_ALLOC,
54 PPC_ADMA_INIT_COHERENT,
55 PPC_ADMA_INIT_CHANNEL,
56 PPC_ADMA_INIT_IRQ1,
57 PPC_ADMA_INIT_IRQ2,
58 PPC_ADMA_INIT_REGISTER
59};
60
61static char *ppc_adma_errors[] = {
62 [PPC_ADMA_INIT_OK] = "ok",
63 [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
64 [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
65 [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
66 "structure",
67 [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
68 "hardware descriptors",
69 [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
70 [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
71 [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
72 [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
73};
74
75static enum ppc_adma_init_code
76ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
77
78struct ppc_dma_chan_ref {
79 struct dma_chan *chan;
80 struct list_head node;
81};
82
83/* The list of channels exported by ppc440spe ADMA */
84struct list_head
85ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
86
87/* This flag is set when want to refetch the xor chain in the interrupt
88 * handler
89 */
90static u32 do_xor_refetch;
91
92/* Pointer to DMA0, DMA1 CP/CS FIFO */
93static void *ppc440spe_dma_fifo_buf;
94
95/* Pointers to last submitted to DMA0, DMA1 CDBs */
96static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
97static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
98
99/* Pointer to last linked and submitted xor CB */
100static struct ppc440spe_adma_desc_slot *xor_last_linked;
101static struct ppc440spe_adma_desc_slot *xor_last_submit;
102
103/* This array is used in data-check operations for storing a pattern */
104static char ppc440spe_qword[16];
105
106static atomic_t ppc440spe_adma_err_irq_ref;
107static dcr_host_t ppc440spe_mq_dcr_host;
108static unsigned int ppc440spe_mq_dcr_len;
109
110/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
111 * the block size in transactions, then we do not allow to activate more than
112 * only one RXOR transactions simultaneously. So use this var to store
113 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
114 * set) or not (PPC440SPE_RXOR_RUN is clear).
115 */
116static unsigned long ppc440spe_rxor_state;
117
118/* These are used in enable & check routines
119 */
120static u32 ppc440spe_r6_enabled;
121static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
122static struct completion ppc440spe_r6_test_comp;
123
124static int ppc440spe_adma_dma2rxor_prep_src(
125 struct ppc440spe_adma_desc_slot *desc,
126 struct ppc440spe_rxor *cursor, int index,
127 int src_cnt, u32 addr);
128static void ppc440spe_adma_dma2rxor_set_src(
129 struct ppc440spe_adma_desc_slot *desc,
130 int index, dma_addr_t addr);
131static void ppc440spe_adma_dma2rxor_set_mult(
132 struct ppc440spe_adma_desc_slot *desc,
133 int index, u8 mult);
134
135#ifdef ADMA_LL_DEBUG
136#define ADMA_LL_DBG(x) ({ if (1) x; 0; })
137#else
138#define ADMA_LL_DBG(x) ({ if (0) x; 0; })
139#endif
140
141static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
142{
143 struct dma_cdb *cdb;
144 struct xor_cb *cb;
145 int i;
146
147 switch (chan->device->id) {
148 case 0:
149 case 1:
150 cdb = block;
151
152 pr_debug("CDB at %p [%d]:\n"
153 "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
154 "\t sg1u 0x%08x sg1l 0x%08x\n"
155 "\t sg2u 0x%08x sg2l 0x%08x\n"
156 "\t sg3u 0x%08x sg3l 0x%08x\n",
157 cdb, chan->device->id,
158 cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
159 le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
160 le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
161 le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
162 );
163 break;
164 case 2:
165 cb = block;
166
167 pr_debug("CB at %p [%d]:\n"
168 "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
169 "\t cbtah 0x%08x cbtal 0x%08x\n"
170 "\t cblah 0x%08x cblal 0x%08x\n",
171 cb, chan->device->id,
172 cb->cbc, cb->cbbc, cb->cbs,
173 cb->cbtah, cb->cbtal,
174 cb->cblah, cb->cblal);
175 for (i = 0; i < 16; i++) {
176 if (i && !cb->ops[i].h && !cb->ops[i].l)
177 continue;
178 pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
179 i, cb->ops[i].h, cb->ops[i].l);
180 }
181 break;
182 }
183}
184
185static void print_cb_list(struct ppc440spe_adma_chan *chan,
186 struct ppc440spe_adma_desc_slot *iter)
187{
188 for (; iter; iter = iter->hw_next)
189 print_cb(chan, iter->hw_desc);
190}
191
192static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
193 unsigned int src_cnt)
194{
195 int i;
196
197 pr_debug("\n%s(%d):\nsrc: ", __func__, id);
198 for (i = 0; i < src_cnt; i++)
199 pr_debug("\t0x%016llx ", src[i]);
200 pr_debug("dst:\n\t0x%016llx\n", dst);
201}
202
203static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
204 unsigned int src_cnt)
205{
206 int i;
207
208 pr_debug("\n%s(%d):\nsrc: ", __func__, id);
209 for (i = 0; i < src_cnt; i++)
210 pr_debug("\t0x%016llx ", src[i]);
211 pr_debug("dst: ");
212 for (i = 0; i < 2; i++)
213 pr_debug("\t0x%016llx ", dst[i]);
214}
215
216static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
217 unsigned int src_cnt,
218 const unsigned char *scf)
219{
220 int i;
221
222 pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
223 if (scf) {
224 for (i = 0; i < src_cnt; i++)
225 pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
226 } else {
227 for (i = 0; i < src_cnt; i++)
228 pr_debug("\t0x%016llx(no) ", src[i]);
229 }
230
231 pr_debug("dst: ");
232 for (i = 0; i < 2; i++)
233 pr_debug("\t0x%016llx ", src[src_cnt + i]);
234}
235
236/******************************************************************************
237 * Command (Descriptor) Blocks low-level routines
238 ******************************************************************************/
239/**
240 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
241 * pseudo operation
242 */
243static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
244 struct ppc440spe_adma_chan *chan)
245{
246 struct xor_cb *p;
247
248 switch (chan->device->id) {
249 case PPC440SPE_XOR_ID:
250 p = desc->hw_desc;
251 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
252 /* NOP with Command Block Complete Enable */
253 p->cbc = XOR_CBCR_CBCE_BIT;
254 break;
255 case PPC440SPE_DMA0_ID:
256 case PPC440SPE_DMA1_ID:
257 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
258 /* NOP with interrupt */
259 set_bit(PPC440SPE_DESC_INT, &desc->flags);
260 break;
261 default:
262 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
263 __func__);
264 break;
265 }
266}
267
268/**
269 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
270 * pseudo operation
271 */
272static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
273{
274 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
275 desc->hw_next = NULL;
276 desc->src_cnt = 0;
277 desc->dst_cnt = 1;
278}
279
280/**
281 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
282 */
283static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
284 int src_cnt, unsigned long flags)
285{
286 struct xor_cb *hw_desc = desc->hw_desc;
287
288 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
289 desc->hw_next = NULL;
290 desc->src_cnt = src_cnt;
291 desc->dst_cnt = 1;
292
293 hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
294 if (flags & DMA_PREP_INTERRUPT)
295 /* Enable interrupt on completion */
296 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
297}
298
299/**
300 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
301 * operation in DMA2 controller
302 */
303static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
304 int dst_cnt, int src_cnt, unsigned long flags)
305{
306 struct xor_cb *hw_desc = desc->hw_desc;
307
308 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
309 desc->hw_next = NULL;
310 desc->src_cnt = src_cnt;
311 desc->dst_cnt = dst_cnt;
312 memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
313 desc->descs_per_op = 0;
314
315 hw_desc->cbc = XOR_CBCR_TGT_BIT;
316 if (flags & DMA_PREP_INTERRUPT)
317 /* Enable interrupt on completion */
318 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
319}
320
321#define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE
322#define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1)
323#define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1)
324
325/**
326 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
327 * with DMA0/1
328 */
329static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
330 int dst_cnt, int src_cnt, unsigned long flags,
331 unsigned long op)
332{
333 struct dma_cdb *hw_desc;
334 struct ppc440spe_adma_desc_slot *iter;
335 u8 dopc;
336
337 /* Common initialization of a PQ descriptors chain */
338 set_bits(op, &desc->flags);
339 desc->src_cnt = src_cnt;
340 desc->dst_cnt = dst_cnt;
341
342 /* WXOR MULTICAST if both P and Q are being computed
343 * MV_SG1_SG2 if Q only
344 */
345 dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
346 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
347
348 list_for_each_entry(iter, &desc->group_list, chain_node) {
349 hw_desc = iter->hw_desc;
350 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
351
352 if (likely(!list_is_last(&iter->chain_node,
353 &desc->group_list))) {
354 /* set 'next' pointer */
355 iter->hw_next = list_entry(iter->chain_node.next,
356 struct ppc440spe_adma_desc_slot, chain_node);
357 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
358 } else {
359 /* this is the last descriptor.
360 * this slot will be pasted from ADMA level
361 * each time it wants to configure parameters
362 * of the transaction (src, dst, ...)
363 */
364 iter->hw_next = NULL;
365 if (flags & DMA_PREP_INTERRUPT)
366 set_bit(PPC440SPE_DESC_INT, &iter->flags);
367 else
368 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
369 }
370 }
371
372 /* Set OPS depending on WXOR/RXOR type of operation */
373 if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
374 /* This is a WXOR only chain:
375 * - first descriptors are for zeroing destinations
376 * if PPC440SPE_ZERO_P/Q set;
377 * - descriptors remained are for GF-XOR operations.
378 */
379 iter = list_first_entry(&desc->group_list,
380 struct ppc440spe_adma_desc_slot,
381 chain_node);
382
383 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
384 hw_desc = iter->hw_desc;
385 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
386 iter = list_first_entry(&iter->chain_node,
387 struct ppc440spe_adma_desc_slot,
388 chain_node);
389 }
390
391 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
392 hw_desc = iter->hw_desc;
393 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
394 iter = list_first_entry(&iter->chain_node,
395 struct ppc440spe_adma_desc_slot,
396 chain_node);
397 }
398
399 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
400 hw_desc = iter->hw_desc;
401 hw_desc->opc = dopc;
402 }
403 } else {
404 /* This is either RXOR-only or mixed RXOR/WXOR */
405
406 /* The first 1 or 2 slots in chain are always RXOR,
407 * if need to calculate P & Q, then there are two
408 * RXOR slots; if only P or only Q, then there is one
409 */
410 iter = list_first_entry(&desc->group_list,
411 struct ppc440spe_adma_desc_slot,
412 chain_node);
413 hw_desc = iter->hw_desc;
414 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
415
416 if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
417 iter = list_first_entry(&iter->chain_node,
418 struct ppc440spe_adma_desc_slot,
419 chain_node);
420 hw_desc = iter->hw_desc;
421 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
422 }
423
424 /* The remaining descs (if any) are WXORs */
425 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
426 iter = list_first_entry(&iter->chain_node,
427 struct ppc440spe_adma_desc_slot,
428 chain_node);
429 list_for_each_entry_from(iter, &desc->group_list,
430 chain_node) {
431 hw_desc = iter->hw_desc;
432 hw_desc->opc = dopc;
433 }
434 }
435 }
436}
437
438/**
439 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
440 * for PQ_ZERO_SUM operation
441 */
442static void ppc440spe_desc_init_dma01pqzero_sum(
443 struct ppc440spe_adma_desc_slot *desc,
444 int dst_cnt, int src_cnt)
445{
446 struct dma_cdb *hw_desc;
447 struct ppc440spe_adma_desc_slot *iter;
448 int i = 0;
449 u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
450 DMA_CDB_OPC_MV_SG1_SG2;
451 /*
452 * Initialize starting from 2nd or 3rd descriptor dependent
453 * on dst_cnt. First one or two slots are for cloning P
454 * and/or Q to chan->pdest and/or chan->qdest as we have
455 * to preserve original P/Q.
456 */
457 iter = list_first_entry(&desc->group_list,
458 struct ppc440spe_adma_desc_slot, chain_node);
459 iter = list_entry(iter->chain_node.next,
460 struct ppc440spe_adma_desc_slot, chain_node);
461
462 if (dst_cnt > 1) {
463 iter = list_entry(iter->chain_node.next,
464 struct ppc440spe_adma_desc_slot, chain_node);
465 }
466 /* initialize each source descriptor in chain */
467 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
468 hw_desc = iter->hw_desc;
469 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
470 iter->src_cnt = 0;
471 iter->dst_cnt = 0;
472
473 /* This is a ZERO_SUM operation:
474 * - <src_cnt> descriptors starting from 2nd or 3rd
475 * descriptor are for GF-XOR operations;
476 * - remaining <dst_cnt> descriptors are for checking the result
477 */
478 if (i++ < src_cnt)
479 /* MV_SG1_SG2 if only Q is being verified
480 * MULTICAST if both P and Q are being verified
481 */
482 hw_desc->opc = dopc;
483 else
484 /* DMA_CDB_OPC_DCHECK128 operation */
485 hw_desc->opc = DMA_CDB_OPC_DCHECK128;
486
487 if (likely(!list_is_last(&iter->chain_node,
488 &desc->group_list))) {
489 /* set 'next' pointer */
490 iter->hw_next = list_entry(iter->chain_node.next,
491 struct ppc440spe_adma_desc_slot,
492 chain_node);
493 } else {
494 /* this is the last descriptor.
495 * this slot will be pasted from ADMA level
496 * each time it wants to configure parameters
497 * of the transaction (src, dst, ...)
498 */
499 iter->hw_next = NULL;
500 /* always enable interrupt generation since we get
501 * the status of pqzero from the handler
502 */
503 set_bit(PPC440SPE_DESC_INT, &iter->flags);
504 }
505 }
506 desc->src_cnt = src_cnt;
507 desc->dst_cnt = dst_cnt;
508}
509
510/**
511 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
512 */
513static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
514 unsigned long flags)
515{
516 struct dma_cdb *hw_desc = desc->hw_desc;
517
518 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
519 desc->hw_next = NULL;
520 desc->src_cnt = 1;
521 desc->dst_cnt = 1;
522
523 if (flags & DMA_PREP_INTERRUPT)
524 set_bit(PPC440SPE_DESC_INT, &desc->flags);
525 else
526 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
527
528 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
529}
530
531/**
532 * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation
533 */
534static void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc,
535 int value, unsigned long flags)
536{
537 struct dma_cdb *hw_desc = desc->hw_desc;
538
539 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
540 desc->hw_next = NULL;
541 desc->src_cnt = 1;
542 desc->dst_cnt = 1;
543
544 if (flags & DMA_PREP_INTERRUPT)
545 set_bit(PPC440SPE_DESC_INT, &desc->flags);
546 else
547 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
548
549 hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value);
550 hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value);
551 hw_desc->opc = DMA_CDB_OPC_DFILL128;
552}
553
554/**
555 * ppc440spe_desc_set_src_addr - set source address into the descriptor
556 */
557static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
558 struct ppc440spe_adma_chan *chan,
559 int src_idx, dma_addr_t addrh,
560 dma_addr_t addrl)
561{
562 struct dma_cdb *dma_hw_desc;
563 struct xor_cb *xor_hw_desc;
564 phys_addr_t addr64, tmplow, tmphi;
565
566 switch (chan->device->id) {
567 case PPC440SPE_DMA0_ID:
568 case PPC440SPE_DMA1_ID:
569 if (!addrh) {
570 addr64 = addrl;
571 tmphi = (addr64 >> 32);
572 tmplow = (addr64 & 0xFFFFFFFF);
573 } else {
574 tmphi = addrh;
575 tmplow = addrl;
576 }
577 dma_hw_desc = desc->hw_desc;
578 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
579 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
580 break;
581 case PPC440SPE_XOR_ID:
582 xor_hw_desc = desc->hw_desc;
583 xor_hw_desc->ops[src_idx].l = addrl;
584 xor_hw_desc->ops[src_idx].h |= addrh;
585 break;
586 }
587}
588
589/**
590 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
591 */
592static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
593 struct ppc440spe_adma_chan *chan, u32 mult_index,
594 int sg_index, unsigned char mult_value)
595{
596 struct dma_cdb *dma_hw_desc;
597 struct xor_cb *xor_hw_desc;
598 u32 *psgu;
599
600 switch (chan->device->id) {
601 case PPC440SPE_DMA0_ID:
602 case PPC440SPE_DMA1_ID:
603 dma_hw_desc = desc->hw_desc;
604
605 switch (sg_index) {
606 /* for RXOR operations set multiplier
607 * into source cued address
608 */
609 case DMA_CDB_SG_SRC:
610 psgu = &dma_hw_desc->sg1u;
611 break;
612 /* for WXOR operations set multiplier
613 * into destination cued address(es)
614 */
615 case DMA_CDB_SG_DST1:
616 psgu = &dma_hw_desc->sg2u;
617 break;
618 case DMA_CDB_SG_DST2:
619 psgu = &dma_hw_desc->sg3u;
620 break;
621 default:
622 BUG();
623 }
624
625 *psgu |= cpu_to_le32(mult_value << mult_index);
626 break;
627 case PPC440SPE_XOR_ID:
628 xor_hw_desc = desc->hw_desc;
629 break;
630 default:
631 BUG();
632 }
633}
634
635/**
636 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
637 */
638static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
639 struct ppc440spe_adma_chan *chan,
640 dma_addr_t addrh, dma_addr_t addrl,
641 u32 dst_idx)
642{
643 struct dma_cdb *dma_hw_desc;
644 struct xor_cb *xor_hw_desc;
645 phys_addr_t addr64, tmphi, tmplow;
646 u32 *psgu, *psgl;
647
648 switch (chan->device->id) {
649 case PPC440SPE_DMA0_ID:
650 case PPC440SPE_DMA1_ID:
651 if (!addrh) {
652 addr64 = addrl;
653 tmphi = (addr64 >> 32);
654 tmplow = (addr64 & 0xFFFFFFFF);
655 } else {
656 tmphi = addrh;
657 tmplow = addrl;
658 }
659 dma_hw_desc = desc->hw_desc;
660
661 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
662 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
663
664 *psgl = cpu_to_le32((u32)tmplow);
665 *psgu |= cpu_to_le32((u32)tmphi);
666 break;
667 case PPC440SPE_XOR_ID:
668 xor_hw_desc = desc->hw_desc;
669 xor_hw_desc->cbtal = addrl;
670 xor_hw_desc->cbtah |= addrh;
671 break;
672 }
673}
674
675/**
676 * ppc440spe_desc_set_byte_count - set number of data bytes involved
677 * into the operation
678 */
679static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
680 struct ppc440spe_adma_chan *chan,
681 u32 byte_count)
682{
683 struct dma_cdb *dma_hw_desc;
684 struct xor_cb *xor_hw_desc;
685
686 switch (chan->device->id) {
687 case PPC440SPE_DMA0_ID:
688 case PPC440SPE_DMA1_ID:
689 dma_hw_desc = desc->hw_desc;
690 dma_hw_desc->cnt = cpu_to_le32(byte_count);
691 break;
692 case PPC440SPE_XOR_ID:
693 xor_hw_desc = desc->hw_desc;
694 xor_hw_desc->cbbc = byte_count;
695 break;
696 }
697}
698
699/**
700 * ppc440spe_desc_set_rxor_block_size - set RXOR block size
701 */
702static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
703{
704 /* assume that byte_count is aligned on the 512-boundary;
705 * thus write it directly to the register (bits 23:31 are
706 * reserved there).
707 */
708 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
709}
710
711/**
712 * ppc440spe_desc_set_dcheck - set CHECK pattern
713 */
714static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
715 struct ppc440spe_adma_chan *chan, u8 *qword)
716{
717 struct dma_cdb *dma_hw_desc;
718
719 switch (chan->device->id) {
720 case PPC440SPE_DMA0_ID:
721 case PPC440SPE_DMA1_ID:
722 dma_hw_desc = desc->hw_desc;
723 iowrite32(qword[0], &dma_hw_desc->sg3l);
724 iowrite32(qword[4], &dma_hw_desc->sg3u);
725 iowrite32(qword[8], &dma_hw_desc->sg2l);
726 iowrite32(qword[12], &dma_hw_desc->sg2u);
727 break;
728 default:
729 BUG();
730 }
731}
732
733/**
734 * ppc440spe_xor_set_link - set link address in xor CB
735 */
736static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
737 struct ppc440spe_adma_desc_slot *next_desc)
738{
739 struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
740
741 if (unlikely(!next_desc || !(next_desc->phys))) {
742 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
743 __func__, next_desc,
744 next_desc ? next_desc->phys : 0);
745 BUG();
746 }
747
748 xor_hw_desc->cbs = 0;
749 xor_hw_desc->cblal = next_desc->phys;
750 xor_hw_desc->cblah = 0;
751 xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
752}
753
754/**
755 * ppc440spe_desc_set_link - set the address of descriptor following this
756 * descriptor in chain
757 */
758static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
759 struct ppc440spe_adma_desc_slot *prev_desc,
760 struct ppc440spe_adma_desc_slot *next_desc)
761{
762 unsigned long flags;
763 struct ppc440spe_adma_desc_slot *tail = next_desc;
764
765 if (unlikely(!prev_desc || !next_desc ||
766 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
767 /* If previous next is overwritten something is wrong.
768 * though we may refetch from append to initiate list
769 * processing; in this case - it's ok.
770 */
771 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
772 "prev->hw_next=0x%p\n", __func__, prev_desc,
773 next_desc, prev_desc ? prev_desc->hw_next : 0);
774 BUG();
775 }
776
777 local_irq_save(flags);
778
779 /* do s/w chaining both for DMA and XOR descriptors */
780 prev_desc->hw_next = next_desc;
781
782 switch (chan->device->id) {
783 case PPC440SPE_DMA0_ID:
784 case PPC440SPE_DMA1_ID:
785 break;
786 case PPC440SPE_XOR_ID:
787 /* bind descriptor to the chain */
788 while (tail->hw_next)
789 tail = tail->hw_next;
790 xor_last_linked = tail;
791
792 if (prev_desc == xor_last_submit)
793 /* do not link to the last submitted CB */
794 break;
795 ppc440spe_xor_set_link(prev_desc, next_desc);
796 break;
797 }
798
799 local_irq_restore(flags);
800}
801
802/**
803 * ppc440spe_desc_get_src_addr - extract the source address from the descriptor
804 */
805static u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc,
806 struct ppc440spe_adma_chan *chan, int src_idx)
807{
808 struct dma_cdb *dma_hw_desc;
809 struct xor_cb *xor_hw_desc;
810
811 switch (chan->device->id) {
812 case PPC440SPE_DMA0_ID:
813 case PPC440SPE_DMA1_ID:
814 dma_hw_desc = desc->hw_desc;
815 /* May have 0, 1, 2, or 3 sources */
816 switch (dma_hw_desc->opc) {
817 case DMA_CDB_OPC_NO_OP:
818 case DMA_CDB_OPC_DFILL128:
819 return 0;
820 case DMA_CDB_OPC_DCHECK128:
821 if (unlikely(src_idx)) {
822 printk(KERN_ERR "%s: try to get %d source for"
823 " DCHECK128\n", __func__, src_idx);
824 BUG();
825 }
826 return le32_to_cpu(dma_hw_desc->sg1l);
827 case DMA_CDB_OPC_MULTICAST:
828 case DMA_CDB_OPC_MV_SG1_SG2:
829 if (unlikely(src_idx > 2)) {
830 printk(KERN_ERR "%s: try to get %d source from"
831 " DMA descr\n", __func__, src_idx);
832 BUG();
833 }
834 if (src_idx) {
835 if (le32_to_cpu(dma_hw_desc->sg1u) &
836 DMA_CUED_XOR_WIN_MSK) {
837 u8 region;
838
839 if (src_idx == 1)
840 return le32_to_cpu(
841 dma_hw_desc->sg1l) +
842 desc->unmap_len;
843
844 region = (le32_to_cpu(
845 dma_hw_desc->sg1u)) >>
846 DMA_CUED_REGION_OFF;
847
848 region &= DMA_CUED_REGION_MSK;
849 switch (region) {
850 case DMA_RXOR123:
851 return le32_to_cpu(
852 dma_hw_desc->sg1l) +
853 (desc->unmap_len << 1);
854 case DMA_RXOR124:
855 return le32_to_cpu(
856 dma_hw_desc->sg1l) +
857 (desc->unmap_len * 3);
858 case DMA_RXOR125:
859 return le32_to_cpu(
860 dma_hw_desc->sg1l) +
861 (desc->unmap_len << 2);
862 default:
863 printk(KERN_ERR
864 "%s: try to"
865 " get src3 for region %02x"
866 "PPC440SPE_DESC_RXOR12?\n",
867 __func__, region);
868 BUG();
869 }
870 } else {
871 printk(KERN_ERR
872 "%s: try to get %d"
873 " source for non-cued descr\n",
874 __func__, src_idx);
875 BUG();
876 }
877 }
878 return le32_to_cpu(dma_hw_desc->sg1l);
879 default:
880 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
881 __func__, dma_hw_desc->opc);
882 BUG();
883 }
884 return le32_to_cpu(dma_hw_desc->sg1l);
885 case PPC440SPE_XOR_ID:
886 /* May have up to 16 sources */
887 xor_hw_desc = desc->hw_desc;
888 return xor_hw_desc->ops[src_idx].l;
889 }
890 return 0;
891}
892
893/**
894 * ppc440spe_desc_get_dest_addr - extract the destination address from the
895 * descriptor
896 */
897static u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc,
898 struct ppc440spe_adma_chan *chan, int idx)
899{
900 struct dma_cdb *dma_hw_desc;
901 struct xor_cb *xor_hw_desc;
902
903 switch (chan->device->id) {
904 case PPC440SPE_DMA0_ID:
905 case PPC440SPE_DMA1_ID:
906 dma_hw_desc = desc->hw_desc;
907
908 if (likely(!idx))
909 return le32_to_cpu(dma_hw_desc->sg2l);
910 return le32_to_cpu(dma_hw_desc->sg3l);
911 case PPC440SPE_XOR_ID:
912 xor_hw_desc = desc->hw_desc;
913 return xor_hw_desc->cbtal;
914 }
915 return 0;
916}
917
918/**
919 * ppc440spe_desc_get_src_num - extract the number of source addresses from
920 * the descriptor
921 */
922static u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc,
923 struct ppc440spe_adma_chan *chan)
924{
925 struct dma_cdb *dma_hw_desc;
926 struct xor_cb *xor_hw_desc;
927
928 switch (chan->device->id) {
929 case PPC440SPE_DMA0_ID:
930 case PPC440SPE_DMA1_ID:
931 dma_hw_desc = desc->hw_desc;
932
933 switch (dma_hw_desc->opc) {
934 case DMA_CDB_OPC_NO_OP:
935 case DMA_CDB_OPC_DFILL128:
936 return 0;
937 case DMA_CDB_OPC_DCHECK128:
938 return 1;
939 case DMA_CDB_OPC_MV_SG1_SG2:
940 case DMA_CDB_OPC_MULTICAST:
941 /*
942 * Only for RXOR operations we have more than
943 * one source
944 */
945 if (le32_to_cpu(dma_hw_desc->sg1u) &
946 DMA_CUED_XOR_WIN_MSK) {
947 /* RXOR op, there are 2 or 3 sources */
948 if (((le32_to_cpu(dma_hw_desc->sg1u) >>
949 DMA_CUED_REGION_OFF) &
950 DMA_CUED_REGION_MSK) == DMA_RXOR12) {
951 /* RXOR 1-2 */
952 return 2;
953 } else {
954 /* RXOR 1-2-3/1-2-4/1-2-5 */
955 return 3;
956 }
957 }
958 return 1;
959 default:
960 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
961 __func__, dma_hw_desc->opc);
962 BUG();
963 }
964 case PPC440SPE_XOR_ID:
965 /* up to 16 sources */
966 xor_hw_desc = desc->hw_desc;
967 return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK;
968 default:
969 BUG();
970 }
971 return 0;
972}
973
974/**
975 * ppc440spe_desc_get_dst_num - get the number of destination addresses in
976 * this descriptor
977 */
978static u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc,
979 struct ppc440spe_adma_chan *chan)
980{
981 struct dma_cdb *dma_hw_desc;
982
983 switch (chan->device->id) {
984 case PPC440SPE_DMA0_ID:
985 case PPC440SPE_DMA1_ID:
986 /* May be 1 or 2 destinations */
987 dma_hw_desc = desc->hw_desc;
988 switch (dma_hw_desc->opc) {
989 case DMA_CDB_OPC_NO_OP:
990 case DMA_CDB_OPC_DCHECK128:
991 return 0;
992 case DMA_CDB_OPC_MV_SG1_SG2:
993 case DMA_CDB_OPC_DFILL128:
994 return 1;
995 case DMA_CDB_OPC_MULTICAST:
996 if (desc->dst_cnt == 2)
997 return 2;
998 else
999 return 1;
1000 default:
1001 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
1002 __func__, dma_hw_desc->opc);
1003 BUG();
1004 }
1005 case PPC440SPE_XOR_ID:
1006 /* Always only 1 destination */
1007 return 1;
1008 default:
1009 BUG();
1010 }
1011 return 0;
1012}
1013
1014/**
1015 * ppc440spe_desc_get_link - get the address of the descriptor that
1016 * follows this one
1017 */
1018static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
1019 struct ppc440spe_adma_chan *chan)
1020{
1021 if (!desc->hw_next)
1022 return 0;
1023
1024 return desc->hw_next->phys;
1025}
1026
1027/**
1028 * ppc440spe_desc_is_aligned - check alignment
1029 */
1030static inline int ppc440spe_desc_is_aligned(
1031 struct ppc440spe_adma_desc_slot *desc, int num_slots)
1032{
1033 return (desc->idx & (num_slots - 1)) ? 0 : 1;
1034}
1035
1036/**
1037 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
1038 * XOR operation
1039 */
1040static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
1041 int *slots_per_op)
1042{
1043 int slot_cnt;
1044
1045 /* each XOR descriptor provides up to 16 source operands */
1046 slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
1047
1048 if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
1049 return slot_cnt;
1050
1051 printk(KERN_ERR "%s: len %d > max %d !!\n",
1052 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
1053 BUG();
1054 return slot_cnt;
1055}
1056
1057/**
1058 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
1059 * DMA2 PQ operation
1060 */
1061static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
1062 int src_cnt, size_t len)
1063{
1064 signed long long order = 0;
1065 int state = 0;
1066 int addr_count = 0;
1067 int i;
1068 for (i = 1; i < src_cnt; i++) {
1069 dma_addr_t cur_addr = srcs[i];
1070 dma_addr_t old_addr = srcs[i-1];
1071 switch (state) {
1072 case 0:
1073 if (cur_addr == old_addr + len) {
1074 /* direct RXOR */
1075 order = 1;
1076 state = 1;
1077 if (i == src_cnt-1)
1078 addr_count++;
1079 } else if (old_addr == cur_addr + len) {
1080 /* reverse RXOR */
1081 order = -1;
1082 state = 1;
1083 if (i == src_cnt-1)
1084 addr_count++;
1085 } else {
1086 state = 3;
1087 }
1088 break;
1089 case 1:
1090 if (i == src_cnt-2 || (order == -1
1091 && cur_addr != old_addr - len)) {
1092 order = 0;
1093 state = 0;
1094 addr_count++;
1095 } else if (cur_addr == old_addr + len*order) {
1096 state = 2;
1097 if (i == src_cnt-1)
1098 addr_count++;
1099 } else if (cur_addr == old_addr + 2*len) {
1100 state = 2;
1101 if (i == src_cnt-1)
1102 addr_count++;
1103 } else if (cur_addr == old_addr + 3*len) {
1104 state = 2;
1105 if (i == src_cnt-1)
1106 addr_count++;
1107 } else {
1108 order = 0;
1109 state = 0;
1110 addr_count++;
1111 }
1112 break;
1113 case 2:
1114 order = 0;
1115 state = 0;
1116 addr_count++;
1117 break;
1118 }
1119 if (state == 3)
1120 break;
1121 }
1122 if (src_cnt <= 1 || (state != 1 && state != 2)) {
1123 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
1124 __func__, src_cnt, state, addr_count, order);
1125 for (i = 0; i < src_cnt; i++)
1126 pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
1127 BUG();
1128 }
1129
1130 return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
1131}
1132
1133
1134/******************************************************************************
1135 * ADMA channel low-level routines
1136 ******************************************************************************/
1137
1138static u32
1139ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
1140static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
1141
1142/**
1143 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
1144 */
1145static void ppc440spe_adma_device_clear_eot_status(
1146 struct ppc440spe_adma_chan *chan)
1147{
1148 struct dma_regs *dma_reg;
1149 struct xor_regs *xor_reg;
1150 u8 *p = chan->device->dma_desc_pool_virt;
1151 struct dma_cdb *cdb;
1152 u32 rv, i;
1153
1154 switch (chan->device->id) {
1155 case PPC440SPE_DMA0_ID:
1156 case PPC440SPE_DMA1_ID:
1157 /* read FIFO to ack */
1158 dma_reg = chan->device->dma_reg;
1159 while ((rv = ioread32(&dma_reg->csfpl))) {
1160 i = rv & DMA_CDB_ADDR_MSK;
1161 cdb = (struct dma_cdb *)&p[i -
1162 (u32)chan->device->dma_desc_pool];
1163
1164 /* Clear opcode to ack. This is necessary for
1165 * ZeroSum operations only
1166 */
1167 cdb->opc = 0;
1168
1169 if (test_bit(PPC440SPE_RXOR_RUN,
1170 &ppc440spe_rxor_state)) {
1171 /* probably this is a completed RXOR op,
1172 * get pointer to CDB using the fact that
1173 * physical and virtual addresses of CDB
1174 * in pools have the same offsets
1175 */
1176 if (le32_to_cpu(cdb->sg1u) &
1177 DMA_CUED_XOR_BASE) {
1178 /* this is a RXOR */
1179 clear_bit(PPC440SPE_RXOR_RUN,
1180 &ppc440spe_rxor_state);
1181 }
1182 }
1183
1184 if (rv & DMA_CDB_STATUS_MSK) {
1185 /* ZeroSum check failed
1186 */
1187 struct ppc440spe_adma_desc_slot *iter;
1188 dma_addr_t phys = rv & ~DMA_CDB_MSK;
1189
1190 /*
1191 * Update the status of corresponding
1192 * descriptor.
1193 */
1194 list_for_each_entry(iter, &chan->chain,
1195 chain_node) {
1196 if (iter->phys == phys)
1197 break;
1198 }
1199 /*
1200 * if cannot find the corresponding
1201 * slot it's a bug
1202 */
1203 BUG_ON(&iter->chain_node == &chan->chain);
1204
1205 if (iter->xor_check_result) {
1206 if (test_bit(PPC440SPE_DESC_PCHECK,
1207 &iter->flags)) {
1208 *iter->xor_check_result |=
1209 SUM_CHECK_P_RESULT;
1210 } else
1211 if (test_bit(PPC440SPE_DESC_QCHECK,
1212 &iter->flags)) {
1213 *iter->xor_check_result |=
1214 SUM_CHECK_Q_RESULT;
1215 } else
1216 BUG();
1217 }
1218 }
1219 }
1220
1221 rv = ioread32(&dma_reg->dsts);
1222 if (rv) {
1223 pr_err("DMA%d err status: 0x%x\n",
1224 chan->device->id, rv);
1225 /* write back to clear */
1226 iowrite32(rv, &dma_reg->dsts);
1227 }
1228 break;
1229 case PPC440SPE_XOR_ID:
1230 /* reset status bits to ack */
1231 xor_reg = chan->device->xor_reg;
1232 rv = ioread32be(&xor_reg->sr);
1233 iowrite32be(rv, &xor_reg->sr);
1234
1235 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
1236 if (rv & XOR_IE_RPTIE_BIT) {
1237 /* Read PLB Timeout Error.
1238 * Try to resubmit the CB
1239 */
1240 u32 val = ioread32be(&xor_reg->ccbalr);
1241
1242 iowrite32be(val, &xor_reg->cblalr);
1243
1244 val = ioread32be(&xor_reg->crsr);
1245 iowrite32be(val | XOR_CRSR_XAE_BIT,
1246 &xor_reg->crsr);
1247 } else
1248 pr_err("XOR ERR 0x%x status\n", rv);
1249 break;
1250 }
1251
1252 /* if the XORcore is idle, but there are unprocessed CBs
1253 * then refetch the s/w chain here
1254 */
1255 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1256 do_xor_refetch)
1257 ppc440spe_chan_append(chan);
1258 break;
1259 }
1260}
1261
1262/**
1263 * ppc440spe_chan_is_busy - get the channel status
1264 */
1265static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1266{
1267 struct dma_regs *dma_reg;
1268 struct xor_regs *xor_reg;
1269 int busy = 0;
1270
1271 switch (chan->device->id) {
1272 case PPC440SPE_DMA0_ID:
1273 case PPC440SPE_DMA1_ID:
1274 dma_reg = chan->device->dma_reg;
1275 /* if command FIFO's head and tail pointers are equal and
1276 * status tail is the same as command, then channel is free
1277 */
1278 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1279 ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1280 busy = 1;
1281 break;
1282 case PPC440SPE_XOR_ID:
1283 /* use the special status bit for the XORcore
1284 */
1285 xor_reg = chan->device->xor_reg;
1286 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1287 break;
1288 }
1289
1290 return busy;
1291}
1292
1293/**
1294 * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain
1295 */
1296static void ppc440spe_chan_set_first_xor_descriptor(
1297 struct ppc440spe_adma_chan *chan,
1298 struct ppc440spe_adma_desc_slot *next_desc)
1299{
1300 struct xor_regs *xor_reg = chan->device->xor_reg;
1301
1302 if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1303 printk(KERN_INFO "%s: Warn: XORcore is running "
1304 "when try to set the first CDB!\n",
1305 __func__);
1306
1307 xor_last_submit = xor_last_linked = next_desc;
1308
1309 iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1310
1311 iowrite32be(next_desc->phys, &xor_reg->cblalr);
1312 iowrite32be(0, &xor_reg->cblahr);
1313 iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1314 &xor_reg->cbcr);
1315
1316 chan->hw_chain_inited = 1;
1317}
1318
1319/**
1320 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1321 * called with irqs disabled
1322 */
1323static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1324 struct ppc440spe_adma_desc_slot *desc)
1325{
1326 u32 pcdb;
1327 struct dma_regs *dma_reg = chan->device->dma_reg;
1328
1329 pcdb = desc->phys;
1330 if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1331 pcdb |= DMA_CDB_NO_INT;
1332
1333 chan_last_sub[chan->device->id] = desc;
1334
1335 ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1336
1337 iowrite32(pcdb, &dma_reg->cpfpl);
1338}
1339
1340/**
1341 * ppc440spe_chan_append - update the h/w chain in the channel
1342 */
1343static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1344{
1345 struct xor_regs *xor_reg;
1346 struct ppc440spe_adma_desc_slot *iter;
1347 struct xor_cb *xcb;
1348 u32 cur_desc;
1349 unsigned long flags;
1350
1351 local_irq_save(flags);
1352
1353 switch (chan->device->id) {
1354 case PPC440SPE_DMA0_ID:
1355 case PPC440SPE_DMA1_ID:
1356 cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1357
1358 if (likely(cur_desc)) {
1359 iter = chan_last_sub[chan->device->id];
1360 BUG_ON(!iter);
1361 } else {
1362 /* first peer */
1363 iter = chan_first_cdb[chan->device->id];
1364 BUG_ON(!iter);
1365 ppc440spe_dma_put_desc(chan, iter);
1366 chan->hw_chain_inited = 1;
1367 }
1368
1369 /* is there something new to append */
1370 if (!iter->hw_next)
1371 break;
1372
1373 /* flush descriptors from the s/w queue to fifo */
1374 list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1375 ppc440spe_dma_put_desc(chan, iter);
1376 if (!iter->hw_next)
1377 break;
1378 }
1379 break;
1380 case PPC440SPE_XOR_ID:
1381 /* update h/w links and refetch */
1382 if (!xor_last_submit->hw_next)
1383 break;
1384
1385 xor_reg = chan->device->xor_reg;
1386 /* the last linked CDB has to generate an interrupt
1387 * that we'd be able to append the next lists to h/w
1388 * regardless of the XOR engine state at the moment of
1389 * appending of these next lists
1390 */
1391 xcb = xor_last_linked->hw_desc;
1392 xcb->cbc |= XOR_CBCR_CBCE_BIT;
1393
1394 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1395 /* XORcore is idle. Refetch now */
1396 do_xor_refetch = 0;
1397 ppc440spe_xor_set_link(xor_last_submit,
1398 xor_last_submit->hw_next);
1399
1400 ADMA_LL_DBG(print_cb_list(chan,
1401 xor_last_submit->hw_next));
1402
1403 xor_last_submit = xor_last_linked;
1404 iowrite32be(ioread32be(&xor_reg->crsr) |
1405 XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1406 &xor_reg->crsr);
1407 } else {
1408 /* XORcore is running. Refetch later in the handler */
1409 do_xor_refetch = 1;
1410 }
1411
1412 break;
1413 }
1414
1415 local_irq_restore(flags);
1416}
1417
1418/**
1419 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1420 */
1421static u32
1422ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1423{
1424 struct dma_regs *dma_reg;
1425 struct xor_regs *xor_reg;
1426
1427 if (unlikely(!chan->hw_chain_inited))
1428 /* h/w descriptor chain is not initialized yet */
1429 return 0;
1430
1431 switch (chan->device->id) {
1432 case PPC440SPE_DMA0_ID:
1433 case PPC440SPE_DMA1_ID:
1434 dma_reg = chan->device->dma_reg;
1435 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1436 case PPC440SPE_XOR_ID:
1437 xor_reg = chan->device->xor_reg;
1438 return ioread32be(&xor_reg->ccbalr);
1439 }
1440 return 0;
1441}
1442
1443/**
1444 * ppc440spe_chan_run - enable the channel
1445 */
1446static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1447{
1448 struct xor_regs *xor_reg;
1449
1450 switch (chan->device->id) {
1451 case PPC440SPE_DMA0_ID:
1452 case PPC440SPE_DMA1_ID:
1453 /* DMAs are always enabled, do nothing */
1454 break;
1455 case PPC440SPE_XOR_ID:
1456 /* drain write buffer */
1457 xor_reg = chan->device->xor_reg;
1458
1459 /* fetch descriptor pointed to in <link> */
1460 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1461 &xor_reg->crsr);
1462 break;
1463 }
1464}
1465
1466/******************************************************************************
1467 * ADMA device level
1468 ******************************************************************************/
1469
1470static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1471static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1472
1473static dma_cookie_t
1474ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1475
1476static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1477 dma_addr_t addr, int index);
1478static void
1479ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1480 dma_addr_t addr, int index);
1481
1482static void
1483ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1484 dma_addr_t *paddr, unsigned long flags);
1485static void
1486ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1487 dma_addr_t addr, int index);
1488static void
1489ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1490 unsigned char mult, int index, int dst_pos);
1491static void
1492ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1493 dma_addr_t paddr, dma_addr_t qaddr);
1494
1495static struct page *ppc440spe_rxor_srcs[32];
1496
1497/**
1498 * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1499 */
1500static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1501{
1502 int i, order = 0, state = 0;
1503 int idx = 0;
1504
1505 if (unlikely(!(src_cnt > 1)))
1506 return 0;
1507
1508 BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1509
1510 /* Skip holes in the source list before checking */
1511 for (i = 0; i < src_cnt; i++) {
1512 if (!srcs[i])
1513 continue;
1514 ppc440spe_rxor_srcs[idx++] = srcs[i];
1515 }
1516 src_cnt = idx;
1517
1518 for (i = 1; i < src_cnt; i++) {
1519 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1520 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1521
1522 switch (state) {
1523 case 0:
1524 if (cur_addr == old_addr + len) {
1525 /* direct RXOR */
1526 order = 1;
1527 state = 1;
1528 } else if (old_addr == cur_addr + len) {
1529 /* reverse RXOR */
1530 order = -1;
1531 state = 1;
1532 } else
1533 goto out;
1534 break;
1535 case 1:
1536 if ((i == src_cnt - 2) ||
1537 (order == -1 && cur_addr != old_addr - len)) {
1538 order = 0;
1539 state = 0;
1540 } else if ((cur_addr == old_addr + len * order) ||
1541 (cur_addr == old_addr + 2 * len) ||
1542 (cur_addr == old_addr + 3 * len)) {
1543 state = 2;
1544 } else {
1545 order = 0;
1546 state = 0;
1547 }
1548 break;
1549 case 2:
1550 order = 0;
1551 state = 0;
1552 break;
1553 }
1554 }
1555
1556out:
1557 if (state == 1 || state == 2)
1558 return 1;
1559
1560 return 0;
1561}
1562
1563/**
1564 * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1565 * the operation given on this channel. It's assumed that 'chan' is
1566 * capable to process 'cap' type of operation.
1567 * @chan: channel to use
1568 * @cap: type of transaction
1569 * @dst_lst: array of destination pointers
1570 * @dst_cnt: number of destination operands
1571 * @src_lst: array of source pointers
1572 * @src_cnt: number of source operands
1573 * @src_sz: size of each source operand
1574 */
1575static int ppc440spe_adma_estimate(struct dma_chan *chan,
1576 enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1577 struct page **src_lst, int src_cnt, size_t src_sz)
1578{
1579 int ef = 1;
1580
1581 if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1582 /* If RAID-6 capabilities were not activated don't try
1583 * to use them
1584 */
1585 if (unlikely(!ppc440spe_r6_enabled))
1586 return -1;
1587 }
1588 /* In the current implementation of ppc440spe ADMA driver it
1589 * makes sense to pick out only pq case, because it may be
1590 * processed:
1591 * (1) either using Biskup method on DMA2;
1592 * (2) or on DMA0/1.
1593 * Thus we give a favour to (1) if the sources are suitable;
1594 * else let it be processed on one of the DMA0/1 engines.
1595 * In the sum_product case where destination is also the
1596 * source process it on DMA0/1 only.
1597 */
1598 if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1599
1600 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1601 ef = 0; /* sum_product case, process on DMA0/1 */
1602 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1603 ef = 3; /* override (DMA0/1 + idle) */
1604 else
1605 ef = 0; /* can't process on DMA2 if !rxor */
1606 }
1607
1608 /* channel idleness increases the priority */
1609 if (likely(ef) &&
1610 !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1611 ef++;
1612
1613 return ef;
1614}
1615
1616struct dma_chan *
1617ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1618 struct page **dst_lst, int dst_cnt, struct page **src_lst,
1619 int src_cnt, size_t src_sz)
1620{
1621 struct dma_chan *best_chan = NULL;
1622 struct ppc_dma_chan_ref *ref;
1623 int best_rank = -1;
1624
1625 if (unlikely(!src_sz))
1626 return NULL;
1627 if (src_sz > PAGE_SIZE) {
1628 /*
1629 * should a user of the api ever pass > PAGE_SIZE requests
1630 * we sort out cases where temporary page-sized buffers
1631 * are used.
1632 */
1633 switch (cap) {
1634 case DMA_PQ:
1635 if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1636 return NULL;
1637 if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1638 return NULL;
1639 break;
1640 case DMA_PQ_VAL:
1641 case DMA_XOR_VAL:
1642 return NULL;
1643 default:
1644 break;
1645 }
1646 }
1647
1648 list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1649 if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1650 int rank;
1651
1652 rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1653 dst_cnt, src_lst, src_cnt, src_sz);
1654 if (rank > best_rank) {
1655 best_rank = rank;
1656 best_chan = ref->chan;
1657 }
1658 }
1659 }
1660
1661 return best_chan;
1662}
1663EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1664
1665/**
1666 * ppc440spe_get_group_entry - get group entry with index idx
1667 * @tdesc: is the last allocated slot in the group.
1668 */
1669static struct ppc440spe_adma_desc_slot *
1670ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1671{
1672 struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1673 int i = 0;
1674
1675 if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1676 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1677 __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1678 BUG();
1679 }
1680
1681 list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1682 if (i++ == entry_idx)
1683 break;
1684 }
1685 return iter;
1686}
1687
1688/**
1689 * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1690 * @slot: Slot to free
1691 * Caller must hold &ppc440spe_chan->lock while calling this function
1692 */
1693static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1694 struct ppc440spe_adma_chan *chan)
1695{
1696 int stride = slot->slots_per_op;
1697
1698 while (stride--) {
1699 slot->slots_per_op = 0;
1700 slot = list_entry(slot->slot_node.next,
1701 struct ppc440spe_adma_desc_slot,
1702 slot_node);
1703 }
1704}
1705
1706static void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan,
1707 struct ppc440spe_adma_desc_slot *desc)
1708{
1709 u32 src_cnt, dst_cnt;
1710 dma_addr_t addr;
1711
1712 /*
1713 * get the number of sources & destination
1714 * included in this descriptor and unmap
1715 * them all
1716 */
1717 src_cnt = ppc440spe_desc_get_src_num(desc, chan);
1718 dst_cnt = ppc440spe_desc_get_dst_num(desc, chan);
1719
1720 /* unmap destinations */
1721 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1722 while (dst_cnt--) {
1723 addr = ppc440spe_desc_get_dest_addr(
1724 desc, chan, dst_cnt);
1725 dma_unmap_page(chan->device->dev,
1726 addr, desc->unmap_len,
1727 DMA_FROM_DEVICE);
1728 }
1729 }
1730
1731 /* unmap sources */
1732 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1733 while (src_cnt--) {
1734 addr = ppc440spe_desc_get_src_addr(
1735 desc, chan, src_cnt);
1736 dma_unmap_page(chan->device->dev,
1737 addr, desc->unmap_len,
1738 DMA_TO_DEVICE);
1739 }
1740 }
1741}
1742
1743/**
1744 * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1745 * upon completion
1746 */
1747static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1748 struct ppc440spe_adma_desc_slot *desc,
1749 struct ppc440spe_adma_chan *chan,
1750 dma_cookie_t cookie)
1751{
1752 int i;
1753
1754 BUG_ON(desc->async_tx.cookie < 0);
1755 if (desc->async_tx.cookie > 0) {
1756 cookie = desc->async_tx.cookie;
1757 desc->async_tx.cookie = 0;
1758
1759 /* call the callback (must not sleep or submit new
1760 * operations to this channel)
1761 */
1762 if (desc->async_tx.callback)
1763 desc->async_tx.callback(
1764 desc->async_tx.callback_param);
1765
1766 /* unmap dma addresses
1767 * (unmap_single vs unmap_page?)
1768 *
1769 * actually, ppc's dma_unmap_page() functions are empty, so
1770 * the following code is just for the sake of completeness
1771 */
1772 if (chan && chan->needs_unmap && desc->group_head &&
1773 desc->unmap_len) {
1774 struct ppc440spe_adma_desc_slot *unmap =
1775 desc->group_head;
1776 /* assume 1 slot per op always */
1777 u32 slot_count = unmap->slot_cnt;
1778
1779 /* Run through the group list and unmap addresses */
1780 for (i = 0; i < slot_count; i++) {
1781 BUG_ON(!unmap);
1782 ppc440spe_adma_unmap(chan, unmap);
1783 unmap = unmap->hw_next;
1784 }
1785 }
1786 }
1787
1788 /* run dependent operations */
1789 dma_run_dependencies(&desc->async_tx);
1790
1791 return cookie;
1792}
1793
1794/**
1795 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1796 */
1797static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1798 struct ppc440spe_adma_chan *chan)
1799{
1800 /* the client is allowed to attach dependent operations
1801 * until 'ack' is set
1802 */
1803 if (!async_tx_test_ack(&desc->async_tx))
1804 return 0;
1805
1806 /* leave the last descriptor in the chain
1807 * so we can append to it
1808 */
1809 if (list_is_last(&desc->chain_node, &chan->chain) ||
1810 desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1811 return 1;
1812
1813 if (chan->device->id != PPC440SPE_XOR_ID) {
1814 /* our DMA interrupt handler clears opc field of
1815 * each processed descriptor. For all types of
1816 * operations except for ZeroSum we do not actually
1817 * need ack from the interrupt handler. ZeroSum is a
1818 * special case since the result of this operation
1819 * is available from the handler only, so if we see
1820 * such type of descriptor (which is unprocessed yet)
1821 * then leave it in chain.
1822 */
1823 struct dma_cdb *cdb = desc->hw_desc;
1824 if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1825 return 1;
1826 }
1827
1828 dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1829 desc->phys, desc->idx, desc->slots_per_op);
1830
1831 list_del(&desc->chain_node);
1832 ppc440spe_adma_free_slots(desc, chan);
1833 return 0;
1834}
1835
1836/**
1837 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1838 * which runs through the channel CDBs list until reach the descriptor
1839 * currently processed. When routine determines that all CDBs of group
1840 * are completed then corresponding callbacks (if any) are called and slots
1841 * are freed.
1842 */
1843static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1844{
1845 struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1846 dma_cookie_t cookie = 0;
1847 u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1848 int busy = ppc440spe_chan_is_busy(chan);
1849 int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1850
1851 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1852 chan->device->id, __func__);
1853
1854 if (!current_desc) {
1855 /* There were no transactions yet, so
1856 * nothing to clean
1857 */
1858 return;
1859 }
1860
1861 /* free completed slots from the chain starting with
1862 * the oldest descriptor
1863 */
1864 list_for_each_entry_safe(iter, _iter, &chan->chain,
1865 chain_node) {
1866 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1867 "busy: %d this_desc: %#llx next_desc: %#x "
1868 "cur: %#x ack: %d\n",
1869 iter->async_tx.cookie, iter->idx, busy, iter->phys,
1870 ppc440spe_desc_get_link(iter, chan), current_desc,
1871 async_tx_test_ack(&iter->async_tx));
1872 prefetch(_iter);
1873 prefetch(&_iter->async_tx);
1874
1875 /* do not advance past the current descriptor loaded into the
1876 * hardware channel,subsequent descriptors are either in process
1877 * or have not been submitted
1878 */
1879 if (seen_current)
1880 break;
1881
1882 /* stop the search if we reach the current descriptor and the
1883 * channel is busy, or if it appears that the current descriptor
1884 * needs to be re-read (i.e. has been appended to)
1885 */
1886 if (iter->phys == current_desc) {
1887 BUG_ON(seen_current++);
1888 if (busy || ppc440spe_desc_get_link(iter, chan)) {
1889 /* not all descriptors of the group have
1890 * been completed; exit.
1891 */
1892 break;
1893 }
1894 }
1895
1896 /* detect the start of a group transaction */
1897 if (!slot_cnt && !slots_per_op) {
1898 slot_cnt = iter->slot_cnt;
1899 slots_per_op = iter->slots_per_op;
1900 if (slot_cnt <= slots_per_op) {
1901 slot_cnt = 0;
1902 slots_per_op = 0;
1903 }
1904 }
1905
1906 if (slot_cnt) {
1907 if (!group_start)
1908 group_start = iter;
1909 slot_cnt -= slots_per_op;
1910 }
1911
1912 /* all the members of a group are complete */
1913 if (slots_per_op != 0 && slot_cnt == 0) {
1914 struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1915 int end_of_chain = 0;
1916
1917 /* clean up the group */
1918 slot_cnt = group_start->slot_cnt;
1919 grp_iter = group_start;
1920 list_for_each_entry_safe_from(grp_iter, _grp_iter,
1921 &chan->chain, chain_node) {
1922
1923 cookie = ppc440spe_adma_run_tx_complete_actions(
1924 grp_iter, chan, cookie);
1925
1926 slot_cnt -= slots_per_op;
1927 end_of_chain = ppc440spe_adma_clean_slot(
1928 grp_iter, chan);
1929 if (end_of_chain && slot_cnt) {
1930 /* Should wait for ZeroSum completion */
1931 if (cookie > 0)
1932 chan->completed_cookie = cookie;
1933 return;
1934 }
1935
1936 if (slot_cnt == 0 || end_of_chain)
1937 break;
1938 }
1939
1940 /* the group should be complete at this point */
1941 BUG_ON(slot_cnt);
1942
1943 slots_per_op = 0;
1944 group_start = NULL;
1945 if (end_of_chain)
1946 break;
1947 else
1948 continue;
1949 } else if (slots_per_op) /* wait for group completion */
1950 continue;
1951
1952 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1953 cookie);
1954
1955 if (ppc440spe_adma_clean_slot(iter, chan))
1956 break;
1957 }
1958
1959 BUG_ON(!seen_current);
1960
1961 if (cookie > 0) {
1962 chan->completed_cookie = cookie;
1963 pr_debug("\tcompleted cookie %d\n", cookie);
1964 }
1965
1966}
1967
1968/**
1969 * ppc440spe_adma_tasklet - clean up watch-dog initiator
1970 */
1971static void ppc440spe_adma_tasklet(unsigned long data)
1972{
1973 struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data;
1974
1975 spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1976 __ppc440spe_adma_slot_cleanup(chan);
1977 spin_unlock(&chan->lock);
1978}
1979
1980/**
1981 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1982 */
1983static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1984{
1985 spin_lock_bh(&chan->lock);
1986 __ppc440spe_adma_slot_cleanup(chan);
1987 spin_unlock_bh(&chan->lock);
1988}
1989
1990/**
1991 * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1992 */
1993static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1994 struct ppc440spe_adma_chan *chan, int num_slots,
1995 int slots_per_op)
1996{
1997 struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1998 struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1999 struct list_head chain = LIST_HEAD_INIT(chain);
2000 int slots_found, retry = 0;
2001
2002
2003 BUG_ON(!num_slots || !slots_per_op);
2004 /* start search from the last allocated descrtiptor
2005 * if a contiguous allocation can not be found start searching
2006 * from the beginning of the list
2007 */
2008retry:
2009 slots_found = 0;
2010 if (retry == 0)
2011 iter = chan->last_used;
2012 else
2013 iter = list_entry(&chan->all_slots,
2014 struct ppc440spe_adma_desc_slot,
2015 slot_node);
2016 list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
2017 slot_node) {
2018 prefetch(_iter);
2019 prefetch(&_iter->async_tx);
2020 if (iter->slots_per_op) {
2021 slots_found = 0;
2022 continue;
2023 }
2024
2025 /* start the allocation if the slot is correctly aligned */
2026 if (!slots_found++)
2027 alloc_start = iter;
2028
2029 if (slots_found == num_slots) {
2030 struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
2031 struct ppc440spe_adma_desc_slot *last_used = NULL;
2032
2033 iter = alloc_start;
2034 while (num_slots) {
2035 int i;
2036 /* pre-ack all but the last descriptor */
2037 if (num_slots != slots_per_op)
2038 async_tx_ack(&iter->async_tx);
2039
2040 list_add_tail(&iter->chain_node, &chain);
2041 alloc_tail = iter;
2042 iter->async_tx.cookie = 0;
2043 iter->hw_next = NULL;
2044 iter->flags = 0;
2045 iter->slot_cnt = num_slots;
2046 iter->xor_check_result = NULL;
2047 for (i = 0; i < slots_per_op; i++) {
2048 iter->slots_per_op = slots_per_op - i;
2049 last_used = iter;
2050 iter = list_entry(iter->slot_node.next,
2051 struct ppc440spe_adma_desc_slot,
2052 slot_node);
2053 }
2054 num_slots -= slots_per_op;
2055 }
2056 alloc_tail->group_head = alloc_start;
2057 alloc_tail->async_tx.cookie = -EBUSY;
2058 list_splice(&chain, &alloc_tail->group_list);
2059 chan->last_used = last_used;
2060 return alloc_tail;
2061 }
2062 }
2063 if (!retry++)
2064 goto retry;
2065
2066 /* try to free some slots if the allocation fails */
2067 tasklet_schedule(&chan->irq_tasklet);
2068 return NULL;
2069}
2070
2071/**
2072 * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots
2073 */
2074static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
2075{
2076 struct ppc440spe_adma_chan *ppc440spe_chan;
2077 struct ppc440spe_adma_desc_slot *slot = NULL;
2078 char *hw_desc;
2079 int i, db_sz;
2080 int init;
2081
2082 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2083 init = ppc440spe_chan->slots_allocated ? 0 : 1;
2084 chan->chan_id = ppc440spe_chan->device->id;
2085
2086 /* Allocate descriptor slots */
2087 i = ppc440spe_chan->slots_allocated;
2088 if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
2089 db_sz = sizeof(struct dma_cdb);
2090 else
2091 db_sz = sizeof(struct xor_cb);
2092
2093 for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
2094 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
2095 GFP_KERNEL);
2096 if (!slot) {
2097 printk(KERN_INFO "SPE ADMA Channel only initialized"
2098 " %d descriptor slots", i--);
2099 break;
2100 }
2101
2102 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
2103 slot->hw_desc = (void *) &hw_desc[i * db_sz];
2104 dma_async_tx_descriptor_init(&slot->async_tx, chan);
2105 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
2106 INIT_LIST_HEAD(&slot->chain_node);
2107 INIT_LIST_HEAD(&slot->slot_node);
2108 INIT_LIST_HEAD(&slot->group_list);
2109 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
2110 slot->idx = i;
2111
2112 spin_lock_bh(&ppc440spe_chan->lock);
2113 ppc440spe_chan->slots_allocated++;
2114 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
2115 spin_unlock_bh(&ppc440spe_chan->lock);
2116 }
2117
2118 if (i && !ppc440spe_chan->last_used) {
2119 ppc440spe_chan->last_used =
2120 list_entry(ppc440spe_chan->all_slots.next,
2121 struct ppc440spe_adma_desc_slot,
2122 slot_node);
2123 }
2124
2125 dev_dbg(ppc440spe_chan->device->common.dev,
2126 "ppc440spe adma%d: allocated %d descriptor slots\n",
2127 ppc440spe_chan->device->id, i);
2128
2129 /* initialize the channel and the chain with a null operation */
2130 if (init) {
2131 switch (ppc440spe_chan->device->id) {
2132 case PPC440SPE_DMA0_ID:
2133 case PPC440SPE_DMA1_ID:
2134 ppc440spe_chan->hw_chain_inited = 0;
2135 /* Use WXOR for self-testing */
2136 if (!ppc440spe_r6_tchan)
2137 ppc440spe_r6_tchan = ppc440spe_chan;
2138 break;
2139 case PPC440SPE_XOR_ID:
2140 ppc440spe_chan_start_null_xor(ppc440spe_chan);
2141 break;
2142 default:
2143 BUG();
2144 }
2145 ppc440spe_chan->needs_unmap = 1;
2146 }
2147
2148 return (i > 0) ? i : -ENOMEM;
2149}
2150
2151/**
2152 * ppc440spe_desc_assign_cookie - assign a cookie
2153 */
2154static dma_cookie_t ppc440spe_desc_assign_cookie(
2155 struct ppc440spe_adma_chan *chan,
2156 struct ppc440spe_adma_desc_slot *desc)
2157{
2158 dma_cookie_t cookie = chan->common.cookie;
2159
2160 cookie++;
2161 if (cookie < 0)
2162 cookie = 1;
2163 chan->common.cookie = desc->async_tx.cookie = cookie;
2164 return cookie;
2165}
2166
2167/**
2168 * ppc440spe_rxor_set_region_data -
2169 */
2170static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
2171 u8 xor_arg_no, u32 mask)
2172{
2173 struct xor_cb *xcb = desc->hw_desc;
2174
2175 xcb->ops[xor_arg_no].h |= mask;
2176}
2177
2178/**
2179 * ppc440spe_rxor_set_src -
2180 */
2181static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
2182 u8 xor_arg_no, dma_addr_t addr)
2183{
2184 struct xor_cb *xcb = desc->hw_desc;
2185
2186 xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
2187 xcb->ops[xor_arg_no].l = addr;
2188}
2189
2190/**
2191 * ppc440spe_rxor_set_mult -
2192 */
2193static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
2194 u8 xor_arg_no, u8 idx, u8 mult)
2195{
2196 struct xor_cb *xcb = desc->hw_desc;
2197
2198 xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
2199}
2200
2201/**
2202 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
2203 * has been achieved
2204 */
2205static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
2206{
2207 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
2208 chan->device->id, chan->pending);
2209
2210 if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
2211 chan->pending = 0;
2212 ppc440spe_chan_append(chan);
2213 }
2214}
2215
2216/**
2217 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
2218 * (it's not necessary that descriptors will be submitted to the h/w
2219 * chains too right now)
2220 */
2221static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
2222{
2223 struct ppc440spe_adma_desc_slot *sw_desc;
2224 struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
2225 struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
2226 int slot_cnt;
2227 int slots_per_op;
2228 dma_cookie_t cookie;
2229
2230 sw_desc = tx_to_ppc440spe_adma_slot(tx);
2231
2232 group_start = sw_desc->group_head;
2233 slot_cnt = group_start->slot_cnt;
2234 slots_per_op = group_start->slots_per_op;
2235
2236 spin_lock_bh(&chan->lock);
2237
2238 cookie = ppc440spe_desc_assign_cookie(chan, sw_desc);
2239
2240 if (unlikely(list_empty(&chan->chain))) {
2241 /* first peer */
2242 list_splice_init(&sw_desc->group_list, &chan->chain);
2243 chan_first_cdb[chan->device->id] = group_start;
2244 } else {
2245 /* isn't first peer, bind CDBs to chain */
2246 old_chain_tail = list_entry(chan->chain.prev,
2247 struct ppc440spe_adma_desc_slot,
2248 chain_node);
2249 list_splice_init(&sw_desc->group_list,
2250 &old_chain_tail->chain_node);
2251 /* fix up the hardware chain */
2252 ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
2253 }
2254
2255 /* increment the pending count by the number of operations */
2256 chan->pending += slot_cnt / slots_per_op;
2257 ppc440spe_adma_check_threshold(chan);
2258 spin_unlock_bh(&chan->lock);
2259
2260 dev_dbg(chan->device->common.dev,
2261 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
2262 chan->device->id, __func__,
2263 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
2264
2265 return cookie;
2266}
2267
2268/**
2269 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
2270 */
2271static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
2272 struct dma_chan *chan, unsigned long flags)
2273{
2274 struct ppc440spe_adma_chan *ppc440spe_chan;
2275 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2276 int slot_cnt, slots_per_op;
2277
2278 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2279
2280 dev_dbg(ppc440spe_chan->device->common.dev,
2281 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
2282 __func__);
2283
2284 spin_lock_bh(&ppc440spe_chan->lock);
2285 slot_cnt = slots_per_op = 1;
2286 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2287 slots_per_op);
2288 if (sw_desc) {
2289 group_start = sw_desc->group_head;
2290 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
2291 group_start->unmap_len = 0;
2292 sw_desc->async_tx.flags = flags;
2293 }
2294 spin_unlock_bh(&ppc440spe_chan->lock);
2295
2296 return sw_desc ? &sw_desc->async_tx : NULL;
2297}
2298
2299/**
2300 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
2301 */
2302static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
2303 struct dma_chan *chan, dma_addr_t dma_dest,
2304 dma_addr_t dma_src, size_t len, unsigned long flags)
2305{
2306 struct ppc440spe_adma_chan *ppc440spe_chan;
2307 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2308 int slot_cnt, slots_per_op;
2309
2310 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2311
2312 if (unlikely(!len))
2313 return NULL;
2314
2315 BUG_ON(unlikely(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT));
2316
2317 spin_lock_bh(&ppc440spe_chan->lock);
2318
2319 dev_dbg(ppc440spe_chan->device->common.dev,
2320 "ppc440spe adma%d: %s len: %u int_en %d\n",
2321 ppc440spe_chan->device->id, __func__, len,
2322 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2323 slot_cnt = slots_per_op = 1;
2324 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2325 slots_per_op);
2326 if (sw_desc) {
2327 group_start = sw_desc->group_head;
2328 ppc440spe_desc_init_memcpy(group_start, flags);
2329 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2330 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2331 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2332 sw_desc->unmap_len = len;
2333 sw_desc->async_tx.flags = flags;
2334 }
2335 spin_unlock_bh(&ppc440spe_chan->lock);
2336
2337 return sw_desc ? &sw_desc->async_tx : NULL;
2338}
2339
2340/**
2341 * ppc440spe_adma_prep_dma_memset - prepare CDB for a MEMSET operation
2342 */
2343static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memset(
2344 struct dma_chan *chan, dma_addr_t dma_dest, int value,
2345 size_t len, unsigned long flags)
2346{
2347 struct ppc440spe_adma_chan *ppc440spe_chan;
2348 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2349 int slot_cnt, slots_per_op;
2350
2351 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2352
2353 if (unlikely(!len))
2354 return NULL;
2355
2356 BUG_ON(unlikely(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT));
2357
2358 spin_lock_bh(&ppc440spe_chan->lock);
2359
2360 dev_dbg(ppc440spe_chan->device->common.dev,
2361 "ppc440spe adma%d: %s cal: %u len: %u int_en %d\n",
2362 ppc440spe_chan->device->id, __func__, value, len,
2363 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2364
2365 slot_cnt = slots_per_op = 1;
2366 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2367 slots_per_op);
2368 if (sw_desc) {
2369 group_start = sw_desc->group_head;
2370 ppc440spe_desc_init_memset(group_start, value, flags);
2371 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2372 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2373 sw_desc->unmap_len = len;
2374 sw_desc->async_tx.flags = flags;
2375 }
2376 spin_unlock_bh(&ppc440spe_chan->lock);
2377
2378 return sw_desc ? &sw_desc->async_tx : NULL;
2379}
2380
2381/**
2382 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2383 */
2384static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2385 struct dma_chan *chan, dma_addr_t dma_dest,
2386 dma_addr_t *dma_src, u32 src_cnt, size_t len,
2387 unsigned long flags)
2388{
2389 struct ppc440spe_adma_chan *ppc440spe_chan;
2390 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2391 int slot_cnt, slots_per_op;
2392
2393 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2394
2395 ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2396 dma_dest, dma_src, src_cnt));
2397 if (unlikely(!len))
2398 return NULL;
2399 BUG_ON(unlikely(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT));
2400
2401 dev_dbg(ppc440spe_chan->device->common.dev,
2402 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2403 ppc440spe_chan->device->id, __func__, src_cnt, len,
2404 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2405
2406 spin_lock_bh(&ppc440spe_chan->lock);
2407 slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2408 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2409 slots_per_op);
2410 if (sw_desc) {
2411 group_start = sw_desc->group_head;
2412 ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2413 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2414 while (src_cnt--)
2415 ppc440spe_adma_memcpy_xor_set_src(group_start,
2416 dma_src[src_cnt], src_cnt);
2417 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2418 sw_desc->unmap_len = len;
2419 sw_desc->async_tx.flags = flags;
2420 }
2421 spin_unlock_bh(&ppc440spe_chan->lock);
2422
2423 return sw_desc ? &sw_desc->async_tx : NULL;
2424}
2425
2426static inline void
2427ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2428 int src_cnt);
2429static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2430
2431/**
2432 * ppc440spe_adma_init_dma2rxor_slot -
2433 */
2434static void ppc440spe_adma_init_dma2rxor_slot(
2435 struct ppc440spe_adma_desc_slot *desc,
2436 dma_addr_t *src, int src_cnt)
2437{
2438 int i;
2439
2440 /* initialize CDB */
2441 for (i = 0; i < src_cnt; i++) {
2442 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2443 desc->src_cnt, (u32)src[i]);
2444 }
2445}
2446
2447/**
2448 * ppc440spe_dma01_prep_mult -
2449 * for Q operation where destination is also the source
2450 */
2451static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2452 struct ppc440spe_adma_chan *ppc440spe_chan,
2453 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2454 const unsigned char *scf, size_t len, unsigned long flags)
2455{
2456 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2457 unsigned long op = 0;
2458 int slot_cnt;
2459
2460 set_bit(PPC440SPE_DESC_WXOR, &op);
2461 slot_cnt = 2;
2462
2463 spin_lock_bh(&ppc440spe_chan->lock);
2464
2465 /* use WXOR, each descriptor occupies one slot */
2466 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2467 if (sw_desc) {
2468 struct ppc440spe_adma_chan *chan;
2469 struct ppc440spe_adma_desc_slot *iter;
2470 struct dma_cdb *hw_desc;
2471
2472 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2473 set_bits(op, &sw_desc->flags);
2474 sw_desc->src_cnt = src_cnt;
2475 sw_desc->dst_cnt = dst_cnt;
2476 /* First descriptor, zero data in the destination and copy it
2477 * to q page using MULTICAST transfer.
2478 */
2479 iter = list_first_entry(&sw_desc->group_list,
2480 struct ppc440spe_adma_desc_slot,
2481 chain_node);
2482 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2483 /* set 'next' pointer */
2484 iter->hw_next = list_entry(iter->chain_node.next,
2485 struct ppc440spe_adma_desc_slot,
2486 chain_node);
2487 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2488 hw_desc = iter->hw_desc;
2489 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2490
2491 ppc440spe_desc_set_dest_addr(iter, chan,
2492 DMA_CUED_XOR_BASE, dst[0], 0);
2493 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2494 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2495 src[0]);
2496 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2497 iter->unmap_len = len;
2498
2499 /*
2500 * Second descriptor, multiply data from the q page
2501 * and store the result in real destination.
2502 */
2503 iter = list_first_entry(&iter->chain_node,
2504 struct ppc440spe_adma_desc_slot,
2505 chain_node);
2506 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2507 iter->hw_next = NULL;
2508 if (flags & DMA_PREP_INTERRUPT)
2509 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2510 else
2511 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2512
2513 hw_desc = iter->hw_desc;
2514 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2515 ppc440spe_desc_set_src_addr(iter, chan, 0,
2516 DMA_CUED_XOR_HB, dst[1]);
2517 ppc440spe_desc_set_dest_addr(iter, chan,
2518 DMA_CUED_XOR_BASE, dst[0], 0);
2519
2520 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2521 DMA_CDB_SG_DST1, scf[0]);
2522 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2523 iter->unmap_len = len;
2524 sw_desc->async_tx.flags = flags;
2525 }
2526
2527 spin_unlock_bh(&ppc440spe_chan->lock);
2528
2529 return sw_desc;
2530}
2531
2532/**
2533 * ppc440spe_dma01_prep_sum_product -
2534 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2535 * the source.
2536 */
2537static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2538 struct ppc440spe_adma_chan *ppc440spe_chan,
2539 dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2540 const unsigned char *scf, size_t len, unsigned long flags)
2541{
2542 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2543 unsigned long op = 0;
2544 int slot_cnt;
2545
2546 set_bit(PPC440SPE_DESC_WXOR, &op);
2547 slot_cnt = 3;
2548
2549 spin_lock_bh(&ppc440spe_chan->lock);
2550
2551 /* WXOR, each descriptor occupies one slot */
2552 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2553 if (sw_desc) {
2554 struct ppc440spe_adma_chan *chan;
2555 struct ppc440spe_adma_desc_slot *iter;
2556 struct dma_cdb *hw_desc;
2557
2558 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2559 set_bits(op, &sw_desc->flags);
2560 sw_desc->src_cnt = src_cnt;
2561 sw_desc->dst_cnt = 1;
2562 /* 1st descriptor, src[1] data to q page and zero destination */
2563 iter = list_first_entry(&sw_desc->group_list,
2564 struct ppc440spe_adma_desc_slot,
2565 chain_node);
2566 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2567 iter->hw_next = list_entry(iter->chain_node.next,
2568 struct ppc440spe_adma_desc_slot,
2569 chain_node);
2570 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2571 hw_desc = iter->hw_desc;
2572 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2573
2574 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2575 *dst, 0);
2576 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2577 ppc440spe_chan->qdest, 1);
2578 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2579 src[1]);
2580 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2581 iter->unmap_len = len;
2582
2583 /* 2nd descriptor, multiply src[1] data and store the
2584 * result in destination */
2585 iter = list_first_entry(&iter->chain_node,
2586 struct ppc440spe_adma_desc_slot,
2587 chain_node);
2588 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2589 /* set 'next' pointer */
2590 iter->hw_next = list_entry(iter->chain_node.next,
2591 struct ppc440spe_adma_desc_slot,
2592 chain_node);
2593 if (flags & DMA_PREP_INTERRUPT)
2594 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2595 else
2596 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2597
2598 hw_desc = iter->hw_desc;
2599 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2600 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2601 ppc440spe_chan->qdest);
2602 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2603 *dst, 0);
2604 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2605 DMA_CDB_SG_DST1, scf[1]);
2606 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2607 iter->unmap_len = len;
2608
2609 /*
2610 * 3rd descriptor, multiply src[0] data and xor it
2611 * with destination
2612 */
2613 iter = list_first_entry(&iter->chain_node,
2614 struct ppc440spe_adma_desc_slot,
2615 chain_node);
2616 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2617 iter->hw_next = NULL;
2618 if (flags & DMA_PREP_INTERRUPT)
2619 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2620 else
2621 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2622
2623 hw_desc = iter->hw_desc;
2624 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2625 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2626 src[0]);
2627 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2628 *dst, 0);
2629 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2630 DMA_CDB_SG_DST1, scf[0]);
2631 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2632 iter->unmap_len = len;
2633 sw_desc->async_tx.flags = flags;
2634 }
2635
2636 spin_unlock_bh(&ppc440spe_chan->lock);
2637
2638 return sw_desc;
2639}
2640
2641static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2642 struct ppc440spe_adma_chan *ppc440spe_chan,
2643 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2644 const unsigned char *scf, size_t len, unsigned long flags)
2645{
2646 int slot_cnt;
2647 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2648 unsigned long op = 0;
2649 unsigned char mult = 1;
2650
2651 pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2652 __func__, dst_cnt, src_cnt, len);
2653 /* select operations WXOR/RXOR depending on the
2654 * source addresses of operators and the number
2655 * of destinations (RXOR support only Q-parity calculations)
2656 */
2657 set_bit(PPC440SPE_DESC_WXOR, &op);
2658 if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2659 /* no active RXOR;
2660 * do RXOR if:
2661 * - there are more than 1 source,
2662 * - len is aligned on 512-byte boundary,
2663 * - source addresses fit to one of 4 possible regions.
2664 */
2665 if (src_cnt > 1 &&
2666 !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2667 (src[0] + len) == src[1]) {
2668 /* may do RXOR R1 R2 */
2669 set_bit(PPC440SPE_DESC_RXOR, &op);
2670 if (src_cnt != 2) {
2671 /* may try to enhance region of RXOR */
2672 if ((src[1] + len) == src[2]) {
2673 /* do RXOR R1 R2 R3 */
2674 set_bit(PPC440SPE_DESC_RXOR123,
2675 &op);
2676 } else if ((src[1] + len * 2) == src[2]) {
2677 /* do RXOR R1 R2 R4 */
2678 set_bit(PPC440SPE_DESC_RXOR124, &op);
2679 } else if ((src[1] + len * 3) == src[2]) {
2680 /* do RXOR R1 R2 R5 */
2681 set_bit(PPC440SPE_DESC_RXOR125,
2682 &op);
2683 } else {
2684 /* do RXOR R1 R2 */
2685 set_bit(PPC440SPE_DESC_RXOR12,
2686 &op);
2687 }
2688 } else {
2689 /* do RXOR R1 R2 */
2690 set_bit(PPC440SPE_DESC_RXOR12, &op);
2691 }
2692 }
2693
2694 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2695 /* can not do this operation with RXOR */
2696 clear_bit(PPC440SPE_RXOR_RUN,
2697 &ppc440spe_rxor_state);
2698 } else {
2699 /* can do; set block size right now */
2700 ppc440spe_desc_set_rxor_block_size(len);
2701 }
2702 }
2703
2704 /* Number of necessary slots depends on operation type selected */
2705 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2706 /* This is a WXOR only chain. Need descriptors for each
2707 * source to GF-XOR them with WXOR, and need descriptors
2708 * for each destination to zero them with WXOR
2709 */
2710 slot_cnt = src_cnt;
2711
2712 if (flags & DMA_PREP_ZERO_P) {
2713 slot_cnt++;
2714 set_bit(PPC440SPE_ZERO_P, &op);
2715 }
2716 if (flags & DMA_PREP_ZERO_Q) {
2717 slot_cnt++;
2718 set_bit(PPC440SPE_ZERO_Q, &op);
2719 }
2720 } else {
2721 /* Need 1/2 descriptor for RXOR operation, and
2722 * need (src_cnt - (2 or 3)) for WXOR of sources
2723 * remained (if any)
2724 */
2725 slot_cnt = dst_cnt;
2726
2727 if (flags & DMA_PREP_ZERO_P)
2728 set_bit(PPC440SPE_ZERO_P, &op);
2729 if (flags & DMA_PREP_ZERO_Q)
2730 set_bit(PPC440SPE_ZERO_Q, &op);
2731
2732 if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2733 slot_cnt += src_cnt - 2;
2734 else
2735 slot_cnt += src_cnt - 3;
2736
2737 /* Thus we have either RXOR only chain or
2738 * mixed RXOR/WXOR
2739 */
2740 if (slot_cnt == dst_cnt)
2741 /* RXOR only chain */
2742 clear_bit(PPC440SPE_DESC_WXOR, &op);
2743 }
2744
2745 spin_lock_bh(&ppc440spe_chan->lock);
2746 /* for both RXOR/WXOR each descriptor occupies one slot */
2747 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2748 if (sw_desc) {
2749 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2750 flags, op);
2751
2752 /* setup dst/src/mult */
2753 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2754 __func__, dst[0], dst[1]);
2755 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2756 while (src_cnt--) {
2757 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2758 src_cnt);
2759
2760 /* NOTE: "Multi = 0 is equivalent to = 1" as it
2761 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2762 * doesn't work for RXOR with DMA0/1! Instead, multi=0
2763 * leads to zeroing source data after RXOR.
2764 * So, for P case set-up mult=1 explicitly.
2765 */
2766 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2767 mult = scf[src_cnt];
2768 ppc440spe_adma_pq_set_src_mult(sw_desc,
2769 mult, src_cnt, dst_cnt - 1);
2770 }
2771
2772 /* Setup byte count foreach slot just allocated */
2773 sw_desc->async_tx.flags = flags;
2774 list_for_each_entry(iter, &sw_desc->group_list,
2775 chain_node) {
2776 ppc440spe_desc_set_byte_count(iter,
2777 ppc440spe_chan, len);
2778 iter->unmap_len = len;
2779 }
2780 }
2781 spin_unlock_bh(&ppc440spe_chan->lock);
2782
2783 return sw_desc;
2784}
2785
2786static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2787 struct ppc440spe_adma_chan *ppc440spe_chan,
2788 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2789 const unsigned char *scf, size_t len, unsigned long flags)
2790{
2791 int slot_cnt, descs_per_op;
2792 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2793 unsigned long op = 0;
2794 unsigned char mult = 1;
2795
2796 BUG_ON(!dst_cnt);
2797 /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2798 __func__, dst_cnt, src_cnt, len);*/
2799
2800 spin_lock_bh(&ppc440spe_chan->lock);
2801 descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2802 if (descs_per_op < 0) {
2803 spin_unlock_bh(&ppc440spe_chan->lock);
2804 return NULL;
2805 }
2806
2807 /* depending on number of sources we have 1 or 2 RXOR chains */
2808 slot_cnt = descs_per_op * dst_cnt;
2809
2810 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2811 if (sw_desc) {
2812 op = slot_cnt;
2813 sw_desc->async_tx.flags = flags;
2814 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2815 ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2816 --op ? 0 : flags);
2817 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2818 len);
2819 iter->unmap_len = len;
2820
2821 ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2822 iter->rxor_cursor.len = len;
2823 iter->descs_per_op = descs_per_op;
2824 }
2825 op = 0;
2826 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2827 op++;
2828 if (op % descs_per_op == 0)
2829 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2830 src_cnt);
2831 if (likely(!list_is_last(&iter->chain_node,
2832 &sw_desc->group_list))) {
2833 /* set 'next' pointer */
2834 iter->hw_next =
2835 list_entry(iter->chain_node.next,
2836 struct ppc440spe_adma_desc_slot,
2837 chain_node);
2838 ppc440spe_xor_set_link(iter, iter->hw_next);
2839 } else {
2840 /* this is the last descriptor. */
2841 iter->hw_next = NULL;
2842 }
2843 }
2844
2845 /* fixup head descriptor */
2846 sw_desc->dst_cnt = dst_cnt;
2847 if (flags & DMA_PREP_ZERO_P)
2848 set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2849 if (flags & DMA_PREP_ZERO_Q)
2850 set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2851
2852 /* setup dst/src/mult */
2853 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2854
2855 while (src_cnt--) {
2856 /* handle descriptors (if dst_cnt == 2) inside
2857 * the ppc440spe_adma_pq_set_srcxxx() functions
2858 */
2859 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2860 src_cnt);
2861 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2862 mult = scf[src_cnt];
2863 ppc440spe_adma_pq_set_src_mult(sw_desc,
2864 mult, src_cnt, dst_cnt - 1);
2865 }
2866 }
2867 spin_unlock_bh(&ppc440spe_chan->lock);
2868 ppc440spe_desc_set_rxor_block_size(len);
2869 return sw_desc;
2870}
2871
2872/**
2873 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2874 */
2875static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2876 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2877 unsigned int src_cnt, const unsigned char *scf,
2878 size_t len, unsigned long flags)
2879{
2880 struct ppc440spe_adma_chan *ppc440spe_chan;
2881 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2882 int dst_cnt = 0;
2883
2884 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2885
2886 ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2887 dst, src, src_cnt));
2888 BUG_ON(!len);
2889 BUG_ON(unlikely(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT));
2890 BUG_ON(!src_cnt);
2891
2892 if (src_cnt == 1 && dst[1] == src[0]) {
2893 dma_addr_t dest[2];
2894
2895 /* dst[1] is real destination (Q) */
2896 dest[0] = dst[1];
2897 /* this is the page to multicast source data to */
2898 dest[1] = ppc440spe_chan->qdest;
2899 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2900 dest, 2, src, src_cnt, scf, len, flags);
2901 return sw_desc ? &sw_desc->async_tx : NULL;
2902 }
2903
2904 if (src_cnt == 2 && dst[1] == src[1]) {
2905 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2906 &dst[1], src, 2, scf, len, flags);
2907 return sw_desc ? &sw_desc->async_tx : NULL;
2908 }
2909
2910 if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2911 BUG_ON(!dst[0]);
2912 dst_cnt++;
2913 flags |= DMA_PREP_ZERO_P;
2914 }
2915
2916 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2917 BUG_ON(!dst[1]);
2918 dst_cnt++;
2919 flags |= DMA_PREP_ZERO_Q;
2920 }
2921
2922 BUG_ON(!dst_cnt);
2923
2924 dev_dbg(ppc440spe_chan->device->common.dev,
2925 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2926 ppc440spe_chan->device->id, __func__, src_cnt, len,
2927 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2928
2929 switch (ppc440spe_chan->device->id) {
2930 case PPC440SPE_DMA0_ID:
2931 case PPC440SPE_DMA1_ID:
2932 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2933 dst, dst_cnt, src, src_cnt, scf,
2934 len, flags);
2935 break;
2936
2937 case PPC440SPE_XOR_ID:
2938 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2939 dst, dst_cnt, src, src_cnt, scf,
2940 len, flags);
2941 break;
2942 }
2943
2944 return sw_desc ? &sw_desc->async_tx : NULL;
2945}
2946
2947/**
2948 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2949 * a PQ_ZERO_SUM operation
2950 */
2951static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2952 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2953 unsigned int src_cnt, const unsigned char *scf, size_t len,
2954 enum sum_check_flags *pqres, unsigned long flags)
2955{
2956 struct ppc440spe_adma_chan *ppc440spe_chan;
2957 struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2958 dma_addr_t pdest, qdest;
2959 int slot_cnt, slots_per_op, idst, dst_cnt;
2960
2961 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2962
2963 if (flags & DMA_PREP_PQ_DISABLE_P)
2964 pdest = 0;
2965 else
2966 pdest = pq[0];
2967
2968 if (flags & DMA_PREP_PQ_DISABLE_Q)
2969 qdest = 0;
2970 else
2971 qdest = pq[1];
2972
2973 ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2974 src, src_cnt, scf));
2975
2976 /* Always use WXOR for P/Q calculations (two destinations).
2977 * Need 1 or 2 extra slots to verify results are zero.
2978 */
2979 idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2980
2981 /* One additional slot per destination to clone P/Q
2982 * before calculation (we have to preserve destinations).
2983 */
2984 slot_cnt = src_cnt + dst_cnt * 2;
2985 slots_per_op = 1;
2986
2987 spin_lock_bh(&ppc440spe_chan->lock);
2988 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2989 slots_per_op);
2990 if (sw_desc) {
2991 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2992
2993 /* Setup byte count for each slot just allocated */
2994 sw_desc->async_tx.flags = flags;
2995 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2996 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2997 len);
2998 iter->unmap_len = len;
2999 }
3000
3001 if (pdest) {
3002 struct dma_cdb *hw_desc;
3003 struct ppc440spe_adma_chan *chan;
3004
3005 iter = sw_desc->group_head;
3006 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3007 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
3008 iter->hw_next = list_entry(iter->chain_node.next,
3009 struct ppc440spe_adma_desc_slot,
3010 chain_node);
3011 hw_desc = iter->hw_desc;
3012 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
3013 iter->src_cnt = 0;
3014 iter->dst_cnt = 0;
3015 ppc440spe_desc_set_dest_addr(iter, chan, 0,
3016 ppc440spe_chan->pdest, 0);
3017 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
3018 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
3019 len);
3020 iter->unmap_len = 0;
3021 /* override pdest to preserve original P */
3022 pdest = ppc440spe_chan->pdest;
3023 }
3024 if (qdest) {
3025 struct dma_cdb *hw_desc;
3026 struct ppc440spe_adma_chan *chan;
3027
3028 iter = list_first_entry(&sw_desc->group_list,
3029 struct ppc440spe_adma_desc_slot,
3030 chain_node);
3031 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3032
3033 if (pdest) {
3034 iter = list_entry(iter->chain_node.next,
3035 struct ppc440spe_adma_desc_slot,
3036 chain_node);
3037 }
3038
3039 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
3040 iter->hw_next = list_entry(iter->chain_node.next,
3041 struct ppc440spe_adma_desc_slot,
3042 chain_node);
3043 hw_desc = iter->hw_desc;
3044 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
3045 iter->src_cnt = 0;
3046 iter->dst_cnt = 0;
3047 ppc440spe_desc_set_dest_addr(iter, chan, 0,
3048 ppc440spe_chan->qdest, 0);
3049 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
3050 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
3051 len);
3052 iter->unmap_len = 0;
3053 /* override qdest to preserve original Q */
3054 qdest = ppc440spe_chan->qdest;
3055 }
3056
3057 /* Setup destinations for P/Q ops */
3058 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
3059
3060 /* Setup zero QWORDs into DCHECK CDBs */
3061 idst = dst_cnt;
3062 list_for_each_entry_reverse(iter, &sw_desc->group_list,
3063 chain_node) {
3064 /*
3065 * The last CDB corresponds to Q-parity check,
3066 * the one before last CDB corresponds
3067 * P-parity check
3068 */
3069 if (idst == DMA_DEST_MAX_NUM) {
3070 if (idst == dst_cnt) {
3071 set_bit(PPC440SPE_DESC_QCHECK,
3072 &iter->flags);
3073 } else {
3074 set_bit(PPC440SPE_DESC_PCHECK,
3075 &iter->flags);
3076 }
3077 } else {
3078 if (qdest) {
3079 set_bit(PPC440SPE_DESC_QCHECK,
3080 &iter->flags);
3081 } else {
3082 set_bit(PPC440SPE_DESC_PCHECK,
3083 &iter->flags);
3084 }
3085 }
3086 iter->xor_check_result = pqres;
3087
3088 /*
3089 * set it to zero, if check fail then result will
3090 * be updated
3091 */
3092 *iter->xor_check_result = 0;
3093 ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
3094 ppc440spe_qword);
3095
3096 if (!(--dst_cnt))
3097 break;
3098 }
3099
3100 /* Setup sources and mults for P/Q ops */
3101 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
3102 chain_node) {
3103 struct ppc440spe_adma_chan *chan;
3104 u32 mult_dst;
3105
3106 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3107 ppc440spe_desc_set_src_addr(iter, chan, 0,
3108 DMA_CUED_XOR_HB,
3109 src[src_cnt - 1]);
3110 if (qdest) {
3111 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
3112 DMA_CDB_SG_DST1;
3113 ppc440spe_desc_set_src_mult(iter, chan,
3114 DMA_CUED_MULT1_OFF,
3115 mult_dst,
3116 scf[src_cnt - 1]);
3117 }
3118 if (!(--src_cnt))
3119 break;
3120 }
3121 }
3122 spin_unlock_bh(&ppc440spe_chan->lock);
3123 return sw_desc ? &sw_desc->async_tx : NULL;
3124}
3125
3126/**
3127 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
3128 * XOR ZERO_SUM operation
3129 */
3130static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
3131 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
3132 size_t len, enum sum_check_flags *result, unsigned long flags)
3133{
3134 struct dma_async_tx_descriptor *tx;
3135 dma_addr_t pq[2];
3136
3137 /* validate P, disable Q */
3138 pq[0] = src[0];
3139 pq[1] = 0;
3140 flags |= DMA_PREP_PQ_DISABLE_Q;
3141
3142 tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
3143 src_cnt - 1, 0, len,
3144 result, flags);
3145 return tx;
3146}
3147
3148/**
3149 * ppc440spe_adma_set_dest - set destination address into descriptor
3150 */
3151static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
3152 dma_addr_t addr, int index)
3153{
3154 struct ppc440spe_adma_chan *chan;
3155
3156 BUG_ON(index >= sw_desc->dst_cnt);
3157
3158 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3159
3160 switch (chan->device->id) {
3161 case PPC440SPE_DMA0_ID:
3162 case PPC440SPE_DMA1_ID:
3163 /* to do: support transfers lengths >
3164 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
3165 */
3166 ppc440spe_desc_set_dest_addr(sw_desc->group_head,
3167 chan, 0, addr, index);
3168 break;
3169 case PPC440SPE_XOR_ID:
3170 sw_desc = ppc440spe_get_group_entry(sw_desc, index);
3171 ppc440spe_desc_set_dest_addr(sw_desc,
3172 chan, 0, addr, index);
3173 break;
3174 }
3175}
3176
3177static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
3178 struct ppc440spe_adma_chan *chan, dma_addr_t addr)
3179{
3180 /* To clear destinations update the descriptor
3181 * (P or Q depending on index) as follows:
3182 * addr is destination (0 corresponds to SG2):
3183 */
3184 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
3185
3186 /* ... and the addr is source: */
3187 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
3188
3189 /* addr is always SG2 then the mult is always DST1 */
3190 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
3191 DMA_CDB_SG_DST1, 1);
3192}
3193
3194/**
3195 * ppc440spe_adma_pq_set_dest - set destination address into descriptor
3196 * for the PQXOR operation
3197 */
3198static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
3199 dma_addr_t *addrs, unsigned long flags)
3200{
3201 struct ppc440spe_adma_desc_slot *iter;
3202 struct ppc440spe_adma_chan *chan;
3203 dma_addr_t paddr, qaddr;
3204 dma_addr_t addr = 0, ppath, qpath;
3205 int index = 0, i;
3206
3207 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3208
3209 if (flags & DMA_PREP_PQ_DISABLE_P)
3210 paddr = 0;
3211 else
3212 paddr = addrs[0];
3213
3214 if (flags & DMA_PREP_PQ_DISABLE_Q)
3215 qaddr = 0;
3216 else
3217 qaddr = addrs[1];
3218
3219 if (!paddr || !qaddr)
3220 addr = paddr ? paddr : qaddr;
3221
3222 switch (chan->device->id) {
3223 case PPC440SPE_DMA0_ID:
3224 case PPC440SPE_DMA1_ID:
3225 /* walk through the WXOR source list and set P/Q-destinations
3226 * for each slot:
3227 */
3228 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3229 /* This is WXOR-only chain; may have 1/2 zero descs */
3230 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3231 index++;
3232 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3233 index++;
3234
3235 iter = ppc440spe_get_group_entry(sw_desc, index);
3236 if (addr) {
3237 /* one destination */
3238 list_for_each_entry_from(iter,
3239 &sw_desc->group_list, chain_node)
3240 ppc440spe_desc_set_dest_addr(iter, chan,
3241 DMA_CUED_XOR_BASE, addr, 0);
3242 } else {
3243 /* two destinations */
3244 list_for_each_entry_from(iter,
3245 &sw_desc->group_list, chain_node) {
3246 ppc440spe_desc_set_dest_addr(iter, chan,
3247 DMA_CUED_XOR_BASE, paddr, 0);
3248 ppc440spe_desc_set_dest_addr(iter, chan,
3249 DMA_CUED_XOR_BASE, qaddr, 1);
3250 }
3251 }
3252
3253 if (index) {
3254 /* To clear destinations update the descriptor
3255 * (1st,2nd, or both depending on flags)
3256 */
3257 index = 0;
3258 if (test_bit(PPC440SPE_ZERO_P,
3259 &sw_desc->flags)) {
3260 iter = ppc440spe_get_group_entry(
3261 sw_desc, index++);
3262 ppc440spe_adma_pq_zero_op(iter, chan,
3263 paddr);
3264 }
3265
3266 if (test_bit(PPC440SPE_ZERO_Q,
3267 &sw_desc->flags)) {
3268 iter = ppc440spe_get_group_entry(
3269 sw_desc, index++);
3270 ppc440spe_adma_pq_zero_op(iter, chan,
3271 qaddr);
3272 }
3273
3274 return;
3275 }
3276 } else {
3277 /* This is RXOR-only or RXOR/WXOR mixed chain */
3278
3279 /* If we want to include destination into calculations,
3280 * then make dest addresses cued with mult=1 (XOR).
3281 */
3282 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3283 DMA_CUED_XOR_HB :
3284 DMA_CUED_XOR_BASE |
3285 (1 << DMA_CUED_MULT1_OFF);
3286 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3287 DMA_CUED_XOR_HB :
3288 DMA_CUED_XOR_BASE |
3289 (1 << DMA_CUED_MULT1_OFF);
3290
3291 /* Setup destination(s) in RXOR slot(s) */
3292 iter = ppc440spe_get_group_entry(sw_desc, index++);
3293 ppc440spe_desc_set_dest_addr(iter, chan,
3294 paddr ? ppath : qpath,
3295 paddr ? paddr : qaddr, 0);
3296 if (!addr) {
3297 /* two destinations */
3298 iter = ppc440spe_get_group_entry(sw_desc,
3299 index++);
3300 ppc440spe_desc_set_dest_addr(iter, chan,
3301 qpath, qaddr, 0);
3302 }
3303
3304 if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
3305 /* Setup destination(s) in remaining WXOR
3306 * slots
3307 */
3308 iter = ppc440spe_get_group_entry(sw_desc,
3309 index);
3310 if (addr) {
3311 /* one destination */
3312 list_for_each_entry_from(iter,
3313 &sw_desc->group_list,
3314 chain_node)
3315 ppc440spe_desc_set_dest_addr(
3316 iter, chan,
3317 DMA_CUED_XOR_BASE,
3318 addr, 0);
3319
3320 } else {
3321 /* two destinations */
3322 list_for_each_entry_from(iter,
3323 &sw_desc->group_list,
3324 chain_node) {
3325 ppc440spe_desc_set_dest_addr(
3326 iter, chan,
3327 DMA_CUED_XOR_BASE,
3328 paddr, 0);
3329 ppc440spe_desc_set_dest_addr(
3330 iter, chan,
3331 DMA_CUED_XOR_BASE,
3332 qaddr, 1);
3333 }
3334 }
3335 }
3336
3337 }
3338 break;
3339
3340 case PPC440SPE_XOR_ID:
3341 /* DMA2 descriptors have only 1 destination, so there are
3342 * two chains - one for each dest.
3343 * If we want to include destination into calculations,
3344 * then make dest addresses cued with mult=1 (XOR).
3345 */
3346 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3347 DMA_CUED_XOR_HB :
3348 DMA_CUED_XOR_BASE |
3349 (1 << DMA_CUED_MULT1_OFF);
3350
3351 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3352 DMA_CUED_XOR_HB :
3353 DMA_CUED_XOR_BASE |
3354 (1 << DMA_CUED_MULT1_OFF);
3355
3356 iter = ppc440spe_get_group_entry(sw_desc, 0);
3357 for (i = 0; i < sw_desc->descs_per_op; i++) {
3358 ppc440spe_desc_set_dest_addr(iter, chan,
3359 paddr ? ppath : qpath,
3360 paddr ? paddr : qaddr, 0);
3361 iter = list_entry(iter->chain_node.next,
3362 struct ppc440spe_adma_desc_slot,
3363 chain_node);
3364 }
3365
3366 if (!addr) {
3367 /* Two destinations; setup Q here */
3368 iter = ppc440spe_get_group_entry(sw_desc,
3369 sw_desc->descs_per_op);
3370 for (i = 0; i < sw_desc->descs_per_op; i++) {
3371 ppc440spe_desc_set_dest_addr(iter,
3372 chan, qpath, qaddr, 0);
3373 iter = list_entry(iter->chain_node.next,
3374 struct ppc440spe_adma_desc_slot,
3375 chain_node);
3376 }
3377 }
3378
3379 break;
3380 }
3381}
3382
3383/**
3384 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3385 * for the PQ_ZERO_SUM operation
3386 */
3387static void ppc440spe_adma_pqzero_sum_set_dest(
3388 struct ppc440spe_adma_desc_slot *sw_desc,
3389 dma_addr_t paddr, dma_addr_t qaddr)
3390{
3391 struct ppc440spe_adma_desc_slot *iter, *end;
3392 struct ppc440spe_adma_chan *chan;
3393 dma_addr_t addr = 0;
3394 int idx;
3395
3396 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3397
3398 /* walk through the WXOR source list and set P/Q-destinations
3399 * for each slot
3400 */
3401 idx = (paddr && qaddr) ? 2 : 1;
3402 /* set end */
3403 list_for_each_entry_reverse(end, &sw_desc->group_list,
3404 chain_node) {
3405 if (!(--idx))
3406 break;
3407 }
3408 /* set start */
3409 idx = (paddr && qaddr) ? 2 : 1;
3410 iter = ppc440spe_get_group_entry(sw_desc, idx);
3411
3412 if (paddr && qaddr) {
3413 /* two destinations */
3414 list_for_each_entry_from(iter, &sw_desc->group_list,
3415 chain_node) {
3416 if (unlikely(iter == end))
3417 break;
3418 ppc440spe_desc_set_dest_addr(iter, chan,
3419 DMA_CUED_XOR_BASE, paddr, 0);
3420 ppc440spe_desc_set_dest_addr(iter, chan,
3421 DMA_CUED_XOR_BASE, qaddr, 1);
3422 }
3423 } else {
3424 /* one destination */
3425 addr = paddr ? paddr : qaddr;
3426 list_for_each_entry_from(iter, &sw_desc->group_list,
3427 chain_node) {
3428 if (unlikely(iter == end))
3429 break;
3430 ppc440spe_desc_set_dest_addr(iter, chan,
3431 DMA_CUED_XOR_BASE, addr, 0);
3432 }
3433 }
3434
3435 /* The remaining descriptors are DATACHECK. These have no need in
3436 * destination. Actually, these destinations are used there
3437 * as sources for check operation. So, set addr as source.
3438 */
3439 ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3440
3441 if (!addr) {
3442 end = list_entry(end->chain_node.next,
3443 struct ppc440spe_adma_desc_slot, chain_node);
3444 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3445 }
3446}
3447
3448/**
3449 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3450 */
3451static inline void ppc440spe_desc_set_xor_src_cnt(
3452 struct ppc440spe_adma_desc_slot *desc,
3453 int src_cnt)
3454{
3455 struct xor_cb *hw_desc = desc->hw_desc;
3456
3457 hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3458 hw_desc->cbc |= src_cnt;
3459}
3460
3461/**
3462 * ppc440spe_adma_pq_set_src - set source address into descriptor
3463 */
3464static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3465 dma_addr_t addr, int index)
3466{
3467 struct ppc440spe_adma_chan *chan;
3468 dma_addr_t haddr = 0;
3469 struct ppc440spe_adma_desc_slot *iter = NULL;
3470
3471 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3472
3473 switch (chan->device->id) {
3474 case PPC440SPE_DMA0_ID:
3475 case PPC440SPE_DMA1_ID:
3476 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3477 */
3478 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3479 /* RXOR-only or RXOR/WXOR operation */
3480 int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3481 &sw_desc->flags) ? 2 : 3;
3482
3483 if (index == 0) {
3484 /* 1st slot (RXOR) */
3485 /* setup sources region (R1-2-3, R1-2-4,
3486 * or R1-2-5)
3487 */
3488 if (test_bit(PPC440SPE_DESC_RXOR12,
3489 &sw_desc->flags))
3490 haddr = DMA_RXOR12 <<
3491 DMA_CUED_REGION_OFF;
3492 else if (test_bit(PPC440SPE_DESC_RXOR123,
3493 &sw_desc->flags))
3494 haddr = DMA_RXOR123 <<
3495 DMA_CUED_REGION_OFF;
3496 else if (test_bit(PPC440SPE_DESC_RXOR124,
3497 &sw_desc->flags))
3498 haddr = DMA_RXOR124 <<
3499 DMA_CUED_REGION_OFF;
3500 else if (test_bit(PPC440SPE_DESC_RXOR125,
3501 &sw_desc->flags))
3502 haddr = DMA_RXOR125 <<
3503 DMA_CUED_REGION_OFF;
3504 else
3505 BUG();
3506 haddr |= DMA_CUED_XOR_BASE;
3507 iter = ppc440spe_get_group_entry(sw_desc, 0);
3508 } else if (index < iskip) {
3509 /* 1st slot (RXOR)
3510 * shall actually set source address only once
3511 * instead of first <iskip>
3512 */
3513 iter = NULL;
3514 } else {
3515 /* 2nd/3d and next slots (WXOR);
3516 * skip first slot with RXOR
3517 */
3518 haddr = DMA_CUED_XOR_HB;
3519 iter = ppc440spe_get_group_entry(sw_desc,
3520 index - iskip + sw_desc->dst_cnt);
3521 }
3522 } else {
3523 int znum = 0;
3524
3525 /* WXOR-only operation; skip first slots with
3526 * zeroing destinations
3527 */
3528 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3529 znum++;
3530 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3531 znum++;
3532
3533 haddr = DMA_CUED_XOR_HB;
3534 iter = ppc440spe_get_group_entry(sw_desc,
3535 index + znum);
3536 }
3537
3538 if (likely(iter)) {
3539 ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3540
3541 if (!index &&
3542 test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3543 sw_desc->dst_cnt == 2) {
3544 /* if we have two destinations for RXOR, then
3545 * setup source in the second descr too
3546 */
3547 iter = ppc440spe_get_group_entry(sw_desc, 1);
3548 ppc440spe_desc_set_src_addr(iter, chan, 0,
3549 haddr, addr);
3550 }
3551 }
3552 break;
3553
3554 case PPC440SPE_XOR_ID:
3555 /* DMA2 may do Biskup */
3556 iter = sw_desc->group_head;
3557 if (iter->dst_cnt == 2) {
3558 /* both P & Q calculations required; set P src here */
3559 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3560
3561 /* this is for Q */
3562 iter = ppc440spe_get_group_entry(sw_desc,
3563 sw_desc->descs_per_op);
3564 }
3565 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3566 break;
3567 }
3568}
3569
3570/**
3571 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3572 */
3573static void ppc440spe_adma_memcpy_xor_set_src(
3574 struct ppc440spe_adma_desc_slot *sw_desc,
3575 dma_addr_t addr, int index)
3576{
3577 struct ppc440spe_adma_chan *chan;
3578
3579 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3580 sw_desc = sw_desc->group_head;
3581
3582 if (likely(sw_desc))
3583 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3584}
3585
3586/**
3587 * ppc440spe_adma_dma2rxor_inc_addr -
3588 */
3589static void ppc440spe_adma_dma2rxor_inc_addr(
3590 struct ppc440spe_adma_desc_slot *desc,
3591 struct ppc440spe_rxor *cursor, int index, int src_cnt)
3592{
3593 cursor->addr_count++;
3594 if (index == src_cnt - 1) {
3595 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3596 } else if (cursor->addr_count == XOR_MAX_OPS) {
3597 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3598 cursor->addr_count = 0;
3599 cursor->desc_count++;
3600 }
3601}
3602
3603/**
3604 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3605 */
3606static int ppc440spe_adma_dma2rxor_prep_src(
3607 struct ppc440spe_adma_desc_slot *hdesc,
3608 struct ppc440spe_rxor *cursor, int index,
3609 int src_cnt, u32 addr)
3610{
3611 int rval = 0;
3612 u32 sign;
3613 struct ppc440spe_adma_desc_slot *desc = hdesc;
3614 int i;
3615
3616 for (i = 0; i < cursor->desc_count; i++) {
3617 desc = list_entry(hdesc->chain_node.next,
3618 struct ppc440spe_adma_desc_slot,
3619 chain_node);
3620 }
3621
3622 switch (cursor->state) {
3623 case 0:
3624 if (addr == cursor->addrl + cursor->len) {
3625 /* direct RXOR */
3626 cursor->state = 1;
3627 cursor->xor_count++;
3628 if (index == src_cnt-1) {
3629 ppc440spe_rxor_set_region(desc,
3630 cursor->addr_count,
3631 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3632 ppc440spe_adma_dma2rxor_inc_addr(
3633 desc, cursor, index, src_cnt);
3634 }
3635 } else if (cursor->addrl == addr + cursor->len) {
3636 /* reverse RXOR */
3637 cursor->state = 1;
3638 cursor->xor_count++;
3639 set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3640 if (index == src_cnt-1) {
3641 ppc440spe_rxor_set_region(desc,
3642 cursor->addr_count,
3643 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3644 ppc440spe_adma_dma2rxor_inc_addr(
3645 desc, cursor, index, src_cnt);
3646 }
3647 } else {
3648 printk(KERN_ERR "Cannot build "
3649 "DMA2 RXOR command block.\n");
3650 BUG();
3651 }
3652 break;
3653 case 1:
3654 sign = test_bit(cursor->addr_count,
3655 desc->reverse_flags)
3656 ? -1 : 1;
3657 if (index == src_cnt-2 || (sign == -1
3658 && addr != cursor->addrl - 2*cursor->len)) {
3659 cursor->state = 0;
3660 cursor->xor_count = 1;
3661 cursor->addrl = addr;
3662 ppc440spe_rxor_set_region(desc,
3663 cursor->addr_count,
3664 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3665 ppc440spe_adma_dma2rxor_inc_addr(
3666 desc, cursor, index, src_cnt);
3667 } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3668 cursor->state = 2;
3669 cursor->xor_count = 0;
3670 ppc440spe_rxor_set_region(desc,
3671 cursor->addr_count,
3672 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3673 if (index == src_cnt-1) {
3674 ppc440spe_adma_dma2rxor_inc_addr(
3675 desc, cursor, index, src_cnt);
3676 }
3677 } else if (addr == cursor->addrl + 3*cursor->len) {
3678 cursor->state = 2;
3679 cursor->xor_count = 0;
3680 ppc440spe_rxor_set_region(desc,
3681 cursor->addr_count,
3682 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3683 if (index == src_cnt-1) {
3684 ppc440spe_adma_dma2rxor_inc_addr(
3685 desc, cursor, index, src_cnt);
3686 }
3687 } else if (addr == cursor->addrl + 4*cursor->len) {
3688 cursor->state = 2;
3689 cursor->xor_count = 0;
3690 ppc440spe_rxor_set_region(desc,
3691 cursor->addr_count,
3692 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3693 if (index == src_cnt-1) {
3694 ppc440spe_adma_dma2rxor_inc_addr(
3695 desc, cursor, index, src_cnt);
3696 }
3697 } else {
3698 cursor->state = 0;
3699 cursor->xor_count = 1;
3700 cursor->addrl = addr;
3701 ppc440spe_rxor_set_region(desc,
3702 cursor->addr_count,
3703 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3704 ppc440spe_adma_dma2rxor_inc_addr(
3705 desc, cursor, index, src_cnt);
3706 }
3707 break;
3708 case 2:
3709 cursor->state = 0;
3710 cursor->addrl = addr;
3711 cursor->xor_count++;
3712 if (index) {
3713 ppc440spe_adma_dma2rxor_inc_addr(
3714 desc, cursor, index, src_cnt);
3715 }
3716 break;
3717 }
3718
3719 return rval;
3720}
3721
3722/**
3723 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3724 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3725 */
3726static void ppc440spe_adma_dma2rxor_set_src(
3727 struct ppc440spe_adma_desc_slot *desc,
3728 int index, dma_addr_t addr)
3729{
3730 struct xor_cb *xcb = desc->hw_desc;
3731 int k = 0, op = 0, lop = 0;
3732
3733 /* get the RXOR operand which corresponds to index addr */
3734 while (op <= index) {
3735 lop = op;
3736 if (k == XOR_MAX_OPS) {
3737 k = 0;
3738 desc = list_entry(desc->chain_node.next,
3739 struct ppc440spe_adma_desc_slot, chain_node);
3740 xcb = desc->hw_desc;
3741
3742 }
3743 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3744 (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3745 op += 2;
3746 else
3747 op += 3;
3748 }
3749
3750 BUG_ON(k < 1);
3751
3752 if (test_bit(k-1, desc->reverse_flags)) {
3753 /* reverse operand order; put last op in RXOR group */
3754 if (index == op - 1)
3755 ppc440spe_rxor_set_src(desc, k - 1, addr);
3756 } else {
3757 /* direct operand order; put first op in RXOR group */
3758 if (index == lop)
3759 ppc440spe_rxor_set_src(desc, k - 1, addr);
3760 }
3761}
3762
3763/**
3764 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3765 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3766 */
3767static void ppc440spe_adma_dma2rxor_set_mult(
3768 struct ppc440spe_adma_desc_slot *desc,
3769 int index, u8 mult)
3770{
3771 struct xor_cb *xcb = desc->hw_desc;
3772 int k = 0, op = 0, lop = 0;
3773
3774 /* get the RXOR operand which corresponds to index mult */
3775 while (op <= index) {
3776 lop = op;
3777 if (k == XOR_MAX_OPS) {
3778 k = 0;
3779 desc = list_entry(desc->chain_node.next,
3780 struct ppc440spe_adma_desc_slot,
3781 chain_node);
3782 xcb = desc->hw_desc;
3783
3784 }
3785 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3786 (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3787 op += 2;
3788 else
3789 op += 3;
3790 }
3791
3792 BUG_ON(k < 1);
3793 if (test_bit(k-1, desc->reverse_flags)) {
3794 /* reverse order */
3795 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3796 } else {
3797 /* direct order */
3798 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3799 }
3800}
3801
3802/**
3803 * ppc440spe_init_rxor_cursor -
3804 */
3805static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3806{
3807 memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3808 cursor->state = 2;
3809}
3810
3811/**
3812 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3813 * descriptor for the PQXOR operation
3814 */
3815static void ppc440spe_adma_pq_set_src_mult(
3816 struct ppc440spe_adma_desc_slot *sw_desc,
3817 unsigned char mult, int index, int dst_pos)
3818{
3819 struct ppc440spe_adma_chan *chan;
3820 u32 mult_idx, mult_dst;
3821 struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3822
3823 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3824
3825 switch (chan->device->id) {
3826 case PPC440SPE_DMA0_ID:
3827 case PPC440SPE_DMA1_ID:
3828 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3829 int region = test_bit(PPC440SPE_DESC_RXOR12,
3830 &sw_desc->flags) ? 2 : 3;
3831
3832 if (index < region) {
3833 /* RXOR multipliers */
3834 iter = ppc440spe_get_group_entry(sw_desc,
3835 sw_desc->dst_cnt - 1);
3836 if (sw_desc->dst_cnt == 2)
3837 iter1 = ppc440spe_get_group_entry(
3838 sw_desc, 0);
3839
3840 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3841 mult_dst = DMA_CDB_SG_SRC;
3842 } else {
3843 /* WXOR multiplier */
3844 iter = ppc440spe_get_group_entry(sw_desc,
3845 index - region +
3846 sw_desc->dst_cnt);
3847 mult_idx = DMA_CUED_MULT1_OFF;
3848 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3849 DMA_CDB_SG_DST1;
3850 }
3851 } else {
3852 int znum = 0;
3853
3854 /* WXOR-only;
3855 * skip first slots with destinations (if ZERO_DST has
3856 * place)
3857 */
3858 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3859 znum++;
3860 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3861 znum++;
3862
3863 iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3864 mult_idx = DMA_CUED_MULT1_OFF;
3865 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3866 }
3867
3868 if (likely(iter)) {
3869 ppc440spe_desc_set_src_mult(iter, chan,
3870 mult_idx, mult_dst, mult);
3871
3872 if (unlikely(iter1)) {
3873 /* if we have two destinations for RXOR, then
3874 * we've just set Q mult. Set-up P now.
3875 */
3876 ppc440spe_desc_set_src_mult(iter1, chan,
3877 mult_idx, mult_dst, 1);
3878 }
3879
3880 }
3881 break;
3882
3883 case PPC440SPE_XOR_ID:
3884 iter = sw_desc->group_head;
3885 if (sw_desc->dst_cnt == 2) {
3886 /* both P & Q calculations required; set P mult here */
3887 ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3888
3889 /* and then set Q mult */
3890 iter = ppc440spe_get_group_entry(sw_desc,
3891 sw_desc->descs_per_op);
3892 }
3893 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3894 break;
3895 }
3896}
3897
3898/**
3899 * ppc440spe_adma_free_chan_resources - free the resources allocated
3900 */
3901static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3902{
3903 struct ppc440spe_adma_chan *ppc440spe_chan;
3904 struct ppc440spe_adma_desc_slot *iter, *_iter;
3905 int in_use_descs = 0;
3906
3907 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3908 ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3909
3910 spin_lock_bh(&ppc440spe_chan->lock);
3911 list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3912 chain_node) {
3913 in_use_descs++;
3914 list_del(&iter->chain_node);
3915 }
3916 list_for_each_entry_safe_reverse(iter, _iter,
3917 &ppc440spe_chan->all_slots, slot_node) {
3918 list_del(&iter->slot_node);
3919 kfree(iter);
3920 ppc440spe_chan->slots_allocated--;
3921 }
3922 ppc440spe_chan->last_used = NULL;
3923
3924 dev_dbg(ppc440spe_chan->device->common.dev,
3925 "ppc440spe adma%d %s slots_allocated %d\n",
3926 ppc440spe_chan->device->id,
3927 __func__, ppc440spe_chan->slots_allocated);
3928 spin_unlock_bh(&ppc440spe_chan->lock);
3929
3930 /* one is ok since we left it on there on purpose */
3931 if (in_use_descs > 1)
3932 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3933 in_use_descs - 1);
3934}
3935
3936/**
3937 * ppc440spe_adma_is_complete - poll the status of an ADMA transaction
3938 * @chan: ADMA channel handle
3939 * @cookie: ADMA transaction identifier
3940 */
3941static enum dma_status ppc440spe_adma_is_complete(struct dma_chan *chan,
3942 dma_cookie_t cookie, dma_cookie_t *done, dma_cookie_t *used)
3943{
3944 struct ppc440spe_adma_chan *ppc440spe_chan;
3945 dma_cookie_t last_used;
3946 dma_cookie_t last_complete;
3947 enum dma_status ret;
3948
3949 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3950 last_used = chan->cookie;
3951 last_complete = ppc440spe_chan->completed_cookie;
3952
3953 if (done)
3954 *done = last_complete;
3955 if (used)
3956 *used = last_used;
3957
3958 ret = dma_async_is_complete(cookie, last_complete, last_used);
3959 if (ret == DMA_SUCCESS)
3960 return ret;
3961
3962 ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3963
3964 last_used = chan->cookie;
3965 last_complete = ppc440spe_chan->completed_cookie;
3966
3967 if (done)
3968 *done = last_complete;
3969 if (used)
3970 *used = last_used;
3971
3972 return dma_async_is_complete(cookie, last_complete, last_used);
3973}
3974
3975/**
3976 * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3977 */
3978static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3979{
3980 struct ppc440spe_adma_chan *chan = data;
3981
3982 dev_dbg(chan->device->common.dev,
3983 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3984
3985 tasklet_schedule(&chan->irq_tasklet);
3986 ppc440spe_adma_device_clear_eot_status(chan);
3987
3988 return IRQ_HANDLED;
3989}
3990
3991/**
3992 * ppc440spe_adma_err_handler - DMA error interrupt handler;
3993 * do the same things as a eot handler
3994 */
3995static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3996{
3997 struct ppc440spe_adma_chan *chan = data;
3998
3999 dev_dbg(chan->device->common.dev,
4000 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
4001
4002 tasklet_schedule(&chan->irq_tasklet);
4003 ppc440spe_adma_device_clear_eot_status(chan);
4004
4005 return IRQ_HANDLED;
4006}
4007
4008/**
4009 * ppc440spe_test_callback - called when test operation has been done
4010 */
4011static void ppc440spe_test_callback(void *unused)
4012{
4013 complete(&ppc440spe_r6_test_comp);
4014}
4015
4016/**
4017 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
4018 */
4019static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
4020{
4021 struct ppc440spe_adma_chan *ppc440spe_chan;
4022
4023 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4024 dev_dbg(ppc440spe_chan->device->common.dev,
4025 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
4026 __func__, ppc440spe_chan->pending);
4027
4028 if (ppc440spe_chan->pending) {
4029 ppc440spe_chan->pending = 0;
4030 ppc440spe_chan_append(ppc440spe_chan);
4031 }
4032}
4033
4034/**
4035 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
4036 * use FIFOs (as opposite to chains used in XOR) so this is a XOR
4037 * specific operation)
4038 */
4039static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
4040{
4041 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
4042 dma_cookie_t cookie;
4043 int slot_cnt, slots_per_op;
4044
4045 dev_dbg(chan->device->common.dev,
4046 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
4047
4048 spin_lock_bh(&chan->lock);
4049 slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
4050 sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
4051 if (sw_desc) {
4052 group_start = sw_desc->group_head;
4053 list_splice_init(&sw_desc->group_list, &chan->chain);
4054 async_tx_ack(&sw_desc->async_tx);
4055 ppc440spe_desc_init_null_xor(group_start);
4056
4057 cookie = chan->common.cookie;
4058 cookie++;
4059 if (cookie <= 1)
4060 cookie = 2;
4061
4062 /* initialize the completed cookie to be less than
4063 * the most recently used cookie
4064 */
4065 chan->completed_cookie = cookie - 1;
4066 chan->common.cookie = sw_desc->async_tx.cookie = cookie;
4067
4068 /* channel should not be busy */
4069 BUG_ON(ppc440spe_chan_is_busy(chan));
4070
4071 /* set the descriptor address */
4072 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
4073
4074 /* run the descriptor */
4075 ppc440spe_chan_run(chan);
4076 } else
4077 printk(KERN_ERR "ppc440spe adma%d"
4078 " failed to allocate null descriptor\n",
4079 chan->device->id);
4080 spin_unlock_bh(&chan->lock);
4081}
4082
4083/**
4084 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
4085 * For this we just perform one WXOR operation with the same source
4086 * and destination addresses, the GF-multiplier is 1; so if RAID-6
4087 * capabilities are enabled then we'll get src/dst filled with zero.
4088 */
4089static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
4090{
4091 struct ppc440spe_adma_desc_slot *sw_desc, *iter;
4092 struct page *pg;
4093 char *a;
4094 dma_addr_t dma_addr, addrs[2];
4095 unsigned long op = 0;
4096 int rval = 0;
4097
4098 set_bit(PPC440SPE_DESC_WXOR, &op);
4099
4100 pg = alloc_page(GFP_KERNEL);
4101 if (!pg)
4102 return -ENOMEM;
4103
4104 spin_lock_bh(&chan->lock);
4105 sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
4106 if (sw_desc) {
4107 /* 1 src, 1 dsr, int_ena, WXOR */
4108 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
4109 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
4110 ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
4111 iter->unmap_len = PAGE_SIZE;
4112 }
4113 } else {
4114 rval = -EFAULT;
4115 spin_unlock_bh(&chan->lock);
4116 goto exit;
4117 }
4118 spin_unlock_bh(&chan->lock);
4119
4120 /* Fill the test page with ones */
4121 memset(page_address(pg), 0xFF, PAGE_SIZE);
4122 dma_addr = dma_map_page(chan->device->dev, pg, 0,
4123 PAGE_SIZE, DMA_BIDIRECTIONAL);
4124
4125 /* Setup addresses */
4126 ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
4127 ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
4128 addrs[0] = dma_addr;
4129 addrs[1] = 0;
4130 ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
4131
4132 async_tx_ack(&sw_desc->async_tx);
4133 sw_desc->async_tx.callback = ppc440spe_test_callback;
4134 sw_desc->async_tx.callback_param = NULL;
4135
4136 init_completion(&ppc440spe_r6_test_comp);
4137
4138 ppc440spe_adma_tx_submit(&sw_desc->async_tx);
4139 ppc440spe_adma_issue_pending(&chan->common);
4140
4141 wait_for_completion(&ppc440spe_r6_test_comp);
4142
4143 /* Now check if the test page is zeroed */
4144 a = page_address(pg);
4145 if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
4146 /* page is zero - RAID-6 enabled */
4147 rval = 0;
4148 } else {
4149 /* RAID-6 was not enabled */
4150 rval = -EINVAL;
4151 }
4152exit:
4153 __free_page(pg);
4154 return rval;
4155}
4156
4157static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
4158{
4159 switch (adev->id) {
4160 case PPC440SPE_DMA0_ID:
4161 case PPC440SPE_DMA1_ID:
4162 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
4163 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
4164 dma_cap_set(DMA_MEMSET, adev->common.cap_mask);
4165 dma_cap_set(DMA_PQ, adev->common.cap_mask);
4166 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
4167 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
4168 break;
4169 case PPC440SPE_XOR_ID:
4170 dma_cap_set(DMA_XOR, adev->common.cap_mask);
4171 dma_cap_set(DMA_PQ, adev->common.cap_mask);
4172 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
4173 adev->common.cap_mask = adev->common.cap_mask;
4174 break;
4175 }
4176
4177 /* Set base routines */
4178 adev->common.device_alloc_chan_resources =
4179 ppc440spe_adma_alloc_chan_resources;
4180 adev->common.device_free_chan_resources =
4181 ppc440spe_adma_free_chan_resources;
4182 adev->common.device_is_tx_complete = ppc440spe_adma_is_complete;
4183 adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
4184
4185 /* Set prep routines based on capability */
4186 if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
4187 adev->common.device_prep_dma_memcpy =
4188 ppc440spe_adma_prep_dma_memcpy;
4189 }
4190 if (dma_has_cap(DMA_MEMSET, adev->common.cap_mask)) {
4191 adev->common.device_prep_dma_memset =
4192 ppc440spe_adma_prep_dma_memset;
4193 }
4194 if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
4195 adev->common.max_xor = XOR_MAX_OPS;
4196 adev->common.device_prep_dma_xor =
4197 ppc440spe_adma_prep_dma_xor;
4198 }
4199 if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
4200 switch (adev->id) {
4201 case PPC440SPE_DMA0_ID:
4202 dma_set_maxpq(&adev->common,
4203 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
4204 break;
4205 case PPC440SPE_DMA1_ID:
4206 dma_set_maxpq(&adev->common,
4207 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
4208 break;
4209 case PPC440SPE_XOR_ID:
4210 adev->common.max_pq = XOR_MAX_OPS * 3;
4211 break;
4212 }
4213 adev->common.device_prep_dma_pq =
4214 ppc440spe_adma_prep_dma_pq;
4215 }
4216 if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
4217 switch (adev->id) {
4218 case PPC440SPE_DMA0_ID:
4219 adev->common.max_pq = DMA0_FIFO_SIZE /
4220 sizeof(struct dma_cdb);
4221 break;
4222 case PPC440SPE_DMA1_ID:
4223 adev->common.max_pq = DMA1_FIFO_SIZE /
4224 sizeof(struct dma_cdb);
4225 break;
4226 }
4227 adev->common.device_prep_dma_pq_val =
4228 ppc440spe_adma_prep_dma_pqzero_sum;
4229 }
4230 if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
4231 switch (adev->id) {
4232 case PPC440SPE_DMA0_ID:
4233 adev->common.max_xor = DMA0_FIFO_SIZE /
4234 sizeof(struct dma_cdb);
4235 break;
4236 case PPC440SPE_DMA1_ID:
4237 adev->common.max_xor = DMA1_FIFO_SIZE /
4238 sizeof(struct dma_cdb);
4239 break;
4240 }
4241 adev->common.device_prep_dma_xor_val =
4242 ppc440spe_adma_prep_dma_xor_zero_sum;
4243 }
4244 if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
4245 adev->common.device_prep_dma_interrupt =
4246 ppc440spe_adma_prep_dma_interrupt;
4247 }
4248 pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
4249 "( %s%s%s%s%s%s%s)\n",
4250 dev_name(adev->dev),
4251 dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
4252 dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
4253 dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
4254 dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
4255 dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
4256 dma_has_cap(DMA_MEMSET, adev->common.cap_mask) ? "memset " : "",
4257 dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
4258}
4259
4260static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
4261 struct ppc440spe_adma_chan *chan,
4262 int *initcode)
4263{
4264 struct device_node *np;
4265 int ret;
4266
4267 np = container_of(adev->dev, struct of_device, dev)->node;
4268 if (adev->id != PPC440SPE_XOR_ID) {
4269 adev->err_irq = irq_of_parse_and_map(np, 1);
4270 if (adev->err_irq == NO_IRQ) {
4271 dev_warn(adev->dev, "no err irq resource?\n");
4272 *initcode = PPC_ADMA_INIT_IRQ2;
4273 adev->err_irq = -ENXIO;
4274 } else
4275 atomic_inc(&ppc440spe_adma_err_irq_ref);
4276 } else {
4277 adev->err_irq = -ENXIO;
4278 }
4279
4280 adev->irq = irq_of_parse_and_map(np, 0);
4281 if (adev->irq == NO_IRQ) {
4282 dev_err(adev->dev, "no irq resource\n");
4283 *initcode = PPC_ADMA_INIT_IRQ1;
4284 ret = -ENXIO;
4285 goto err_irq_map;
4286 }
4287 dev_dbg(adev->dev, "irq %d, err irq %d\n",
4288 adev->irq, adev->err_irq);
4289
4290 ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
4291 0, dev_driver_string(adev->dev), chan);
4292 if (ret) {
4293 dev_err(adev->dev, "can't request irq %d\n",
4294 adev->irq);
4295 *initcode = PPC_ADMA_INIT_IRQ1;
4296 ret = -EIO;
4297 goto err_req1;
4298 }
4299
4300 /* only DMA engines have a separate error IRQ
4301 * so it's Ok if err_irq < 0 in XOR engine case.
4302 */
4303 if (adev->err_irq > 0) {
4304 /* both DMA engines share common error IRQ */
4305 ret = request_irq(adev->err_irq,
4306 ppc440spe_adma_err_handler,
4307 IRQF_SHARED,
4308 dev_driver_string(adev->dev),
4309 chan);
4310 if (ret) {
4311 dev_err(adev->dev, "can't request irq %d\n",
4312 adev->err_irq);
4313 *initcode = PPC_ADMA_INIT_IRQ2;
4314 ret = -EIO;
4315 goto err_req2;
4316 }
4317 }
4318
4319 if (adev->id == PPC440SPE_XOR_ID) {
4320 /* enable XOR engine interrupts */
4321 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4322 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
4323 &adev->xor_reg->ier);
4324 } else {
4325 u32 mask, enable;
4326
4327 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4328 if (!np) {
4329 pr_err("%s: can't find I2O device tree node\n",
4330 __func__);
4331 ret = -ENODEV;
4332 goto err_req2;
4333 }
4334 adev->i2o_reg = of_iomap(np, 0);
4335 if (!adev->i2o_reg) {
4336 pr_err("%s: failed to map I2O registers\n", __func__);
4337 of_node_put(np);
4338 ret = -EINVAL;
4339 goto err_req2;
4340 }
4341 of_node_put(np);
4342 /* Unmask 'CS FIFO Attention' interrupts and
4343 * enable generating interrupts on errors
4344 */
4345 enable = (adev->id == PPC440SPE_DMA0_ID) ?
4346 ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4347 ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4348 mask = ioread32(&adev->i2o_reg->iopim) & enable;
4349 iowrite32(mask, &adev->i2o_reg->iopim);
4350 }
4351 return 0;
4352
4353err_req2:
4354 free_irq(adev->irq, chan);
4355err_req1:
4356 irq_dispose_mapping(adev->irq);
4357err_irq_map:
4358 if (adev->err_irq > 0) {
4359 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
4360 irq_dispose_mapping(adev->err_irq);
4361 }
4362 return ret;
4363}
4364
4365static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
4366 struct ppc440spe_adma_chan *chan)
4367{
4368 u32 mask, disable;
4369
4370 if (adev->id == PPC440SPE_XOR_ID) {
4371 /* disable XOR engine interrupts */
4372 mask = ioread32be(&adev->xor_reg->ier);
4373 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4374 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
4375 iowrite32be(mask, &adev->xor_reg->ier);
4376 } else {
4377 /* disable DMAx engine interrupts */
4378 disable = (adev->id == PPC440SPE_DMA0_ID) ?
4379 (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4380 (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4381 mask = ioread32(&adev->i2o_reg->iopim) | disable;
4382 iowrite32(mask, &adev->i2o_reg->iopim);
4383 }
4384 free_irq(adev->irq, chan);
4385 irq_dispose_mapping(adev->irq);
4386 if (adev->err_irq > 0) {
4387 free_irq(adev->err_irq, chan);
4388 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
4389 irq_dispose_mapping(adev->err_irq);
4390 iounmap(adev->i2o_reg);
4391 }
4392 }
4393}
4394
4395/**
4396 * ppc440spe_adma_probe - probe the asynch device
4397 */
4398static int __devinit ppc440spe_adma_probe(struct of_device *ofdev,
4399 const struct of_device_id *match)
4400{
4401 struct device_node *np = ofdev->node;
4402 struct resource res;
4403 struct ppc440spe_adma_device *adev;
4404 struct ppc440spe_adma_chan *chan;
4405 struct ppc_dma_chan_ref *ref, *_ref;
4406 int ret = 0, initcode = PPC_ADMA_INIT_OK;
4407 const u32 *idx;
4408 int len;
4409 void *regs;
4410 u32 id, pool_size;
4411
4412 if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4413 id = PPC440SPE_XOR_ID;
4414 /* As far as the XOR engine is concerned, it does not
4415 * use FIFOs but uses linked list. So there is no dependency
4416 * between pool size to allocate and the engine configuration.
4417 */
4418 pool_size = PAGE_SIZE << 1;
4419 } else {
4420 /* it is DMA0 or DMA1 */
4421 idx = of_get_property(np, "cell-index", &len);
4422 if (!idx || (len != sizeof(u32))) {
4423 dev_err(&ofdev->dev, "Device node %s has missing "
4424 "or invalid cell-index property\n",
4425 np->full_name);
4426 return -EINVAL;
4427 }
4428 id = *idx;
4429 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4430 * should allocate the pool accordingly to size of this
4431 * FIFO. Thus, the pool size depends on the FIFO depth:
4432 * how much CDBs pointers the FIFO may contain then so
4433 * much CDBs we should provide in the pool.
4434 * That is
4435 * CDB size = 32B;
4436 * CDBs number = (DMA0_FIFO_SIZE >> 3);
4437 * Pool size = CDBs number * CDB size =
4438 * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4439 */
4440 pool_size = (id == PPC440SPE_DMA0_ID) ?
4441 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4442 pool_size <<= 2;
4443 }
4444
4445 if (of_address_to_resource(np, 0, &res)) {
4446 dev_err(&ofdev->dev, "failed to get memory resource\n");
4447 initcode = PPC_ADMA_INIT_MEMRES;
4448 ret = -ENODEV;
4449 goto out;
4450 }
4451
4452 if (!request_mem_region(res.start, resource_size(&res),
4453 dev_driver_string(&ofdev->dev))) {
4454 dev_err(&ofdev->dev, "failed to request memory region "
4455 "(0x%016llx-0x%016llx)\n",
4456 (u64)res.start, (u64)res.end);
4457 initcode = PPC_ADMA_INIT_MEMREG;
4458 ret = -EBUSY;
4459 goto out;
4460 }
4461
4462 /* create a device */
4463 adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4464 if (!adev) {
4465 dev_err(&ofdev->dev, "failed to allocate device\n");
4466 initcode = PPC_ADMA_INIT_ALLOC;
4467 ret = -ENOMEM;
4468 goto err_adev_alloc;
4469 }
4470
4471 adev->id = id;
4472 adev->pool_size = pool_size;
4473 /* allocate coherent memory for hardware descriptors */
4474 adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4475 adev->pool_size, &adev->dma_desc_pool,
4476 GFP_KERNEL);
4477 if (adev->dma_desc_pool_virt == NULL) {
4478 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4479 "memory for hardware descriptors\n",
4480 adev->pool_size);
4481 initcode = PPC_ADMA_INIT_COHERENT;
4482 ret = -ENOMEM;
4483 goto err_dma_alloc;
4484 }
4485 dev_dbg(&ofdev->dev, "allocted descriptor pool virt 0x%p phys 0x%llx\n",
4486 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4487
4488 regs = ioremap(res.start, resource_size(&res));
4489 if (!regs) {
4490 dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4491 goto err_regs_alloc;
4492 }
4493
4494 if (adev->id == PPC440SPE_XOR_ID) {
4495 adev->xor_reg = regs;
4496 /* Reset XOR */
4497 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4498 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4499 } else {
4500 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4501 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4502 adev->dma_reg = regs;
4503 /* DMAx_FIFO_SIZE is defined in bytes,
4504 * <fsiz> - is defined in number of CDB pointers (8byte).
4505 * DMA FIFO Length = CSlength + CPlength, where
4506 * CSlength = CPlength = (fsiz + 1) * 8.
4507 */
4508 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4509 &adev->dma_reg->fsiz);
4510 /* Configure DMA engine */
4511 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4512 &adev->dma_reg->cfg);
4513 /* Clear Status */
4514 iowrite32(~0, &adev->dma_reg->dsts);
4515 }
4516
4517 adev->dev = &ofdev->dev;
4518 adev->common.dev = &ofdev->dev;
4519 INIT_LIST_HEAD(&adev->common.channels);
4520 dev_set_drvdata(&ofdev->dev, adev);
4521
4522 /* create a channel */
4523 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4524 if (!chan) {
4525 dev_err(&ofdev->dev, "can't allocate channel structure\n");
4526 initcode = PPC_ADMA_INIT_CHANNEL;
4527 ret = -ENOMEM;
4528 goto err_chan_alloc;
4529 }
4530
4531 spin_lock_init(&chan->lock);
4532 INIT_LIST_HEAD(&chan->chain);
4533 INIT_LIST_HEAD(&chan->all_slots);
4534 chan->device = adev;
4535 chan->common.device = &adev->common;
4536 list_add_tail(&chan->common.device_node, &adev->common.channels);
4537 tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet,
4538 (unsigned long)chan);
4539
4540 /* allocate and map helper pages for async validation or
4541 * async_mult/async_sum_product operations on DMA0/1.
4542 */
4543 if (adev->id != PPC440SPE_XOR_ID) {
4544 chan->pdest_page = alloc_page(GFP_KERNEL);
4545 chan->qdest_page = alloc_page(GFP_KERNEL);
4546 if (!chan->pdest_page ||
4547 !chan->qdest_page) {
4548 if (chan->pdest_page)
4549 __free_page(chan->pdest_page);
4550 if (chan->qdest_page)
4551 __free_page(chan->qdest_page);
4552 ret = -ENOMEM;
4553 goto err_page_alloc;
4554 }
4555 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4556 PAGE_SIZE, DMA_BIDIRECTIONAL);
4557 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4558 PAGE_SIZE, DMA_BIDIRECTIONAL);
4559 }
4560
4561 ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4562 if (ref) {
4563 ref->chan = &chan->common;
4564 INIT_LIST_HEAD(&ref->node);
4565 list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4566 } else {
4567 dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4568 ret = -ENOMEM;
4569 goto err_ref_alloc;
4570 }
4571
4572 ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4573 if (ret)
4574 goto err_irq;
4575
4576 ppc440spe_adma_init_capabilities(adev);
4577
4578 ret = dma_async_device_register(&adev->common);
4579 if (ret) {
4580 initcode = PPC_ADMA_INIT_REGISTER;
4581 dev_err(&ofdev->dev, "failed to register dma device\n");
4582 goto err_dev_reg;
4583 }
4584
4585 goto out;
4586
4587err_dev_reg:
4588 ppc440spe_adma_release_irqs(adev, chan);
4589err_irq:
4590 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4591 if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4592 list_del(&ref->node);
4593 kfree(ref);
4594 }
4595 }
4596err_ref_alloc:
4597 if (adev->id != PPC440SPE_XOR_ID) {
4598 dma_unmap_page(&ofdev->dev, chan->pdest,
4599 PAGE_SIZE, DMA_BIDIRECTIONAL);
4600 dma_unmap_page(&ofdev->dev, chan->qdest,
4601 PAGE_SIZE, DMA_BIDIRECTIONAL);
4602 __free_page(chan->pdest_page);
4603 __free_page(chan->qdest_page);
4604 }
4605err_page_alloc:
4606 kfree(chan);
4607err_chan_alloc:
4608 if (adev->id == PPC440SPE_XOR_ID)
4609 iounmap(adev->xor_reg);
4610 else
4611 iounmap(adev->dma_reg);
4612err_regs_alloc:
4613 dma_free_coherent(adev->dev, adev->pool_size,
4614 adev->dma_desc_pool_virt,
4615 adev->dma_desc_pool);
4616err_dma_alloc:
4617 kfree(adev);
4618err_adev_alloc:
4619 release_mem_region(res.start, resource_size(&res));
4620out:
4621 if (id < PPC440SPE_ADMA_ENGINES_NUM)
4622 ppc440spe_adma_devices[id] = initcode;
4623
4624 return ret;
4625}
4626
4627/**
4628 * ppc440spe_adma_remove - remove the asynch device
4629 */
4630static int __devexit ppc440spe_adma_remove(struct of_device *ofdev)
4631{
4632 struct ppc440spe_adma_device *adev = dev_get_drvdata(&ofdev->dev);
4633 struct device_node *np = ofdev->node;
4634 struct resource res;
4635 struct dma_chan *chan, *_chan;
4636 struct ppc_dma_chan_ref *ref, *_ref;
4637 struct ppc440spe_adma_chan *ppc440spe_chan;
4638
4639 dev_set_drvdata(&ofdev->dev, NULL);
4640 if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4641 ppc440spe_adma_devices[adev->id] = -1;
4642
4643 dma_async_device_unregister(&adev->common);
4644
4645 list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4646 device_node) {
4647 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4648 ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4649 tasklet_kill(&ppc440spe_chan->irq_tasklet);
4650 if (adev->id != PPC440SPE_XOR_ID) {
4651 dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4652 PAGE_SIZE, DMA_BIDIRECTIONAL);
4653 dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4654 PAGE_SIZE, DMA_BIDIRECTIONAL);
4655 __free_page(ppc440spe_chan->pdest_page);
4656 __free_page(ppc440spe_chan->qdest_page);
4657 }
4658 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4659 node) {
4660 if (ppc440spe_chan ==
4661 to_ppc440spe_adma_chan(ref->chan)) {
4662 list_del(&ref->node);
4663 kfree(ref);
4664 }
4665 }
4666 list_del(&chan->device_node);
4667 kfree(ppc440spe_chan);
4668 }
4669
4670 dma_free_coherent(adev->dev, adev->pool_size,
4671 adev->dma_desc_pool_virt, adev->dma_desc_pool);
4672 if (adev->id == PPC440SPE_XOR_ID)
4673 iounmap(adev->xor_reg);
4674 else
4675 iounmap(adev->dma_reg);
4676 of_address_to_resource(np, 0, &res);
4677 release_mem_region(res.start, resource_size(&res));
4678 kfree(adev);
4679 return 0;
4680}
4681
4682/*
4683 * /sys driver interface to enable h/w RAID-6 capabilities
4684 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4685 * directory are "devices", "enable" and "poly".
4686 * "devices" shows available engines.
4687 * "enable" is used to enable RAID-6 capabilities or to check
4688 * whether these has been activated.
4689 * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4690 */
4691
4692static ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf)
4693{
4694 ssize_t size = 0;
4695 int i;
4696
4697 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4698 if (ppc440spe_adma_devices[i] == -1)
4699 continue;
4700 size += snprintf(buf + size, PAGE_SIZE - size,
4701 "PPC440SP(E)-ADMA.%d: %s\n", i,
4702 ppc_adma_errors[ppc440spe_adma_devices[i]]);
4703 }
4704 return size;
4705}
4706
4707static ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf)
4708{
4709 return snprintf(buf, PAGE_SIZE,
4710 "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4711 ppc440spe_r6_enabled ? "EN" : "DIS");
4712}
4713
4714static ssize_t store_ppc440spe_r6enable(struct device_driver *dev,
4715 const char *buf, size_t count)
4716{
4717 unsigned long val;
4718
4719 if (!count || count > 11)
4720 return -EINVAL;
4721
4722 if (!ppc440spe_r6_tchan)
4723 return -EFAULT;
4724
4725 /* Write a key */
4726 sscanf(buf, "%lx", &val);
4727 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4728 isync();
4729
4730 /* Verify whether it really works now */
4731 if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4732 pr_info("PPC440SP(e) RAID-6 has been activated "
4733 "successfully\n");
4734 ppc440spe_r6_enabled = 1;
4735 } else {
4736 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4737 " Error key ?\n");
4738 ppc440spe_r6_enabled = 0;
4739 }
4740 return count;
4741}
4742
4743static ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf)
4744{
4745 ssize_t size = 0;
4746 u32 reg;
4747
4748#ifdef CONFIG_440SP
4749 /* 440SP has fixed polynomial */
4750 reg = 0x4d;
4751#else
4752 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4753 reg >>= MQ0_CFBHL_POLY;
4754 reg &= 0xFF;
4755#endif
4756
4757 size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4758 "uses 0x1%02x polynomial.\n", reg);
4759 return size;
4760}
4761
4762static ssize_t store_ppc440spe_r6poly(struct device_driver *dev,
4763 const char *buf, size_t count)
4764{
4765 unsigned long reg, val;
4766
4767#ifdef CONFIG_440SP
4768 /* 440SP uses default 0x14D polynomial only */
4769 return -EINVAL;
4770#endif
4771
4772 if (!count || count > 6)
4773 return -EINVAL;
4774
4775 /* e.g., 0x14D or 0x11D */
4776 sscanf(buf, "%lx", &val);
4777
4778 if (val & ~0x1FF)
4779 return -EINVAL;
4780
4781 val &= 0xFF;
4782 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4783 reg &= ~(0xFF << MQ0_CFBHL_POLY);
4784 reg |= val << MQ0_CFBHL_POLY;
4785 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4786
4787 return count;
4788}
4789
4790static DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL);
4791static DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable,
4792 store_ppc440spe_r6enable);
4793static DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly,
4794 store_ppc440spe_r6poly);
4795
4796/*
4797 * Common initialisation for RAID engines; allocate memory for
4798 * DMAx FIFOs, perform configuration common for all DMA engines.
4799 * Further DMA engine specific configuration is done at probe time.
4800 */
4801static int ppc440spe_configure_raid_devices(void)
4802{
4803 struct device_node *np;
4804 struct resource i2o_res;
4805 struct i2o_regs __iomem *i2o_reg;
4806 dcr_host_t i2o_dcr_host;
4807 unsigned int dcr_base, dcr_len;
4808 int i, ret;
4809
4810 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4811 if (!np) {
4812 pr_err("%s: can't find I2O device tree node\n",
4813 __func__);
4814 return -ENODEV;
4815 }
4816
4817 if (of_address_to_resource(np, 0, &i2o_res)) {
4818 of_node_put(np);
4819 return -EINVAL;
4820 }
4821
4822 i2o_reg = of_iomap(np, 0);
4823 if (!i2o_reg) {
4824 pr_err("%s: failed to map I2O registers\n", __func__);
4825 of_node_put(np);
4826 return -EINVAL;
4827 }
4828
4829 /* Get I2O DCRs base */
4830 dcr_base = dcr_resource_start(np, 0);
4831 dcr_len = dcr_resource_len(np, 0);
4832 if (!dcr_base && !dcr_len) {
4833 pr_err("%s: can't get DCR registers base/len!\n",
4834 np->full_name);
4835 of_node_put(np);
4836 iounmap(i2o_reg);
4837 return -ENODEV;
4838 }
4839
4840 i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4841 if (!DCR_MAP_OK(i2o_dcr_host)) {
4842 pr_err("%s: failed to map DCRs!\n", np->full_name);
4843 of_node_put(np);
4844 iounmap(i2o_reg);
4845 return -ENODEV;
4846 }
4847 of_node_put(np);
4848
4849 /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4850 * the base address of FIFO memory space.
4851 * Actually we need twice more physical memory than programmed in the
4852 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4853 */
4854 ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4855 GFP_KERNEL);
4856 if (!ppc440spe_dma_fifo_buf) {
4857 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4858 iounmap(i2o_reg);
4859 dcr_unmap(i2o_dcr_host, dcr_len);
4860 return -ENOMEM;
4861 }
4862
4863 /*
4864 * Configure h/w
4865 */
4866 /* Reset I2O/DMA */
4867 mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4868 mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4869
4870 /* Setup the base address of mmaped registers */
4871 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4872 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4873 I2O_REG_ENABLE);
4874 dcr_unmap(i2o_dcr_host, dcr_len);
4875
4876 /* Setup FIFO memory space base address */
4877 iowrite32(0, &i2o_reg->ifbah);
4878 iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4879
4880 /* set zero FIFO size for I2O, so the whole
4881 * ppc440spe_dma_fifo_buf is used by DMAs.
4882 * DMAx_FIFOs will be configured while probe.
4883 */
4884 iowrite32(0, &i2o_reg->ifsiz);
4885 iounmap(i2o_reg);
4886
4887 /* To prepare WXOR/RXOR functionality we need access to
4888 * Memory Queue Module DCRs (finally it will be enabled
4889 * via /sys interface of the ppc440spe ADMA driver).
4890 */
4891 np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4892 if (!np) {
4893 pr_err("%s: can't find MQ device tree node\n",
4894 __func__);
4895 ret = -ENODEV;
4896 goto out_free;
4897 }
4898
4899 /* Get MQ DCRs base */
4900 dcr_base = dcr_resource_start(np, 0);
4901 dcr_len = dcr_resource_len(np, 0);
4902 if (!dcr_base && !dcr_len) {
4903 pr_err("%s: can't get DCR registers base/len!\n",
4904 np->full_name);
4905 ret = -ENODEV;
4906 goto out_mq;
4907 }
4908
4909 ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4910 if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4911 pr_err("%s: failed to map DCRs!\n", np->full_name);
4912 ret = -ENODEV;
4913 goto out_mq;
4914 }
4915 of_node_put(np);
4916 ppc440spe_mq_dcr_len = dcr_len;
4917
4918 /* Set HB alias */
4919 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4920
4921 /* Set:
4922 * - LL transaction passing limit to 1;
4923 * - Memory controller cycle limit to 1;
4924 * - Galois Polynomial to 0x14d (default)
4925 */
4926 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4927 (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4928 (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4929
4930 atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4931 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4932 ppc440spe_adma_devices[i] = -1;
4933
4934 return 0;
4935
4936out_mq:
4937 of_node_put(np);
4938out_free:
4939 kfree(ppc440spe_dma_fifo_buf);
4940 return ret;
4941}
4942
4943static struct of_device_id __devinitdata ppc440spe_adma_of_match[] = {
4944 { .compatible = "ibm,dma-440spe", },
4945 { .compatible = "amcc,xor-accelerator", },
4946 {},
4947};
4948MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4949
4950static struct of_platform_driver ppc440spe_adma_driver = {
4951 .match_table = ppc440spe_adma_of_match,
4952 .probe = ppc440spe_adma_probe,
4953 .remove = __devexit_p(ppc440spe_adma_remove),
4954 .driver = {
4955 .name = "PPC440SP(E)-ADMA",
4956 .owner = THIS_MODULE,
4957 },
4958};
4959
4960static __init int ppc440spe_adma_init(void)
4961{
4962 int ret;
4963
4964 ret = ppc440spe_configure_raid_devices();
4965 if (ret)
4966 return ret;
4967
4968 ret = of_register_platform_driver(&ppc440spe_adma_driver);
4969 if (ret) {
4970 pr_err("%s: failed to register platform driver\n",
4971 __func__);
4972 goto out_reg;
4973 }
4974
4975 /* Initialization status */
4976 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4977 &driver_attr_devices);
4978 if (ret)
4979 goto out_dev;
4980
4981 /* RAID-6 h/w enable entry */
4982 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4983 &driver_attr_enable);
4984 if (ret)
4985 goto out_en;
4986
4987 /* GF polynomial to use */
4988 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4989 &driver_attr_poly);
4990 if (!ret)
4991 return ret;
4992
4993 driver_remove_file(&ppc440spe_adma_driver.driver,
4994 &driver_attr_enable);
4995out_en:
4996 driver_remove_file(&ppc440spe_adma_driver.driver,
4997 &driver_attr_devices);
4998out_dev:
4999 /* User will not be able to enable h/w RAID-6 */
5000 pr_err("%s: failed to create RAID-6 driver interface\n",
5001 __func__);
5002 of_unregister_platform_driver(&ppc440spe_adma_driver);
5003out_reg:
5004 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
5005 kfree(ppc440spe_dma_fifo_buf);
5006 return ret;
5007}
5008
5009static void __exit ppc440spe_adma_exit(void)
5010{
5011 driver_remove_file(&ppc440spe_adma_driver.driver,
5012 &driver_attr_poly);
5013 driver_remove_file(&ppc440spe_adma_driver.driver,
5014 &driver_attr_enable);
5015 driver_remove_file(&ppc440spe_adma_driver.driver,
5016 &driver_attr_devices);
5017 of_unregister_platform_driver(&ppc440spe_adma_driver);
5018 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
5019 kfree(ppc440spe_dma_fifo_buf);
5020}
5021
5022arch_initcall(ppc440spe_adma_init);
5023module_exit(ppc440spe_adma_exit);
5024
5025MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
5026MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
5027MODULE_LICENSE("GPL");
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-20 11:03:52 -0500