aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/powerpc/dts-bindings/4xx/ppc440spe-adma.txt93
-rw-r--r--arch/powerpc/include/asm/async_tx.h47
-rw-r--r--arch/powerpc/include/asm/dcr-regs.h23
-rw-r--r--drivers/dma/Kconfig11
-rw-r--r--drivers/dma/Makefile1
-rw-r--r--drivers/dma/ppc4xx/Makefile1
-rw-r--r--drivers/dma/ppc4xx/adma.c5027
-rw-r--r--drivers/dma/ppc4xx/adma.h195
-rw-r--r--drivers/dma/ppc4xx/dma.h223
-rw-r--r--drivers/dma/ppc4xx/xor.h110
10 files changed, 5731 insertions, 0 deletions
diff --git a/Documentation/powerpc/dts-bindings/4xx/ppc440spe-adma.txt b/Documentation/powerpc/dts-bindings/4xx/ppc440spe-adma.txt
new file mode 100644
index 000000000000..515ebcf1b97d
--- /dev/null
+++ b/Documentation/powerpc/dts-bindings/4xx/ppc440spe-adma.txt
@@ -0,0 +1,93 @@
1PPC440SPe DMA/XOR (DMA Controller and XOR Accelerator)
2
3Device nodes needed for operation of the ppc440spe-adma driver
4are specified hereby. These are I2O/DMA, DMA and XOR nodes
5for DMA engines and Memory Queue Module node. The latter is used
6by ADMA driver for configuration of RAID-6 H/W capabilities of
7the PPC440SPe. In addition to the nodes and properties described
8below, the ranges property of PLB node must specify ranges for
9DMA devices.
10
11 i) The I2O node
12
13 Required properties:
14
15 - compatible : "ibm,i2o-440spe";
16 - reg : <registers mapping>
17 - dcr-reg : <DCR registers range>
18
19 Example:
20
21 I2O: i2o@400100000 {
22 compatible = "ibm,i2o-440spe";
23 reg = <0x00000004 0x00100000 0x100>;
24 dcr-reg = <0x060 0x020>;
25 };
26
27
28 ii) The DMA node
29
30 Required properties:
31
32 - compatible : "ibm,dma-440spe";
33 - cell-index : 1 cell, hardware index of the DMA engine
34 (typically 0x0 and 0x1 for DMA0 and DMA1)
35 - reg : <registers mapping>
36 - dcr-reg : <DCR registers range>
37 - interrupts : <interrupt mapping for DMA0/1 interrupts sources:
38 2 sources: DMAx CS FIFO Needs Service IRQ (on UIC0)
39 and DMA Error IRQ (on UIC1). The latter is common
40 for both DMA engines>.
41 - interrupt-parent : needed for interrupt mapping
42
43 Example:
44
45 DMA0: dma0@400100100 {
46 compatible = "ibm,dma-440spe";
47 cell-index = <0>;
48 reg = <0x00000004 0x00100100 0x100>;
49 dcr-reg = <0x060 0x020>;
50 interrupt-parent = <&DMA0>;
51 interrupts = <0 1>;
52 #interrupt-cells = <1>;
53 #address-cells = <0>;
54 #size-cells = <0>;
55 interrupt-map = <
56 0 &UIC0 0x14 4
57 1 &UIC1 0x16 4>;
58 };
59
60
61 iii) XOR Accelerator node
62
63 Required properties:
64
65 - compatible : "amcc,xor-accelerator";
66 - reg : <registers mapping>
67 - interrupts : <interrupt mapping for XOR interrupt source>
68 - interrupt-parent : for interrupt mapping
69
70 Example:
71
72 xor-accel@400200000 {
73 compatible = "amcc,xor-accelerator";
74 reg = <0x00000004 0x00200000 0x400>;
75 interrupt-parent = <&UIC1>;
76 interrupts = <0x1f 4>;
77 };
78
79
80 iv) Memory Queue Module node
81
82 Required properties:
83
84 - compatible : "ibm,mq-440spe";
85 - dcr-reg : <DCR registers range>
86
87 Example:
88
89 MQ0: mq {
90 compatible = "ibm,mq-440spe";
91 dcr-reg = <0x040 0x020>;
92 };
93
diff --git a/arch/powerpc/include/asm/async_tx.h b/arch/powerpc/include/asm/async_tx.h
new file mode 100644
index 000000000000..8b2dc55d01ab
--- /dev/null
+++ b/arch/powerpc/include/asm/async_tx.h
@@ -0,0 +1,47 @@
1/*
2 * Copyright (C) 2008-2009 DENX Software Engineering.
3 *
4 * Author: Yuri Tikhonov <yur@emcraft.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the Free
8 * Software Foundation; either version 2 of the License, or (at your option)
9 * any later version.
10 *
11 * This program is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * this program; if not, write to the Free Software Foundation, Inc., 59
18 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 *
20 * The full GNU General Public License is included in this distribution in the
21 * file called COPYING.
22 */
23#ifndef _ASM_POWERPC_ASYNC_TX_H_
24#define _ASM_POWERPC_ASYNC_TX_H_
25
26#if defined(CONFIG_440SPe) || defined(CONFIG_440SP)
27extern struct dma_chan *
28ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
29 struct page **dst_lst, int dst_cnt, struct page **src_lst,
30 int src_cnt, size_t src_sz);
31
32#define async_tx_find_channel(dep, cap, dst_lst, dst_cnt, src_lst, \
33 src_cnt, src_sz) \
34 ppc440spe_async_tx_find_best_channel(cap, dst_lst, dst_cnt, src_lst, \
35 src_cnt, src_sz)
36#else
37
38#define async_tx_find_channel(dep, type, dst, dst_count, src, src_count, len) \
39 __async_tx_find_channel(dep, type)
40
41struct dma_chan *
42__async_tx_find_channel(struct async_submit_ctl *submit,
43 enum dma_transaction_type tx_type);
44
45#endif
46
47#endif
diff --git a/arch/powerpc/include/asm/dcr-regs.h b/arch/powerpc/include/asm/dcr-regs.h
index 828e3aa1f2fc..380274de429f 100644
--- a/arch/powerpc/include/asm/dcr-regs.h
+++ b/arch/powerpc/include/asm/dcr-regs.h
@@ -157,4 +157,27 @@
157#define L2C_SNP_SSR_32G 0x0000f000 157#define L2C_SNP_SSR_32G 0x0000f000
158#define L2C_SNP_ESR 0x00000800 158#define L2C_SNP_ESR 0x00000800
159 159
160/*
161 * DCR register offsets for 440SP/440SPe I2O/DMA controller.
162 * The base address is configured in the device tree.
163 */
164#define DCRN_I2O0_IBAL 0x006
165#define DCRN_I2O0_IBAH 0x007
166#define I2O_REG_ENABLE 0x00000001 /* Enable I2O/DMA access */
167
168/* 440SP/440SPe Software Reset DCR */
169#define DCRN_SDR0_SRST 0x0200
170#define DCRN_SDR0_SRST_I2ODMA (0x80000000 >> 15) /* Reset I2O/DMA */
171
172/* 440SP/440SPe Memory Queue DCR offsets */
173#define DCRN_MQ0_XORBA 0x04
174#define DCRN_MQ0_CF2H 0x06
175#define DCRN_MQ0_CFBHL 0x0f
176#define DCRN_MQ0_BAUH 0x10
177
178/* HB/LL Paths Configuration Register */
179#define MQ0_CFBHL_TPLM 28
180#define MQ0_CFBHL_HBCL 23
181#define MQ0_CFBHL_POLY 15
182
160#endif /* __DCR_REGS_H__ */ 183#endif /* __DCR_REGS_H__ */
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 24cdd20fe462..fe93d70f2e37 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -116,6 +116,17 @@ config COH901318
116 help 116 help
117 Enable support for ST-Ericsson COH 901 318 DMA. 117 Enable support for ST-Ericsson COH 901 318 DMA.
118 118
119config AMCC_PPC440SPE_ADMA
120 tristate "AMCC PPC440SPe ADMA support"
121 depends on 440SPe || 440SP
122 select DMA_ENGINE
123 select ARCH_HAS_ASYNC_TX_FIND_CHANNEL
124 help
125 Enable support for the AMCC PPC440SPe RAID engines.
126
127config ARCH_HAS_ASYNC_TX_FIND_CHANNEL
128 bool
129
119config DMA_ENGINE 130config DMA_ENGINE
120 bool 131 bool
121 132
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index 4db768e09cf3..807053d48232 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -11,3 +11,4 @@ obj-$(CONFIG_MX3_IPU) += ipu/
11obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o 11obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
12obj-$(CONFIG_SH_DMAE) += shdma.o 12obj-$(CONFIG_SH_DMAE) += shdma.o
13obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o 13obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o
14obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/
diff --git a/drivers/dma/ppc4xx/Makefile b/drivers/dma/ppc4xx/Makefile
new file mode 100644
index 000000000000..b3d259b3e52a
--- /dev/null
+++ b/drivers/dma/ppc4xx/Makefile
@@ -0,0 +1 @@
obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += adma.o
diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c
new file mode 100644
index 000000000000..0a3478e910f0
--- /dev/null
+++ b/drivers/dma/ppc4xx/adma.c
@@ -0,0 +1,5027 @@
1/*
2 * Copyright (C) 2006-2009 DENX Software Engineering.
3 *
4 * Author: Yuri Tikhonov <yur@emcraft.com>
5 *
6 * Further porting to arch/powerpc by
7 * Anatolij Gustschin <agust@denx.de>
8 *
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU General Public License as published by the Free
11 * Software Foundation; either version 2 of the License, or (at your option)
12 * any later version.
13 *
14 * This program is distributed in the hope that it will be useful, but WITHOUT
15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 * more details.
18 *
19 * You should have received a copy of the GNU General Public License along with
20 * this program; if not, write to the Free Software Foundation, Inc., 59
21 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 *
23 * The full GNU General Public License is included in this distribution in the
24 * file called COPYING.
25 */
26
27/*
28 * This driver supports the asynchrounous DMA copy and RAID engines available
29 * on the AMCC PPC440SPe Processors.
30 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
31 * ADMA driver written by D.Williams.
32 */
33
34#include <linux/init.h>
35#include <linux/module.h>
36#include <linux/async_tx.h>
37#include <linux/delay.h>
38#include <linux/dma-mapping.h>
39#include <linux/spinlock.h>
40#include <linux/interrupt.h>
41#include <linux/uaccess.h>
42#include <linux/proc_fs.h>
43#include <linux/of.h>
44#include <linux/of_platform.h>
45#include <asm/dcr.h>
46#include <asm/dcr-regs.h>
47#include "adma.h"
48
49enum ppc_adma_init_code {
50 PPC_ADMA_INIT_OK = 0,
51 PPC_ADMA_INIT_MEMRES,
52 PPC_ADMA_INIT_MEMREG,
53 PPC_ADMA_INIT_ALLOC,
54 PPC_ADMA_INIT_COHERENT,
55 PPC_ADMA_INIT_CHANNEL,
56 PPC_ADMA_INIT_IRQ1,
57 PPC_ADMA_INIT_IRQ2,
58 PPC_ADMA_INIT_REGISTER
59};
60
61static char *ppc_adma_errors[] = {
62 [PPC_ADMA_INIT_OK] = "ok",
63 [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
64 [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
65 [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
66 "structure",
67 [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
68 "hardware descriptors",
69 [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
70 [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
71 [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
72 [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
73};
74
75static enum ppc_adma_init_code
76ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
77
78struct ppc_dma_chan_ref {
79 struct dma_chan *chan;
80 struct list_head node;
81};
82
83/* The list of channels exported by ppc440spe ADMA */
84struct list_head
85ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
86
87/* This flag is set when want to refetch the xor chain in the interrupt
88 * handler
89 */
90static u32 do_xor_refetch;
91
92/* Pointer to DMA0, DMA1 CP/CS FIFO */
93static void *ppc440spe_dma_fifo_buf;
94
95/* Pointers to last submitted to DMA0, DMA1 CDBs */
96static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
97static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
98
99/* Pointer to last linked and submitted xor CB */
100static struct ppc440spe_adma_desc_slot *xor_last_linked;
101static struct ppc440spe_adma_desc_slot *xor_last_submit;
102
103/* This array is used in data-check operations for storing a pattern */
104static char ppc440spe_qword[16];
105
106static atomic_t ppc440spe_adma_err_irq_ref;
107static dcr_host_t ppc440spe_mq_dcr_host;
108static unsigned int ppc440spe_mq_dcr_len;
109
110/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
111 * the block size in transactions, then we do not allow to activate more than
112 * only one RXOR transactions simultaneously. So use this var to store
113 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
114 * set) or not (PPC440SPE_RXOR_RUN is clear).
115 */
116static unsigned long ppc440spe_rxor_state;
117
118/* These are used in enable & check routines
119 */
120static u32 ppc440spe_r6_enabled;
121static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
122static struct completion ppc440spe_r6_test_comp;
123
124static int ppc440spe_adma_dma2rxor_prep_src(
125 struct ppc440spe_adma_desc_slot *desc,
126 struct ppc440spe_rxor *cursor, int index,
127 int src_cnt, u32 addr);
128static void ppc440spe_adma_dma2rxor_set_src(
129 struct ppc440spe_adma_desc_slot *desc,
130 int index, dma_addr_t addr);
131static void ppc440spe_adma_dma2rxor_set_mult(
132 struct ppc440spe_adma_desc_slot *desc,
133 int index, u8 mult);
134
135#ifdef ADMA_LL_DEBUG
136#define ADMA_LL_DBG(x) ({ if (1) x; 0; })
137#else
138#define ADMA_LL_DBG(x) ({ if (0) x; 0; })
139#endif
140
141static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
142{
143 struct dma_cdb *cdb;
144 struct xor_cb *cb;
145 int i;
146
147 switch (chan->device->id) {
148 case 0:
149 case 1:
150 cdb = block;
151
152 pr_debug("CDB at %p [%d]:\n"
153 "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
154 "\t sg1u 0x%08x sg1l 0x%08x\n"
155 "\t sg2u 0x%08x sg2l 0x%08x\n"
156 "\t sg3u 0x%08x sg3l 0x%08x\n",
157 cdb, chan->device->id,
158 cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
159 le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
160 le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
161 le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
162 );
163 break;
164 case 2:
165 cb = block;
166
167 pr_debug("CB at %p [%d]:\n"
168 "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
169 "\t cbtah 0x%08x cbtal 0x%08x\n"
170 "\t cblah 0x%08x cblal 0x%08x\n",
171 cb, chan->device->id,
172 cb->cbc, cb->cbbc, cb->cbs,
173 cb->cbtah, cb->cbtal,
174 cb->cblah, cb->cblal);
175 for (i = 0; i < 16; i++) {
176 if (i && !cb->ops[i].h && !cb->ops[i].l)
177 continue;
178 pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
179 i, cb->ops[i].h, cb->ops[i].l);
180 }
181 break;
182 }
183}
184
185static void print_cb_list(struct ppc440spe_adma_chan *chan,
186 struct ppc440spe_adma_desc_slot *iter)
187{
188 for (; iter; iter = iter->hw_next)
189 print_cb(chan, iter->hw_desc);
190}
191
192static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
193 unsigned int src_cnt)
194{
195 int i;
196
197 pr_debug("\n%s(%d):\nsrc: ", __func__, id);
198 for (i = 0; i < src_cnt; i++)
199 pr_debug("\t0x%016llx ", src[i]);
200 pr_debug("dst:\n\t0x%016llx\n", dst);
201}
202
203static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
204 unsigned int src_cnt)
205{
206 int i;
207
208 pr_debug("\n%s(%d):\nsrc: ", __func__, id);
209 for (i = 0; i < src_cnt; i++)
210 pr_debug("\t0x%016llx ", src[i]);
211 pr_debug("dst: ");
212 for (i = 0; i < 2; i++)
213 pr_debug("\t0x%016llx ", dst[i]);
214}
215
216static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
217 unsigned int src_cnt,
218 const unsigned char *scf)
219{
220 int i;
221
222 pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
223 if (scf) {
224 for (i = 0; i < src_cnt; i++)
225 pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
226 } else {
227 for (i = 0; i < src_cnt; i++)
228 pr_debug("\t0x%016llx(no) ", src[i]);
229 }
230
231 pr_debug("dst: ");
232 for (i = 0; i < 2; i++)
233 pr_debug("\t0x%016llx ", src[src_cnt + i]);
234}
235
236/******************************************************************************
237 * Command (Descriptor) Blocks low-level routines
238 ******************************************************************************/
239/**
240 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
241 * pseudo operation
242 */
243static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
244 struct ppc440spe_adma_chan *chan)
245{
246 struct xor_cb *p;
247
248 switch (chan->device->id) {
249 case PPC440SPE_XOR_ID:
250 p = desc->hw_desc;
251 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
252 /* NOP with Command Block Complete Enable */
253 p->cbc = XOR_CBCR_CBCE_BIT;
254 break;
255 case PPC440SPE_DMA0_ID:
256 case PPC440SPE_DMA1_ID:
257 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
258 /* NOP with interrupt */
259 set_bit(PPC440SPE_DESC_INT, &desc->flags);
260 break;
261 default:
262 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
263 __func__);
264 break;
265 }
266}
267
268/**
269 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
270 * pseudo operation
271 */
272static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
273{
274 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
275 desc->hw_next = NULL;
276 desc->src_cnt = 0;
277 desc->dst_cnt = 1;
278}
279
280/**
281 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
282 */
283static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
284 int src_cnt, unsigned long flags)
285{
286 struct xor_cb *hw_desc = desc->hw_desc;
287
288 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
289 desc->hw_next = NULL;
290 desc->src_cnt = src_cnt;
291 desc->dst_cnt = 1;
292
293 hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
294 if (flags & DMA_PREP_INTERRUPT)
295 /* Enable interrupt on completion */
296 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
297}
298
299/**
300 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
301 * operation in DMA2 controller
302 */
303static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
304 int dst_cnt, int src_cnt, unsigned long flags)
305{
306 struct xor_cb *hw_desc = desc->hw_desc;
307
308 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
309 desc->hw_next = NULL;
310 desc->src_cnt = src_cnt;
311 desc->dst_cnt = dst_cnt;
312 memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
313 desc->descs_per_op = 0;
314
315 hw_desc->cbc = XOR_CBCR_TGT_BIT;
316 if (flags & DMA_PREP_INTERRUPT)
317 /* Enable interrupt on completion */
318 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
319}
320
321#define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE
322#define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1)
323#define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1)
324
325/**
326 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
327 * with DMA0/1
328 */
329static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
330 int dst_cnt, int src_cnt, unsigned long flags,
331 unsigned long op)
332{
333 struct dma_cdb *hw_desc;
334 struct ppc440spe_adma_desc_slot *iter;
335 u8 dopc;
336
337 /* Common initialization of a PQ descriptors chain */
338 set_bits(op, &desc->flags);
339 desc->src_cnt = src_cnt;
340 desc->dst_cnt = dst_cnt;
341
342 /* WXOR MULTICAST if both P and Q are being computed
343 * MV_SG1_SG2 if Q only
344 */
345 dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
346 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
347
348 list_for_each_entry(iter, &desc->group_list, chain_node) {
349 hw_desc = iter->hw_desc;
350 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
351
352 if (likely(!list_is_last(&iter->chain_node,
353 &desc->group_list))) {
354 /* set 'next' pointer */
355 iter->hw_next = list_entry(iter->chain_node.next,
356 struct ppc440spe_adma_desc_slot, chain_node);
357 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
358 } else {
359 /* this is the last descriptor.
360 * this slot will be pasted from ADMA level
361 * each time it wants to configure parameters
362 * of the transaction (src, dst, ...)
363 */
364 iter->hw_next = NULL;
365 if (flags & DMA_PREP_INTERRUPT)
366 set_bit(PPC440SPE_DESC_INT, &iter->flags);
367 else
368 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
369 }
370 }
371
372 /* Set OPS depending on WXOR/RXOR type of operation */
373 if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
374 /* This is a WXOR only chain:
375 * - first descriptors are for zeroing destinations
376 * if PPC440SPE_ZERO_P/Q set;
377 * - descriptors remained are for GF-XOR operations.
378 */
379 iter = list_first_entry(&desc->group_list,
380 struct ppc440spe_adma_desc_slot,
381 chain_node);
382
383 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
384 hw_desc = iter->hw_desc;
385 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
386 iter = list_first_entry(&iter->chain_node,
387 struct ppc440spe_adma_desc_slot,
388 chain_node);
389 }
390
391 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
392 hw_desc = iter->hw_desc;
393 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
394 iter = list_first_entry(&iter->chain_node,
395 struct ppc440spe_adma_desc_slot,
396 chain_node);
397 }
398
399 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
400 hw_desc = iter->hw_desc;
401 hw_desc->opc = dopc;
402 }
403 } else {
404 /* This is either RXOR-only or mixed RXOR/WXOR */
405
406 /* The first 1 or 2 slots in chain are always RXOR,
407 * if need to calculate P & Q, then there are two
408 * RXOR slots; if only P or only Q, then there is one
409 */
410 iter = list_first_entry(&desc->group_list,
411 struct ppc440spe_adma_desc_slot,
412 chain_node);
413 hw_desc = iter->hw_desc;
414 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
415
416 if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
417 iter = list_first_entry(&iter->chain_node,
418 struct ppc440spe_adma_desc_slot,
419 chain_node);
420 hw_desc = iter->hw_desc;
421 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
422 }
423
424 /* The remaining descs (if any) are WXORs */
425 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
426 iter = list_first_entry(&iter->chain_node,
427 struct ppc440spe_adma_desc_slot,
428 chain_node);
429 list_for_each_entry_from(iter, &desc->group_list,
430 chain_node) {
431 hw_desc = iter->hw_desc;
432 hw_desc->opc = dopc;
433 }
434 }
435 }
436}
437
438/**
439 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
440 * for PQ_ZERO_SUM operation
441 */
442static void ppc440spe_desc_init_dma01pqzero_sum(
443 struct ppc440spe_adma_desc_slot *desc,
444 int dst_cnt, int src_cnt)
445{
446 struct dma_cdb *hw_desc;
447 struct ppc440spe_adma_desc_slot *iter;
448 int i = 0;
449 u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
450 DMA_CDB_OPC_MV_SG1_SG2;
451 /*
452 * Initialize starting from 2nd or 3rd descriptor dependent
453 * on dst_cnt. First one or two slots are for cloning P
454 * and/or Q to chan->pdest and/or chan->qdest as we have
455 * to preserve original P/Q.
456 */
457 iter = list_first_entry(&desc->group_list,
458 struct ppc440spe_adma_desc_slot, chain_node);
459 iter = list_entry(iter->chain_node.next,
460 struct ppc440spe_adma_desc_slot, chain_node);
461
462 if (dst_cnt > 1) {
463 iter = list_entry(iter->chain_node.next,
464 struct ppc440spe_adma_desc_slot, chain_node);
465 }
466 /* initialize each source descriptor in chain */
467 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
468 hw_desc = iter->hw_desc;
469 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
470 iter->src_cnt = 0;
471 iter->dst_cnt = 0;
472
473 /* This is a ZERO_SUM operation:
474 * - <src_cnt> descriptors starting from 2nd or 3rd
475 * descriptor are for GF-XOR operations;
476 * - remaining <dst_cnt> descriptors are for checking the result
477 */
478 if (i++ < src_cnt)
479 /* MV_SG1_SG2 if only Q is being verified
480 * MULTICAST if both P and Q are being verified
481 */
482 hw_desc->opc = dopc;
483 else
484 /* DMA_CDB_OPC_DCHECK128 operation */
485 hw_desc->opc = DMA_CDB_OPC_DCHECK128;
486
487 if (likely(!list_is_last(&iter->chain_node,
488 &desc->group_list))) {
489 /* set 'next' pointer */
490 iter->hw_next = list_entry(iter->chain_node.next,
491 struct ppc440spe_adma_desc_slot,
492 chain_node);
493 } else {
494 /* this is the last descriptor.
495 * this slot will be pasted from ADMA level
496 * each time it wants to configure parameters
497 * of the transaction (src, dst, ...)
498 */
499 iter->hw_next = NULL;
500 /* always enable interrupt generation since we get
501 * the status of pqzero from the handler
502 */
503 set_bit(PPC440SPE_DESC_INT, &iter->flags);
504 }
505 }
506 desc->src_cnt = src_cnt;
507 desc->dst_cnt = dst_cnt;
508}
509
510/**
511 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
512 */
513static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
514 unsigned long flags)
515{
516 struct dma_cdb *hw_desc = desc->hw_desc;
517
518 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
519 desc->hw_next = NULL;
520 desc->src_cnt = 1;
521 desc->dst_cnt = 1;
522
523 if (flags & DMA_PREP_INTERRUPT)
524 set_bit(PPC440SPE_DESC_INT, &desc->flags);
525 else
526 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
527
528 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
529}
530
531/**
532 * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation
533 */
534static void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc,
535 int value, unsigned long flags)
536{
537 struct dma_cdb *hw_desc = desc->hw_desc;
538
539 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
540 desc->hw_next = NULL;
541 desc->src_cnt = 1;
542 desc->dst_cnt = 1;
543
544 if (flags & DMA_PREP_INTERRUPT)
545 set_bit(PPC440SPE_DESC_INT, &desc->flags);
546 else
547 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
548
549 hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value);
550 hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value);
551 hw_desc->opc = DMA_CDB_OPC_DFILL128;
552}
553
554/**
555 * ppc440spe_desc_set_src_addr - set source address into the descriptor
556 */
557static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
558 struct ppc440spe_adma_chan *chan,
559 int src_idx, dma_addr_t addrh,
560 dma_addr_t addrl)
561{
562 struct dma_cdb *dma_hw_desc;
563 struct xor_cb *xor_hw_desc;
564 phys_addr_t addr64, tmplow, tmphi;
565
566 switch (chan->device->id) {
567 case PPC440SPE_DMA0_ID:
568 case PPC440SPE_DMA1_ID:
569 if (!addrh) {
570 addr64 = addrl;
571 tmphi = (addr64 >> 32);
572 tmplow = (addr64 & 0xFFFFFFFF);
573 } else {
574 tmphi = addrh;
575 tmplow = addrl;
576 }
577 dma_hw_desc = desc->hw_desc;
578 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
579 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
580 break;
581 case PPC440SPE_XOR_ID:
582 xor_hw_desc = desc->hw_desc;
583 xor_hw_desc->ops[src_idx].l = addrl;
584 xor_hw_desc->ops[src_idx].h |= addrh;
585 break;
586 }
587}
588
589/**
590 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
591 */
592static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
593 struct ppc440spe_adma_chan *chan, u32 mult_index,
594 int sg_index, unsigned char mult_value)
595{
596 struct dma_cdb *dma_hw_desc;
597 struct xor_cb *xor_hw_desc;
598 u32 *psgu;
599
600 switch (chan->device->id) {
601 case PPC440SPE_DMA0_ID:
602 case PPC440SPE_DMA1_ID:
603 dma_hw_desc = desc->hw_desc;
604
605 switch (sg_index) {
606 /* for RXOR operations set multiplier
607 * into source cued address
608 */
609 case DMA_CDB_SG_SRC:
610 psgu = &dma_hw_desc->sg1u;
611 break;
612 /* for WXOR operations set multiplier
613 * into destination cued address(es)
614 */
615 case DMA_CDB_SG_DST1:
616 psgu = &dma_hw_desc->sg2u;
617 break;
618 case DMA_CDB_SG_DST2:
619 psgu = &dma_hw_desc->sg3u;
620 break;
621 default:
622 BUG();
623 }
624
625 *psgu |= cpu_to_le32(mult_value << mult_index);
626 break;
627 case PPC440SPE_XOR_ID:
628 xor_hw_desc = desc->hw_desc;
629 break;
630 default:
631 BUG();
632 }
633}
634
635/**
636 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
637 */
638static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
639 struct ppc440spe_adma_chan *chan,
640 dma_addr_t addrh, dma_addr_t addrl,
641 u32 dst_idx)
642{
643 struct dma_cdb *dma_hw_desc;
644 struct xor_cb *xor_hw_desc;
645 phys_addr_t addr64, tmphi, tmplow;
646 u32 *psgu, *psgl;
647
648 switch (chan->device->id) {
649 case PPC440SPE_DMA0_ID:
650 case PPC440SPE_DMA1_ID:
651 if (!addrh) {
652 addr64 = addrl;
653 tmphi = (addr64 >> 32);
654 tmplow = (addr64 & 0xFFFFFFFF);
655 } else {
656 tmphi = addrh;
657 tmplow = addrl;
658 }
659 dma_hw_desc = desc->hw_desc;
660
661 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
662 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
663
664 *psgl = cpu_to_le32((u32)tmplow);
665 *psgu |= cpu_to_le32((u32)tmphi);
666 break;
667 case PPC440SPE_XOR_ID:
668 xor_hw_desc = desc->hw_desc;
669 xor_hw_desc->cbtal = addrl;
670 xor_hw_desc->cbtah |= addrh;
671 break;
672 }
673}
674
675/**
676 * ppc440spe_desc_set_byte_count - set number of data bytes involved
677 * into the operation
678 */
679static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
680 struct ppc440spe_adma_chan *chan,
681 u32 byte_count)
682{
683 struct dma_cdb *dma_hw_desc;
684 struct xor_cb *xor_hw_desc;
685
686 switch (chan->device->id) {
687 case PPC440SPE_DMA0_ID:
688 case PPC440SPE_DMA1_ID:
689 dma_hw_desc = desc->hw_desc;
690 dma_hw_desc->cnt = cpu_to_le32(byte_count);
691 break;
692 case PPC440SPE_XOR_ID:
693 xor_hw_desc = desc->hw_desc;
694 xor_hw_desc->cbbc = byte_count;
695 break;
696 }
697}
698
699/**
700 * ppc440spe_desc_set_rxor_block_size - set RXOR block size
701 */
702static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
703{
704 /* assume that byte_count is aligned on the 512-boundary;
705 * thus write it directly to the register (bits 23:31 are
706 * reserved there).
707 */
708 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
709}
710
711/**
712 * ppc440spe_desc_set_dcheck - set CHECK pattern
713 */
714static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
715 struct ppc440spe_adma_chan *chan, u8 *qword)
716{
717 struct dma_cdb *dma_hw_desc;
718
719 switch (chan->device->id) {
720 case PPC440SPE_DMA0_ID:
721 case PPC440SPE_DMA1_ID:
722 dma_hw_desc = desc->hw_desc;
723 iowrite32(qword[0], &dma_hw_desc->sg3l);
724 iowrite32(qword[4], &dma_hw_desc->sg3u);
725 iowrite32(qword[8], &dma_hw_desc->sg2l);
726 iowrite32(qword[12], &dma_hw_desc->sg2u);
727 break;
728 default:
729 BUG();
730 }
731}
732
733/**
734 * ppc440spe_xor_set_link - set link address in xor CB
735 */
736static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
737 struct ppc440spe_adma_desc_slot *next_desc)
738{
739 struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
740
741 if (unlikely(!next_desc || !(next_desc->phys))) {
742 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
743 __func__, next_desc,
744 next_desc ? next_desc->phys : 0);
745 BUG();
746 }
747
748 xor_hw_desc->cbs = 0;
749 xor_hw_desc->cblal = next_desc->phys;
750 xor_hw_desc->cblah = 0;
751 xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
752}
753
754/**
755 * ppc440spe_desc_set_link - set the address of descriptor following this
756 * descriptor in chain
757 */
758static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
759 struct ppc440spe_adma_desc_slot *prev_desc,
760 struct ppc440spe_adma_desc_slot *next_desc)
761{
762 unsigned long flags;
763 struct ppc440spe_adma_desc_slot *tail = next_desc;
764
765 if (unlikely(!prev_desc || !next_desc ||
766 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
767 /* If previous next is overwritten something is wrong.
768 * though we may refetch from append to initiate list
769 * processing; in this case - it's ok.
770 */
771 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
772 "prev->hw_next=0x%p\n", __func__, prev_desc,
773 next_desc, prev_desc ? prev_desc->hw_next : 0);
774 BUG();
775 }
776
777 local_irq_save(flags);
778
779 /* do s/w chaining both for DMA and XOR descriptors */
780 prev_desc->hw_next = next_desc;
781
782 switch (chan->device->id) {
783 case PPC440SPE_DMA0_ID:
784 case PPC440SPE_DMA1_ID:
785 break;
786 case PPC440SPE_XOR_ID:
787 /* bind descriptor to the chain */
788 while (tail->hw_next)
789 tail = tail->hw_next;
790 xor_last_linked = tail;
791
792 if (prev_desc == xor_last_submit)
793 /* do not link to the last submitted CB */
794 break;
795 ppc440spe_xor_set_link(prev_desc, next_desc);
796 break;
797 }
798
799 local_irq_restore(flags);
800}
801
802/**
803 * ppc440spe_desc_get_src_addr - extract the source address from the descriptor
804 */
805static u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc,
806 struct ppc440spe_adma_chan *chan, int src_idx)
807{
808 struct dma_cdb *dma_hw_desc;
809 struct xor_cb *xor_hw_desc;
810
811 switch (chan->device->id) {
812 case PPC440SPE_DMA0_ID:
813 case PPC440SPE_DMA1_ID:
814 dma_hw_desc = desc->hw_desc;
815 /* May have 0, 1, 2, or 3 sources */
816 switch (dma_hw_desc->opc) {
817 case DMA_CDB_OPC_NO_OP:
818 case DMA_CDB_OPC_DFILL128:
819 return 0;
820 case DMA_CDB_OPC_DCHECK128:
821 if (unlikely(src_idx)) {
822 printk(KERN_ERR "%s: try to get %d source for"
823 " DCHECK128\n", __func__, src_idx);
824 BUG();
825 }
826 return le32_to_cpu(dma_hw_desc->sg1l);
827 case DMA_CDB_OPC_MULTICAST:
828 case DMA_CDB_OPC_MV_SG1_SG2:
829 if (unlikely(src_idx > 2)) {
830 printk(KERN_ERR "%s: try to get %d source from"
831 " DMA descr\n", __func__, src_idx);
832 BUG();
833 }
834 if (src_idx) {
835 if (le32_to_cpu(dma_hw_desc->sg1u) &
836 DMA_CUED_XOR_WIN_MSK) {
837 u8 region;
838
839 if (src_idx == 1)
840 return le32_to_cpu(
841 dma_hw_desc->sg1l) +
842 desc->unmap_len;
843
844 region = (le32_to_cpu(
845 dma_hw_desc->sg1u)) >>
846 DMA_CUED_REGION_OFF;
847
848 region &= DMA_CUED_REGION_MSK;
849 switch (region) {
850 case DMA_RXOR123:
851 return le32_to_cpu(
852 dma_hw_desc->sg1l) +
853 (desc->unmap_len << 1);
854 case DMA_RXOR124:
855 return le32_to_cpu(
856 dma_hw_desc->sg1l) +
857 (desc->unmap_len * 3);
858 case DMA_RXOR125:
859 return le32_to_cpu(
860 dma_hw_desc->sg1l) +
861 (desc->unmap_len << 2);
862 default:
863 printk(KERN_ERR
864 "%s: try to"
865 " get src3 for region %02x"
866 "PPC440SPE_DESC_RXOR12?\n",
867 __func__, region);
868 BUG();
869 }
870 } else {
871 printk(KERN_ERR
872 "%s: try to get %d"
873 " source for non-cued descr\n",
874 __func__, src_idx);
875 BUG();
876 }
877 }
878 return le32_to_cpu(dma_hw_desc->sg1l);
879 default:
880 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
881 __func__, dma_hw_desc->opc);
882 BUG();
883 }
884 return le32_to_cpu(dma_hw_desc->sg1l);
885 case PPC440SPE_XOR_ID:
886 /* May have up to 16 sources */
887 xor_hw_desc = desc->hw_desc;
888 return xor_hw_desc->ops[src_idx].l;
889 }
890 return 0;
891}
892
893/**
894 * ppc440spe_desc_get_dest_addr - extract the destination address from the
895 * descriptor
896 */
897static u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc,
898 struct ppc440spe_adma_chan *chan, int idx)
899{
900 struct dma_cdb *dma_hw_desc;
901 struct xor_cb *xor_hw_desc;
902
903 switch (chan->device->id) {
904 case PPC440SPE_DMA0_ID:
905 case PPC440SPE_DMA1_ID:
906 dma_hw_desc = desc->hw_desc;
907
908 if (likely(!idx))
909 return le32_to_cpu(dma_hw_desc->sg2l);
910 return le32_to_cpu(dma_hw_desc->sg3l);
911 case PPC440SPE_XOR_ID:
912 xor_hw_desc = desc->hw_desc;
913 return xor_hw_desc->cbtal;
914 }
915 return 0;
916}
917
918/**
919 * ppc440spe_desc_get_src_num - extract the number of source addresses from
920 * the descriptor
921 */
922static u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc,
923 struct ppc440spe_adma_chan *chan)
924{
925 struct dma_cdb *dma_hw_desc;
926 struct xor_cb *xor_hw_desc;
927
928 switch (chan->device->id) {
929 case PPC440SPE_DMA0_ID:
930 case PPC440SPE_DMA1_ID:
931 dma_hw_desc = desc->hw_desc;
932
933 switch (dma_hw_desc->opc) {
934 case DMA_CDB_OPC_NO_OP:
935 case DMA_CDB_OPC_DFILL128:
936 return 0;
937 case DMA_CDB_OPC_DCHECK128:
938 return 1;
939 case DMA_CDB_OPC_MV_SG1_SG2:
940 case DMA_CDB_OPC_MULTICAST:
941 /*
942 * Only for RXOR operations we have more than
943 * one source
944 */
945 if (le32_to_cpu(dma_hw_desc->sg1u) &
946 DMA_CUED_XOR_WIN_MSK) {
947 /* RXOR op, there are 2 or 3 sources */
948 if (((le32_to_cpu(dma_hw_desc->sg1u) >>
949 DMA_CUED_REGION_OFF) &
950 DMA_CUED_REGION_MSK) == DMA_RXOR12) {
951 /* RXOR 1-2 */
952 return 2;
953 } else {
954 /* RXOR 1-2-3/1-2-4/1-2-5 */
955 return 3;
956 }
957 }
958 return 1;
959 default:
960 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
961 __func__, dma_hw_desc->opc);
962 BUG();
963 }
964 case PPC440SPE_XOR_ID:
965 /* up to 16 sources */
966 xor_hw_desc = desc->hw_desc;
967 return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK;
968 default:
969 BUG();
970 }
971 return 0;
972}
973
974/**
975 * ppc440spe_desc_get_dst_num - get the number of destination addresses in
976 * this descriptor
977 */
978static u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc,
979 struct ppc440spe_adma_chan *chan)
980{
981 struct dma_cdb *dma_hw_desc;
982
983 switch (chan->device->id) {
984 case PPC440SPE_DMA0_ID:
985 case PPC440SPE_DMA1_ID:
986 /* May be 1 or 2 destinations */
987 dma_hw_desc = desc->hw_desc;
988 switch (dma_hw_desc->opc) {
989 case DMA_CDB_OPC_NO_OP:
990 case DMA_CDB_OPC_DCHECK128:
991 return 0;
992 case DMA_CDB_OPC_MV_SG1_SG2:
993 case DMA_CDB_OPC_DFILL128:
994 return 1;
995 case DMA_CDB_OPC_MULTICAST:
996 if (desc->dst_cnt == 2)
997 return 2;
998 else
999 return 1;
1000 default:
1001 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
1002 __func__, dma_hw_desc->opc);
1003 BUG();
1004 }
1005 case PPC440SPE_XOR_ID:
1006 /* Always only 1 destination */
1007 return 1;
1008 default:
1009 BUG();
1010 }
1011 return 0;
1012}
1013
1014/**
1015 * ppc440spe_desc_get_link - get the address of the descriptor that
1016 * follows this one
1017 */
1018static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
1019 struct ppc440spe_adma_chan *chan)
1020{
1021 if (!desc->hw_next)
1022 return 0;
1023
1024 return desc->hw_next->phys;
1025}
1026
1027/**
1028 * ppc440spe_desc_is_aligned - check alignment
1029 */
1030static inline int ppc440spe_desc_is_aligned(
1031 struct ppc440spe_adma_desc_slot *desc, int num_slots)
1032{
1033 return (desc->idx & (num_slots - 1)) ? 0 : 1;
1034}
1035
1036/**
1037 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
1038 * XOR operation
1039 */
1040static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
1041 int *slots_per_op)
1042{
1043 int slot_cnt;
1044
1045 /* each XOR descriptor provides up to 16 source operands */
1046 slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
1047
1048 if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
1049 return slot_cnt;
1050
1051 printk(KERN_ERR "%s: len %d > max %d !!\n",
1052 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
1053 BUG();
1054 return slot_cnt;
1055}
1056
1057/**
1058 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
1059 * DMA2 PQ operation
1060 */
1061static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
1062 int src_cnt, size_t len)
1063{
1064 signed long long order = 0;
1065 int state = 0;
1066 int addr_count = 0;
1067 int i;
1068 for (i = 1; i < src_cnt; i++) {
1069 dma_addr_t cur_addr = srcs[i];
1070 dma_addr_t old_addr = srcs[i-1];
1071 switch (state) {
1072 case 0:
1073 if (cur_addr == old_addr + len) {
1074 /* direct RXOR */
1075 order = 1;
1076 state = 1;
1077 if (i == src_cnt-1)
1078 addr_count++;
1079 } else if (old_addr == cur_addr + len) {
1080 /* reverse RXOR */
1081 order = -1;
1082 state = 1;
1083 if (i == src_cnt-1)
1084 addr_count++;
1085 } else {
1086 state = 3;
1087 }
1088 break;
1089 case 1:
1090 if (i == src_cnt-2 || (order == -1
1091 && cur_addr != old_addr - len)) {
1092 order = 0;
1093 state = 0;
1094 addr_count++;
1095 } else if (cur_addr == old_addr + len*order) {
1096 state = 2;
1097 if (i == src_cnt-1)
1098 addr_count++;
1099 } else if (cur_addr == old_addr + 2*len) {
1100 state = 2;
1101 if (i == src_cnt-1)
1102 addr_count++;
1103 } else if (cur_addr == old_addr + 3*len) {
1104 state = 2;
1105 if (i == src_cnt-1)
1106 addr_count++;
1107 } else {
1108 order = 0;
1109 state = 0;
1110 addr_count++;
1111 }
1112 break;
1113 case 2:
1114 order = 0;
1115 state = 0;
1116 addr_count++;
1117 break;
1118 }
1119 if (state == 3)
1120 break;
1121 }
1122 if (src_cnt <= 1 || (state != 1 && state != 2)) {
1123 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
1124 __func__, src_cnt, state, addr_count, order);
1125 for (i = 0; i < src_cnt; i++)
1126 pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
1127 BUG();
1128 }
1129
1130 return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
1131}
1132
1133
1134/******************************************************************************
1135 * ADMA channel low-level routines
1136 ******************************************************************************/
1137
1138static u32
1139ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
1140static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
1141
1142/**
1143 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
1144 */
1145static void ppc440spe_adma_device_clear_eot_status(
1146 struct ppc440spe_adma_chan *chan)
1147{
1148 struct dma_regs *dma_reg;
1149 struct xor_regs *xor_reg;
1150 u8 *p = chan->device->dma_desc_pool_virt;
1151 struct dma_cdb *cdb;
1152 u32 rv, i;
1153
1154 switch (chan->device->id) {
1155 case PPC440SPE_DMA0_ID:
1156 case PPC440SPE_DMA1_ID:
1157 /* read FIFO to ack */
1158 dma_reg = chan->device->dma_reg;
1159 while ((rv = ioread32(&dma_reg->csfpl))) {
1160 i = rv & DMA_CDB_ADDR_MSK;
1161 cdb = (struct dma_cdb *)&p[i -
1162 (u32)chan->device->dma_desc_pool];
1163
1164 /* Clear opcode to ack. This is necessary for
1165 * ZeroSum operations only
1166 */
1167 cdb->opc = 0;
1168
1169 if (test_bit(PPC440SPE_RXOR_RUN,
1170 &ppc440spe_rxor_state)) {
1171 /* probably this is a completed RXOR op,
1172 * get pointer to CDB using the fact that
1173 * physical and virtual addresses of CDB
1174 * in pools have the same offsets
1175 */
1176 if (le32_to_cpu(cdb->sg1u) &
1177 DMA_CUED_XOR_BASE) {
1178 /* this is a RXOR */
1179 clear_bit(PPC440SPE_RXOR_RUN,
1180 &ppc440spe_rxor_state);
1181 }
1182 }
1183
1184 if (rv & DMA_CDB_STATUS_MSK) {
1185 /* ZeroSum check failed
1186 */
1187 struct ppc440spe_adma_desc_slot *iter;
1188 dma_addr_t phys = rv & ~DMA_CDB_MSK;
1189
1190 /*
1191 * Update the status of corresponding
1192 * descriptor.
1193 */
1194 list_for_each_entry(iter, &chan->chain,
1195 chain_node) {
1196 if (iter->phys == phys)
1197 break;
1198 }
1199 /*
1200 * if cannot find the corresponding
1201 * slot it's a bug
1202 */
1203 BUG_ON(&iter->chain_node == &chan->chain);
1204
1205 if (iter->xor_check_result) {
1206 if (test_bit(PPC440SPE_DESC_PCHECK,
1207 &iter->flags)) {
1208 *iter->xor_check_result |=
1209 SUM_CHECK_P_RESULT;
1210 } else
1211 if (test_bit(PPC440SPE_DESC_QCHECK,
1212 &iter->flags)) {
1213 *iter->xor_check_result |=
1214 SUM_CHECK_Q_RESULT;
1215 } else
1216 BUG();
1217 }
1218 }
1219 }
1220
1221 rv = ioread32(&dma_reg->dsts);
1222 if (rv) {
1223 pr_err("DMA%d err status: 0x%x\n",
1224 chan->device->id, rv);
1225 /* write back to clear */
1226 iowrite32(rv, &dma_reg->dsts);
1227 }
1228 break;
1229 case PPC440SPE_XOR_ID:
1230 /* reset status bits to ack */
1231 xor_reg = chan->device->xor_reg;
1232 rv = ioread32be(&xor_reg->sr);
1233 iowrite32be(rv, &xor_reg->sr);
1234
1235 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
1236 if (rv & XOR_IE_RPTIE_BIT) {
1237 /* Read PLB Timeout Error.
1238 * Try to resubmit the CB
1239 */
1240 u32 val = ioread32be(&xor_reg->ccbalr);
1241
1242 iowrite32be(val, &xor_reg->cblalr);
1243
1244 val = ioread32be(&xor_reg->crsr);
1245 iowrite32be(val | XOR_CRSR_XAE_BIT,
1246 &xor_reg->crsr);
1247 } else
1248 pr_err("XOR ERR 0x%x status\n", rv);
1249 break;
1250 }
1251
1252 /* if the XORcore is idle, but there are unprocessed CBs
1253 * then refetch the s/w chain here
1254 */
1255 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1256 do_xor_refetch)
1257 ppc440spe_chan_append(chan);
1258 break;
1259 }
1260}
1261
1262/**
1263 * ppc440spe_chan_is_busy - get the channel status
1264 */
1265static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1266{
1267 struct dma_regs *dma_reg;
1268 struct xor_regs *xor_reg;
1269 int busy = 0;
1270
1271 switch (chan->device->id) {
1272 case PPC440SPE_DMA0_ID:
1273 case PPC440SPE_DMA1_ID:
1274 dma_reg = chan->device->dma_reg;
1275 /* if command FIFO's head and tail pointers are equal and
1276 * status tail is the same as command, then channel is free
1277 */
1278 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1279 ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1280 busy = 1;
1281 break;
1282 case PPC440SPE_XOR_ID:
1283 /* use the special status bit for the XORcore
1284 */
1285 xor_reg = chan->device->xor_reg;
1286 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1287 break;
1288 }
1289
1290 return busy;
1291}
1292
1293/**
1294 * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain
1295 */
1296static void ppc440spe_chan_set_first_xor_descriptor(
1297 struct ppc440spe_adma_chan *chan,
1298 struct ppc440spe_adma_desc_slot *next_desc)
1299{
1300 struct xor_regs *xor_reg = chan->device->xor_reg;
1301
1302 if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1303 printk(KERN_INFO "%s: Warn: XORcore is running "
1304 "when try to set the first CDB!\n",
1305 __func__);
1306
1307 xor_last_submit = xor_last_linked = next_desc;
1308
1309 iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1310
1311 iowrite32be(next_desc->phys, &xor_reg->cblalr);
1312 iowrite32be(0, &xor_reg->cblahr);
1313 iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1314 &xor_reg->cbcr);
1315
1316 chan->hw_chain_inited = 1;
1317}
1318
1319/**
1320 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1321 * called with irqs disabled
1322 */
1323static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1324 struct ppc440spe_adma_desc_slot *desc)
1325{
1326 u32 pcdb;
1327 struct dma_regs *dma_reg = chan->device->dma_reg;
1328
1329 pcdb = desc->phys;
1330 if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1331 pcdb |= DMA_CDB_NO_INT;
1332
1333 chan_last_sub[chan->device->id] = desc;
1334
1335 ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1336
1337 iowrite32(pcdb, &dma_reg->cpfpl);
1338}
1339
1340/**
1341 * ppc440spe_chan_append - update the h/w chain in the channel
1342 */
1343static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1344{
1345 struct xor_regs *xor_reg;
1346 struct ppc440spe_adma_desc_slot *iter;
1347 struct xor_cb *xcb;
1348 u32 cur_desc;
1349 unsigned long flags;
1350
1351 local_irq_save(flags);
1352
1353 switch (chan->device->id) {
1354 case PPC440SPE_DMA0_ID:
1355 case PPC440SPE_DMA1_ID:
1356 cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1357
1358 if (likely(cur_desc)) {
1359 iter = chan_last_sub[chan->device->id];
1360 BUG_ON(!iter);
1361 } else {
1362 /* first peer */
1363 iter = chan_first_cdb[chan->device->id];
1364 BUG_ON(!iter);
1365 ppc440spe_dma_put_desc(chan, iter);
1366 chan->hw_chain_inited = 1;
1367 }
1368
1369 /* is there something new to append */
1370 if (!iter->hw_next)
1371 break;
1372
1373 /* flush descriptors from the s/w queue to fifo */
1374 list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1375 ppc440spe_dma_put_desc(chan, iter);
1376 if (!iter->hw_next)
1377 break;
1378 }
1379 break;
1380 case PPC440SPE_XOR_ID:
1381 /* update h/w links and refetch */
1382 if (!xor_last_submit->hw_next)
1383 break;
1384
1385 xor_reg = chan->device->xor_reg;
1386 /* the last linked CDB has to generate an interrupt
1387 * that we'd be able to append the next lists to h/w
1388 * regardless of the XOR engine state at the moment of
1389 * appending of these next lists
1390 */
1391 xcb = xor_last_linked->hw_desc;
1392 xcb->cbc |= XOR_CBCR_CBCE_BIT;
1393
1394 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1395 /* XORcore is idle. Refetch now */
1396 do_xor_refetch = 0;
1397 ppc440spe_xor_set_link(xor_last_submit,
1398 xor_last_submit->hw_next);
1399
1400 ADMA_LL_DBG(print_cb_list(chan,
1401 xor_last_submit->hw_next));
1402
1403 xor_last_submit = xor_last_linked;
1404 iowrite32be(ioread32be(&xor_reg->crsr) |
1405 XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1406 &xor_reg->crsr);
1407 } else {
1408 /* XORcore is running. Refetch later in the handler */
1409 do_xor_refetch = 1;
1410 }
1411
1412 break;
1413 }
1414
1415 local_irq_restore(flags);
1416}
1417
1418/**
1419 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1420 */
1421static u32
1422ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1423{
1424 struct dma_regs *dma_reg;
1425 struct xor_regs *xor_reg;
1426
1427 if (unlikely(!chan->hw_chain_inited))
1428 /* h/w descriptor chain is not initialized yet */
1429 return 0;
1430
1431 switch (chan->device->id) {
1432 case PPC440SPE_DMA0_ID:
1433 case PPC440SPE_DMA1_ID:
1434 dma_reg = chan->device->dma_reg;
1435 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1436 case PPC440SPE_XOR_ID:
1437 xor_reg = chan->device->xor_reg;
1438 return ioread32be(&xor_reg->ccbalr);
1439 }
1440 return 0;
1441}
1442
1443/**
1444 * ppc440spe_chan_run - enable the channel
1445 */
1446static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1447{
1448 struct xor_regs *xor_reg;
1449
1450 switch (chan->device->id) {
1451 case PPC440SPE_DMA0_ID:
1452 case PPC440SPE_DMA1_ID:
1453 /* DMAs are always enabled, do nothing */
1454 break;
1455 case PPC440SPE_XOR_ID:
1456 /* drain write buffer */
1457 xor_reg = chan->device->xor_reg;
1458
1459 /* fetch descriptor pointed to in <link> */
1460 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1461 &xor_reg->crsr);
1462 break;
1463 }
1464}
1465
1466/******************************************************************************
1467 * ADMA device level
1468 ******************************************************************************/
1469
1470static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1471static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1472
1473static dma_cookie_t
1474ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1475
1476static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1477 dma_addr_t addr, int index);
1478static void
1479ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1480 dma_addr_t addr, int index);
1481
1482static void
1483ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1484 dma_addr_t *paddr, unsigned long flags);
1485static void
1486ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1487 dma_addr_t addr, int index);
1488static void
1489ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1490 unsigned char mult, int index, int dst_pos);
1491static void
1492ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1493 dma_addr_t paddr, dma_addr_t qaddr);
1494
1495static struct page *ppc440spe_rxor_srcs[32];
1496
1497/**
1498 * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1499 */
1500static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1501{
1502 int i, order = 0, state = 0;
1503 int idx = 0;
1504
1505 if (unlikely(!(src_cnt > 1)))
1506 return 0;
1507
1508 BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1509
1510 /* Skip holes in the source list before checking */
1511 for (i = 0; i < src_cnt; i++) {
1512 if (!srcs[i])
1513 continue;
1514 ppc440spe_rxor_srcs[idx++] = srcs[i];
1515 }
1516 src_cnt = idx;
1517
1518 for (i = 1; i < src_cnt; i++) {
1519 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1520 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1521
1522 switch (state) {
1523 case 0:
1524 if (cur_addr == old_addr + len) {
1525 /* direct RXOR */
1526 order = 1;
1527 state = 1;
1528 } else if (old_addr == cur_addr + len) {
1529 /* reverse RXOR */
1530 order = -1;
1531 state = 1;
1532 } else
1533 goto out;
1534 break;
1535 case 1:
1536 if ((i == src_cnt - 2) ||
1537 (order == -1 && cur_addr != old_addr - len)) {
1538 order = 0;
1539 state = 0;
1540 } else if ((cur_addr == old_addr + len * order) ||
1541 (cur_addr == old_addr + 2 * len) ||
1542 (cur_addr == old_addr + 3 * len)) {
1543 state = 2;
1544 } else {
1545 order = 0;
1546 state = 0;
1547 }
1548 break;
1549 case 2:
1550 order = 0;
1551 state = 0;
1552 break;
1553 }
1554 }
1555
1556out:
1557 if (state == 1 || state == 2)
1558 return 1;
1559
1560 return 0;
1561}
1562
1563/**
1564 * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1565 * the operation given on this channel. It's assumed that 'chan' is
1566 * capable to process 'cap' type of operation.
1567 * @chan: channel to use
1568 * @cap: type of transaction
1569 * @dst_lst: array of destination pointers
1570 * @dst_cnt: number of destination operands
1571 * @src_lst: array of source pointers
1572 * @src_cnt: number of source operands
1573 * @src_sz: size of each source operand
1574 */
1575static int ppc440spe_adma_estimate(struct dma_chan *chan,
1576 enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1577 struct page **src_lst, int src_cnt, size_t src_sz)
1578{
1579 int ef = 1;
1580
1581 if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1582 /* If RAID-6 capabilities were not activated don't try
1583 * to use them
1584 */
1585 if (unlikely(!ppc440spe_r6_enabled))
1586 return -1;
1587 }
1588 /* In the current implementation of ppc440spe ADMA driver it
1589 * makes sense to pick out only pq case, because it may be
1590 * processed:
1591 * (1) either using Biskup method on DMA2;
1592 * (2) or on DMA0/1.
1593 * Thus we give a favour to (1) if the sources are suitable;
1594 * else let it be processed on one of the DMA0/1 engines.
1595 * In the sum_product case where destination is also the
1596 * source process it on DMA0/1 only.
1597 */
1598 if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1599
1600 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1601 ef = 0; /* sum_product case, process on DMA0/1 */
1602 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1603 ef = 3; /* override (DMA0/1 + idle) */
1604 else
1605 ef = 0; /* can't process on DMA2 if !rxor */
1606 }
1607
1608 /* channel idleness increases the priority */
1609 if (likely(ef) &&
1610 !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1611 ef++;
1612
1613 return ef;
1614}
1615
1616struct dma_chan *
1617ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1618 struct page **dst_lst, int dst_cnt, struct page **src_lst,
1619 int src_cnt, size_t src_sz)
1620{
1621 struct dma_chan *best_chan = NULL;
1622 struct ppc_dma_chan_ref *ref;
1623 int best_rank = -1;
1624
1625 if (unlikely(!src_sz))
1626 return NULL;
1627 if (src_sz > PAGE_SIZE) {
1628 /*
1629 * should a user of the api ever pass > PAGE_SIZE requests
1630 * we sort out cases where temporary page-sized buffers
1631 * are used.
1632 */
1633 switch (cap) {
1634 case DMA_PQ:
1635 if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1636 return NULL;
1637 if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1638 return NULL;
1639 break;
1640 case DMA_PQ_VAL:
1641 case DMA_XOR_VAL:
1642 return NULL;
1643 default:
1644 break;
1645 }
1646 }
1647
1648 list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1649 if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1650 int rank;
1651
1652 rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1653 dst_cnt, src_lst, src_cnt, src_sz);
1654 if (rank > best_rank) {
1655 best_rank = rank;
1656 best_chan = ref->chan;
1657 }
1658 }
1659 }
1660
1661 return best_chan;
1662}
1663EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1664
1665/**
1666 * ppc440spe_get_group_entry - get group entry with index idx
1667 * @tdesc: is the last allocated slot in the group.
1668 */
1669static struct ppc440spe_adma_desc_slot *
1670ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1671{
1672 struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1673 int i = 0;
1674
1675 if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1676 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1677 __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1678 BUG();
1679 }
1680
1681 list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1682 if (i++ == entry_idx)
1683 break;
1684 }
1685 return iter;
1686}
1687
1688/**
1689 * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1690 * @slot: Slot to free
1691 * Caller must hold &ppc440spe_chan->lock while calling this function
1692 */
1693static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1694 struct ppc440spe_adma_chan *chan)
1695{
1696 int stride = slot->slots_per_op;
1697
1698 while (stride--) {
1699 slot->slots_per_op = 0;
1700 slot = list_entry(slot->slot_node.next,
1701 struct ppc440spe_adma_desc_slot,
1702 slot_node);
1703 }
1704}
1705
1706static void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan,
1707 struct ppc440spe_adma_desc_slot *desc)
1708{
1709 u32 src_cnt, dst_cnt;
1710 dma_addr_t addr;
1711
1712 /*
1713 * get the number of sources & destination
1714 * included in this descriptor and unmap
1715 * them all
1716 */
1717 src_cnt = ppc440spe_desc_get_src_num(desc, chan);
1718 dst_cnt = ppc440spe_desc_get_dst_num(desc, chan);
1719
1720 /* unmap destinations */
1721 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1722 while (dst_cnt--) {
1723 addr = ppc440spe_desc_get_dest_addr(
1724 desc, chan, dst_cnt);
1725 dma_unmap_page(chan->device->dev,
1726 addr, desc->unmap_len,
1727 DMA_FROM_DEVICE);
1728 }
1729 }
1730
1731 /* unmap sources */
1732 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1733 while (src_cnt--) {
1734 addr = ppc440spe_desc_get_src_addr(
1735 desc, chan, src_cnt);
1736 dma_unmap_page(chan->device->dev,
1737 addr, desc->unmap_len,
1738 DMA_TO_DEVICE);
1739 }
1740 }
1741}
1742
1743/**
1744 * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1745 * upon completion
1746 */
1747static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1748 struct ppc440spe_adma_desc_slot *desc,
1749 struct ppc440spe_adma_chan *chan,
1750 dma_cookie_t cookie)
1751{
1752 int i;
1753
1754 BUG_ON(desc->async_tx.cookie < 0);
1755 if (desc->async_tx.cookie > 0) {
1756 cookie = desc->async_tx.cookie;
1757 desc->async_tx.cookie = 0;
1758
1759 /* call the callback (must not sleep or submit new
1760 * operations to this channel)
1761 */
1762 if (desc->async_tx.callback)
1763 desc->async_tx.callback(
1764 desc->async_tx.callback_param);
1765
1766 /* unmap dma addresses
1767 * (unmap_single vs unmap_page?)
1768 *
1769 * actually, ppc's dma_unmap_page() functions are empty, so
1770 * the following code is just for the sake of completeness
1771 */
1772 if (chan && chan->needs_unmap && desc->group_head &&
1773 desc->unmap_len) {
1774 struct ppc440spe_adma_desc_slot *unmap =
1775 desc->group_head;
1776 /* assume 1 slot per op always */
1777 u32 slot_count = unmap->slot_cnt;
1778
1779 /* Run through the group list and unmap addresses */
1780 for (i = 0; i < slot_count; i++) {
1781 BUG_ON(!unmap);
1782 ppc440spe_adma_unmap(chan, unmap);
1783 unmap = unmap->hw_next;
1784 }
1785 }
1786 }
1787
1788 /* run dependent operations */
1789 dma_run_dependencies(&desc->async_tx);
1790
1791 return cookie;
1792}
1793
1794/**
1795 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1796 */
1797static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1798 struct ppc440spe_adma_chan *chan)
1799{
1800 /* the client is allowed to attach dependent operations
1801 * until 'ack' is set
1802 */
1803 if (!async_tx_test_ack(&desc->async_tx))
1804 return 0;
1805
1806 /* leave the last descriptor in the chain
1807 * so we can append to it
1808 */
1809 if (list_is_last(&desc->chain_node, &chan->chain) ||
1810 desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1811 return 1;
1812
1813 if (chan->device->id != PPC440SPE_XOR_ID) {
1814 /* our DMA interrupt handler clears opc field of
1815 * each processed descriptor. For all types of
1816 * operations except for ZeroSum we do not actually
1817 * need ack from the interrupt handler. ZeroSum is a
1818 * special case since the result of this operation
1819 * is available from the handler only, so if we see
1820 * such type of descriptor (which is unprocessed yet)
1821 * then leave it in chain.
1822 */
1823 struct dma_cdb *cdb = desc->hw_desc;
1824 if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1825 return 1;
1826 }
1827
1828 dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1829 desc->phys, desc->idx, desc->slots_per_op);
1830
1831 list_del(&desc->chain_node);
1832 ppc440spe_adma_free_slots(desc, chan);
1833 return 0;
1834}
1835
1836/**
1837 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1838 * which runs through the channel CDBs list until reach the descriptor
1839 * currently processed. When routine determines that all CDBs of group
1840 * are completed then corresponding callbacks (if any) are called and slots
1841 * are freed.
1842 */
1843static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1844{
1845 struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1846 dma_cookie_t cookie = 0;
1847 u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1848 int busy = ppc440spe_chan_is_busy(chan);
1849 int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1850
1851 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1852 chan->device->id, __func__);
1853
1854 if (!current_desc) {
1855 /* There were no transactions yet, so
1856 * nothing to clean
1857 */
1858 return;
1859 }
1860
1861 /* free completed slots from the chain starting with
1862 * the oldest descriptor
1863 */
1864 list_for_each_entry_safe(iter, _iter, &chan->chain,
1865 chain_node) {
1866 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1867 "busy: %d this_desc: %#llx next_desc: %#x "
1868 "cur: %#x ack: %d\n",
1869 iter->async_tx.cookie, iter->idx, busy, iter->phys,
1870 ppc440spe_desc_get_link(iter, chan), current_desc,
1871 async_tx_test_ack(&iter->async_tx));
1872 prefetch(_iter);
1873 prefetch(&_iter->async_tx);
1874
1875 /* do not advance past the current descriptor loaded into the
1876 * hardware channel,subsequent descriptors are either in process
1877 * or have not been submitted
1878 */
1879 if (seen_current)
1880 break;
1881
1882 /* stop the search if we reach the current descriptor and the
1883 * channel is busy, or if it appears that the current descriptor
1884 * needs to be re-read (i.e. has been appended to)
1885 */
1886 if (iter->phys == current_desc) {
1887 BUG_ON(seen_current++);
1888 if (busy || ppc440spe_desc_get_link(iter, chan)) {
1889 /* not all descriptors of the group have
1890 * been completed; exit.
1891 */
1892 break;
1893 }
1894 }
1895
1896 /* detect the start of a group transaction */
1897 if (!slot_cnt && !slots_per_op) {
1898 slot_cnt = iter->slot_cnt;
1899 slots_per_op = iter->slots_per_op;
1900 if (slot_cnt <= slots_per_op) {
1901 slot_cnt = 0;
1902 slots_per_op = 0;
1903 }
1904 }
1905
1906 if (slot_cnt) {
1907 if (!group_start)
1908 group_start = iter;
1909 slot_cnt -= slots_per_op;
1910 }
1911
1912 /* all the members of a group are complete */
1913 if (slots_per_op != 0 && slot_cnt == 0) {
1914 struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1915 int end_of_chain = 0;
1916
1917 /* clean up the group */
1918 slot_cnt = group_start->slot_cnt;
1919 grp_iter = group_start;
1920 list_for_each_entry_safe_from(grp_iter, _grp_iter,
1921 &chan->chain, chain_node) {
1922
1923 cookie = ppc440spe_adma_run_tx_complete_actions(
1924 grp_iter, chan, cookie);
1925
1926 slot_cnt -= slots_per_op;
1927 end_of_chain = ppc440spe_adma_clean_slot(
1928 grp_iter, chan);
1929 if (end_of_chain && slot_cnt) {
1930 /* Should wait for ZeroSum completion */
1931 if (cookie > 0)
1932 chan->completed_cookie = cookie;
1933 return;
1934 }
1935
1936 if (slot_cnt == 0 || end_of_chain)
1937 break;
1938 }
1939
1940 /* the group should be complete at this point */
1941 BUG_ON(slot_cnt);
1942
1943 slots_per_op = 0;
1944 group_start = NULL;
1945 if (end_of_chain)
1946 break;
1947 else
1948 continue;
1949 } else if (slots_per_op) /* wait for group completion */
1950 continue;
1951
1952 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1953 cookie);
1954
1955 if (ppc440spe_adma_clean_slot(iter, chan))
1956 break;
1957 }
1958
1959 BUG_ON(!seen_current);
1960
1961 if (cookie > 0) {
1962 chan->completed_cookie = cookie;
1963 pr_debug("\tcompleted cookie %d\n", cookie);
1964 }
1965
1966}
1967
1968/**
1969 * ppc440spe_adma_tasklet - clean up watch-dog initiator
1970 */
1971static void ppc440spe_adma_tasklet(unsigned long data)
1972{
1973 struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data;
1974
1975 spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1976 __ppc440spe_adma_slot_cleanup(chan);
1977 spin_unlock(&chan->lock);
1978}
1979
1980/**
1981 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1982 */
1983static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1984{
1985 spin_lock_bh(&chan->lock);
1986 __ppc440spe_adma_slot_cleanup(chan);
1987 spin_unlock_bh(&chan->lock);
1988}
1989
1990/**
1991 * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1992 */
1993static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1994 struct ppc440spe_adma_chan *chan, int num_slots,
1995 int slots_per_op)
1996{
1997 struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1998 struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1999 struct list_head chain = LIST_HEAD_INIT(chain);
2000 int slots_found, retry = 0;
2001
2002
2003 BUG_ON(!num_slots || !slots_per_op);
2004 /* start search from the last allocated descrtiptor
2005 * if a contiguous allocation can not be found start searching
2006 * from the beginning of the list
2007 */
2008retry:
2009 slots_found = 0;
2010 if (retry == 0)
2011 iter = chan->last_used;
2012 else
2013 iter = list_entry(&chan->all_slots,
2014 struct ppc440spe_adma_desc_slot,
2015 slot_node);
2016 list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
2017 slot_node) {
2018 prefetch(_iter);
2019 prefetch(&_iter->async_tx);
2020 if (iter->slots_per_op) {
2021 slots_found = 0;
2022 continue;
2023 }
2024
2025 /* start the allocation if the slot is correctly aligned */
2026 if (!slots_found++)
2027 alloc_start = iter;
2028
2029 if (slots_found == num_slots) {
2030 struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
2031 struct ppc440spe_adma_desc_slot *last_used = NULL;
2032
2033 iter = alloc_start;
2034 while (num_slots) {
2035 int i;
2036 /* pre-ack all but the last descriptor */
2037 if (num_slots != slots_per_op)
2038 async_tx_ack(&iter->async_tx);
2039
2040 list_add_tail(&iter->chain_node, &chain);
2041 alloc_tail = iter;
2042 iter->async_tx.cookie = 0;
2043 iter->hw_next = NULL;
2044 iter->flags = 0;
2045 iter->slot_cnt = num_slots;
2046 iter->xor_check_result = NULL;
2047 for (i = 0; i < slots_per_op; i++) {
2048 iter->slots_per_op = slots_per_op - i;
2049 last_used = iter;
2050 iter = list_entry(iter->slot_node.next,
2051 struct ppc440spe_adma_desc_slot,
2052 slot_node);
2053 }
2054 num_slots -= slots_per_op;
2055 }
2056 alloc_tail->group_head = alloc_start;
2057 alloc_tail->async_tx.cookie = -EBUSY;
2058 list_splice(&chain, &alloc_tail->group_list);
2059 chan->last_used = last_used;
2060 return alloc_tail;
2061 }
2062 }
2063 if (!retry++)
2064 goto retry;
2065
2066 /* try to free some slots if the allocation fails */
2067 tasklet_schedule(&chan->irq_tasklet);
2068 return NULL;
2069}
2070
2071/**
2072 * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots
2073 */
2074static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
2075{
2076 struct ppc440spe_adma_chan *ppc440spe_chan;
2077 struct ppc440spe_adma_desc_slot *slot = NULL;
2078 char *hw_desc;
2079 int i, db_sz;
2080 int init;
2081
2082 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2083 init = ppc440spe_chan->slots_allocated ? 0 : 1;
2084 chan->chan_id = ppc440spe_chan->device->id;
2085
2086 /* Allocate descriptor slots */
2087 i = ppc440spe_chan->slots_allocated;
2088 if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
2089 db_sz = sizeof(struct dma_cdb);
2090 else
2091 db_sz = sizeof(struct xor_cb);
2092
2093 for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
2094 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
2095 GFP_KERNEL);
2096 if (!slot) {
2097 printk(KERN_INFO "SPE ADMA Channel only initialized"
2098 " %d descriptor slots", i--);
2099 break;
2100 }
2101
2102 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
2103 slot->hw_desc = (void *) &hw_desc[i * db_sz];
2104 dma_async_tx_descriptor_init(&slot->async_tx, chan);
2105 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
2106 INIT_LIST_HEAD(&slot->chain_node);
2107 INIT_LIST_HEAD(&slot->slot_node);
2108 INIT_LIST_HEAD(&slot->group_list);
2109 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
2110 slot->idx = i;
2111
2112 spin_lock_bh(&ppc440spe_chan->lock);
2113 ppc440spe_chan->slots_allocated++;
2114 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
2115 spin_unlock_bh(&ppc440spe_chan->lock);
2116 }
2117
2118 if (i && !ppc440spe_chan->last_used) {
2119 ppc440spe_chan->last_used =
2120 list_entry(ppc440spe_chan->all_slots.next,
2121 struct ppc440spe_adma_desc_slot,
2122 slot_node);
2123 }
2124
2125 dev_dbg(ppc440spe_chan->device->common.dev,
2126 "ppc440spe adma%d: allocated %d descriptor slots\n",
2127 ppc440spe_chan->device->id, i);
2128
2129 /* initialize the channel and the chain with a null operation */
2130 if (init) {
2131 switch (ppc440spe_chan->device->id) {
2132 case PPC440SPE_DMA0_ID:
2133 case PPC440SPE_DMA1_ID:
2134 ppc440spe_chan->hw_chain_inited = 0;
2135 /* Use WXOR for self-testing */
2136 if (!ppc440spe_r6_tchan)
2137 ppc440spe_r6_tchan = ppc440spe_chan;
2138 break;
2139 case PPC440SPE_XOR_ID:
2140 ppc440spe_chan_start_null_xor(ppc440spe_chan);
2141 break;
2142 default:
2143 BUG();
2144 }
2145 ppc440spe_chan->needs_unmap = 1;
2146 }
2147
2148 return (i > 0) ? i : -ENOMEM;
2149}
2150
2151/**
2152 * ppc440spe_desc_assign_cookie - assign a cookie
2153 */
2154static dma_cookie_t ppc440spe_desc_assign_cookie(
2155 struct ppc440spe_adma_chan *chan,
2156 struct ppc440spe_adma_desc_slot *desc)
2157{
2158 dma_cookie_t cookie = chan->common.cookie;
2159
2160 cookie++;
2161 if (cookie < 0)
2162 cookie = 1;
2163 chan->common.cookie = desc->async_tx.cookie = cookie;
2164 return cookie;
2165}
2166
2167/**
2168 * ppc440spe_rxor_set_region_data -
2169 */
2170static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
2171 u8 xor_arg_no, u32 mask)
2172{
2173 struct xor_cb *xcb = desc->hw_desc;
2174
2175 xcb->ops[xor_arg_no].h |= mask;
2176}
2177
2178/**
2179 * ppc440spe_rxor_set_src -
2180 */
2181static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
2182 u8 xor_arg_no, dma_addr_t addr)
2183{
2184 struct xor_cb *xcb = desc->hw_desc;
2185
2186 xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
2187 xcb->ops[xor_arg_no].l = addr;
2188}
2189
2190/**
2191 * ppc440spe_rxor_set_mult -
2192 */
2193static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
2194 u8 xor_arg_no, u8 idx, u8 mult)
2195{
2196 struct xor_cb *xcb = desc->hw_desc;
2197
2198 xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
2199}
2200
2201/**
2202 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
2203 * has been achieved
2204 */
2205static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
2206{
2207 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
2208 chan->device->id, chan->pending);
2209
2210 if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
2211 chan->pending = 0;
2212 ppc440spe_chan_append(chan);
2213 }
2214}
2215
2216/**
2217 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
2218 * (it's not necessary that descriptors will be submitted to the h/w
2219 * chains too right now)
2220 */
2221static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
2222{
2223 struct ppc440spe_adma_desc_slot *sw_desc;
2224 struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
2225 struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
2226 int slot_cnt;
2227 int slots_per_op;
2228 dma_cookie_t cookie;
2229
2230 sw_desc = tx_to_ppc440spe_adma_slot(tx);
2231
2232 group_start = sw_desc->group_head;
2233 slot_cnt = group_start->slot_cnt;
2234 slots_per_op = group_start->slots_per_op;
2235
2236 spin_lock_bh(&chan->lock);
2237
2238 cookie = ppc440spe_desc_assign_cookie(chan, sw_desc);
2239
2240 if (unlikely(list_empty(&chan->chain))) {
2241 /* first peer */
2242 list_splice_init(&sw_desc->group_list, &chan->chain);
2243 chan_first_cdb[chan->device->id] = group_start;
2244 } else {
2245 /* isn't first peer, bind CDBs to chain */
2246 old_chain_tail = list_entry(chan->chain.prev,
2247 struct ppc440spe_adma_desc_slot,
2248 chain_node);
2249 list_splice_init(&sw_desc->group_list,
2250 &old_chain_tail->chain_node);
2251 /* fix up the hardware chain */
2252 ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
2253 }
2254
2255 /* increment the pending count by the number of operations */
2256 chan->pending += slot_cnt / slots_per_op;
2257 ppc440spe_adma_check_threshold(chan);
2258 spin_unlock_bh(&chan->lock);
2259
2260 dev_dbg(chan->device->common.dev,
2261 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
2262 chan->device->id, __func__,
2263 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
2264
2265 return cookie;
2266}
2267
2268/**
2269 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
2270 */
2271static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
2272 struct dma_chan *chan, unsigned long flags)
2273{
2274 struct ppc440spe_adma_chan *ppc440spe_chan;
2275 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2276 int slot_cnt, slots_per_op;
2277
2278 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2279
2280 dev_dbg(ppc440spe_chan->device->common.dev,
2281 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
2282 __func__);
2283
2284 spin_lock_bh(&ppc440spe_chan->lock);
2285 slot_cnt = slots_per_op = 1;
2286 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2287 slots_per_op);
2288 if (sw_desc) {
2289 group_start = sw_desc->group_head;
2290 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
2291 group_start->unmap_len = 0;
2292 sw_desc->async_tx.flags = flags;
2293 }
2294 spin_unlock_bh(&ppc440spe_chan->lock);
2295
2296 return sw_desc ? &sw_desc->async_tx : NULL;
2297}
2298
2299/**
2300 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
2301 */
2302static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
2303 struct dma_chan *chan, dma_addr_t dma_dest,
2304 dma_addr_t dma_src, size_t len, unsigned long flags)
2305{
2306 struct ppc440spe_adma_chan *ppc440spe_chan;
2307 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2308 int slot_cnt, slots_per_op;
2309
2310 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2311
2312 if (unlikely(!len))
2313 return NULL;
2314
2315 BUG_ON(unlikely(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT));
2316
2317 spin_lock_bh(&ppc440spe_chan->lock);
2318
2319 dev_dbg(ppc440spe_chan->device->common.dev,
2320 "ppc440spe adma%d: %s len: %u int_en %d\n",
2321 ppc440spe_chan->device->id, __func__, len,
2322 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2323 slot_cnt = slots_per_op = 1;
2324 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2325 slots_per_op);
2326 if (sw_desc) {
2327 group_start = sw_desc->group_head;
2328 ppc440spe_desc_init_memcpy(group_start, flags);
2329 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2330 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2331 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2332 sw_desc->unmap_len = len;
2333 sw_desc->async_tx.flags = flags;
2334 }
2335 spin_unlock_bh(&ppc440spe_chan->lock);
2336
2337 return sw_desc ? &sw_desc->async_tx : NULL;
2338}
2339
2340/**
2341 * ppc440spe_adma_prep_dma_memset - prepare CDB for a MEMSET operation
2342 */
2343static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memset(
2344 struct dma_chan *chan, dma_addr_t dma_dest, int value,
2345 size_t len, unsigned long flags)
2346{
2347 struct ppc440spe_adma_chan *ppc440spe_chan;
2348 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2349 int slot_cnt, slots_per_op;
2350
2351 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2352
2353 if (unlikely(!len))
2354 return NULL;
2355
2356 BUG_ON(unlikely(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT));
2357
2358 spin_lock_bh(&ppc440spe_chan->lock);
2359
2360 dev_dbg(ppc440spe_chan->device->common.dev,
2361 "ppc440spe adma%d: %s cal: %u len: %u int_en %d\n",
2362 ppc440spe_chan->device->id, __func__, value, len,
2363 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2364
2365 slot_cnt = slots_per_op = 1;
2366 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2367 slots_per_op);
2368 if (sw_desc) {
2369 group_start = sw_desc->group_head;
2370 ppc440spe_desc_init_memset(group_start, value, flags);
2371 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2372 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2373 sw_desc->unmap_len = len;
2374 sw_desc->async_tx.flags = flags;
2375 }
2376 spin_unlock_bh(&ppc440spe_chan->lock);
2377
2378 return sw_desc ? &sw_desc->async_tx : NULL;
2379}
2380
2381/**
2382 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2383 */
2384static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2385 struct dma_chan *chan, dma_addr_t dma_dest,
2386 dma_addr_t *dma_src, u32 src_cnt, size_t len,
2387 unsigned long flags)
2388{
2389 struct ppc440spe_adma_chan *ppc440spe_chan;
2390 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2391 int slot_cnt, slots_per_op;
2392
2393 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2394
2395 ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2396 dma_dest, dma_src, src_cnt));
2397 if (unlikely(!len))
2398 return NULL;
2399 BUG_ON(unlikely(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT));
2400
2401 dev_dbg(ppc440spe_chan->device->common.dev,
2402 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2403 ppc440spe_chan->device->id, __func__, src_cnt, len,
2404 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2405
2406 spin_lock_bh(&ppc440spe_chan->lock);
2407 slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2408 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2409 slots_per_op);
2410 if (sw_desc) {
2411 group_start = sw_desc->group_head;
2412 ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2413 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2414 while (src_cnt--)
2415 ppc440spe_adma_memcpy_xor_set_src(group_start,
2416 dma_src[src_cnt], src_cnt);
2417 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2418 sw_desc->unmap_len = len;
2419 sw_desc->async_tx.flags = flags;
2420 }
2421 spin_unlock_bh(&ppc440spe_chan->lock);
2422
2423 return sw_desc ? &sw_desc->async_tx : NULL;
2424}
2425
2426static inline void
2427ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2428 int src_cnt);
2429static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2430
2431/**
2432 * ppc440spe_adma_init_dma2rxor_slot -
2433 */
2434static void ppc440spe_adma_init_dma2rxor_slot(
2435 struct ppc440spe_adma_desc_slot *desc,
2436 dma_addr_t *src, int src_cnt)
2437{
2438 int i;
2439
2440 /* initialize CDB */
2441 for (i = 0; i < src_cnt; i++) {
2442 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2443 desc->src_cnt, (u32)src[i]);
2444 }
2445}
2446
2447/**
2448 * ppc440spe_dma01_prep_mult -
2449 * for Q operation where destination is also the source
2450 */
2451static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2452 struct ppc440spe_adma_chan *ppc440spe_chan,
2453 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2454 const unsigned char *scf, size_t len, unsigned long flags)
2455{
2456 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2457 unsigned long op = 0;
2458 int slot_cnt;
2459
2460 set_bit(PPC440SPE_DESC_WXOR, &op);
2461 slot_cnt = 2;
2462
2463 spin_lock_bh(&ppc440spe_chan->lock);
2464
2465 /* use WXOR, each descriptor occupies one slot */
2466 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2467 if (sw_desc) {
2468 struct ppc440spe_adma_chan *chan;
2469 struct ppc440spe_adma_desc_slot *iter;
2470 struct dma_cdb *hw_desc;
2471
2472 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2473 set_bits(op, &sw_desc->flags);
2474 sw_desc->src_cnt = src_cnt;
2475 sw_desc->dst_cnt = dst_cnt;
2476 /* First descriptor, zero data in the destination and copy it
2477 * to q page using MULTICAST transfer.
2478 */
2479 iter = list_first_entry(&sw_desc->group_list,
2480 struct ppc440spe_adma_desc_slot,
2481 chain_node);
2482 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2483 /* set 'next' pointer */
2484 iter->hw_next = list_entry(iter->chain_node.next,
2485 struct ppc440spe_adma_desc_slot,
2486 chain_node);
2487 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2488 hw_desc = iter->hw_desc;
2489 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2490
2491 ppc440spe_desc_set_dest_addr(iter, chan,
2492 DMA_CUED_XOR_BASE, dst[0], 0);
2493 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2494 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2495 src[0]);
2496 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2497 iter->unmap_len = len;
2498
2499 /*
2500 * Second descriptor, multiply data from the q page
2501 * and store the result in real destination.
2502 */
2503 iter = list_first_entry(&iter->chain_node,
2504 struct ppc440spe_adma_desc_slot,
2505 chain_node);
2506 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2507 iter->hw_next = NULL;
2508 if (flags & DMA_PREP_INTERRUPT)
2509 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2510 else
2511 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2512
2513 hw_desc = iter->hw_desc;
2514 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2515 ppc440spe_desc_set_src_addr(iter, chan, 0,
2516 DMA_CUED_XOR_HB, dst[1]);
2517 ppc440spe_desc_set_dest_addr(iter, chan,
2518 DMA_CUED_XOR_BASE, dst[0], 0);
2519
2520 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2521 DMA_CDB_SG_DST1, scf[0]);
2522 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2523 iter->unmap_len = len;
2524 sw_desc->async_tx.flags = flags;
2525 }
2526
2527 spin_unlock_bh(&ppc440spe_chan->lock);
2528
2529 return sw_desc;
2530}
2531
2532/**
2533 * ppc440spe_dma01_prep_sum_product -
2534 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2535 * the source.
2536 */
2537static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2538 struct ppc440spe_adma_chan *ppc440spe_chan,
2539 dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2540 const unsigned char *scf, size_t len, unsigned long flags)
2541{
2542 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2543 unsigned long op = 0;
2544 int slot_cnt;
2545
2546 set_bit(PPC440SPE_DESC_WXOR, &op);
2547 slot_cnt = 3;
2548
2549 spin_lock_bh(&ppc440spe_chan->lock);
2550
2551 /* WXOR, each descriptor occupies one slot */
2552 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2553 if (sw_desc) {
2554 struct ppc440spe_adma_chan *chan;
2555 struct ppc440spe_adma_desc_slot *iter;
2556 struct dma_cdb *hw_desc;
2557
2558 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2559 set_bits(op, &sw_desc->flags);
2560 sw_desc->src_cnt = src_cnt;
2561 sw_desc->dst_cnt = 1;
2562 /* 1st descriptor, src[1] data to q page and zero destination */
2563 iter = list_first_entry(&sw_desc->group_list,
2564 struct ppc440spe_adma_desc_slot,
2565 chain_node);
2566 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2567 iter->hw_next = list_entry(iter->chain_node.next,
2568 struct ppc440spe_adma_desc_slot,
2569 chain_node);
2570 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2571 hw_desc = iter->hw_desc;
2572 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2573
2574 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2575 *dst, 0);
2576 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2577 ppc440spe_chan->qdest, 1);
2578 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2579 src[1]);
2580 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2581 iter->unmap_len = len;
2582
2583 /* 2nd descriptor, multiply src[1] data and store the
2584 * result in destination */
2585 iter = list_first_entry(&iter->chain_node,
2586 struct ppc440spe_adma_desc_slot,
2587 chain_node);
2588 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2589 /* set 'next' pointer */
2590 iter->hw_next = list_entry(iter->chain_node.next,
2591 struct ppc440spe_adma_desc_slot,
2592 chain_node);
2593 if (flags & DMA_PREP_INTERRUPT)
2594 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2595 else
2596 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2597
2598 hw_desc = iter->hw_desc;
2599 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2600 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2601 ppc440spe_chan->qdest);
2602 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2603 *dst, 0);
2604 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2605 DMA_CDB_SG_DST1, scf[1]);
2606 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2607 iter->unmap_len = len;
2608
2609 /*
2610 * 3rd descriptor, multiply src[0] data and xor it
2611 * with destination
2612 */
2613 iter = list_first_entry(&iter->chain_node,
2614 struct ppc440spe_adma_desc_slot,
2615 chain_node);
2616 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2617 iter->hw_next = NULL;
2618 if (flags & DMA_PREP_INTERRUPT)
2619 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2620 else
2621 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2622
2623 hw_desc = iter->hw_desc;
2624 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2625 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2626 src[0]);
2627 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2628 *dst, 0);
2629 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2630 DMA_CDB_SG_DST1, scf[0]);
2631 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2632 iter->unmap_len = len;
2633 sw_desc->async_tx.flags = flags;
2634 }
2635
2636 spin_unlock_bh(&ppc440spe_chan->lock);
2637
2638 return sw_desc;
2639}
2640
2641static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2642 struct ppc440spe_adma_chan *ppc440spe_chan,
2643 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2644 const unsigned char *scf, size_t len, unsigned long flags)
2645{
2646 int slot_cnt;
2647 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2648 unsigned long op = 0;
2649 unsigned char mult = 1;
2650
2651 pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2652 __func__, dst_cnt, src_cnt, len);
2653 /* select operations WXOR/RXOR depending on the
2654 * source addresses of operators and the number
2655 * of destinations (RXOR support only Q-parity calculations)
2656 */
2657 set_bit(PPC440SPE_DESC_WXOR, &op);
2658 if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2659 /* no active RXOR;
2660 * do RXOR if:
2661 * - there are more than 1 source,
2662 * - len is aligned on 512-byte boundary,
2663 * - source addresses fit to one of 4 possible regions.
2664 */
2665 if (src_cnt > 1 &&
2666 !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2667 (src[0] + len) == src[1]) {
2668 /* may do RXOR R1 R2 */
2669 set_bit(PPC440SPE_DESC_RXOR, &op);
2670 if (src_cnt != 2) {
2671 /* may try to enhance region of RXOR */
2672 if ((src[1] + len) == src[2]) {
2673 /* do RXOR R1 R2 R3 */
2674 set_bit(PPC440SPE_DESC_RXOR123,
2675 &op);
2676 } else if ((src[1] + len * 2) == src[2]) {
2677 /* do RXOR R1 R2 R4 */
2678 set_bit(PPC440SPE_DESC_RXOR124, &op);
2679 } else if ((src[1] + len * 3) == src[2]) {
2680 /* do RXOR R1 R2 R5 */
2681 set_bit(PPC440SPE_DESC_RXOR125,
2682 &op);
2683 } else {
2684 /* do RXOR R1 R2 */
2685 set_bit(PPC440SPE_DESC_RXOR12,
2686 &op);
2687 }
2688 } else {
2689 /* do RXOR R1 R2 */
2690 set_bit(PPC440SPE_DESC_RXOR12, &op);
2691 }
2692 }
2693
2694 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2695 /* can not do this operation with RXOR */
2696 clear_bit(PPC440SPE_RXOR_RUN,
2697 &ppc440spe_rxor_state);
2698 } else {
2699 /* can do; set block size right now */
2700 ppc440spe_desc_set_rxor_block_size(len);
2701 }
2702 }
2703
2704 /* Number of necessary slots depends on operation type selected */
2705 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2706 /* This is a WXOR only chain. Need descriptors for each
2707 * source to GF-XOR them with WXOR, and need descriptors
2708 * for each destination to zero them with WXOR
2709 */
2710 slot_cnt = src_cnt;
2711
2712 if (flags & DMA_PREP_ZERO_P) {
2713 slot_cnt++;
2714 set_bit(PPC440SPE_ZERO_P, &op);
2715 }
2716 if (flags & DMA_PREP_ZERO_Q) {
2717 slot_cnt++;
2718 set_bit(PPC440SPE_ZERO_Q, &op);
2719 }
2720 } else {
2721 /* Need 1/2 descriptor for RXOR operation, and
2722 * need (src_cnt - (2 or 3)) for WXOR of sources
2723 * remained (if any)
2724 */
2725 slot_cnt = dst_cnt;
2726
2727 if (flags & DMA_PREP_ZERO_P)
2728 set_bit(PPC440SPE_ZERO_P, &op);
2729 if (flags & DMA_PREP_ZERO_Q)
2730 set_bit(PPC440SPE_ZERO_Q, &op);
2731
2732 if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2733 slot_cnt += src_cnt - 2;
2734 else
2735 slot_cnt += src_cnt - 3;
2736
2737 /* Thus we have either RXOR only chain or
2738 * mixed RXOR/WXOR
2739 */
2740 if (slot_cnt == dst_cnt)
2741 /* RXOR only chain */
2742 clear_bit(PPC440SPE_DESC_WXOR, &op);
2743 }
2744
2745 spin_lock_bh(&ppc440spe_chan->lock);
2746 /* for both RXOR/WXOR each descriptor occupies one slot */
2747 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2748 if (sw_desc) {
2749 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2750 flags, op);
2751
2752 /* setup dst/src/mult */
2753 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2754 __func__, dst[0], dst[1]);
2755 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2756 while (src_cnt--) {
2757 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2758 src_cnt);
2759
2760 /* NOTE: "Multi = 0 is equivalent to = 1" as it
2761 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2762 * doesn't work for RXOR with DMA0/1! Instead, multi=0
2763 * leads to zeroing source data after RXOR.
2764 * So, for P case set-up mult=1 explicitly.
2765 */
2766 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2767 mult = scf[src_cnt];
2768 ppc440spe_adma_pq_set_src_mult(sw_desc,
2769 mult, src_cnt, dst_cnt - 1);
2770 }
2771
2772 /* Setup byte count foreach slot just allocated */
2773 sw_desc->async_tx.flags = flags;
2774 list_for_each_entry(iter, &sw_desc->group_list,
2775 chain_node) {
2776 ppc440spe_desc_set_byte_count(iter,
2777 ppc440spe_chan, len);
2778 iter->unmap_len = len;
2779 }
2780 }
2781 spin_unlock_bh(&ppc440spe_chan->lock);
2782
2783 return sw_desc;
2784}
2785
2786static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2787 struct ppc440spe_adma_chan *ppc440spe_chan,
2788 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2789 const unsigned char *scf, size_t len, unsigned long flags)
2790{
2791 int slot_cnt, descs_per_op;
2792 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2793 unsigned long op = 0;
2794 unsigned char mult = 1;
2795
2796 BUG_ON(!dst_cnt);
2797 /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2798 __func__, dst_cnt, src_cnt, len);*/
2799
2800 spin_lock_bh(&ppc440spe_chan->lock);
2801 descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2802 if (descs_per_op < 0) {
2803 spin_unlock_bh(&ppc440spe_chan->lock);
2804 return NULL;
2805 }
2806
2807 /* depending on number of sources we have 1 or 2 RXOR chains */
2808 slot_cnt = descs_per_op * dst_cnt;
2809
2810 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2811 if (sw_desc) {
2812 op = slot_cnt;
2813 sw_desc->async_tx.flags = flags;
2814 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2815 ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2816 --op ? 0 : flags);
2817 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2818 len);
2819 iter->unmap_len = len;
2820
2821 ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2822 iter->rxor_cursor.len = len;
2823 iter->descs_per_op = descs_per_op;
2824 }
2825 op = 0;
2826 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2827 op++;
2828 if (op % descs_per_op == 0)
2829 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2830 src_cnt);
2831 if (likely(!list_is_last(&iter->chain_node,
2832 &sw_desc->group_list))) {
2833 /* set 'next' pointer */
2834 iter->hw_next =
2835 list_entry(iter->chain_node.next,
2836 struct ppc440spe_adma_desc_slot,
2837 chain_node);
2838 ppc440spe_xor_set_link(iter, iter->hw_next);
2839 } else {
2840 /* this is the last descriptor. */
2841 iter->hw_next = NULL;
2842 }
2843 }
2844
2845 /* fixup head descriptor */
2846 sw_desc->dst_cnt = dst_cnt;
2847 if (flags & DMA_PREP_ZERO_P)
2848 set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2849 if (flags & DMA_PREP_ZERO_Q)
2850 set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2851
2852 /* setup dst/src/mult */
2853 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2854
2855 while (src_cnt--) {
2856 /* handle descriptors (if dst_cnt == 2) inside
2857 * the ppc440spe_adma_pq_set_srcxxx() functions
2858 */
2859 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2860 src_cnt);
2861 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2862 mult = scf[src_cnt];
2863 ppc440spe_adma_pq_set_src_mult(sw_desc,
2864 mult, src_cnt, dst_cnt - 1);
2865 }
2866 }
2867 spin_unlock_bh(&ppc440spe_chan->lock);
2868 ppc440spe_desc_set_rxor_block_size(len);
2869 return sw_desc;
2870}
2871
2872/**
2873 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2874 */
2875static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2876 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2877 unsigned int src_cnt, const unsigned char *scf,
2878 size_t len, unsigned long flags)
2879{
2880 struct ppc440spe_adma_chan *ppc440spe_chan;
2881 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2882 int dst_cnt = 0;
2883
2884 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2885
2886 ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2887 dst, src, src_cnt));
2888 BUG_ON(!len);
2889 BUG_ON(unlikely(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT));
2890 BUG_ON(!src_cnt);
2891
2892 if (src_cnt == 1 && dst[1] == src[0]) {
2893 dma_addr_t dest[2];
2894
2895 /* dst[1] is real destination (Q) */
2896 dest[0] = dst[1];
2897 /* this is the page to multicast source data to */
2898 dest[1] = ppc440spe_chan->qdest;
2899 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2900 dest, 2, src, src_cnt, scf, len, flags);
2901 return sw_desc ? &sw_desc->async_tx : NULL;
2902 }
2903
2904 if (src_cnt == 2 && dst[1] == src[1]) {
2905 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2906 &dst[1], src, 2, scf, len, flags);
2907 return sw_desc ? &sw_desc->async_tx : NULL;
2908 }
2909
2910 if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2911 BUG_ON(!dst[0]);
2912 dst_cnt++;
2913 flags |= DMA_PREP_ZERO_P;
2914 }
2915
2916 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2917 BUG_ON(!dst[1]);
2918 dst_cnt++;
2919 flags |= DMA_PREP_ZERO_Q;
2920 }
2921
2922 BUG_ON(!dst_cnt);
2923
2924 dev_dbg(ppc440spe_chan->device->common.dev,
2925 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2926 ppc440spe_chan->device->id, __func__, src_cnt, len,
2927 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2928
2929 switch (ppc440spe_chan->device->id) {
2930 case PPC440SPE_DMA0_ID:
2931 case PPC440SPE_DMA1_ID:
2932 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2933 dst, dst_cnt, src, src_cnt, scf,
2934 len, flags);
2935 break;
2936
2937 case PPC440SPE_XOR_ID:
2938 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2939 dst, dst_cnt, src, src_cnt, scf,
2940 len, flags);
2941 break;
2942 }
2943
2944 return sw_desc ? &sw_desc->async_tx : NULL;
2945}
2946
2947/**
2948 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2949 * a PQ_ZERO_SUM operation
2950 */
2951static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2952 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2953 unsigned int src_cnt, const unsigned char *scf, size_t len,
2954 enum sum_check_flags *pqres, unsigned long flags)
2955{
2956 struct ppc440spe_adma_chan *ppc440spe_chan;
2957 struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2958 dma_addr_t pdest, qdest;
2959 int slot_cnt, slots_per_op, idst, dst_cnt;
2960
2961 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2962
2963 if (flags & DMA_PREP_PQ_DISABLE_P)
2964 pdest = 0;
2965 else
2966 pdest = pq[0];
2967
2968 if (flags & DMA_PREP_PQ_DISABLE_Q)
2969 qdest = 0;
2970 else
2971 qdest = pq[1];
2972
2973 ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2974 src, src_cnt, scf));
2975
2976 /* Always use WXOR for P/Q calculations (two destinations).
2977 * Need 1 or 2 extra slots to verify results are zero.
2978 */
2979 idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2980
2981 /* One additional slot per destination to clone P/Q
2982 * before calculation (we have to preserve destinations).
2983 */
2984 slot_cnt = src_cnt + dst_cnt * 2;
2985 slots_per_op = 1;
2986
2987 spin_lock_bh(&ppc440spe_chan->lock);
2988 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2989 slots_per_op);
2990 if (sw_desc) {
2991 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2992
2993 /* Setup byte count for each slot just allocated */
2994 sw_desc->async_tx.flags = flags;
2995 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2996 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2997 len);
2998 iter->unmap_len = len;
2999 }
3000
3001 if (pdest) {
3002 struct dma_cdb *hw_desc;
3003 struct ppc440spe_adma_chan *chan;
3004
3005 iter = sw_desc->group_head;
3006 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3007 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
3008 iter->hw_next = list_entry(iter->chain_node.next,
3009 struct ppc440spe_adma_desc_slot,
3010 chain_node);
3011 hw_desc = iter->hw_desc;
3012 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
3013 iter->src_cnt = 0;
3014 iter->dst_cnt = 0;
3015 ppc440spe_desc_set_dest_addr(iter, chan, 0,
3016 ppc440spe_chan->pdest, 0);
3017 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
3018 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
3019 len);
3020 iter->unmap_len = 0;
3021 /* override pdest to preserve original P */
3022 pdest = ppc440spe_chan->pdest;
3023 }
3024 if (qdest) {
3025 struct dma_cdb *hw_desc;
3026 struct ppc440spe_adma_chan *chan;
3027
3028 iter = list_first_entry(&sw_desc->group_list,
3029 struct ppc440spe_adma_desc_slot,
3030 chain_node);
3031 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3032
3033 if (pdest) {
3034 iter = list_entry(iter->chain_node.next,
3035 struct ppc440spe_adma_desc_slot,
3036 chain_node);
3037 }
3038
3039 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
3040 iter->hw_next = list_entry(iter->chain_node.next,
3041 struct ppc440spe_adma_desc_slot,
3042 chain_node);
3043 hw_desc = iter->hw_desc;
3044 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
3045 iter->src_cnt = 0;
3046 iter->dst_cnt = 0;
3047 ppc440spe_desc_set_dest_addr(iter, chan, 0,
3048 ppc440spe_chan->qdest, 0);
3049 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
3050 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
3051 len);
3052 iter->unmap_len = 0;
3053 /* override qdest to preserve original Q */
3054 qdest = ppc440spe_chan->qdest;
3055 }
3056
3057 /* Setup destinations for P/Q ops */
3058 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
3059
3060 /* Setup zero QWORDs into DCHECK CDBs */
3061 idst = dst_cnt;
3062 list_for_each_entry_reverse(iter, &sw_desc->group_list,
3063 chain_node) {
3064 /*
3065 * The last CDB corresponds to Q-parity check,
3066 * the one before last CDB corresponds
3067 * P-parity check
3068 */
3069 if (idst == DMA_DEST_MAX_NUM) {
3070 if (idst == dst_cnt) {
3071 set_bit(PPC440SPE_DESC_QCHECK,
3072 &iter->flags);
3073 } else {
3074 set_bit(PPC440SPE_DESC_PCHECK,
3075 &iter->flags);
3076 }
3077 } else {
3078 if (qdest) {
3079 set_bit(PPC440SPE_DESC_QCHECK,
3080 &iter->flags);
3081 } else {
3082 set_bit(PPC440SPE_DESC_PCHECK,
3083 &iter->flags);
3084 }
3085 }
3086 iter->xor_check_result = pqres;
3087
3088 /*
3089 * set it to zero, if check fail then result will
3090 * be updated
3091 */
3092 *iter->xor_check_result = 0;
3093 ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
3094 ppc440spe_qword);
3095
3096 if (!(--dst_cnt))
3097 break;
3098 }
3099
3100 /* Setup sources and mults for P/Q ops */
3101 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
3102 chain_node) {
3103 struct ppc440spe_adma_chan *chan;
3104 u32 mult_dst;
3105
3106 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3107 ppc440spe_desc_set_src_addr(iter, chan, 0,
3108 DMA_CUED_XOR_HB,
3109 src[src_cnt - 1]);
3110 if (qdest) {
3111 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
3112 DMA_CDB_SG_DST1;
3113 ppc440spe_desc_set_src_mult(iter, chan,
3114 DMA_CUED_MULT1_OFF,
3115 mult_dst,
3116 scf[src_cnt - 1]);
3117 }
3118 if (!(--src_cnt))
3119 break;
3120 }
3121 }
3122 spin_unlock_bh(&ppc440spe_chan->lock);
3123 return sw_desc ? &sw_desc->async_tx : NULL;
3124}
3125
3126/**
3127 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
3128 * XOR ZERO_SUM operation
3129 */
3130static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
3131 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
3132 size_t len, enum sum_check_flags *result, unsigned long flags)
3133{
3134 struct dma_async_tx_descriptor *tx;
3135 dma_addr_t pq[2];
3136
3137 /* validate P, disable Q */
3138 pq[0] = src[0];
3139 pq[1] = 0;
3140 flags |= DMA_PREP_PQ_DISABLE_Q;
3141
3142 tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
3143 src_cnt - 1, 0, len,
3144 result, flags);
3145 return tx;
3146}
3147
3148/**
3149 * ppc440spe_adma_set_dest - set destination address into descriptor
3150 */
3151static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
3152 dma_addr_t addr, int index)
3153{
3154 struct ppc440spe_adma_chan *chan;
3155
3156 BUG_ON(index >= sw_desc->dst_cnt);
3157
3158 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3159
3160 switch (chan->device->id) {
3161 case PPC440SPE_DMA0_ID:
3162 case PPC440SPE_DMA1_ID:
3163 /* to do: support transfers lengths >
3164 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
3165 */
3166 ppc440spe_desc_set_dest_addr(sw_desc->group_head,
3167 chan, 0, addr, index);
3168 break;
3169 case PPC440SPE_XOR_ID:
3170 sw_desc = ppc440spe_get_group_entry(sw_desc, index);
3171 ppc440spe_desc_set_dest_addr(sw_desc,
3172 chan, 0, addr, index);
3173 break;
3174 }
3175}
3176
3177static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
3178 struct ppc440spe_adma_chan *chan, dma_addr_t addr)
3179{
3180 /* To clear destinations update the descriptor
3181 * (P or Q depending on index) as follows:
3182 * addr is destination (0 corresponds to SG2):
3183 */
3184 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
3185
3186 /* ... and the addr is source: */
3187 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
3188
3189 /* addr is always SG2 then the mult is always DST1 */
3190 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
3191 DMA_CDB_SG_DST1, 1);
3192}
3193
3194/**
3195 * ppc440spe_adma_pq_set_dest - set destination address into descriptor
3196 * for the PQXOR operation
3197 */
3198static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
3199 dma_addr_t *addrs, unsigned long flags)
3200{
3201 struct ppc440spe_adma_desc_slot *iter;
3202 struct ppc440spe_adma_chan *chan;
3203 dma_addr_t paddr, qaddr;
3204 dma_addr_t addr = 0, ppath, qpath;
3205 int index = 0, i;
3206
3207 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3208
3209 if (flags & DMA_PREP_PQ_DISABLE_P)
3210 paddr = 0;
3211 else
3212 paddr = addrs[0];
3213
3214 if (flags & DMA_PREP_PQ_DISABLE_Q)
3215 qaddr = 0;
3216 else
3217 qaddr = addrs[1];
3218
3219 if (!paddr || !qaddr)
3220 addr = paddr ? paddr : qaddr;
3221
3222 switch (chan->device->id) {
3223 case PPC440SPE_DMA0_ID:
3224 case PPC440SPE_DMA1_ID:
3225 /* walk through the WXOR source list and set P/Q-destinations
3226 * for each slot:
3227 */
3228 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3229 /* This is WXOR-only chain; may have 1/2 zero descs */
3230 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3231 index++;
3232 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3233 index++;
3234
3235 iter = ppc440spe_get_group_entry(sw_desc, index);
3236 if (addr) {
3237 /* one destination */
3238 list_for_each_entry_from(iter,
3239 &sw_desc->group_list, chain_node)
3240 ppc440spe_desc_set_dest_addr(iter, chan,
3241 DMA_CUED_XOR_BASE, addr, 0);
3242 } else {
3243 /* two destinations */
3244 list_for_each_entry_from(iter,
3245 &sw_desc->group_list, chain_node) {
3246 ppc440spe_desc_set_dest_addr(iter, chan,
3247 DMA_CUED_XOR_BASE, paddr, 0);
3248 ppc440spe_desc_set_dest_addr(iter, chan,
3249 DMA_CUED_XOR_BASE, qaddr, 1);
3250 }
3251 }
3252
3253 if (index) {
3254 /* To clear destinations update the descriptor
3255 * (1st,2nd, or both depending on flags)
3256 */
3257 index = 0;
3258 if (test_bit(PPC440SPE_ZERO_P,
3259 &sw_desc->flags)) {
3260 iter = ppc440spe_get_group_entry(
3261 sw_desc, index++);
3262 ppc440spe_adma_pq_zero_op(iter, chan,
3263 paddr);
3264 }
3265
3266 if (test_bit(PPC440SPE_ZERO_Q,
3267 &sw_desc->flags)) {
3268 iter = ppc440spe_get_group_entry(
3269 sw_desc, index++);
3270 ppc440spe_adma_pq_zero_op(iter, chan,
3271 qaddr);
3272 }
3273
3274 return;
3275 }
3276 } else {
3277 /* This is RXOR-only or RXOR/WXOR mixed chain */
3278
3279 /* If we want to include destination into calculations,
3280 * then make dest addresses cued with mult=1 (XOR).
3281 */
3282 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3283 DMA_CUED_XOR_HB :
3284 DMA_CUED_XOR_BASE |
3285 (1 << DMA_CUED_MULT1_OFF);
3286 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3287 DMA_CUED_XOR_HB :
3288 DMA_CUED_XOR_BASE |
3289 (1 << DMA_CUED_MULT1_OFF);
3290
3291 /* Setup destination(s) in RXOR slot(s) */
3292 iter = ppc440spe_get_group_entry(sw_desc, index++);
3293 ppc440spe_desc_set_dest_addr(iter, chan,
3294 paddr ? ppath : qpath,
3295 paddr ? paddr : qaddr, 0);
3296 if (!addr) {
3297 /* two destinations */
3298 iter = ppc440spe_get_group_entry(sw_desc,
3299 index++);
3300 ppc440spe_desc_set_dest_addr(iter, chan,
3301 qpath, qaddr, 0);
3302 }
3303
3304 if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
3305 /* Setup destination(s) in remaining WXOR
3306 * slots
3307 */
3308 iter = ppc440spe_get_group_entry(sw_desc,
3309 index);
3310 if (addr) {
3311 /* one destination */
3312 list_for_each_entry_from(iter,
3313 &sw_desc->group_list,
3314 chain_node)
3315 ppc440spe_desc_set_dest_addr(
3316 iter, chan,
3317 DMA_CUED_XOR_BASE,
3318 addr, 0);
3319
3320 } else {
3321 /* two destinations */
3322 list_for_each_entry_from(iter,
3323 &sw_desc->group_list,
3324 chain_node) {
3325 ppc440spe_desc_set_dest_addr(
3326 iter, chan,
3327 DMA_CUED_XOR_BASE,
3328 paddr, 0);
3329 ppc440spe_desc_set_dest_addr(
3330 iter, chan,
3331 DMA_CUED_XOR_BASE,
3332 qaddr, 1);
3333 }
3334 }
3335 }
3336
3337 }
3338 break;
3339
3340 case PPC440SPE_XOR_ID:
3341 /* DMA2 descriptors have only 1 destination, so there are
3342 * two chains - one for each dest.
3343 * If we want to include destination into calculations,
3344 * then make dest addresses cued with mult=1 (XOR).
3345 */
3346 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3347 DMA_CUED_XOR_HB :
3348 DMA_CUED_XOR_BASE |
3349 (1 << DMA_CUED_MULT1_OFF);
3350
3351 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3352 DMA_CUED_XOR_HB :
3353 DMA_CUED_XOR_BASE |
3354 (1 << DMA_CUED_MULT1_OFF);
3355
3356 iter = ppc440spe_get_group_entry(sw_desc, 0);
3357 for (i = 0; i < sw_desc->descs_per_op; i++) {
3358 ppc440spe_desc_set_dest_addr(iter, chan,
3359 paddr ? ppath : qpath,
3360 paddr ? paddr : qaddr, 0);
3361 iter = list_entry(iter->chain_node.next,
3362 struct ppc440spe_adma_desc_slot,
3363 chain_node);
3364 }
3365
3366 if (!addr) {
3367 /* Two destinations; setup Q here */
3368 iter = ppc440spe_get_group_entry(sw_desc,
3369 sw_desc->descs_per_op);
3370 for (i = 0; i < sw_desc->descs_per_op; i++) {
3371 ppc440spe_desc_set_dest_addr(iter,
3372 chan, qpath, qaddr, 0);
3373 iter = list_entry(iter->chain_node.next,
3374 struct ppc440spe_adma_desc_slot,
3375 chain_node);
3376 }
3377 }
3378
3379 break;
3380 }
3381}
3382
3383/**
3384 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3385 * for the PQ_ZERO_SUM operation
3386 */
3387static void ppc440spe_adma_pqzero_sum_set_dest(
3388 struct ppc440spe_adma_desc_slot *sw_desc,
3389 dma_addr_t paddr, dma_addr_t qaddr)
3390{
3391 struct ppc440spe_adma_desc_slot *iter, *end;
3392 struct ppc440spe_adma_chan *chan;
3393 dma_addr_t addr = 0;
3394 int idx;
3395
3396 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3397
3398 /* walk through the WXOR source list and set P/Q-destinations
3399 * for each slot
3400 */
3401 idx = (paddr && qaddr) ? 2 : 1;
3402 /* set end */
3403 list_for_each_entry_reverse(end, &sw_desc->group_list,
3404 chain_node) {
3405 if (!(--idx))
3406 break;
3407 }
3408 /* set start */
3409 idx = (paddr && qaddr) ? 2 : 1;
3410 iter = ppc440spe_get_group_entry(sw_desc, idx);
3411
3412 if (paddr && qaddr) {
3413 /* two destinations */
3414 list_for_each_entry_from(iter, &sw_desc->group_list,
3415 chain_node) {
3416 if (unlikely(iter == end))
3417 break;
3418 ppc440spe_desc_set_dest_addr(iter, chan,
3419 DMA_CUED_XOR_BASE, paddr, 0);
3420 ppc440spe_desc_set_dest_addr(iter, chan,
3421 DMA_CUED_XOR_BASE, qaddr, 1);
3422 }
3423 } else {
3424 /* one destination */
3425 addr = paddr ? paddr : qaddr;
3426 list_for_each_entry_from(iter, &sw_desc->group_list,
3427 chain_node) {
3428 if (unlikely(iter == end))
3429 break;
3430 ppc440spe_desc_set_dest_addr(iter, chan,
3431 DMA_CUED_XOR_BASE, addr, 0);
3432 }
3433 }
3434
3435 /* The remaining descriptors are DATACHECK. These have no need in
3436 * destination. Actually, these destinations are used there
3437 * as sources for check operation. So, set addr as source.
3438 */
3439 ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3440
3441 if (!addr) {
3442 end = list_entry(end->chain_node.next,
3443 struct ppc440spe_adma_desc_slot, chain_node);
3444 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3445 }
3446}
3447
3448/**
3449 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3450 */
3451static inline void ppc440spe_desc_set_xor_src_cnt(
3452 struct ppc440spe_adma_desc_slot *desc,
3453 int src_cnt)
3454{
3455 struct xor_cb *hw_desc = desc->hw_desc;
3456
3457 hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3458 hw_desc->cbc |= src_cnt;
3459}
3460
3461/**
3462 * ppc440spe_adma_pq_set_src - set source address into descriptor
3463 */
3464static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3465 dma_addr_t addr, int index)
3466{
3467 struct ppc440spe_adma_chan *chan;
3468 dma_addr_t haddr = 0;
3469 struct ppc440spe_adma_desc_slot *iter = NULL;
3470
3471 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3472
3473 switch (chan->device->id) {
3474 case PPC440SPE_DMA0_ID:
3475 case PPC440SPE_DMA1_ID:
3476 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3477 */
3478 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3479 /* RXOR-only or RXOR/WXOR operation */
3480 int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3481 &sw_desc->flags) ? 2 : 3;
3482
3483 if (index == 0) {
3484 /* 1st slot (RXOR) */
3485 /* setup sources region (R1-2-3, R1-2-4,
3486 * or R1-2-5)
3487 */
3488 if (test_bit(PPC440SPE_DESC_RXOR12,
3489 &sw_desc->flags))
3490 haddr = DMA_RXOR12 <<
3491 DMA_CUED_REGION_OFF;
3492 else if (test_bit(PPC440SPE_DESC_RXOR123,
3493 &sw_desc->flags))
3494 haddr = DMA_RXOR123 <<
3495 DMA_CUED_REGION_OFF;
3496 else if (test_bit(PPC440SPE_DESC_RXOR124,
3497 &sw_desc->flags))
3498 haddr = DMA_RXOR124 <<
3499 DMA_CUED_REGION_OFF;
3500 else if (test_bit(PPC440SPE_DESC_RXOR125,
3501 &sw_desc->flags))
3502 haddr = DMA_RXOR125 <<
3503 DMA_CUED_REGION_OFF;
3504 else
3505 BUG();
3506 haddr |= DMA_CUED_XOR_BASE;
3507 iter = ppc440spe_get_group_entry(sw_desc, 0);
3508 } else if (index < iskip) {
3509 /* 1st slot (RXOR)
3510 * shall actually set source address only once
3511 * instead of first <iskip>
3512 */
3513 iter = NULL;
3514 } else {
3515 /* 2nd/3d and next slots (WXOR);
3516 * skip first slot with RXOR
3517 */
3518 haddr = DMA_CUED_XOR_HB;
3519 iter = ppc440spe_get_group_entry(sw_desc,
3520 index - iskip + sw_desc->dst_cnt);
3521 }
3522 } else {
3523 int znum = 0;
3524
3525 /* WXOR-only operation; skip first slots with
3526 * zeroing destinations
3527 */
3528 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3529 znum++;
3530 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3531 znum++;
3532
3533 haddr = DMA_CUED_XOR_HB;
3534 iter = ppc440spe_get_group_entry(sw_desc,
3535 index + znum);
3536 }
3537
3538 if (likely(iter)) {
3539 ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3540
3541 if (!index &&
3542 test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3543 sw_desc->dst_cnt == 2) {
3544 /* if we have two destinations for RXOR, then
3545 * setup source in the second descr too
3546 */
3547 iter = ppc440spe_get_group_entry(sw_desc, 1);
3548 ppc440spe_desc_set_src_addr(iter, chan, 0,
3549 haddr, addr);
3550 }
3551 }
3552 break;
3553
3554 case PPC440SPE_XOR_ID:
3555 /* DMA2 may do Biskup */
3556 iter = sw_desc->group_head;
3557 if (iter->dst_cnt == 2) {
3558 /* both P & Q calculations required; set P src here */
3559 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3560
3561 /* this is for Q */
3562 iter = ppc440spe_get_group_entry(sw_desc,
3563 sw_desc->descs_per_op);
3564 }
3565 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3566 break;
3567 }
3568}
3569
3570/**
3571 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3572 */
3573static void ppc440spe_adma_memcpy_xor_set_src(
3574 struct ppc440spe_adma_desc_slot *sw_desc,
3575 dma_addr_t addr, int index)
3576{
3577 struct ppc440spe_adma_chan *chan;
3578
3579 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3580 sw_desc = sw_desc->group_head;
3581
3582 if (likely(sw_desc))
3583 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3584}
3585
3586/**
3587 * ppc440spe_adma_dma2rxor_inc_addr -
3588 */
3589static void ppc440spe_adma_dma2rxor_inc_addr(
3590 struct ppc440spe_adma_desc_slot *desc,
3591 struct ppc440spe_rxor *cursor, int index, int src_cnt)
3592{
3593 cursor->addr_count++;
3594 if (index == src_cnt - 1) {
3595 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3596 } else if (cursor->addr_count == XOR_MAX_OPS) {
3597 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3598 cursor->addr_count = 0;
3599 cursor->desc_count++;
3600 }
3601}
3602
3603/**
3604 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3605 */
3606static int ppc440spe_adma_dma2rxor_prep_src(
3607 struct ppc440spe_adma_desc_slot *hdesc,
3608 struct ppc440spe_rxor *cursor, int index,
3609 int src_cnt, u32 addr)
3610{
3611 int rval = 0;
3612 u32 sign;
3613 struct ppc440spe_adma_desc_slot *desc = hdesc;
3614 int i;
3615
3616 for (i = 0; i < cursor->desc_count; i++) {
3617 desc = list_entry(hdesc->chain_node.next,
3618 struct ppc440spe_adma_desc_slot,
3619 chain_node);
3620 }
3621
3622 switch (cursor->state) {
3623 case 0:
3624 if (addr == cursor->addrl + cursor->len) {
3625 /* direct RXOR */
3626 cursor->state = 1;
3627 cursor->xor_count++;
3628 if (index == src_cnt-1) {
3629 ppc440spe_rxor_set_region(desc,
3630 cursor->addr_count,
3631 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3632 ppc440spe_adma_dma2rxor_inc_addr(
3633 desc, cursor, index, src_cnt);
3634 }
3635 } else if (cursor->addrl == addr + cursor->len) {
3636 /* reverse RXOR */
3637 cursor->state = 1;
3638 cursor->xor_count++;
3639 set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3640 if (index == src_cnt-1) {
3641 ppc440spe_rxor_set_region(desc,
3642 cursor->addr_count,
3643 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3644 ppc440spe_adma_dma2rxor_inc_addr(
3645 desc, cursor, index, src_cnt);
3646 }
3647 } else {
3648 printk(KERN_ERR "Cannot build "
3649 "DMA2 RXOR command block.\n");
3650 BUG();
3651 }
3652 break;
3653 case 1:
3654 sign = test_bit(cursor->addr_count,
3655 desc->reverse_flags)
3656 ? -1 : 1;
3657 if (index == src_cnt-2 || (sign == -1
3658 && addr != cursor->addrl - 2*cursor->len)) {
3659 cursor->state = 0;
3660 cursor->xor_count = 1;
3661 cursor->addrl = addr;
3662 ppc440spe_rxor_set_region(desc,
3663 cursor->addr_count,
3664 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3665 ppc440spe_adma_dma2rxor_inc_addr(
3666 desc, cursor, index, src_cnt);
3667 } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3668 cursor->state = 2;
3669 cursor->xor_count = 0;
3670 ppc440spe_rxor_set_region(desc,
3671 cursor->addr_count,
3672 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3673 if (index == src_cnt-1) {
3674 ppc440spe_adma_dma2rxor_inc_addr(
3675 desc, cursor, index, src_cnt);
3676 }
3677 } else if (addr == cursor->addrl + 3*cursor->len) {
3678 cursor->state = 2;
3679 cursor->xor_count = 0;
3680 ppc440spe_rxor_set_region(desc,
3681 cursor->addr_count,
3682 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3683 if (index == src_cnt-1) {
3684 ppc440spe_adma_dma2rxor_inc_addr(
3685 desc, cursor, index, src_cnt);
3686 }
3687 } else if (addr == cursor->addrl + 4*cursor->len) {
3688 cursor->state = 2;
3689 cursor->xor_count = 0;
3690 ppc440spe_rxor_set_region(desc,
3691 cursor->addr_count,
3692 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3693 if (index == src_cnt-1) {
3694 ppc440spe_adma_dma2rxor_inc_addr(
3695 desc, cursor, index, src_cnt);
3696 }
3697 } else {
3698 cursor->state = 0;
3699 cursor->xor_count = 1;
3700 cursor->addrl = addr;
3701 ppc440spe_rxor_set_region(desc,
3702 cursor->addr_count,
3703 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3704 ppc440spe_adma_dma2rxor_inc_addr(
3705 desc, cursor, index, src_cnt);
3706 }
3707 break;
3708 case 2:
3709 cursor->state = 0;
3710 cursor->addrl = addr;
3711 cursor->xor_count++;
3712 if (index) {
3713 ppc440spe_adma_dma2rxor_inc_addr(
3714 desc, cursor, index, src_cnt);
3715 }
3716 break;
3717 }
3718
3719 return rval;
3720}
3721
3722/**
3723 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3724 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3725 */
3726static void ppc440spe_adma_dma2rxor_set_src(
3727 struct ppc440spe_adma_desc_slot *desc,
3728 int index, dma_addr_t addr)
3729{
3730 struct xor_cb *xcb = desc->hw_desc;
3731 int k = 0, op = 0, lop = 0;
3732
3733 /* get the RXOR operand which corresponds to index addr */
3734 while (op <= index) {
3735 lop = op;
3736 if (k == XOR_MAX_OPS) {
3737 k = 0;
3738 desc = list_entry(desc->chain_node.next,
3739 struct ppc440spe_adma_desc_slot, chain_node);
3740 xcb = desc->hw_desc;
3741
3742 }
3743 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3744 (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3745 op += 2;
3746 else
3747 op += 3;
3748 }
3749
3750 BUG_ON(k < 1);
3751
3752 if (test_bit(k-1, desc->reverse_flags)) {
3753 /* reverse operand order; put last op in RXOR group */
3754 if (index == op - 1)
3755 ppc440spe_rxor_set_src(desc, k - 1, addr);
3756 } else {
3757 /* direct operand order; put first op in RXOR group */
3758 if (index == lop)
3759 ppc440spe_rxor_set_src(desc, k - 1, addr);
3760 }
3761}
3762
3763/**
3764 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3765 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3766 */
3767static void ppc440spe_adma_dma2rxor_set_mult(
3768 struct ppc440spe_adma_desc_slot *desc,
3769 int index, u8 mult)
3770{
3771 struct xor_cb *xcb = desc->hw_desc;
3772 int k = 0, op = 0, lop = 0;
3773
3774 /* get the RXOR operand which corresponds to index mult */
3775 while (op <= index) {
3776 lop = op;
3777 if (k == XOR_MAX_OPS) {
3778 k = 0;
3779 desc = list_entry(desc->chain_node.next,
3780 struct ppc440spe_adma_desc_slot,
3781 chain_node);
3782 xcb = desc->hw_desc;
3783
3784 }
3785 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3786 (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3787 op += 2;
3788 else
3789 op += 3;
3790 }
3791
3792 BUG_ON(k < 1);
3793 if (test_bit(k-1, desc->reverse_flags)) {
3794 /* reverse order */
3795 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3796 } else {
3797 /* direct order */
3798 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3799 }
3800}
3801
3802/**
3803 * ppc440spe_init_rxor_cursor -
3804 */
3805static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3806{
3807 memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3808 cursor->state = 2;
3809}
3810
3811/**
3812 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3813 * descriptor for the PQXOR operation
3814 */
3815static void ppc440spe_adma_pq_set_src_mult(
3816 struct ppc440spe_adma_desc_slot *sw_desc,
3817 unsigned char mult, int index, int dst_pos)
3818{
3819 struct ppc440spe_adma_chan *chan;
3820 u32 mult_idx, mult_dst;
3821 struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3822
3823 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3824
3825 switch (chan->device->id) {
3826 case PPC440SPE_DMA0_ID:
3827 case PPC440SPE_DMA1_ID:
3828 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3829 int region = test_bit(PPC440SPE_DESC_RXOR12,
3830 &sw_desc->flags) ? 2 : 3;
3831
3832 if (index < region) {
3833 /* RXOR multipliers */
3834 iter = ppc440spe_get_group_entry(sw_desc,
3835 sw_desc->dst_cnt - 1);
3836 if (sw_desc->dst_cnt == 2)
3837 iter1 = ppc440spe_get_group_entry(
3838 sw_desc, 0);
3839
3840 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3841 mult_dst = DMA_CDB_SG_SRC;
3842 } else {
3843 /* WXOR multiplier */
3844 iter = ppc440spe_get_group_entry(sw_desc,
3845 index - region +
3846 sw_desc->dst_cnt);
3847 mult_idx = DMA_CUED_MULT1_OFF;
3848 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3849 DMA_CDB_SG_DST1;
3850 }
3851 } else {
3852 int znum = 0;
3853
3854 /* WXOR-only;
3855 * skip first slots with destinations (if ZERO_DST has
3856 * place)
3857 */
3858 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3859 znum++;
3860 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3861 znum++;
3862
3863 iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3864 mult_idx = DMA_CUED_MULT1_OFF;
3865 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3866 }
3867
3868 if (likely(iter)) {
3869 ppc440spe_desc_set_src_mult(iter, chan,
3870 mult_idx, mult_dst, mult);
3871
3872 if (unlikely(iter1)) {
3873 /* if we have two destinations for RXOR, then
3874 * we've just set Q mult. Set-up P now.
3875 */
3876 ppc440spe_desc_set_src_mult(iter1, chan,
3877 mult_idx, mult_dst, 1);
3878 }
3879
3880 }
3881 break;
3882
3883 case PPC440SPE_XOR_ID:
3884 iter = sw_desc->group_head;
3885 if (sw_desc->dst_cnt == 2) {
3886 /* both P & Q calculations required; set P mult here */
3887 ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3888
3889 /* and then set Q mult */
3890 iter = ppc440spe_get_group_entry(sw_desc,
3891 sw_desc->descs_per_op);
3892 }
3893 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3894 break;
3895 }
3896}
3897
3898/**
3899 * ppc440spe_adma_free_chan_resources - free the resources allocated
3900 */
3901static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3902{
3903 struct ppc440spe_adma_chan *ppc440spe_chan;
3904 struct ppc440spe_adma_desc_slot *iter, *_iter;
3905 int in_use_descs = 0;
3906
3907 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3908 ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3909
3910 spin_lock_bh(&ppc440spe_chan->lock);
3911 list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3912 chain_node) {
3913 in_use_descs++;
3914 list_del(&iter->chain_node);
3915 }
3916 list_for_each_entry_safe_reverse(iter, _iter,
3917 &ppc440spe_chan->all_slots, slot_node) {
3918 list_del(&iter->slot_node);
3919 kfree(iter);
3920 ppc440spe_chan->slots_allocated--;
3921 }
3922 ppc440spe_chan->last_used = NULL;
3923
3924 dev_dbg(ppc440spe_chan->device->common.dev,
3925 "ppc440spe adma%d %s slots_allocated %d\n",
3926 ppc440spe_chan->device->id,
3927 __func__, ppc440spe_chan->slots_allocated);
3928 spin_unlock_bh(&ppc440spe_chan->lock);
3929
3930 /* one is ok since we left it on there on purpose */
3931 if (in_use_descs > 1)
3932 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3933 in_use_descs - 1);
3934}
3935
3936/**
3937 * ppc440spe_adma_is_complete - poll the status of an ADMA transaction
3938 * @chan: ADMA channel handle
3939 * @cookie: ADMA transaction identifier
3940 */
3941static enum dma_status ppc440spe_adma_is_complete(struct dma_chan *chan,
3942 dma_cookie_t cookie, dma_cookie_t *done, dma_cookie_t *used)
3943{
3944 struct ppc440spe_adma_chan *ppc440spe_chan;
3945 dma_cookie_t last_used;
3946 dma_cookie_t last_complete;
3947 enum dma_status ret;
3948
3949 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3950 last_used = chan->cookie;
3951 last_complete = ppc440spe_chan->completed_cookie;
3952
3953 if (done)
3954 *done = last_complete;
3955 if (used)
3956 *used = last_used;
3957
3958 ret = dma_async_is_complete(cookie, last_complete, last_used);
3959 if (ret == DMA_SUCCESS)
3960 return ret;
3961
3962 ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3963
3964 last_used = chan->cookie;
3965 last_complete = ppc440spe_chan->completed_cookie;
3966
3967 if (done)
3968 *done = last_complete;
3969 if (used)
3970 *used = last_used;
3971
3972 return dma_async_is_complete(cookie, last_complete, last_used);
3973}
3974
3975/**
3976 * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3977 */
3978static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3979{
3980 struct ppc440spe_adma_chan *chan = data;
3981
3982 dev_dbg(chan->device->common.dev,
3983 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3984
3985 tasklet_schedule(&chan->irq_tasklet);
3986 ppc440spe_adma_device_clear_eot_status(chan);
3987
3988 return IRQ_HANDLED;
3989}
3990
3991/**
3992 * ppc440spe_adma_err_handler - DMA error interrupt handler;
3993 * do the same things as a eot handler
3994 */
3995static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3996{
3997 struct ppc440spe_adma_chan *chan = data;
3998
3999 dev_dbg(chan->device->common.dev,
4000 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
4001
4002 tasklet_schedule(&chan->irq_tasklet);
4003 ppc440spe_adma_device_clear_eot_status(chan);
4004
4005 return IRQ_HANDLED;
4006}
4007
4008/**
4009 * ppc440spe_test_callback - called when test operation has been done
4010 */
4011static void ppc440spe_test_callback(void *unused)
4012{
4013 complete(&ppc440spe_r6_test_comp);
4014}
4015
4016/**
4017 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
4018 */
4019static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
4020{
4021 struct ppc440spe_adma_chan *ppc440spe_chan;
4022
4023 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4024 dev_dbg(ppc440spe_chan->device->common.dev,
4025 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
4026 __func__, ppc440spe_chan->pending);
4027
4028 if (ppc440spe_chan->pending) {
4029 ppc440spe_chan->pending = 0;
4030 ppc440spe_chan_append(ppc440spe_chan);
4031 }
4032}
4033
4034/**
4035 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
4036 * use FIFOs (as opposite to chains used in XOR) so this is a XOR
4037 * specific operation)
4038 */
4039static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
4040{
4041 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
4042 dma_cookie_t cookie;
4043 int slot_cnt, slots_per_op;
4044
4045 dev_dbg(chan->device->common.dev,
4046 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
4047
4048 spin_lock_bh(&chan->lock);
4049 slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
4050 sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
4051 if (sw_desc) {
4052 group_start = sw_desc->group_head;
4053 list_splice_init(&sw_desc->group_list, &chan->chain);
4054 async_tx_ack(&sw_desc->async_tx);
4055 ppc440spe_desc_init_null_xor(group_start);
4056
4057 cookie = chan->common.cookie;
4058 cookie++;
4059 if (cookie <= 1)
4060 cookie = 2;
4061
4062 /* initialize the completed cookie to be less than
4063 * the most recently used cookie
4064 */
4065 chan->completed_cookie = cookie - 1;
4066 chan->common.cookie = sw_desc->async_tx.cookie = cookie;
4067
4068 /* channel should not be busy */
4069 BUG_ON(ppc440spe_chan_is_busy(chan));
4070
4071 /* set the descriptor address */
4072 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
4073
4074 /* run the descriptor */
4075 ppc440spe_chan_run(chan);
4076 } else
4077 printk(KERN_ERR "ppc440spe adma%d"
4078 " failed to allocate null descriptor\n",
4079 chan->device->id);
4080 spin_unlock_bh(&chan->lock);
4081}
4082
4083/**
4084 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
4085 * For this we just perform one WXOR operation with the same source
4086 * and destination addresses, the GF-multiplier is 1; so if RAID-6
4087 * capabilities are enabled then we'll get src/dst filled with zero.
4088 */
4089static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
4090{
4091 struct ppc440spe_adma_desc_slot *sw_desc, *iter;
4092 struct page *pg;
4093 char *a;
4094 dma_addr_t dma_addr, addrs[2];
4095 unsigned long op = 0;
4096 int rval = 0;
4097
4098 set_bit(PPC440SPE_DESC_WXOR, &op);
4099
4100 pg = alloc_page(GFP_KERNEL);
4101 if (!pg)
4102 return -ENOMEM;
4103
4104 spin_lock_bh(&chan->lock);
4105 sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
4106 if (sw_desc) {
4107 /* 1 src, 1 dsr, int_ena, WXOR */
4108 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
4109 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
4110 ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
4111 iter->unmap_len = PAGE_SIZE;
4112 }
4113 } else {
4114 rval = -EFAULT;
4115 spin_unlock_bh(&chan->lock);
4116 goto exit;
4117 }
4118 spin_unlock_bh(&chan->lock);
4119
4120 /* Fill the test page with ones */
4121 memset(page_address(pg), 0xFF, PAGE_SIZE);
4122 dma_addr = dma_map_page(chan->device->dev, pg, 0,
4123 PAGE_SIZE, DMA_BIDIRECTIONAL);
4124
4125 /* Setup addresses */
4126 ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
4127 ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
4128 addrs[0] = dma_addr;
4129 addrs[1] = 0;
4130 ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
4131
4132 async_tx_ack(&sw_desc->async_tx);
4133 sw_desc->async_tx.callback = ppc440spe_test_callback;
4134 sw_desc->async_tx.callback_param = NULL;
4135
4136 init_completion(&ppc440spe_r6_test_comp);
4137
4138 ppc440spe_adma_tx_submit(&sw_desc->async_tx);
4139 ppc440spe_adma_issue_pending(&chan->common);
4140
4141 wait_for_completion(&ppc440spe_r6_test_comp);
4142
4143 /* Now check if the test page is zeroed */
4144 a = page_address(pg);
4145 if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
4146 /* page is zero - RAID-6 enabled */
4147 rval = 0;
4148 } else {
4149 /* RAID-6 was not enabled */
4150 rval = -EINVAL;
4151 }
4152exit:
4153 __free_page(pg);
4154 return rval;
4155}
4156
4157static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
4158{
4159 switch (adev->id) {
4160 case PPC440SPE_DMA0_ID:
4161 case PPC440SPE_DMA1_ID:
4162 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
4163 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
4164 dma_cap_set(DMA_MEMSET, adev->common.cap_mask);
4165 dma_cap_set(DMA_PQ, adev->common.cap_mask);
4166 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
4167 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
4168 break;
4169 case PPC440SPE_XOR_ID:
4170 dma_cap_set(DMA_XOR, adev->common.cap_mask);
4171 dma_cap_set(DMA_PQ, adev->common.cap_mask);
4172 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
4173 adev->common.cap_mask = adev->common.cap_mask;
4174 break;
4175 }
4176
4177 /* Set base routines */
4178 adev->common.device_alloc_chan_resources =
4179 ppc440spe_adma_alloc_chan_resources;
4180 adev->common.device_free_chan_resources =
4181 ppc440spe_adma_free_chan_resources;
4182 adev->common.device_is_tx_complete = ppc440spe_adma_is_complete;
4183 adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
4184
4185 /* Set prep routines based on capability */
4186 if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
4187 adev->common.device_prep_dma_memcpy =
4188 ppc440spe_adma_prep_dma_memcpy;
4189 }
4190 if (dma_has_cap(DMA_MEMSET, adev->common.cap_mask)) {
4191 adev->common.device_prep_dma_memset =
4192 ppc440spe_adma_prep_dma_memset;
4193 }
4194 if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
4195 adev->common.max_xor = XOR_MAX_OPS;
4196 adev->common.device_prep_dma_xor =
4197 ppc440spe_adma_prep_dma_xor;
4198 }
4199 if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
4200 switch (adev->id) {
4201 case PPC440SPE_DMA0_ID:
4202 dma_set_maxpq(&adev->common,
4203 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
4204 break;
4205 case PPC440SPE_DMA1_ID:
4206 dma_set_maxpq(&adev->common,
4207 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
4208 break;
4209 case PPC440SPE_XOR_ID:
4210 adev->common.max_pq = XOR_MAX_OPS * 3;
4211 break;
4212 }
4213 adev->common.device_prep_dma_pq =
4214 ppc440spe_adma_prep_dma_pq;
4215 }
4216 if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
4217 switch (adev->id) {
4218 case PPC440SPE_DMA0_ID:
4219 adev->common.max_pq = DMA0_FIFO_SIZE /
4220 sizeof(struct dma_cdb);
4221 break;
4222 case PPC440SPE_DMA1_ID:
4223 adev->common.max_pq = DMA1_FIFO_SIZE /
4224 sizeof(struct dma_cdb);
4225 break;
4226 }
4227 adev->common.device_prep_dma_pq_val =
4228 ppc440spe_adma_prep_dma_pqzero_sum;
4229 }
4230 if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
4231 switch (adev->id) {
4232 case PPC440SPE_DMA0_ID:
4233 adev->common.max_xor = DMA0_FIFO_SIZE /
4234 sizeof(struct dma_cdb);
4235 break;
4236 case PPC440SPE_DMA1_ID:
4237 adev->common.max_xor = DMA1_FIFO_SIZE /
4238 sizeof(struct dma_cdb);
4239 break;
4240 }
4241 adev->common.device_prep_dma_xor_val =
4242 ppc440spe_adma_prep_dma_xor_zero_sum;
4243 }
4244 if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
4245 adev->common.device_prep_dma_interrupt =
4246 ppc440spe_adma_prep_dma_interrupt;
4247 }
4248 pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
4249 "( %s%s%s%s%s%s%s)\n",
4250 dev_name(adev->dev),
4251 dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
4252 dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
4253 dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
4254 dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
4255 dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
4256 dma_has_cap(DMA_MEMSET, adev->common.cap_mask) ? "memset " : "",
4257 dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
4258}
4259
4260static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
4261 struct ppc440spe_adma_chan *chan,
4262 int *initcode)
4263{
4264 struct device_node *np;
4265 int ret;
4266
4267 np = container_of(adev->dev, struct of_device, dev)->node;
4268 if (adev->id != PPC440SPE_XOR_ID) {
4269 adev->err_irq = irq_of_parse_and_map(np, 1);
4270 if (adev->err_irq == NO_IRQ) {
4271 dev_warn(adev->dev, "no err irq resource?\n");
4272 *initcode = PPC_ADMA_INIT_IRQ2;
4273 adev->err_irq = -ENXIO;
4274 } else
4275 atomic_inc(&ppc440spe_adma_err_irq_ref);
4276 } else {
4277 adev->err_irq = -ENXIO;
4278 }
4279
4280 adev->irq = irq_of_parse_and_map(np, 0);
4281 if (adev->irq == NO_IRQ) {
4282 dev_err(adev->dev, "no irq resource\n");
4283 *initcode = PPC_ADMA_INIT_IRQ1;
4284 ret = -ENXIO;
4285 goto err_irq_map;
4286 }
4287 dev_dbg(adev->dev, "irq %d, err irq %d\n",
4288 adev->irq, adev->err_irq);
4289
4290 ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
4291 0, dev_driver_string(adev->dev), chan);
4292 if (ret) {
4293 dev_err(adev->dev, "can't request irq %d\n",
4294 adev->irq);
4295 *initcode = PPC_ADMA_INIT_IRQ1;
4296 ret = -EIO;
4297 goto err_req1;
4298 }
4299
4300 /* only DMA engines have a separate error IRQ
4301 * so it's Ok if err_irq < 0 in XOR engine case.
4302 */
4303 if (adev->err_irq > 0) {
4304 /* both DMA engines share common error IRQ */
4305 ret = request_irq(adev->err_irq,
4306 ppc440spe_adma_err_handler,
4307 IRQF_SHARED,
4308 dev_driver_string(adev->dev),
4309 chan);
4310 if (ret) {
4311 dev_err(adev->dev, "can't request irq %d\n",
4312 adev->err_irq);
4313 *initcode = PPC_ADMA_INIT_IRQ2;
4314 ret = -EIO;
4315 goto err_req2;
4316 }
4317 }
4318
4319 if (adev->id == PPC440SPE_XOR_ID) {
4320 /* enable XOR engine interrupts */
4321 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4322 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
4323 &adev->xor_reg->ier);
4324 } else {
4325 u32 mask, enable;
4326
4327 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4328 if (!np) {
4329 pr_err("%s: can't find I2O device tree node\n",
4330 __func__);
4331 ret = -ENODEV;
4332 goto err_req2;
4333 }
4334 adev->i2o_reg = of_iomap(np, 0);
4335 if (!adev->i2o_reg) {
4336 pr_err("%s: failed to map I2O registers\n", __func__);
4337 of_node_put(np);
4338 ret = -EINVAL;
4339 goto err_req2;
4340 }
4341 of_node_put(np);
4342 /* Unmask 'CS FIFO Attention' interrupts and
4343 * enable generating interrupts on errors
4344 */
4345 enable = (adev->id == PPC440SPE_DMA0_ID) ?
4346 ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4347 ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4348 mask = ioread32(&adev->i2o_reg->iopim) & enable;
4349 iowrite32(mask, &adev->i2o_reg->iopim);
4350 }
4351 return 0;
4352
4353err_req2:
4354 free_irq(adev->irq, chan);
4355err_req1:
4356 irq_dispose_mapping(adev->irq);
4357err_irq_map:
4358 if (adev->err_irq > 0) {
4359 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
4360 irq_dispose_mapping(adev->err_irq);
4361 }
4362 return ret;
4363}
4364
4365static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
4366 struct ppc440spe_adma_chan *chan)
4367{
4368 u32 mask, disable;
4369
4370 if (adev->id == PPC440SPE_XOR_ID) {
4371 /* disable XOR engine interrupts */
4372 mask = ioread32be(&adev->xor_reg->ier);
4373 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4374 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
4375 iowrite32be(mask, &adev->xor_reg->ier);
4376 } else {
4377 /* disable DMAx engine interrupts */
4378 disable = (adev->id == PPC440SPE_DMA0_ID) ?
4379 (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4380 (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4381 mask = ioread32(&adev->i2o_reg->iopim) | disable;
4382 iowrite32(mask, &adev->i2o_reg->iopim);
4383 }
4384 free_irq(adev->irq, chan);
4385 irq_dispose_mapping(adev->irq);
4386 if (adev->err_irq > 0) {
4387 free_irq(adev->err_irq, chan);
4388 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
4389 irq_dispose_mapping(adev->err_irq);
4390 iounmap(adev->i2o_reg);
4391 }
4392 }
4393}
4394
4395/**
4396 * ppc440spe_adma_probe - probe the asynch device
4397 */
4398static int __devinit ppc440spe_adma_probe(struct of_device *ofdev,
4399 const struct of_device_id *match)
4400{
4401 struct device_node *np = ofdev->node;
4402 struct resource res;
4403 struct ppc440spe_adma_device *adev;
4404 struct ppc440spe_adma_chan *chan;
4405 struct ppc_dma_chan_ref *ref, *_ref;
4406 int ret = 0, initcode = PPC_ADMA_INIT_OK;
4407 const u32 *idx;
4408 int len;
4409 void *regs;
4410 u32 id, pool_size;
4411
4412 if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4413 id = PPC440SPE_XOR_ID;
4414 /* As far as the XOR engine is concerned, it does not
4415 * use FIFOs but uses linked list. So there is no dependency
4416 * between pool size to allocate and the engine configuration.
4417 */
4418 pool_size = PAGE_SIZE << 1;
4419 } else {
4420 /* it is DMA0 or DMA1 */
4421 idx = of_get_property(np, "cell-index", &len);
4422 if (!idx || (len != sizeof(u32))) {
4423 dev_err(&ofdev->dev, "Device node %s has missing "
4424 "or invalid cell-index property\n",
4425 np->full_name);
4426 return -EINVAL;
4427 }
4428 id = *idx;
4429 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4430 * should allocate the pool accordingly to size of this
4431 * FIFO. Thus, the pool size depends on the FIFO depth:
4432 * how much CDBs pointers the FIFO may contain then so
4433 * much CDBs we should provide in the pool.
4434 * That is
4435 * CDB size = 32B;
4436 * CDBs number = (DMA0_FIFO_SIZE >> 3);
4437 * Pool size = CDBs number * CDB size =
4438 * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4439 */
4440 pool_size = (id == PPC440SPE_DMA0_ID) ?
4441 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4442 pool_size <<= 2;
4443 }
4444
4445 if (of_address_to_resource(np, 0, &res)) {
4446 dev_err(&ofdev->dev, "failed to get memory resource\n");
4447 initcode = PPC_ADMA_INIT_MEMRES;
4448 ret = -ENODEV;
4449 goto out;
4450 }
4451
4452 if (!request_mem_region(res.start, resource_size(&res),
4453 dev_driver_string(&ofdev->dev))) {
4454 dev_err(&ofdev->dev, "failed to request memory region "
4455 "(0x%016llx-0x%016llx)\n",
4456 (u64)res.start, (u64)res.end);
4457 initcode = PPC_ADMA_INIT_MEMREG;
4458 ret = -EBUSY;
4459 goto out;
4460 }
4461
4462 /* create a device */
4463 adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4464 if (!adev) {
4465 dev_err(&ofdev->dev, "failed to allocate device\n");
4466 initcode = PPC_ADMA_INIT_ALLOC;
4467 ret = -ENOMEM;
4468 goto err_adev_alloc;
4469 }
4470
4471 adev->id = id;
4472 adev->pool_size = pool_size;
4473 /* allocate coherent memory for hardware descriptors */
4474 adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4475 adev->pool_size, &adev->dma_desc_pool,
4476 GFP_KERNEL);
4477 if (adev->dma_desc_pool_virt == NULL) {
4478 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4479 "memory for hardware descriptors\n",
4480 adev->pool_size);
4481 initcode = PPC_ADMA_INIT_COHERENT;
4482 ret = -ENOMEM;
4483 goto err_dma_alloc;
4484 }
4485 dev_dbg(&ofdev->dev, "allocted descriptor pool virt 0x%p phys 0x%llx\n",
4486 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4487
4488 regs = ioremap(res.start, resource_size(&res));
4489 if (!regs) {
4490 dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4491 goto err_regs_alloc;
4492 }
4493
4494 if (adev->id == PPC440SPE_XOR_ID) {
4495 adev->xor_reg = regs;
4496 /* Reset XOR */
4497 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4498 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4499 } else {
4500 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4501 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4502 adev->dma_reg = regs;
4503 /* DMAx_FIFO_SIZE is defined in bytes,
4504 * <fsiz> - is defined in number of CDB pointers (8byte).
4505 * DMA FIFO Length = CSlength + CPlength, where
4506 * CSlength = CPlength = (fsiz + 1) * 8.
4507 */
4508 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4509 &adev->dma_reg->fsiz);
4510 /* Configure DMA engine */
4511 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4512 &adev->dma_reg->cfg);
4513 /* Clear Status */
4514 iowrite32(~0, &adev->dma_reg->dsts);
4515 }
4516
4517 adev->dev = &ofdev->dev;
4518 adev->common.dev = &ofdev->dev;
4519 INIT_LIST_HEAD(&adev->common.channels);
4520 dev_set_drvdata(&ofdev->dev, adev);
4521
4522 /* create a channel */
4523 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4524 if (!chan) {
4525 dev_err(&ofdev->dev, "can't allocate channel structure\n");
4526 initcode = PPC_ADMA_INIT_CHANNEL;
4527 ret = -ENOMEM;
4528 goto err_chan_alloc;
4529 }
4530
4531 spin_lock_init(&chan->lock);
4532 INIT_LIST_HEAD(&chan->chain);
4533 INIT_LIST_HEAD(&chan->all_slots);
4534 chan->device = adev;
4535 chan->common.device = &adev->common;
4536 list_add_tail(&chan->common.device_node, &adev->common.channels);
4537 tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet,
4538 (unsigned long)chan);
4539
4540 /* allocate and map helper pages for async validation or
4541 * async_mult/async_sum_product operations on DMA0/1.
4542 */
4543 if (adev->id != PPC440SPE_XOR_ID) {
4544 chan->pdest_page = alloc_page(GFP_KERNEL);
4545 chan->qdest_page = alloc_page(GFP_KERNEL);
4546 if (!chan->pdest_page ||
4547 !chan->qdest_page) {
4548 if (chan->pdest_page)
4549 __free_page(chan->pdest_page);
4550 if (chan->qdest_page)
4551 __free_page(chan->qdest_page);
4552 ret = -ENOMEM;
4553 goto err_page_alloc;
4554 }
4555 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4556 PAGE_SIZE, DMA_BIDIRECTIONAL);
4557 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4558 PAGE_SIZE, DMA_BIDIRECTIONAL);
4559 }
4560
4561 ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4562 if (ref) {
4563 ref->chan = &chan->common;
4564 INIT_LIST_HEAD(&ref->node);
4565 list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4566 } else {
4567 dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4568 ret = -ENOMEM;
4569 goto err_ref_alloc;
4570 }
4571
4572 ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4573 if (ret)
4574 goto err_irq;
4575
4576 ppc440spe_adma_init_capabilities(adev);
4577
4578 ret = dma_async_device_register(&adev->common);
4579 if (ret) {
4580 initcode = PPC_ADMA_INIT_REGISTER;
4581 dev_err(&ofdev->dev, "failed to register dma device\n");
4582 goto err_dev_reg;
4583 }
4584
4585 goto out;
4586
4587err_dev_reg:
4588 ppc440spe_adma_release_irqs(adev, chan);
4589err_irq:
4590 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4591 if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4592 list_del(&ref->node);
4593 kfree(ref);
4594 }
4595 }
4596err_ref_alloc:
4597 if (adev->id != PPC440SPE_XOR_ID) {
4598 dma_unmap_page(&ofdev->dev, chan->pdest,
4599 PAGE_SIZE, DMA_BIDIRECTIONAL);
4600 dma_unmap_page(&ofdev->dev, chan->qdest,
4601 PAGE_SIZE, DMA_BIDIRECTIONAL);
4602 __free_page(chan->pdest_page);
4603 __free_page(chan->qdest_page);
4604 }
4605err_page_alloc:
4606 kfree(chan);
4607err_chan_alloc:
4608 if (adev->id == PPC440SPE_XOR_ID)
4609 iounmap(adev->xor_reg);
4610 else
4611 iounmap(adev->dma_reg);
4612err_regs_alloc:
4613 dma_free_coherent(adev->dev, adev->pool_size,
4614 adev->dma_desc_pool_virt,
4615 adev->dma_desc_pool);
4616err_dma_alloc:
4617 kfree(adev);
4618err_adev_alloc:
4619 release_mem_region(res.start, resource_size(&res));
4620out:
4621 if (id < PPC440SPE_ADMA_ENGINES_NUM)
4622 ppc440spe_adma_devices[id] = initcode;
4623
4624 return ret;
4625}
4626
4627/**
4628 * ppc440spe_adma_remove - remove the asynch device
4629 */
4630static int __devexit ppc440spe_adma_remove(struct of_device *ofdev)
4631{
4632 struct ppc440spe_adma_device *adev = dev_get_drvdata(&ofdev->dev);
4633 struct device_node *np = ofdev->node;
4634 struct resource res;
4635 struct dma_chan *chan, *_chan;
4636 struct ppc_dma_chan_ref *ref, *_ref;
4637 struct ppc440spe_adma_chan *ppc440spe_chan;
4638
4639 dev_set_drvdata(&ofdev->dev, NULL);
4640 if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4641 ppc440spe_adma_devices[adev->id] = -1;
4642
4643 dma_async_device_unregister(&adev->common);
4644
4645 list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4646 device_node) {
4647 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4648 ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4649 tasklet_kill(&ppc440spe_chan->irq_tasklet);
4650 if (adev->id != PPC440SPE_XOR_ID) {
4651 dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4652 PAGE_SIZE, DMA_BIDIRECTIONAL);
4653 dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4654 PAGE_SIZE, DMA_BIDIRECTIONAL);
4655 __free_page(ppc440spe_chan->pdest_page);
4656 __free_page(ppc440spe_chan->qdest_page);
4657 }
4658 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4659 node) {
4660 if (ppc440spe_chan ==
4661 to_ppc440spe_adma_chan(ref->chan)) {
4662 list_del(&ref->node);
4663 kfree(ref);
4664 }
4665 }
4666 list_del(&chan->device_node);
4667 kfree(ppc440spe_chan);
4668 }
4669
4670 dma_free_coherent(adev->dev, adev->pool_size,
4671 adev->dma_desc_pool_virt, adev->dma_desc_pool);
4672 if (adev->id == PPC440SPE_XOR_ID)
4673 iounmap(adev->xor_reg);
4674 else
4675 iounmap(adev->dma_reg);
4676 of_address_to_resource(np, 0, &res);
4677 release_mem_region(res.start, resource_size(&res));
4678 kfree(adev);
4679 return 0;
4680}
4681
4682/*
4683 * /sys driver interface to enable h/w RAID-6 capabilities
4684 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4685 * directory are "devices", "enable" and "poly".
4686 * "devices" shows available engines.
4687 * "enable" is used to enable RAID-6 capabilities or to check
4688 * whether these has been activated.
4689 * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4690 */
4691
4692static ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf)
4693{
4694 ssize_t size = 0;
4695 int i;
4696
4697 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4698 if (ppc440spe_adma_devices[i] == -1)
4699 continue;
4700 size += snprintf(buf + size, PAGE_SIZE - size,
4701 "PPC440SP(E)-ADMA.%d: %s\n", i,
4702 ppc_adma_errors[ppc440spe_adma_devices[i]]);
4703 }
4704 return size;
4705}
4706
4707static ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf)
4708{
4709 return snprintf(buf, PAGE_SIZE,
4710 "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4711 ppc440spe_r6_enabled ? "EN" : "DIS");
4712}
4713
4714static ssize_t store_ppc440spe_r6enable(struct device_driver *dev,
4715 const char *buf, size_t count)
4716{
4717 unsigned long val;
4718
4719 if (!count || count > 11)
4720 return -EINVAL;
4721
4722 if (!ppc440spe_r6_tchan)
4723 return -EFAULT;
4724
4725 /* Write a key */
4726 sscanf(buf, "%lx", &val);
4727 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4728 isync();
4729
4730 /* Verify whether it really works now */
4731 if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4732 pr_info("PPC440SP(e) RAID-6 has been activated "
4733 "successfully\n");
4734 ppc440spe_r6_enabled = 1;
4735 } else {
4736 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4737 " Error key ?\n");
4738 ppc440spe_r6_enabled = 0;
4739 }
4740 return count;
4741}
4742
4743static ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf)
4744{
4745 ssize_t size = 0;
4746 u32 reg;
4747
4748#ifdef CONFIG_440SP
4749 /* 440SP has fixed polynomial */
4750 reg = 0x4d;
4751#else
4752 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4753 reg >>= MQ0_CFBHL_POLY;
4754 reg &= 0xFF;
4755#endif
4756
4757 size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4758 "uses 0x1%02x polynomial.\n", reg);
4759 return size;
4760}
4761
4762static ssize_t store_ppc440spe_r6poly(struct device_driver *dev,
4763 const char *buf, size_t count)
4764{
4765 unsigned long reg, val;
4766
4767#ifdef CONFIG_440SP
4768 /* 440SP uses default 0x14D polynomial only */
4769 return -EINVAL;
4770#endif
4771
4772 if (!count || count > 6)
4773 return -EINVAL;
4774
4775 /* e.g., 0x14D or 0x11D */
4776 sscanf(buf, "%lx", &val);
4777
4778 if (val & ~0x1FF)
4779 return -EINVAL;
4780
4781 val &= 0xFF;
4782 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4783 reg &= ~(0xFF << MQ0_CFBHL_POLY);
4784 reg |= val << MQ0_CFBHL_POLY;
4785 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4786
4787 return count;
4788}
4789
4790static DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL);
4791static DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable,
4792 store_ppc440spe_r6enable);
4793static DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly,
4794 store_ppc440spe_r6poly);
4795
4796/*
4797 * Common initialisation for RAID engines; allocate memory for
4798 * DMAx FIFOs, perform configuration common for all DMA engines.
4799 * Further DMA engine specific configuration is done at probe time.
4800 */
4801static int ppc440spe_configure_raid_devices(void)
4802{
4803 struct device_node *np;
4804 struct resource i2o_res;
4805 struct i2o_regs __iomem *i2o_reg;
4806 dcr_host_t i2o_dcr_host;
4807 unsigned int dcr_base, dcr_len;
4808 int i, ret;
4809
4810 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4811 if (!np) {
4812 pr_err("%s: can't find I2O device tree node\n",
4813 __func__);
4814 return -ENODEV;
4815 }
4816
4817 if (of_address_to_resource(np, 0, &i2o_res)) {
4818 of_node_put(np);
4819 return -EINVAL;
4820 }
4821
4822 i2o_reg = of_iomap(np, 0);
4823 if (!i2o_reg) {
4824 pr_err("%s: failed to map I2O registers\n", __func__);
4825 of_node_put(np);
4826 return -EINVAL;
4827 }
4828
4829 /* Get I2O DCRs base */
4830 dcr_base = dcr_resource_start(np, 0);
4831 dcr_len = dcr_resource_len(np, 0);
4832 if (!dcr_base && !dcr_len) {
4833 pr_err("%s: can't get DCR registers base/len!\n",
4834 np->full_name);
4835 of_node_put(np);
4836 iounmap(i2o_reg);
4837 return -ENODEV;
4838 }
4839
4840 i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4841 if (!DCR_MAP_OK(i2o_dcr_host)) {
4842 pr_err("%s: failed to map DCRs!\n", np->full_name);
4843 of_node_put(np);
4844 iounmap(i2o_reg);
4845 return -ENODEV;
4846 }
4847 of_node_put(np);
4848
4849 /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4850 * the base address of FIFO memory space.
4851 * Actually we need twice more physical memory than programmed in the
4852 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4853 */
4854 ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4855 GFP_KERNEL);
4856 if (!ppc440spe_dma_fifo_buf) {
4857 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4858 iounmap(i2o_reg);
4859 dcr_unmap(i2o_dcr_host, dcr_len);
4860 return -ENOMEM;
4861 }
4862
4863 /*
4864 * Configure h/w
4865 */
4866 /* Reset I2O/DMA */
4867 mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4868 mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4869
4870 /* Setup the base address of mmaped registers */
4871 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4872 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4873 I2O_REG_ENABLE);
4874 dcr_unmap(i2o_dcr_host, dcr_len);
4875
4876 /* Setup FIFO memory space base address */
4877 iowrite32(0, &i2o_reg->ifbah);
4878 iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4879
4880 /* set zero FIFO size for I2O, so the whole
4881 * ppc440spe_dma_fifo_buf is used by DMAs.
4882 * DMAx_FIFOs will be configured while probe.
4883 */
4884 iowrite32(0, &i2o_reg->ifsiz);
4885 iounmap(i2o_reg);
4886
4887 /* To prepare WXOR/RXOR functionality we need access to
4888 * Memory Queue Module DCRs (finally it will be enabled
4889 * via /sys interface of the ppc440spe ADMA driver).
4890 */
4891 np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4892 if (!np) {
4893 pr_err("%s: can't find MQ device tree node\n",
4894 __func__);
4895 ret = -ENODEV;
4896 goto out_free;
4897 }
4898
4899 /* Get MQ DCRs base */
4900 dcr_base = dcr_resource_start(np, 0);
4901 dcr_len = dcr_resource_len(np, 0);
4902 if (!dcr_base && !dcr_len) {
4903 pr_err("%s: can't get DCR registers base/len!\n",
4904 np->full_name);
4905 ret = -ENODEV;
4906 goto out_mq;
4907 }
4908
4909 ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4910 if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4911 pr_err("%s: failed to map DCRs!\n", np->full_name);
4912 ret = -ENODEV;
4913 goto out_mq;
4914 }
4915 of_node_put(np);
4916 ppc440spe_mq_dcr_len = dcr_len;
4917
4918 /* Set HB alias */
4919 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4920
4921 /* Set:
4922 * - LL transaction passing limit to 1;
4923 * - Memory controller cycle limit to 1;
4924 * - Galois Polynomial to 0x14d (default)
4925 */
4926 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4927 (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4928 (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4929
4930 atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4931 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4932 ppc440spe_adma_devices[i] = -1;
4933
4934 return 0;
4935
4936out_mq:
4937 of_node_put(np);
4938out_free:
4939 kfree(ppc440spe_dma_fifo_buf);
4940 return ret;
4941}
4942
4943static struct of_device_id __devinitdata ppc440spe_adma_of_match[] = {
4944 { .compatible = "ibm,dma-440spe", },
4945 { .compatible = "amcc,xor-accelerator", },
4946 {},
4947};
4948MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4949
4950static struct of_platform_driver ppc440spe_adma_driver = {
4951 .match_table = ppc440spe_adma_of_match,
4952 .probe = ppc440spe_adma_probe,
4953 .remove = __devexit_p(ppc440spe_adma_remove),
4954 .driver = {
4955 .name = "PPC440SP(E)-ADMA",
4956 .owner = THIS_MODULE,
4957 },
4958};
4959
4960static __init int ppc440spe_adma_init(void)
4961{
4962 int ret;
4963
4964 ret = ppc440spe_configure_raid_devices();
4965 if (ret)
4966 return ret;
4967
4968 ret = of_register_platform_driver(&ppc440spe_adma_driver);
4969 if (ret) {
4970 pr_err("%s: failed to register platform driver\n",
4971 __func__);
4972 goto out_reg;
4973 }
4974
4975 /* Initialization status */
4976 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4977 &driver_attr_devices);
4978 if (ret)
4979 goto out_dev;
4980
4981 /* RAID-6 h/w enable entry */
4982 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4983 &driver_attr_enable);
4984 if (ret)
4985 goto out_en;
4986
4987 /* GF polynomial to use */
4988 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4989 &driver_attr_poly);
4990 if (!ret)
4991 return ret;
4992
4993 driver_remove_file(&ppc440spe_adma_driver.driver,
4994 &driver_attr_enable);
4995out_en:
4996 driver_remove_file(&ppc440spe_adma_driver.driver,
4997 &driver_attr_devices);
4998out_dev:
4999 /* User will not be able to enable h/w RAID-6 */
5000 pr_err("%s: failed to create RAID-6 driver interface\n",
5001 __func__);
5002 of_unregister_platform_driver(&ppc440spe_adma_driver);
5003out_reg:
5004 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
5005 kfree(ppc440spe_dma_fifo_buf);
5006 return ret;
5007}
5008
5009static void __exit ppc440spe_adma_exit(void)
5010{
5011 driver_remove_file(&ppc440spe_adma_driver.driver,
5012 &driver_attr_poly);
5013 driver_remove_file(&ppc440spe_adma_driver.driver,
5014 &driver_attr_enable);
5015 driver_remove_file(&ppc440spe_adma_driver.driver,
5016 &driver_attr_devices);
5017 of_unregister_platform_driver(&ppc440spe_adma_driver);
5018 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
5019 kfree(ppc440spe_dma_fifo_buf);
5020}
5021
5022arch_initcall(ppc440spe_adma_init);
5023module_exit(ppc440spe_adma_exit);
5024
5025MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
5026MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
5027MODULE_LICENSE("GPL");
diff --git a/drivers/dma/ppc4xx/adma.h b/drivers/dma/ppc4xx/adma.h
new file mode 100644
index 000000000000..8ada5a812e3b
--- /dev/null
+++ b/drivers/dma/ppc4xx/adma.h
@@ -0,0 +1,195 @@
1/*
2 * 2006-2009 (C) DENX Software Engineering.
3 *
4 * Author: Yuri Tikhonov <yur@emcraft.com>
5 *
6 * This file is licensed under the terms of the GNU General Public License
7 * version 2. This program is licensed "as is" without any warranty of
8 * any kind, whether express or implied.
9 */
10
11#ifndef _PPC440SPE_ADMA_H
12#define _PPC440SPE_ADMA_H
13
14#include <linux/types.h>
15#include "dma.h"
16#include "xor.h"
17
18#define to_ppc440spe_adma_chan(chan) \
19 container_of(chan, struct ppc440spe_adma_chan, common)
20#define to_ppc440spe_adma_device(dev) \
21 container_of(dev, struct ppc440spe_adma_device, common)
22#define tx_to_ppc440spe_adma_slot(tx) \
23 container_of(tx, struct ppc440spe_adma_desc_slot, async_tx)
24
25/* Default polynomial (for 440SP is only available) */
26#define PPC440SPE_DEFAULT_POLY 0x4d
27
28#define PPC440SPE_ADMA_ENGINES_NUM (XOR_ENGINES_NUM + DMA_ENGINES_NUM)
29
30#define PPC440SPE_ADMA_WATCHDOG_MSEC 3
31#define PPC440SPE_ADMA_THRESHOLD 1
32
33#define PPC440SPE_DMA0_ID 0
34#define PPC440SPE_DMA1_ID 1
35#define PPC440SPE_XOR_ID 2
36
37#define PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT 0xFFFFFFUL
38/* this is the XOR_CBBCR width */
39#define PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT (1 << 31)
40#define PPC440SPE_ADMA_ZERO_SUM_MAX_BYTE_COUNT PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT
41
42#define PPC440SPE_RXOR_RUN 0
43
44#define MQ0_CF2H_RXOR_BS_MASK 0x1FF
45
46#undef ADMA_LL_DEBUG
47
48/**
49 * struct ppc440spe_adma_device - internal representation of an ADMA device
50 * @dev: device
51 * @dma_reg: base for DMAx register access
52 * @xor_reg: base for XOR register access
53 * @i2o_reg: base for I2O register access
54 * @id: HW ADMA Device selector
55 * @dma_desc_pool_virt: base of DMA descriptor region (CPU address)
56 * @dma_desc_pool: base of DMA descriptor region (DMA address)
57 * @pool_size: size of the pool
58 * @irq: DMAx or XOR irq number
59 * @err_irq: DMAx error irq number
60 * @common: embedded struct dma_device
61 */
62struct ppc440spe_adma_device {
63 struct device *dev;
64 struct dma_regs __iomem *dma_reg;
65 struct xor_regs __iomem *xor_reg;
66 struct i2o_regs __iomem *i2o_reg;
67 int id;
68 void *dma_desc_pool_virt;
69 dma_addr_t dma_desc_pool;
70 size_t pool_size;
71 int irq;
72 int err_irq;
73 struct dma_device common;
74};
75
76/**
77 * struct ppc440spe_adma_chan - internal representation of an ADMA channel
78 * @lock: serializes enqueue/dequeue operations to the slot pool
79 * @device: parent device
80 * @chain: device chain view of the descriptors
81 * @common: common dmaengine channel object members
82 * @all_slots: complete domain of slots usable by the channel
83 * @pending: allows batching of hardware operations
84 * @completed_cookie: identifier for the most recently completed operation
85 * @slots_allocated: records the actual size of the descriptor slot pool
86 * @hw_chain_inited: h/w descriptor chain initialization flag
87 * @irq_tasklet: bottom half where ppc440spe_adma_slot_cleanup runs
88 * @needs_unmap: if buffers should not be unmapped upon final processing
89 * @pdest_page: P destination page for async validate operation
90 * @qdest_page: Q destination page for async validate operation
91 * @pdest: P dma addr for async validate operation
92 * @qdest: Q dma addr for async validate operation
93 */
94struct ppc440spe_adma_chan {
95 spinlock_t lock;
96 struct ppc440spe_adma_device *device;
97 struct list_head chain;
98 struct dma_chan common;
99 struct list_head all_slots;
100 struct ppc440spe_adma_desc_slot *last_used;
101 int pending;
102 dma_cookie_t completed_cookie;
103 int slots_allocated;
104 int hw_chain_inited;
105 struct tasklet_struct irq_tasklet;
106 u8 needs_unmap;
107 struct page *pdest_page;
108 struct page *qdest_page;
109 dma_addr_t pdest;
110 dma_addr_t qdest;
111};
112
113struct ppc440spe_rxor {
114 u32 addrl;
115 u32 addrh;
116 int len;
117 int xor_count;
118 int addr_count;
119 int desc_count;
120 int state;
121};
122
123/**
124 * struct ppc440spe_adma_desc_slot - PPC440SPE-ADMA software descriptor
125 * @phys: hardware address of the hardware descriptor chain
126 * @group_head: first operation in a transaction
127 * @hw_next: pointer to the next descriptor in chain
128 * @async_tx: support for the async_tx api
129 * @slot_node: node on the iop_adma_chan.all_slots list
130 * @chain_node: node on the op_adma_chan.chain list
131 * @group_list: list of slots that make up a multi-descriptor transaction
132 * for example transfer lengths larger than the supported hw max
133 * @unmap_len: transaction bytecount
134 * @hw_desc: virtual address of the hardware descriptor chain
135 * @stride: currently chained or not
136 * @idx: pool index
137 * @slot_cnt: total slots used in an transaction (group of operations)
138 * @src_cnt: number of sources set in this descriptor
139 * @dst_cnt: number of destinations set in the descriptor
140 * @slots_per_op: number of slots per operation
141 * @descs_per_op: number of slot per P/Q operation see comment
142 * for ppc440spe_prep_dma_pqxor function
143 * @flags: desc state/type
144 * @reverse_flags: 1 if a corresponding rxor address uses reversed address order
145 * @xor_check_result: result of zero sum
146 * @crc32_result: result crc calculation
147 */
148struct ppc440spe_adma_desc_slot {
149 dma_addr_t phys;
150 struct ppc440spe_adma_desc_slot *group_head;
151 struct ppc440spe_adma_desc_slot *hw_next;
152 struct dma_async_tx_descriptor async_tx;
153 struct list_head slot_node;
154 struct list_head chain_node; /* node in channel ops list */
155 struct list_head group_list; /* list */
156 unsigned int unmap_len;
157 void *hw_desc;
158 u16 stride;
159 u16 idx;
160 u16 slot_cnt;
161 u8 src_cnt;
162 u8 dst_cnt;
163 u8 slots_per_op;
164 u8 descs_per_op;
165 unsigned long flags;
166 unsigned long reverse_flags[8];
167
168#define PPC440SPE_DESC_INT 0 /* generate interrupt on complete */
169#define PPC440SPE_ZERO_P 1 /* clear P destionaion */
170#define PPC440SPE_ZERO_Q 2 /* clear Q destination */
171#define PPC440SPE_COHERENT 3 /* src/dst are coherent */
172
173#define PPC440SPE_DESC_WXOR 4 /* WXORs are in chain */
174#define PPC440SPE_DESC_RXOR 5 /* RXOR is in chain */
175
176#define PPC440SPE_DESC_RXOR123 8 /* CDB for RXOR123 operation */
177#define PPC440SPE_DESC_RXOR124 9 /* CDB for RXOR124 operation */
178#define PPC440SPE_DESC_RXOR125 10 /* CDB for RXOR125 operation */
179#define PPC440SPE_DESC_RXOR12 11 /* CDB for RXOR12 operation */
180#define PPC440SPE_DESC_RXOR_REV 12 /* CDB has srcs in reversed order */
181
182#define PPC440SPE_DESC_PCHECK 13
183#define PPC440SPE_DESC_QCHECK 14
184
185#define PPC440SPE_DESC_RXOR_MSK 0x3
186
187 struct ppc440spe_rxor rxor_cursor;
188
189 union {
190 u32 *xor_check_result;
191 u32 *crc32_result;
192 };
193};
194
195#endif /* _PPC440SPE_ADMA_H */
diff --git a/drivers/dma/ppc4xx/dma.h b/drivers/dma/ppc4xx/dma.h
new file mode 100644
index 000000000000..bcde2df2f373
--- /dev/null
+++ b/drivers/dma/ppc4xx/dma.h
@@ -0,0 +1,223 @@
1/*
2 * 440SPe's DMA engines support header file
3 *
4 * 2006-2009 (C) DENX Software Engineering.
5 *
6 * Author: Yuri Tikhonov <yur@emcraft.com>
7 *
8 * This file is licensed under the term of the GNU General Public License
9 * version 2. The program licensed "as is" without any warranty of any
10 * kind, whether express or implied.
11 */
12
13#ifndef _PPC440SPE_DMA_H
14#define _PPC440SPE_DMA_H
15
16#include <linux/types.h>
17
18/* Number of elements in the array with statical CDBs */
19#define MAX_STAT_DMA_CDBS 16
20/* Number of DMA engines available on the contoller */
21#define DMA_ENGINES_NUM 2
22
23/* Maximum h/w supported number of destinations */
24#define DMA_DEST_MAX_NUM 2
25
26/* FIFO's params */
27#define DMA0_FIFO_SIZE 0x1000
28#define DMA1_FIFO_SIZE 0x1000
29#define DMA_FIFO_ENABLE (1<<12)
30
31/* DMA Configuration Register. Data Transfer Engine PLB Priority: */
32#define DMA_CFG_DXEPR_LP (0<<26)
33#define DMA_CFG_DXEPR_HP (3<<26)
34#define DMA_CFG_DXEPR_HHP (2<<26)
35#define DMA_CFG_DXEPR_HHHP (1<<26)
36
37/* DMA Configuration Register. DMA FIFO Manager PLB Priority: */
38#define DMA_CFG_DFMPP_LP (0<<23)
39#define DMA_CFG_DFMPP_HP (3<<23)
40#define DMA_CFG_DFMPP_HHP (2<<23)
41#define DMA_CFG_DFMPP_HHHP (1<<23)
42
43/* DMA Configuration Register. Force 64-byte Alignment */
44#define DMA_CFG_FALGN (1 << 19)
45
46/*UIC0:*/
47#define D0CPF_INT (1<<12)
48#define D0CSF_INT (1<<11)
49#define D1CPF_INT (1<<10)
50#define D1CSF_INT (1<<9)
51/*UIC1:*/
52#define DMAE_INT (1<<9)
53
54/* I2O IOP Interrupt Mask Register */
55#define I2O_IOPIM_P0SNE (1<<3)
56#define I2O_IOPIM_P0EM (1<<5)
57#define I2O_IOPIM_P1SNE (1<<6)
58#define I2O_IOPIM_P1EM (1<<8)
59
60/* DMA CDB fields */
61#define DMA_CDB_MSK (0xF)
62#define DMA_CDB_64B_ADDR (1<<2)
63#define DMA_CDB_NO_INT (1<<3)
64#define DMA_CDB_STATUS_MSK (0x3)
65#define DMA_CDB_ADDR_MSK (0xFFFFFFF0)
66
67/* DMA CDB OpCodes */
68#define DMA_CDB_OPC_NO_OP (0x00)
69#define DMA_CDB_OPC_MV_SG1_SG2 (0x01)
70#define DMA_CDB_OPC_MULTICAST (0x05)
71#define DMA_CDB_OPC_DFILL128 (0x24)
72#define DMA_CDB_OPC_DCHECK128 (0x23)
73
74#define DMA_CUED_XOR_BASE (0x10000000)
75#define DMA_CUED_XOR_HB (0x00000008)
76
77#ifdef CONFIG_440SP
78#define DMA_CUED_MULT1_OFF 0
79#define DMA_CUED_MULT2_OFF 8
80#define DMA_CUED_MULT3_OFF 16
81#define DMA_CUED_REGION_OFF 24
82#define DMA_CUED_XOR_WIN_MSK (0xFC000000)
83#else
84#define DMA_CUED_MULT1_OFF 2
85#define DMA_CUED_MULT2_OFF 10
86#define DMA_CUED_MULT3_OFF 18
87#define DMA_CUED_REGION_OFF 26
88#define DMA_CUED_XOR_WIN_MSK (0xF0000000)
89#endif
90
91#define DMA_CUED_REGION_MSK 0x3
92#define DMA_RXOR123 0x0
93#define DMA_RXOR124 0x1
94#define DMA_RXOR125 0x2
95#define DMA_RXOR12 0x3
96
97/* S/G addresses */
98#define DMA_CDB_SG_SRC 1
99#define DMA_CDB_SG_DST1 2
100#define DMA_CDB_SG_DST2 3
101
102/*
103 * DMAx engines Command Descriptor Block Type
104 */
105struct dma_cdb {
106 /*
107 * Basic CDB structure (Table 20-17, p.499, 440spe_um_1_22.pdf)
108 */
109 u8 pad0[2]; /* reserved */
110 u8 attr; /* attributes */
111 u8 opc; /* opcode */
112 u32 sg1u; /* upper SG1 address */
113 u32 sg1l; /* lower SG1 address */
114 u32 cnt; /* SG count, 3B used */
115 u32 sg2u; /* upper SG2 address */
116 u32 sg2l; /* lower SG2 address */
117 u32 sg3u; /* upper SG3 address */
118 u32 sg3l; /* lower SG3 address */
119};
120
121/*
122 * DMAx hardware registers (p.515 in 440SPe UM 1.22)
123 */
124struct dma_regs {
125 u32 cpfpl;
126 u32 cpfph;
127 u32 csfpl;
128 u32 csfph;
129 u32 dsts;
130 u32 cfg;
131 u8 pad0[0x8];
132 u16 cpfhp;
133 u16 cpftp;
134 u16 csfhp;
135 u16 csftp;
136 u8 pad1[0x8];
137 u32 acpl;
138 u32 acph;
139 u32 s1bpl;
140 u32 s1bph;
141 u32 s2bpl;
142 u32 s2bph;
143 u32 s3bpl;
144 u32 s3bph;
145 u8 pad2[0x10];
146 u32 earl;
147 u32 earh;
148 u8 pad3[0x8];
149 u32 seat;
150 u32 sead;
151 u32 op;
152 u32 fsiz;
153};
154
155/*
156 * I2O hardware registers (p.528 in 440SPe UM 1.22)
157 */
158struct i2o_regs {
159 u32 ists;
160 u32 iseat;
161 u32 isead;
162 u8 pad0[0x14];
163 u32 idbel;
164 u8 pad1[0xc];
165 u32 ihis;
166 u32 ihim;
167 u8 pad2[0x8];
168 u32 ihiq;
169 u32 ihoq;
170 u8 pad3[0x8];
171 u32 iopis;
172 u32 iopim;
173 u32 iopiq;
174 u8 iopoq;
175 u8 pad4[3];
176 u16 iiflh;
177 u16 iiflt;
178 u16 iiplh;
179 u16 iiplt;
180 u16 ioflh;
181 u16 ioflt;
182 u16 ioplh;
183 u16 ioplt;
184 u32 iidc;
185 u32 ictl;
186 u32 ifcpp;
187 u8 pad5[0x4];
188 u16 mfac0;
189 u16 mfac1;
190 u16 mfac2;
191 u16 mfac3;
192 u16 mfac4;
193 u16 mfac5;
194 u16 mfac6;
195 u16 mfac7;
196 u16 ifcfh;
197 u16 ifcht;
198 u8 pad6[0x4];
199 u32 iifmc;
200 u32 iodb;
201 u32 iodbc;
202 u32 ifbal;
203 u32 ifbah;
204 u32 ifsiz;
205 u32 ispd0;
206 u32 ispd1;
207 u32 ispd2;
208 u32 ispd3;
209 u32 ihipl;
210 u32 ihiph;
211 u32 ihopl;
212 u32 ihoph;
213 u32 iiipl;
214 u32 iiiph;
215 u32 iiopl;
216 u32 iioph;
217 u32 ifcpl;
218 u32 ifcph;
219 u8 pad7[0x8];
220 u32 iopt;
221};
222
223#endif /* _PPC440SPE_DMA_H */
diff --git a/drivers/dma/ppc4xx/xor.h b/drivers/dma/ppc4xx/xor.h
new file mode 100644
index 000000000000..daed7384daac
--- /dev/null
+++ b/drivers/dma/ppc4xx/xor.h
@@ -0,0 +1,110 @@
1/*
2 * 440SPe's XOR engines support header file
3 *
4 * 2006-2009 (C) DENX Software Engineering.
5 *
6 * Author: Yuri Tikhonov <yur@emcraft.com>
7 *
8 * This file is licensed under the term of the GNU General Public License
9 * version 2. The program licensed "as is" without any warranty of any
10 * kind, whether express or implied.
11 */
12
13#ifndef _PPC440SPE_XOR_H
14#define _PPC440SPE_XOR_H
15
16#include <linux/types.h>
17
18/* Number of XOR engines available on the contoller */
19#define XOR_ENGINES_NUM 1
20
21/* Number of operands supported in the h/w */
22#define XOR_MAX_OPS 16
23
24/*
25 * XOR Command Block Control Register bits
26 */
27#define XOR_CBCR_LNK_BIT (1<<31) /* link present */
28#define XOR_CBCR_TGT_BIT (1<<30) /* target present */
29#define XOR_CBCR_CBCE_BIT (1<<29) /* command block compete enable */
30#define XOR_CBCR_RNZE_BIT (1<<28) /* result not zero enable */
31#define XOR_CBCR_XNOR_BIT (1<<15) /* XOR/XNOR */
32#define XOR_CDCR_OAC_MSK (0x7F) /* operand address count */
33
34/*
35 * XORCore Status Register bits
36 */
37#define XOR_SR_XCP_BIT (1<<31) /* core processing */
38#define XOR_SR_ICB_BIT (1<<17) /* invalid CB */
39#define XOR_SR_IC_BIT (1<<16) /* invalid command */
40#define XOR_SR_IPE_BIT (1<<15) /* internal parity error */
41#define XOR_SR_RNZ_BIT (1<<2) /* result not Zero */
42#define XOR_SR_CBC_BIT (1<<1) /* CB complete */
43#define XOR_SR_CBLC_BIT (1<<0) /* CB list complete */
44
45/*
46 * XORCore Control Set and Reset Register bits
47 */
48#define XOR_CRSR_XASR_BIT (1<<31) /* soft reset */
49#define XOR_CRSR_XAE_BIT (1<<30) /* enable */
50#define XOR_CRSR_RCBE_BIT (1<<29) /* refetch CB enable */
51#define XOR_CRSR_PAUS_BIT (1<<28) /* pause */
52#define XOR_CRSR_64BA_BIT (1<<27) /* 64/32 CB format */
53#define XOR_CRSR_CLP_BIT (1<<25) /* continue list processing */
54
55/*
56 * XORCore Interrupt Enable Register
57 */
58#define XOR_IE_ICBIE_BIT (1<<17) /* Invalid Command Block IRQ Enable */
59#define XOR_IE_ICIE_BIT (1<<16) /* Invalid Command IRQ Enable */
60#define XOR_IE_RPTIE_BIT (1<<14) /* Read PLB Timeout Error IRQ Enable */
61#define XOR_IE_CBCIE_BIT (1<<1) /* CB complete interrupt enable */
62#define XOR_IE_CBLCI_BIT (1<<0) /* CB list complete interrupt enable */
63
64/*
65 * XOR Accelerator engine Command Block Type
66 */
67struct xor_cb {
68 /*
69 * Basic 64-bit format XOR CB (Table 19-1, p.463, 440spe_um_1_22.pdf)
70 */
71 u32 cbc; /* control */
72 u32 cbbc; /* byte count */
73 u32 cbs; /* status */
74 u8 pad0[4]; /* reserved */
75 u32 cbtah; /* target address high */
76 u32 cbtal; /* target address low */
77 u32 cblah; /* link address high */
78 u32 cblal; /* link address low */
79 struct {
80 u32 h;
81 u32 l;
82 } __attribute__ ((packed)) ops[16];
83} __attribute__ ((packed));
84
85/*
86 * XOR hardware registers Table 19-3, UM 1.22
87 */
88struct xor_regs {
89 u32 op_ar[16][2]; /* operand address[0]-high,[1]-low registers */
90 u8 pad0[352]; /* reserved */
91 u32 cbcr; /* CB control register */
92 u32 cbbcr; /* CB byte count register */
93 u32 cbsr; /* CB status register */
94 u8 pad1[4]; /* reserved */
95 u32 cbtahr; /* operand target address high register */
96 u32 cbtalr; /* operand target address low register */
97 u32 cblahr; /* CB link address high register */
98 u32 cblalr; /* CB link address low register */
99 u32 crsr; /* control set register */
100 u32 crrr; /* control reset register */
101 u32 ccbahr; /* current CB address high register */
102 u32 ccbalr; /* current CB address low register */
103 u32 plbr; /* PLB configuration register */
104 u32 ier; /* interrupt enable register */
105 u32 pecr; /* parity error count register */
106 u32 sr; /* status register */
107 u32 revidr; /* revision ID register */
108};
109
110#endif /* _PPC440SPE_XOR_H */
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-25 12:22:29 -0400