aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2009-12-16 13:28:56 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2009-12-16 13:28:56 -0500
commit7949456b1b96924c2d9ae5aea5fa7d4c81c946ed (patch)
tree819e64dcd686c8b53c698c164aea96a002e8b5f8
parent60d9aa758c00f20ade0cb1951f6a934f628dd2d7 (diff)
parent12458ea06efd7b44281e68fe59c950ec7d59c649 (diff)
Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx
* 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx: ppc440spe-adma: adds updated ppc440spe adma driver iop-adma.c: use resource_size() dmaengine: clarify the meaning of the DMA_CTRL_ACK flag sh: stylistic improvements for the DMA driver dmaengine: fix dmatest to verify minimum transfer length and test buffer size sh: DMA driver has to specify its alignment requirements Add COH 901 318 DMA block driver v5
Diffstat
-rw-r--r--Documentation/powerpc/dts-bindings/4xx/ppc440spe-adma.txt93
-rw-r--r--arch/arm/mach-u300/include/mach/coh901318.h281
-rw-r--r--arch/powerpc/include/asm/async_tx.h47
-rw-r--r--arch/powerpc/include/asm/dcr-regs.h23
-rw-r--r--drivers/dma/Kconfig18
-rw-r--r--drivers/dma/Makefile2
-rw-r--r--drivers/dma/coh901318.c1325
-rw-r--r--drivers/dma/coh901318_lli.c318
-rw-r--r--drivers/dma/coh901318_lli.h124
-rw-r--r--drivers/dma/dmatest.c16
-rw-r--r--drivers/dma/iop-adma.c4
-rw-r--r--drivers/dma/ppc4xx/Makefile1
-rw-r--r--drivers/dma/ppc4xx/adma.c5027
-rw-r--r--drivers/dma/ppc4xx/adma.h195
-rw-r--r--drivers/dma/ppc4xx/dma.h223
-rw-r--r--drivers/dma/ppc4xx/xor.h110
-rw-r--r--drivers/dma/shdma.c36
-rw-r--r--drivers/dma/shdma.h14
-rw-r--r--include/linux/dmaengine.h2
19 files changed, 7828 insertions, 31 deletions
diff --git a/Documentation/powerpc/dts-bindings/4xx/ppc440spe-adma.txt b/Documentation/powerpc/dts-bindings/4xx/ppc440spe-adma.txt
new file mode 100644
index 000000000000..515ebcf1b97d
--- /dev/null
+++ b/Documentation/powerpc/dts-bindings/4xx/ppc440spe-adma.txt
@@ -0,0 +1,93 @@
1PPC440SPe DMA/XOR (DMA Controller and XOR Accelerator)
2
3Device nodes needed for operation of the ppc440spe-adma driver
4are specified hereby. These are I2O/DMA, DMA and XOR nodes
5for DMA engines and Memory Queue Module node. The latter is used
6by ADMA driver for configuration of RAID-6 H/W capabilities of
7the PPC440SPe. In addition to the nodes and properties described
8below, the ranges property of PLB node must specify ranges for
9DMA devices.
10
11 i) The I2O node
12
13 Required properties:
14
15 - compatible : "ibm,i2o-440spe";
16 - reg : <registers mapping>
17 - dcr-reg : <DCR registers range>
18
19 Example:
20
21 I2O: i2o@400100000 {
22 compatible = "ibm,i2o-440spe";
23 reg = <0x00000004 0x00100000 0x100>;
24 dcr-reg = <0x060 0x020>;
25 };
26
27
28 ii) The DMA node
29
30 Required properties:
31
32 - compatible : "ibm,dma-440spe";
33 - cell-index : 1 cell, hardware index of the DMA engine
34 (typically 0x0 and 0x1 for DMA0 and DMA1)
35 - reg : <registers mapping>
36 - dcr-reg : <DCR registers range>
37 - interrupts : <interrupt mapping for DMA0/1 interrupts sources:
38 2 sources: DMAx CS FIFO Needs Service IRQ (on UIC0)
39 and DMA Error IRQ (on UIC1). The latter is common
40 for both DMA engines>.
41 - interrupt-parent : needed for interrupt mapping
42
43 Example:
44
45 DMA0: dma0@400100100 {
46 compatible = "ibm,dma-440spe";
47 cell-index = <0>;
48 reg = <0x00000004 0x00100100 0x100>;
49 dcr-reg = <0x060 0x020>;
50 interrupt-parent = <&DMA0>;
51 interrupts = <0 1>;
52 #interrupt-cells = <1>;
53 #address-cells = <0>;
54 #size-cells = <0>;
55 interrupt-map = <
56 0 &UIC0 0x14 4
57 1 &UIC1 0x16 4>;
58 };
59
60
61 iii) XOR Accelerator node
62
63 Required properties:
64
65 - compatible : "amcc,xor-accelerator";
66 - reg : <registers mapping>
67 - interrupts : <interrupt mapping for XOR interrupt source>
68 - interrupt-parent : for interrupt mapping
69
70 Example:
71
72 xor-accel@400200000 {
73 compatible = "amcc,xor-accelerator";
74 reg = <0x00000004 0x00200000 0x400>;
75 interrupt-parent = <&UIC1>;
76 interrupts = <0x1f 4>;
77 };
78
79
80 iv) Memory Queue Module node
81
82 Required properties:
83
84 - compatible : "ibm,mq-440spe";
85 - dcr-reg : <DCR registers range>
86
87 Example:
88
89 MQ0: mq {
90 compatible = "ibm,mq-440spe";
91 dcr-reg = <0x040 0x020>;
92 };
93
diff --git a/arch/arm/mach-u300/include/mach/coh901318.h b/arch/arm/mach-u300/include/mach/coh901318.h
new file mode 100644
index 000000000000..f4cfee9c7d28
--- /dev/null
+++ b/arch/arm/mach-u300/include/mach/coh901318.h
@@ -0,0 +1,281 @@
1/*
2 *
3 * include/linux/coh901318.h
4 *
5 *
6 * Copyright (C) 2007-2009 ST-Ericsson
7 * License terms: GNU General Public License (GPL) version 2
8 * DMA driver for COH 901 318
9 * Author: Per Friden <per.friden@stericsson.com>
10 */
11
12#ifndef COH901318_H
13#define COH901318_H
14
15#include <linux/device.h>
16#include <linux/dmaengine.h>
17
18#define MAX_DMA_PACKET_SIZE_SHIFT 11
19#define MAX_DMA_PACKET_SIZE (1 << MAX_DMA_PACKET_SIZE_SHIFT)
20
21/**
22 * struct coh901318_lli - linked list item for DMAC
23 * @control: control settings for DMAC
24 * @src_addr: transfer source address
25 * @dst_addr: transfer destination address
26 * @link_addr: physical address to next lli
27 * @virt_link_addr: virtual addres of next lli (only used by pool_free)
28 * @phy_this: physical address of current lli (only used by pool_free)
29 */
30struct coh901318_lli {
31 u32 control;
32 dma_addr_t src_addr;
33 dma_addr_t dst_addr;
34 dma_addr_t link_addr;
35
36 void *virt_link_addr;
37 dma_addr_t phy_this;
38};
39/**
40 * struct coh901318_params - parameters for DMAC configuration
41 * @config: DMA config register
42 * @ctrl_lli_last: DMA control register for the last lli in the list
43 * @ctrl_lli: DMA control register for an lli
44 * @ctrl_lli_chained: DMA control register for a chained lli
45 */
46struct coh901318_params {
47 u32 config;
48 u32 ctrl_lli_last;
49 u32 ctrl_lli;
50 u32 ctrl_lli_chained;
51};
52/**
53 * struct coh_dma_channel - dma channel base
54 * @name: ascii name of dma channel
55 * @number: channel id number
56 * @desc_nbr_max: number of preallocated descriptortors
57 * @priority_high: prio of channel, 0 low otherwise high.
58 * @param: configuration parameters
59 * @dev_addr: physical address of periphal connected to channel
60 */
61struct coh_dma_channel {
62 const char name[32];
63 const int number;
64 const int desc_nbr_max;
65 const int priority_high;
66 const struct coh901318_params param;
67 const dma_addr_t dev_addr;
68};
69
70/**
71 * dma_access_memory_state_t - register dma for memory access
72 *
73 * @dev: The dma device
74 * @active: 1 means dma intends to access memory
75 * 0 means dma wont access memory
76 */
77typedef void (*dma_access_memory_state_t)(struct device *dev,
78 bool active);
79
80/**
81 * struct powersave - DMA power save structure
82 * @lock: lock protecting data in this struct
83 * @started_channels: bit mask indicating active dma channels
84 */
85struct powersave {
86 spinlock_t lock;
87 u64 started_channels;
88};
89/**
90 * struct coh901318_platform - platform arch structure
91 * @chans_slave: specifying dma slave channels
92 * @chans_memcpy: specifying dma memcpy channels
93 * @access_memory_state: requesting DMA memeory access (on / off)
94 * @chan_conf: dma channel configurations
95 * @max_channels: max number of dma chanenls
96 */
97struct coh901318_platform {
98 const int *chans_slave;
99 const int *chans_memcpy;
100 const dma_access_memory_state_t access_memory_state;
101 const struct coh_dma_channel *chan_conf;
102 const int max_channels;
103};
104
105/**
106 * coh901318_get_bytes_left() - Get number of bytes left on a current transfer
107 * @chan: dma channel handle
108 * return number of bytes left, or negative on error
109 */
110u32 coh901318_get_bytes_left(struct dma_chan *chan);
111
112/**
113 * coh901318_stop() - Stops dma transfer
114 * @chan: dma channel handle
115 * return 0 on success otherwise negative value
116 */
117void coh901318_stop(struct dma_chan *chan);
118
119/**
120 * coh901318_continue() - Resumes a stopped dma transfer
121 * @chan: dma channel handle
122 * return 0 on success otherwise negative value
123 */
124void coh901318_continue(struct dma_chan *chan);
125
126/**
127 * coh901318_filter_id() - DMA channel filter function
128 * @chan: dma channel handle
129 * @chan_id: id of dma channel to be filter out
130 *
131 * In dma_request_channel() it specifies what channel id to be requested
132 */
133bool coh901318_filter_id(struct dma_chan *chan, void *chan_id);
134
135/*
136 * DMA Controller - this access the static mappings of the coh901318 dma.
137 *
138 */
139
140#define COH901318_MOD32_MASK (0x1F)
141#define COH901318_WORD_MASK (0xFFFFFFFF)
142/* INT_STATUS - Interrupt Status Registers 32bit (R/-) */
143#define COH901318_INT_STATUS1 (0x0000)
144#define COH901318_INT_STATUS2 (0x0004)
145/* TC_INT_STATUS - Terminal Count Interrupt Status Registers 32bit (R/-) */
146#define COH901318_TC_INT_STATUS1 (0x0008)
147#define COH901318_TC_INT_STATUS2 (0x000C)
148/* TC_INT_CLEAR - Terminal Count Interrupt Clear Registers 32bit (-/W) */
149#define COH901318_TC_INT_CLEAR1 (0x0010)
150#define COH901318_TC_INT_CLEAR2 (0x0014)
151/* RAW_TC_INT_STATUS - Raw Term Count Interrupt Status Registers 32bit (R/-) */
152#define COH901318_RAW_TC_INT_STATUS1 (0x0018)
153#define COH901318_RAW_TC_INT_STATUS2 (0x001C)
154/* BE_INT_STATUS - Bus Error Interrupt Status Registers 32bit (R/-) */
155#define COH901318_BE_INT_STATUS1 (0x0020)
156#define COH901318_BE_INT_STATUS2 (0x0024)
157/* BE_INT_CLEAR - Bus Error Interrupt Clear Registers 32bit (-/W) */
158#define COH901318_BE_INT_CLEAR1 (0x0028)
159#define COH901318_BE_INT_CLEAR2 (0x002C)
160/* RAW_BE_INT_STATUS - Raw Term Count Interrupt Status Registers 32bit (R/-) */
161#define COH901318_RAW_BE_INT_STATUS1 (0x0030)
162#define COH901318_RAW_BE_INT_STATUS2 (0x0034)
163
164/*
165 * CX_CFG - Channel Configuration Registers 32bit (R/W)
166 */
167#define COH901318_CX_CFG (0x0100)
168#define COH901318_CX_CFG_SPACING (0x04)
169/* Channel enable activates tha dma job */
170#define COH901318_CX_CFG_CH_ENABLE (0x00000001)
171#define COH901318_CX_CFG_CH_DISABLE (0x00000000)
172/* Request Mode */
173#define COH901318_CX_CFG_RM_MASK (0x00000006)
174#define COH901318_CX_CFG_RM_MEMORY_TO_MEMORY (0x0 << 1)
175#define COH901318_CX_CFG_RM_PRIMARY_TO_MEMORY (0x1 << 1)
176#define COH901318_CX_CFG_RM_MEMORY_TO_PRIMARY (0x1 << 1)
177#define COH901318_CX_CFG_RM_PRIMARY_TO_SECONDARY (0x3 << 1)
178#define COH901318_CX_CFG_RM_SECONDARY_TO_PRIMARY (0x3 << 1)
179/* Linked channel request field. RM must == 11 */
180#define COH901318_CX_CFG_LCRF_SHIFT 3
181#define COH901318_CX_CFG_LCRF_MASK (0x000001F8)
182#define COH901318_CX_CFG_LCR_DISABLE (0x00000000)
183/* Terminal Counter Interrupt Request Mask */
184#define COH901318_CX_CFG_TC_IRQ_ENABLE (0x00000200)
185#define COH901318_CX_CFG_TC_IRQ_DISABLE (0x00000000)
186/* Bus Error interrupt Mask */
187#define COH901318_CX_CFG_BE_IRQ_ENABLE (0x00000400)
188#define COH901318_CX_CFG_BE_IRQ_DISABLE (0x00000000)
189
190/*
191 * CX_STAT - Channel Status Registers 32bit (R/-)
192 */
193#define COH901318_CX_STAT (0x0200)
194#define COH901318_CX_STAT_SPACING (0x04)
195#define COH901318_CX_STAT_RBE_IRQ_IND (0x00000008)
196#define COH901318_CX_STAT_RTC_IRQ_IND (0x00000004)
197#define COH901318_CX_STAT_ACTIVE (0x00000002)
198#define COH901318_CX_STAT_ENABLED (0x00000001)
199
200/*
201 * CX_CTRL - Channel Control Registers 32bit (R/W)
202 */
203#define COH901318_CX_CTRL (0x0400)
204#define COH901318_CX_CTRL_SPACING (0x10)
205/* Transfer Count Enable */
206#define COH901318_CX_CTRL_TC_ENABLE (0x00001000)
207#define COH901318_CX_CTRL_TC_DISABLE (0x00000000)
208/* Transfer Count Value 0 - 4095 */
209#define COH901318_CX_CTRL_TC_VALUE_MASK (0x00000FFF)
210/* Burst count */
211#define COH901318_CX_CTRL_BURST_COUNT_MASK (0x0000E000)
212#define COH901318_CX_CTRL_BURST_COUNT_64_BYTES (0x7 << 13)
213#define COH901318_CX_CTRL_BURST_COUNT_48_BYTES (0x6 << 13)
214#define COH901318_CX_CTRL_BURST_COUNT_32_BYTES (0x5 << 13)
215#define COH901318_CX_CTRL_BURST_COUNT_16_BYTES (0x4 << 13)
216#define COH901318_CX_CTRL_BURST_COUNT_8_BYTES (0x3 << 13)
217#define COH901318_CX_CTRL_BURST_COUNT_4_BYTES (0x2 << 13)
218#define COH901318_CX_CTRL_BURST_COUNT_2_BYTES (0x1 << 13)
219#define COH901318_CX_CTRL_BURST_COUNT_1_BYTE (0x0 << 13)
220/* Source bus size */
221#define COH901318_CX_CTRL_SRC_BUS_SIZE_MASK (0x00030000)
222#define COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS (0x2 << 16)
223#define COH901318_CX_CTRL_SRC_BUS_SIZE_16_BITS (0x1 << 16)
224#define COH901318_CX_CTRL_SRC_BUS_SIZE_8_BITS (0x0 << 16)
225/* Source address increment */
226#define COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE (0x00040000)
227#define COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE (0x00000000)
228/* Destination Bus Size */
229#define COH901318_CX_CTRL_DST_BUS_SIZE_MASK (0x00180000)
230#define COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS (0x2 << 19)
231#define COH901318_CX_CTRL_DST_BUS_SIZE_16_BITS (0x1 << 19)
232#define COH901318_CX_CTRL_DST_BUS_SIZE_8_BITS (0x0 << 19)
233/* Destination address increment */
234#define COH901318_CX_CTRL_DST_ADDR_INC_ENABLE (0x00200000)
235#define COH901318_CX_CTRL_DST_ADDR_INC_DISABLE (0x00000000)
236/* Master Mode (Master2 is only connected to MSL) */
237#define COH901318_CX_CTRL_MASTER_MODE_MASK (0x00C00000)
238#define COH901318_CX_CTRL_MASTER_MODE_M2R_M1W (0x3 << 22)
239#define COH901318_CX_CTRL_MASTER_MODE_M1R_M2W (0x2 << 22)
240#define COH901318_CX_CTRL_MASTER_MODE_M2RW (0x1 << 22)
241#define COH901318_CX_CTRL_MASTER_MODE_M1RW (0x0 << 22)
242/* Terminal Count flag to PER enable */
243#define COH901318_CX_CTRL_TCP_ENABLE (0x01000000)
244#define COH901318_CX_CTRL_TCP_DISABLE (0x00000000)
245/* Terminal Count flags to CPU enable */
246#define COH901318_CX_CTRL_TC_IRQ_ENABLE (0x02000000)
247#define COH901318_CX_CTRL_TC_IRQ_DISABLE (0x00000000)
248/* Hand shake to peripheral */
249#define COH901318_CX_CTRL_HSP_ENABLE (0x04000000)
250#define COH901318_CX_CTRL_HSP_DISABLE (0x00000000)
251#define COH901318_CX_CTRL_HSS_ENABLE (0x08000000)
252#define COH901318_CX_CTRL_HSS_DISABLE (0x00000000)
253/* DMA mode */
254#define COH901318_CX_CTRL_DDMA_MASK (0x30000000)
255#define COH901318_CX_CTRL_DDMA_LEGACY (0x0 << 28)
256#define COH901318_CX_CTRL_DDMA_DEMAND_DMA1 (0x1 << 28)
257#define COH901318_CX_CTRL_DDMA_DEMAND_DMA2 (0x2 << 28)
258/* Primary Request Data Destination */
259#define COH901318_CX_CTRL_PRDD_MASK (0x40000000)
260#define COH901318_CX_CTRL_PRDD_DEST (0x1 << 30)
261#define COH901318_CX_CTRL_PRDD_SOURCE (0x0 << 30)
262
263/*
264 * CX_SRC_ADDR - Channel Source Address Registers 32bit (R/W)
265 */
266#define COH901318_CX_SRC_ADDR (0x0404)
267#define COH901318_CX_SRC_ADDR_SPACING (0x10)
268
269/*
270 * CX_DST_ADDR - Channel Destination Address Registers 32bit R/W
271 */
272#define COH901318_CX_DST_ADDR (0x0408)
273#define COH901318_CX_DST_ADDR_SPACING (0x10)
274
275/*
276 * CX_LNK_ADDR - Channel Link Address Registers 32bit (R/W)
277 */
278#define COH901318_CX_LNK_ADDR (0x040C)
279#define COH901318_CX_LNK_ADDR_SPACING (0x10)
280#define COH901318_CX_LNK_LINK_IMMEDIATE (0x00000001)
281#endif /* COH901318_H */
diff --git a/arch/powerpc/include/asm/async_tx.h b/arch/powerpc/include/asm/async_tx.h
new file mode 100644
index 000000000000..8b2dc55d01ab
--- /dev/null
+++ b/arch/powerpc/include/asm/async_tx.h
@@ -0,0 +1,47 @@
1/*
2 * Copyright (C) 2008-2009 DENX Software Engineering.
3 *
4 * Author: Yuri Tikhonov <yur@emcraft.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the Free
8 * Software Foundation; either version 2 of the License, or (at your option)
9 * any later version.
10 *
11 * This program is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * this program; if not, write to the Free Software Foundation, Inc., 59
18 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
19 *
20 * The full GNU General Public License is included in this distribution in the
21 * file called COPYING.
22 */
23#ifndef _ASM_POWERPC_ASYNC_TX_H_
24#define _ASM_POWERPC_ASYNC_TX_H_
25
26#if defined(CONFIG_440SPe) || defined(CONFIG_440SP)
27extern struct dma_chan *
28ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
29 struct page **dst_lst, int dst_cnt, struct page **src_lst,
30 int src_cnt, size_t src_sz);
31
32#define async_tx_find_channel(dep, cap, dst_lst, dst_cnt, src_lst, \
33 src_cnt, src_sz) \
34 ppc440spe_async_tx_find_best_channel(cap, dst_lst, dst_cnt, src_lst, \
35 src_cnt, src_sz)
36#else
37
38#define async_tx_find_channel(dep, type, dst, dst_count, src, src_count, len) \
39 __async_tx_find_channel(dep, type)
40
41struct dma_chan *
42__async_tx_find_channel(struct async_submit_ctl *submit,
43 enum dma_transaction_type tx_type);
44
45#endif
46
47#endif
diff --git a/arch/powerpc/include/asm/dcr-regs.h b/arch/powerpc/include/asm/dcr-regs.h
index 828e3aa1f2fc..380274de429f 100644
--- a/arch/powerpc/include/asm/dcr-regs.h
+++ b/arch/powerpc/include/asm/dcr-regs.h
@@ -157,4 +157,27 @@
157#define L2C_SNP_SSR_32G 0x0000f000 157#define L2C_SNP_SSR_32G 0x0000f000
158#define L2C_SNP_ESR 0x00000800 158#define L2C_SNP_ESR 0x00000800
159 159
160/*
161 * DCR register offsets for 440SP/440SPe I2O/DMA controller.
162 * The base address is configured in the device tree.
163 */
164#define DCRN_I2O0_IBAL 0x006
165#define DCRN_I2O0_IBAH 0x007
166#define I2O_REG_ENABLE 0x00000001 /* Enable I2O/DMA access */
167
168/* 440SP/440SPe Software Reset DCR */
169#define DCRN_SDR0_SRST 0x0200
170#define DCRN_SDR0_SRST_I2ODMA (0x80000000 >> 15) /* Reset I2O/DMA */
171
172/* 440SP/440SPe Memory Queue DCR offsets */
173#define DCRN_MQ0_XORBA 0x04
174#define DCRN_MQ0_CF2H 0x06
175#define DCRN_MQ0_CFBHL 0x0f
176#define DCRN_MQ0_BAUH 0x10
177
178/* HB/LL Paths Configuration Register */
179#define MQ0_CFBHL_TPLM 28
180#define MQ0_CFBHL_HBCL 23
181#define MQ0_CFBHL_POLY 15
182
160#endif /* __DCR_REGS_H__ */ 183#endif /* __DCR_REGS_H__ */
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index eb140ff38c27..e02d74b1e892 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -111,6 +111,24 @@ config SH_DMAE
111 help 111 help
112 Enable support for the Renesas SuperH DMA controllers. 112 Enable support for the Renesas SuperH DMA controllers.
113 113
114config COH901318
115 bool "ST-Ericsson COH901318 DMA support"
116 select DMA_ENGINE
117 depends on ARCH_U300
118 help
119 Enable support for ST-Ericsson COH 901 318 DMA.
120
121config AMCC_PPC440SPE_ADMA
122 tristate "AMCC PPC440SPe ADMA support"
123 depends on 440SPe || 440SP
124 select DMA_ENGINE
125 select ARCH_HAS_ASYNC_TX_FIND_CHANNEL
126 help
127 Enable support for the AMCC PPC440SPe RAID engines.
128
129config ARCH_HAS_ASYNC_TX_FIND_CHANNEL
130 bool
131
114config DMA_ENGINE 132config DMA_ENGINE
115 bool 133 bool
116 134
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index eca71ba78ae9..807053d48232 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -10,3 +10,5 @@ obj-$(CONFIG_AT_HDMAC) += at_hdmac.o
10obj-$(CONFIG_MX3_IPU) += ipu/ 10obj-$(CONFIG_MX3_IPU) += ipu/
11obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o 11obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
12obj-$(CONFIG_SH_DMAE) += shdma.o 12obj-$(CONFIG_SH_DMAE) += shdma.o
13obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o
14obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/
diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c
new file mode 100644
index 000000000000..4a99cd94536b
--- /dev/null
+++ b/drivers/dma/coh901318.c
@@ -0,0 +1,1325 @@
1/*
2 * driver/dma/coh901318.c
3 *
4 * Copyright (C) 2007-2009 ST-Ericsson
5 * License terms: GNU General Public License (GPL) version 2
6 * DMA driver for COH 901 318
7 * Author: Per Friden <per.friden@stericsson.com>
8 */
9
10#include <linux/init.h>
11#include <linux/module.h>
12#include <linux/kernel.h> /* printk() */
13#include <linux/fs.h> /* everything... */
14#include <linux/slab.h> /* kmalloc() */
15#include <linux/dmaengine.h>
16#include <linux/platform_device.h>
17#include <linux/device.h>
18#include <linux/irqreturn.h>
19#include <linux/interrupt.h>
20#include <linux/io.h>
21#include <linux/uaccess.h>
22#include <linux/debugfs.h>
23#include <mach/coh901318.h>
24
25#include "coh901318_lli.h"
26
27#define COHC_2_DEV(cohc) (&cohc->chan.dev->device)
28
29#ifdef VERBOSE_DEBUG
30#define COH_DBG(x) ({ if (1) x; 0; })
31#else
32#define COH_DBG(x) ({ if (0) x; 0; })
33#endif
34
35struct coh901318_desc {
36 struct dma_async_tx_descriptor desc;
37 struct list_head node;
38 struct scatterlist *sg;
39 unsigned int sg_len;
40 struct coh901318_lli *data;
41 enum dma_data_direction dir;
42 int pending_irqs;
43 unsigned long flags;
44};
45
46struct coh901318_base {
47 struct device *dev;
48 void __iomem *virtbase;
49 struct coh901318_pool pool;
50 struct powersave pm;
51 struct dma_device dma_slave;
52 struct dma_device dma_memcpy;
53 struct coh901318_chan *chans;
54 struct coh901318_platform *platform;
55};
56
57struct coh901318_chan {
58 spinlock_t lock;
59 int allocated;
60 int completed;
61 int id;
62 int stopped;
63
64 struct work_struct free_work;
65 struct dma_chan chan;
66
67 struct tasklet_struct tasklet;
68
69 struct list_head active;
70 struct list_head queue;
71 struct list_head free;
72
73 unsigned long nbr_active_done;
74 unsigned long busy;
75 int pending_irqs;
76
77 struct coh901318_base *base;
78};
79
80static void coh901318_list_print(struct coh901318_chan *cohc,
81 struct coh901318_lli *lli)
82{
83 struct coh901318_lli *l;
84 dma_addr_t addr = virt_to_phys(lli);
85 int i = 0;
86
87 while (addr) {
88 l = phys_to_virt(addr);
89 dev_vdbg(COHC_2_DEV(cohc), "i %d, lli %p, ctrl 0x%x, src 0x%x"
90 ", dst 0x%x, link 0x%x link_virt 0x%p\n",
91 i, l, l->control, l->src_addr, l->dst_addr,
92 l->link_addr, phys_to_virt(l->link_addr));
93 i++;
94 addr = l->link_addr;
95 }
96}
97
98#ifdef CONFIG_DEBUG_FS
99
100#define COH901318_DEBUGFS_ASSIGN(x, y) (x = y)
101
102static struct coh901318_base *debugfs_dma_base;
103static struct dentry *dma_dentry;
104
105static int coh901318_debugfs_open(struct inode *inode, struct file *file)
106{
107
108 file->private_data = inode->i_private;
109 return 0;
110}
111
112static int coh901318_debugfs_read(struct file *file, char __user *buf,
113 size_t count, loff_t *f_pos)
114{
115 u64 started_channels = debugfs_dma_base->pm.started_channels;
116 int pool_count = debugfs_dma_base->pool.debugfs_pool_counter;
117 int i;
118 int ret = 0;
119 char *dev_buf;
120 char *tmp;
121 int dev_size;
122
123 dev_buf = kmalloc(4*1024, GFP_KERNEL);
124 if (dev_buf == NULL)
125 goto err_kmalloc;
126 tmp = dev_buf;
127
128 tmp += sprintf(tmp, "DMA -- enable dma channels\n");
129
130 for (i = 0; i < debugfs_dma_base->platform->max_channels; i++)
131 if (started_channels & (1 << i))
132 tmp += sprintf(tmp, "channel %d\n", i);
133
134 tmp += sprintf(tmp, "Pool alloc nbr %d\n", pool_count);
135 dev_size = tmp - dev_buf;
136
137 /* No more to read if offset != 0 */
138 if (*f_pos > dev_size)
139 goto out;
140
141 if (count > dev_size - *f_pos)
142 count = dev_size - *f_pos;
143
144 if (copy_to_user(buf, dev_buf + *f_pos, count))
145 ret = -EINVAL;
146 ret = count;
147 *f_pos += count;
148
149 out:
150 kfree(dev_buf);
151 return ret;
152
153 err_kmalloc:
154 return 0;
155}
156
157static const struct file_operations coh901318_debugfs_status_operations = {
158 .owner = THIS_MODULE,
159 .open = coh901318_debugfs_open,
160 .read = coh901318_debugfs_read,
161};
162
163
164static int __init init_coh901318_debugfs(void)
165{
166
167 dma_dentry = debugfs_create_dir("dma", NULL);
168
169 (void) debugfs_create_file("status",
170 S_IFREG | S_IRUGO,
171 dma_dentry, NULL,
172 &coh901318_debugfs_status_operations);
173 return 0;
174}
175
176static void __exit exit_coh901318_debugfs(void)
177{
178 debugfs_remove_recursive(dma_dentry);
179}
180
181module_init(init_coh901318_debugfs);
182module_exit(exit_coh901318_debugfs);
183#else
184
185#define COH901318_DEBUGFS_ASSIGN(x, y)
186
187#endif /* CONFIG_DEBUG_FS */
188
189static inline struct coh901318_chan *to_coh901318_chan(struct dma_chan *chan)
190{
191 return container_of(chan, struct coh901318_chan, chan);
192}
193
194static inline dma_addr_t
195cohc_dev_addr(struct coh901318_chan *cohc)
196{
197 return cohc->base->platform->chan_conf[cohc->id].dev_addr;
198}
199
200static inline const struct coh901318_params *
201cohc_chan_param(struct coh901318_chan *cohc)
202{
203 return &cohc->base->platform->chan_conf[cohc->id].param;
204}
205
206static inline const struct coh_dma_channel *
207cohc_chan_conf(struct coh901318_chan *cohc)
208{
209 return &cohc->base->platform->chan_conf[cohc->id];
210}
211
212static void enable_powersave(struct coh901318_chan *cohc)
213{
214 unsigned long flags;
215 struct powersave *pm = &cohc->base->pm;
216
217 spin_lock_irqsave(&pm->lock, flags);
218
219 pm->started_channels &= ~(1ULL << cohc->id);
220
221 if (!pm->started_channels) {
222 /* DMA no longer intends to access memory */
223 cohc->base->platform->access_memory_state(cohc->base->dev,
224 false);
225 }
226
227 spin_unlock_irqrestore(&pm->lock, flags);
228}
229static void disable_powersave(struct coh901318_chan *cohc)
230{
231 unsigned long flags;
232 struct powersave *pm = &cohc->base->pm;
233
234 spin_lock_irqsave(&pm->lock, flags);
235
236 if (!pm->started_channels) {
237 /* DMA intends to access memory */
238 cohc->base->platform->access_memory_state(cohc->base->dev,
239 true);
240 }
241
242 pm->started_channels |= (1ULL << cohc->id);
243
244 spin_unlock_irqrestore(&pm->lock, flags);
245}
246
247static inline int coh901318_set_ctrl(struct coh901318_chan *cohc, u32 control)
248{
249 int channel = cohc->id;
250 void __iomem *virtbase = cohc->base->virtbase;
251
252 writel(control,
253 virtbase + COH901318_CX_CTRL +
254 COH901318_CX_CTRL_SPACING * channel);
255 return 0;
256}
257
258static inline int coh901318_set_conf(struct coh901318_chan *cohc, u32 conf)
259{
260 int channel = cohc->id;
261 void __iomem *virtbase = cohc->base->virtbase;
262
263 writel(conf,
264 virtbase + COH901318_CX_CFG +
265 COH901318_CX_CFG_SPACING*channel);
266 return 0;
267}
268
269
270static int coh901318_start(struct coh901318_chan *cohc)
271{
272 u32 val;
273 int channel = cohc->id;
274 void __iomem *virtbase = cohc->base->virtbase;
275
276 disable_powersave(cohc);
277
278 val = readl(virtbase + COH901318_CX_CFG +
279 COH901318_CX_CFG_SPACING * channel);
280
281 /* Enable channel */
282 val |= COH901318_CX_CFG_CH_ENABLE;
283 writel(val, virtbase + COH901318_CX_CFG +
284 COH901318_CX_CFG_SPACING * channel);
285
286 return 0;
287}
288
289static int coh901318_prep_linked_list(struct coh901318_chan *cohc,
290 struct coh901318_lli *data)
291{
292 int channel = cohc->id;
293 void __iomem *virtbase = cohc->base->virtbase;
294
295 BUG_ON(readl(virtbase + COH901318_CX_STAT +
296 COH901318_CX_STAT_SPACING*channel) &
297 COH901318_CX_STAT_ACTIVE);
298
299 writel(data->src_addr,
300 virtbase + COH901318_CX_SRC_ADDR +
301 COH901318_CX_SRC_ADDR_SPACING * channel);
302
303 writel(data->dst_addr, virtbase +
304 COH901318_CX_DST_ADDR +
305 COH901318_CX_DST_ADDR_SPACING * channel);
306
307 writel(data->link_addr, virtbase + COH901318_CX_LNK_ADDR +
308 COH901318_CX_LNK_ADDR_SPACING * channel);
309
310 writel(data->control, virtbase + COH901318_CX_CTRL +
311 COH901318_CX_CTRL_SPACING * channel);
312
313 return 0;
314}
315static dma_cookie_t
316coh901318_assign_cookie(struct coh901318_chan *cohc,
317 struct coh901318_desc *cohd)
318{
319 dma_cookie_t cookie = cohc->chan.cookie;
320
321 if (++cookie < 0)
322 cookie = 1;
323
324 cohc->chan.cookie = cookie;
325 cohd->desc.cookie = cookie;
326
327 return cookie;
328}
329
330static struct coh901318_desc *
331coh901318_desc_get(struct coh901318_chan *cohc)
332{
333 struct coh901318_desc *desc;
334
335 if (list_empty(&cohc->free)) {
336 /* alloc new desc because we're out of used ones
337 * TODO: alloc a pile of descs instead of just one,
338 * avoid many small allocations.
339 */
340 desc = kmalloc(sizeof(struct coh901318_desc), GFP_NOWAIT);
341 if (desc == NULL)
342 goto out;
343 INIT_LIST_HEAD(&desc->node);
344 } else {
345 /* Reuse an old desc. */
346 desc = list_first_entry(&cohc->free,
347 struct coh901318_desc,
348 node);
349 list_del(&desc->node);
350 }
351
352 out:
353 return desc;
354}
355
356static void
357coh901318_desc_free(struct coh901318_chan *cohc, struct coh901318_desc *cohd)
358{
359 list_add_tail(&cohd->node, &cohc->free);
360}
361
362/* call with irq lock held */
363static void
364coh901318_desc_submit(struct coh901318_chan *cohc, struct coh901318_desc *desc)
365{
366 list_add_tail(&desc->node, &cohc->active);
367
368 BUG_ON(cohc->pending_irqs != 0);
369
370 cohc->pending_irqs = desc->pending_irqs;
371}
372
373static struct coh901318_desc *
374coh901318_first_active_get(struct coh901318_chan *cohc)
375{
376 struct coh901318_desc *d;
377
378 if (list_empty(&cohc->active))
379 return NULL;
380
381 d = list_first_entry(&cohc->active,
382 struct coh901318_desc,
383 node);
384 return d;
385}
386
387static void
388coh901318_desc_remove(struct coh901318_desc *cohd)
389{
390 list_del(&cohd->node);
391}
392
393static void
394coh901318_desc_queue(struct coh901318_chan *cohc, struct coh901318_desc *desc)
395{
396 list_add_tail(&desc->node, &cohc->queue);
397}
398
399static struct coh901318_desc *
400coh901318_first_queued(struct coh901318_chan *cohc)
401{
402 struct coh901318_desc *d;
403
404 if (list_empty(&cohc->queue))
405 return NULL;
406
407 d = list_first_entry(&cohc->queue,
408 struct coh901318_desc,
409 node);
410 return d;
411}
412
413/*
414 * DMA start/stop controls
415 */
416u32 coh901318_get_bytes_left(struct dma_chan *chan)
417{
418 unsigned long flags;
419 u32 ret;
420 struct coh901318_chan *cohc = to_coh901318_chan(chan);
421
422 spin_lock_irqsave(&cohc->lock, flags);
423
424 /* Read transfer count value */
425 ret = readl(cohc->base->virtbase +
426 COH901318_CX_CTRL+COH901318_CX_CTRL_SPACING *
427 cohc->id) & COH901318_CX_CTRL_TC_VALUE_MASK;
428
429 spin_unlock_irqrestore(&cohc->lock, flags);
430
431 return ret;
432}
433EXPORT_SYMBOL(coh901318_get_bytes_left);
434
435
436/* Stops a transfer without losing data. Enables power save.
437 Use this function in conjunction with coh901318_continue(..)
438*/
439void coh901318_stop(struct dma_chan *chan)
440{
441 u32 val;
442 unsigned long flags;
443 struct coh901318_chan *cohc = to_coh901318_chan(chan);
444 int channel = cohc->id;
445 void __iomem *virtbase = cohc->base->virtbase;
446
447 spin_lock_irqsave(&cohc->lock, flags);
448
449 /* Disable channel in HW */
450 val = readl(virtbase + COH901318_CX_CFG +
451 COH901318_CX_CFG_SPACING * channel);
452
453 /* Stopping infinit transfer */
454 if ((val & COH901318_CX_CTRL_TC_ENABLE) == 0 &&
455 (val & COH901318_CX_CFG_CH_ENABLE))
456 cohc->stopped = 1;
457
458
459 val &= ~COH901318_CX_CFG_CH_ENABLE;
460 /* Enable twice, HW bug work around */
461 writel(val, virtbase + COH901318_CX_CFG +
462 COH901318_CX_CFG_SPACING * channel);
463 writel(val, virtbase + COH901318_CX_CFG +
464 COH901318_CX_CFG_SPACING * channel);
465
466 /* Spin-wait for it to actually go inactive */
467 while (readl(virtbase + COH901318_CX_STAT+COH901318_CX_STAT_SPACING *
468 channel) & COH901318_CX_STAT_ACTIVE)
469 cpu_relax();
470
471 /* Check if we stopped an active job */
472 if ((readl(virtbase + COH901318_CX_CTRL+COH901318_CX_CTRL_SPACING *
473 channel) & COH901318_CX_CTRL_TC_VALUE_MASK) > 0)
474 cohc->stopped = 1;
475
476 enable_powersave(cohc);
477
478 spin_unlock_irqrestore(&cohc->lock, flags);
479}
480EXPORT_SYMBOL(coh901318_stop);
481
482/* Continues a transfer that has been stopped via 300_dma_stop(..).
483 Power save is handled.
484*/
485void coh901318_continue(struct dma_chan *chan)
486{
487 u32 val;
488 unsigned long flags;
489 struct coh901318_chan *cohc = to_coh901318_chan(chan);
490 int channel = cohc->id;
491
492 spin_lock_irqsave(&cohc->lock, flags);
493
494 disable_powersave(cohc);
495
496 if (cohc->stopped) {
497 /* Enable channel in HW */
498 val = readl(cohc->base->virtbase + COH901318_CX_CFG +
499 COH901318_CX_CFG_SPACING * channel);
500
501 val |= COH901318_CX_CFG_CH_ENABLE;
502
503 writel(val, cohc->base->virtbase + COH901318_CX_CFG +
504 COH901318_CX_CFG_SPACING*channel);
505
506 cohc->stopped = 0;
507 }
508
509 spin_unlock_irqrestore(&cohc->lock, flags);
510}
511EXPORT_SYMBOL(coh901318_continue);
512
513bool coh901318_filter_id(struct dma_chan *chan, void *chan_id)
514{
515 unsigned int ch_nr = (unsigned int) chan_id;
516
517 if (ch_nr == to_coh901318_chan(chan)->id)
518 return true;
519
520 return false;
521}
522EXPORT_SYMBOL(coh901318_filter_id);
523
524/*
525 * DMA channel allocation
526 */
527static int coh901318_config(struct coh901318_chan *cohc,
528 struct coh901318_params *param)
529{
530 unsigned long flags;
531 const struct coh901318_params *p;
532 int channel = cohc->id;
533 void __iomem *virtbase = cohc->base->virtbase;
534
535 spin_lock_irqsave(&cohc->lock, flags);
536
537 if (param)
538 p = param;
539 else
540 p = &cohc->base->platform->chan_conf[channel].param;
541
542 /* Clear any pending BE or TC interrupt */
543 if (channel < 32) {
544 writel(1 << channel, virtbase + COH901318_BE_INT_CLEAR1);
545 writel(1 << channel, virtbase + COH901318_TC_INT_CLEAR1);
546 } else {
547 writel(1 << (channel - 32), virtbase +
548 COH901318_BE_INT_CLEAR2);
549 writel(1 << (channel - 32), virtbase +
550 COH901318_TC_INT_CLEAR2);
551 }
552
553 coh901318_set_conf(cohc, p->config);
554 coh901318_set_ctrl(cohc, p->ctrl_lli_last);
555
556 spin_unlock_irqrestore(&cohc->lock, flags);
557
558 return 0;
559}
560
561/* must lock when calling this function
562 * start queued jobs, if any
563 * TODO: start all queued jobs in one go
564 *
565 * Returns descriptor if queued job is started otherwise NULL.
566 * If the queue is empty NULL is returned.
567 */
568static struct coh901318_desc *coh901318_queue_start(struct coh901318_chan *cohc)
569{
570 struct coh901318_desc *cohd_que;
571
572 /* start queued jobs, if any
573 * TODO: transmit all queued jobs in one go
574 */
575 cohd_que = coh901318_first_queued(cohc);
576
577 if (cohd_que != NULL) {
578 /* Remove from queue */
579 coh901318_desc_remove(cohd_que);
580 /* initiate DMA job */
581 cohc->busy = 1;
582
583 coh901318_desc_submit(cohc, cohd_que);
584
585 coh901318_prep_linked_list(cohc, cohd_que->data);
586
587 /* start dma job */
588 coh901318_start(cohc);
589
590 }
591
592 return cohd_que;
593}
594
595static void dma_tasklet(unsigned long data)
596{
597 struct coh901318_chan *cohc = (struct coh901318_chan *) data;
598 struct coh901318_desc *cohd_fin;
599 unsigned long flags;
600 dma_async_tx_callback callback;
601 void *callback_param;
602
603 spin_lock_irqsave(&cohc->lock, flags);
604
605 /* get first active entry from list */
606 cohd_fin = coh901318_first_active_get(cohc);
607
608 BUG_ON(cohd_fin->pending_irqs == 0);
609
610 if (cohd_fin == NULL)
611 goto err;
612
613 cohd_fin->pending_irqs--;
614 cohc->completed = cohd_fin->desc.cookie;
615
616 BUG_ON(cohc->nbr_active_done && cohd_fin == NULL);
617
618 if (cohc->nbr_active_done == 0)
619 return;
620
621 if (!cohd_fin->pending_irqs) {
622 /* release the lli allocation*/
623 coh901318_lli_free(&cohc->base->pool, &cohd_fin->data);
624 }
625
626 dev_vdbg(COHC_2_DEV(cohc), "[%s] chan_id %d pending_irqs %d"
627 " nbr_active_done %ld\n", __func__,
628 cohc->id, cohc->pending_irqs, cohc->nbr_active_done);
629
630 /* callback to client */
631 callback = cohd_fin->desc.callback;
632 callback_param = cohd_fin->desc.callback_param;
633
634 if (!cohd_fin->pending_irqs) {
635 coh901318_desc_remove(cohd_fin);
636
637 /* return desc to free-list */
638 coh901318_desc_free(cohc, cohd_fin);
639 }
640
641 if (cohc->nbr_active_done)
642 cohc->nbr_active_done--;
643
644 if (cohc->nbr_active_done) {
645 if (cohc_chan_conf(cohc)->priority_high)
646 tasklet_hi_schedule(&cohc->tasklet);
647 else
648 tasklet_schedule(&cohc->tasklet);
649 }
650 spin_unlock_irqrestore(&cohc->lock, flags);
651
652 if (callback)
653 callback(callback_param);
654
655 return;
656
657 err:
658 spin_unlock_irqrestore(&cohc->lock, flags);
659 dev_err(COHC_2_DEV(cohc), "[%s] No active dma desc\n", __func__);
660}
661
662
663/* called from interrupt context */
664static void dma_tc_handle(struct coh901318_chan *cohc)
665{
666 BUG_ON(!cohc->allocated && (list_empty(&cohc->active) ||
667 list_empty(&cohc->queue)));
668
669 if (!cohc->allocated)
670 return;
671
672 BUG_ON(cohc->pending_irqs == 0);
673
674 cohc->pending_irqs--;
675 cohc->nbr_active_done++;
676
677 if (cohc->pending_irqs == 0 && coh901318_queue_start(cohc) == NULL)
678 cohc->busy = 0;
679
680 BUG_ON(list_empty(&cohc->active));
681
682 if (cohc_chan_conf(cohc)->priority_high)
683 tasklet_hi_schedule(&cohc->tasklet);
684 else
685 tasklet_schedule(&cohc->tasklet);
686}
687
688
689static irqreturn_t dma_irq_handler(int irq, void *dev_id)
690{
691 u32 status1;
692 u32 status2;
693 int i;
694 int ch;
695 struct coh901318_base *base = dev_id;
696 struct coh901318_chan *cohc;
697 void __iomem *virtbase = base->virtbase;
698
699 status1 = readl(virtbase + COH901318_INT_STATUS1);
700 status2 = readl(virtbase + COH901318_INT_STATUS2);
701
702 if (unlikely(status1 == 0 && status2 == 0)) {
703 dev_warn(base->dev, "spurious DMA IRQ from no channel!\n");
704 return IRQ_HANDLED;
705 }
706
707 /* TODO: consider handle IRQ in tasklet here to
708 * minimize interrupt latency */
709
710 /* Check the first 32 DMA channels for IRQ */
711 while (status1) {
712 /* Find first bit set, return as a number. */
713 i = ffs(status1) - 1;
714 ch = i;
715
716 cohc = &base->chans[ch];
717 spin_lock(&cohc->lock);
718
719 /* Mask off this bit */
720 status1 &= ~(1 << i);
721 /* Check the individual channel bits */
722 if (test_bit(i, virtbase + COH901318_BE_INT_STATUS1)) {
723 dev_crit(COHC_2_DEV(cohc),
724 "DMA bus error on channel %d!\n", ch);
725 BUG_ON(1);
726 /* Clear BE interrupt */
727 __set_bit(i, virtbase + COH901318_BE_INT_CLEAR1);
728 } else {
729 /* Caused by TC, really? */
730 if (unlikely(!test_bit(i, virtbase +
731 COH901318_TC_INT_STATUS1))) {
732 dev_warn(COHC_2_DEV(cohc),
733 "ignoring interrupt not caused by terminal count on channel %d\n", ch);
734 /* Clear TC interrupt */
735 BUG_ON(1);
736 __set_bit(i, virtbase + COH901318_TC_INT_CLEAR1);
737 } else {
738 /* Enable powersave if transfer has finished */
739 if (!(readl(virtbase + COH901318_CX_STAT +
740 COH901318_CX_STAT_SPACING*ch) &
741 COH901318_CX_STAT_ENABLED)) {
742 enable_powersave(cohc);
743 }
744
745 /* Must clear TC interrupt before calling
746 * dma_tc_handle
747 * in case tc_handle initate a new dma job
748 */
749 __set_bit(i, virtbase + COH901318_TC_INT_CLEAR1);
750
751 dma_tc_handle(cohc);
752 }
753 }
754 spin_unlock(&cohc->lock);
755 }
756
757 /* Check the remaining 32 DMA channels for IRQ */
758 while (status2) {
759 /* Find first bit set, return as a number. */
760 i = ffs(status2) - 1;
761 ch = i + 32;
762 cohc = &base->chans[ch];
763 spin_lock(&cohc->lock);
764
765 /* Mask off this bit */
766 status2 &= ~(1 << i);
767 /* Check the individual channel bits */
768 if (test_bit(i, virtbase + COH901318_BE_INT_STATUS2)) {
769 dev_crit(COHC_2_DEV(cohc),
770 "DMA bus error on channel %d!\n", ch);
771 /* Clear BE interrupt */
772 BUG_ON(1);
773 __set_bit(i, virtbase + COH901318_BE_INT_CLEAR2);
774 } else {
775 /* Caused by TC, really? */
776 if (unlikely(!test_bit(i, virtbase +
777 COH901318_TC_INT_STATUS2))) {
778 dev_warn(COHC_2_DEV(cohc),
779 "ignoring interrupt not caused by terminal count on channel %d\n", ch);
780 /* Clear TC interrupt */
781 __set_bit(i, virtbase + COH901318_TC_INT_CLEAR2);
782 BUG_ON(1);
783 } else {
784 /* Enable powersave if transfer has finished */
785 if (!(readl(virtbase + COH901318_CX_STAT +
786 COH901318_CX_STAT_SPACING*ch) &
787 COH901318_CX_STAT_ENABLED)) {
788 enable_powersave(cohc);
789 }
790 /* Must clear TC interrupt before calling
791 * dma_tc_handle
792 * in case tc_handle initate a new dma job
793 */
794 __set_bit(i, virtbase + COH901318_TC_INT_CLEAR2);
795
796 dma_tc_handle(cohc);
797 }
798 }
799 spin_unlock(&cohc->lock);
800 }
801
802 return IRQ_HANDLED;
803}
804
805static int coh901318_alloc_chan_resources(struct dma_chan *chan)
806{
807 struct coh901318_chan *cohc = to_coh901318_chan(chan);
808
809 dev_vdbg(COHC_2_DEV(cohc), "[%s] DMA channel %d\n",
810 __func__, cohc->id);
811
812 if (chan->client_count > 1)
813 return -EBUSY;
814
815 coh901318_config(cohc, NULL);
816
817 cohc->allocated = 1;
818 cohc->completed = chan->cookie = 1;
819
820 return 1;
821}
822
823static void
824coh901318_free_chan_resources(struct dma_chan *chan)
825{
826 struct coh901318_chan *cohc = to_coh901318_chan(chan);
827 int channel = cohc->id;
828 unsigned long flags;
829
830 spin_lock_irqsave(&cohc->lock, flags);
831
832 /* Disable HW */
833 writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CFG +
834 COH901318_CX_CFG_SPACING*channel);
835 writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CTRL +
836 COH901318_CX_CTRL_SPACING*channel);
837
838 cohc->allocated = 0;
839
840 spin_unlock_irqrestore(&cohc->lock, flags);
841
842 chan->device->device_terminate_all(chan);
843}
844
845
846static dma_cookie_t
847coh901318_tx_submit(struct dma_async_tx_descriptor *tx)
848{
849 struct coh901318_desc *cohd = container_of(tx, struct coh901318_desc,
850 desc);
851 struct coh901318_chan *cohc = to_coh901318_chan(tx->chan);
852 unsigned long flags;
853
854 spin_lock_irqsave(&cohc->lock, flags);
855
856 tx->cookie = coh901318_assign_cookie(cohc, cohd);
857
858 coh901318_desc_queue(cohc, cohd);
859
860 spin_unlock_irqrestore(&cohc->lock, flags);
861
862 return tx->cookie;
863}
864
865static struct dma_async_tx_descriptor *
866coh901318_prep_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
867 size_t size, unsigned long flags)
868{
869 struct coh901318_lli *data;
870 struct coh901318_desc *cohd;
871 unsigned long flg;
872 struct coh901318_chan *cohc = to_coh901318_chan(chan);
873 int lli_len;
874 u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last;
875
876 spin_lock_irqsave(&cohc->lock, flg);
877
878 dev_vdbg(COHC_2_DEV(cohc),
879 "[%s] channel %d src 0x%x dest 0x%x size %d\n",
880 __func__, cohc->id, src, dest, size);
881
882 if (flags & DMA_PREP_INTERRUPT)
883 /* Trigger interrupt after last lli */
884 ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE;
885
886 lli_len = size >> MAX_DMA_PACKET_SIZE_SHIFT;
887 if ((lli_len << MAX_DMA_PACKET_SIZE_SHIFT) < size)
888 lli_len++;
889
890 data = coh901318_lli_alloc(&cohc->base->pool, lli_len);
891
892 if (data == NULL)
893 goto err;
894
895 cohd = coh901318_desc_get(cohc);
896 cohd->sg = NULL;
897 cohd->sg_len = 0;
898 cohd->data = data;
899
900 cohd->pending_irqs =
901 coh901318_lli_fill_memcpy(
902 &cohc->base->pool, data, src, size, dest,
903 cohc_chan_param(cohc)->ctrl_lli_chained,
904 ctrl_last);
905 cohd->flags = flags;
906
907 COH_DBG(coh901318_list_print(cohc, data));
908
909 dma_async_tx_descriptor_init(&cohd->desc, chan);
910
911 cohd->desc.tx_submit = coh901318_tx_submit;
912
913 spin_unlock_irqrestore(&cohc->lock, flg);
914
915 return &cohd->desc;
916 err:
917 spin_unlock_irqrestore(&cohc->lock, flg);
918 return NULL;
919}
920
921static struct dma_async_tx_descriptor *
922coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
923 unsigned int sg_len, enum dma_data_direction direction,
924 unsigned long flags)
925{
926 struct coh901318_chan *cohc = to_coh901318_chan(chan);
927 struct coh901318_lli *data;
928 struct coh901318_desc *cohd;
929 struct scatterlist *sg;
930 int len = 0;
931 int size;
932 int i;
933 u32 ctrl_chained = cohc_chan_param(cohc)->ctrl_lli_chained;
934 u32 ctrl = cohc_chan_param(cohc)->ctrl_lli;
935 u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last;
936 unsigned long flg;
937
938 if (!sgl)
939 goto out;
940 if (sgl->length == 0)
941 goto out;
942
943 spin_lock_irqsave(&cohc->lock, flg);
944
945 dev_vdbg(COHC_2_DEV(cohc), "[%s] sg_len %d dir %d\n",
946 __func__, sg_len, direction);
947
948 if (flags & DMA_PREP_INTERRUPT)
949 /* Trigger interrupt after last lli */
950 ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE;
951
952 cohd = coh901318_desc_get(cohc);
953 cohd->sg = NULL;
954 cohd->sg_len = 0;
955 cohd->dir = direction;
956
957 if (direction == DMA_TO_DEVICE) {
958 u32 tx_flags = COH901318_CX_CTRL_PRDD_SOURCE |
959 COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE;
960
961 ctrl_chained |= tx_flags;
962 ctrl_last |= tx_flags;
963 ctrl |= tx_flags;
964 } else if (direction == DMA_FROM_DEVICE) {
965 u32 rx_flags = COH901318_CX_CTRL_PRDD_DEST |
966 COH901318_CX_CTRL_DST_ADDR_INC_ENABLE;
967
968 ctrl_chained |= rx_flags;
969 ctrl_last |= rx_flags;
970 ctrl |= rx_flags;
971 } else
972 goto err_direction;
973
974 dma_async_tx_descriptor_init(&cohd->desc, chan);
975
976 cohd->desc.tx_submit = coh901318_tx_submit;
977
978
979 /* The dma only supports transmitting packages up to
980 * MAX_DMA_PACKET_SIZE. Calculate to total number of
981 * dma elemts required to send the entire sg list
982 */
983 for_each_sg(sgl, sg, sg_len, i) {
984 unsigned int factor;
985 size = sg_dma_len(sg);
986
987 if (size <= MAX_DMA_PACKET_SIZE) {
988 len++;
989 continue;
990 }
991
992 factor = size >> MAX_DMA_PACKET_SIZE_SHIFT;
993 if ((factor << MAX_DMA_PACKET_SIZE_SHIFT) < size)
994 factor++;
995
996 len += factor;
997 }
998
999 data = coh901318_lli_alloc(&cohc->base->pool, len);
1000
1001 if (data == NULL)
1002 goto err_dma_alloc;
1003
1004 /* initiate allocated data list */
1005 cohd->pending_irqs =
1006 coh901318_lli_fill_sg(&cohc->base->pool, data, sgl, sg_len,
1007 cohc_dev_addr(cohc),
1008 ctrl_chained,
1009 ctrl,
1010 ctrl_last,
1011 direction, COH901318_CX_CTRL_TC_IRQ_ENABLE);
1012 cohd->data = data;
1013
1014 cohd->flags = flags;
1015
1016 COH_DBG(coh901318_list_print(cohc, data));
1017
1018 spin_unlock_irqrestore(&cohc->lock, flg);
1019
1020 return &cohd->desc;
1021 err_dma_alloc:
1022 err_direction:
1023 coh901318_desc_remove(cohd);
1024 coh901318_desc_free(cohc, cohd);
1025 spin_unlock_irqrestore(&cohc->lock, flg);
1026 out:
1027 return NULL;
1028}
1029
1030static enum dma_status
1031coh901318_is_tx_complete(struct dma_chan *chan,
1032 dma_cookie_t cookie, dma_cookie_t *done,
1033 dma_cookie_t *used)
1034{
1035 struct coh901318_chan *cohc = to_coh901318_chan(chan);
1036 dma_cookie_t last_used;
1037 dma_cookie_t last_complete;
1038 int ret;
1039
1040 last_complete = cohc->completed;
1041 last_used = chan->cookie;
1042
1043 ret = dma_async_is_complete(cookie, last_complete, last_used);
1044
1045 if (done)
1046 *done = last_complete;
1047 if (used)
1048 *used = last_used;
1049
1050 return ret;
1051}
1052
1053static void
1054coh901318_issue_pending(struct dma_chan *chan)
1055{
1056 struct coh901318_chan *cohc = to_coh901318_chan(chan);
1057 unsigned long flags;
1058
1059 spin_lock_irqsave(&cohc->lock, flags);
1060
1061 /* Busy means that pending jobs are already being processed */
1062 if (!cohc->busy)
1063 coh901318_queue_start(cohc);
1064
1065 spin_unlock_irqrestore(&cohc->lock, flags);
1066}
1067
1068static void
1069coh901318_terminate_all(struct dma_chan *chan)
1070{
1071 unsigned long flags;
1072 struct coh901318_chan *cohc = to_coh901318_chan(chan);
1073 struct coh901318_desc *cohd;
1074 void __iomem *virtbase = cohc->base->virtbase;
1075
1076 coh901318_stop(chan);
1077
1078 spin_lock_irqsave(&cohc->lock, flags);
1079
1080 /* Clear any pending BE or TC interrupt */
1081 if (cohc->id < 32) {
1082 writel(1 << cohc->id, virtbase + COH901318_BE_INT_CLEAR1);
1083 writel(1 << cohc->id, virtbase + COH901318_TC_INT_CLEAR1);
1084 } else {
1085 writel(1 << (cohc->id - 32), virtbase +
1086 COH901318_BE_INT_CLEAR2);
1087 writel(1 << (cohc->id - 32), virtbase +
1088 COH901318_TC_INT_CLEAR2);
1089 }
1090
1091 enable_powersave(cohc);
1092
1093 while ((cohd = coh901318_first_active_get(cohc))) {
1094 /* release the lli allocation*/
1095 coh901318_lli_free(&cohc->base->pool, &cohd->data);
1096
1097 coh901318_desc_remove(cohd);
1098
1099 /* return desc to free-list */
1100 coh901318_desc_free(cohc, cohd);
1101 }
1102
1103 while ((cohd = coh901318_first_queued(cohc))) {
1104 /* release the lli allocation*/
1105 coh901318_lli_free(&cohc->base->pool, &cohd->data);
1106
1107 coh901318_desc_remove(cohd);
1108
1109 /* return desc to free-list */
1110 coh901318_desc_free(cohc, cohd);
1111 }
1112
1113
1114 cohc->nbr_active_done = 0;
1115 cohc->busy = 0;
1116 cohc->pending_irqs = 0;
1117
1118 spin_unlock_irqrestore(&cohc->lock, flags);
1119}
1120void coh901318_base_init(struct dma_device *dma, const int *pick_chans,
1121 struct coh901318_base *base)
1122{
1123 int chans_i;
1124 int i = 0;
1125 struct coh901318_chan *cohc;
1126
1127 INIT_LIST_HEAD(&dma->channels);
1128
1129 for (chans_i = 0; pick_chans[chans_i] != -1; chans_i += 2) {
1130 for (i = pick_chans[chans_i]; i <= pick_chans[chans_i+1]; i++) {
1131 cohc = &base->chans[i];
1132
1133 cohc->base = base;
1134 cohc->chan.device = dma;
1135 cohc->id = i;
1136
1137 /* TODO: do we really need this lock if only one
1138 * client is connected to each channel?
1139 */
1140
1141 spin_lock_init(&cohc->lock);
1142
1143 cohc->pending_irqs = 0;
1144 cohc->nbr_active_done = 0;
1145 cohc->busy = 0;
1146 INIT_LIST_HEAD(&cohc->free);
1147 INIT_LIST_HEAD(&cohc->active);
1148 INIT_LIST_HEAD(&cohc->queue);
1149
1150 tasklet_init(&cohc->tasklet, dma_tasklet,
1151 (unsigned long) cohc);
1152
1153 list_add_tail(&cohc->chan.device_node,
1154 &dma->channels);
1155 }
1156 }
1157}
1158
1159static int __init coh901318_probe(struct platform_device *pdev)
1160{
1161 int err = 0;
1162 struct coh901318_platform *pdata;
1163 struct coh901318_base *base;
1164 int irq;
1165 struct resource *io;
1166
1167 io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1168 if (!io)
1169 goto err_get_resource;
1170
1171 /* Map DMA controller registers to virtual memory */
1172 if (request_mem_region(io->start,
1173 resource_size(io),
1174 pdev->dev.driver->name) == NULL) {
1175 err = -EBUSY;
1176 goto err_request_mem;
1177 }
1178
1179 pdata = pdev->dev.platform_data;
1180 if (!pdata)
1181 goto err_no_platformdata;
1182
1183 base = kmalloc(ALIGN(sizeof(struct coh901318_base), 4) +
1184 pdata->max_channels *
1185 sizeof(struct coh901318_chan),
1186 GFP_KERNEL);
1187 if (!base)
1188 goto err_alloc_coh_dma_channels;
1189
1190 base->chans = ((void *)base) + ALIGN(sizeof(struct coh901318_base), 4);
1191
1192 base->virtbase = ioremap(io->start, resource_size(io));
1193 if (!base->virtbase) {
1194 err = -ENOMEM;
1195 goto err_no_ioremap;
1196 }
1197
1198 base->dev = &pdev->dev;
1199 base->platform = pdata;
1200 spin_lock_init(&base->pm.lock);
1201 base->pm.started_channels = 0;
1202
1203 COH901318_DEBUGFS_ASSIGN(debugfs_dma_base, base);
1204
1205 platform_set_drvdata(pdev, base);
1206
1207 irq = platform_get_irq(pdev, 0);
1208 if (irq < 0)
1209 goto err_no_irq;
1210
1211 err = request_irq(irq, dma_irq_handler, IRQF_DISABLED,
1212 "coh901318", base);
1213 if (err) {
1214 dev_crit(&pdev->dev,
1215 "Cannot allocate IRQ for DMA controller!\n");
1216 goto err_request_irq;
1217 }
1218
1219 err = coh901318_pool_create(&base->pool, &pdev->dev,
1220 sizeof(struct coh901318_lli),
1221 32);
1222 if (err)
1223 goto err_pool_create;
1224
1225 /* init channels for device transfers */
1226 coh901318_base_init(&base->dma_slave, base->platform->chans_slave,
1227 base);
1228
1229 dma_cap_zero(base->dma_slave.cap_mask);
1230 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
1231
1232 base->dma_slave.device_alloc_chan_resources = coh901318_alloc_chan_resources;
1233 base->dma_slave.device_free_chan_resources = coh901318_free_chan_resources;
1234 base->dma_slave.device_prep_slave_sg = coh901318_prep_slave_sg;
1235 base->dma_slave.device_is_tx_complete = coh901318_is_tx_complete;
1236 base->dma_slave.device_issue_pending = coh901318_issue_pending;
1237 base->dma_slave.device_terminate_all = coh901318_terminate_all;
1238 base->dma_slave.dev = &pdev->dev;
1239
1240 err = dma_async_device_register(&base->dma_slave);
1241
1242 if (err)
1243 goto err_register_slave;
1244
1245 /* init channels for memcpy */
1246 coh901318_base_init(&base->dma_memcpy, base->platform->chans_memcpy,
1247 base);
1248
1249 dma_cap_zero(base->dma_memcpy.cap_mask);
1250 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
1251
1252 base->dma_memcpy.device_alloc_chan_resources = coh901318_alloc_chan_resources;
1253 base->dma_memcpy.device_free_chan_resources = coh901318_free_chan_resources;
1254 base->dma_memcpy.device_prep_dma_memcpy = coh901318_prep_memcpy;
1255 base->dma_memcpy.device_is_tx_complete = coh901318_is_tx_complete;
1256 base->dma_memcpy.device_issue_pending = coh901318_issue_pending;
1257 base->dma_memcpy.device_terminate_all = coh901318_terminate_all;
1258 base->dma_memcpy.dev = &pdev->dev;
1259 err = dma_async_device_register(&base->dma_memcpy);
1260
1261 if (err)
1262 goto err_register_memcpy;
1263
1264 dev_dbg(&pdev->dev, "Initialized COH901318 DMA on virtual base 0x%08x\n",
1265 (u32) base->virtbase);
1266
1267 return err;
1268
1269 err_register_memcpy:
1270 dma_async_device_unregister(&base->dma_slave);
1271 err_register_slave:
1272 coh901318_pool_destroy(&base->pool);
1273 err_pool_create:
1274 free_irq(platform_get_irq(pdev, 0), base);
1275 err_request_irq:
1276 err_no_irq:
1277 iounmap(base->virtbase);
1278 err_no_ioremap:
1279 kfree(base);
1280 err_alloc_coh_dma_channels:
1281 err_no_platformdata:
1282 release_mem_region(pdev->resource->start,
1283 resource_size(pdev->resource));
1284 err_request_mem:
1285 err_get_resource:
1286 return err;
1287}
1288
1289static int __exit coh901318_remove(struct platform_device *pdev)
1290{
1291 struct coh901318_base *base = platform_get_drvdata(pdev);
1292
1293 dma_async_device_unregister(&base->dma_memcpy);
1294 dma_async_device_unregister(&base->dma_slave);
1295 coh901318_pool_destroy(&base->pool);
1296 free_irq(platform_get_irq(pdev, 0), base);
1297 kfree(base);
1298 iounmap(base->virtbase);
1299 release_mem_region(pdev->resource->start,
1300 resource_size(pdev->resource));
1301 return 0;
1302}
1303
1304
1305static struct platform_driver coh901318_driver = {
1306 .remove = __exit_p(coh901318_remove),
1307 .driver = {
1308 .name = "coh901318",
1309 },
1310};
1311
1312int __init coh901318_init(void)
1313{
1314 return platform_driver_probe(&coh901318_driver, coh901318_probe);
1315}
1316subsys_initcall(coh901318_init);
1317
1318void __exit coh901318_exit(void)
1319{
1320 platform_driver_unregister(&coh901318_driver);
1321}
1322module_exit(coh901318_exit);
1323
1324MODULE_LICENSE("GPL");
1325MODULE_AUTHOR("Per Friden");
diff --git a/drivers/dma/coh901318_lli.c b/drivers/dma/coh901318_lli.c
new file mode 100644
index 000000000000..f5120f238a4d
--- /dev/null
+++ b/drivers/dma/coh901318_lli.c
@@ -0,0 +1,318 @@
1/*
2 * driver/dma/coh901318_lli.c
3 *
4 * Copyright (C) 2007-2009 ST-Ericsson
5 * License terms: GNU General Public License (GPL) version 2
6 * Support functions for handling lli for dma
7 * Author: Per Friden <per.friden@stericsson.com>
8 */
9
10#include <linux/dma-mapping.h>
11#include <linux/spinlock.h>
12#include <linux/dmapool.h>
13#include <linux/memory.h>
14#include <mach/coh901318.h>
15
16#include "coh901318_lli.h"
17
18#if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_U300_DEBUG))
19#define DEBUGFS_POOL_COUNTER_RESET(pool) (pool->debugfs_pool_counter = 0)
20#define DEBUGFS_POOL_COUNTER_ADD(pool, add) (pool->debugfs_pool_counter += add)
21#else
22#define DEBUGFS_POOL_COUNTER_RESET(pool)
23#define DEBUGFS_POOL_COUNTER_ADD(pool, add)
24#endif
25
26static struct coh901318_lli *
27coh901318_lli_next(struct coh901318_lli *data)
28{
29 if (data == NULL || data->link_addr == 0)
30 return NULL;
31
32 return (struct coh901318_lli *) data->virt_link_addr;
33}
34
35int coh901318_pool_create(struct coh901318_pool *pool,
36 struct device *dev,
37 size_t size, size_t align)
38{
39 spin_lock_init(&pool->lock);
40 pool->dev = dev;
41 pool->dmapool = dma_pool_create("lli_pool", dev, size, align, 0);
42
43 DEBUGFS_POOL_COUNTER_RESET(pool);
44 return 0;
45}
46
47int coh901318_pool_destroy(struct coh901318_pool *pool)
48{
49
50 dma_pool_destroy(pool->dmapool);
51 return 0;
52}
53
54struct coh901318_lli *
55coh901318_lli_alloc(struct coh901318_pool *pool, unsigned int len)
56{
57 int i;
58 struct coh901318_lli *head;
59 struct coh901318_lli *lli;
60 struct coh901318_lli *lli_prev;
61 dma_addr_t phy;
62
63 if (len == 0)
64 goto err;
65
66 spin_lock(&pool->lock);
67
68 head = dma_pool_alloc(pool->dmapool, GFP_NOWAIT, &phy);
69
70 if (head == NULL)
71 goto err;
72
73 DEBUGFS_POOL_COUNTER_ADD(pool, 1);
74
75 lli = head;
76 lli->phy_this = phy;
77
78 for (i = 1; i < len; i++) {
79 lli_prev = lli;
80
81 lli = dma_pool_alloc(pool->dmapool, GFP_NOWAIT, &phy);
82
83 if (lli == NULL)
84 goto err_clean_up;
85
86 DEBUGFS_POOL_COUNTER_ADD(pool, 1);
87 lli->phy_this = phy;
88
89 lli_prev->link_addr = phy;
90 lli_prev->virt_link_addr = lli;
91 }
92
93 lli->link_addr = 0x00000000U;
94
95 spin_unlock(&pool->lock);
96
97 return head;
98
99 err:
100 spin_unlock(&pool->lock);
101 return NULL;
102
103 err_clean_up:
104 lli_prev->link_addr = 0x00000000U;
105 spin_unlock(&pool->lock);
106 coh901318_lli_free(pool, &head);
107 return NULL;
108}
109
110void coh901318_lli_free(struct coh901318_pool *pool,
111 struct coh901318_lli **lli)
112{
113 struct coh901318_lli *l;
114 struct coh901318_lli *next;
115
116 if (lli == NULL)
117 return;
118
119 l = *lli;
120
121 if (l == NULL)
122 return;
123
124 spin_lock(&pool->lock);
125
126 while (l->link_addr) {
127 next = l->virt_link_addr;
128 dma_pool_free(pool->dmapool, l, l->phy_this);
129 DEBUGFS_POOL_COUNTER_ADD(pool, -1);
130 l = next;
131 }
132 dma_pool_free(pool->dmapool, l, l->phy_this);
133 DEBUGFS_POOL_COUNTER_ADD(pool, -1);
134
135 spin_unlock(&pool->lock);
136 *lli = NULL;
137}
138
139int
140coh901318_lli_fill_memcpy(struct coh901318_pool *pool,
141 struct coh901318_lli *lli,
142 dma_addr_t source, unsigned int size,
143 dma_addr_t destination, u32 ctrl_chained,
144 u32 ctrl_eom)
145{
146 int s = size;
147 dma_addr_t src = source;
148 dma_addr_t dst = destination;
149
150 lli->src_addr = src;
151 lli->dst_addr = dst;
152
153 while (lli->link_addr) {
154 lli->control = ctrl_chained | MAX_DMA_PACKET_SIZE;
155 lli->src_addr = src;
156 lli->dst_addr = dst;
157
158 s -= MAX_DMA_PACKET_SIZE;
159 lli = coh901318_lli_next(lli);
160
161 src += MAX_DMA_PACKET_SIZE;
162 dst += MAX_DMA_PACKET_SIZE;
163 }
164
165 lli->control = ctrl_eom | s;
166 lli->src_addr = src;
167 lli->dst_addr = dst;
168
169 /* One irq per single transfer */
170 return 1;
171}
172
173int
174coh901318_lli_fill_single(struct coh901318_pool *pool,
175 struct coh901318_lli *lli,
176 dma_addr_t buf, unsigned int size,
177 dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl_eom,
178 enum dma_data_direction dir)
179{
180 int s = size;
181 dma_addr_t src;
182 dma_addr_t dst;
183
184
185 if (dir == DMA_TO_DEVICE) {
186 src = buf;
187 dst = dev_addr;
188
189 } else if (dir == DMA_FROM_DEVICE) {
190
191 src = dev_addr;
192 dst = buf;
193 } else {
194 return -EINVAL;
195 }
196
197 while (lli->link_addr) {
198 size_t block_size = MAX_DMA_PACKET_SIZE;
199 lli->control = ctrl_chained | MAX_DMA_PACKET_SIZE;
200
201 /* If we are on the next-to-final block and there will
202 * be less than half a DMA packet left for the last
203 * block, then we want to make this block a little
204 * smaller to balance the sizes. This is meant to
205 * avoid too small transfers if the buffer size is
206 * (MAX_DMA_PACKET_SIZE*N + 1) */
207 if (s < (MAX_DMA_PACKET_SIZE + MAX_DMA_PACKET_SIZE/2))
208 block_size = MAX_DMA_PACKET_SIZE/2;
209
210 s -= block_size;
211 lli->src_addr = src;
212 lli->dst_addr = dst;
213
214 lli = coh901318_lli_next(lli);
215
216 if (dir == DMA_TO_DEVICE)
217 src += block_size;
218 else if (dir == DMA_FROM_DEVICE)
219 dst += block_size;
220 }
221
222 lli->control = ctrl_eom | s;
223 lli->src_addr = src;
224 lli->dst_addr = dst;
225
226 /* One irq per single transfer */
227 return 1;
228}
229
230int
231coh901318_lli_fill_sg(struct coh901318_pool *pool,
232 struct coh901318_lli *lli,
233 struct scatterlist *sgl, unsigned int nents,
234 dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl,
235 u32 ctrl_last,
236 enum dma_data_direction dir, u32 ctrl_irq_mask)
237{
238 int i;
239 struct scatterlist *sg;
240 u32 ctrl_sg;
241 dma_addr_t src = 0;
242 dma_addr_t dst = 0;
243 int nbr_of_irq = 0;
244 u32 bytes_to_transfer;
245 u32 elem_size;
246
247 if (lli == NULL)
248 goto err;
249
250 spin_lock(&pool->lock);
251
252 if (dir == DMA_TO_DEVICE)
253 dst = dev_addr;
254 else if (dir == DMA_FROM_DEVICE)
255 src = dev_addr;
256 else
257 goto err;
258
259 for_each_sg(sgl, sg, nents, i) {
260 if (sg_is_chain(sg)) {
261 /* sg continues to the next sg-element don't
262 * send ctrl_finish until the last
263 * sg-element in the chain
264 */
265 ctrl_sg = ctrl_chained;
266 } else if (i == nents - 1)
267 ctrl_sg = ctrl_last;
268 else
269 ctrl_sg = ctrl ? ctrl : ctrl_last;
270
271
272 if ((ctrl_sg & ctrl_irq_mask))
273 nbr_of_irq++;
274
275 if (dir == DMA_TO_DEVICE)
276 /* increment source address */
277 src = sg_dma_address(sg);
278 else
279 /* increment destination address */
280 dst = sg_dma_address(sg);
281
282 bytes_to_transfer = sg_dma_len(sg);
283
284 while (bytes_to_transfer) {
285 u32 val;
286
287 if (bytes_to_transfer > MAX_DMA_PACKET_SIZE) {
288 elem_size = MAX_DMA_PACKET_SIZE;
289 val = ctrl_chained;
290 } else {
291 elem_size = bytes_to_transfer;
292 val = ctrl_sg;
293 }
294
295 lli->control = val | elem_size;
296 lli->src_addr = src;
297 lli->dst_addr = dst;
298
299 if (dir == DMA_FROM_DEVICE)
300 dst += elem_size;
301 else
302 src += elem_size;
303
304 BUG_ON(lli->link_addr & 3);
305
306 bytes_to_transfer -= elem_size;
307 lli = coh901318_lli_next(lli);
308 }
309
310 }
311 spin_unlock(&pool->lock);
312
313 /* There can be many IRQs per sg transfer */
314 return nbr_of_irq;
315 err:
316 spin_unlock(&pool->lock);
317 return -EINVAL;
318}
diff --git a/drivers/dma/coh901318_lli.h b/drivers/dma/coh901318_lli.h
new file mode 100644
index 000000000000..7bf713b79c6b
--- /dev/null
+++ b/drivers/dma/coh901318_lli.h
@@ -0,0 +1,124 @@
1/*
2 * driver/dma/coh901318_lli.h
3 *
4 * Copyright (C) 2007-2009 ST-Ericsson
5 * License terms: GNU General Public License (GPL) version 2
6 * Support functions for handling lli for coh901318
7 * Author: Per Friden <per.friden@stericsson.com>
8 */
9
10#ifndef COH901318_LLI_H
11#define COH901318_LLI_H
12
13#include <mach/coh901318.h>
14
15struct device;
16
17struct coh901318_pool {
18 spinlock_t lock;
19 struct dma_pool *dmapool;
20 struct device *dev;
21
22#ifdef CONFIG_DEBUG_FS
23 int debugfs_pool_counter;
24#endif
25};
26
27struct device;
28/**
29 * coh901318_pool_create() - Creates an dma pool for lli:s
30 * @pool: pool handle
31 * @dev: dma device
32 * @lli_nbr: number of lli:s in the pool
33 * @algin: adress alignemtn of lli:s
34 * returns 0 on success otherwise none zero
35 */
36int coh901318_pool_create(struct coh901318_pool *pool,
37 struct device *dev,
38 size_t lli_nbr, size_t align);
39
40/**
41 * coh901318_pool_destroy() - Destroys the dma pool
42 * @pool: pool handle
43 * returns 0 on success otherwise none zero
44 */
45int coh901318_pool_destroy(struct coh901318_pool *pool);
46
47/**
48 * coh901318_lli_alloc() - Allocates a linked list
49 *
50 * @pool: pool handle
51 * @len: length to list
52 * return: none NULL if success otherwise NULL
53 */
54struct coh901318_lli *
55coh901318_lli_alloc(struct coh901318_pool *pool,
56 unsigned int len);
57
58/**
59 * coh901318_lli_free() - Returns the linked list items to the pool
60 * @pool: pool handle
61 * @lli: reference to lli pointer to be freed
62 */
63void coh901318_lli_free(struct coh901318_pool *pool,
64 struct coh901318_lli **lli);
65
66/**
67 * coh901318_lli_fill_memcpy() - Prepares the lli:s for dma memcpy
68 * @pool: pool handle
69 * @lli: allocated lli
70 * @src: src address
71 * @size: transfer size
72 * @dst: destination address
73 * @ctrl_chained: ctrl for chained lli
74 * @ctrl_last: ctrl for the last lli
75 * returns number of CPU interrupts for the lli, negative on error.
76 */
77int
78coh901318_lli_fill_memcpy(struct coh901318_pool *pool,
79 struct coh901318_lli *lli,
80 dma_addr_t src, unsigned int size,
81 dma_addr_t dst, u32 ctrl_chained, u32 ctrl_last);
82
83/**
84 * coh901318_lli_fill_single() - Prepares the lli:s for dma single transfer
85 * @pool: pool handle
86 * @lli: allocated lli
87 * @buf: transfer buffer
88 * @size: transfer size
89 * @dev_addr: address of periphal
90 * @ctrl_chained: ctrl for chained lli
91 * @ctrl_last: ctrl for the last lli
92 * @dir: direction of transfer (to or from device)
93 * returns number of CPU interrupts for the lli, negative on error.
94 */
95int
96coh901318_lli_fill_single(struct coh901318_pool *pool,
97 struct coh901318_lli *lli,
98 dma_addr_t buf, unsigned int size,
99 dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl_last,
100 enum dma_data_direction dir);
101
102/**
103 * coh901318_lli_fill_single() - Prepares the lli:s for dma scatter list transfer
104 * @pool: pool handle
105 * @lli: allocated lli
106 * @sg: scatter gather list
107 * @nents: number of entries in sg
108 * @dev_addr: address of periphal
109 * @ctrl_chained: ctrl for chained lli
110 * @ctrl: ctrl of middle lli
111 * @ctrl_last: ctrl for the last lli
112 * @dir: direction of transfer (to or from device)
113 * @ctrl_irq_mask: ctrl mask for CPU interrupt
114 * returns number of CPU interrupts for the lli, negative on error.
115 */
116int
117coh901318_lli_fill_sg(struct coh901318_pool *pool,
118 struct coh901318_lli *lli,
119 struct scatterlist *sg, unsigned int nents,
120 dma_addr_t dev_addr, u32 ctrl_chained,
121 u32 ctrl, u32 ctrl_last,
122 enum dma_data_direction dir, u32 ctrl_irq_mask);
123
124#endif /* COH901318_LLI_H */
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index a32a4cf7b1e0..8b905161fbf4 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -298,10 +298,6 @@ static int dmatest_func(void *data)
298 298
299 total_tests++; 299 total_tests++;
300 300
301 len = dmatest_random() % test_buf_size + 1;
302 src_off = dmatest_random() % (test_buf_size - len + 1);
303 dst_off = dmatest_random() % (test_buf_size - len + 1);
304
305 /* honor alignment restrictions */ 301 /* honor alignment restrictions */
306 if (thread->type == DMA_MEMCPY) 302 if (thread->type == DMA_MEMCPY)
307 align = dev->copy_align; 303 align = dev->copy_align;
@@ -310,7 +306,19 @@ static int dmatest_func(void *data)
310 else if (thread->type == DMA_PQ) 306 else if (thread->type == DMA_PQ)
311 align = dev->pq_align; 307 align = dev->pq_align;
312 308
309 if (1 << align > test_buf_size) {
310 pr_err("%u-byte buffer too small for %d-byte alignment\n",
311 test_buf_size, 1 << align);
312 break;
313 }
314
315 len = dmatest_random() % test_buf_size + 1;
313 len = (len >> align) << align; 316 len = (len >> align) << align;
317 if (!len)
318 len = 1 << align;
319 src_off = dmatest_random() % (test_buf_size - len + 1);
320 dst_off = dmatest_random() % (test_buf_size - len + 1);
321
314 src_off = (src_off >> align) << align; 322 src_off = (src_off >> align) << align;
315 dst_off = (dst_off >> align) << align; 323 dst_off = (dst_off >> align) << align;
316 324
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
index 645ca8d54ec4..ca6e6a0cb793 100644
--- a/drivers/dma/iop-adma.c
+++ b/drivers/dma/iop-adma.c
@@ -1470,7 +1470,7 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
1470 return -ENODEV; 1470 return -ENODEV;
1471 1471
1472 if (!devm_request_mem_region(&pdev->dev, res->start, 1472 if (!devm_request_mem_region(&pdev->dev, res->start,
1473 res->end - res->start, pdev->name)) 1473 resource_size(res), pdev->name))
1474 return -EBUSY; 1474 return -EBUSY;
1475 1475
1476 adev = kzalloc(sizeof(*adev), GFP_KERNEL); 1476 adev = kzalloc(sizeof(*adev), GFP_KERNEL);
@@ -1542,7 +1542,7 @@ static int __devinit iop_adma_probe(struct platform_device *pdev)
1542 iop_chan->device = adev; 1542 iop_chan->device = adev;
1543 1543
1544 iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start, 1544 iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1545 res->end - res->start); 1545 resource_size(res));
1546 if (!iop_chan->mmr_base) { 1546 if (!iop_chan->mmr_base) {
1547 ret = -ENOMEM; 1547 ret = -ENOMEM;
1548 goto err_free_iop_chan; 1548 goto err_free_iop_chan;
diff --git a/drivers/dma/ppc4xx/Makefile b/drivers/dma/ppc4xx/Makefile
new file mode 100644
index 000000000000..b3d259b3e52a
--- /dev/null
+++ b/drivers/dma/ppc4xx/Makefile
@@ -0,0 +1 @@
obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += adma.o
diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c
new file mode 100644
index 000000000000..0a3478e910f0
--- /dev/null
+++ b/drivers/dma/ppc4xx/adma.c
@@ -0,0 +1,5027 @@
1/*
2 * Copyright (C) 2006-2009 DENX Software Engineering.
3 *
4 * Author: Yuri Tikhonov <yur@emcraft.com>
5 *
6 * Further porting to arch/powerpc by
7 * Anatolij Gustschin <agust@denx.de>
8 *
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU General Public License as published by the Free
11 * Software Foundation; either version 2 of the License, or (at your option)
12 * any later version.
13 *
14 * This program is distributed in the hope that it will be useful, but WITHOUT
15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 * more details.
18 *
19 * You should have received a copy of the GNU General Public License along with
20 * this program; if not, write to the Free Software Foundation, Inc., 59
21 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
22 *
23 * The full GNU General Public License is included in this distribution in the
24 * file called COPYING.
25 */
26
27/*
28 * This driver supports the asynchrounous DMA copy and RAID engines available
29 * on the AMCC PPC440SPe Processors.
30 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
31 * ADMA driver written by D.Williams.
32 */
33
34#include <linux/init.h>
35#include <linux/module.h>
36#include <linux/async_tx.h>
37#include <linux/delay.h>
38#include <linux/dma-mapping.h>
39#include <linux/spinlock.h>
40#include <linux/interrupt.h>
41#include <linux/uaccess.h>
42#include <linux/proc_fs.h>
43#include <linux/of.h>
44#include <linux/of_platform.h>
45#include <asm/dcr.h>
46#include <asm/dcr-regs.h>
47#include "adma.h"
48
49enum ppc_adma_init_code {
50 PPC_ADMA_INIT_OK = 0,
51 PPC_ADMA_INIT_MEMRES,
52 PPC_ADMA_INIT_MEMREG,
53 PPC_ADMA_INIT_ALLOC,
54 PPC_ADMA_INIT_COHERENT,
55 PPC_ADMA_INIT_CHANNEL,
56 PPC_ADMA_INIT_IRQ1,
57 PPC_ADMA_INIT_IRQ2,
58 PPC_ADMA_INIT_REGISTER
59};
60
61static char *ppc_adma_errors[] = {
62 [PPC_ADMA_INIT_OK] = "ok",
63 [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
64 [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
65 [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
66 "structure",
67 [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
68 "hardware descriptors",
69 [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
70 [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
71 [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
72 [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
73};
74
75static enum ppc_adma_init_code
76ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
77
78struct ppc_dma_chan_ref {
79 struct dma_chan *chan;
80 struct list_head node;
81};
82
83/* The list of channels exported by ppc440spe ADMA */
84struct list_head
85ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
86
87/* This flag is set when want to refetch the xor chain in the interrupt
88 * handler
89 */
90static u32 do_xor_refetch;
91
92/* Pointer to DMA0, DMA1 CP/CS FIFO */
93static void *ppc440spe_dma_fifo_buf;
94
95/* Pointers to last submitted to DMA0, DMA1 CDBs */
96static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
97static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
98
99/* Pointer to last linked and submitted xor CB */
100static struct ppc440spe_adma_desc_slot *xor_last_linked;
101static struct ppc440spe_adma_desc_slot *xor_last_submit;
102
103/* This array is used in data-check operations for storing a pattern */
104static char ppc440spe_qword[16];
105
106static atomic_t ppc440spe_adma_err_irq_ref;
107static dcr_host_t ppc440spe_mq_dcr_host;
108static unsigned int ppc440spe_mq_dcr_len;
109
110/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
111 * the block size in transactions, then we do not allow to activate more than
112 * only one RXOR transactions simultaneously. So use this var to store
113 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
114 * set) or not (PPC440SPE_RXOR_RUN is clear).
115 */
116static unsigned long ppc440spe_rxor_state;
117
118/* These are used in enable & check routines
119 */
120static u32 ppc440spe_r6_enabled;
121static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
122static struct completion ppc440spe_r6_test_comp;
123
124static int ppc440spe_adma_dma2rxor_prep_src(
125 struct ppc440spe_adma_desc_slot *desc,
126 struct ppc440spe_rxor *cursor, int index,
127 int src_cnt, u32 addr);
128static void ppc440spe_adma_dma2rxor_set_src(
129 struct ppc440spe_adma_desc_slot *desc,
130 int index, dma_addr_t addr);
131static void ppc440spe_adma_dma2rxor_set_mult(
132 struct ppc440spe_adma_desc_slot *desc,
133 int index, u8 mult);
134
135#ifdef ADMA_LL_DEBUG
136#define ADMA_LL_DBG(x) ({ if (1) x; 0; })
137#else
138#define ADMA_LL_DBG(x) ({ if (0) x; 0; })
139#endif
140
141static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
142{
143 struct dma_cdb *cdb;
144 struct xor_cb *cb;
145 int i;
146
147 switch (chan->device->id) {
148 case 0:
149 case 1:
150 cdb = block;
151
152 pr_debug("CDB at %p [%d]:\n"
153 "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
154 "\t sg1u 0x%08x sg1l 0x%08x\n"
155 "\t sg2u 0x%08x sg2l 0x%08x\n"
156 "\t sg3u 0x%08x sg3l 0x%08x\n",
157 cdb, chan->device->id,
158 cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
159 le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
160 le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
161 le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
162 );
163 break;
164 case 2:
165 cb = block;
166
167 pr_debug("CB at %p [%d]:\n"
168 "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
169 "\t cbtah 0x%08x cbtal 0x%08x\n"
170 "\t cblah 0x%08x cblal 0x%08x\n",
171 cb, chan->device->id,
172 cb->cbc, cb->cbbc, cb->cbs,
173 cb->cbtah, cb->cbtal,
174 cb->cblah, cb->cblal);
175 for (i = 0; i < 16; i++) {
176 if (i && !cb->ops[i].h && !cb->ops[i].l)
177 continue;
178 pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
179 i, cb->ops[i].h, cb->ops[i].l);
180 }
181 break;
182 }
183}
184
185static void print_cb_list(struct ppc440spe_adma_chan *chan,
186 struct ppc440spe_adma_desc_slot *iter)
187{
188 for (; iter; iter = iter->hw_next)
189 print_cb(chan, iter->hw_desc);
190}
191
192static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
193 unsigned int src_cnt)
194{
195 int i;
196
197 pr_debug("\n%s(%d):\nsrc: ", __func__, id);
198 for (i = 0; i < src_cnt; i++)
199 pr_debug("\t0x%016llx ", src[i]);
200 pr_debug("dst:\n\t0x%016llx\n", dst);
201}
202
203static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
204 unsigned int src_cnt)
205{
206 int i;
207
208 pr_debug("\n%s(%d):\nsrc: ", __func__, id);
209 for (i = 0; i < src_cnt; i++)
210 pr_debug("\t0x%016llx ", src[i]);
211 pr_debug("dst: ");
212 for (i = 0; i < 2; i++)
213 pr_debug("\t0x%016llx ", dst[i]);
214}
215
216static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
217 unsigned int src_cnt,
218 const unsigned char *scf)
219{
220 int i;
221
222 pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
223 if (scf) {
224 for (i = 0; i < src_cnt; i++)
225 pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
226 } else {
227 for (i = 0; i < src_cnt; i++)
228 pr_debug("\t0x%016llx(no) ", src[i]);
229 }
230
231 pr_debug("dst: ");
232 for (i = 0; i < 2; i++)
233 pr_debug("\t0x%016llx ", src[src_cnt + i]);
234}
235
236/******************************************************************************
237 * Command (Descriptor) Blocks low-level routines
238 ******************************************************************************/
239/**
240 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
241 * pseudo operation
242 */
243static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
244 struct ppc440spe_adma_chan *chan)
245{
246 struct xor_cb *p;
247
248 switch (chan->device->id) {
249 case PPC440SPE_XOR_ID:
250 p = desc->hw_desc;
251 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
252 /* NOP with Command Block Complete Enable */
253 p->cbc = XOR_CBCR_CBCE_BIT;
254 break;
255 case PPC440SPE_DMA0_ID:
256 case PPC440SPE_DMA1_ID:
257 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
258 /* NOP with interrupt */
259 set_bit(PPC440SPE_DESC_INT, &desc->flags);
260 break;
261 default:
262 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
263 __func__);
264 break;
265 }
266}
267
268/**
269 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
270 * pseudo operation
271 */
272static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
273{
274 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
275 desc->hw_next = NULL;
276 desc->src_cnt = 0;
277 desc->dst_cnt = 1;
278}
279
280/**
281 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
282 */
283static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
284 int src_cnt, unsigned long flags)
285{
286 struct xor_cb *hw_desc = desc->hw_desc;
287
288 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
289 desc->hw_next = NULL;
290 desc->src_cnt = src_cnt;
291 desc->dst_cnt = 1;
292
293 hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
294 if (flags & DMA_PREP_INTERRUPT)
295 /* Enable interrupt on completion */
296 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
297}
298
299/**
300 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
301 * operation in DMA2 controller
302 */
303static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
304 int dst_cnt, int src_cnt, unsigned long flags)
305{
306 struct xor_cb *hw_desc = desc->hw_desc;
307
308 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
309 desc->hw_next = NULL;
310 desc->src_cnt = src_cnt;
311 desc->dst_cnt = dst_cnt;
312 memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
313 desc->descs_per_op = 0;
314
315 hw_desc->cbc = XOR_CBCR_TGT_BIT;
316 if (flags & DMA_PREP_INTERRUPT)
317 /* Enable interrupt on completion */
318 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
319}
320
321#define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE
322#define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1)
323#define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1)
324
325/**
326 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
327 * with DMA0/1
328 */
329static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
330 int dst_cnt, int src_cnt, unsigned long flags,
331 unsigned long op)
332{
333 struct dma_cdb *hw_desc;
334 struct ppc440spe_adma_desc_slot *iter;
335 u8 dopc;
336
337 /* Common initialization of a PQ descriptors chain */
338 set_bits(op, &desc->flags);
339 desc->src_cnt = src_cnt;
340 desc->dst_cnt = dst_cnt;
341
342 /* WXOR MULTICAST if both P and Q are being computed
343 * MV_SG1_SG2 if Q only
344 */
345 dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
346 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
347
348 list_for_each_entry(iter, &desc->group_list, chain_node) {
349 hw_desc = iter->hw_desc;
350 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
351
352 if (likely(!list_is_last(&iter->chain_node,
353 &desc->group_list))) {
354 /* set 'next' pointer */
355 iter->hw_next = list_entry(iter->chain_node.next,
356 struct ppc440spe_adma_desc_slot, chain_node);
357 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
358 } else {
359 /* this is the last descriptor.
360 * this slot will be pasted from ADMA level
361 * each time it wants to configure parameters
362 * of the transaction (src, dst, ...)
363 */
364 iter->hw_next = NULL;
365 if (flags & DMA_PREP_INTERRUPT)
366 set_bit(PPC440SPE_DESC_INT, &iter->flags);
367 else
368 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
369 }
370 }
371
372 /* Set OPS depending on WXOR/RXOR type of operation */
373 if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
374 /* This is a WXOR only chain:
375 * - first descriptors are for zeroing destinations
376 * if PPC440SPE_ZERO_P/Q set;
377 * - descriptors remained are for GF-XOR operations.
378 */
379 iter = list_first_entry(&desc->group_list,
380 struct ppc440spe_adma_desc_slot,
381 chain_node);
382
383 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
384 hw_desc = iter->hw_desc;
385 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
386 iter = list_first_entry(&iter->chain_node,
387 struct ppc440spe_adma_desc_slot,
388 chain_node);
389 }
390
391 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
392 hw_desc = iter->hw_desc;
393 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
394 iter = list_first_entry(&iter->chain_node,
395 struct ppc440spe_adma_desc_slot,
396 chain_node);
397 }
398
399 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
400 hw_desc = iter->hw_desc;
401 hw_desc->opc = dopc;
402 }
403 } else {
404 /* This is either RXOR-only or mixed RXOR/WXOR */
405
406 /* The first 1 or 2 slots in chain are always RXOR,
407 * if need to calculate P & Q, then there are two
408 * RXOR slots; if only P or only Q, then there is one
409 */
410 iter = list_first_entry(&desc->group_list,
411 struct ppc440spe_adma_desc_slot,
412 chain_node);
413 hw_desc = iter->hw_desc;
414 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
415
416 if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
417 iter = list_first_entry(&iter->chain_node,
418 struct ppc440spe_adma_desc_slot,
419 chain_node);
420 hw_desc = iter->hw_desc;
421 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
422 }
423
424 /* The remaining descs (if any) are WXORs */
425 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
426 iter = list_first_entry(&iter->chain_node,
427 struct ppc440spe_adma_desc_slot,
428 chain_node);
429 list_for_each_entry_from(iter, &desc->group_list,
430 chain_node) {
431 hw_desc = iter->hw_desc;
432 hw_desc->opc = dopc;
433 }
434 }
435 }
436}
437
438/**
439 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
440 * for PQ_ZERO_SUM operation
441 */
442static void ppc440spe_desc_init_dma01pqzero_sum(
443 struct ppc440spe_adma_desc_slot *desc,
444 int dst_cnt, int src_cnt)
445{
446 struct dma_cdb *hw_desc;
447 struct ppc440spe_adma_desc_slot *iter;
448 int i = 0;
449 u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
450 DMA_CDB_OPC_MV_SG1_SG2;
451 /*
452 * Initialize starting from 2nd or 3rd descriptor dependent
453 * on dst_cnt. First one or two slots are for cloning P
454 * and/or Q to chan->pdest and/or chan->qdest as we have
455 * to preserve original P/Q.
456 */
457 iter = list_first_entry(&desc->group_list,
458 struct ppc440spe_adma_desc_slot, chain_node);
459 iter = list_entry(iter->chain_node.next,
460 struct ppc440spe_adma_desc_slot, chain_node);
461
462 if (dst_cnt > 1) {
463 iter = list_entry(iter->chain_node.next,
464 struct ppc440spe_adma_desc_slot, chain_node);
465 }
466 /* initialize each source descriptor in chain */
467 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
468 hw_desc = iter->hw_desc;
469 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
470 iter->src_cnt = 0;
471 iter->dst_cnt = 0;
472
473 /* This is a ZERO_SUM operation:
474 * - <src_cnt> descriptors starting from 2nd or 3rd
475 * descriptor are for GF-XOR operations;
476 * - remaining <dst_cnt> descriptors are for checking the result
477 */
478 if (i++ < src_cnt)
479 /* MV_SG1_SG2 if only Q is being verified
480 * MULTICAST if both P and Q are being verified
481 */
482 hw_desc->opc = dopc;
483 else
484 /* DMA_CDB_OPC_DCHECK128 operation */
485 hw_desc->opc = DMA_CDB_OPC_DCHECK128;
486
487 if (likely(!list_is_last(&iter->chain_node,
488 &desc->group_list))) {
489 /* set 'next' pointer */
490 iter->hw_next = list_entry(iter->chain_node.next,
491 struct ppc440spe_adma_desc_slot,
492 chain_node);
493 } else {
494 /* this is the last descriptor.
495 * this slot will be pasted from ADMA level
496 * each time it wants to configure parameters
497 * of the transaction (src, dst, ...)
498 */
499 iter->hw_next = NULL;
500 /* always enable interrupt generation since we get
501 * the status of pqzero from the handler
502 */
503 set_bit(PPC440SPE_DESC_INT, &iter->flags);
504 }
505 }
506 desc->src_cnt = src_cnt;
507 desc->dst_cnt = dst_cnt;
508}
509
510/**
511 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
512 */
513static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
514 unsigned long flags)
515{
516 struct dma_cdb *hw_desc = desc->hw_desc;
517
518 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
519 desc->hw_next = NULL;
520 desc->src_cnt = 1;
521 desc->dst_cnt = 1;
522
523 if (flags & DMA_PREP_INTERRUPT)
524 set_bit(PPC440SPE_DESC_INT, &desc->flags);
525 else
526 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
527
528 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
529}
530
531/**
532 * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation
533 */
534static void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc,
535 int value, unsigned long flags)
536{
537 struct dma_cdb *hw_desc = desc->hw_desc;
538
539 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
540 desc->hw_next = NULL;
541 desc->src_cnt = 1;
542 desc->dst_cnt = 1;
543
544 if (flags & DMA_PREP_INTERRUPT)
545 set_bit(PPC440SPE_DESC_INT, &desc->flags);
546 else
547 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
548
549 hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value);
550 hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value);
551 hw_desc->opc = DMA_CDB_OPC_DFILL128;
552}
553
554/**
555 * ppc440spe_desc_set_src_addr - set source address into the descriptor
556 */
557static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
558 struct ppc440spe_adma_chan *chan,
559 int src_idx, dma_addr_t addrh,
560 dma_addr_t addrl)
561{
562 struct dma_cdb *dma_hw_desc;
563 struct xor_cb *xor_hw_desc;
564 phys_addr_t addr64, tmplow, tmphi;
565
566 switch (chan->device->id) {
567 case PPC440SPE_DMA0_ID:
568 case PPC440SPE_DMA1_ID:
569 if (!addrh) {
570 addr64 = addrl;
571 tmphi = (addr64 >> 32);
572 tmplow = (addr64 & 0xFFFFFFFF);
573 } else {
574 tmphi = addrh;
575 tmplow = addrl;
576 }
577 dma_hw_desc = desc->hw_desc;
578 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
579 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
580 break;
581 case PPC440SPE_XOR_ID:
582 xor_hw_desc = desc->hw_desc;
583 xor_hw_desc->ops[src_idx].l = addrl;
584 xor_hw_desc->ops[src_idx].h |= addrh;
585 break;
586 }
587}
588
589/**
590 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
591 */
592static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
593 struct ppc440spe_adma_chan *chan, u32 mult_index,
594 int sg_index, unsigned char mult_value)
595{
596 struct dma_cdb *dma_hw_desc;
597 struct xor_cb *xor_hw_desc;
598 u32 *psgu;
599
600 switch (chan->device->id) {
601 case PPC440SPE_DMA0_ID:
602 case PPC440SPE_DMA1_ID:
603 dma_hw_desc = desc->hw_desc;
604
605 switch (sg_index) {
606 /* for RXOR operations set multiplier
607 * into source cued address
608 */
609 case DMA_CDB_SG_SRC:
610 psgu = &dma_hw_desc->sg1u;
611 break;
612 /* for WXOR operations set multiplier
613 * into destination cued address(es)
614 */
615 case DMA_CDB_SG_DST1:
616 psgu = &dma_hw_desc->sg2u;
617 break;
618 case DMA_CDB_SG_DST2:
619 psgu = &dma_hw_desc->sg3u;
620 break;
621 default:
622 BUG();
623 }
624
625 *psgu |= cpu_to_le32(mult_value << mult_index);
626 break;
627 case PPC440SPE_XOR_ID:
628 xor_hw_desc = desc->hw_desc;
629 break;
630 default:
631 BUG();
632 }
633}
634
635/**
636 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
637 */
638static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
639 struct ppc440spe_adma_chan *chan,
640 dma_addr_t addrh, dma_addr_t addrl,
641 u32 dst_idx)
642{
643 struct dma_cdb *dma_hw_desc;
644 struct xor_cb *xor_hw_desc;
645 phys_addr_t addr64, tmphi, tmplow;
646 u32 *psgu, *psgl;
647
648 switch (chan->device->id) {
649 case PPC440SPE_DMA0_ID:
650 case PPC440SPE_DMA1_ID:
651 if (!addrh) {
652 addr64 = addrl;
653 tmphi = (addr64 >> 32);
654 tmplow = (addr64 & 0xFFFFFFFF);
655 } else {
656 tmphi = addrh;
657 tmplow = addrl;
658 }
659 dma_hw_desc = desc->hw_desc;
660
661 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
662 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
663
664 *psgl = cpu_to_le32((u32)tmplow);
665 *psgu |= cpu_to_le32((u32)tmphi);
666 break;
667 case PPC440SPE_XOR_ID:
668 xor_hw_desc = desc->hw_desc;
669 xor_hw_desc->cbtal = addrl;
670 xor_hw_desc->cbtah |= addrh;
671 break;
672 }
673}
674
675/**
676 * ppc440spe_desc_set_byte_count - set number of data bytes involved
677 * into the operation
678 */
679static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
680 struct ppc440spe_adma_chan *chan,
681 u32 byte_count)
682{
683 struct dma_cdb *dma_hw_desc;
684 struct xor_cb *xor_hw_desc;
685
686 switch (chan->device->id) {
687 case PPC440SPE_DMA0_ID:
688 case PPC440SPE_DMA1_ID:
689 dma_hw_desc = desc->hw_desc;
690 dma_hw_desc->cnt = cpu_to_le32(byte_count);
691 break;
692 case PPC440SPE_XOR_ID:
693 xor_hw_desc = desc->hw_desc;
694 xor_hw_desc->cbbc = byte_count;
695 break;
696 }
697}
698
699/**
700 * ppc440spe_desc_set_rxor_block_size - set RXOR block size
701 */
702static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
703{
704 /* assume that byte_count is aligned on the 512-boundary;
705 * thus write it directly to the register (bits 23:31 are
706 * reserved there).
707 */
708 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
709}
710
711/**
712 * ppc440spe_desc_set_dcheck - set CHECK pattern
713 */
714static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
715 struct ppc440spe_adma_chan *chan, u8 *qword)
716{
717 struct dma_cdb *dma_hw_desc;
718
719 switch (chan->device->id) {
720 case PPC440SPE_DMA0_ID:
721 case PPC440SPE_DMA1_ID:
722 dma_hw_desc = desc->hw_desc;
723 iowrite32(qword[0], &dma_hw_desc->sg3l);
724 iowrite32(qword[4], &dma_hw_desc->sg3u);
725 iowrite32(qword[8], &dma_hw_desc->sg2l);
726 iowrite32(qword[12], &dma_hw_desc->sg2u);
727 break;
728 default:
729 BUG();
730 }
731}
732
733/**
734 * ppc440spe_xor_set_link - set link address in xor CB
735 */
736static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
737 struct ppc440spe_adma_desc_slot *next_desc)
738{
739 struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
740
741 if (unlikely(!next_desc || !(next_desc->phys))) {
742 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
743 __func__, next_desc,
744 next_desc ? next_desc->phys : 0);
745 BUG();
746 }
747
748 xor_hw_desc->cbs = 0;
749 xor_hw_desc->cblal = next_desc->phys;
750 xor_hw_desc->cblah = 0;
751 xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
752}
753
754/**
755 * ppc440spe_desc_set_link - set the address of descriptor following this
756 * descriptor in chain
757 */
758static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
759 struct ppc440spe_adma_desc_slot *prev_desc,
760 struct ppc440spe_adma_desc_slot *next_desc)
761{
762 unsigned long flags;
763 struct ppc440spe_adma_desc_slot *tail = next_desc;
764
765 if (unlikely(!prev_desc || !next_desc ||
766 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
767 /* If previous next is overwritten something is wrong.
768 * though we may refetch from append to initiate list
769 * processing; in this case - it's ok.
770 */
771 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
772 "prev->hw_next=0x%p\n", __func__, prev_desc,
773 next_desc, prev_desc ? prev_desc->hw_next : 0);
774 BUG();
775 }
776
777 local_irq_save(flags);
778
779 /* do s/w chaining both for DMA and XOR descriptors */
780 prev_desc->hw_next = next_desc;
781
782 switch (chan->device->id) {
783 case PPC440SPE_DMA0_ID:
784 case PPC440SPE_DMA1_ID:
785 break;
786 case PPC440SPE_XOR_ID:
787 /* bind descriptor to the chain */
788 while (tail->hw_next)
789 tail = tail->hw_next;
790 xor_last_linked = tail;
791
792 if (prev_desc == xor_last_submit)
793 /* do not link to the last submitted CB */
794 break;
795 ppc440spe_xor_set_link(prev_desc, next_desc);
796 break;
797 }
798
799 local_irq_restore(flags);
800}
801
802/**
803 * ppc440spe_desc_get_src_addr - extract the source address from the descriptor
804 */
805static u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc,
806 struct ppc440spe_adma_chan *chan, int src_idx)
807{
808 struct dma_cdb *dma_hw_desc;
809 struct xor_cb *xor_hw_desc;
810
811 switch (chan->device->id) {
812 case PPC440SPE_DMA0_ID:
813 case PPC440SPE_DMA1_ID:
814 dma_hw_desc = desc->hw_desc;
815 /* May have 0, 1, 2, or 3 sources */
816 switch (dma_hw_desc->opc) {
817 case DMA_CDB_OPC_NO_OP:
818 case DMA_CDB_OPC_DFILL128:
819 return 0;
820 case DMA_CDB_OPC_DCHECK128:
821 if (unlikely(src_idx)) {
822 printk(KERN_ERR "%s: try to get %d source for"
823 " DCHECK128\n", __func__, src_idx);
824 BUG();
825 }
826 return le32_to_cpu(dma_hw_desc->sg1l);
827 case DMA_CDB_OPC_MULTICAST:
828 case DMA_CDB_OPC_MV_SG1_SG2:
829 if (unlikely(src_idx > 2)) {
830 printk(KERN_ERR "%s: try to get %d source from"
831 " DMA descr\n", __func__, src_idx);
832 BUG();
833 }
834 if (src_idx) {
835 if (le32_to_cpu(dma_hw_desc->sg1u) &
836 DMA_CUED_XOR_WIN_MSK) {
837 u8 region;
838
839 if (src_idx == 1)
840 return le32_to_cpu(
841 dma_hw_desc->sg1l) +
842 desc->unmap_len;
843
844 region = (le32_to_cpu(
845 dma_hw_desc->sg1u)) >>
846 DMA_CUED_REGION_OFF;
847
848 region &= DMA_CUED_REGION_MSK;
849 switch (region) {
850 case DMA_RXOR123:
851 return le32_to_cpu(
852 dma_hw_desc->sg1l) +
853 (desc->unmap_len << 1);
854 case DMA_RXOR124:
855 return le32_to_cpu(
856 dma_hw_desc->sg1l) +
857 (desc->unmap_len * 3);
858 case DMA_RXOR125:
859 return le32_to_cpu(
860 dma_hw_desc->sg1l) +
861 (desc->unmap_len << 2);
862 default:
863 printk(KERN_ERR
864 "%s: try to"
865 " get src3 for region %02x"
866 "PPC440SPE_DESC_RXOR12?\n",
867 __func__, region);
868 BUG();
869 }
870 } else {
871 printk(KERN_ERR
872 "%s: try to get %d"
873 " source for non-cued descr\n",
874 __func__, src_idx);
875 BUG();
876 }
877 }
878 return le32_to_cpu(dma_hw_desc->sg1l);
879 default:
880 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
881 __func__, dma_hw_desc->opc);
882 BUG();
883 }
884 return le32_to_cpu(dma_hw_desc->sg1l);
885 case PPC440SPE_XOR_ID:
886 /* May have up to 16 sources */
887 xor_hw_desc = desc->hw_desc;
888 return xor_hw_desc->ops[src_idx].l;
889 }
890 return 0;
891}
892
893/**
894 * ppc440spe_desc_get_dest_addr - extract the destination address from the
895 * descriptor
896 */
897static u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc,
898 struct ppc440spe_adma_chan *chan, int idx)
899{
900 struct dma_cdb *dma_hw_desc;
901 struct xor_cb *xor_hw_desc;
902
903 switch (chan->device->id) {
904 case PPC440SPE_DMA0_ID:
905 case PPC440SPE_DMA1_ID:
906 dma_hw_desc = desc->hw_desc;
907
908 if (likely(!idx))
909 return le32_to_cpu(dma_hw_desc->sg2l);
910 return le32_to_cpu(dma_hw_desc->sg3l);
911 case PPC440SPE_XOR_ID:
912 xor_hw_desc = desc->hw_desc;
913 return xor_hw_desc->cbtal;
914 }
915 return 0;
916}
917
918/**
919 * ppc440spe_desc_get_src_num - extract the number of source addresses from
920 * the descriptor
921 */
922static u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc,
923 struct ppc440spe_adma_chan *chan)
924{
925 struct dma_cdb *dma_hw_desc;
926 struct xor_cb *xor_hw_desc;
927
928 switch (chan->device->id) {
929 case PPC440SPE_DMA0_ID:
930 case PPC440SPE_DMA1_ID:
931 dma_hw_desc = desc->hw_desc;
932
933 switch (dma_hw_desc->opc) {
934 case DMA_CDB_OPC_NO_OP:
935 case DMA_CDB_OPC_DFILL128:
936 return 0;
937 case DMA_CDB_OPC_DCHECK128:
938 return 1;
939 case DMA_CDB_OPC_MV_SG1_SG2:
940 case DMA_CDB_OPC_MULTICAST:
941 /*
942 * Only for RXOR operations we have more than
943 * one source
944 */
945 if (le32_to_cpu(dma_hw_desc->sg1u) &
946 DMA_CUED_XOR_WIN_MSK) {
947 /* RXOR op, there are 2 or 3 sources */
948 if (((le32_to_cpu(dma_hw_desc->sg1u) >>
949 DMA_CUED_REGION_OFF) &
950 DMA_CUED_REGION_MSK) == DMA_RXOR12) {
951 /* RXOR 1-2 */
952 return 2;
953 } else {
954 /* RXOR 1-2-3/1-2-4/1-2-5 */
955 return 3;
956 }
957 }
958 return 1;
959 default:
960 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
961 __func__, dma_hw_desc->opc);
962 BUG();
963 }
964 case PPC440SPE_XOR_ID:
965 /* up to 16 sources */
966 xor_hw_desc = desc->hw_desc;
967 return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK;
968 default:
969 BUG();
970 }
971 return 0;
972}
973
974/**
975 * ppc440spe_desc_get_dst_num - get the number of destination addresses in
976 * this descriptor
977 */
978static u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc,
979 struct ppc440spe_adma_chan *chan)
980{
981 struct dma_cdb *dma_hw_desc;
982
983 switch (chan->device->id) {
984 case PPC440SPE_DMA0_ID:
985 case PPC440SPE_DMA1_ID:
986 /* May be 1 or 2 destinations */
987 dma_hw_desc = desc->hw_desc;
988 switch (dma_hw_desc->opc) {
989 case DMA_CDB_OPC_NO_OP:
990 case DMA_CDB_OPC_DCHECK128:
991 return 0;
992 case DMA_CDB_OPC_MV_SG1_SG2:
993 case DMA_CDB_OPC_DFILL128:
994 return 1;
995 case DMA_CDB_OPC_MULTICAST:
996 if (desc->dst_cnt == 2)
997 return 2;
998 else
999 return 1;
1000 default:
1001 printk(KERN_ERR "%s: unknown OPC 0x%02x\n",
1002 __func__, dma_hw_desc->opc);
1003 BUG();
1004 }
1005 case PPC440SPE_XOR_ID:
1006 /* Always only 1 destination */
1007 return 1;
1008 default:
1009 BUG();
1010 }
1011 return 0;
1012}
1013
1014/**
1015 * ppc440spe_desc_get_link - get the address of the descriptor that
1016 * follows this one
1017 */
1018static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
1019 struct ppc440spe_adma_chan *chan)
1020{
1021 if (!desc->hw_next)
1022 return 0;
1023
1024 return desc->hw_next->phys;
1025}
1026
1027/**
1028 * ppc440spe_desc_is_aligned - check alignment
1029 */
1030static inline int ppc440spe_desc_is_aligned(
1031 struct ppc440spe_adma_desc_slot *desc, int num_slots)
1032{
1033 return (desc->idx & (num_slots - 1)) ? 0 : 1;
1034}
1035
1036/**
1037 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
1038 * XOR operation
1039 */
1040static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
1041 int *slots_per_op)
1042{
1043 int slot_cnt;
1044
1045 /* each XOR descriptor provides up to 16 source operands */
1046 slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
1047
1048 if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
1049 return slot_cnt;
1050
1051 printk(KERN_ERR "%s: len %d > max %d !!\n",
1052 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
1053 BUG();
1054 return slot_cnt;
1055}
1056
1057/**
1058 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
1059 * DMA2 PQ operation
1060 */
1061static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
1062 int src_cnt, size_t len)
1063{
1064 signed long long order = 0;
1065 int state = 0;
1066 int addr_count = 0;
1067 int i;
1068 for (i = 1; i < src_cnt; i++) {
1069 dma_addr_t cur_addr = srcs[i];
1070 dma_addr_t old_addr = srcs[i-1];
1071 switch (state) {
1072 case 0:
1073 if (cur_addr == old_addr + len) {
1074 /* direct RXOR */
1075 order = 1;
1076 state = 1;
1077 if (i == src_cnt-1)
1078 addr_count++;
1079 } else if (old_addr == cur_addr + len) {
1080 /* reverse RXOR */
1081 order = -1;
1082 state = 1;
1083 if (i == src_cnt-1)
1084 addr_count++;
1085 } else {
1086 state = 3;
1087 }
1088 break;
1089 case 1:
1090 if (i == src_cnt-2 || (order == -1
1091 && cur_addr != old_addr - len)) {
1092 order = 0;
1093 state = 0;
1094 addr_count++;
1095 } else if (cur_addr == old_addr + len*order) {
1096 state = 2;
1097 if (i == src_cnt-1)
1098 addr_count++;
1099 } else if (cur_addr == old_addr + 2*len) {
1100 state = 2;
1101 if (i == src_cnt-1)
1102 addr_count++;
1103 } else if (cur_addr == old_addr + 3*len) {
1104 state = 2;
1105 if (i == src_cnt-1)
1106 addr_count++;
1107 } else {
1108 order = 0;
1109 state = 0;
1110 addr_count++;
1111 }
1112 break;
1113 case 2:
1114 order = 0;
1115 state = 0;
1116 addr_count++;
1117 break;
1118 }
1119 if (state == 3)
1120 break;
1121 }
1122 if (src_cnt <= 1 || (state != 1 && state != 2)) {
1123 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
1124 __func__, src_cnt, state, addr_count, order);
1125 for (i = 0; i < src_cnt; i++)
1126 pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
1127 BUG();
1128 }
1129
1130 return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
1131}
1132
1133
1134/******************************************************************************
1135 * ADMA channel low-level routines
1136 ******************************************************************************/
1137
1138static u32
1139ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
1140static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
1141
1142/**
1143 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
1144 */
1145static void ppc440spe_adma_device_clear_eot_status(
1146 struct ppc440spe_adma_chan *chan)
1147{
1148 struct dma_regs *dma_reg;
1149 struct xor_regs *xor_reg;
1150 u8 *p = chan->device->dma_desc_pool_virt;
1151 struct dma_cdb *cdb;
1152 u32 rv, i;
1153
1154 switch (chan->device->id) {
1155 case PPC440SPE_DMA0_ID:
1156 case PPC440SPE_DMA1_ID:
1157 /* read FIFO to ack */
1158 dma_reg = chan->device->dma_reg;
1159 while ((rv = ioread32(&dma_reg->csfpl))) {
1160 i = rv & DMA_CDB_ADDR_MSK;
1161 cdb = (struct dma_cdb *)&p[i -
1162 (u32)chan->device->dma_desc_pool];
1163
1164 /* Clear opcode to ack. This is necessary for
1165 * ZeroSum operations only
1166 */
1167 cdb->opc = 0;
1168
1169 if (test_bit(PPC440SPE_RXOR_RUN,
1170 &ppc440spe_rxor_state)) {
1171 /* probably this is a completed RXOR op,
1172 * get pointer to CDB using the fact that
1173 * physical and virtual addresses of CDB
1174 * in pools have the same offsets
1175 */
1176 if (le32_to_cpu(cdb->sg1u) &
1177 DMA_CUED_XOR_BASE) {
1178 /* this is a RXOR */
1179 clear_bit(PPC440SPE_RXOR_RUN,
1180 &ppc440spe_rxor_state);
1181 }
1182 }
1183
1184 if (rv & DMA_CDB_STATUS_MSK) {
1185 /* ZeroSum check failed
1186 */
1187 struct ppc440spe_adma_desc_slot *iter;
1188 dma_addr_t phys = rv & ~DMA_CDB_MSK;
1189
1190 /*
1191 * Update the status of corresponding
1192 * descriptor.
1193 */
1194 list_for_each_entry(iter, &chan->chain,
1195 chain_node) {
1196 if (iter->phys == phys)
1197 break;
1198 }
1199 /*
1200 * if cannot find the corresponding
1201 * slot it's a bug
1202 */
1203 BUG_ON(&iter->chain_node == &chan->chain);
1204
1205 if (iter->xor_check_result) {
1206 if (test_bit(PPC440SPE_DESC_PCHECK,
1207 &iter->flags)) {
1208 *iter->xor_check_result |=
1209 SUM_CHECK_P_RESULT;
1210 } else
1211 if (test_bit(PPC440SPE_DESC_QCHECK,
1212 &iter->flags)) {
1213 *iter->xor_check_result |=
1214 SUM_CHECK_Q_RESULT;
1215 } else
1216 BUG();
1217 }
1218 }
1219 }
1220
1221 rv = ioread32(&dma_reg->dsts);
1222 if (rv) {
1223 pr_err("DMA%d err status: 0x%x\n",
1224 chan->device->id, rv);
1225 /* write back to clear */
1226 iowrite32(rv, &dma_reg->dsts);
1227 }
1228 break;
1229 case PPC440SPE_XOR_ID:
1230 /* reset status bits to ack */
1231 xor_reg = chan->device->xor_reg;
1232 rv = ioread32be(&xor_reg->sr);
1233 iowrite32be(rv, &xor_reg->sr);
1234
1235 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
1236 if (rv & XOR_IE_RPTIE_BIT) {
1237 /* Read PLB Timeout Error.
1238 * Try to resubmit the CB
1239 */
1240 u32 val = ioread32be(&xor_reg->ccbalr);
1241
1242 iowrite32be(val, &xor_reg->cblalr);
1243
1244 val = ioread32be(&xor_reg->crsr);
1245 iowrite32be(val | XOR_CRSR_XAE_BIT,
1246 &xor_reg->crsr);
1247 } else
1248 pr_err("XOR ERR 0x%x status\n", rv);
1249 break;
1250 }
1251
1252 /* if the XORcore is idle, but there are unprocessed CBs
1253 * then refetch the s/w chain here
1254 */
1255 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1256 do_xor_refetch)
1257 ppc440spe_chan_append(chan);
1258 break;
1259 }
1260}
1261
1262/**
1263 * ppc440spe_chan_is_busy - get the channel status
1264 */
1265static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1266{
1267 struct dma_regs *dma_reg;
1268 struct xor_regs *xor_reg;
1269 int busy = 0;
1270
1271 switch (chan->device->id) {
1272 case PPC440SPE_DMA0_ID:
1273 case PPC440SPE_DMA1_ID:
1274 dma_reg = chan->device->dma_reg;
1275 /* if command FIFO's head and tail pointers are equal and
1276 * status tail is the same as command, then channel is free
1277 */
1278 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1279 ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1280 busy = 1;
1281 break;
1282 case PPC440SPE_XOR_ID:
1283 /* use the special status bit for the XORcore
1284 */
1285 xor_reg = chan->device->xor_reg;
1286 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1287 break;
1288 }
1289
1290 return busy;
1291}
1292
1293/**
1294 * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain
1295 */
1296static void ppc440spe_chan_set_first_xor_descriptor(
1297 struct ppc440spe_adma_chan *chan,
1298 struct ppc440spe_adma_desc_slot *next_desc)
1299{
1300 struct xor_regs *xor_reg = chan->device->xor_reg;
1301
1302 if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1303 printk(KERN_INFO "%s: Warn: XORcore is running "
1304 "when try to set the first CDB!\n",
1305 __func__);
1306
1307 xor_last_submit = xor_last_linked = next_desc;
1308
1309 iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1310
1311 iowrite32be(next_desc->phys, &xor_reg->cblalr);
1312 iowrite32be(0, &xor_reg->cblahr);
1313 iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1314 &xor_reg->cbcr);
1315
1316 chan->hw_chain_inited = 1;
1317}
1318
1319/**
1320 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1321 * called with irqs disabled
1322 */
1323static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1324 struct ppc440spe_adma_desc_slot *desc)
1325{
1326 u32 pcdb;
1327 struct dma_regs *dma_reg = chan->device->dma_reg;
1328
1329 pcdb = desc->phys;
1330 if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1331 pcdb |= DMA_CDB_NO_INT;
1332
1333 chan_last_sub[chan->device->id] = desc;
1334
1335 ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1336
1337 iowrite32(pcdb, &dma_reg->cpfpl);
1338}
1339
1340/**
1341 * ppc440spe_chan_append - update the h/w chain in the channel
1342 */
1343static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1344{
1345 struct xor_regs *xor_reg;
1346 struct ppc440spe_adma_desc_slot *iter;
1347 struct xor_cb *xcb;
1348 u32 cur_desc;
1349 unsigned long flags;
1350
1351 local_irq_save(flags);
1352
1353 switch (chan->device->id) {
1354 case PPC440SPE_DMA0_ID:
1355 case PPC440SPE_DMA1_ID:
1356 cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1357
1358 if (likely(cur_desc)) {
1359 iter = chan_last_sub[chan->device->id];
1360 BUG_ON(!iter);
1361 } else {
1362 /* first peer */
1363 iter = chan_first_cdb[chan->device->id];
1364 BUG_ON(!iter);
1365 ppc440spe_dma_put_desc(chan, iter);
1366 chan->hw_chain_inited = 1;
1367 }
1368
1369 /* is there something new to append */
1370 if (!iter->hw_next)
1371 break;
1372
1373 /* flush descriptors from the s/w queue to fifo */
1374 list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1375 ppc440spe_dma_put_desc(chan, iter);
1376 if (!iter->hw_next)
1377 break;
1378 }
1379 break;
1380 case PPC440SPE_XOR_ID:
1381 /* update h/w links and refetch */
1382 if (!xor_last_submit->hw_next)
1383 break;
1384
1385 xor_reg = chan->device->xor_reg;
1386 /* the last linked CDB has to generate an interrupt
1387 * that we'd be able to append the next lists to h/w
1388 * regardless of the XOR engine state at the moment of
1389 * appending of these next lists
1390 */
1391 xcb = xor_last_linked->hw_desc;
1392 xcb->cbc |= XOR_CBCR_CBCE_BIT;
1393
1394 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1395 /* XORcore is idle. Refetch now */
1396 do_xor_refetch = 0;
1397 ppc440spe_xor_set_link(xor_last_submit,
1398 xor_last_submit->hw_next);
1399
1400 ADMA_LL_DBG(print_cb_list(chan,
1401 xor_last_submit->hw_next));
1402
1403 xor_last_submit = xor_last_linked;
1404 iowrite32be(ioread32be(&xor_reg->crsr) |
1405 XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1406 &xor_reg->crsr);
1407 } else {
1408 /* XORcore is running. Refetch later in the handler */
1409 do_xor_refetch = 1;
1410 }
1411
1412 break;
1413 }
1414
1415 local_irq_restore(flags);
1416}
1417
1418/**
1419 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1420 */
1421static u32
1422ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1423{
1424 struct dma_regs *dma_reg;
1425 struct xor_regs *xor_reg;
1426
1427 if (unlikely(!chan->hw_chain_inited))
1428 /* h/w descriptor chain is not initialized yet */
1429 return 0;
1430
1431 switch (chan->device->id) {
1432 case PPC440SPE_DMA0_ID:
1433 case PPC440SPE_DMA1_ID:
1434 dma_reg = chan->device->dma_reg;
1435 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1436 case PPC440SPE_XOR_ID:
1437 xor_reg = chan->device->xor_reg;
1438 return ioread32be(&xor_reg->ccbalr);
1439 }
1440 return 0;
1441}
1442
1443/**
1444 * ppc440spe_chan_run - enable the channel
1445 */
1446static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1447{
1448 struct xor_regs *xor_reg;
1449
1450 switch (chan->device->id) {
1451 case PPC440SPE_DMA0_ID:
1452 case PPC440SPE_DMA1_ID:
1453 /* DMAs are always enabled, do nothing */
1454 break;
1455 case PPC440SPE_XOR_ID:
1456 /* drain write buffer */
1457 xor_reg = chan->device->xor_reg;
1458
1459 /* fetch descriptor pointed to in <link> */
1460 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1461 &xor_reg->crsr);
1462 break;
1463 }
1464}
1465
1466/******************************************************************************
1467 * ADMA device level
1468 ******************************************************************************/
1469
1470static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1471static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1472
1473static dma_cookie_t
1474ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1475
1476static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1477 dma_addr_t addr, int index);
1478static void
1479ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1480 dma_addr_t addr, int index);
1481
1482static void
1483ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1484 dma_addr_t *paddr, unsigned long flags);
1485static void
1486ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1487 dma_addr_t addr, int index);
1488static void
1489ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1490 unsigned char mult, int index, int dst_pos);
1491static void
1492ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1493 dma_addr_t paddr, dma_addr_t qaddr);
1494
1495static struct page *ppc440spe_rxor_srcs[32];
1496
1497/**
1498 * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1499 */
1500static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1501{
1502 int i, order = 0, state = 0;
1503 int idx = 0;
1504
1505 if (unlikely(!(src_cnt > 1)))
1506 return 0;
1507
1508 BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1509
1510 /* Skip holes in the source list before checking */
1511 for (i = 0; i < src_cnt; i++) {
1512 if (!srcs[i])
1513 continue;
1514 ppc440spe_rxor_srcs[idx++] = srcs[i];
1515 }
1516 src_cnt = idx;
1517
1518 for (i = 1; i < src_cnt; i++) {
1519 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1520 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1521
1522 switch (state) {
1523 case 0:
1524 if (cur_addr == old_addr + len) {
1525 /* direct RXOR */
1526 order = 1;
1527 state = 1;
1528 } else if (old_addr == cur_addr + len) {
1529 /* reverse RXOR */
1530 order = -1;
1531 state = 1;
1532 } else
1533 goto out;
1534 break;
1535 case 1:
1536 if ((i == src_cnt - 2) ||
1537 (order == -1 && cur_addr != old_addr - len)) {
1538 order = 0;
1539 state = 0;
1540 } else if ((cur_addr == old_addr + len * order) ||
1541 (cur_addr == old_addr + 2 * len) ||
1542 (cur_addr == old_addr + 3 * len)) {
1543 state = 2;
1544 } else {
1545 order = 0;
1546 state = 0;
1547 }
1548 break;
1549 case 2:
1550 order = 0;
1551 state = 0;
1552 break;
1553 }
1554 }
1555
1556out:
1557 if (state == 1 || state == 2)
1558 return 1;
1559
1560 return 0;
1561}
1562
1563/**
1564 * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1565 * the operation given on this channel. It's assumed that 'chan' is
1566 * capable to process 'cap' type of operation.
1567 * @chan: channel to use
1568 * @cap: type of transaction
1569 * @dst_lst: array of destination pointers
1570 * @dst_cnt: number of destination operands
1571 * @src_lst: array of source pointers
1572 * @src_cnt: number of source operands
1573 * @src_sz: size of each source operand
1574 */
1575static int ppc440spe_adma_estimate(struct dma_chan *chan,
1576 enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1577 struct page **src_lst, int src_cnt, size_t src_sz)
1578{
1579 int ef = 1;
1580
1581 if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1582 /* If RAID-6 capabilities were not activated don't try
1583 * to use them
1584 */
1585 if (unlikely(!ppc440spe_r6_enabled))
1586 return -1;
1587 }
1588 /* In the current implementation of ppc440spe ADMA driver it
1589 * makes sense to pick out only pq case, because it may be
1590 * processed:
1591 * (1) either using Biskup method on DMA2;
1592 * (2) or on DMA0/1.
1593 * Thus we give a favour to (1) if the sources are suitable;
1594 * else let it be processed on one of the DMA0/1 engines.
1595 * In the sum_product case where destination is also the
1596 * source process it on DMA0/1 only.
1597 */
1598 if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1599
1600 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1601 ef = 0; /* sum_product case, process on DMA0/1 */
1602 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1603 ef = 3; /* override (DMA0/1 + idle) */
1604 else
1605 ef = 0; /* can't process on DMA2 if !rxor */
1606 }
1607
1608 /* channel idleness increases the priority */
1609 if (likely(ef) &&
1610 !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1611 ef++;
1612
1613 return ef;
1614}
1615
1616struct dma_chan *
1617ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1618 struct page **dst_lst, int dst_cnt, struct page **src_lst,
1619 int src_cnt, size_t src_sz)
1620{
1621 struct dma_chan *best_chan = NULL;
1622 struct ppc_dma_chan_ref *ref;
1623 int best_rank = -1;
1624
1625 if (unlikely(!src_sz))
1626 return NULL;
1627 if (src_sz > PAGE_SIZE) {
1628 /*
1629 * should a user of the api ever pass > PAGE_SIZE requests
1630 * we sort out cases where temporary page-sized buffers
1631 * are used.
1632 */
1633 switch (cap) {
1634 case DMA_PQ:
1635 if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1636 return NULL;
1637 if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1638 return NULL;
1639 break;
1640 case DMA_PQ_VAL:
1641 case DMA_XOR_VAL:
1642 return NULL;
1643 default:
1644 break;
1645 }
1646 }
1647
1648 list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1649 if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1650 int rank;
1651
1652 rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1653 dst_cnt, src_lst, src_cnt, src_sz);
1654 if (rank > best_rank) {
1655 best_rank = rank;
1656 best_chan = ref->chan;
1657 }
1658 }
1659 }
1660
1661 return best_chan;
1662}
1663EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1664
1665/**
1666 * ppc440spe_get_group_entry - get group entry with index idx
1667 * @tdesc: is the last allocated slot in the group.
1668 */
1669static struct ppc440spe_adma_desc_slot *
1670ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1671{
1672 struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1673 int i = 0;
1674
1675 if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1676 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1677 __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1678 BUG();
1679 }
1680
1681 list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1682 if (i++ == entry_idx)
1683 break;
1684 }
1685 return iter;
1686}
1687
1688/**
1689 * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1690 * @slot: Slot to free
1691 * Caller must hold &ppc440spe_chan->lock while calling this function
1692 */
1693static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1694 struct ppc440spe_adma_chan *chan)
1695{
1696 int stride = slot->slots_per_op;
1697
1698 while (stride--) {
1699 slot->slots_per_op = 0;
1700 slot = list_entry(slot->slot_node.next,
1701 struct ppc440spe_adma_desc_slot,
1702 slot_node);
1703 }
1704}
1705
1706static void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan,
1707 struct ppc440spe_adma_desc_slot *desc)
1708{
1709 u32 src_cnt, dst_cnt;
1710 dma_addr_t addr;
1711
1712 /*
1713 * get the number of sources & destination
1714 * included in this descriptor and unmap
1715 * them all
1716 */
1717 src_cnt = ppc440spe_desc_get_src_num(desc, chan);
1718 dst_cnt = ppc440spe_desc_get_dst_num(desc, chan);
1719
1720 /* unmap destinations */
1721 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1722 while (dst_cnt--) {
1723 addr = ppc440spe_desc_get_dest_addr(
1724 desc, chan, dst_cnt);
1725 dma_unmap_page(chan->device->dev,
1726 addr, desc->unmap_len,
1727 DMA_FROM_DEVICE);
1728 }
1729 }
1730
1731 /* unmap sources */
1732 if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1733 while (src_cnt--) {
1734 addr = ppc440spe_desc_get_src_addr(
1735 desc, chan, src_cnt);
1736 dma_unmap_page(chan->device->dev,
1737 addr, desc->unmap_len,
1738 DMA_TO_DEVICE);
1739 }
1740 }
1741}
1742
1743/**
1744 * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1745 * upon completion
1746 */
1747static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1748 struct ppc440spe_adma_desc_slot *desc,
1749 struct ppc440spe_adma_chan *chan,
1750 dma_cookie_t cookie)
1751{
1752 int i;
1753
1754 BUG_ON(desc->async_tx.cookie < 0);
1755 if (desc->async_tx.cookie > 0) {
1756 cookie = desc->async_tx.cookie;
1757 desc->async_tx.cookie = 0;
1758
1759 /* call the callback (must not sleep or submit new
1760 * operations to this channel)
1761 */
1762 if (desc->async_tx.callback)
1763 desc->async_tx.callback(
1764 desc->async_tx.callback_param);
1765
1766 /* unmap dma addresses
1767 * (unmap_single vs unmap_page?)
1768 *
1769 * actually, ppc's dma_unmap_page() functions are empty, so
1770 * the following code is just for the sake of completeness
1771 */
1772 if (chan && chan->needs_unmap && desc->group_head &&
1773 desc->unmap_len) {
1774 struct ppc440spe_adma_desc_slot *unmap =
1775 desc->group_head;
1776 /* assume 1 slot per op always */
1777 u32 slot_count = unmap->slot_cnt;
1778
1779 /* Run through the group list and unmap addresses */
1780 for (i = 0; i < slot_count; i++) {
1781 BUG_ON(!unmap);
1782 ppc440spe_adma_unmap(chan, unmap);
1783 unmap = unmap->hw_next;
1784 }
1785 }
1786 }
1787
1788 /* run dependent operations */
1789 dma_run_dependencies(&desc->async_tx);
1790
1791 return cookie;
1792}
1793
1794/**
1795 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1796 */
1797static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1798 struct ppc440spe_adma_chan *chan)
1799{
1800 /* the client is allowed to attach dependent operations
1801 * until 'ack' is set
1802 */
1803 if (!async_tx_test_ack(&desc->async_tx))
1804 return 0;
1805
1806 /* leave the last descriptor in the chain
1807 * so we can append to it
1808 */
1809 if (list_is_last(&desc->chain_node, &chan->chain) ||
1810 desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1811 return 1;
1812
1813 if (chan->device->id != PPC440SPE_XOR_ID) {
1814 /* our DMA interrupt handler clears opc field of
1815 * each processed descriptor. For all types of
1816 * operations except for ZeroSum we do not actually
1817 * need ack from the interrupt handler. ZeroSum is a
1818 * special case since the result of this operation
1819 * is available from the handler only, so if we see
1820 * such type of descriptor (which is unprocessed yet)
1821 * then leave it in chain.
1822 */
1823 struct dma_cdb *cdb = desc->hw_desc;
1824 if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1825 return 1;
1826 }
1827
1828 dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1829 desc->phys, desc->idx, desc->slots_per_op);
1830
1831 list_del(&desc->chain_node);
1832 ppc440spe_adma_free_slots(desc, chan);
1833 return 0;
1834}
1835
1836/**
1837 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1838 * which runs through the channel CDBs list until reach the descriptor
1839 * currently processed. When routine determines that all CDBs of group
1840 * are completed then corresponding callbacks (if any) are called and slots
1841 * are freed.
1842 */
1843static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1844{
1845 struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1846 dma_cookie_t cookie = 0;
1847 u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1848 int busy = ppc440spe_chan_is_busy(chan);
1849 int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1850
1851 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1852 chan->device->id, __func__);
1853
1854 if (!current_desc) {
1855 /* There were no transactions yet, so
1856 * nothing to clean
1857 */
1858 return;
1859 }
1860
1861 /* free completed slots from the chain starting with
1862 * the oldest descriptor
1863 */
1864 list_for_each_entry_safe(iter, _iter, &chan->chain,
1865 chain_node) {
1866 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1867 "busy: %d this_desc: %#llx next_desc: %#x "
1868 "cur: %#x ack: %d\n",
1869 iter->async_tx.cookie, iter->idx, busy, iter->phys,
1870 ppc440spe_desc_get_link(iter, chan), current_desc,
1871 async_tx_test_ack(&iter->async_tx));
1872 prefetch(_iter);
1873 prefetch(&_iter->async_tx);
1874
1875 /* do not advance past the current descriptor loaded into the
1876 * hardware channel,subsequent descriptors are either in process
1877 * or have not been submitted
1878 */
1879 if (seen_current)
1880 break;
1881
1882 /* stop the search if we reach the current descriptor and the
1883 * channel is busy, or if it appears that the current descriptor
1884 * needs to be re-read (i.e. has been appended to)
1885 */
1886 if (iter->phys == current_desc) {
1887 BUG_ON(seen_current++);
1888 if (busy || ppc440spe_desc_get_link(iter, chan)) {
1889 /* not all descriptors of the group have
1890 * been completed; exit.
1891 */
1892 break;
1893 }
1894 }
1895
1896 /* detect the start of a group transaction */
1897 if (!slot_cnt && !slots_per_op) {
1898 slot_cnt = iter->slot_cnt;
1899 slots_per_op = iter->slots_per_op;
1900 if (slot_cnt <= slots_per_op) {
1901 slot_cnt = 0;
1902 slots_per_op = 0;
1903 }
1904 }
1905
1906 if (slot_cnt) {
1907 if (!group_start)
1908 group_start = iter;
1909 slot_cnt -= slots_per_op;
1910 }
1911
1912 /* all the members of a group are complete */
1913 if (slots_per_op != 0 && slot_cnt == 0) {
1914 struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1915 int end_of_chain = 0;
1916
1917 /* clean up the group */
1918 slot_cnt = group_start->slot_cnt;
1919 grp_iter = group_start;
1920 list_for_each_entry_safe_from(grp_iter, _grp_iter,
1921 &chan->chain, chain_node) {
1922
1923 cookie = ppc440spe_adma_run_tx_complete_actions(
1924 grp_iter, chan, cookie);
1925
1926 slot_cnt -= slots_per_op;
1927 end_of_chain = ppc440spe_adma_clean_slot(
1928 grp_iter, chan);
1929 if (end_of_chain && slot_cnt) {
1930 /* Should wait for ZeroSum completion */
1931 if (cookie > 0)
1932 chan->completed_cookie = cookie;
1933 return;
1934 }
1935
1936 if (slot_cnt == 0 || end_of_chain)
1937 break;
1938 }
1939
1940 /* the group should be complete at this point */
1941 BUG_ON(slot_cnt);
1942
1943 slots_per_op = 0;
1944 group_start = NULL;
1945 if (end_of_chain)
1946 break;
1947 else
1948 continue;
1949 } else if (slots_per_op) /* wait for group completion */
1950 continue;
1951
1952 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1953 cookie);
1954
1955 if (ppc440spe_adma_clean_slot(iter, chan))
1956 break;
1957 }
1958
1959 BUG_ON(!seen_current);
1960
1961 if (cookie > 0) {
1962 chan->completed_cookie = cookie;
1963 pr_debug("\tcompleted cookie %d\n", cookie);
1964 }
1965
1966}
1967
1968/**
1969 * ppc440spe_adma_tasklet - clean up watch-dog initiator
1970 */
1971static void ppc440spe_adma_tasklet(unsigned long data)
1972{
1973 struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data;
1974
1975 spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1976 __ppc440spe_adma_slot_cleanup(chan);
1977 spin_unlock(&chan->lock);
1978}
1979
1980/**
1981 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1982 */
1983static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1984{
1985 spin_lock_bh(&chan->lock);
1986 __ppc440spe_adma_slot_cleanup(chan);
1987 spin_unlock_bh(&chan->lock);
1988}
1989
1990/**
1991 * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1992 */
1993static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1994 struct ppc440spe_adma_chan *chan, int num_slots,
1995 int slots_per_op)
1996{
1997 struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1998 struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1999 struct list_head chain = LIST_HEAD_INIT(chain);
2000 int slots_found, retry = 0;
2001
2002
2003 BUG_ON(!num_slots || !slots_per_op);
2004 /* start search from the last allocated descrtiptor
2005 * if a contiguous allocation can not be found start searching
2006 * from the beginning of the list
2007 */
2008retry:
2009 slots_found = 0;
2010 if (retry == 0)
2011 iter = chan->last_used;
2012 else
2013 iter = list_entry(&chan->all_slots,
2014 struct ppc440spe_adma_desc_slot,
2015 slot_node);
2016 list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
2017 slot_node) {
2018 prefetch(_iter);
2019 prefetch(&_iter->async_tx);
2020 if (iter->slots_per_op) {
2021 slots_found = 0;
2022 continue;
2023 }
2024
2025 /* start the allocation if the slot is correctly aligned */
2026 if (!slots_found++)
2027 alloc_start = iter;
2028
2029 if (slots_found == num_slots) {
2030 struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
2031 struct ppc440spe_adma_desc_slot *last_used = NULL;
2032
2033 iter = alloc_start;
2034 while (num_slots) {
2035 int i;
2036 /* pre-ack all but the last descriptor */
2037 if (num_slots != slots_per_op)
2038 async_tx_ack(&iter->async_tx);
2039
2040 list_add_tail(&iter->chain_node, &chain);
2041 alloc_tail = iter;
2042 iter->async_tx.cookie = 0;
2043 iter->hw_next = NULL;
2044 iter->flags = 0;
2045 iter->slot_cnt = num_slots;
2046 iter->xor_check_result = NULL;
2047 for (i = 0; i < slots_per_op; i++) {
2048 iter->slots_per_op = slots_per_op - i;
2049 last_used = iter;
2050 iter = list_entry(iter->slot_node.next,
2051 struct ppc440spe_adma_desc_slot,
2052 slot_node);
2053 }
2054 num_slots -= slots_per_op;
2055 }
2056 alloc_tail->group_head = alloc_start;
2057 alloc_tail->async_tx.cookie = -EBUSY;
2058 list_splice(&chain, &alloc_tail->group_list);
2059 chan->last_used = last_used;
2060 return alloc_tail;
2061 }
2062 }
2063 if (!retry++)
2064 goto retry;
2065
2066 /* try to free some slots if the allocation fails */
2067 tasklet_schedule(&chan->irq_tasklet);
2068 return NULL;
2069}
2070
2071/**
2072 * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots
2073 */
2074static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
2075{
2076 struct ppc440spe_adma_chan *ppc440spe_chan;
2077 struct ppc440spe_adma_desc_slot *slot = NULL;
2078 char *hw_desc;
2079 int i, db_sz;
2080 int init;
2081
2082 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2083 init = ppc440spe_chan->slots_allocated ? 0 : 1;
2084 chan->chan_id = ppc440spe_chan->device->id;
2085
2086 /* Allocate descriptor slots */
2087 i = ppc440spe_chan->slots_allocated;
2088 if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
2089 db_sz = sizeof(struct dma_cdb);
2090 else
2091 db_sz = sizeof(struct xor_cb);
2092
2093 for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
2094 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
2095 GFP_KERNEL);
2096 if (!slot) {
2097 printk(KERN_INFO "SPE ADMA Channel only initialized"
2098 " %d descriptor slots", i--);
2099 break;
2100 }
2101
2102 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
2103 slot->hw_desc = (void *) &hw_desc[i * db_sz];
2104 dma_async_tx_descriptor_init(&slot->async_tx, chan);
2105 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
2106 INIT_LIST_HEAD(&slot->chain_node);
2107 INIT_LIST_HEAD(&slot->slot_node);
2108 INIT_LIST_HEAD(&slot->group_list);
2109 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
2110 slot->idx = i;
2111
2112 spin_lock_bh(&ppc440spe_chan->lock);
2113 ppc440spe_chan->slots_allocated++;
2114 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
2115 spin_unlock_bh(&ppc440spe_chan->lock);
2116 }
2117
2118 if (i && !ppc440spe_chan->last_used) {
2119 ppc440spe_chan->last_used =
2120 list_entry(ppc440spe_chan->all_slots.next,
2121 struct ppc440spe_adma_desc_slot,
2122 slot_node);
2123 }
2124
2125 dev_dbg(ppc440spe_chan->device->common.dev,
2126 "ppc440spe adma%d: allocated %d descriptor slots\n",
2127 ppc440spe_chan->device->id, i);
2128
2129 /* initialize the channel and the chain with a null operation */
2130 if (init) {
2131 switch (ppc440spe_chan->device->id) {
2132 case PPC440SPE_DMA0_ID:
2133 case PPC440SPE_DMA1_ID:
2134 ppc440spe_chan->hw_chain_inited = 0;
2135 /* Use WXOR for self-testing */
2136 if (!ppc440spe_r6_tchan)
2137 ppc440spe_r6_tchan = ppc440spe_chan;
2138 break;
2139 case PPC440SPE_XOR_ID:
2140 ppc440spe_chan_start_null_xor(ppc440spe_chan);
2141 break;
2142 default:
2143 BUG();
2144 }
2145 ppc440spe_chan->needs_unmap = 1;
2146 }
2147
2148 return (i > 0) ? i : -ENOMEM;
2149}
2150
2151/**
2152 * ppc440spe_desc_assign_cookie - assign a cookie
2153 */
2154static dma_cookie_t ppc440spe_desc_assign_cookie(
2155 struct ppc440spe_adma_chan *chan,
2156 struct ppc440spe_adma_desc_slot *desc)
2157{
2158 dma_cookie_t cookie = chan->common.cookie;
2159
2160 cookie++;
2161 if (cookie < 0)
2162 cookie = 1;
2163 chan->common.cookie = desc->async_tx.cookie = cookie;
2164 return cookie;
2165}
2166
2167/**
2168 * ppc440spe_rxor_set_region_data -
2169 */
2170static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
2171 u8 xor_arg_no, u32 mask)
2172{
2173 struct xor_cb *xcb = desc->hw_desc;
2174
2175 xcb->ops[xor_arg_no].h |= mask;
2176}
2177
2178/**
2179 * ppc440spe_rxor_set_src -
2180 */
2181static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
2182 u8 xor_arg_no, dma_addr_t addr)
2183{
2184 struct xor_cb *xcb = desc->hw_desc;
2185
2186 xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
2187 xcb->ops[xor_arg_no].l = addr;
2188}
2189
2190/**
2191 * ppc440spe_rxor_set_mult -
2192 */
2193static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
2194 u8 xor_arg_no, u8 idx, u8 mult)
2195{
2196 struct xor_cb *xcb = desc->hw_desc;
2197
2198 xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
2199}
2200
2201/**
2202 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
2203 * has been achieved
2204 */
2205static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
2206{
2207 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
2208 chan->device->id, chan->pending);
2209
2210 if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
2211 chan->pending = 0;
2212 ppc440spe_chan_append(chan);
2213 }
2214}
2215
2216/**
2217 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
2218 * (it's not necessary that descriptors will be submitted to the h/w
2219 * chains too right now)
2220 */
2221static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
2222{
2223 struct ppc440spe_adma_desc_slot *sw_desc;
2224 struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
2225 struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
2226 int slot_cnt;
2227 int slots_per_op;
2228 dma_cookie_t cookie;
2229
2230 sw_desc = tx_to_ppc440spe_adma_slot(tx);
2231
2232 group_start = sw_desc->group_head;
2233 slot_cnt = group_start->slot_cnt;
2234 slots_per_op = group_start->slots_per_op;
2235
2236 spin_lock_bh(&chan->lock);
2237
2238 cookie = ppc440spe_desc_assign_cookie(chan, sw_desc);
2239
2240 if (unlikely(list_empty(&chan->chain))) {
2241 /* first peer */
2242 list_splice_init(&sw_desc->group_list, &chan->chain);
2243 chan_first_cdb[chan->device->id] = group_start;
2244 } else {
2245 /* isn't first peer, bind CDBs to chain */
2246 old_chain_tail = list_entry(chan->chain.prev,
2247 struct ppc440spe_adma_desc_slot,
2248 chain_node);
2249 list_splice_init(&sw_desc->group_list,
2250 &old_chain_tail->chain_node);
2251 /* fix up the hardware chain */
2252 ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
2253 }
2254
2255 /* increment the pending count by the number of operations */
2256 chan->pending += slot_cnt / slots_per_op;
2257 ppc440spe_adma_check_threshold(chan);
2258 spin_unlock_bh(&chan->lock);
2259
2260 dev_dbg(chan->device->common.dev,
2261 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
2262 chan->device->id, __func__,
2263 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
2264
2265 return cookie;
2266}
2267
2268/**
2269 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
2270 */
2271static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
2272 struct dma_chan *chan, unsigned long flags)
2273{
2274 struct ppc440spe_adma_chan *ppc440spe_chan;
2275 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2276 int slot_cnt, slots_per_op;
2277
2278 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2279
2280 dev_dbg(ppc440spe_chan->device->common.dev,
2281 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
2282 __func__);
2283
2284 spin_lock_bh(&ppc440spe_chan->lock);
2285 slot_cnt = slots_per_op = 1;
2286 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2287 slots_per_op);
2288 if (sw_desc) {
2289 group_start = sw_desc->group_head;
2290 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
2291 group_start->unmap_len = 0;
2292 sw_desc->async_tx.flags = flags;
2293 }
2294 spin_unlock_bh(&ppc440spe_chan->lock);
2295
2296 return sw_desc ? &sw_desc->async_tx : NULL;
2297}
2298
2299/**
2300 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
2301 */
2302static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
2303 struct dma_chan *chan, dma_addr_t dma_dest,
2304 dma_addr_t dma_src, size_t len, unsigned long flags)
2305{
2306 struct ppc440spe_adma_chan *ppc440spe_chan;
2307 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2308 int slot_cnt, slots_per_op;
2309
2310 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2311
2312 if (unlikely(!len))
2313 return NULL;
2314
2315 BUG_ON(unlikely(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT));
2316
2317 spin_lock_bh(&ppc440spe_chan->lock);
2318
2319 dev_dbg(ppc440spe_chan->device->common.dev,
2320 "ppc440spe adma%d: %s len: %u int_en %d\n",
2321 ppc440spe_chan->device->id, __func__, len,
2322 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2323 slot_cnt = slots_per_op = 1;
2324 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2325 slots_per_op);
2326 if (sw_desc) {
2327 group_start = sw_desc->group_head;
2328 ppc440spe_desc_init_memcpy(group_start, flags);
2329 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2330 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2331 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2332 sw_desc->unmap_len = len;
2333 sw_desc->async_tx.flags = flags;
2334 }
2335 spin_unlock_bh(&ppc440spe_chan->lock);
2336
2337 return sw_desc ? &sw_desc->async_tx : NULL;
2338}
2339
2340/**
2341 * ppc440spe_adma_prep_dma_memset - prepare CDB for a MEMSET operation
2342 */
2343static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memset(
2344 struct dma_chan *chan, dma_addr_t dma_dest, int value,
2345 size_t len, unsigned long flags)
2346{
2347 struct ppc440spe_adma_chan *ppc440spe_chan;
2348 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2349 int slot_cnt, slots_per_op;
2350
2351 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2352
2353 if (unlikely(!len))
2354 return NULL;
2355
2356 BUG_ON(unlikely(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT));
2357
2358 spin_lock_bh(&ppc440spe_chan->lock);
2359
2360 dev_dbg(ppc440spe_chan->device->common.dev,
2361 "ppc440spe adma%d: %s cal: %u len: %u int_en %d\n",
2362 ppc440spe_chan->device->id, __func__, value, len,
2363 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2364
2365 slot_cnt = slots_per_op = 1;
2366 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2367 slots_per_op);
2368 if (sw_desc) {
2369 group_start = sw_desc->group_head;
2370 ppc440spe_desc_init_memset(group_start, value, flags);
2371 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2372 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2373 sw_desc->unmap_len = len;
2374 sw_desc->async_tx.flags = flags;
2375 }
2376 spin_unlock_bh(&ppc440spe_chan->lock);
2377
2378 return sw_desc ? &sw_desc->async_tx : NULL;
2379}
2380
2381/**
2382 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2383 */
2384static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2385 struct dma_chan *chan, dma_addr_t dma_dest,
2386 dma_addr_t *dma_src, u32 src_cnt, size_t len,
2387 unsigned long flags)
2388{
2389 struct ppc440spe_adma_chan *ppc440spe_chan;
2390 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2391 int slot_cnt, slots_per_op;
2392
2393 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2394
2395 ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2396 dma_dest, dma_src, src_cnt));
2397 if (unlikely(!len))
2398 return NULL;
2399 BUG_ON(unlikely(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT));
2400
2401 dev_dbg(ppc440spe_chan->device->common.dev,
2402 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2403 ppc440spe_chan->device->id, __func__, src_cnt, len,
2404 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2405
2406 spin_lock_bh(&ppc440spe_chan->lock);
2407 slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2408 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2409 slots_per_op);
2410 if (sw_desc) {
2411 group_start = sw_desc->group_head;
2412 ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2413 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2414 while (src_cnt--)
2415 ppc440spe_adma_memcpy_xor_set_src(group_start,
2416 dma_src[src_cnt], src_cnt);
2417 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2418 sw_desc->unmap_len = len;
2419 sw_desc->async_tx.flags = flags;
2420 }
2421 spin_unlock_bh(&ppc440spe_chan->lock);
2422
2423 return sw_desc ? &sw_desc->async_tx : NULL;
2424}
2425
2426static inline void
2427ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2428 int src_cnt);
2429static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2430
2431/**
2432 * ppc440spe_adma_init_dma2rxor_slot -
2433 */
2434static void ppc440spe_adma_init_dma2rxor_slot(
2435 struct ppc440spe_adma_desc_slot *desc,
2436 dma_addr_t *src, int src_cnt)
2437{
2438 int i;
2439
2440 /* initialize CDB */
2441 for (i = 0; i < src_cnt; i++) {
2442 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2443 desc->src_cnt, (u32)src[i]);
2444 }
2445}
2446
2447/**
2448 * ppc440spe_dma01_prep_mult -
2449 * for Q operation where destination is also the source
2450 */
2451static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2452 struct ppc440spe_adma_chan *ppc440spe_chan,
2453 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2454 const unsigned char *scf, size_t len, unsigned long flags)
2455{
2456 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2457 unsigned long op = 0;
2458 int slot_cnt;
2459
2460 set_bit(PPC440SPE_DESC_WXOR, &op);
2461 slot_cnt = 2;
2462
2463 spin_lock_bh(&ppc440spe_chan->lock);
2464
2465 /* use WXOR, each descriptor occupies one slot */
2466 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2467 if (sw_desc) {
2468 struct ppc440spe_adma_chan *chan;
2469 struct ppc440spe_adma_desc_slot *iter;
2470 struct dma_cdb *hw_desc;
2471
2472 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2473 set_bits(op, &sw_desc->flags);
2474 sw_desc->src_cnt = src_cnt;
2475 sw_desc->dst_cnt = dst_cnt;
2476 /* First descriptor, zero data in the destination and copy it
2477 * to q page using MULTICAST transfer.
2478 */
2479 iter = list_first_entry(&sw_desc->group_list,
2480 struct ppc440spe_adma_desc_slot,
2481 chain_node);
2482 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2483 /* set 'next' pointer */
2484 iter->hw_next = list_entry(iter->chain_node.next,
2485 struct ppc440spe_adma_desc_slot,
2486 chain_node);
2487 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2488 hw_desc = iter->hw_desc;
2489 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2490
2491 ppc440spe_desc_set_dest_addr(iter, chan,
2492 DMA_CUED_XOR_BASE, dst[0], 0);
2493 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2494 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2495 src[0]);
2496 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2497 iter->unmap_len = len;
2498
2499 /*
2500 * Second descriptor, multiply data from the q page
2501 * and store the result in real destination.
2502 */
2503 iter = list_first_entry(&iter->chain_node,
2504 struct ppc440spe_adma_desc_slot,
2505 chain_node);
2506 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2507 iter->hw_next = NULL;
2508 if (flags & DMA_PREP_INTERRUPT)
2509 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2510 else
2511 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2512
2513 hw_desc = iter->hw_desc;
2514 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2515 ppc440spe_desc_set_src_addr(iter, chan, 0,
2516 DMA_CUED_XOR_HB, dst[1]);
2517 ppc440spe_desc_set_dest_addr(iter, chan,
2518 DMA_CUED_XOR_BASE, dst[0], 0);
2519
2520 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2521 DMA_CDB_SG_DST1, scf[0]);
2522 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2523 iter->unmap_len = len;
2524 sw_desc->async_tx.flags = flags;
2525 }
2526
2527 spin_unlock_bh(&ppc440spe_chan->lock);
2528
2529 return sw_desc;
2530}
2531
2532/**
2533 * ppc440spe_dma01_prep_sum_product -
2534 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2535 * the source.
2536 */
2537static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2538 struct ppc440spe_adma_chan *ppc440spe_chan,
2539 dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2540 const unsigned char *scf, size_t len, unsigned long flags)
2541{
2542 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2543 unsigned long op = 0;
2544 int slot_cnt;
2545
2546 set_bit(PPC440SPE_DESC_WXOR, &op);
2547 slot_cnt = 3;
2548
2549 spin_lock_bh(&ppc440spe_chan->lock);
2550
2551 /* WXOR, each descriptor occupies one slot */
2552 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2553 if (sw_desc) {
2554 struct ppc440spe_adma_chan *chan;
2555 struct ppc440spe_adma_desc_slot *iter;
2556 struct dma_cdb *hw_desc;
2557
2558 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2559 set_bits(op, &sw_desc->flags);
2560 sw_desc->src_cnt = src_cnt;
2561 sw_desc->dst_cnt = 1;
2562 /* 1st descriptor, src[1] data to q page and zero destination */
2563 iter = list_first_entry(&sw_desc->group_list,
2564 struct ppc440spe_adma_desc_slot,
2565 chain_node);
2566 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2567 iter->hw_next = list_entry(iter->chain_node.next,
2568 struct ppc440spe_adma_desc_slot,
2569 chain_node);
2570 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2571 hw_desc = iter->hw_desc;
2572 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2573
2574 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2575 *dst, 0);
2576 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2577 ppc440spe_chan->qdest, 1);
2578 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2579 src[1]);
2580 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2581 iter->unmap_len = len;
2582
2583 /* 2nd descriptor, multiply src[1] data and store the
2584 * result in destination */
2585 iter = list_first_entry(&iter->chain_node,
2586 struct ppc440spe_adma_desc_slot,
2587 chain_node);
2588 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2589 /* set 'next' pointer */
2590 iter->hw_next = list_entry(iter->chain_node.next,
2591 struct ppc440spe_adma_desc_slot,
2592 chain_node);
2593 if (flags & DMA_PREP_INTERRUPT)
2594 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2595 else
2596 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2597
2598 hw_desc = iter->hw_desc;
2599 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2600 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2601 ppc440spe_chan->qdest);
2602 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2603 *dst, 0);
2604 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2605 DMA_CDB_SG_DST1, scf[1]);
2606 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2607 iter->unmap_len = len;
2608
2609 /*
2610 * 3rd descriptor, multiply src[0] data and xor it
2611 * with destination
2612 */
2613 iter = list_first_entry(&iter->chain_node,
2614 struct ppc440spe_adma_desc_slot,
2615 chain_node);
2616 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2617 iter->hw_next = NULL;
2618 if (flags & DMA_PREP_INTERRUPT)
2619 set_bit(PPC440SPE_DESC_INT, &iter->flags);
2620 else
2621 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2622
2623 hw_desc = iter->hw_desc;
2624 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2625 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2626 src[0]);
2627 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2628 *dst, 0);
2629 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2630 DMA_CDB_SG_DST1, scf[0]);
2631 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2632 iter->unmap_len = len;
2633 sw_desc->async_tx.flags = flags;
2634 }
2635
2636 spin_unlock_bh(&ppc440spe_chan->lock);
2637
2638 return sw_desc;
2639}
2640
2641static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2642 struct ppc440spe_adma_chan *ppc440spe_chan,
2643 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2644 const unsigned char *scf, size_t len, unsigned long flags)
2645{
2646 int slot_cnt;
2647 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2648 unsigned long op = 0;
2649 unsigned char mult = 1;
2650
2651 pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2652 __func__, dst_cnt, src_cnt, len);
2653 /* select operations WXOR/RXOR depending on the
2654 * source addresses of operators and the number
2655 * of destinations (RXOR support only Q-parity calculations)
2656 */
2657 set_bit(PPC440SPE_DESC_WXOR, &op);
2658 if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2659 /* no active RXOR;
2660 * do RXOR if:
2661 * - there are more than 1 source,
2662 * - len is aligned on 512-byte boundary,
2663 * - source addresses fit to one of 4 possible regions.
2664 */
2665 if (src_cnt > 1 &&
2666 !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2667 (src[0] + len) == src[1]) {
2668 /* may do RXOR R1 R2 */
2669 set_bit(PPC440SPE_DESC_RXOR, &op);
2670 if (src_cnt != 2) {
2671 /* may try to enhance region of RXOR */
2672 if ((src[1] + len) == src[2]) {
2673 /* do RXOR R1 R2 R3 */
2674 set_bit(PPC440SPE_DESC_RXOR123,
2675 &op);
2676 } else if ((src[1] + len * 2) == src[2]) {
2677 /* do RXOR R1 R2 R4 */
2678 set_bit(PPC440SPE_DESC_RXOR124, &op);
2679 } else if ((src[1] + len * 3) == src[2]) {
2680 /* do RXOR R1 R2 R5 */
2681 set_bit(PPC440SPE_DESC_RXOR125,
2682 &op);
2683 } else {
2684 /* do RXOR R1 R2 */
2685 set_bit(PPC440SPE_DESC_RXOR12,
2686 &op);
2687 }
2688 } else {
2689 /* do RXOR R1 R2 */
2690 set_bit(PPC440SPE_DESC_RXOR12, &op);
2691 }
2692 }
2693
2694 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2695 /* can not do this operation with RXOR */
2696 clear_bit(PPC440SPE_RXOR_RUN,
2697 &ppc440spe_rxor_state);
2698 } else {
2699 /* can do; set block size right now */
2700 ppc440spe_desc_set_rxor_block_size(len);
2701 }
2702 }
2703
2704 /* Number of necessary slots depends on operation type selected */
2705 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2706 /* This is a WXOR only chain. Need descriptors for each
2707 * source to GF-XOR them with WXOR, and need descriptors
2708 * for each destination to zero them with WXOR
2709 */
2710 slot_cnt = src_cnt;
2711
2712 if (flags & DMA_PREP_ZERO_P) {
2713 slot_cnt++;
2714 set_bit(PPC440SPE_ZERO_P, &op);
2715 }
2716 if (flags & DMA_PREP_ZERO_Q) {
2717 slot_cnt++;
2718 set_bit(PPC440SPE_ZERO_Q, &op);
2719 }
2720 } else {
2721 /* Need 1/2 descriptor for RXOR operation, and
2722 * need (src_cnt - (2 or 3)) for WXOR of sources
2723 * remained (if any)
2724 */
2725 slot_cnt = dst_cnt;
2726
2727 if (flags & DMA_PREP_ZERO_P)
2728 set_bit(PPC440SPE_ZERO_P, &op);
2729 if (flags & DMA_PREP_ZERO_Q)
2730 set_bit(PPC440SPE_ZERO_Q, &op);
2731
2732 if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2733 slot_cnt += src_cnt - 2;
2734 else
2735 slot_cnt += src_cnt - 3;
2736
2737 /* Thus we have either RXOR only chain or
2738 * mixed RXOR/WXOR
2739 */
2740 if (slot_cnt == dst_cnt)
2741 /* RXOR only chain */
2742 clear_bit(PPC440SPE_DESC_WXOR, &op);
2743 }
2744
2745 spin_lock_bh(&ppc440spe_chan->lock);
2746 /* for both RXOR/WXOR each descriptor occupies one slot */
2747 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2748 if (sw_desc) {
2749 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2750 flags, op);
2751
2752 /* setup dst/src/mult */
2753 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2754 __func__, dst[0], dst[1]);
2755 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2756 while (src_cnt--) {
2757 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2758 src_cnt);
2759
2760 /* NOTE: "Multi = 0 is equivalent to = 1" as it
2761 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2762 * doesn't work for RXOR with DMA0/1! Instead, multi=0
2763 * leads to zeroing source data after RXOR.
2764 * So, for P case set-up mult=1 explicitly.
2765 */
2766 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2767 mult = scf[src_cnt];
2768 ppc440spe_adma_pq_set_src_mult(sw_desc,
2769 mult, src_cnt, dst_cnt - 1);
2770 }
2771
2772 /* Setup byte count foreach slot just allocated */
2773 sw_desc->async_tx.flags = flags;
2774 list_for_each_entry(iter, &sw_desc->group_list,
2775 chain_node) {
2776 ppc440spe_desc_set_byte_count(iter,
2777 ppc440spe_chan, len);
2778 iter->unmap_len = len;
2779 }
2780 }
2781 spin_unlock_bh(&ppc440spe_chan->lock);
2782
2783 return sw_desc;
2784}
2785
2786static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2787 struct ppc440spe_adma_chan *ppc440spe_chan,
2788 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2789 const unsigned char *scf, size_t len, unsigned long flags)
2790{
2791 int slot_cnt, descs_per_op;
2792 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2793 unsigned long op = 0;
2794 unsigned char mult = 1;
2795
2796 BUG_ON(!dst_cnt);
2797 /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2798 __func__, dst_cnt, src_cnt, len);*/
2799
2800 spin_lock_bh(&ppc440spe_chan->lock);
2801 descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2802 if (descs_per_op < 0) {
2803 spin_unlock_bh(&ppc440spe_chan->lock);
2804 return NULL;
2805 }
2806
2807 /* depending on number of sources we have 1 or 2 RXOR chains */
2808 slot_cnt = descs_per_op * dst_cnt;
2809
2810 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2811 if (sw_desc) {
2812 op = slot_cnt;
2813 sw_desc->async_tx.flags = flags;
2814 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2815 ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2816 --op ? 0 : flags);
2817 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2818 len);
2819 iter->unmap_len = len;
2820
2821 ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2822 iter->rxor_cursor.len = len;
2823 iter->descs_per_op = descs_per_op;
2824 }
2825 op = 0;
2826 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2827 op++;
2828 if (op % descs_per_op == 0)
2829 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2830 src_cnt);
2831 if (likely(!list_is_last(&iter->chain_node,
2832 &sw_desc->group_list))) {
2833 /* set 'next' pointer */
2834 iter->hw_next =
2835 list_entry(iter->chain_node.next,
2836 struct ppc440spe_adma_desc_slot,
2837 chain_node);
2838 ppc440spe_xor_set_link(iter, iter->hw_next);
2839 } else {
2840 /* this is the last descriptor. */
2841 iter->hw_next = NULL;
2842 }
2843 }
2844
2845 /* fixup head descriptor */
2846 sw_desc->dst_cnt = dst_cnt;
2847 if (flags & DMA_PREP_ZERO_P)
2848 set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2849 if (flags & DMA_PREP_ZERO_Q)
2850 set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2851
2852 /* setup dst/src/mult */
2853 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2854
2855 while (src_cnt--) {
2856 /* handle descriptors (if dst_cnt == 2) inside
2857 * the ppc440spe_adma_pq_set_srcxxx() functions
2858 */
2859 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2860 src_cnt);
2861 if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2862 mult = scf[src_cnt];
2863 ppc440spe_adma_pq_set_src_mult(sw_desc,
2864 mult, src_cnt, dst_cnt - 1);
2865 }
2866 }
2867 spin_unlock_bh(&ppc440spe_chan->lock);
2868 ppc440spe_desc_set_rxor_block_size(len);
2869 return sw_desc;
2870}
2871
2872/**
2873 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2874 */
2875static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2876 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2877 unsigned int src_cnt, const unsigned char *scf,
2878 size_t len, unsigned long flags)
2879{
2880 struct ppc440spe_adma_chan *ppc440spe_chan;
2881 struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2882 int dst_cnt = 0;
2883
2884 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2885
2886 ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2887 dst, src, src_cnt));
2888 BUG_ON(!len);
2889 BUG_ON(unlikely(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT));
2890 BUG_ON(!src_cnt);
2891
2892 if (src_cnt == 1 && dst[1] == src[0]) {
2893 dma_addr_t dest[2];
2894
2895 /* dst[1] is real destination (Q) */
2896 dest[0] = dst[1];
2897 /* this is the page to multicast source data to */
2898 dest[1] = ppc440spe_chan->qdest;
2899 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2900 dest, 2, src, src_cnt, scf, len, flags);
2901 return sw_desc ? &sw_desc->async_tx : NULL;
2902 }
2903
2904 if (src_cnt == 2 && dst[1] == src[1]) {
2905 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2906 &dst[1], src, 2, scf, len, flags);
2907 return sw_desc ? &sw_desc->async_tx : NULL;
2908 }
2909
2910 if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2911 BUG_ON(!dst[0]);
2912 dst_cnt++;
2913 flags |= DMA_PREP_ZERO_P;
2914 }
2915
2916 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2917 BUG_ON(!dst[1]);
2918 dst_cnt++;
2919 flags |= DMA_PREP_ZERO_Q;
2920 }
2921
2922 BUG_ON(!dst_cnt);
2923
2924 dev_dbg(ppc440spe_chan->device->common.dev,
2925 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2926 ppc440spe_chan->device->id, __func__, src_cnt, len,
2927 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2928
2929 switch (ppc440spe_chan->device->id) {
2930 case PPC440SPE_DMA0_ID:
2931 case PPC440SPE_DMA1_ID:
2932 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2933 dst, dst_cnt, src, src_cnt, scf,
2934 len, flags);
2935 break;
2936
2937 case PPC440SPE_XOR_ID:
2938 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2939 dst, dst_cnt, src, src_cnt, scf,
2940 len, flags);
2941 break;
2942 }
2943
2944 return sw_desc ? &sw_desc->async_tx : NULL;
2945}
2946
2947/**
2948 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2949 * a PQ_ZERO_SUM operation
2950 */
2951static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2952 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2953 unsigned int src_cnt, const unsigned char *scf, size_t len,
2954 enum sum_check_flags *pqres, unsigned long flags)
2955{
2956 struct ppc440spe_adma_chan *ppc440spe_chan;
2957 struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2958 dma_addr_t pdest, qdest;
2959 int slot_cnt, slots_per_op, idst, dst_cnt;
2960
2961 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2962
2963 if (flags & DMA_PREP_PQ_DISABLE_P)
2964 pdest = 0;
2965 else
2966 pdest = pq[0];
2967
2968 if (flags & DMA_PREP_PQ_DISABLE_Q)
2969 qdest = 0;
2970 else
2971 qdest = pq[1];
2972
2973 ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2974 src, src_cnt, scf));
2975
2976 /* Always use WXOR for P/Q calculations (two destinations).
2977 * Need 1 or 2 extra slots to verify results are zero.
2978 */
2979 idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2980
2981 /* One additional slot per destination to clone P/Q
2982 * before calculation (we have to preserve destinations).
2983 */
2984 slot_cnt = src_cnt + dst_cnt * 2;
2985 slots_per_op = 1;
2986
2987 spin_lock_bh(&ppc440spe_chan->lock);
2988 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2989 slots_per_op);
2990 if (sw_desc) {
2991 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2992
2993 /* Setup byte count for each slot just allocated */
2994 sw_desc->async_tx.flags = flags;
2995 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2996 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2997 len);
2998 iter->unmap_len = len;
2999 }
3000
3001 if (pdest) {
3002 struct dma_cdb *hw_desc;
3003 struct ppc440spe_adma_chan *chan;
3004
3005 iter = sw_desc->group_head;
3006 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3007 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
3008 iter->hw_next = list_entry(iter->chain_node.next,
3009 struct ppc440spe_adma_desc_slot,
3010 chain_node);
3011 hw_desc = iter->hw_desc;
3012 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
3013 iter->src_cnt = 0;
3014 iter->dst_cnt = 0;
3015 ppc440spe_desc_set_dest_addr(iter, chan, 0,
3016 ppc440spe_chan->pdest, 0);
3017 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
3018 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
3019 len);
3020 iter->unmap_len = 0;
3021 /* override pdest to preserve original P */
3022 pdest = ppc440spe_chan->pdest;
3023 }
3024 if (qdest) {
3025 struct dma_cdb *hw_desc;
3026 struct ppc440spe_adma_chan *chan;
3027
3028 iter = list_first_entry(&sw_desc->group_list,
3029 struct ppc440spe_adma_desc_slot,
3030 chain_node);
3031 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3032
3033 if (pdest) {
3034 iter = list_entry(iter->chain_node.next,
3035 struct ppc440spe_adma_desc_slot,
3036 chain_node);
3037 }
3038
3039 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
3040 iter->hw_next = list_entry(iter->chain_node.next,
3041 struct ppc440spe_adma_desc_slot,
3042 chain_node);
3043 hw_desc = iter->hw_desc;
3044 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
3045 iter->src_cnt = 0;
3046 iter->dst_cnt = 0;
3047 ppc440spe_desc_set_dest_addr(iter, chan, 0,
3048 ppc440spe_chan->qdest, 0);
3049 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
3050 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
3051 len);
3052 iter->unmap_len = 0;
3053 /* override qdest to preserve original Q */
3054 qdest = ppc440spe_chan->qdest;
3055 }
3056
3057 /* Setup destinations for P/Q ops */
3058 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
3059
3060 /* Setup zero QWORDs into DCHECK CDBs */
3061 idst = dst_cnt;
3062 list_for_each_entry_reverse(iter, &sw_desc->group_list,
3063 chain_node) {
3064 /*
3065 * The last CDB corresponds to Q-parity check,
3066 * the one before last CDB corresponds
3067 * P-parity check
3068 */
3069 if (idst == DMA_DEST_MAX_NUM) {
3070 if (idst == dst_cnt) {
3071 set_bit(PPC440SPE_DESC_QCHECK,
3072 &iter->flags);
3073 } else {
3074 set_bit(PPC440SPE_DESC_PCHECK,
3075 &iter->flags);
3076 }
3077 } else {
3078 if (qdest) {
3079 set_bit(PPC440SPE_DESC_QCHECK,
3080 &iter->flags);
3081 } else {
3082 set_bit(PPC440SPE_DESC_PCHECK,
3083 &iter->flags);
3084 }
3085 }
3086 iter->xor_check_result = pqres;
3087
3088 /*
3089 * set it to zero, if check fail then result will
3090 * be updated
3091 */
3092 *iter->xor_check_result = 0;
3093 ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
3094 ppc440spe_qword);
3095
3096 if (!(--dst_cnt))
3097 break;
3098 }
3099
3100 /* Setup sources and mults for P/Q ops */
3101 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
3102 chain_node) {
3103 struct ppc440spe_adma_chan *chan;
3104 u32 mult_dst;
3105
3106 chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
3107 ppc440spe_desc_set_src_addr(iter, chan, 0,
3108 DMA_CUED_XOR_HB,
3109 src[src_cnt - 1]);
3110 if (qdest) {
3111 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
3112 DMA_CDB_SG_DST1;
3113 ppc440spe_desc_set_src_mult(iter, chan,
3114 DMA_CUED_MULT1_OFF,
3115 mult_dst,
3116 scf[src_cnt - 1]);
3117 }
3118 if (!(--src_cnt))
3119 break;
3120 }
3121 }
3122 spin_unlock_bh(&ppc440spe_chan->lock);
3123 return sw_desc ? &sw_desc->async_tx : NULL;
3124}
3125
3126/**
3127 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
3128 * XOR ZERO_SUM operation
3129 */
3130static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
3131 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
3132 size_t len, enum sum_check_flags *result, unsigned long flags)
3133{
3134 struct dma_async_tx_descriptor *tx;
3135 dma_addr_t pq[2];
3136
3137 /* validate P, disable Q */
3138 pq[0] = src[0];
3139 pq[1] = 0;
3140 flags |= DMA_PREP_PQ_DISABLE_Q;
3141
3142 tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
3143 src_cnt - 1, 0, len,
3144 result, flags);
3145 return tx;
3146}
3147
3148/**
3149 * ppc440spe_adma_set_dest - set destination address into descriptor
3150 */
3151static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
3152 dma_addr_t addr, int index)
3153{
3154 struct ppc440spe_adma_chan *chan;
3155
3156 BUG_ON(index >= sw_desc->dst_cnt);
3157
3158 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3159
3160 switch (chan->device->id) {
3161 case PPC440SPE_DMA0_ID:
3162 case PPC440SPE_DMA1_ID:
3163 /* to do: support transfers lengths >
3164 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
3165 */
3166 ppc440spe_desc_set_dest_addr(sw_desc->group_head,
3167 chan, 0, addr, index);
3168 break;
3169 case PPC440SPE_XOR_ID:
3170 sw_desc = ppc440spe_get_group_entry(sw_desc, index);
3171 ppc440spe_desc_set_dest_addr(sw_desc,
3172 chan, 0, addr, index);
3173 break;
3174 }
3175}
3176
3177static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
3178 struct ppc440spe_adma_chan *chan, dma_addr_t addr)
3179{
3180 /* To clear destinations update the descriptor
3181 * (P or Q depending on index) as follows:
3182 * addr is destination (0 corresponds to SG2):
3183 */
3184 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
3185
3186 /* ... and the addr is source: */
3187 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
3188
3189 /* addr is always SG2 then the mult is always DST1 */
3190 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
3191 DMA_CDB_SG_DST1, 1);
3192}
3193
3194/**
3195 * ppc440spe_adma_pq_set_dest - set destination address into descriptor
3196 * for the PQXOR operation
3197 */
3198static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
3199 dma_addr_t *addrs, unsigned long flags)
3200{
3201 struct ppc440spe_adma_desc_slot *iter;
3202 struct ppc440spe_adma_chan *chan;
3203 dma_addr_t paddr, qaddr;
3204 dma_addr_t addr = 0, ppath, qpath;
3205 int index = 0, i;
3206
3207 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3208
3209 if (flags & DMA_PREP_PQ_DISABLE_P)
3210 paddr = 0;
3211 else
3212 paddr = addrs[0];
3213
3214 if (flags & DMA_PREP_PQ_DISABLE_Q)
3215 qaddr = 0;
3216 else
3217 qaddr = addrs[1];
3218
3219 if (!paddr || !qaddr)
3220 addr = paddr ? paddr : qaddr;
3221
3222 switch (chan->device->id) {
3223 case PPC440SPE_DMA0_ID:
3224 case PPC440SPE_DMA1_ID:
3225 /* walk through the WXOR source list and set P/Q-destinations
3226 * for each slot:
3227 */
3228 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3229 /* This is WXOR-only chain; may have 1/2 zero descs */
3230 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3231 index++;
3232 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3233 index++;
3234
3235 iter = ppc440spe_get_group_entry(sw_desc, index);
3236 if (addr) {
3237 /* one destination */
3238 list_for_each_entry_from(iter,
3239 &sw_desc->group_list, chain_node)
3240 ppc440spe_desc_set_dest_addr(iter, chan,
3241 DMA_CUED_XOR_BASE, addr, 0);
3242 } else {
3243 /* two destinations */
3244 list_for_each_entry_from(iter,
3245 &sw_desc->group_list, chain_node) {
3246 ppc440spe_desc_set_dest_addr(iter, chan,
3247 DMA_CUED_XOR_BASE, paddr, 0);
3248 ppc440spe_desc_set_dest_addr(iter, chan,
3249 DMA_CUED_XOR_BASE, qaddr, 1);
3250 }
3251 }
3252
3253 if (index) {
3254 /* To clear destinations update the descriptor
3255 * (1st,2nd, or both depending on flags)
3256 */
3257 index = 0;
3258 if (test_bit(PPC440SPE_ZERO_P,
3259 &sw_desc->flags)) {
3260 iter = ppc440spe_get_group_entry(
3261 sw_desc, index++);
3262 ppc440spe_adma_pq_zero_op(iter, chan,
3263 paddr);
3264 }
3265
3266 if (test_bit(PPC440SPE_ZERO_Q,
3267 &sw_desc->flags)) {
3268 iter = ppc440spe_get_group_entry(
3269 sw_desc, index++);
3270 ppc440spe_adma_pq_zero_op(iter, chan,
3271 qaddr);
3272 }
3273
3274 return;
3275 }
3276 } else {
3277 /* This is RXOR-only or RXOR/WXOR mixed chain */
3278
3279 /* If we want to include destination into calculations,
3280 * then make dest addresses cued with mult=1 (XOR).
3281 */
3282 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3283 DMA_CUED_XOR_HB :
3284 DMA_CUED_XOR_BASE |
3285 (1 << DMA_CUED_MULT1_OFF);
3286 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3287 DMA_CUED_XOR_HB :
3288 DMA_CUED_XOR_BASE |
3289 (1 << DMA_CUED_MULT1_OFF);
3290
3291 /* Setup destination(s) in RXOR slot(s) */
3292 iter = ppc440spe_get_group_entry(sw_desc, index++);
3293 ppc440spe_desc_set_dest_addr(iter, chan,
3294 paddr ? ppath : qpath,
3295 paddr ? paddr : qaddr, 0);
3296 if (!addr) {
3297 /* two destinations */
3298 iter = ppc440spe_get_group_entry(sw_desc,
3299 index++);
3300 ppc440spe_desc_set_dest_addr(iter, chan,
3301 qpath, qaddr, 0);
3302 }
3303
3304 if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
3305 /* Setup destination(s) in remaining WXOR
3306 * slots
3307 */
3308 iter = ppc440spe_get_group_entry(sw_desc,
3309 index);
3310 if (addr) {
3311 /* one destination */
3312 list_for_each_entry_from(iter,
3313 &sw_desc->group_list,
3314 chain_node)
3315 ppc440spe_desc_set_dest_addr(
3316 iter, chan,
3317 DMA_CUED_XOR_BASE,
3318 addr, 0);
3319
3320 } else {
3321 /* two destinations */
3322 list_for_each_entry_from(iter,
3323 &sw_desc->group_list,
3324 chain_node) {
3325 ppc440spe_desc_set_dest_addr(
3326 iter, chan,
3327 DMA_CUED_XOR_BASE,
3328 paddr, 0);
3329 ppc440spe_desc_set_dest_addr(
3330 iter, chan,
3331 DMA_CUED_XOR_BASE,
3332 qaddr, 1);
3333 }
3334 }
3335 }
3336
3337 }
3338 break;
3339
3340 case PPC440SPE_XOR_ID:
3341 /* DMA2 descriptors have only 1 destination, so there are
3342 * two chains - one for each dest.
3343 * If we want to include destination into calculations,
3344 * then make dest addresses cued with mult=1 (XOR).
3345 */
3346 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3347 DMA_CUED_XOR_HB :
3348 DMA_CUED_XOR_BASE |
3349 (1 << DMA_CUED_MULT1_OFF);
3350
3351 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3352 DMA_CUED_XOR_HB :
3353 DMA_CUED_XOR_BASE |
3354 (1 << DMA_CUED_MULT1_OFF);
3355
3356 iter = ppc440spe_get_group_entry(sw_desc, 0);
3357 for (i = 0; i < sw_desc->descs_per_op; i++) {
3358 ppc440spe_desc_set_dest_addr(iter, chan,
3359 paddr ? ppath : qpath,
3360 paddr ? paddr : qaddr, 0);
3361 iter = list_entry(iter->chain_node.next,
3362 struct ppc440spe_adma_desc_slot,
3363 chain_node);
3364 }
3365
3366 if (!addr) {
3367 /* Two destinations; setup Q here */
3368 iter = ppc440spe_get_group_entry(sw_desc,
3369 sw_desc->descs_per_op);
3370 for (i = 0; i < sw_desc->descs_per_op; i++) {
3371 ppc440spe_desc_set_dest_addr(iter,
3372 chan, qpath, qaddr, 0);
3373 iter = list_entry(iter->chain_node.next,
3374 struct ppc440spe_adma_desc_slot,
3375 chain_node);
3376 }
3377 }
3378
3379 break;
3380 }
3381}
3382
3383/**
3384 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3385 * for the PQ_ZERO_SUM operation
3386 */
3387static void ppc440spe_adma_pqzero_sum_set_dest(
3388 struct ppc440spe_adma_desc_slot *sw_desc,
3389 dma_addr_t paddr, dma_addr_t qaddr)
3390{
3391 struct ppc440spe_adma_desc_slot *iter, *end;
3392 struct ppc440spe_adma_chan *chan;
3393 dma_addr_t addr = 0;
3394 int idx;
3395
3396 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3397
3398 /* walk through the WXOR source list and set P/Q-destinations
3399 * for each slot
3400 */
3401 idx = (paddr && qaddr) ? 2 : 1;
3402 /* set end */
3403 list_for_each_entry_reverse(end, &sw_desc->group_list,
3404 chain_node) {
3405 if (!(--idx))
3406 break;
3407 }
3408 /* set start */
3409 idx = (paddr && qaddr) ? 2 : 1;
3410 iter = ppc440spe_get_group_entry(sw_desc, idx);
3411
3412 if (paddr && qaddr) {
3413 /* two destinations */
3414 list_for_each_entry_from(iter, &sw_desc->group_list,
3415 chain_node) {
3416 if (unlikely(iter == end))
3417 break;
3418 ppc440spe_desc_set_dest_addr(iter, chan,
3419 DMA_CUED_XOR_BASE, paddr, 0);
3420 ppc440spe_desc_set_dest_addr(iter, chan,
3421 DMA_CUED_XOR_BASE, qaddr, 1);
3422 }
3423 } else {
3424 /* one destination */
3425 addr = paddr ? paddr : qaddr;
3426 list_for_each_entry_from(iter, &sw_desc->group_list,
3427 chain_node) {
3428 if (unlikely(iter == end))
3429 break;
3430 ppc440spe_desc_set_dest_addr(iter, chan,
3431 DMA_CUED_XOR_BASE, addr, 0);
3432 }
3433 }
3434
3435 /* The remaining descriptors are DATACHECK. These have no need in
3436 * destination. Actually, these destinations are used there
3437 * as sources for check operation. So, set addr as source.
3438 */
3439 ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3440
3441 if (!addr) {
3442 end = list_entry(end->chain_node.next,
3443 struct ppc440spe_adma_desc_slot, chain_node);
3444 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3445 }
3446}
3447
3448/**
3449 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3450 */
3451static inline void ppc440spe_desc_set_xor_src_cnt(
3452 struct ppc440spe_adma_desc_slot *desc,
3453 int src_cnt)
3454{
3455 struct xor_cb *hw_desc = desc->hw_desc;
3456
3457 hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3458 hw_desc->cbc |= src_cnt;
3459}
3460
3461/**
3462 * ppc440spe_adma_pq_set_src - set source address into descriptor
3463 */
3464static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3465 dma_addr_t addr, int index)
3466{
3467 struct ppc440spe_adma_chan *chan;
3468 dma_addr_t haddr = 0;
3469 struct ppc440spe_adma_desc_slot *iter = NULL;
3470
3471 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3472
3473 switch (chan->device->id) {
3474 case PPC440SPE_DMA0_ID:
3475 case PPC440SPE_DMA1_ID:
3476 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3477 */
3478 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3479 /* RXOR-only or RXOR/WXOR operation */
3480 int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3481 &sw_desc->flags) ? 2 : 3;
3482
3483 if (index == 0) {
3484 /* 1st slot (RXOR) */
3485 /* setup sources region (R1-2-3, R1-2-4,
3486 * or R1-2-5)
3487 */
3488 if (test_bit(PPC440SPE_DESC_RXOR12,
3489 &sw_desc->flags))
3490 haddr = DMA_RXOR12 <<
3491 DMA_CUED_REGION_OFF;
3492 else if (test_bit(PPC440SPE_DESC_RXOR123,
3493 &sw_desc->flags))
3494 haddr = DMA_RXOR123 <<
3495 DMA_CUED_REGION_OFF;
3496 else if (test_bit(PPC440SPE_DESC_RXOR124,
3497 &sw_desc->flags))
3498 haddr = DMA_RXOR124 <<
3499 DMA_CUED_REGION_OFF;
3500 else if (test_bit(PPC440SPE_DESC_RXOR125,
3501 &sw_desc->flags))
3502 haddr = DMA_RXOR125 <<
3503 DMA_CUED_REGION_OFF;
3504 else
3505 BUG();
3506 haddr |= DMA_CUED_XOR_BASE;
3507 iter = ppc440spe_get_group_entry(sw_desc, 0);
3508 } else if (index < iskip) {
3509 /* 1st slot (RXOR)
3510 * shall actually set source address only once
3511 * instead of first <iskip>
3512 */
3513 iter = NULL;
3514 } else {
3515 /* 2nd/3d and next slots (WXOR);
3516 * skip first slot with RXOR
3517 */
3518 haddr = DMA_CUED_XOR_HB;
3519 iter = ppc440spe_get_group_entry(sw_desc,
3520 index - iskip + sw_desc->dst_cnt);
3521 }
3522 } else {
3523 int znum = 0;
3524
3525 /* WXOR-only operation; skip first slots with
3526 * zeroing destinations
3527 */
3528 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3529 znum++;
3530 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3531 znum++;
3532
3533 haddr = DMA_CUED_XOR_HB;
3534 iter = ppc440spe_get_group_entry(sw_desc,
3535 index + znum);
3536 }
3537
3538 if (likely(iter)) {
3539 ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3540
3541 if (!index &&
3542 test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3543 sw_desc->dst_cnt == 2) {
3544 /* if we have two destinations for RXOR, then
3545 * setup source in the second descr too
3546 */
3547 iter = ppc440spe_get_group_entry(sw_desc, 1);
3548 ppc440spe_desc_set_src_addr(iter, chan, 0,
3549 haddr, addr);
3550 }
3551 }
3552 break;
3553
3554 case PPC440SPE_XOR_ID:
3555 /* DMA2 may do Biskup */
3556 iter = sw_desc->group_head;
3557 if (iter->dst_cnt == 2) {
3558 /* both P & Q calculations required; set P src here */
3559 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3560
3561 /* this is for Q */
3562 iter = ppc440spe_get_group_entry(sw_desc,
3563 sw_desc->descs_per_op);
3564 }
3565 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3566 break;
3567 }
3568}
3569
3570/**
3571 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3572 */
3573static void ppc440spe_adma_memcpy_xor_set_src(
3574 struct ppc440spe_adma_desc_slot *sw_desc,
3575 dma_addr_t addr, int index)
3576{
3577 struct ppc440spe_adma_chan *chan;
3578
3579 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3580 sw_desc = sw_desc->group_head;
3581
3582 if (likely(sw_desc))
3583 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3584}
3585
3586/**
3587 * ppc440spe_adma_dma2rxor_inc_addr -
3588 */
3589static void ppc440spe_adma_dma2rxor_inc_addr(
3590 struct ppc440spe_adma_desc_slot *desc,
3591 struct ppc440spe_rxor *cursor, int index, int src_cnt)
3592{
3593 cursor->addr_count++;
3594 if (index == src_cnt - 1) {
3595 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3596 } else if (cursor->addr_count == XOR_MAX_OPS) {
3597 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3598 cursor->addr_count = 0;
3599 cursor->desc_count++;
3600 }
3601}
3602
3603/**
3604 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3605 */
3606static int ppc440spe_adma_dma2rxor_prep_src(
3607 struct ppc440spe_adma_desc_slot *hdesc,
3608 struct ppc440spe_rxor *cursor, int index,
3609 int src_cnt, u32 addr)
3610{
3611 int rval = 0;
3612 u32 sign;
3613 struct ppc440spe_adma_desc_slot *desc = hdesc;
3614 int i;
3615
3616 for (i = 0; i < cursor->desc_count; i++) {
3617 desc = list_entry(hdesc->chain_node.next,
3618 struct ppc440spe_adma_desc_slot,
3619 chain_node);
3620 }
3621
3622 switch (cursor->state) {
3623 case 0:
3624 if (addr == cursor->addrl + cursor->len) {
3625 /* direct RXOR */
3626 cursor->state = 1;
3627 cursor->xor_count++;
3628 if (index == src_cnt-1) {
3629 ppc440spe_rxor_set_region(desc,
3630 cursor->addr_count,
3631 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3632 ppc440spe_adma_dma2rxor_inc_addr(
3633 desc, cursor, index, src_cnt);
3634 }
3635 } else if (cursor->addrl == addr + cursor->len) {
3636 /* reverse RXOR */
3637 cursor->state = 1;
3638 cursor->xor_count++;
3639 set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3640 if (index == src_cnt-1) {
3641 ppc440spe_rxor_set_region(desc,
3642 cursor->addr_count,
3643 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3644 ppc440spe_adma_dma2rxor_inc_addr(
3645 desc, cursor, index, src_cnt);
3646 }
3647 } else {
3648 printk(KERN_ERR "Cannot build "
3649 "DMA2 RXOR command block.\n");
3650 BUG();
3651 }
3652 break;
3653 case 1:
3654 sign = test_bit(cursor->addr_count,
3655 desc->reverse_flags)
3656 ? -1 : 1;
3657 if (index == src_cnt-2 || (sign == -1
3658 && addr != cursor->addrl - 2*cursor->len)) {
3659 cursor->state = 0;
3660 cursor->xor_count = 1;
3661 cursor->addrl = addr;
3662 ppc440spe_rxor_set_region(desc,
3663 cursor->addr_count,
3664 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3665 ppc440spe_adma_dma2rxor_inc_addr(
3666 desc, cursor, index, src_cnt);
3667 } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3668 cursor->state = 2;
3669 cursor->xor_count = 0;
3670 ppc440spe_rxor_set_region(desc,
3671 cursor->addr_count,
3672 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3673 if (index == src_cnt-1) {
3674 ppc440spe_adma_dma2rxor_inc_addr(
3675 desc, cursor, index, src_cnt);
3676 }
3677 } else if (addr == cursor->addrl + 3*cursor->len) {
3678 cursor->state = 2;
3679 cursor->xor_count = 0;
3680 ppc440spe_rxor_set_region(desc,
3681 cursor->addr_count,
3682 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3683 if (index == src_cnt-1) {
3684 ppc440spe_adma_dma2rxor_inc_addr(
3685 desc, cursor, index, src_cnt);
3686 }
3687 } else if (addr == cursor->addrl + 4*cursor->len) {
3688 cursor->state = 2;
3689 cursor->xor_count = 0;
3690 ppc440spe_rxor_set_region(desc,
3691 cursor->addr_count,
3692 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3693 if (index == src_cnt-1) {
3694 ppc440spe_adma_dma2rxor_inc_addr(
3695 desc, cursor, index, src_cnt);
3696 }
3697 } else {
3698 cursor->state = 0;
3699 cursor->xor_count = 1;
3700 cursor->addrl = addr;
3701 ppc440spe_rxor_set_region(desc,
3702 cursor->addr_count,
3703 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3704 ppc440spe_adma_dma2rxor_inc_addr(
3705 desc, cursor, index, src_cnt);
3706 }
3707 break;
3708 case 2:
3709 cursor->state = 0;
3710 cursor->addrl = addr;
3711 cursor->xor_count++;
3712 if (index) {
3713 ppc440spe_adma_dma2rxor_inc_addr(
3714 desc, cursor, index, src_cnt);
3715 }
3716 break;
3717 }
3718
3719 return rval;
3720}
3721
3722/**
3723 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3724 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3725 */
3726static void ppc440spe_adma_dma2rxor_set_src(
3727 struct ppc440spe_adma_desc_slot *desc,
3728 int index, dma_addr_t addr)
3729{
3730 struct xor_cb *xcb = desc->hw_desc;
3731 int k = 0, op = 0, lop = 0;
3732
3733 /* get the RXOR operand which corresponds to index addr */
3734 while (op <= index) {
3735 lop = op;
3736 if (k == XOR_MAX_OPS) {
3737 k = 0;
3738 desc = list_entry(desc->chain_node.next,
3739 struct ppc440spe_adma_desc_slot, chain_node);
3740 xcb = desc->hw_desc;
3741
3742 }
3743 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3744 (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3745 op += 2;
3746 else
3747 op += 3;
3748 }
3749
3750 BUG_ON(k < 1);
3751
3752 if (test_bit(k-1, desc->reverse_flags)) {
3753 /* reverse operand order; put last op in RXOR group */
3754 if (index == op - 1)
3755 ppc440spe_rxor_set_src(desc, k - 1, addr);
3756 } else {
3757 /* direct operand order; put first op in RXOR group */
3758 if (index == lop)
3759 ppc440spe_rxor_set_src(desc, k - 1, addr);
3760 }
3761}
3762
3763/**
3764 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3765 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3766 */
3767static void ppc440spe_adma_dma2rxor_set_mult(
3768 struct ppc440spe_adma_desc_slot *desc,
3769 int index, u8 mult)
3770{
3771 struct xor_cb *xcb = desc->hw_desc;
3772 int k = 0, op = 0, lop = 0;
3773
3774 /* get the RXOR operand which corresponds to index mult */
3775 while (op <= index) {
3776 lop = op;
3777 if (k == XOR_MAX_OPS) {
3778 k = 0;
3779 desc = list_entry(desc->chain_node.next,
3780 struct ppc440spe_adma_desc_slot,
3781 chain_node);
3782 xcb = desc->hw_desc;
3783
3784 }
3785 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3786 (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3787 op += 2;
3788 else
3789 op += 3;
3790 }
3791
3792 BUG_ON(k < 1);
3793 if (test_bit(k-1, desc->reverse_flags)) {
3794 /* reverse order */
3795 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3796 } else {
3797 /* direct order */
3798 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3799 }
3800}
3801
3802/**
3803 * ppc440spe_init_rxor_cursor -
3804 */
3805static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3806{
3807 memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3808 cursor->state = 2;
3809}
3810
3811/**
3812 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3813 * descriptor for the PQXOR operation
3814 */
3815static void ppc440spe_adma_pq_set_src_mult(
3816 struct ppc440spe_adma_desc_slot *sw_desc,
3817 unsigned char mult, int index, int dst_pos)
3818{
3819 struct ppc440spe_adma_chan *chan;
3820 u32 mult_idx, mult_dst;
3821 struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3822
3823 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3824
3825 switch (chan->device->id) {
3826 case PPC440SPE_DMA0_ID:
3827 case PPC440SPE_DMA1_ID:
3828 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3829 int region = test_bit(PPC440SPE_DESC_RXOR12,
3830 &sw_desc->flags) ? 2 : 3;
3831
3832 if (index < region) {
3833 /* RXOR multipliers */
3834 iter = ppc440spe_get_group_entry(sw_desc,
3835 sw_desc->dst_cnt - 1);
3836 if (sw_desc->dst_cnt == 2)
3837 iter1 = ppc440spe_get_group_entry(
3838 sw_desc, 0);
3839
3840 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3841 mult_dst = DMA_CDB_SG_SRC;
3842 } else {
3843 /* WXOR multiplier */
3844 iter = ppc440spe_get_group_entry(sw_desc,
3845 index - region +
3846 sw_desc->dst_cnt);
3847 mult_idx = DMA_CUED_MULT1_OFF;
3848 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3849 DMA_CDB_SG_DST1;
3850 }
3851 } else {
3852 int znum = 0;
3853
3854 /* WXOR-only;
3855 * skip first slots with destinations (if ZERO_DST has
3856 * place)
3857 */
3858 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3859 znum++;
3860 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3861 znum++;
3862
3863 iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3864 mult_idx = DMA_CUED_MULT1_OFF;
3865 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3866 }
3867
3868 if (likely(iter)) {
3869 ppc440spe_desc_set_src_mult(iter, chan,
3870 mult_idx, mult_dst, mult);
3871
3872 if (unlikely(iter1)) {
3873 /* if we have two destinations for RXOR, then
3874 * we've just set Q mult. Set-up P now.
3875 */
3876 ppc440spe_desc_set_src_mult(iter1, chan,
3877 mult_idx, mult_dst, 1);
3878 }
3879
3880 }
3881 break;
3882
3883 case PPC440SPE_XOR_ID:
3884 iter = sw_desc->group_head;
3885 if (sw_desc->dst_cnt == 2) {
3886 /* both P & Q calculations required; set P mult here */
3887 ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3888
3889 /* and then set Q mult */
3890 iter = ppc440spe_get_group_entry(sw_desc,
3891 sw_desc->descs_per_op);
3892 }
3893 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3894 break;
3895 }
3896}
3897
3898/**
3899 * ppc440spe_adma_free_chan_resources - free the resources allocated
3900 */
3901static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3902{
3903 struct ppc440spe_adma_chan *ppc440spe_chan;
3904 struct ppc440spe_adma_desc_slot *iter, *_iter;
3905 int in_use_descs = 0;
3906
3907 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3908 ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3909
3910 spin_lock_bh(&ppc440spe_chan->lock);
3911 list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3912 chain_node) {
3913 in_use_descs++;
3914 list_del(&iter->chain_node);
3915 }
3916 list_for_each_entry_safe_reverse(iter, _iter,
3917 &ppc440spe_chan->all_slots, slot_node) {
3918 list_del(&iter->slot_node);
3919 kfree(iter);
3920 ppc440spe_chan->slots_allocated--;
3921 }
3922 ppc440spe_chan->last_used = NULL;
3923
3924 dev_dbg(ppc440spe_chan->device->common.dev,
3925 "ppc440spe adma%d %s slots_allocated %d\n",
3926 ppc440spe_chan->device->id,
3927 __func__, ppc440spe_chan->slots_allocated);
3928 spin_unlock_bh(&ppc440spe_chan->lock);
3929
3930 /* one is ok since we left it on there on purpose */
3931 if (in_use_descs > 1)
3932 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3933 in_use_descs - 1);
3934}
3935
3936/**
3937 * ppc440spe_adma_is_complete - poll the status of an ADMA transaction
3938 * @chan: ADMA channel handle
3939 * @cookie: ADMA transaction identifier
3940 */
3941static enum dma_status ppc440spe_adma_is_complete(struct dma_chan *chan,
3942 dma_cookie_t cookie, dma_cookie_t *done, dma_cookie_t *used)
3943{
3944 struct ppc440spe_adma_chan *ppc440spe_chan;
3945 dma_cookie_t last_used;
3946 dma_cookie_t last_complete;
3947 enum dma_status ret;
3948
3949 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3950 last_used = chan->cookie;
3951 last_complete = ppc440spe_chan->completed_cookie;
3952
3953 if (done)
3954 *done = last_complete;
3955 if (used)
3956 *used = last_used;
3957
3958 ret = dma_async_is_complete(cookie, last_complete, last_used);
3959 if (ret == DMA_SUCCESS)
3960 return ret;
3961
3962 ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3963
3964 last_used = chan->cookie;
3965 last_complete = ppc440spe_chan->completed_cookie;
3966
3967 if (done)
3968 *done = last_complete;
3969 if (used)
3970 *used = last_used;
3971
3972 return dma_async_is_complete(cookie, last_complete, last_used);
3973}
3974
3975/**
3976 * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3977 */
3978static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3979{
3980 struct ppc440spe_adma_chan *chan = data;
3981
3982 dev_dbg(chan->device->common.dev,
3983 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3984
3985 tasklet_schedule(&chan->irq_tasklet);
3986 ppc440spe_adma_device_clear_eot_status(chan);
3987
3988 return IRQ_HANDLED;
3989}
3990
3991/**
3992 * ppc440spe_adma_err_handler - DMA error interrupt handler;
3993 * do the same things as a eot handler
3994 */
3995static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3996{
3997 struct ppc440spe_adma_chan *chan = data;
3998
3999 dev_dbg(chan->device->common.dev,
4000 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
4001
4002 tasklet_schedule(&chan->irq_tasklet);
4003 ppc440spe_adma_device_clear_eot_status(chan);
4004
4005 return IRQ_HANDLED;
4006}
4007
4008/**
4009 * ppc440spe_test_callback - called when test operation has been done
4010 */
4011static void ppc440spe_test_callback(void *unused)
4012{
4013 complete(&ppc440spe_r6_test_comp);
4014}
4015
4016/**
4017 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
4018 */
4019static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
4020{
4021 struct ppc440spe_adma_chan *ppc440spe_chan;
4022
4023 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4024 dev_dbg(ppc440spe_chan->device->common.dev,
4025 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
4026 __func__, ppc440spe_chan->pending);
4027
4028 if (ppc440spe_chan->pending) {
4029 ppc440spe_chan->pending = 0;
4030 ppc440spe_chan_append(ppc440spe_chan);
4031 }
4032}
4033
4034/**
4035 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
4036 * use FIFOs (as opposite to chains used in XOR) so this is a XOR
4037 * specific operation)
4038 */
4039static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
4040{
4041 struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
4042 dma_cookie_t cookie;
4043 int slot_cnt, slots_per_op;
4044
4045 dev_dbg(chan->device->common.dev,
4046 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
4047
4048 spin_lock_bh(&chan->lock);
4049 slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
4050 sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
4051 if (sw_desc) {
4052 group_start = sw_desc->group_head;
4053 list_splice_init(&sw_desc->group_list, &chan->chain);
4054 async_tx_ack(&sw_desc->async_tx);
4055 ppc440spe_desc_init_null_xor(group_start);
4056
4057 cookie = chan->common.cookie;
4058 cookie++;
4059 if (cookie <= 1)
4060 cookie = 2;
4061
4062 /* initialize the completed cookie to be less than
4063 * the most recently used cookie
4064 */
4065 chan->completed_cookie = cookie - 1;
4066 chan->common.cookie = sw_desc->async_tx.cookie = cookie;
4067
4068 /* channel should not be busy */
4069 BUG_ON(ppc440spe_chan_is_busy(chan));
4070
4071 /* set the descriptor address */
4072 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
4073
4074 /* run the descriptor */
4075 ppc440spe_chan_run(chan);
4076 } else
4077 printk(KERN_ERR "ppc440spe adma%d"
4078 " failed to allocate null descriptor\n",
4079 chan->device->id);
4080 spin_unlock_bh(&chan->lock);
4081}
4082
4083/**
4084 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
4085 * For this we just perform one WXOR operation with the same source
4086 * and destination addresses, the GF-multiplier is 1; so if RAID-6
4087 * capabilities are enabled then we'll get src/dst filled with zero.
4088 */
4089static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
4090{
4091 struct ppc440spe_adma_desc_slot *sw_desc, *iter;
4092 struct page *pg;
4093 char *a;
4094 dma_addr_t dma_addr, addrs[2];
4095 unsigned long op = 0;
4096 int rval = 0;
4097
4098 set_bit(PPC440SPE_DESC_WXOR, &op);
4099
4100 pg = alloc_page(GFP_KERNEL);
4101 if (!pg)
4102 return -ENOMEM;
4103
4104 spin_lock_bh(&chan->lock);
4105 sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
4106 if (sw_desc) {
4107 /* 1 src, 1 dsr, int_ena, WXOR */
4108 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
4109 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
4110 ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
4111 iter->unmap_len = PAGE_SIZE;
4112 }
4113 } else {
4114 rval = -EFAULT;
4115 spin_unlock_bh(&chan->lock);
4116 goto exit;
4117 }
4118 spin_unlock_bh(&chan->lock);
4119
4120 /* Fill the test page with ones */
4121 memset(page_address(pg), 0xFF, PAGE_SIZE);
4122 dma_addr = dma_map_page(chan->device->dev, pg, 0,
4123 PAGE_SIZE, DMA_BIDIRECTIONAL);
4124
4125 /* Setup addresses */
4126 ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
4127 ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
4128 addrs[0] = dma_addr;
4129 addrs[1] = 0;
4130 ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
4131
4132 async_tx_ack(&sw_desc->async_tx);
4133 sw_desc->async_tx.callback = ppc440spe_test_callback;
4134 sw_desc->async_tx.callback_param = NULL;
4135
4136 init_completion(&ppc440spe_r6_test_comp);
4137
4138 ppc440spe_adma_tx_submit(&sw_desc->async_tx);
4139 ppc440spe_adma_issue_pending(&chan->common);
4140
4141 wait_for_completion(&ppc440spe_r6_test_comp);
4142
4143 /* Now check if the test page is zeroed */
4144 a = page_address(pg);
4145 if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
4146 /* page is zero - RAID-6 enabled */
4147 rval = 0;
4148 } else {
4149 /* RAID-6 was not enabled */
4150 rval = -EINVAL;
4151 }
4152exit:
4153 __free_page(pg);
4154 return rval;
4155}
4156
4157static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
4158{
4159 switch (adev->id) {
4160 case PPC440SPE_DMA0_ID:
4161 case PPC440SPE_DMA1_ID:
4162 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
4163 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
4164 dma_cap_set(DMA_MEMSET, adev->common.cap_mask);
4165 dma_cap_set(DMA_PQ, adev->common.cap_mask);
4166 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
4167 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
4168 break;
4169 case PPC440SPE_XOR_ID:
4170 dma_cap_set(DMA_XOR, adev->common.cap_mask);
4171 dma_cap_set(DMA_PQ, adev->common.cap_mask);
4172 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
4173 adev->common.cap_mask = adev->common.cap_mask;
4174 break;
4175 }
4176
4177 /* Set base routines */
4178 adev->common.device_alloc_chan_resources =
4179 ppc440spe_adma_alloc_chan_resources;
4180 adev->common.device_free_chan_resources =
4181 ppc440spe_adma_free_chan_resources;
4182 adev->common.device_is_tx_complete = ppc440spe_adma_is_complete;
4183 adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
4184
4185 /* Set prep routines based on capability */
4186 if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
4187 adev->common.device_prep_dma_memcpy =
4188 ppc440spe_adma_prep_dma_memcpy;
4189 }
4190 if (dma_has_cap(DMA_MEMSET, adev->common.cap_mask)) {
4191 adev->common.device_prep_dma_memset =
4192 ppc440spe_adma_prep_dma_memset;
4193 }
4194 if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
4195 adev->common.max_xor = XOR_MAX_OPS;
4196 adev->common.device_prep_dma_xor =
4197 ppc440spe_adma_prep_dma_xor;
4198 }
4199 if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
4200 switch (adev->id) {
4201 case PPC440SPE_DMA0_ID:
4202 dma_set_maxpq(&adev->common,
4203 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
4204 break;
4205 case PPC440SPE_DMA1_ID:
4206 dma_set_maxpq(&adev->common,
4207 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
4208 break;
4209 case PPC440SPE_XOR_ID:
4210 adev->common.max_pq = XOR_MAX_OPS * 3;
4211 break;
4212 }
4213 adev->common.device_prep_dma_pq =
4214 ppc440spe_adma_prep_dma_pq;
4215 }
4216 if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
4217 switch (adev->id) {
4218 case PPC440SPE_DMA0_ID:
4219 adev->common.max_pq = DMA0_FIFO_SIZE /
4220 sizeof(struct dma_cdb);
4221 break;
4222 case PPC440SPE_DMA1_ID:
4223 adev->common.max_pq = DMA1_FIFO_SIZE /
4224 sizeof(struct dma_cdb);
4225 break;
4226 }
4227 adev->common.device_prep_dma_pq_val =
4228 ppc440spe_adma_prep_dma_pqzero_sum;
4229 }
4230 if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
4231 switch (adev->id) {
4232 case PPC440SPE_DMA0_ID:
4233 adev->common.max_xor = DMA0_FIFO_SIZE /
4234 sizeof(struct dma_cdb);
4235 break;
4236 case PPC440SPE_DMA1_ID:
4237 adev->common.max_xor = DMA1_FIFO_SIZE /
4238 sizeof(struct dma_cdb);
4239 break;
4240 }
4241 adev->common.device_prep_dma_xor_val =
4242 ppc440spe_adma_prep_dma_xor_zero_sum;
4243 }
4244 if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
4245 adev->common.device_prep_dma_interrupt =
4246 ppc440spe_adma_prep_dma_interrupt;
4247 }
4248 pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
4249 "( %s%s%s%s%s%s%s)\n",
4250 dev_name(adev->dev),
4251 dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
4252 dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
4253 dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
4254 dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
4255 dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
4256 dma_has_cap(DMA_MEMSET, adev->common.cap_mask) ? "memset " : "",
4257 dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
4258}
4259
4260static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
4261 struct ppc440spe_adma_chan *chan,
4262 int *initcode)
4263{
4264 struct device_node *np;
4265 int ret;
4266
4267 np = container_of(adev->dev, struct of_device, dev)->node;
4268 if (adev->id != PPC440SPE_XOR_ID) {
4269 adev->err_irq = irq_of_parse_and_map(np, 1);
4270 if (adev->err_irq == NO_IRQ) {
4271 dev_warn(adev->dev, "no err irq resource?\n");
4272 *initcode = PPC_ADMA_INIT_IRQ2;
4273 adev->err_irq = -ENXIO;
4274 } else
4275 atomic_inc(&ppc440spe_adma_err_irq_ref);
4276 } else {
4277 adev->err_irq = -ENXIO;
4278 }
4279
4280 adev->irq = irq_of_parse_and_map(np, 0);
4281 if (adev->irq == NO_IRQ) {
4282 dev_err(adev->dev, "no irq resource\n");
4283 *initcode = PPC_ADMA_INIT_IRQ1;
4284 ret = -ENXIO;
4285 goto err_irq_map;
4286 }
4287 dev_dbg(adev->dev, "irq %d, err irq %d\n",
4288 adev->irq, adev->err_irq);
4289
4290 ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
4291 0, dev_driver_string(adev->dev), chan);
4292 if (ret) {
4293 dev_err(adev->dev, "can't request irq %d\n",
4294 adev->irq);
4295 *initcode = PPC_ADMA_INIT_IRQ1;
4296 ret = -EIO;
4297 goto err_req1;
4298 }
4299
4300 /* only DMA engines have a separate error IRQ
4301 * so it's Ok if err_irq < 0 in XOR engine case.
4302 */
4303 if (adev->err_irq > 0) {
4304 /* both DMA engines share common error IRQ */
4305 ret = request_irq(adev->err_irq,
4306 ppc440spe_adma_err_handler,
4307 IRQF_SHARED,
4308 dev_driver_string(adev->dev),
4309 chan);
4310 if (ret) {
4311 dev_err(adev->dev, "can't request irq %d\n",
4312 adev->err_irq);
4313 *initcode = PPC_ADMA_INIT_IRQ2;
4314 ret = -EIO;
4315 goto err_req2;
4316 }
4317 }
4318
4319 if (adev->id == PPC440SPE_XOR_ID) {
4320 /* enable XOR engine interrupts */
4321 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4322 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
4323 &adev->xor_reg->ier);
4324 } else {
4325 u32 mask, enable;
4326
4327 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4328 if (!np) {
4329 pr_err("%s: can't find I2O device tree node\n",
4330 __func__);
4331 ret = -ENODEV;
4332 goto err_req2;
4333 }
4334 adev->i2o_reg = of_iomap(np, 0);
4335 if (!adev->i2o_reg) {
4336 pr_err("%s: failed to map I2O registers\n", __func__);
4337 of_node_put(np);
4338 ret = -EINVAL;
4339 goto err_req2;
4340 }
4341 of_node_put(np);
4342 /* Unmask 'CS FIFO Attention' interrupts and
4343 * enable generating interrupts on errors
4344 */
4345 enable = (adev->id == PPC440SPE_DMA0_ID) ?
4346 ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4347 ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4348 mask = ioread32(&adev->i2o_reg->iopim) & enable;
4349 iowrite32(mask, &adev->i2o_reg->iopim);
4350 }
4351 return 0;
4352
4353err_req2:
4354 free_irq(adev->irq, chan);
4355err_req1:
4356 irq_dispose_mapping(adev->irq);
4357err_irq_map:
4358 if (adev->err_irq > 0) {
4359 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
4360 irq_dispose_mapping(adev->err_irq);
4361 }
4362 return ret;
4363}
4364
4365static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
4366 struct ppc440spe_adma_chan *chan)
4367{
4368 u32 mask, disable;
4369
4370 if (adev->id == PPC440SPE_XOR_ID) {
4371 /* disable XOR engine interrupts */
4372 mask = ioread32be(&adev->xor_reg->ier);
4373 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4374 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
4375 iowrite32be(mask, &adev->xor_reg->ier);
4376 } else {
4377 /* disable DMAx engine interrupts */
4378 disable = (adev->id == PPC440SPE_DMA0_ID) ?
4379 (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4380 (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4381 mask = ioread32(&adev->i2o_reg->iopim) | disable;
4382 iowrite32(mask, &adev->i2o_reg->iopim);
4383 }
4384 free_irq(adev->irq, chan);
4385 irq_dispose_mapping(adev->irq);
4386 if (adev->err_irq > 0) {
4387 free_irq(adev->err_irq, chan);
4388 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
4389 irq_dispose_mapping(adev->err_irq);
4390 iounmap(adev->i2o_reg);
4391 }
4392 }
4393}
4394
4395/**
4396 * ppc440spe_adma_probe - probe the asynch device
4397 */
4398static int __devinit ppc440spe_adma_probe(struct of_device *ofdev,
4399 const struct of_device_id *match)
4400{
4401 struct device_node *np = ofdev->node;
4402 struct resource res;
4403 struct ppc440spe_adma_device *adev;
4404 struct ppc440spe_adma_chan *chan;
4405 struct ppc_dma_chan_ref *ref, *_ref;
4406 int ret = 0, initcode = PPC_ADMA_INIT_OK;
4407 const u32 *idx;
4408 int len;
4409 void *regs;
4410 u32 id, pool_size;
4411
4412 if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4413 id = PPC440SPE_XOR_ID;
4414 /* As far as the XOR engine is concerned, it does not
4415 * use FIFOs but uses linked list. So there is no dependency
4416 * between pool size to allocate and the engine configuration.
4417 */
4418 pool_size = PAGE_SIZE << 1;
4419 } else {
4420 /* it is DMA0 or DMA1 */
4421 idx = of_get_property(np, "cell-index", &len);
4422 if (!idx || (len != sizeof(u32))) {
4423 dev_err(&ofdev->dev, "Device node %s has missing "
4424 "or invalid cell-index property\n",
4425 np->full_name);
4426 return -EINVAL;
4427 }
4428 id = *idx;
4429 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4430 * should allocate the pool accordingly to size of this
4431 * FIFO. Thus, the pool size depends on the FIFO depth:
4432 * how much CDBs pointers the FIFO may contain then so
4433 * much CDBs we should provide in the pool.
4434 * That is
4435 * CDB size = 32B;
4436 * CDBs number = (DMA0_FIFO_SIZE >> 3);
4437 * Pool size = CDBs number * CDB size =
4438 * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4439 */
4440 pool_size = (id == PPC440SPE_DMA0_ID) ?
4441 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4442 pool_size <<= 2;
4443 }
4444
4445 if (of_address_to_resource(np, 0, &res)) {
4446 dev_err(&ofdev->dev, "failed to get memory resource\n");
4447 initcode = PPC_ADMA_INIT_MEMRES;
4448 ret = -ENODEV;
4449 goto out;
4450 }
4451
4452 if (!request_mem_region(res.start, resource_size(&res),
4453 dev_driver_string(&ofdev->dev))) {
4454 dev_err(&ofdev->dev, "failed to request memory region "
4455 "(0x%016llx-0x%016llx)\n",
4456 (u64)res.start, (u64)res.end);
4457 initcode = PPC_ADMA_INIT_MEMREG;
4458 ret = -EBUSY;
4459 goto out;
4460 }
4461
4462 /* create a device */
4463 adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4464 if (!adev) {
4465 dev_err(&ofdev->dev, "failed to allocate device\n");
4466 initcode = PPC_ADMA_INIT_ALLOC;
4467 ret = -ENOMEM;
4468 goto err_adev_alloc;
4469 }
4470
4471 adev->id = id;
4472 adev->pool_size = pool_size;
4473 /* allocate coherent memory for hardware descriptors */
4474 adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4475 adev->pool_size, &adev->dma_desc_pool,
4476 GFP_KERNEL);
4477 if (adev->dma_desc_pool_virt == NULL) {
4478 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4479 "memory for hardware descriptors\n",
4480 adev->pool_size);
4481 initcode = PPC_ADMA_INIT_COHERENT;
4482 ret = -ENOMEM;
4483 goto err_dma_alloc;
4484 }
4485 dev_dbg(&ofdev->dev, "allocted descriptor pool virt 0x%p phys 0x%llx\n",
4486 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4487
4488 regs = ioremap(res.start, resource_size(&res));
4489 if (!regs) {
4490 dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4491 goto err_regs_alloc;
4492 }
4493
4494 if (adev->id == PPC440SPE_XOR_ID) {
4495 adev->xor_reg = regs;
4496 /* Reset XOR */
4497 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4498 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4499 } else {
4500 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4501 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4502 adev->dma_reg = regs;
4503 /* DMAx_FIFO_SIZE is defined in bytes,
4504 * <fsiz> - is defined in number of CDB pointers (8byte).
4505 * DMA FIFO Length = CSlength + CPlength, where
4506 * CSlength = CPlength = (fsiz + 1) * 8.
4507 */
4508 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4509 &adev->dma_reg->fsiz);
4510 /* Configure DMA engine */
4511 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4512 &adev->dma_reg->cfg);
4513 /* Clear Status */
4514 iowrite32(~0, &adev->dma_reg->dsts);
4515 }
4516
4517 adev->dev = &ofdev->dev;
4518 adev->common.dev = &ofdev->dev;
4519 INIT_LIST_HEAD(&adev->common.channels);
4520 dev_set_drvdata(&ofdev->dev, adev);
4521
4522 /* create a channel */
4523 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4524 if (!chan) {
4525 dev_err(&ofdev->dev, "can't allocate channel structure\n");
4526 initcode = PPC_ADMA_INIT_CHANNEL;
4527 ret = -ENOMEM;
4528 goto err_chan_alloc;
4529 }
4530
4531 spin_lock_init(&chan->lock);
4532 INIT_LIST_HEAD(&chan->chain);
4533 INIT_LIST_HEAD(&chan->all_slots);
4534 chan->device = adev;
4535 chan->common.device = &adev->common;
4536 list_add_tail(&chan->common.device_node, &adev->common.channels);
4537 tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet,
4538 (unsigned long)chan);
4539
4540 /* allocate and map helper pages for async validation or
4541 * async_mult/async_sum_product operations on DMA0/1.
4542 */
4543 if (adev->id != PPC440SPE_XOR_ID) {
4544 chan->pdest_page = alloc_page(GFP_KERNEL);
4545 chan->qdest_page = alloc_page(GFP_KERNEL);
4546 if (!chan->pdest_page ||
4547 !chan->qdest_page) {
4548 if (chan->pdest_page)
4549 __free_page(chan->pdest_page);
4550 if (chan->qdest_page)
4551 __free_page(chan->qdest_page);
4552 ret = -ENOMEM;
4553 goto err_page_alloc;
4554 }
4555 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4556 PAGE_SIZE, DMA_BIDIRECTIONAL);
4557 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4558 PAGE_SIZE, DMA_BIDIRECTIONAL);
4559 }
4560
4561 ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4562 if (ref) {
4563 ref->chan = &chan->common;
4564 INIT_LIST_HEAD(&ref->node);
4565 list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4566 } else {
4567 dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4568 ret = -ENOMEM;
4569 goto err_ref_alloc;
4570 }
4571
4572 ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4573 if (ret)
4574 goto err_irq;
4575
4576 ppc440spe_adma_init_capabilities(adev);
4577
4578 ret = dma_async_device_register(&adev->common);
4579 if (ret) {
4580 initcode = PPC_ADMA_INIT_REGISTER;
4581 dev_err(&ofdev->dev, "failed to register dma device\n");
4582 goto err_dev_reg;
4583 }
4584
4585 goto out;
4586
4587err_dev_reg:
4588 ppc440spe_adma_release_irqs(adev, chan);
4589err_irq:
4590 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4591 if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4592 list_del(&ref->node);
4593 kfree(ref);
4594 }
4595 }
4596err_ref_alloc:
4597 if (adev->id != PPC440SPE_XOR_ID) {
4598 dma_unmap_page(&ofdev->dev, chan->pdest,
4599 PAGE_SIZE, DMA_BIDIRECTIONAL);
4600 dma_unmap_page(&ofdev->dev, chan->qdest,
4601 PAGE_SIZE, DMA_BIDIRECTIONAL);
4602 __free_page(chan->pdest_page);
4603 __free_page(chan->qdest_page);
4604 }
4605err_page_alloc:
4606 kfree(chan);
4607err_chan_alloc:
4608 if (adev->id == PPC440SPE_XOR_ID)
4609 iounmap(adev->xor_reg);
4610 else
4611 iounmap(adev->dma_reg);
4612err_regs_alloc:
4613 dma_free_coherent(adev->dev, adev->pool_size,
4614 adev->dma_desc_pool_virt,
4615 adev->dma_desc_pool);
4616err_dma_alloc:
4617 kfree(adev);
4618err_adev_alloc:
4619 release_mem_region(res.start, resource_size(&res));
4620out:
4621 if (id < PPC440SPE_ADMA_ENGINES_NUM)
4622 ppc440spe_adma_devices[id] = initcode;
4623
4624 return ret;
4625}
4626
4627/**
4628 * ppc440spe_adma_remove - remove the asynch device
4629 */
4630static int __devexit ppc440spe_adma_remove(struct of_device *ofdev)
4631{
4632 struct ppc440spe_adma_device *adev = dev_get_drvdata(&ofdev->dev);
4633 struct device_node *np = ofdev->node;
4634 struct resource res;
4635 struct dma_chan *chan, *_chan;
4636 struct ppc_dma_chan_ref *ref, *_ref;
4637 struct ppc440spe_adma_chan *ppc440spe_chan;
4638
4639 dev_set_drvdata(&ofdev->dev, NULL);
4640 if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4641 ppc440spe_adma_devices[adev->id] = -1;
4642
4643 dma_async_device_unregister(&adev->common);
4644
4645 list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4646 device_node) {
4647 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4648 ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4649 tasklet_kill(&ppc440spe_chan->irq_tasklet);
4650 if (adev->id != PPC440SPE_XOR_ID) {
4651 dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4652 PAGE_SIZE, DMA_BIDIRECTIONAL);
4653 dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4654 PAGE_SIZE, DMA_BIDIRECTIONAL);
4655 __free_page(ppc440spe_chan->pdest_page);
4656 __free_page(ppc440spe_chan->qdest_page);
4657 }
4658 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4659 node) {
4660 if (ppc440spe_chan ==
4661 to_ppc440spe_adma_chan(ref->chan)) {
4662 list_del(&ref->node);
4663 kfree(ref);
4664 }
4665 }
4666 list_del(&chan->device_node);
4667 kfree(ppc440spe_chan);
4668 }
4669
4670 dma_free_coherent(adev->dev, adev->pool_size,
4671 adev->dma_desc_pool_virt, adev->dma_desc_pool);
4672 if (adev->id == PPC440SPE_XOR_ID)
4673 iounmap(adev->xor_reg);
4674 else
4675 iounmap(adev->dma_reg);
4676 of_address_to_resource(np, 0, &res);
4677 release_mem_region(res.start, resource_size(&res));
4678 kfree(adev);
4679 return 0;
4680}
4681
4682/*
4683 * /sys driver interface to enable h/w RAID-6 capabilities
4684 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4685 * directory are "devices", "enable" and "poly".
4686 * "devices" shows available engines.
4687 * "enable" is used to enable RAID-6 capabilities or to check
4688 * whether these has been activated.
4689 * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4690 */
4691
4692static ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf)
4693{
4694 ssize_t size = 0;
4695 int i;
4696
4697 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4698 if (ppc440spe_adma_devices[i] == -1)
4699 continue;
4700 size += snprintf(buf + size, PAGE_SIZE - size,
4701 "PPC440SP(E)-ADMA.%d: %s\n", i,
4702 ppc_adma_errors[ppc440spe_adma_devices[i]]);
4703 }
4704 return size;
4705}
4706
4707static ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf)
4708{
4709 return snprintf(buf, PAGE_SIZE,
4710 "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4711 ppc440spe_r6_enabled ? "EN" : "DIS");
4712}
4713
4714static ssize_t store_ppc440spe_r6enable(struct device_driver *dev,
4715 const char *buf, size_t count)
4716{
4717 unsigned long val;
4718
4719 if (!count || count > 11)
4720 return -EINVAL;
4721
4722 if (!ppc440spe_r6_tchan)
4723 return -EFAULT;
4724
4725 /* Write a key */
4726 sscanf(buf, "%lx", &val);
4727 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4728 isync();
4729
4730 /* Verify whether it really works now */
4731 if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4732 pr_info("PPC440SP(e) RAID-6 has been activated "
4733 "successfully\n");
4734 ppc440spe_r6_enabled = 1;
4735 } else {
4736 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4737 " Error key ?\n");
4738 ppc440spe_r6_enabled = 0;
4739 }
4740 return count;
4741}
4742
4743static ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf)
4744{
4745 ssize_t size = 0;
4746 u32 reg;
4747
4748#ifdef CONFIG_440SP
4749 /* 440SP has fixed polynomial */
4750 reg = 0x4d;
4751#else
4752 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4753 reg >>= MQ0_CFBHL_POLY;
4754 reg &= 0xFF;
4755#endif
4756
4757 size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4758 "uses 0x1%02x polynomial.\n", reg);
4759 return size;
4760}
4761
4762static ssize_t store_ppc440spe_r6poly(struct device_driver *dev,
4763 const char *buf, size_t count)
4764{
4765 unsigned long reg, val;
4766
4767#ifdef CONFIG_440SP
4768 /* 440SP uses default 0x14D polynomial only */
4769 return -EINVAL;
4770#endif
4771
4772 if (!count || count > 6)
4773 return -EINVAL;
4774
4775 /* e.g., 0x14D or 0x11D */
4776 sscanf(buf, "%lx", &val);
4777
4778 if (val & ~0x1FF)
4779 return -EINVAL;
4780
4781 val &= 0xFF;
4782 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4783 reg &= ~(0xFF << MQ0_CFBHL_POLY);
4784 reg |= val << MQ0_CFBHL_POLY;
4785 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4786
4787 return count;
4788}
4789
4790static DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL);
4791static DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable,
4792 store_ppc440spe_r6enable);
4793static DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly,
4794 store_ppc440spe_r6poly);
4795
4796/*
4797 * Common initialisation for RAID engines; allocate memory for
4798 * DMAx FIFOs, perform configuration common for all DMA engines.
4799 * Further DMA engine specific configuration is done at probe time.
4800 */
4801static int ppc440spe_configure_raid_devices(void)
4802{
4803 struct device_node *np;
4804 struct resource i2o_res;
4805 struct i2o_regs __iomem *i2o_reg;
4806 dcr_host_t i2o_dcr_host;
4807 unsigned int dcr_base, dcr_len;
4808 int i, ret;
4809
4810 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4811 if (!np) {
4812 pr_err("%s: can't find I2O device tree node\n",
4813 __func__);
4814 return -ENODEV;
4815 }
4816
4817 if (of_address_to_resource(np, 0, &i2o_res)) {
4818 of_node_put(np);
4819 return -EINVAL;
4820 }
4821
4822 i2o_reg = of_iomap(np, 0);
4823 if (!i2o_reg) {
4824 pr_err("%s: failed to map I2O registers\n", __func__);
4825 of_node_put(np);
4826 return -EINVAL;
4827 }
4828
4829 /* Get I2O DCRs base */
4830 dcr_base = dcr_resource_start(np, 0);
4831 dcr_len = dcr_resource_len(np, 0);
4832 if (!dcr_base && !dcr_len) {
4833 pr_err("%s: can't get DCR registers base/len!\n",
4834 np->full_name);
4835 of_node_put(np);
4836 iounmap(i2o_reg);
4837 return -ENODEV;
4838 }
4839
4840 i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4841 if (!DCR_MAP_OK(i2o_dcr_host)) {
4842 pr_err("%s: failed to map DCRs!\n", np->full_name);
4843 of_node_put(np);
4844 iounmap(i2o_reg);
4845 return -ENODEV;
4846 }
4847 of_node_put(np);
4848
4849 /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4850 * the base address of FIFO memory space.
4851 * Actually we need twice more physical memory than programmed in the
4852 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4853 */
4854 ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4855 GFP_KERNEL);
4856 if (!ppc440spe_dma_fifo_buf) {
4857 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4858 iounmap(i2o_reg);
4859 dcr_unmap(i2o_dcr_host, dcr_len);
4860 return -ENOMEM;
4861 }
4862
4863 /*
4864 * Configure h/w
4865 */
4866 /* Reset I2O/DMA */
4867 mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4868 mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4869
4870 /* Setup the base address of mmaped registers */
4871 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4872 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4873 I2O_REG_ENABLE);
4874 dcr_unmap(i2o_dcr_host, dcr_len);
4875
4876 /* Setup FIFO memory space base address */
4877 iowrite32(0, &i2o_reg->ifbah);
4878 iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4879
4880 /* set zero FIFO size for I2O, so the whole
4881 * ppc440spe_dma_fifo_buf is used by DMAs.
4882 * DMAx_FIFOs will be configured while probe.
4883 */
4884 iowrite32(0, &i2o_reg->ifsiz);
4885 iounmap(i2o_reg);
4886
4887 /* To prepare WXOR/RXOR functionality we need access to
4888 * Memory Queue Module DCRs (finally it will be enabled
4889 * via /sys interface of the ppc440spe ADMA driver).
4890 */
4891 np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4892 if (!np) {
4893 pr_err("%s: can't find MQ device tree node\n",
4894 __func__);
4895 ret = -ENODEV;
4896 goto out_free;
4897 }
4898
4899 /* Get MQ DCRs base */
4900 dcr_base = dcr_resource_start(np, 0);
4901 dcr_len = dcr_resource_len(np, 0);
4902 if (!dcr_base && !dcr_len) {
4903 pr_err("%s: can't get DCR registers base/len!\n",
4904 np->full_name);
4905 ret = -ENODEV;
4906 goto out_mq;
4907 }
4908
4909 ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4910 if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4911 pr_err("%s: failed to map DCRs!\n", np->full_name);
4912 ret = -ENODEV;
4913 goto out_mq;
4914 }
4915 of_node_put(np);
4916 ppc440spe_mq_dcr_len = dcr_len;
4917
4918 /* Set HB alias */
4919 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4920
4921 /* Set:
4922 * - LL transaction passing limit to 1;
4923 * - Memory controller cycle limit to 1;
4924 * - Galois Polynomial to 0x14d (default)
4925 */
4926 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4927 (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4928 (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4929
4930 atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4931 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4932 ppc440spe_adma_devices[i] = -1;
4933
4934 return 0;
4935
4936out_mq:
4937 of_node_put(np);
4938out_free:
4939 kfree(ppc440spe_dma_fifo_buf);
4940 return ret;
4941}
4942
4943static struct of_device_id __devinitdata ppc440spe_adma_of_match[] = {
4944 { .compatible = "ibm,dma-440spe", },
4945 { .compatible = "amcc,xor-accelerator", },
4946 {},
4947};
4948MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4949
4950static struct of_platform_driver ppc440spe_adma_driver = {
4951 .match_table = ppc440spe_adma_of_match,
4952 .probe = ppc440spe_adma_probe,
4953 .remove = __devexit_p(ppc440spe_adma_remove),
4954 .driver = {
4955 .name = "PPC440SP(E)-ADMA",
4956 .owner = THIS_MODULE,
4957 },
4958};
4959
4960static __init int ppc440spe_adma_init(void)
4961{
4962 int ret;
4963
4964 ret = ppc440spe_configure_raid_devices();
4965 if (ret)
4966 return ret;
4967
4968 ret = of_register_platform_driver(&ppc440spe_adma_driver);
4969 if (ret) {
4970 pr_err("%s: failed to register platform driver\n",
4971 __func__);
4972 goto out_reg;
4973 }
4974
4975 /* Initialization status */
4976 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4977 &driver_attr_devices);
4978 if (ret)
4979 goto out_dev;
4980
4981 /* RAID-6 h/w enable entry */
4982 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4983 &driver_attr_enable);
4984 if (ret)
4985 goto out_en;
4986
4987 /* GF polynomial to use */
4988 ret = driver_create_file(&ppc440spe_adma_driver.driver,
4989 &driver_attr_poly);
4990 if (!ret)
4991 return ret;
4992
4993 driver_remove_file(&ppc440spe_adma_driver.driver,
4994 &driver_attr_enable);
4995out_en:
4996 driver_remove_file(&ppc440spe_adma_driver.driver,
4997 &driver_attr_devices);
4998out_dev:
4999 /* User will not be able to enable h/w RAID-6 */
5000 pr_err("%s: failed to create RAID-6 driver interface\n",
5001 __func__);
5002 of_unregister_platform_driver(&ppc440spe_adma_driver);
5003out_reg:
5004 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
5005 kfree(ppc440spe_dma_fifo_buf);
5006 return ret;
5007}
5008
5009static void __exit ppc440spe_adma_exit(void)
5010{
5011 driver_remove_file(&ppc440spe_adma_driver.driver,
5012 &driver_attr_poly);
5013 driver_remove_file(&ppc440spe_adma_driver.driver,
5014 &driver_attr_enable);
5015 driver_remove_file(&ppc440spe_adma_driver.driver,
5016 &driver_attr_devices);
5017 of_unregister_platform_driver(&ppc440spe_adma_driver);
5018 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
5019 kfree(ppc440spe_dma_fifo_buf);
5020}
5021
5022arch_initcall(ppc440spe_adma_init);
5023module_exit(ppc440spe_adma_exit);
5024
5025MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
5026MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
5027MODULE_LICENSE("GPL");
diff --git a/drivers/dma/ppc4xx/adma.h b/drivers/dma/ppc4xx/adma.h
new file mode 100644
index 000000000000..8ada5a812e3b
--- /dev/null
+++ b/drivers/dma/ppc4xx/adma.h
@@ -0,0 +1,195 @@
1/*
2 * 2006-2009 (C) DENX Software Engineering.
3 *
4 * Author: Yuri Tikhonov <yur@emcraft.com>
5 *
6 * This file is licensed under the terms of the GNU General Public License
7 * version 2. This program is licensed "as is" without any warranty of
8 * any kind, whether express or implied.
9 */
10
11#ifndef _PPC440SPE_ADMA_H
12#define _PPC440SPE_ADMA_H
13
14#include <linux/types.h>
15#include "dma.h"
16#include "xor.h"
17
18#define to_ppc440spe_adma_chan(chan) \
19 container_of(chan, struct ppc440spe_adma_chan, common)
20#define to_ppc440spe_adma_device(dev) \
21 container_of(dev, struct ppc440spe_adma_device, common)
22#define tx_to_ppc440spe_adma_slot(tx) \
23 container_of(tx, struct ppc440spe_adma_desc_slot, async_tx)
24
25/* Default polynomial (for 440SP is only available) */
26#define PPC440SPE_DEFAULT_POLY 0x4d
27
28#define PPC440SPE_ADMA_ENGINES_NUM (XOR_ENGINES_NUM + DMA_ENGINES_NUM)
29
30#define PPC440SPE_ADMA_WATCHDOG_MSEC 3
31#define PPC440SPE_ADMA_THRESHOLD 1
32
33#define PPC440SPE_DMA0_ID 0
34#define PPC440SPE_DMA1_ID 1
35#define PPC440SPE_XOR_ID 2
36
37#define PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT 0xFFFFFFUL
38/* this is the XOR_CBBCR width */
39#define PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT (1 << 31)
40#define PPC440SPE_ADMA_ZERO_SUM_MAX_BYTE_COUNT PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT
41
42#define PPC440SPE_RXOR_RUN 0
43
44#define MQ0_CF2H_RXOR_BS_MASK 0x1FF
45
46#undef ADMA_LL_DEBUG
47
48/**
49 * struct ppc440spe_adma_device - internal representation of an ADMA device
50 * @dev: device
51 * @dma_reg: base for DMAx register access
52 * @xor_reg: base for XOR register access
53 * @i2o_reg: base for I2O register access
54 * @id: HW ADMA Device selector
55 * @dma_desc_pool_virt: base of DMA descriptor region (CPU address)
56 * @dma_desc_pool: base of DMA descriptor region (DMA address)
57 * @pool_size: size of the pool
58 * @irq: DMAx or XOR irq number
59 * @err_irq: DMAx error irq number
60 * @common: embedded struct dma_device
61 */
62struct ppc440spe_adma_device {
63 struct device *dev;
64 struct dma_regs __iomem *dma_reg;
65 struct xor_regs __iomem *xor_reg;
66 struct i2o_regs __iomem *i2o_reg;
67 int id;
68 void *dma_desc_pool_virt;
69 dma_addr_t dma_desc_pool;
70 size_t pool_size;
71 int irq;
72 int err_irq;
73 struct dma_device common;
74};
75
76/**
77 * struct ppc440spe_adma_chan - internal representation of an ADMA channel
78 * @lock: serializes enqueue/dequeue operations to the slot pool
79 * @device: parent device
80 * @chain: device chain view of the descriptors
81 * @common: common dmaengine channel object members
82 * @all_slots: complete domain of slots usable by the channel
83 * @pending: allows batching of hardware operations
84 * @completed_cookie: identifier for the most recently completed operation
85 * @slots_allocated: records the actual size of the descriptor slot pool
86 * @hw_chain_inited: h/w descriptor chain initialization flag
87 * @irq_tasklet: bottom half where ppc440spe_adma_slot_cleanup runs
88 * @needs_unmap: if buffers should not be unmapped upon final processing
89 * @pdest_page: P destination page for async validate operation
90 * @qdest_page: Q destination page for async validate operation
91 * @pdest: P dma addr for async validate operation
92 * @qdest: Q dma addr for async validate operation
93 */
94struct ppc440spe_adma_chan {
95 spinlock_t lock;
96 struct ppc440spe_adma_device *device;
97 struct list_head chain;
98 struct dma_chan common;
99 struct list_head all_slots;
100 struct ppc440spe_adma_desc_slot *last_used;
101 int pending;
102 dma_cookie_t completed_cookie;
103 int slots_allocated;
104 int hw_chain_inited;
105 struct tasklet_struct irq_tasklet;
106 u8 needs_unmap;
107 struct page *pdest_page;
108 struct page *qdest_page;
109 dma_addr_t pdest;
110 dma_addr_t qdest;
111};
112
113struct ppc440spe_rxor {
114 u32 addrl;
115 u32 addrh;
116 int len;
117 int xor_count;
118 int addr_count;
119 int desc_count;
120 int state;
121};
122
123/**
124 * struct ppc440spe_adma_desc_slot - PPC440SPE-ADMA software descriptor
125 * @phys: hardware address of the hardware descriptor chain
126 * @group_head: first operation in a transaction
127 * @hw_next: pointer to the next descriptor in chain
128 * @async_tx: support for the async_tx api
129 * @slot_node: node on the iop_adma_chan.all_slots list
130 * @chain_node: node on the op_adma_chan.chain list
131 * @group_list: list of slots that make up a multi-descriptor transaction
132 * for example transfer lengths larger than the supported hw max
133 * @unmap_len: transaction bytecount
134 * @hw_desc: virtual address of the hardware descriptor chain
135 * @stride: currently chained or not
136 * @idx: pool index
137 * @slot_cnt: total slots used in an transaction (group of operations)
138 * @src_cnt: number of sources set in this descriptor
139 * @dst_cnt: number of destinations set in the descriptor
140 * @slots_per_op: number of slots per operation
141 * @descs_per_op: number of slot per P/Q operation see comment
142 * for ppc440spe_prep_dma_pqxor function
143 * @flags: desc state/type
144 * @reverse_flags: 1 if a corresponding rxor address uses reversed address order
145 * @xor_check_result: result of zero sum
146 * @crc32_result: result crc calculation
147 */
148struct ppc440spe_adma_desc_slot {
149 dma_addr_t phys;
150 struct ppc440spe_adma_desc_slot *group_head;
151 struct ppc440spe_adma_desc_slot *hw_next;
152 struct dma_async_tx_descriptor async_tx;
153 struct list_head slot_node;
154 struct list_head chain_node; /* node in channel ops list */
155 struct list_head group_list; /* list */
156 unsigned int unmap_len;
157 void *hw_desc;
158 u16 stride;
159 u16 idx;
160 u16 slot_cnt;
161 u8 src_cnt;
162 u8 dst_cnt;
163 u8 slots_per_op;
164 u8 descs_per_op;
165 unsigned long flags;
166 unsigned long reverse_flags[8];
167
168#define PPC440SPE_DESC_INT 0 /* generate interrupt on complete */
169#define PPC440SPE_ZERO_P 1 /* clear P destionaion */
170#define PPC440SPE_ZERO_Q 2 /* clear Q destination */
171#define PPC440SPE_COHERENT 3 /* src/dst are coherent */
172
173#define PPC440SPE_DESC_WXOR 4 /* WXORs are in chain */
174#define PPC440SPE_DESC_RXOR 5 /* RXOR is in chain */
175
176#define PPC440SPE_DESC_RXOR123 8 /* CDB for RXOR123 operation */
177#define PPC440SPE_DESC_RXOR124 9 /* CDB for RXOR124 operation */
178#define PPC440SPE_DESC_RXOR125 10 /* CDB for RXOR125 operation */
179#define PPC440SPE_DESC_RXOR12 11 /* CDB for RXOR12 operation */
180#define PPC440SPE_DESC_RXOR_REV 12 /* CDB has srcs in reversed order */
181
182#define PPC440SPE_DESC_PCHECK 13
183#define PPC440SPE_DESC_QCHECK 14
184
185#define PPC440SPE_DESC_RXOR_MSK 0x3
186
187 struct ppc440spe_rxor rxor_cursor;
188
189 union {
190 u32 *xor_check_result;
191 u32 *crc32_result;
192 };
193};
194
195#endif /* _PPC440SPE_ADMA_H */
diff --git a/drivers/dma/ppc4xx/dma.h b/drivers/dma/ppc4xx/dma.h
new file mode 100644
index 000000000000..bcde2df2f373
--- /dev/null
+++ b/drivers/dma/ppc4xx/dma.h
@@ -0,0 +1,223 @@
1/*
2 * 440SPe's DMA engines support header file
3 *
4 * 2006-2009 (C) DENX Software Engineering.
5 *
6 * Author: Yuri Tikhonov <yur@emcraft.com>
7 *
8 * This file is licensed under the term of the GNU General Public License
9 * version 2. The program licensed "as is" without any warranty of any
10 * kind, whether express or implied.
11 */
12
13#ifndef _PPC440SPE_DMA_H
14#define _PPC440SPE_DMA_H
15
16#include <linux/types.h>
17
18/* Number of elements in the array with statical CDBs */
19#define MAX_STAT_DMA_CDBS 16
20/* Number of DMA engines available on the contoller */
21#define DMA_ENGINES_NUM 2
22
23/* Maximum h/w supported number of destinations */
24#define DMA_DEST_MAX_NUM 2
25
26/* FIFO's params */
27#define DMA0_FIFO_SIZE 0x1000
28#define DMA1_FIFO_SIZE 0x1000
29#define DMA_FIFO_ENABLE (1<<12)
30
31/* DMA Configuration Register. Data Transfer Engine PLB Priority: */
32#define DMA_CFG_DXEPR_LP (0<<26)
33#define DMA_CFG_DXEPR_HP (3<<26)
34#define DMA_CFG_DXEPR_HHP (2<<26)
35#define DMA_CFG_DXEPR_HHHP (1<<26)
36
37/* DMA Configuration Register. DMA FIFO Manager PLB Priority: */
38#define DMA_CFG_DFMPP_LP (0<<23)
39#define DMA_CFG_DFMPP_HP (3<<23)
40#define DMA_CFG_DFMPP_HHP (2<<23)
41#define DMA_CFG_DFMPP_HHHP (1<<23)
42
43/* DMA Configuration Register. Force 64-byte Alignment */
44#define DMA_CFG_FALGN (1 << 19)
45
46/*UIC0:*/
47#define D0CPF_INT (1<<12)
48#define D0CSF_INT (1<<11)
49#define D1CPF_INT (1<<10)
50#define D1CSF_INT (1<<9)
51/*UIC1:*/
52#define DMAE_INT (1<<9)
53
54/* I2O IOP Interrupt Mask Register */
55#define I2O_IOPIM_P0SNE (1<<3)
56#define I2O_IOPIM_P0EM (1<<5)
57#define I2O_IOPIM_P1SNE (1<<6)
58#define I2O_IOPIM_P1EM (1<<8)
59
60/* DMA CDB fields */
61#define DMA_CDB_MSK (0xF)
62#define DMA_CDB_64B_ADDR (1<<2)
63#define DMA_CDB_NO_INT (1<<3)
64#define DMA_CDB_STATUS_MSK (0x3)
65#define DMA_CDB_ADDR_MSK (0xFFFFFFF0)
66
67/* DMA CDB OpCodes */
68#define DMA_CDB_OPC_NO_OP (0x00)
69#define DMA_CDB_OPC_MV_SG1_SG2 (0x01)
70#define DMA_CDB_OPC_MULTICAST (0x05)
71#define DMA_CDB_OPC_DFILL128 (0x24)
72#define DMA_CDB_OPC_DCHECK128 (0x23)
73
74#define DMA_CUED_XOR_BASE (0x10000000)
75#define DMA_CUED_XOR_HB (0x00000008)
76
77#ifdef CONFIG_440SP
78#define DMA_CUED_MULT1_OFF 0
79#define DMA_CUED_MULT2_OFF 8
80#define DMA_CUED_MULT3_OFF 16
81#define DMA_CUED_REGION_OFF 24
82#define DMA_CUED_XOR_WIN_MSK (0xFC000000)
83#else
84#define DMA_CUED_MULT1_OFF 2
85#define DMA_CUED_MULT2_OFF 10
86#define DMA_CUED_MULT3_OFF 18
87#define DMA_CUED_REGION_OFF 26
88#define DMA_CUED_XOR_WIN_MSK (0xF0000000)
89#endif
90
91#define DMA_CUED_REGION_MSK 0x3
92#define DMA_RXOR123 0x0
93#define DMA_RXOR124 0x1
94#define DMA_RXOR125 0x2
95#define DMA_RXOR12 0x3
96
97/* S/G addresses */
98#define DMA_CDB_SG_SRC 1
99#define DMA_CDB_SG_DST1 2
100#define DMA_CDB_SG_DST2 3
101
102/*
103 * DMAx engines Command Descriptor Block Type
104 */
105struct dma_cdb {
106 /*
107 * Basic CDB structure (Table 20-17, p.499, 440spe_um_1_22.pdf)
108 */
109 u8 pad0[2]; /* reserved */
110 u8 attr; /* attributes */
111 u8 opc; /* opcode */
112 u32 sg1u; /* upper SG1 address */
113 u32 sg1l; /* lower SG1 address */
114 u32 cnt; /* SG count, 3B used */
115 u32 sg2u; /* upper SG2 address */
116 u32 sg2l; /* lower SG2 address */
117 u32 sg3u; /* upper SG3 address */
118 u32 sg3l; /* lower SG3 address */
119};
120
121/*
122 * DMAx hardware registers (p.515 in 440SPe UM 1.22)
123 */
124struct dma_regs {
125 u32 cpfpl;
126 u32 cpfph;
127 u32 csfpl;
128 u32 csfph;
129 u32 dsts;
130 u32 cfg;
131 u8 pad0[0x8];
132 u16 cpfhp;
133 u16 cpftp;
134 u16 csfhp;
135 u16 csftp;
136 u8 pad1[0x8];
137 u32 acpl;
138 u32 acph;
139 u32 s1bpl;
140 u32 s1bph;
141 u32 s2bpl;
142 u32 s2bph;
143 u32 s3bpl;
144 u32 s3bph;
145 u8 pad2[0x10];
146 u32 earl;
147 u32 earh;
148 u8 pad3[0x8];
149 u32 seat;
150 u32 sead;
151 u32 op;
152 u32 fsiz;
153};
154
155/*
156 * I2O hardware registers (p.528 in 440SPe UM 1.22)
157 */
158struct i2o_regs {
159 u32 ists;
160 u32 iseat;
161 u32 isead;
162 u8 pad0[0x14];
163 u32 idbel;
164 u8 pad1[0xc];
165 u32 ihis;
166 u32 ihim;
167 u8 pad2[0x8];
168 u32 ihiq;
169 u32 ihoq;
170 u8 pad3[0x8];
171 u32 iopis;
172 u32 iopim;
173 u32 iopiq;
174 u8 iopoq;
175 u8 pad4[3];
176 u16 iiflh;
177 u16 iiflt;
178 u16 iiplh;
179 u16 iiplt;
180 u16 ioflh;
181 u16 ioflt;
182 u16 ioplh;
183 u16 ioplt;
184 u32 iidc;
185 u32 ictl;
186 u32 ifcpp;
187 u8 pad5[0x4];
188 u16 mfac0;
189 u16 mfac1;
190 u16 mfac2;
191 u16 mfac3;
192 u16 mfac4;
193 u16 mfac5;
194 u16 mfac6;
195 u16 mfac7;
196 u16 ifcfh;
197 u16 ifcht;
198 u8 pad6[0x4];
199 u32 iifmc;
200 u32 iodb;
201 u32 iodbc;
202 u32 ifbal;
203 u32 ifbah;
204 u32 ifsiz;
205 u32 ispd0;
206 u32 ispd1;
207 u32 ispd2;
208 u32 ispd3;
209 u32 ihipl;
210 u32 ihiph;
211 u32 ihopl;
212 u32 ihoph;
213 u32 iiipl;
214 u32 iiiph;
215 u32 iiopl;
216 u32 iioph;
217 u32 ifcpl;
218 u32 ifcph;
219 u8 pad7[0x8];
220 u32 iopt;
221};
222
223#endif /* _PPC440SPE_DMA_H */
diff --git a/drivers/dma/ppc4xx/xor.h b/drivers/dma/ppc4xx/xor.h
new file mode 100644
index 000000000000..daed7384daac
--- /dev/null
+++ b/drivers/dma/ppc4xx/xor.h
@@ -0,0 +1,110 @@
1/*
2 * 440SPe's XOR engines support header file
3 *
4 * 2006-2009 (C) DENX Software Engineering.
5 *
6 * Author: Yuri Tikhonov <yur@emcraft.com>
7 *
8 * This file is licensed under the term of the GNU General Public License
9 * version 2. The program licensed "as is" without any warranty of any
10 * kind, whether express or implied.
11 */
12
13#ifndef _PPC440SPE_XOR_H
14#define _PPC440SPE_XOR_H
15
16#include <linux/types.h>
17
18/* Number of XOR engines available on the contoller */
19#define XOR_ENGINES_NUM 1
20
21/* Number of operands supported in the h/w */
22#define XOR_MAX_OPS 16
23
24/*
25 * XOR Command Block Control Register bits
26 */
27#define XOR_CBCR_LNK_BIT (1<<31) /* link present */
28#define XOR_CBCR_TGT_BIT (1<<30) /* target present */
29#define XOR_CBCR_CBCE_BIT (1<<29) /* command block compete enable */
30#define XOR_CBCR_RNZE_BIT (1<<28) /* result not zero enable */
31#define XOR_CBCR_XNOR_BIT (1<<15) /* XOR/XNOR */
32#define XOR_CDCR_OAC_MSK (0x7F) /* operand address count */
33
34/*
35 * XORCore Status Register bits
36 */
37#define XOR_SR_XCP_BIT (1<<31) /* core processing */
38#define XOR_SR_ICB_BIT (1<<17) /* invalid CB */
39#define XOR_SR_IC_BIT (1<<16) /* invalid command */
40#define XOR_SR_IPE_BIT (1<<15) /* internal parity error */
41#define XOR_SR_RNZ_BIT (1<<2) /* result not Zero */
42#define XOR_SR_CBC_BIT (1<<1) /* CB complete */
43#define XOR_SR_CBLC_BIT (1<<0) /* CB list complete */
44
45/*
46 * XORCore Control Set and Reset Register bits
47 */
48#define XOR_CRSR_XASR_BIT (1<<31) /* soft reset */
49#define XOR_CRSR_XAE_BIT (1<<30) /* enable */
50#define XOR_CRSR_RCBE_BIT (1<<29) /* refetch CB enable */
51#define XOR_CRSR_PAUS_BIT (1<<28) /* pause */
52#define XOR_CRSR_64BA_BIT (1<<27) /* 64/32 CB format */
53#define XOR_CRSR_CLP_BIT (1<<25) /* continue list processing */
54
55/*
56 * XORCore Interrupt Enable Register
57 */
58#define XOR_IE_ICBIE_BIT (1<<17) /* Invalid Command Block IRQ Enable */
59#define XOR_IE_ICIE_BIT (1<<16) /* Invalid Command IRQ Enable */
60#define XOR_IE_RPTIE_BIT (1<<14) /* Read PLB Timeout Error IRQ Enable */
61#define XOR_IE_CBCIE_BIT (1<<1) /* CB complete interrupt enable */
62#define XOR_IE_CBLCI_BIT (1<<0) /* CB list complete interrupt enable */
63
64/*
65 * XOR Accelerator engine Command Block Type
66 */
67struct xor_cb {
68 /*
69 * Basic 64-bit format XOR CB (Table 19-1, p.463, 440spe_um_1_22.pdf)
70 */
71 u32 cbc; /* control */
72 u32 cbbc; /* byte count */
73 u32 cbs; /* status */
74 u8 pad0[4]; /* reserved */
75 u32 cbtah; /* target address high */
76 u32 cbtal; /* target address low */
77 u32 cblah; /* link address high */
78 u32 cblal; /* link address low */
79 struct {
80 u32 h;
81 u32 l;
82 } __attribute__ ((packed)) ops[16];
83} __attribute__ ((packed));
84
85/*
86 * XOR hardware registers Table 19-3, UM 1.22
87 */
88struct xor_regs {
89 u32 op_ar[16][2]; /* operand address[0]-high,[1]-low registers */
90 u8 pad0[352]; /* reserved */
91 u32 cbcr; /* CB control register */
92 u32 cbbcr; /* CB byte count register */
93 u32 cbsr; /* CB status register */
94 u8 pad1[4]; /* reserved */
95 u32 cbtahr; /* operand target address high register */
96 u32 cbtalr; /* operand target address low register */
97 u32 cblahr; /* CB link address high register */
98 u32 cblalr; /* CB link address low register */
99 u32 crsr; /* control set register */
100 u32 crrr; /* control reset register */
101 u32 ccbahr; /* current CB address high register */
102 u32 ccbalr; /* current CB address low register */
103 u32 plbr; /* PLB configuration register */
104 u32 ier; /* interrupt enable register */
105 u32 pecr; /* parity error count register */
106 u32 sr; /* status register */
107 u32 revidr; /* revision ID register */
108};
109
110#endif /* _PPC440SPE_XOR_H */
diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c
index 034ecf0ace03..2e4a54c8afeb 100644
--- a/drivers/dma/shdma.c
+++ b/drivers/dma/shdma.c
@@ -80,17 +80,17 @@ static int sh_dmae_rst(int id)
80 unsigned short dmaor; 80 unsigned short dmaor;
81 81
82 sh_dmae_ctl_stop(id); 82 sh_dmae_ctl_stop(id);
83 dmaor = (dmaor_read_reg(id)|DMAOR_INIT); 83 dmaor = dmaor_read_reg(id) | DMAOR_INIT;
84 84
85 dmaor_write_reg(id, dmaor); 85 dmaor_write_reg(id, dmaor);
86 if ((dmaor_read_reg(id) & (DMAOR_AE | DMAOR_NMIF))) { 86 if (dmaor_read_reg(id) & (DMAOR_AE | DMAOR_NMIF)) {
87 pr_warning(KERN_ERR "dma-sh: Can't initialize DMAOR.\n"); 87 pr_warning(KERN_ERR "dma-sh: Can't initialize DMAOR.\n");
88 return -EINVAL; 88 return -EINVAL;
89 } 89 }
90 return 0; 90 return 0;
91} 91}
92 92
93static int dmae_is_idle(struct sh_dmae_chan *sh_chan) 93static int dmae_is_busy(struct sh_dmae_chan *sh_chan)
94{ 94{
95 u32 chcr = sh_dmae_readl(sh_chan, CHCR); 95 u32 chcr = sh_dmae_readl(sh_chan, CHCR);
96 if (chcr & CHCR_DE) { 96 if (chcr & CHCR_DE) {
@@ -110,15 +110,14 @@ static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs hw)
110{ 110{
111 sh_dmae_writel(sh_chan, hw.sar, SAR); 111 sh_dmae_writel(sh_chan, hw.sar, SAR);
112 sh_dmae_writel(sh_chan, hw.dar, DAR); 112 sh_dmae_writel(sh_chan, hw.dar, DAR);
113 sh_dmae_writel(sh_chan, 113 sh_dmae_writel(sh_chan, hw.tcr >> calc_xmit_shift(sh_chan), TCR);
114 (hw.tcr >> calc_xmit_shift(sh_chan)), TCR);
115} 114}
116 115
117static void dmae_start(struct sh_dmae_chan *sh_chan) 116static void dmae_start(struct sh_dmae_chan *sh_chan)
118{ 117{
119 u32 chcr = sh_dmae_readl(sh_chan, CHCR); 118 u32 chcr = sh_dmae_readl(sh_chan, CHCR);
120 119
121 chcr |= (CHCR_DE|CHCR_IE); 120 chcr |= CHCR_DE | CHCR_IE;
122 sh_dmae_writel(sh_chan, chcr, CHCR); 121 sh_dmae_writel(sh_chan, chcr, CHCR);
123} 122}
124 123
@@ -132,7 +131,7 @@ static void dmae_halt(struct sh_dmae_chan *sh_chan)
132 131
133static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val) 132static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val)
134{ 133{
135 int ret = dmae_is_idle(sh_chan); 134 int ret = dmae_is_busy(sh_chan);
136 /* When DMA was working, can not set data to CHCR */ 135 /* When DMA was working, can not set data to CHCR */
137 if (ret) 136 if (ret)
138 return ret; 137 return ret;
@@ -149,7 +148,7 @@ static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val)
149{ 148{
150 u32 addr; 149 u32 addr;
151 int shift = 0; 150 int shift = 0;
152 int ret = dmae_is_idle(sh_chan); 151 int ret = dmae_is_busy(sh_chan);
153 if (ret) 152 if (ret)
154 return ret; 153 return ret;
155 154
@@ -307,7 +306,7 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy(
307 new = sh_dmae_get_desc(sh_chan); 306 new = sh_dmae_get_desc(sh_chan);
308 if (!new) { 307 if (!new) {
309 dev_err(sh_chan->dev, 308 dev_err(sh_chan->dev,
310 "No free memory for link descriptor\n"); 309 "No free memory for link descriptor\n");
311 goto err_get_desc; 310 goto err_get_desc;
312 } 311 }
313 312
@@ -388,7 +387,7 @@ static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan)
388 struct sh_dmae_regs hw; 387 struct sh_dmae_regs hw;
389 388
390 /* DMA work check */ 389 /* DMA work check */
391 if (dmae_is_idle(sh_chan)) 390 if (dmae_is_busy(sh_chan))
392 return; 391 return;
393 392
394 /* Find the first un-transfer desciptor */ 393 /* Find the first un-transfer desciptor */
@@ -497,8 +496,9 @@ static void dmae_do_tasklet(unsigned long data)
497 struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data; 496 struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
498 struct sh_desc *desc, *_desc, *cur_desc = NULL; 497 struct sh_desc *desc, *_desc, *cur_desc = NULL;
499 u32 sar_buf = sh_dmae_readl(sh_chan, SAR); 498 u32 sar_buf = sh_dmae_readl(sh_chan, SAR);
499
500 list_for_each_entry_safe(desc, _desc, 500 list_for_each_entry_safe(desc, _desc,
501 &sh_chan->ld_queue, node) { 501 &sh_chan->ld_queue, node) {
502 if ((desc->hw.sar + desc->hw.tcr) == sar_buf) { 502 if ((desc->hw.sar + desc->hw.tcr) == sar_buf) {
503 cur_desc = desc; 503 cur_desc = desc;
504 break; 504 break;
@@ -543,8 +543,8 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id)
543 /* alloc channel */ 543 /* alloc channel */
544 new_sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL); 544 new_sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL);
545 if (!new_sh_chan) { 545 if (!new_sh_chan) {
546 dev_err(shdev->common.dev, "No free memory for allocating " 546 dev_err(shdev->common.dev,
547 "dma channels!\n"); 547 "No free memory for allocating dma channels!\n");
548 return -ENOMEM; 548 return -ENOMEM;
549 } 549 }
550 550
@@ -586,8 +586,8 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id)
586 "sh-dmae%d", new_sh_chan->id); 586 "sh-dmae%d", new_sh_chan->id);
587 587
588 /* set up channel irq */ 588 /* set up channel irq */
589 err = request_irq(irq, &sh_dmae_interrupt, 589 err = request_irq(irq, &sh_dmae_interrupt, irqflags,
590 irqflags, new_sh_chan->dev_id, new_sh_chan); 590 new_sh_chan->dev_id, new_sh_chan);
591 if (err) { 591 if (err) {
592 dev_err(shdev->common.dev, "DMA channel %d request_irq error " 592 dev_err(shdev->common.dev, "DMA channel %d request_irq error "
593 "with return %d\n", id, err); 593 "with return %d\n", id, err);
@@ -676,6 +676,8 @@ static int __init sh_dmae_probe(struct platform_device *pdev)
676 shdev->common.device_is_tx_complete = sh_dmae_is_complete; 676 shdev->common.device_is_tx_complete = sh_dmae_is_complete;
677 shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending; 677 shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending;
678 shdev->common.dev = &pdev->dev; 678 shdev->common.dev = &pdev->dev;
679 /* Default transfer size of 32 bytes requires 32-byte alignment */
680 shdev->common.copy_align = 5;
679 681
680#if defined(CONFIG_CPU_SH4) 682#if defined(CONFIG_CPU_SH4)
681 /* Non Mix IRQ mode SH7722/SH7730 etc... */ 683 /* Non Mix IRQ mode SH7722/SH7730 etc... */
@@ -688,8 +690,8 @@ static int __init sh_dmae_probe(struct platform_device *pdev)
688 } 690 }
689 691
690 for (ecnt = 0 ; ecnt < ARRAY_SIZE(eirq); ecnt++) { 692 for (ecnt = 0 ; ecnt < ARRAY_SIZE(eirq); ecnt++) {
691 err = request_irq(eirq[ecnt], sh_dmae_err, 693 err = request_irq(eirq[ecnt], sh_dmae_err, irqflags,
692 irqflags, "DMAC Address Error", shdev); 694 "DMAC Address Error", shdev);
693 if (err) { 695 if (err) {
694 dev_err(&pdev->dev, "DMA device request_irq" 696 dev_err(&pdev->dev, "DMA device request_irq"
695 "error (irq %d) with return %d\n", 697 "error (irq %d) with return %d\n",
diff --git a/drivers/dma/shdma.h b/drivers/dma/shdma.h
index 2b4bc15a2c0a..60b81e529b42 100644
--- a/drivers/dma/shdma.h
+++ b/drivers/dma/shdma.h
@@ -35,15 +35,15 @@ struct sh_desc {
35 35
36struct sh_dmae_chan { 36struct sh_dmae_chan {
37 dma_cookie_t completed_cookie; /* The maximum cookie completed */ 37 dma_cookie_t completed_cookie; /* The maximum cookie completed */
38 spinlock_t desc_lock; /* Descriptor operation lock */ 38 spinlock_t desc_lock; /* Descriptor operation lock */
39 struct list_head ld_queue; /* Link descriptors queue */ 39 struct list_head ld_queue; /* Link descriptors queue */
40 struct list_head ld_free; /* Link descriptors free */ 40 struct list_head ld_free; /* Link descriptors free */
41 struct dma_chan common; /* DMA common channel */ 41 struct dma_chan common; /* DMA common channel */
42 struct device *dev; /* Channel device */ 42 struct device *dev; /* Channel device */
43 struct tasklet_struct tasklet; /* Tasklet */ 43 struct tasklet_struct tasklet; /* Tasklet */
44 int descs_allocated; /* desc count */ 44 int descs_allocated; /* desc count */
45 int id; /* Raw id of this channel */ 45 int id; /* Raw id of this channel */
46 char dev_id[16]; /* unique name per DMAC of channel */ 46 char dev_id[16]; /* unique name per DMAC of channel */
47 47
48 /* Set chcr */ 48 /* Set chcr */
49 int (*set_chcr)(struct sh_dmae_chan *sh_chan, u32 regs); 49 int (*set_chcr)(struct sh_dmae_chan *sh_chan, u32 regs);
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index 2b9f2ac7ed60..78784982b33e 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -74,7 +74,7 @@ enum dma_transaction_type {
74 * control completion, and communicate status. 74 * control completion, and communicate status.
75 * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of 75 * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of
76 * this transaction 76 * this transaction
77 * @DMA_CTRL_ACK - the descriptor cannot be reused until the client 77 * @DMA_CTRL_ACK - if clear, the descriptor cannot be reused until the client
78 * acknowledges receipt, i.e. has has a chance to establish any dependency 78 * acknowledges receipt, i.e. has has a chance to establish any dependency
79 * chains 79 * chains
80 * @DMA_COMPL_SKIP_SRC_UNMAP - set to disable dma-unmapping the source buffer(s) 80 * @DMA_COMPL_SKIP_SRC_UNMAP - set to disable dma-unmapping the source buffer(s)
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-18 11:26:23 -0500