aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma')
-rw-r--r--drivers/dma/Kconfig607
-rw-r--r--drivers/dma/Makefile88
-rw-r--r--drivers/dma/amba-pl08x.c192
-rw-r--r--drivers/dma/at_hdmac.c127
-rw-r--r--drivers/dma/at_hdmac_regs.h6
-rw-r--r--drivers/dma/at_xdmac.c183
-rw-r--r--drivers/dma/coh901318.c2
-rw-r--r--drivers/dma/dma-axi-dmac.c691
-rw-r--r--drivers/dma/dma-jz4780.c124
-rw-r--r--drivers/dma/dw/Kconfig6
-rw-r--r--drivers/dma/edma.c2
-rw-r--r--drivers/dma/hsu/hsu.c39
-rw-r--r--drivers/dma/hsu/hsu.h1
-rw-r--r--drivers/dma/imx-dma.c25
-rw-r--r--drivers/dma/imx-sdma.c254
-rw-r--r--drivers/dma/ioat/Makefile2
-rw-r--r--drivers/dma/ioat/dca.c374
-rw-r--r--drivers/dma/ioat/dma.c1655
-rw-r--r--drivers/dma/ioat/dma.h353
-rw-r--r--drivers/dma/ioat/dma_v2.c916
-rw-r--r--drivers/dma/ioat/dma_v2.h175
-rw-r--r--drivers/dma/ioat/dma_v3.c1717
-rw-r--r--drivers/dma/ioat/hw.h16
-rw-r--r--drivers/dma/ioat/init.c1314
-rw-r--r--drivers/dma/ioat/pci.c258
-rw-r--r--drivers/dma/ioat/prep.c715
-rw-r--r--drivers/dma/ioat/sysfs.c135
-rw-r--r--drivers/dma/ipu/ipu_irq.c64
-rw-r--r--drivers/dma/k3dma.c3
-rw-r--r--drivers/dma/lpc18xx-dmamux.c183
-rw-r--r--drivers/dma/mic_x100_dma.h2
-rw-r--r--drivers/dma/mmp_pdma.c3
-rw-r--r--drivers/dma/mmp_tdma.c3
-rw-r--r--drivers/dma/mv_xor.c69
-rw-r--r--drivers/dma/pch_dma.c4
-rw-r--r--drivers/dma/pl330.c3
-rw-r--r--drivers/dma/pxa_dma.c15
-rw-r--r--drivers/dma/sh/Kconfig24
-rw-r--r--drivers/dma/sh/Makefile4
-rw-r--r--drivers/dma/sirf-dma.c3
-rw-r--r--drivers/dma/ste_dma40.c2
-rw-r--r--drivers/dma/sun4i-dma.c1288
-rw-r--r--drivers/dma/sun6i-dma.c2
-rw-r--r--drivers/dma/tegra20-apb-dma.c63
-rw-r--r--drivers/dma/ti-dma-crossbar.c41
-rw-r--r--drivers/dma/timb_dma.c4
-rw-r--r--drivers/dma/xgene-dma.c66
-rw-r--r--drivers/dma/zx296702_dma.c951
48 files changed, 7606 insertions, 5168 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index bdbbe5bcfb83..b4584757dae0 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -33,27 +33,29 @@ if DMADEVICES
33 33
34comment "DMA Devices" 34comment "DMA Devices"
35 35
36config INTEL_MIC_X100_DMA 36#core
37 tristate "Intel MIC X100 DMA Driver" 37config ASYNC_TX_ENABLE_CHANNEL_SWITCH
38 depends on 64BIT && X86 && INTEL_MIC_BUS 38 bool
39 select DMA_ENGINE
40 help
41 This enables DMA support for the Intel Many Integrated Core
42 (MIC) family of PCIe form factor coprocessor X100 devices that
43 run a 64 bit Linux OS. This driver will be used by both MIC
44 host and card drivers.
45
46 If you are building host kernel with a MIC device or a card
47 kernel for a MIC device, then say M (recommended) or Y, else
48 say N. If unsure say N.
49 39
50 More information about the Intel MIC family as well as the Linux 40config ARCH_HAS_ASYNC_TX_FIND_CHANNEL
51 OS and tools for MIC to use with this driver are available from 41 bool
52 <http://software.intel.com/en-us/mic-developer>.
53 42
54config ASYNC_TX_ENABLE_CHANNEL_SWITCH 43config DMA_ENGINE
55 bool 44 bool
56 45
46config DMA_VIRTUAL_CHANNELS
47 tristate
48
49config DMA_ACPI
50 def_bool y
51 depends on ACPI
52
53config DMA_OF
54 def_bool y
55 depends on OF
56 select DMA_ENGINE
57
58#devices
57config AMBA_PL08X 59config AMBA_PL08X
58 bool "ARM PrimeCell PL080 or PL081 support" 60 bool "ARM PrimeCell PL080 or PL081 support"
59 depends on ARM_AMBA 61 depends on ARM_AMBA
@@ -63,51 +65,112 @@ config AMBA_PL08X
63 Platform has a PL08x DMAC device 65 Platform has a PL08x DMAC device
64 which can provide DMA engine support 66 which can provide DMA engine support
65 67
66config INTEL_IOATDMA 68config AMCC_PPC440SPE_ADMA
67 tristate "Intel I/OAT DMA support" 69 tristate "AMCC PPC440SPe ADMA support"
68 depends on PCI && X86 70 depends on 440SPe || 440SP
69 select DMA_ENGINE 71 select DMA_ENGINE
70 select DMA_ENGINE_RAID 72 select DMA_ENGINE_RAID
71 select DCA 73 select ARCH_HAS_ASYNC_TX_FIND_CHANNEL
74 select ASYNC_TX_ENABLE_CHANNEL_SWITCH
72 help 75 help
73 Enable support for the Intel(R) I/OAT DMA engine present 76 Enable support for the AMCC PPC440SPe RAID engines.
74 in recent Intel Xeon chipsets.
75 77
76 Say Y here if you have such a chipset. 78config AT_HDMAC
79 tristate "Atmel AHB DMA support"
80 depends on ARCH_AT91
81 select DMA_ENGINE
82 help
83 Support the Atmel AHB DMA controller.
77 84
78 If unsure, say N. 85config AT_XDMAC
86 tristate "Atmel XDMA support"
87 depends on ARCH_AT91
88 select DMA_ENGINE
89 help
90 Support the Atmel XDMA controller.
79 91
80config INTEL_IOP_ADMA 92config AXI_DMAC
81 tristate "Intel IOP ADMA support" 93 tristate "Analog Devices AXI-DMAC DMA support"
82 depends on ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX 94 depends on MICROBLAZE || NIOS2 || ARCH_ZYNQ || ARCH_SOCFPGA || COMPILE_TEST
83 select DMA_ENGINE 95 select DMA_ENGINE
84 select ASYNC_TX_ENABLE_CHANNEL_SWITCH 96 select DMA_VIRTUAL_CHANNELS
85 help 97 help
86 Enable support for the Intel(R) IOP Series RAID engines. 98 Enable support for the Analog Devices AXI-DMAC peripheral. This DMA
99 controller is often used in Analog Device's reference designs for FPGA
100 platforms.
87 101
88config IDMA64 102config COH901318
89 tristate "Intel integrated DMA 64-bit support" 103 bool "ST-Ericsson COH901318 DMA support"
104 select DMA_ENGINE
105 depends on ARCH_U300
106 help
107 Enable support for ST-Ericsson COH 901 318 DMA.
108
109config DMA_BCM2835
110 tristate "BCM2835 DMA engine support"
111 depends on ARCH_BCM2835
112 select DMA_ENGINE
113 select DMA_VIRTUAL_CHANNELS
114
115config DMA_JZ4740
116 tristate "JZ4740 DMA support"
117 depends on MACH_JZ4740
118 select DMA_ENGINE
119 select DMA_VIRTUAL_CHANNELS
120
121config DMA_JZ4780
122 tristate "JZ4780 DMA support"
123 depends on MACH_JZ4780
90 select DMA_ENGINE 124 select DMA_ENGINE
91 select DMA_VIRTUAL_CHANNELS 125 select DMA_VIRTUAL_CHANNELS
92 help 126 help
93 Enable DMA support for Intel Low Power Subsystem such as found on 127 This selects support for the DMA controller in Ingenic JZ4780 SoCs.
94 Intel Skylake PCH. 128 If you have a board based on such a SoC and wish to use DMA for
129 devices which can use the DMA controller, say Y or M here.
95 130
96source "drivers/dma/dw/Kconfig" 131config DMA_OMAP
132 tristate "OMAP DMA support"
133 depends on ARCH_OMAP
134 select DMA_ENGINE
135 select DMA_VIRTUAL_CHANNELS
136 select TI_DMA_CROSSBAR if SOC_DRA7XX
97 137
98config AT_HDMAC 138config DMA_SA11X0
99 tristate "Atmel AHB DMA support" 139 tristate "SA-11x0 DMA support"
100 depends on ARCH_AT91 140 depends on ARCH_SA1100
101 select DMA_ENGINE 141 select DMA_ENGINE
142 select DMA_VIRTUAL_CHANNELS
102 help 143 help
103 Support the Atmel AHB DMA controller. 144 Support the DMA engine found on Intel StrongARM SA-1100 and
145 SA-1110 SoCs. This DMA engine can only be used with on-chip
146 devices.
104 147
105config AT_XDMAC 148config DMA_SUN4I
106 tristate "Atmel XDMA support" 149 tristate "Allwinner A10 DMA SoCs support"
107 depends on ARCH_AT91 150 depends on MACH_SUN4I || MACH_SUN5I || MACH_SUN7I
151 default (MACH_SUN4I || MACH_SUN5I || MACH_SUN7I)
108 select DMA_ENGINE 152 select DMA_ENGINE
153 select DMA_OF
154 select DMA_VIRTUAL_CHANNELS
109 help 155 help
110 Support the Atmel XDMA controller. 156 Enable support for the DMA controller present in the sun4i,
157 sun5i and sun7i Allwinner ARM SoCs.
158
159config DMA_SUN6I
160 tristate "Allwinner A31 SoCs DMA support"
161 depends on MACH_SUN6I || MACH_SUN8I || COMPILE_TEST
162 depends on RESET_CONTROLLER
163 select DMA_ENGINE
164 select DMA_VIRTUAL_CHANNELS
165 help
166 Support for the DMA engine first found in Allwinner A31 SoCs.
167
168config EP93XX_DMA
169 bool "Cirrus Logic EP93xx DMA support"
170 depends on ARCH_EP93XX
171 select DMA_ENGINE
172 help
173 Enable support for the Cirrus Logic EP93xx M2P/M2M DMA controller.
111 174
112config FSL_DMA 175config FSL_DMA
113 tristate "Freescale Elo series DMA support" 176 tristate "Freescale Elo series DMA support"
@@ -120,6 +183,16 @@ config FSL_DMA
120 EloPlus is on mpc85xx and mpc86xx and Pxxx parts, and the Elo3 is on 183 EloPlus is on mpc85xx and mpc86xx and Pxxx parts, and the Elo3 is on
121 some Txxx and Bxxx parts. 184 some Txxx and Bxxx parts.
122 185
186config FSL_EDMA
187 tristate "Freescale eDMA engine support"
188 depends on OF
189 select DMA_ENGINE
190 select DMA_VIRTUAL_CHANNELS
191 help
192 Support the Freescale eDMA engine with programmable channel
193 multiplexing capability for DMA request sources(slot).
194 This module can be found on Freescale Vybrid and LS-1 SoCs.
195
123config FSL_RAID 196config FSL_RAID
124 tristate "Freescale RAID engine Support" 197 tristate "Freescale RAID engine Support"
125 depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH 198 depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH
@@ -131,153 +204,175 @@ config FSL_RAID
131 the capability to offload memcpy, xor and pq computation 204 the capability to offload memcpy, xor and pq computation
132 for raid5/6. 205 for raid5/6.
133 206
134source "drivers/dma/hsu/Kconfig" 207config IMG_MDC_DMA
135 208 tristate "IMG MDC support"
136config MPC512X_DMA 209 depends on MIPS || COMPILE_TEST
137 tristate "Freescale MPC512x built-in DMA engine support" 210 depends on MFD_SYSCON
138 depends on PPC_MPC512x || PPC_MPC831x
139 select DMA_ENGINE 211 select DMA_ENGINE
140 ---help--- 212 select DMA_VIRTUAL_CHANNELS
141 Enable support for the Freescale MPC512x built-in DMA engine. 213 help
142 214 Enable support for the IMG multi-threaded DMA controller (MDC).
143source "drivers/dma/bestcomm/Kconfig"
144 215
145config MV_XOR 216config IMX_DMA
146 bool "Marvell XOR engine support" 217 tristate "i.MX DMA support"
147 depends on PLAT_ORION 218 depends on ARCH_MXC
148 select DMA_ENGINE 219 select DMA_ENGINE
149 select DMA_ENGINE_RAID 220 help
150 select ASYNC_TX_ENABLE_CHANNEL_SWITCH 221 Support the i.MX DMA engine. This engine is integrated into
151 ---help--- 222 Freescale i.MX1/21/27 chips.
152 Enable support for the Marvell XOR engine.
153 223
154config MX3_IPU 224config IMX_SDMA
155 bool "MX3x Image Processing Unit support" 225 tristate "i.MX SDMA support"
156 depends on ARCH_MXC 226 depends on ARCH_MXC
157 select DMA_ENGINE 227 select DMA_ENGINE
158 default y
159 help 228 help
160 If you plan to use the Image Processing unit in the i.MX3x, say 229 Support the i.MX SDMA engine. This engine is integrated into
161 Y here. If unsure, select Y. 230 Freescale i.MX25/31/35/51/53/6 chips.
162 231
163config MX3_IPU_IRQS 232config IDMA64
164 int "Number of dynamically mapped interrupts for IPU" 233 tristate "Intel integrated DMA 64-bit support"
165 depends on MX3_IPU 234 select DMA_ENGINE
166 range 2 137 235 select DMA_VIRTUAL_CHANNELS
167 default 4
168 help 236 help
169 Out of 137 interrupt sources on i.MX31 IPU only very few are used. 237 Enable DMA support for Intel Low Power Subsystem such as found on
170 To avoid bloating the irq_desc[] array we allocate a sufficient 238 Intel Skylake PCH.
171 number of IRQ slots and map them dynamically to specific sources.
172 239
173config PXA_DMA 240config INTEL_IOATDMA
174 bool "PXA DMA support" 241 tristate "Intel I/OAT DMA support"
175 depends on (ARCH_MMP || ARCH_PXA) 242 depends on PCI && X86_64
176 select DMA_ENGINE 243 select DMA_ENGINE
177 select DMA_VIRTUAL_CHANNELS 244 select DMA_ENGINE_RAID
245 select DCA
178 help 246 help
179 Support the DMA engine for PXA. It is also compatible with MMP PDMA 247 Enable support for the Intel(R) I/OAT DMA engine present
180 platform. The internal DMA IP of all PXA variants is supported, with 248 in recent Intel Xeon chipsets.
181 16 to 32 channels for peripheral to memory or memory to memory
182 transfers.
183 249
184config TXX9_DMAC 250 Say Y here if you have such a chipset.
185 tristate "Toshiba TXx9 SoC DMA support" 251
186 depends on MACH_TX49XX || MACH_TX39XX 252 If unsure, say N.
253
254config INTEL_IOP_ADMA
255 tristate "Intel IOP ADMA support"
256 depends on ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX
187 select DMA_ENGINE 257 select DMA_ENGINE
258 select ASYNC_TX_ENABLE_CHANNEL_SWITCH
188 help 259 help
189 Support the TXx9 SoC internal DMA controller. This can be 260 Enable support for the Intel(R) IOP Series RAID engines.
190 integrated in chips such as the Toshiba TX4927/38/39.
191 261
192config TEGRA20_APB_DMA 262config INTEL_MIC_X100_DMA
193 bool "NVIDIA Tegra20 APB DMA support" 263 tristate "Intel MIC X100 DMA Driver"
194 depends on ARCH_TEGRA 264 depends on 64BIT && X86 && INTEL_MIC_BUS
195 select DMA_ENGINE 265 select DMA_ENGINE
196 help 266 help
197 Support for the NVIDIA Tegra20 APB DMA controller driver. The 267 This enables DMA support for the Intel Many Integrated Core
198 DMA controller is having multiple DMA channel which can be 268 (MIC) family of PCIe form factor coprocessor X100 devices that
199 configured for different peripherals like audio, UART, SPI, 269 run a 64 bit Linux OS. This driver will be used by both MIC
200 I2C etc which is in APB bus. 270 host and card drivers.
201 This DMA controller transfers data from memory to peripheral fifo
202 or vice versa. It does not support memory to memory data transfer.
203 271
204config S3C24XX_DMAC 272 If you are building host kernel with a MIC device or a card
205 tristate "Samsung S3C24XX DMA support" 273 kernel for a MIC device, then say M (recommended) or Y, else
206 depends on ARCH_S3C24XX 274 say N. If unsure say N.
275
276 More information about the Intel MIC family as well as the Linux
277 OS and tools for MIC to use with this driver are available from
278 <http://software.intel.com/en-us/mic-developer>.
279
280config K3_DMA
281 tristate "Hisilicon K3 DMA support"
282 depends on ARCH_HI3xxx
207 select DMA_ENGINE 283 select DMA_ENGINE
208 select DMA_VIRTUAL_CHANNELS 284 select DMA_VIRTUAL_CHANNELS
209 help 285 help
210 Support for the Samsung S3C24XX DMA controller driver. The 286 Support the DMA engine for Hisilicon K3 platform
211 DMA controller is having multiple DMA channels which can be 287 devices.
212 configured for different peripherals like audio, UART, SPI.
213 The DMA controller can transfer data from memory to peripheral,
214 periphal to memory, periphal to periphal and memory to memory.
215 288
216source "drivers/dma/sh/Kconfig" 289config LPC18XX_DMAMUX
290 bool "NXP LPC18xx/43xx DMA MUX for PL080"
291 depends on ARCH_LPC18XX || COMPILE_TEST
292 depends on OF && AMBA_PL08X
293 select MFD_SYSCON
294 help
295 Enable support for DMA on NXP LPC18xx/43xx platforms
296 with PL080 and multiplexed DMA request lines.
217 297
218config COH901318 298config MMP_PDMA
219 bool "ST-Ericsson COH901318 DMA support" 299 bool "MMP PDMA support"
300 depends on (ARCH_MMP || ARCH_PXA)
220 select DMA_ENGINE 301 select DMA_ENGINE
221 depends on ARCH_U300
222 help 302 help
223 Enable support for ST-Ericsson COH 901 318 DMA. 303 Support the MMP PDMA engine for PXA and MMP platform.
224 304
225config STE_DMA40 305config MMP_TDMA
226 bool "ST-Ericsson DMA40 support" 306 bool "MMP Two-Channel DMA support"
227 depends on ARCH_U8500 307 depends on ARCH_MMP
228 select DMA_ENGINE 308 select DMA_ENGINE
309 select MMP_SRAM
229 help 310 help
230 Support for ST-Ericsson DMA40 controller 311 Support the MMP Two-Channel DMA engine.
312 This engine used for MMP Audio DMA and pxa910 SQU.
313 It needs sram driver under mach-mmp.
231 314
232config AMCC_PPC440SPE_ADMA 315config MOXART_DMA
233 tristate "AMCC PPC440SPe ADMA support" 316 tristate "MOXART DMA support"
234 depends on 440SPe || 440SP 317 depends on ARCH_MOXART
235 select DMA_ENGINE 318 select DMA_ENGINE
236 select DMA_ENGINE_RAID 319 select DMA_OF
237 select ARCH_HAS_ASYNC_TX_FIND_CHANNEL 320 select DMA_VIRTUAL_CHANNELS
238 select ASYNC_TX_ENABLE_CHANNEL_SWITCH
239 help 321 help
240 Enable support for the AMCC PPC440SPe RAID engines. 322 Enable support for the MOXA ART SoC DMA controller.
323
324 Say Y here if you enabled MMP ADMA, otherwise say N.
241 325
242config TIMB_DMA 326config MPC512X_DMA
243 tristate "Timberdale FPGA DMA support" 327 tristate "Freescale MPC512x built-in DMA engine support"
244 depends on MFD_TIMBERDALE 328 depends on PPC_MPC512x || PPC_MPC831x
245 select DMA_ENGINE 329 select DMA_ENGINE
246 help 330 ---help---
247 Enable support for the Timberdale FPGA DMA engine. 331 Enable support for the Freescale MPC512x built-in DMA engine.
248 332
249config SIRF_DMA 333config MV_XOR
250 tristate "CSR SiRFprimaII/SiRFmarco DMA support" 334 bool "Marvell XOR engine support"
251 depends on ARCH_SIRF 335 depends on PLAT_ORION
252 select DMA_ENGINE 336 select DMA_ENGINE
253 help 337 select DMA_ENGINE_RAID
254 Enable support for the CSR SiRFprimaII DMA engine. 338 select ASYNC_TX_ENABLE_CHANNEL_SWITCH
339 ---help---
340 Enable support for the Marvell XOR engine.
255 341
256config TI_EDMA 342config MXS_DMA
257 bool "TI EDMA support" 343 bool "MXS DMA support"
258 depends on ARCH_DAVINCI || ARCH_OMAP || ARCH_KEYSTONE 344 depends on SOC_IMX23 || SOC_IMX28 || SOC_IMX6Q
345 select STMP_DEVICE
259 select DMA_ENGINE 346 select DMA_ENGINE
260 select DMA_VIRTUAL_CHANNELS
261 select TI_PRIV_EDMA
262 default n
263 help 347 help
264 Enable support for the TI EDMA controller. This DMA 348 Support the MXS DMA engine. This engine including APBH-DMA
265 engine is found on TI DaVinci and AM33xx parts. 349 and APBX-DMA is integrated into Freescale i.MX23/28/MX6Q/MX6DL chips.
266 350
267config TI_DMA_CROSSBAR 351config MX3_IPU
268 bool 352 bool "MX3x Image Processing Unit support"
353 depends on ARCH_MXC
354 select DMA_ENGINE
355 default y
356 help
357 If you plan to use the Image Processing unit in the i.MX3x, say
358 Y here. If unsure, select Y.
269 359
270config ARCH_HAS_ASYNC_TX_FIND_CHANNEL 360config MX3_IPU_IRQS
271 bool 361 int "Number of dynamically mapped interrupts for IPU"
362 depends on MX3_IPU
363 range 2 137
364 default 4
365 help
366 Out of 137 interrupt sources on i.MX31 IPU only very few are used.
367 To avoid bloating the irq_desc[] array we allocate a sufficient
368 number of IRQ slots and map them dynamically to specific sources.
272 369
273config PL330_DMA 370config NBPFAXI_DMA
274 tristate "DMA API Driver for PL330" 371 tristate "Renesas Type-AXI NBPF DMA support"
275 select DMA_ENGINE 372 select DMA_ENGINE
276 depends on ARM_AMBA 373 depends on ARM || COMPILE_TEST
277 help 374 help
278 Select if your platform has one or more PL330 DMACs. 375 Support for "Type-AXI" NBPF DMA IPs from Renesas
279 You need to provide platform specific settings via
280 platform_data for a dma-pl330 device.
281 376
282config PCH_DMA 377config PCH_DMA
283 tristate "Intel EG20T PCH / LAPIS Semicon IOH(ML7213/ML7223/ML7831) DMA" 378 tristate "Intel EG20T PCH / LAPIS Semicon IOH(ML7213/ML7223/ML7831) DMA"
@@ -293,72 +388,87 @@ config PCH_DMA
293 ML7213/ML7223/ML7831 is companion chip for Intel Atom E6xx series. 388 ML7213/ML7223/ML7831 is companion chip for Intel Atom E6xx series.
294 ML7213/ML7223/ML7831 is completely compatible for Intel EG20T PCH. 389 ML7213/ML7223/ML7831 is completely compatible for Intel EG20T PCH.
295 390
296config IMX_SDMA 391config PL330_DMA
297 tristate "i.MX SDMA support" 392 tristate "DMA API Driver for PL330"
298 depends on ARCH_MXC
299 select DMA_ENGINE 393 select DMA_ENGINE
394 depends on ARM_AMBA
300 help 395 help
301 Support the i.MX SDMA engine. This engine is integrated into 396 Select if your platform has one or more PL330 DMACs.
302 Freescale i.MX25/31/35/51/53/6 chips. 397 You need to provide platform specific settings via
398 platform_data for a dma-pl330 device.
303 399
304config IMX_DMA 400config PXA_DMA
305 tristate "i.MX DMA support" 401 bool "PXA DMA support"
306 depends on ARCH_MXC 402 depends on (ARCH_MMP || ARCH_PXA)
307 select DMA_ENGINE 403 select DMA_ENGINE
404 select DMA_VIRTUAL_CHANNELS
308 help 405 help
309 Support the i.MX DMA engine. This engine is integrated into 406 Support the DMA engine for PXA. It is also compatible with MMP PDMA
310 Freescale i.MX1/21/27 chips. 407 platform. The internal DMA IP of all PXA variants is supported, with
408 16 to 32 channels for peripheral to memory or memory to memory
409 transfers.
311 410
312config MXS_DMA 411config QCOM_BAM_DMA
313 bool "MXS DMA support" 412 tristate "QCOM BAM DMA support"
314 depends on SOC_IMX23 || SOC_IMX28 || SOC_IMX6Q 413 depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM)
315 select STMP_DEVICE 414 select DMA_ENGINE
415 select DMA_VIRTUAL_CHANNELS
416 ---help---
417 Enable support for the QCOM BAM DMA controller. This controller
418 provides DMA capabilities for a variety of on-chip devices.
419
420config SIRF_DMA
421 tristate "CSR SiRFprimaII/SiRFmarco DMA support"
422 depends on ARCH_SIRF
316 select DMA_ENGINE 423 select DMA_ENGINE
317 help 424 help
318 Support the MXS DMA engine. This engine including APBH-DMA 425 Enable support for the CSR SiRFprimaII DMA engine.
319 and APBX-DMA is integrated into Freescale i.MX23/28/MX6Q/MX6DL chips.
320 426
321config EP93XX_DMA 427config STE_DMA40
322 bool "Cirrus Logic EP93xx DMA support" 428 bool "ST-Ericsson DMA40 support"
323 depends on ARCH_EP93XX 429 depends on ARCH_U8500
324 select DMA_ENGINE 430 select DMA_ENGINE
325 help 431 help
326 Enable support for the Cirrus Logic EP93xx M2P/M2M DMA controller. 432 Support for ST-Ericsson DMA40 controller
327 433
328config DMA_SA11X0 434config S3C24XX_DMAC
329 tristate "SA-11x0 DMA support" 435 tristate "Samsung S3C24XX DMA support"
330 depends on ARCH_SA1100 436 depends on ARCH_S3C24XX
331 select DMA_ENGINE 437 select DMA_ENGINE
332 select DMA_VIRTUAL_CHANNELS 438 select DMA_VIRTUAL_CHANNELS
333 help 439 help
334 Support the DMA engine found on Intel StrongARM SA-1100 and 440 Support for the Samsung S3C24XX DMA controller driver. The
335 SA-1110 SoCs. This DMA engine can only be used with on-chip 441 DMA controller is having multiple DMA channels which can be
336 devices. 442 configured for different peripherals like audio, UART, SPI.
443 The DMA controller can transfer data from memory to peripheral,
444 periphal to memory, periphal to periphal and memory to memory.
337 445
338config MMP_TDMA 446config TXX9_DMAC
339 bool "MMP Two-Channel DMA support" 447 tristate "Toshiba TXx9 SoC DMA support"
340 depends on ARCH_MMP 448 depends on MACH_TX49XX || MACH_TX39XX
341 select DMA_ENGINE 449 select DMA_ENGINE
342 select MMP_SRAM
343 help 450 help
344 Support the MMP Two-Channel DMA engine. 451 Support the TXx9 SoC internal DMA controller. This can be
345 This engine used for MMP Audio DMA and pxa910 SQU. 452 integrated in chips such as the Toshiba TX4927/38/39.
346 It needs sram driver under mach-mmp.
347
348 Say Y here if you enabled MMP ADMA, otherwise say N.
349 453
350config DMA_OMAP 454config TEGRA20_APB_DMA
351 tristate "OMAP DMA support" 455 bool "NVIDIA Tegra20 APB DMA support"
352 depends on ARCH_OMAP 456 depends on ARCH_TEGRA
353 select DMA_ENGINE 457 select DMA_ENGINE
354 select DMA_VIRTUAL_CHANNELS 458 help
355 select TI_DMA_CROSSBAR if SOC_DRA7XX 459 Support for the NVIDIA Tegra20 APB DMA controller driver. The
460 DMA controller is having multiple DMA channel which can be
461 configured for different peripherals like audio, UART, SPI,
462 I2C etc which is in APB bus.
463 This DMA controller transfers data from memory to peripheral fifo
464 or vice versa. It does not support memory to memory data transfer.
356 465
357config DMA_BCM2835 466config TIMB_DMA
358 tristate "BCM2835 DMA engine support" 467 tristate "Timberdale FPGA DMA support"
359 depends on ARCH_BCM2835 468 depends on MFD_TIMBERDALE
360 select DMA_ENGINE 469 select DMA_ENGINE
361 select DMA_VIRTUAL_CHANNELS 470 help
471 Enable support for the Timberdale FPGA DMA engine.
362 472
363config TI_CPPI41 473config TI_CPPI41
364 tristate "AM33xx CPPI41 DMA support" 474 tristate "AM33xx CPPI41 DMA support"
@@ -368,56 +478,28 @@ config TI_CPPI41
368 The Communications Port Programming Interface (CPPI) 4.1 DMA engine 478 The Communications Port Programming Interface (CPPI) 4.1 DMA engine
369 is currently used by the USB driver on AM335x platforms. 479 is currently used by the USB driver on AM335x platforms.
370 480
371config MMP_PDMA 481config TI_DMA_CROSSBAR
372 bool "MMP PDMA support" 482 bool
373 depends on (ARCH_MMP || ARCH_PXA)
374 select DMA_ENGINE
375 help
376 Support the MMP PDMA engine for PXA and MMP platform.
377
378config DMA_JZ4740
379 tristate "JZ4740 DMA support"
380 depends on MACH_JZ4740
381 select DMA_ENGINE
382 select DMA_VIRTUAL_CHANNELS
383
384config DMA_JZ4780
385 tristate "JZ4780 DMA support"
386 depends on MACH_JZ4780
387 select DMA_ENGINE
388 select DMA_VIRTUAL_CHANNELS
389 help
390 This selects support for the DMA controller in Ingenic JZ4780 SoCs.
391 If you have a board based on such a SoC and wish to use DMA for
392 devices which can use the DMA controller, say Y or M here.
393 483
394config K3_DMA 484config TI_EDMA
395 tristate "Hisilicon K3 DMA support" 485 bool "TI EDMA support"
396 depends on ARCH_HI3xxx 486 depends on ARCH_DAVINCI || ARCH_OMAP || ARCH_KEYSTONE
397 select DMA_ENGINE 487 select DMA_ENGINE
398 select DMA_VIRTUAL_CHANNELS 488 select DMA_VIRTUAL_CHANNELS
489 select TI_PRIV_EDMA
490 default n
399 help 491 help
400 Support the DMA engine for Hisilicon K3 platform 492 Enable support for the TI EDMA controller. This DMA
401 devices. 493 engine is found on TI DaVinci and AM33xx parts.
402 494
403config MOXART_DMA 495config XGENE_DMA
404 tristate "MOXART DMA support" 496 tristate "APM X-Gene DMA support"
405 depends on ARCH_MOXART 497 depends on ARCH_XGENE || COMPILE_TEST
406 select DMA_ENGINE
407 select DMA_OF
408 select DMA_VIRTUAL_CHANNELS
409 help
410 Enable support for the MOXA ART SoC DMA controller.
411
412config FSL_EDMA
413 tristate "Freescale eDMA engine support"
414 depends on OF
415 select DMA_ENGINE 498 select DMA_ENGINE
416 select DMA_VIRTUAL_CHANNELS 499 select DMA_ENGINE_RAID
500 select ASYNC_TX_ENABLE_CHANNEL_SWITCH
417 help 501 help
418 Support the Freescale eDMA engine with programmable channel 502 Enable support for the APM X-Gene SoC DMA engine.
419 multiplexing capability for DMA request sources(slot).
420 This module can be found on Freescale Vybrid and LS-1 SoCs.
421 503
422config XILINX_VDMA 504config XILINX_VDMA
423 tristate "Xilinx AXI VDMA Engine" 505 tristate "Xilinx AXI VDMA Engine"
@@ -433,55 +515,25 @@ config XILINX_VDMA
433 channels, Memory Mapped to Stream (MM2S) and Stream to 515 channels, Memory Mapped to Stream (MM2S) and Stream to
434 Memory Mapped (S2MM) for the data transfers. 516 Memory Mapped (S2MM) for the data transfers.
435 517
436config DMA_SUN6I 518config ZX_DMA
437 tristate "Allwinner A31 SoCs DMA support" 519 tristate "ZTE ZX296702 DMA support"
438 depends on MACH_SUN6I || MACH_SUN8I || COMPILE_TEST 520 depends on ARCH_ZX
439 depends on RESET_CONTROLLER
440 select DMA_ENGINE
441 select DMA_VIRTUAL_CHANNELS
442 help
443 Support for the DMA engine first found in Allwinner A31 SoCs.
444
445config NBPFAXI_DMA
446 tristate "Renesas Type-AXI NBPF DMA support"
447 select DMA_ENGINE
448 depends on ARM || COMPILE_TEST
449 help
450 Support for "Type-AXI" NBPF DMA IPs from Renesas
451
452config IMG_MDC_DMA
453 tristate "IMG MDC support"
454 depends on MIPS || COMPILE_TEST
455 depends on MFD_SYSCON
456 select DMA_ENGINE 521 select DMA_ENGINE
457 select DMA_VIRTUAL_CHANNELS 522 select DMA_VIRTUAL_CHANNELS
458 help 523 help
459 Enable support for the IMG multi-threaded DMA controller (MDC). 524 Support the DMA engine for ZTE ZX296702 platform devices.
460 525
461config XGENE_DMA
462 tristate "APM X-Gene DMA support"
463 depends on ARCH_XGENE || COMPILE_TEST
464 select DMA_ENGINE
465 select DMA_ENGINE_RAID
466 select ASYNC_TX_ENABLE_CHANNEL_SWITCH
467 help
468 Enable support for the APM X-Gene SoC DMA engine.
469 526
470config DMA_ENGINE 527# driver files
471 bool 528source "drivers/dma/bestcomm/Kconfig"
472 529
473config DMA_VIRTUAL_CHANNELS 530source "drivers/dma/dw/Kconfig"
474 tristate
475 531
476config DMA_ACPI 532source "drivers/dma/hsu/Kconfig"
477 def_bool y
478 depends on ACPI
479 533
480config DMA_OF 534source "drivers/dma/sh/Kconfig"
481 def_bool y
482 depends on OF
483 select DMA_ENGINE
484 535
536# clients
485comment "DMA Clients" 537comment "DMA Clients"
486 depends on DMA_ENGINE 538 depends on DMA_ENGINE
487 539
@@ -506,13 +558,4 @@ config DMATEST
506config DMA_ENGINE_RAID 558config DMA_ENGINE_RAID
507 bool 559 bool
508 560
509config QCOM_BAM_DMA
510 tristate "QCOM BAM DMA support"
511 depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM)
512 select DMA_ENGINE
513 select DMA_VIRTUAL_CHANNELS
514 ---help---
515 Enable support for the QCOM BAM DMA controller. This controller
516 provides DMA capabilities for a variety of on-chip devices.
517
518endif 561endif
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index 56ff8c705c00..7711a7180726 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -1,59 +1,69 @@
1#dmaengine debug flags
1subdir-ccflags-$(CONFIG_DMADEVICES_DEBUG) := -DDEBUG 2subdir-ccflags-$(CONFIG_DMADEVICES_DEBUG) := -DDEBUG
2subdir-ccflags-$(CONFIG_DMADEVICES_VDEBUG) += -DVERBOSE_DEBUG 3subdir-ccflags-$(CONFIG_DMADEVICES_VDEBUG) += -DVERBOSE_DEBUG
3 4
5#core
4obj-$(CONFIG_DMA_ENGINE) += dmaengine.o 6obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
5obj-$(CONFIG_DMA_VIRTUAL_CHANNELS) += virt-dma.o 7obj-$(CONFIG_DMA_VIRTUAL_CHANNELS) += virt-dma.o
6obj-$(CONFIG_DMA_ACPI) += acpi-dma.o 8obj-$(CONFIG_DMA_ACPI) += acpi-dma.o
7obj-$(CONFIG_DMA_OF) += of-dma.o 9obj-$(CONFIG_DMA_OF) += of-dma.o
8 10
11#dmatest
9obj-$(CONFIG_DMATEST) += dmatest.o 12obj-$(CONFIG_DMATEST) += dmatest.o
10obj-$(CONFIG_INTEL_IOATDMA) += ioat/ 13
11obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o 14#devices
12obj-$(CONFIG_FSL_DMA) += fsldma.o 15obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o
13obj-$(CONFIG_HSU_DMA) += hsu/ 16obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/
14obj-$(CONFIG_MPC512X_DMA) += mpc512x_dma.o
15obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/
16obj-$(CONFIG_MV_XOR) += mv_xor.o
17obj-$(CONFIG_IDMA64) += idma64.o
18obj-$(CONFIG_DW_DMAC_CORE) += dw/
19obj-$(CONFIG_AT_HDMAC) += at_hdmac.o 17obj-$(CONFIG_AT_HDMAC) += at_hdmac.o
20obj-$(CONFIG_AT_XDMAC) += at_xdmac.o 18obj-$(CONFIG_AT_XDMAC) += at_xdmac.o
21obj-$(CONFIG_MX3_IPU) += ipu/ 19obj-$(CONFIG_AXI_DMAC) += dma-axi-dmac.o
22obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
23obj-$(CONFIG_RENESAS_DMA) += sh/
24obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o 20obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o
25obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ 21obj-$(CONFIG_DMA_BCM2835) += bcm2835-dma.o
26obj-$(CONFIG_IMX_SDMA) += imx-sdma.o 22obj-$(CONFIG_DMA_JZ4740) += dma-jz4740.o
23obj-$(CONFIG_DMA_JZ4780) += dma-jz4780.o
24obj-$(CONFIG_DMA_OMAP) += omap-dma.o
25obj-$(CONFIG_DMA_SA11X0) += sa11x0-dma.o
26obj-$(CONFIG_DMA_SUN4I) += sun4i-dma.o
27obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o
28obj-$(CONFIG_DW_DMAC_CORE) += dw/
29obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o
30obj-$(CONFIG_FSL_DMA) += fsldma.o
31obj-$(CONFIG_FSL_EDMA) += fsl-edma.o
32obj-$(CONFIG_FSL_RAID) += fsl_raid.o
33obj-$(CONFIG_HSU_DMA) += hsu/
34obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o
27obj-$(CONFIG_IMX_DMA) += imx-dma.o 35obj-$(CONFIG_IMX_DMA) += imx-dma.o
36obj-$(CONFIG_IMX_SDMA) += imx-sdma.o
37obj-$(CONFIG_IDMA64) += idma64.o
38obj-$(CONFIG_INTEL_IOATDMA) += ioat/
39obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
40obj-$(CONFIG_INTEL_MIC_X100_DMA) += mic_x100_dma.o
41obj-$(CONFIG_K3_DMA) += k3dma.o
42obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o
43obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o
44obj-$(CONFIG_MMP_TDMA) += mmp_tdma.o
45obj-$(CONFIG_MOXART_DMA) += moxart-dma.o
46obj-$(CONFIG_MPC512X_DMA) += mpc512x_dma.o
47obj-$(CONFIG_MV_XOR) += mv_xor.o
28obj-$(CONFIG_MXS_DMA) += mxs-dma.o 48obj-$(CONFIG_MXS_DMA) += mxs-dma.o
49obj-$(CONFIG_MX3_IPU) += ipu/
50obj-$(CONFIG_NBPFAXI_DMA) += nbpfaxi.o
51obj-$(CONFIG_PCH_DMA) += pch_dma.o
52obj-$(CONFIG_PL330_DMA) += pl330.o
53obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/
29obj-$(CONFIG_PXA_DMA) += pxa_dma.o 54obj-$(CONFIG_PXA_DMA) += pxa_dma.o
30obj-$(CONFIG_TIMB_DMA) += timb_dma.o 55obj-$(CONFIG_QCOM_BAM_DMA) += qcom_bam_dma.o
56obj-$(CONFIG_RENESAS_DMA) += sh/
31obj-$(CONFIG_SIRF_DMA) += sirf-dma.o 57obj-$(CONFIG_SIRF_DMA) += sirf-dma.o
32obj-$(CONFIG_TI_EDMA) += edma.o
33obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o 58obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o
34obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o
35obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o 59obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o
36obj-$(CONFIG_PL330_DMA) += pl330.o 60obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
37obj-$(CONFIG_PCH_DMA) += pch_dma.o 61obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o
38obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o 62obj-$(CONFIG_TIMB_DMA) += timb_dma.o
39obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o
40obj-$(CONFIG_DMA_SA11X0) += sa11x0-dma.o
41obj-$(CONFIG_MMP_TDMA) += mmp_tdma.o
42obj-$(CONFIG_DMA_OMAP) += omap-dma.o
43obj-$(CONFIG_TI_DMA_CROSSBAR) += ti-dma-crossbar.o
44obj-$(CONFIG_DMA_BCM2835) += bcm2835-dma.o
45obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o
46obj-$(CONFIG_DMA_JZ4740) += dma-jz4740.o
47obj-$(CONFIG_DMA_JZ4780) += dma-jz4780.o
48obj-$(CONFIG_TI_CPPI41) += cppi41.o 63obj-$(CONFIG_TI_CPPI41) += cppi41.o
49obj-$(CONFIG_K3_DMA) += k3dma.o 64obj-$(CONFIG_TI_DMA_CROSSBAR) += ti-dma-crossbar.o
50obj-$(CONFIG_MOXART_DMA) += moxart-dma.o 65obj-$(CONFIG_TI_EDMA) += edma.o
51obj-$(CONFIG_FSL_RAID) += fsl_raid.o
52obj-$(CONFIG_FSL_EDMA) += fsl-edma.o
53obj-$(CONFIG_QCOM_BAM_DMA) += qcom_bam_dma.o
54obj-y += xilinx/
55obj-$(CONFIG_INTEL_MIC_X100_DMA) += mic_x100_dma.o
56obj-$(CONFIG_NBPFAXI_DMA) += nbpfaxi.o
57obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o
58obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o
59obj-$(CONFIG_XGENE_DMA) += xgene-dma.o 66obj-$(CONFIG_XGENE_DMA) += xgene-dma.o
67obj-$(CONFIG_ZX_DMA) += zx296702_dma.o
68
69obj-y += xilinx/
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index 5de3cf453f35..9b42c0588550 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -83,6 +83,8 @@
83#include <linux/init.h> 83#include <linux/init.h>
84#include <linux/interrupt.h> 84#include <linux/interrupt.h>
85#include <linux/module.h> 85#include <linux/module.h>
86#include <linux/of.h>
87#include <linux/of_dma.h>
86#include <linux/pm_runtime.h> 88#include <linux/pm_runtime.h>
87#include <linux/seq_file.h> 89#include <linux/seq_file.h>
88#include <linux/slab.h> 90#include <linux/slab.h>
@@ -2030,10 +2032,188 @@ static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
2030} 2032}
2031#endif 2033#endif
2032 2034
2035#ifdef CONFIG_OF
2036static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x,
2037 u32 id)
2038{
2039 struct pl08x_dma_chan *chan;
2040
2041 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {
2042 if (chan->signal == id)
2043 return &chan->vc.chan;
2044 }
2045
2046 return NULL;
2047}
2048
2049static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec,
2050 struct of_dma *ofdma)
2051{
2052 struct pl08x_driver_data *pl08x = ofdma->of_dma_data;
2053 struct pl08x_channel_data *data;
2054 struct pl08x_dma_chan *chan;
2055 struct dma_chan *dma_chan;
2056
2057 if (!pl08x)
2058 return NULL;
2059
2060 if (dma_spec->args_count != 2)
2061 return NULL;
2062
2063 dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]);
2064 if (dma_chan)
2065 return dma_get_slave_channel(dma_chan);
2066
2067 chan = devm_kzalloc(pl08x->slave.dev, sizeof(*chan) + sizeof(*data),
2068 GFP_KERNEL);
2069 if (!chan)
2070 return NULL;
2071
2072 data = (void *)&chan[1];
2073 data->bus_id = "(none)";
2074 data->periph_buses = dma_spec->args[1];
2075
2076 chan->cd = data;
2077 chan->host = pl08x;
2078 chan->slave = true;
2079 chan->name = data->bus_id;
2080 chan->state = PL08X_CHAN_IDLE;
2081 chan->signal = dma_spec->args[0];
2082 chan->vc.desc_free = pl08x_desc_free;
2083
2084 vchan_init(&chan->vc, &pl08x->slave);
2085
2086 return dma_get_slave_channel(&chan->vc.chan);
2087}
2088
2089static int pl08x_of_probe(struct amba_device *adev,
2090 struct pl08x_driver_data *pl08x,
2091 struct device_node *np)
2092{
2093 struct pl08x_platform_data *pd;
2094 u32 cctl_memcpy = 0;
2095 u32 val;
2096 int ret;
2097
2098 pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL);
2099 if (!pd)
2100 return -ENOMEM;
2101
2102 /* Eligible bus masters for fetching LLIs */
2103 if (of_property_read_bool(np, "lli-bus-interface-ahb1"))
2104 pd->lli_buses |= PL08X_AHB1;
2105 if (of_property_read_bool(np, "lli-bus-interface-ahb2"))
2106 pd->lli_buses |= PL08X_AHB2;
2107 if (!pd->lli_buses) {
2108 dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n");
2109 pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2;
2110 }
2111
2112 /* Eligible bus masters for memory access */
2113 if (of_property_read_bool(np, "mem-bus-interface-ahb1"))
2114 pd->mem_buses |= PL08X_AHB1;
2115 if (of_property_read_bool(np, "mem-bus-interface-ahb2"))
2116 pd->mem_buses |= PL08X_AHB2;
2117 if (!pd->mem_buses) {
2118 dev_info(&adev->dev, "no bus masters for memory stated, assume all\n");
2119 pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2;
2120 }
2121
2122 /* Parse the memcpy channel properties */
2123 ret = of_property_read_u32(np, "memcpy-burst-size", &val);
2124 if (ret) {
2125 dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n");
2126 val = 1;
2127 }
2128 switch (val) {
2129 default:
2130 dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n");
2131 /* Fall through */
2132 case 1:
2133 cctl_memcpy |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT |
2134 PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT;
2135 break;
2136 case 4:
2137 cctl_memcpy |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT |
2138 PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT;
2139 break;
2140 case 8:
2141 cctl_memcpy |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT |
2142 PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT;
2143 break;
2144 case 16:
2145 cctl_memcpy |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT |
2146 PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT;
2147 break;
2148 case 32:
2149 cctl_memcpy |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT |
2150 PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT;
2151 break;
2152 case 64:
2153 cctl_memcpy |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT |
2154 PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT;
2155 break;
2156 case 128:
2157 cctl_memcpy |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT |
2158 PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT;
2159 break;
2160 case 256:
2161 cctl_memcpy |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT |
2162 PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT;
2163 break;
2164 }
2165
2166 ret = of_property_read_u32(np, "memcpy-bus-width", &val);
2167 if (ret) {
2168 dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n");
2169 val = 8;
2170 }
2171 switch (val) {
2172 default:
2173 dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n");
2174 /* Fall through */
2175 case 8:
2176 cctl_memcpy |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT |
2177 PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;
2178 break;
2179 case 16:
2180 cctl_memcpy |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT |
2181 PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;
2182 break;
2183 case 32:
2184 cctl_memcpy |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT |
2185 PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;
2186 break;
2187 }
2188
2189 /* This is currently the only thing making sense */
2190 cctl_memcpy |= PL080_CONTROL_PROT_SYS;
2191
2192 /* Set up memcpy channel */
2193 pd->memcpy_channel.bus_id = "memcpy";
2194 pd->memcpy_channel.cctl_memcpy = cctl_memcpy;
2195 /* Use the buses that can access memory, obviously */
2196 pd->memcpy_channel.periph_buses = pd->mem_buses;
2197
2198 pl08x->pd = pd;
2199
2200 return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate,
2201 pl08x);
2202}
2203#else
2204static inline int pl08x_of_probe(struct amba_device *adev,
2205 struct pl08x_driver_data *pl08x,
2206 struct device_node *np)
2207{
2208 return -EINVAL;
2209}
2210#endif
2211
2033static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) 2212static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
2034{ 2213{
2035 struct pl08x_driver_data *pl08x; 2214 struct pl08x_driver_data *pl08x;
2036 const struct vendor_data *vd = id->data; 2215 const struct vendor_data *vd = id->data;
2216 struct device_node *np = adev->dev.of_node;
2037 u32 tsfr_size; 2217 u32 tsfr_size;
2038 int ret = 0; 2218 int ret = 0;
2039 int i; 2219 int i;
@@ -2093,9 +2273,15 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
2093 /* Get the platform data */ 2273 /* Get the platform data */
2094 pl08x->pd = dev_get_platdata(&adev->dev); 2274 pl08x->pd = dev_get_platdata(&adev->dev);
2095 if (!pl08x->pd) { 2275 if (!pl08x->pd) {
2096 dev_err(&adev->dev, "no platform data supplied\n"); 2276 if (np) {
2097 ret = -EINVAL; 2277 ret = pl08x_of_probe(adev, pl08x, np);
2098 goto out_no_platdata; 2278 if (ret)
2279 goto out_no_platdata;
2280 } else {
2281 dev_err(&adev->dev, "no platform data supplied\n");
2282 ret = -EINVAL;
2283 goto out_no_platdata;
2284 }
2099 } 2285 }
2100 2286
2101 /* Assign useful pointers to the driver state */ 2287 /* Assign useful pointers to the driver state */
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c
index d3629b7482dd..58d406230d89 100644
--- a/drivers/dma/at_hdmac.c
+++ b/drivers/dma/at_hdmac.c
@@ -448,6 +448,7 @@ static void
448atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) 448atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
449{ 449{
450 struct dma_async_tx_descriptor *txd = &desc->txd; 450 struct dma_async_tx_descriptor *txd = &desc->txd;
451 struct at_dma *atdma = to_at_dma(atchan->chan_common.device);
451 452
452 dev_vdbg(chan2dev(&atchan->chan_common), 453 dev_vdbg(chan2dev(&atchan->chan_common),
453 "descriptor %u complete\n", txd->cookie); 454 "descriptor %u complete\n", txd->cookie);
@@ -456,6 +457,13 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
456 if (!atc_chan_is_cyclic(atchan)) 457 if (!atc_chan_is_cyclic(atchan))
457 dma_cookie_complete(txd); 458 dma_cookie_complete(txd);
458 459
460 /* If the transfer was a memset, free our temporary buffer */
461 if (desc->memset) {
462 dma_pool_free(atdma->memset_pool, desc->memset_vaddr,
463 desc->memset_paddr);
464 desc->memset = false;
465 }
466
459 /* move children to free_list */ 467 /* move children to free_list */
460 list_splice_init(&desc->tx_list, &atchan->free_list); 468 list_splice_init(&desc->tx_list, &atchan->free_list);
461 /* move myself to free_list */ 469 /* move myself to free_list */
@@ -717,14 +725,14 @@ atc_prep_dma_interleaved(struct dma_chan *chan,
717 size_t len = 0; 725 size_t len = 0;
718 int i; 726 int i;
719 727
728 if (unlikely(!xt || xt->numf != 1 || !xt->frame_size))
729 return NULL;
730
720 dev_info(chan2dev(chan), 731 dev_info(chan2dev(chan),
721 "%s: src=0x%08x, dest=0x%08x, numf=%d, frame_size=%d, flags=0x%lx\n", 732 "%s: src=0x%08x, dest=0x%08x, numf=%d, frame_size=%d, flags=0x%lx\n",
722 __func__, xt->src_start, xt->dst_start, xt->numf, 733 __func__, xt->src_start, xt->dst_start, xt->numf,
723 xt->frame_size, flags); 734 xt->frame_size, flags);
724 735
725 if (unlikely(!xt || xt->numf != 1 || !xt->frame_size))
726 return NULL;
727
728 /* 736 /*
729 * The controller can only "skip" X bytes every Y bytes, so we 737 * The controller can only "skip" X bytes every Y bytes, so we
730 * need to make sure we are given a template that fit that 738 * need to make sure we are given a template that fit that
@@ -873,6 +881,93 @@ err_desc_get:
873 return NULL; 881 return NULL;
874} 882}
875 883
884/**
885 * atc_prep_dma_memset - prepare a memcpy operation
886 * @chan: the channel to prepare operation on
887 * @dest: operation virtual destination address
888 * @value: value to set memory buffer to
889 * @len: operation length
890 * @flags: tx descriptor status flags
891 */
892static struct dma_async_tx_descriptor *
893atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,
894 size_t len, unsigned long flags)
895{
896 struct at_dma_chan *atchan = to_at_dma_chan(chan);
897 struct at_dma *atdma = to_at_dma(chan->device);
898 struct at_desc *desc = NULL;
899 size_t xfer_count;
900 u32 ctrla;
901 u32 ctrlb;
902
903 dev_vdbg(chan2dev(chan), "%s: d0x%x v0x%x l0x%zx f0x%lx\n", __func__,
904 dest, value, len, flags);
905
906 if (unlikely(!len)) {
907 dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
908 return NULL;
909 }
910
911 if (!is_dma_fill_aligned(chan->device, dest, 0, len)) {
912 dev_dbg(chan2dev(chan), "%s: buffer is not aligned\n",
913 __func__);
914 return NULL;
915 }
916
917 xfer_count = len >> 2;
918 if (xfer_count > ATC_BTSIZE_MAX) {
919 dev_err(chan2dev(chan), "%s: buffer is too big\n",
920 __func__);
921 return NULL;
922 }
923
924 ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN
925 | ATC_SRC_ADDR_MODE_FIXED
926 | ATC_DST_ADDR_MODE_INCR
927 | ATC_FC_MEM2MEM;
928
929 ctrla = ATC_SRC_WIDTH(2) |
930 ATC_DST_WIDTH(2);
931
932 desc = atc_desc_get(atchan);
933 if (!desc) {
934 dev_err(chan2dev(chan), "%s: can't get a descriptor\n",
935 __func__);
936 return NULL;
937 }
938
939 desc->memset_vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC,
940 &desc->memset_paddr);
941 if (!desc->memset_vaddr) {
942 dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n",
943 __func__);
944 goto err_put_desc;
945 }
946
947 *desc->memset_vaddr = value;
948 desc->memset = true;
949
950 desc->lli.saddr = desc->memset_paddr;
951 desc->lli.daddr = dest;
952 desc->lli.ctrla = ctrla | xfer_count;
953 desc->lli.ctrlb = ctrlb;
954
955 desc->txd.cookie = -EBUSY;
956 desc->len = len;
957 desc->total_len = len;
958
959 /* set end-of-link on the descriptor */
960 set_desc_eol(desc);
961
962 desc->txd.flags = flags;
963
964 return &desc->txd;
965
966err_put_desc:
967 atc_desc_put(atchan, desc);
968 return NULL;
969}
970
876 971
877/** 972/**
878 * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction 973 * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
@@ -1755,6 +1850,8 @@ static int __init at_dma_probe(struct platform_device *pdev)
1755 dma_cap_set(DMA_SG, at91sam9rl_config.cap_mask); 1850 dma_cap_set(DMA_SG, at91sam9rl_config.cap_mask);
1756 dma_cap_set(DMA_INTERLEAVE, at91sam9g45_config.cap_mask); 1851 dma_cap_set(DMA_INTERLEAVE, at91sam9g45_config.cap_mask);
1757 dma_cap_set(DMA_MEMCPY, at91sam9g45_config.cap_mask); 1852 dma_cap_set(DMA_MEMCPY, at91sam9g45_config.cap_mask);
1853 dma_cap_set(DMA_MEMSET, at91sam9g45_config.cap_mask);
1854 dma_cap_set(DMA_PRIVATE, at91sam9g45_config.cap_mask);
1758 dma_cap_set(DMA_SLAVE, at91sam9g45_config.cap_mask); 1855 dma_cap_set(DMA_SLAVE, at91sam9g45_config.cap_mask);
1759 dma_cap_set(DMA_SG, at91sam9g45_config.cap_mask); 1856 dma_cap_set(DMA_SG, at91sam9g45_config.cap_mask);
1760 1857
@@ -1818,7 +1915,16 @@ static int __init at_dma_probe(struct platform_device *pdev)
1818 if (!atdma->dma_desc_pool) { 1915 if (!atdma->dma_desc_pool) {
1819 dev_err(&pdev->dev, "No memory for descriptors dma pool\n"); 1916 dev_err(&pdev->dev, "No memory for descriptors dma pool\n");
1820 err = -ENOMEM; 1917 err = -ENOMEM;
1821 goto err_pool_create; 1918 goto err_desc_pool_create;
1919 }
1920
1921 /* create a pool of consistent memory blocks for memset blocks */
1922 atdma->memset_pool = dma_pool_create("at_hdmac_memset_pool",
1923 &pdev->dev, sizeof(int), 4, 0);
1924 if (!atdma->memset_pool) {
1925 dev_err(&pdev->dev, "No memory for memset dma pool\n");
1926 err = -ENOMEM;
1927 goto err_memset_pool_create;
1822 } 1928 }
1823 1929
1824 /* clear any pending interrupt */ 1930 /* clear any pending interrupt */
@@ -1864,6 +1970,11 @@ static int __init at_dma_probe(struct platform_device *pdev)
1864 if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask)) 1970 if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))
1865 atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy; 1971 atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;
1866 1972
1973 if (dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask)) {
1974 atdma->dma_common.device_prep_dma_memset = atc_prep_dma_memset;
1975 atdma->dma_common.fill_align = DMAENGINE_ALIGN_4_BYTES;
1976 }
1977
1867 if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) { 1978 if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {
1868 atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg; 1979 atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;
1869 /* controller can do slave DMA: can trigger cyclic transfers */ 1980 /* controller can do slave DMA: can trigger cyclic transfers */
@@ -1884,8 +1995,9 @@ static int __init at_dma_probe(struct platform_device *pdev)
1884 1995
1885 dma_writel(atdma, EN, AT_DMA_ENABLE); 1996 dma_writel(atdma, EN, AT_DMA_ENABLE);
1886 1997
1887 dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s), %d channels\n", 1998 dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s%s), %d channels\n",
1888 dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "", 1999 dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",
2000 dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask) ? "set " : "",
1889 dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "", 2001 dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "",
1890 dma_has_cap(DMA_SG, atdma->dma_common.cap_mask) ? "sg-cpy " : "", 2002 dma_has_cap(DMA_SG, atdma->dma_common.cap_mask) ? "sg-cpy " : "",
1891 plat_dat->nr_channels); 2003 plat_dat->nr_channels);
@@ -1910,8 +2022,10 @@ static int __init at_dma_probe(struct platform_device *pdev)
1910 2022
1911err_of_dma_controller_register: 2023err_of_dma_controller_register:
1912 dma_async_device_unregister(&atdma->dma_common); 2024 dma_async_device_unregister(&atdma->dma_common);
2025 dma_pool_destroy(atdma->memset_pool);
2026err_memset_pool_create:
1913 dma_pool_destroy(atdma->dma_desc_pool); 2027 dma_pool_destroy(atdma->dma_desc_pool);
1914err_pool_create: 2028err_desc_pool_create:
1915 free_irq(platform_get_irq(pdev, 0), atdma); 2029 free_irq(platform_get_irq(pdev, 0), atdma);
1916err_irq: 2030err_irq:
1917 clk_disable_unprepare(atdma->clk); 2031 clk_disable_unprepare(atdma->clk);
@@ -1936,6 +2050,7 @@ static int at_dma_remove(struct platform_device *pdev)
1936 at_dma_off(atdma); 2050 at_dma_off(atdma);
1937 dma_async_device_unregister(&atdma->dma_common); 2051 dma_async_device_unregister(&atdma->dma_common);
1938 2052
2053 dma_pool_destroy(atdma->memset_pool);
1939 dma_pool_destroy(atdma->dma_desc_pool); 2054 dma_pool_destroy(atdma->dma_desc_pool);
1940 free_irq(platform_get_irq(pdev, 0), atdma); 2055 free_irq(platform_get_irq(pdev, 0), atdma);
1941 2056
diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h
index 7f5a08230f76..c3bebbe899ac 100644
--- a/drivers/dma/at_hdmac_regs.h
+++ b/drivers/dma/at_hdmac_regs.h
@@ -200,6 +200,11 @@ struct at_desc {
200 size_t boundary; 200 size_t boundary;
201 size_t dst_hole; 201 size_t dst_hole;
202 size_t src_hole; 202 size_t src_hole;
203
204 /* Memset temporary buffer */
205 bool memset;
206 dma_addr_t memset_paddr;
207 int *memset_vaddr;
203}; 208};
204 209
205static inline struct at_desc * 210static inline struct at_desc *
@@ -330,6 +335,7 @@ struct at_dma {
330 u8 all_chan_mask; 335 u8 all_chan_mask;
331 336
332 struct dma_pool *dma_desc_pool; 337 struct dma_pool *dma_desc_pool;
338 struct dma_pool *memset_pool;
333 /* AT THE END channels table */ 339 /* AT THE END channels table */
334 struct at_dma_chan chan[0]; 340 struct at_dma_chan chan[0];
335}; 341};
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
index 40afa2a16cfc..a165b4bfd330 100644
--- a/drivers/dma/at_xdmac.c
+++ b/drivers/dma/at_xdmac.c
@@ -625,12 +625,12 @@ at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
625 unsigned int sg_len, enum dma_transfer_direction direction, 625 unsigned int sg_len, enum dma_transfer_direction direction,
626 unsigned long flags, void *context) 626 unsigned long flags, void *context)
627{ 627{
628 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); 628 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
629 struct at_xdmac_desc *first = NULL, *prev = NULL; 629 struct at_xdmac_desc *first = NULL, *prev = NULL;
630 struct scatterlist *sg; 630 struct scatterlist *sg;
631 int i; 631 int i;
632 unsigned int xfer_size = 0; 632 unsigned int xfer_size = 0;
633 unsigned long irqflags; 633 unsigned long irqflags;
634 struct dma_async_tx_descriptor *ret = NULL; 634 struct dma_async_tx_descriptor *ret = NULL;
635 635
636 if (!sgl) 636 if (!sgl)
@@ -797,10 +797,7 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
797 list_add_tail(&desc->desc_node, &first->descs_list); 797 list_add_tail(&desc->desc_node, &first->descs_list);
798 } 798 }
799 799
800 prev->lld.mbr_nda = first->tx_dma_desc.phys; 800 at_xdmac_queue_desc(chan, prev, first);
801 dev_dbg(chan2dev(chan),
802 "%s: chain lld: prev=0x%p, mbr_nda=%pad\n",
803 __func__, prev, &prev->lld.mbr_nda);
804 first->tx_dma_desc.flags = flags; 801 first->tx_dma_desc.flags = flags;
805 first->xfer_size = buf_len; 802 first->xfer_size = buf_len;
806 first->direction = direction; 803 first->direction = direction;
@@ -1135,7 +1132,7 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan,
1135 * SAMA5D4x), so we can use the same interface for source and dest, 1132 * SAMA5D4x), so we can use the same interface for source and dest,
1136 * that solves the fact we don't know the direction. 1133 * that solves the fact we don't know the direction.
1137 */ 1134 */
1138 u32 chan_cc = AT_XDMAC_CC_DAM_INCREMENTED_AM 1135 u32 chan_cc = AT_XDMAC_CC_DAM_UBS_AM
1139 | AT_XDMAC_CC_SAM_INCREMENTED_AM 1136 | AT_XDMAC_CC_SAM_INCREMENTED_AM
1140 | AT_XDMAC_CC_DIF(0) 1137 | AT_XDMAC_CC_DIF(0)
1141 | AT_XDMAC_CC_SIF(0) 1138 | AT_XDMAC_CC_SIF(0)
@@ -1203,6 +1200,168 @@ at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,
1203 return &desc->tx_dma_desc; 1200 return &desc->tx_dma_desc;
1204} 1201}
1205 1202
1203static struct dma_async_tx_descriptor *
1204at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl,
1205 unsigned int sg_len, int value,
1206 unsigned long flags)
1207{
1208 struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
1209 struct at_xdmac_desc *desc, *pdesc = NULL,
1210 *ppdesc = NULL, *first = NULL;
1211 struct scatterlist *sg, *psg = NULL, *ppsg = NULL;
1212 size_t stride = 0, pstride = 0, len = 0;
1213 int i;
1214
1215 if (!sgl)
1216 return NULL;
1217
1218 dev_dbg(chan2dev(chan), "%s: sg_len=%d, value=0x%x, flags=0x%lx\n",
1219 __func__, sg_len, value, flags);
1220
1221 /* Prepare descriptors. */
1222 for_each_sg(sgl, sg, sg_len, i) {
1223 dev_dbg(chan2dev(chan), "%s: dest=0x%08x, len=%d, pattern=0x%x, flags=0x%lx\n",
1224 __func__, sg_dma_address(sg), sg_dma_len(sg),
1225 value, flags);
1226 desc = at_xdmac_memset_create_desc(chan, atchan,
1227 sg_dma_address(sg),
1228 sg_dma_len(sg),
1229 value);
1230 if (!desc && first)
1231 list_splice_init(&first->descs_list,
1232 &atchan->free_descs_list);
1233
1234 if (!first)
1235 first = desc;
1236
1237 /* Update our strides */
1238 pstride = stride;
1239 if (psg)
1240 stride = sg_dma_address(sg) -
1241 (sg_dma_address(psg) + sg_dma_len(psg));
1242
1243 /*
1244 * The scatterlist API gives us only the address and
1245 * length of each elements.
1246 *
1247 * Unfortunately, we don't have the stride, which we
1248 * will need to compute.
1249 *
1250 * That make us end up in a situation like this one:
1251 * len stride len stride len
1252 * +-------+ +-------+ +-------+
1253 * | N-2 | | N-1 | | N |
1254 * +-------+ +-------+ +-------+
1255 *
1256 * We need all these three elements (N-2, N-1 and N)
1257 * to actually take the decision on whether we need to
1258 * queue N-1 or reuse N-2.
1259 *
1260 * We will only consider N if it is the last element.
1261 */
1262 if (ppdesc && pdesc) {
1263 if ((stride == pstride) &&
1264 (sg_dma_len(ppsg) == sg_dma_len(psg))) {
1265 dev_dbg(chan2dev(chan),
1266 "%s: desc 0x%p can be merged with desc 0x%p\n",
1267 __func__, pdesc, ppdesc);
1268
1269 /*
1270 * Increment the block count of the
1271 * N-2 descriptor
1272 */
1273 at_xdmac_increment_block_count(chan, ppdesc);
1274 ppdesc->lld.mbr_dus = stride;
1275
1276 /*
1277 * Put back the N-1 descriptor in the
1278 * free descriptor list
1279 */
1280 list_add_tail(&pdesc->desc_node,
1281 &atchan->free_descs_list);
1282
1283 /*
1284 * Make our N-1 descriptor pointer
1285 * point to the N-2 since they were
1286 * actually merged.
1287 */
1288 pdesc = ppdesc;
1289
1290 /*
1291 * Rule out the case where we don't have
1292 * pstride computed yet (our second sg
1293 * element)
1294 *
1295 * We also want to catch the case where there
1296 * would be a negative stride,
1297 */
1298 } else if (pstride ||
1299 sg_dma_address(sg) < sg_dma_address(psg)) {
1300 /*
1301 * Queue the N-1 descriptor after the
1302 * N-2
1303 */
1304 at_xdmac_queue_desc(chan, ppdesc, pdesc);
1305
1306 /*
1307 * Add the N-1 descriptor to the list
1308 * of the descriptors used for this
1309 * transfer
1310 */
1311 list_add_tail(&desc->desc_node,
1312 &first->descs_list);
1313 dev_dbg(chan2dev(chan),
1314 "%s: add desc 0x%p to descs_list 0x%p\n",
1315 __func__, desc, first);
1316 }
1317 }
1318
1319 /*
1320 * If we are the last element, just see if we have the
1321 * same size than the previous element.
1322 *
1323 * If so, we can merge it with the previous descriptor
1324 * since we don't care about the stride anymore.
1325 */
1326 if ((i == (sg_len - 1)) &&
1327 sg_dma_len(ppsg) == sg_dma_len(psg)) {
1328 dev_dbg(chan2dev(chan),
1329 "%s: desc 0x%p can be merged with desc 0x%p\n",
1330 __func__, desc, pdesc);
1331
1332 /*
1333 * Increment the block count of the N-1
1334 * descriptor
1335 */
1336 at_xdmac_increment_block_count(chan, pdesc);
1337 pdesc->lld.mbr_dus = stride;
1338
1339 /*
1340 * Put back the N descriptor in the free
1341 * descriptor list
1342 */
1343 list_add_tail(&desc->desc_node,
1344 &atchan->free_descs_list);
1345 }
1346
1347 /* Update our descriptors */
1348 ppdesc = pdesc;
1349 pdesc = desc;
1350
1351 /* Update our scatter pointers */
1352 ppsg = psg;
1353 psg = sg;
1354
1355 len += sg_dma_len(sg);
1356 }
1357
1358 first->tx_dma_desc.cookie = -EBUSY;
1359 first->tx_dma_desc.flags = flags;
1360 first->xfer_size = len;
1361
1362 return &first->tx_dma_desc;
1363}
1364
1206static enum dma_status 1365static enum dma_status
1207at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, 1366at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
1208 struct dma_tx_state *txstate) 1367 struct dma_tx_state *txstate)
@@ -1736,6 +1895,7 @@ static int at_xdmac_probe(struct platform_device *pdev)
1736 dma_cap_set(DMA_INTERLEAVE, atxdmac->dma.cap_mask); 1895 dma_cap_set(DMA_INTERLEAVE, atxdmac->dma.cap_mask);
1737 dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask); 1896 dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask);
1738 dma_cap_set(DMA_MEMSET, atxdmac->dma.cap_mask); 1897 dma_cap_set(DMA_MEMSET, atxdmac->dma.cap_mask);
1898 dma_cap_set(DMA_MEMSET_SG, atxdmac->dma.cap_mask);
1739 dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask); 1899 dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask);
1740 /* 1900 /*
1741 * Without DMA_PRIVATE the driver is not able to allocate more than 1901 * Without DMA_PRIVATE the driver is not able to allocate more than
@@ -1751,6 +1911,7 @@ static int at_xdmac_probe(struct platform_device *pdev)
1751 atxdmac->dma.device_prep_interleaved_dma = at_xdmac_prep_interleaved; 1911 atxdmac->dma.device_prep_interleaved_dma = at_xdmac_prep_interleaved;
1752 atxdmac->dma.device_prep_dma_memcpy = at_xdmac_prep_dma_memcpy; 1912 atxdmac->dma.device_prep_dma_memcpy = at_xdmac_prep_dma_memcpy;
1753 atxdmac->dma.device_prep_dma_memset = at_xdmac_prep_dma_memset; 1913 atxdmac->dma.device_prep_dma_memset = at_xdmac_prep_dma_memset;
1914 atxdmac->dma.device_prep_dma_memset_sg = at_xdmac_prep_dma_memset_sg;
1754 atxdmac->dma.device_prep_slave_sg = at_xdmac_prep_slave_sg; 1915 atxdmac->dma.device_prep_slave_sg = at_xdmac_prep_slave_sg;
1755 atxdmac->dma.device_config = at_xdmac_device_config; 1916 atxdmac->dma.device_config = at_xdmac_device_config;
1756 atxdmac->dma.device_pause = at_xdmac_device_pause; 1917 atxdmac->dma.device_pause = at_xdmac_device_pause;
diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c
index fd22dd36985f..c340ca9bd2b5 100644
--- a/drivers/dma/coh901318.c
+++ b/drivers/dma/coh901318.c
@@ -2730,7 +2730,7 @@ static int __init coh901318_probe(struct platform_device *pdev)
2730 * This controller can only access address at even 32bit boundaries, 2730 * This controller can only access address at even 32bit boundaries,
2731 * i.e. 2^2 2731 * i.e. 2^2
2732 */ 2732 */
2733 base->dma_memcpy.copy_align = 2; 2733 base->dma_memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES;
2734 err = dma_async_device_register(&base->dma_memcpy); 2734 err = dma_async_device_register(&base->dma_memcpy);
2735 2735
2736 if (err) 2736 if (err)
diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c
new file mode 100644
index 000000000000..5b2395e7e04d
--- /dev/null
+++ b/drivers/dma/dma-axi-dmac.c
@@ -0,0 +1,691 @@
1/*
2 * Driver for the Analog Devices AXI-DMAC core
3 *
4 * Copyright 2013-2015 Analog Devices Inc.
5 * Author: Lars-Peter Clausen <lars@metafoo.de>
6 *
7 * Licensed under the GPL-2.
8 */
9
10#include <linux/clk.h>
11#include <linux/device.h>
12#include <linux/dma-mapping.h>
13#include <linux/dmaengine.h>
14#include <linux/err.h>
15#include <linux/interrupt.h>
16#include <linux/io.h>
17#include <linux/kernel.h>
18#include <linux/module.h>
19#include <linux/of.h>
20#include <linux/of_dma.h>
21#include <linux/platform_device.h>
22#include <linux/slab.h>
23
24#include <dt-bindings/dma/axi-dmac.h>
25
26#include "dmaengine.h"
27#include "virt-dma.h"
28
29/*
30 * The AXI-DMAC is a soft IP core that is used in FPGA designs. The core has
31 * various instantiation parameters which decided the exact feature set support
32 * by the core.
33 *
34 * Each channel of the core has a source interface and a destination interface.
35 * The number of channels and the type of the channel interfaces is selected at
36 * configuration time. A interface can either be a connected to a central memory
37 * interconnect, which allows access to system memory, or it can be connected to
38 * a dedicated bus which is directly connected to a data port on a peripheral.
39 * Given that those are configuration options of the core that are selected when
40 * it is instantiated this means that they can not be changed by software at
41 * runtime. By extension this means that each channel is uni-directional. It can
42 * either be device to memory or memory to device, but not both. Also since the
43 * device side is a dedicated data bus only connected to a single peripheral
44 * there is no address than can or needs to be configured for the device side.
45 */
46
47#define AXI_DMAC_REG_IRQ_MASK 0x80
48#define AXI_DMAC_REG_IRQ_PENDING 0x84
49#define AXI_DMAC_REG_IRQ_SOURCE 0x88
50
51#define AXI_DMAC_REG_CTRL 0x400
52#define AXI_DMAC_REG_TRANSFER_ID 0x404
53#define AXI_DMAC_REG_START_TRANSFER 0x408
54#define AXI_DMAC_REG_FLAGS 0x40c
55#define AXI_DMAC_REG_DEST_ADDRESS 0x410
56#define AXI_DMAC_REG_SRC_ADDRESS 0x414
57#define AXI_DMAC_REG_X_LENGTH 0x418
58#define AXI_DMAC_REG_Y_LENGTH 0x41c
59#define AXI_DMAC_REG_DEST_STRIDE 0x420
60#define AXI_DMAC_REG_SRC_STRIDE 0x424
61#define AXI_DMAC_REG_TRANSFER_DONE 0x428
62#define AXI_DMAC_REG_ACTIVE_TRANSFER_ID 0x42c
63#define AXI_DMAC_REG_STATUS 0x430
64#define AXI_DMAC_REG_CURRENT_SRC_ADDR 0x434
65#define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438
66
67#define AXI_DMAC_CTRL_ENABLE BIT(0)
68#define AXI_DMAC_CTRL_PAUSE BIT(1)
69
70#define AXI_DMAC_IRQ_SOT BIT(0)
71#define AXI_DMAC_IRQ_EOT BIT(1)
72
73#define AXI_DMAC_FLAG_CYCLIC BIT(0)
74
75struct axi_dmac_sg {
76 dma_addr_t src_addr;
77 dma_addr_t dest_addr;
78 unsigned int x_len;
79 unsigned int y_len;
80 unsigned int dest_stride;
81 unsigned int src_stride;
82 unsigned int id;
83};
84
85struct axi_dmac_desc {
86 struct virt_dma_desc vdesc;
87 bool cyclic;
88
89 unsigned int num_submitted;
90 unsigned int num_completed;
91 unsigned int num_sgs;
92 struct axi_dmac_sg sg[];
93};
94
95struct axi_dmac_chan {
96 struct virt_dma_chan vchan;
97
98 struct axi_dmac_desc *next_desc;
99 struct list_head active_descs;
100 enum dma_transfer_direction direction;
101
102 unsigned int src_width;
103 unsigned int dest_width;
104 unsigned int src_type;
105 unsigned int dest_type;
106
107 unsigned int max_length;
108 unsigned int align_mask;
109
110 bool hw_cyclic;
111 bool hw_2d;
112};
113
114struct axi_dmac {
115 void __iomem *base;
116 int irq;
117
118 struct clk *clk;
119
120 struct dma_device dma_dev;
121 struct axi_dmac_chan chan;
122
123 struct device_dma_parameters dma_parms;
124};
125
126static struct axi_dmac *chan_to_axi_dmac(struct axi_dmac_chan *chan)
127{
128 return container_of(chan->vchan.chan.device, struct axi_dmac,
129 dma_dev);
130}
131
132static struct axi_dmac_chan *to_axi_dmac_chan(struct dma_chan *c)
133{
134 return container_of(c, struct axi_dmac_chan, vchan.chan);
135}
136
137static struct axi_dmac_desc *to_axi_dmac_desc(struct virt_dma_desc *vdesc)
138{
139 return container_of(vdesc, struct axi_dmac_desc, vdesc);
140}
141
142static void axi_dmac_write(struct axi_dmac *axi_dmac, unsigned int reg,
143 unsigned int val)
144{
145 writel(val, axi_dmac->base + reg);
146}
147
148static int axi_dmac_read(struct axi_dmac *axi_dmac, unsigned int reg)
149{
150 return readl(axi_dmac->base + reg);
151}
152
153static int axi_dmac_src_is_mem(struct axi_dmac_chan *chan)
154{
155 return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM;
156}
157
158static int axi_dmac_dest_is_mem(struct axi_dmac_chan *chan)
159{
160 return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM;
161}
162
163static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len)
164{
165 if (len == 0 || len > chan->max_length)
166 return false;
167 if ((len & chan->align_mask) != 0) /* Not aligned */
168 return false;
169 return true;
170}
171
172static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr)
173{
174 if ((addr & chan->align_mask) != 0) /* Not aligned */
175 return false;
176 return true;
177}
178
179static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
180{
181 struct axi_dmac *dmac = chan_to_axi_dmac(chan);
182 struct virt_dma_desc *vdesc;
183 struct axi_dmac_desc *desc;
184 struct axi_dmac_sg *sg;
185 unsigned int flags = 0;
186 unsigned int val;
187
188 val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER);
189 if (val) /* Queue is full, wait for the next SOT IRQ */
190 return;
191
192 desc = chan->next_desc;
193
194 if (!desc) {
195 vdesc = vchan_next_desc(&chan->vchan);
196 if (!vdesc)
197 return;
198 list_move_tail(&vdesc->node, &chan->active_descs);
199 desc = to_axi_dmac_desc(vdesc);
200 }
201 sg = &desc->sg[desc->num_submitted];
202
203 desc->num_submitted++;
204 if (desc->num_submitted == desc->num_sgs)
205 chan->next_desc = NULL;
206 else
207 chan->next_desc = desc;
208
209 sg->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID);
210
211 if (axi_dmac_dest_is_mem(chan)) {
212 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->dest_addr);
213 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->dest_stride);
214 }
215
216 if (axi_dmac_src_is_mem(chan)) {
217 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->src_addr);
218 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->src_stride);
219 }
220
221 /*
222 * If the hardware supports cyclic transfers and there is no callback to
223 * call, enable hw cyclic mode to avoid unnecessary interrupts.
224 */
225 if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback)
226 flags |= AXI_DMAC_FLAG_CYCLIC;
227
228 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->x_len - 1);
229 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->y_len - 1);
230 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags);
231 axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1);
232}
233
234static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan)
235{
236 return list_first_entry_or_null(&chan->active_descs,
237 struct axi_dmac_desc, vdesc.node);
238}
239
240static void axi_dmac_transfer_done(struct axi_dmac_chan *chan,
241 unsigned int completed_transfers)
242{
243 struct axi_dmac_desc *active;
244 struct axi_dmac_sg *sg;
245
246 active = axi_dmac_active_desc(chan);
247 if (!active)
248 return;
249
250 if (active->cyclic) {
251 vchan_cyclic_callback(&active->vdesc);
252 } else {
253 do {
254 sg = &active->sg[active->num_completed];
255 if (!(BIT(sg->id) & completed_transfers))
256 break;
257 active->num_completed++;
258 if (active->num_completed == active->num_sgs) {
259 list_del(&active->vdesc.node);
260 vchan_cookie_complete(&active->vdesc);
261 active = axi_dmac_active_desc(chan);
262 }
263 } while (active);
264 }
265}
266
267static irqreturn_t axi_dmac_interrupt_handler(int irq, void *devid)
268{
269 struct axi_dmac *dmac = devid;
270 unsigned int pending;
271
272 pending = axi_dmac_read(dmac, AXI_DMAC_REG_IRQ_PENDING);
273 axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_PENDING, pending);
274
275 spin_lock(&dmac->chan.vchan.lock);
276 /* One or more transfers have finished */
277 if (pending & AXI_DMAC_IRQ_EOT) {
278 unsigned int completed;
279
280 completed = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE);
281 axi_dmac_transfer_done(&dmac->chan, completed);
282 }
283 /* Space has become available in the descriptor queue */
284 if (pending & AXI_DMAC_IRQ_SOT)
285 axi_dmac_start_transfer(&dmac->chan);
286 spin_unlock(&dmac->chan.vchan.lock);
287
288 return IRQ_HANDLED;
289}
290
291static int axi_dmac_terminate_all(struct dma_chan *c)
292{
293 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
294 struct axi_dmac *dmac = chan_to_axi_dmac(chan);
295 unsigned long flags;
296 LIST_HEAD(head);
297
298 spin_lock_irqsave(&chan->vchan.lock, flags);
299 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0);
300 chan->next_desc = NULL;
301 vchan_get_all_descriptors(&chan->vchan, &head);
302 list_splice_tail_init(&chan->active_descs, &head);
303 spin_unlock_irqrestore(&chan->vchan.lock, flags);
304
305 vchan_dma_desc_free_list(&chan->vchan, &head);
306
307 return 0;
308}
309
310static void axi_dmac_issue_pending(struct dma_chan *c)
311{
312 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
313 struct axi_dmac *dmac = chan_to_axi_dmac(chan);
314 unsigned long flags;
315
316 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE);
317
318 spin_lock_irqsave(&chan->vchan.lock, flags);
319 if (vchan_issue_pending(&chan->vchan))
320 axi_dmac_start_transfer(chan);
321 spin_unlock_irqrestore(&chan->vchan.lock, flags);
322}
323
324static struct axi_dmac_desc *axi_dmac_alloc_desc(unsigned int num_sgs)
325{
326 struct axi_dmac_desc *desc;
327
328 desc = kzalloc(sizeof(struct axi_dmac_desc) +
329 sizeof(struct axi_dmac_sg) * num_sgs, GFP_NOWAIT);
330 if (!desc)
331 return NULL;
332
333 desc->num_sgs = num_sgs;
334
335 return desc;
336}
337
338static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg(
339 struct dma_chan *c, struct scatterlist *sgl,
340 unsigned int sg_len, enum dma_transfer_direction direction,
341 unsigned long flags, void *context)
342{
343 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
344 struct axi_dmac_desc *desc;
345 struct scatterlist *sg;
346 unsigned int i;
347
348 if (direction != chan->direction)
349 return NULL;
350
351 desc = axi_dmac_alloc_desc(sg_len);
352 if (!desc)
353 return NULL;
354
355 for_each_sg(sgl, sg, sg_len, i) {
356 if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) ||
357 !axi_dmac_check_len(chan, sg_dma_len(sg))) {
358 kfree(desc);
359 return NULL;
360 }
361
362 if (direction == DMA_DEV_TO_MEM)
363 desc->sg[i].dest_addr = sg_dma_address(sg);
364 else
365 desc->sg[i].src_addr = sg_dma_address(sg);
366 desc->sg[i].x_len = sg_dma_len(sg);
367 desc->sg[i].y_len = 1;
368 }
369
370 desc->cyclic = false;
371
372 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
373}
374
375static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic(
376 struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,
377 size_t period_len, enum dma_transfer_direction direction,
378 unsigned long flags)
379{
380 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
381 struct axi_dmac_desc *desc;
382 unsigned int num_periods, i;
383
384 if (direction != chan->direction)
385 return NULL;
386
387 if (!axi_dmac_check_len(chan, buf_len) ||
388 !axi_dmac_check_addr(chan, buf_addr))
389 return NULL;
390
391 if (period_len == 0 || buf_len % period_len)
392 return NULL;
393
394 num_periods = buf_len / period_len;
395
396 desc = axi_dmac_alloc_desc(num_periods);
397 if (!desc)
398 return NULL;
399
400 for (i = 0; i < num_periods; i++) {
401 if (direction == DMA_DEV_TO_MEM)
402 desc->sg[i].dest_addr = buf_addr;
403 else
404 desc->sg[i].src_addr = buf_addr;
405 desc->sg[i].x_len = period_len;
406 desc->sg[i].y_len = 1;
407 buf_addr += period_len;
408 }
409
410 desc->cyclic = true;
411
412 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
413}
414
415static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved(
416 struct dma_chan *c, struct dma_interleaved_template *xt,
417 unsigned long flags)
418{
419 struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
420 struct axi_dmac_desc *desc;
421 size_t dst_icg, src_icg;
422
423 if (xt->frame_size != 1)
424 return NULL;
425
426 if (xt->dir != chan->direction)
427 return NULL;
428
429 if (axi_dmac_src_is_mem(chan)) {
430 if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start))
431 return NULL;
432 }
433
434 if (axi_dmac_dest_is_mem(chan)) {
435 if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start))
436 return NULL;
437 }
438
439 dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
440 src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
441
442 if (chan->hw_2d) {
443 if (!axi_dmac_check_len(chan, xt->sgl[0].size) ||
444 !axi_dmac_check_len(chan, xt->numf))
445 return NULL;
446 if (xt->sgl[0].size + dst_icg > chan->max_length ||
447 xt->sgl[0].size + src_icg > chan->max_length)
448 return NULL;
449 } else {
450 if (dst_icg != 0 || src_icg != 0)
451 return NULL;
452 if (chan->max_length / xt->sgl[0].size < xt->numf)
453 return NULL;
454 if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf))
455 return NULL;
456 }
457
458 desc = axi_dmac_alloc_desc(1);
459 if (!desc)
460 return NULL;
461
462 if (axi_dmac_src_is_mem(chan)) {
463 desc->sg[0].src_addr = xt->src_start;
464 desc->sg[0].src_stride = xt->sgl[0].size + src_icg;
465 }
466
467 if (axi_dmac_dest_is_mem(chan)) {
468 desc->sg[0].dest_addr = xt->dst_start;
469 desc->sg[0].dest_stride = xt->sgl[0].size + dst_icg;
470 }
471
472 if (chan->hw_2d) {
473 desc->sg[0].x_len = xt->sgl[0].size;
474 desc->sg[0].y_len = xt->numf;
475 } else {
476 desc->sg[0].x_len = xt->sgl[0].size * xt->numf;
477 desc->sg[0].y_len = 1;
478 }
479
480 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
481}
482
483static void axi_dmac_free_chan_resources(struct dma_chan *c)
484{
485 vchan_free_chan_resources(to_virt_chan(c));
486}
487
488static void axi_dmac_desc_free(struct virt_dma_desc *vdesc)
489{
490 kfree(container_of(vdesc, struct axi_dmac_desc, vdesc));
491}
492
493/*
494 * The configuration stored in the devicetree matches the configuration
495 * parameters of the peripheral instance and allows the driver to know which
496 * features are implemented and how it should behave.
497 */
498static int axi_dmac_parse_chan_dt(struct device_node *of_chan,
499 struct axi_dmac_chan *chan)
500{
501 u32 val;
502 int ret;
503
504 ret = of_property_read_u32(of_chan, "reg", &val);
505 if (ret)
506 return ret;
507
508 /* We only support 1 channel for now */
509 if (val != 0)
510 return -EINVAL;
511
512 ret = of_property_read_u32(of_chan, "adi,source-bus-type", &val);
513 if (ret)
514 return ret;
515 if (val > AXI_DMAC_BUS_TYPE_FIFO)
516 return -EINVAL;
517 chan->src_type = val;
518
519 ret = of_property_read_u32(of_chan, "adi,destination-bus-type", &val);
520 if (ret)
521 return ret;
522 if (val > AXI_DMAC_BUS_TYPE_FIFO)
523 return -EINVAL;
524 chan->dest_type = val;
525
526 ret = of_property_read_u32(of_chan, "adi,source-bus-width", &val);
527 if (ret)
528 return ret;
529 chan->src_width = val / 8;
530
531 ret = of_property_read_u32(of_chan, "adi,destination-bus-width", &val);
532 if (ret)
533 return ret;
534 chan->dest_width = val / 8;
535
536 ret = of_property_read_u32(of_chan, "adi,length-width", &val);
537 if (ret)
538 return ret;
539
540 if (val >= 32)
541 chan->max_length = UINT_MAX;
542 else
543 chan->max_length = (1ULL << val) - 1;
544
545 chan->align_mask = max(chan->dest_width, chan->src_width) - 1;
546
547 if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
548 chan->direction = DMA_MEM_TO_MEM;
549 else if (!axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
550 chan->direction = DMA_MEM_TO_DEV;
551 else if (axi_dmac_dest_is_mem(chan) && !axi_dmac_src_is_mem(chan))
552 chan->direction = DMA_DEV_TO_MEM;
553 else
554 chan->direction = DMA_DEV_TO_DEV;
555
556 chan->hw_cyclic = of_property_read_bool(of_chan, "adi,cyclic");
557 chan->hw_2d = of_property_read_bool(of_chan, "adi,2d");
558
559 return 0;
560}
561
562static int axi_dmac_probe(struct platform_device *pdev)
563{
564 struct device_node *of_channels, *of_chan;
565 struct dma_device *dma_dev;
566 struct axi_dmac *dmac;
567 struct resource *res;
568 int ret;
569
570 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
571 if (!dmac)
572 return -ENOMEM;
573
574 dmac->irq = platform_get_irq(pdev, 0);
575 if (dmac->irq <= 0)
576 return -EINVAL;
577
578 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
579 dmac->base = devm_ioremap_resource(&pdev->dev, res);
580 if (IS_ERR(dmac->base))
581 return PTR_ERR(dmac->base);
582
583 dmac->clk = devm_clk_get(&pdev->dev, NULL);
584 if (IS_ERR(dmac->clk))
585 return PTR_ERR(dmac->clk);
586
587 INIT_LIST_HEAD(&dmac->chan.active_descs);
588
589 of_channels = of_get_child_by_name(pdev->dev.of_node, "adi,channels");
590 if (of_channels == NULL)
591 return -ENODEV;
592
593 for_each_child_of_node(of_channels, of_chan) {
594 ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan);
595 if (ret) {
596 of_node_put(of_chan);
597 of_node_put(of_channels);
598 return -EINVAL;
599 }
600 }
601 of_node_put(of_channels);
602
603 pdev->dev.dma_parms = &dmac->dma_parms;
604 dma_set_max_seg_size(&pdev->dev, dmac->chan.max_length);
605
606 dma_dev = &dmac->dma_dev;
607 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
608 dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
609 dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources;
610 dma_dev->device_tx_status = dma_cookie_status;
611 dma_dev->device_issue_pending = axi_dmac_issue_pending;
612 dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg;
613 dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic;
614 dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved;
615 dma_dev->device_terminate_all = axi_dmac_terminate_all;
616 dma_dev->dev = &pdev->dev;
617 dma_dev->chancnt = 1;
618 dma_dev->src_addr_widths = BIT(dmac->chan.src_width);
619 dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width);
620 dma_dev->directions = BIT(dmac->chan.direction);
621 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
622 INIT_LIST_HEAD(&dma_dev->channels);
623
624 dmac->chan.vchan.desc_free = axi_dmac_desc_free;
625 vchan_init(&dmac->chan.vchan, dma_dev);
626
627 ret = clk_prepare_enable(dmac->clk);
628 if (ret < 0)
629 return ret;
630
631 axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, 0x00);
632
633 ret = dma_async_device_register(dma_dev);
634 if (ret)
635 goto err_clk_disable;
636
637 ret = of_dma_controller_register(pdev->dev.of_node,
638 of_dma_xlate_by_chan_id, dma_dev);
639 if (ret)
640 goto err_unregister_device;
641
642 ret = request_irq(dmac->irq, axi_dmac_interrupt_handler, 0,
643 dev_name(&pdev->dev), dmac);
644 if (ret)
645 goto err_unregister_of;
646
647 platform_set_drvdata(pdev, dmac);
648
649 return 0;
650
651err_unregister_of:
652 of_dma_controller_free(pdev->dev.of_node);
653err_unregister_device:
654 dma_async_device_unregister(&dmac->dma_dev);
655err_clk_disable:
656 clk_disable_unprepare(dmac->clk);
657
658 return ret;
659}
660
661static int axi_dmac_remove(struct platform_device *pdev)
662{
663 struct axi_dmac *dmac = platform_get_drvdata(pdev);
664
665 of_dma_controller_free(pdev->dev.of_node);
666 free_irq(dmac->irq, dmac);
667 tasklet_kill(&dmac->chan.vchan.task);
668 dma_async_device_unregister(&dmac->dma_dev);
669 clk_disable_unprepare(dmac->clk);
670
671 return 0;
672}
673
674static const struct of_device_id axi_dmac_of_match_table[] = {
675 { .compatible = "adi,axi-dmac-1.00.a" },
676 { },
677};
678
679static struct platform_driver axi_dmac_driver = {
680 .driver = {
681 .name = "dma-axi-dmac",
682 .of_match_table = axi_dmac_of_match_table,
683 },
684 .probe = axi_dmac_probe,
685 .remove = axi_dmac_remove,
686};
687module_platform_driver(axi_dmac_driver);
688
689MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
690MODULE_DESCRIPTION("DMA controller driver for the AXI-DMAC controller");
691MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c
index 26d2f0e09ea3..dade7c47ff18 100644
--- a/drivers/dma/dma-jz4780.c
+++ b/drivers/dma/dma-jz4780.c
@@ -145,7 +145,8 @@ struct jz4780_dma_dev {
145 struct jz4780_dma_chan chan[JZ_DMA_NR_CHANNELS]; 145 struct jz4780_dma_chan chan[JZ_DMA_NR_CHANNELS];
146}; 146};
147 147
148struct jz4780_dma_data { 148struct jz4780_dma_filter_data {
149 struct device_node *of_node;
149 uint32_t transfer_type; 150 uint32_t transfer_type;
150 int channel; 151 int channel;
151}; 152};
@@ -214,11 +215,25 @@ static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc)
214 kfree(desc); 215 kfree(desc);
215} 216}
216 217
217static uint32_t jz4780_dma_transfer_size(unsigned long val, int *ord) 218static uint32_t jz4780_dma_transfer_size(unsigned long val, uint32_t *shift)
218{ 219{
219 *ord = ffs(val) - 1; 220 int ord = ffs(val) - 1;
220 221
221 switch (*ord) { 222 /*
223 * 8 byte transfer sizes unsupported so fall back on 4. If it's larger
224 * than the maximum, just limit it. It is perfectly safe to fall back
225 * in this way since we won't exceed the maximum burst size supported
226 * by the device, the only effect is reduced efficiency. This is better
227 * than refusing to perform the request at all.
228 */
229 if (ord == 3)
230 ord = 2;
231 else if (ord > 7)
232 ord = 7;
233
234 *shift = ord;
235
236 switch (ord) {
222 case 0: 237 case 0:
223 return JZ_DMA_SIZE_1_BYTE; 238 return JZ_DMA_SIZE_1_BYTE;
224 case 1: 239 case 1:
@@ -231,20 +246,17 @@ static uint32_t jz4780_dma_transfer_size(unsigned long val, int *ord)
231 return JZ_DMA_SIZE_32_BYTE; 246 return JZ_DMA_SIZE_32_BYTE;
232 case 6: 247 case 6:
233 return JZ_DMA_SIZE_64_BYTE; 248 return JZ_DMA_SIZE_64_BYTE;
234 case 7:
235 return JZ_DMA_SIZE_128_BYTE;
236 default: 249 default:
237 return -EINVAL; 250 return JZ_DMA_SIZE_128_BYTE;
238 } 251 }
239} 252}
240 253
241static uint32_t jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, 254static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan,
242 struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len, 255 struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len,
243 enum dma_transfer_direction direction) 256 enum dma_transfer_direction direction)
244{ 257{
245 struct dma_slave_config *config = &jzchan->config; 258 struct dma_slave_config *config = &jzchan->config;
246 uint32_t width, maxburst, tsz; 259 uint32_t width, maxburst, tsz;
247 int ord;
248 260
249 if (direction == DMA_MEM_TO_DEV) { 261 if (direction == DMA_MEM_TO_DEV) {
250 desc->dcm = JZ_DMA_DCM_SAI; 262 desc->dcm = JZ_DMA_DCM_SAI;
@@ -271,8 +283,8 @@ static uint32_t jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan,
271 * divisible by the transfer size, and we must not use more than the 283 * divisible by the transfer size, and we must not use more than the
272 * maximum burst specified by the user. 284 * maximum burst specified by the user.
273 */ 285 */
274 tsz = jz4780_dma_transfer_size(addr | len | (width * maxburst), &ord); 286 tsz = jz4780_dma_transfer_size(addr | len | (width * maxburst),
275 jzchan->transfer_shift = ord; 287 &jzchan->transfer_shift);
276 288
277 switch (width) { 289 switch (width) {
278 case DMA_SLAVE_BUSWIDTH_1_BYTE: 290 case DMA_SLAVE_BUSWIDTH_1_BYTE:
@@ -289,12 +301,14 @@ static uint32_t jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan,
289 desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT; 301 desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT;
290 desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT; 302 desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT;
291 303
292 desc->dtc = len >> ord; 304 desc->dtc = len >> jzchan->transfer_shift;
305 return 0;
293} 306}
294 307
295static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( 308static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg(
296 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, 309 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
297 enum dma_transfer_direction direction, unsigned long flags) 310 enum dma_transfer_direction direction, unsigned long flags,
311 void *context)
298{ 312{
299 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 313 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
300 struct jz4780_dma_desc *desc; 314 struct jz4780_dma_desc *desc;
@@ -307,12 +321,11 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg(
307 321
308 for (i = 0; i < sg_len; i++) { 322 for (i = 0; i < sg_len; i++) {
309 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], 323 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i],
310 sg_dma_address(&sgl[i]), 324 sg_dma_address(&sgl[i]),
311 sg_dma_len(&sgl[i]), 325 sg_dma_len(&sgl[i]),
312 direction); 326 direction);
313 if (err < 0) 327 if (err < 0)
314 return ERR_PTR(err); 328 return NULL;
315
316 329
317 desc->desc[i].dcm |= JZ_DMA_DCM_TIE; 330 desc->desc[i].dcm |= JZ_DMA_DCM_TIE;
318 331
@@ -354,9 +367,9 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic(
354 367
355 for (i = 0; i < periods; i++) { 368 for (i = 0; i < periods; i++) {
356 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr, 369 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr,
357 period_len, direction); 370 period_len, direction);
358 if (err < 0) 371 if (err < 0)
359 return ERR_PTR(err); 372 return NULL;
360 373
361 buf_addr += period_len; 374 buf_addr += period_len;
362 375
@@ -390,15 +403,13 @@ struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy(
390 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 403 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
391 struct jz4780_dma_desc *desc; 404 struct jz4780_dma_desc *desc;
392 uint32_t tsz; 405 uint32_t tsz;
393 int ord;
394 406
395 desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY); 407 desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY);
396 if (!desc) 408 if (!desc)
397 return NULL; 409 return NULL;
398 410
399 tsz = jz4780_dma_transfer_size(dest | src | len, &ord); 411 tsz = jz4780_dma_transfer_size(dest | src | len,
400 if (tsz < 0) 412 &jzchan->transfer_shift);
401 return ERR_PTR(tsz);
402 413
403 desc->desc[0].dsa = src; 414 desc->desc[0].dsa = src;
404 desc->desc[0].dta = dest; 415 desc->desc[0].dta = dest;
@@ -407,7 +418,7 @@ struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy(
407 tsz << JZ_DMA_DCM_TSZ_SHIFT | 418 tsz << JZ_DMA_DCM_TSZ_SHIFT |
408 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT | 419 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT |
409 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT; 420 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT;
410 desc->desc[0].dtc = len >> ord; 421 desc->desc[0].dtc = len >> jzchan->transfer_shift;
411 422
412 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 423 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags);
413} 424}
@@ -484,8 +495,9 @@ static void jz4780_dma_issue_pending(struct dma_chan *chan)
484 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 495 spin_unlock_irqrestore(&jzchan->vchan.lock, flags);
485} 496}
486 497
487static int jz4780_dma_terminate_all(struct jz4780_dma_chan *jzchan) 498static int jz4780_dma_terminate_all(struct dma_chan *chan)
488{ 499{
500 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
489 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 501 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
490 unsigned long flags; 502 unsigned long flags;
491 LIST_HEAD(head); 503 LIST_HEAD(head);
@@ -507,9 +519,11 @@ static int jz4780_dma_terminate_all(struct jz4780_dma_chan *jzchan)
507 return 0; 519 return 0;
508} 520}
509 521
510static int jz4780_dma_slave_config(struct jz4780_dma_chan *jzchan, 522static int jz4780_dma_config(struct dma_chan *chan,
511 const struct dma_slave_config *config) 523 struct dma_slave_config *config)
512{ 524{
525 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
526
513 if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 527 if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
514 || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)) 528 || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES))
515 return -EINVAL; 529 return -EINVAL;
@@ -567,8 +581,8 @@ static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan,
567 txstate->residue = 0; 581 txstate->residue = 0;
568 582
569 if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc 583 if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc
570 && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) 584 && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT))
571 status = DMA_ERROR; 585 status = DMA_ERROR;
572 586
573 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 587 spin_unlock_irqrestore(&jzchan->vchan.lock, flags);
574 return status; 588 return status;
@@ -671,7 +685,10 @@ static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param)
671{ 685{
672 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 686 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
673 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 687 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan);
674 struct jz4780_dma_data *data = param; 688 struct jz4780_dma_filter_data *data = param;
689
690 if (jzdma->dma_device.dev->of_node != data->of_node)
691 return false;
675 692
676 if (data->channel > -1) { 693 if (data->channel > -1) {
677 if (data->channel != jzchan->id) 694 if (data->channel != jzchan->id)
@@ -690,11 +707,12 @@ static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec,
690{ 707{
691 struct jz4780_dma_dev *jzdma = ofdma->of_dma_data; 708 struct jz4780_dma_dev *jzdma = ofdma->of_dma_data;
692 dma_cap_mask_t mask = jzdma->dma_device.cap_mask; 709 dma_cap_mask_t mask = jzdma->dma_device.cap_mask;
693 struct jz4780_dma_data data; 710 struct jz4780_dma_filter_data data;
694 711
695 if (dma_spec->args_count != 2) 712 if (dma_spec->args_count != 2)
696 return NULL; 713 return NULL;
697 714
715 data.of_node = ofdma->of_node;
698 data.transfer_type = dma_spec->args[0]; 716 data.transfer_type = dma_spec->args[0];
699 data.channel = dma_spec->args[1]; 717 data.channel = dma_spec->args[1];
700 718
@@ -713,9 +731,14 @@ static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec,
713 data.channel); 731 data.channel);
714 return NULL; 732 return NULL;
715 } 733 }
716 }
717 734
718 return dma_request_channel(mask, jz4780_dma_filter_fn, &data); 735 jzdma->chan[data.channel].transfer_type = data.transfer_type;
736
737 return dma_get_slave_channel(
738 &jzdma->chan[data.channel].vchan.chan);
739 } else {
740 return dma_request_channel(mask, jz4780_dma_filter_fn, &data);
741 }
719} 742}
720 743
721static int jz4780_dma_probe(struct platform_device *pdev) 744static int jz4780_dma_probe(struct platform_device *pdev)
@@ -743,23 +766,26 @@ static int jz4780_dma_probe(struct platform_device *pdev)
743 if (IS_ERR(jzdma->base)) 766 if (IS_ERR(jzdma->base))
744 return PTR_ERR(jzdma->base); 767 return PTR_ERR(jzdma->base);
745 768
746 jzdma->irq = platform_get_irq(pdev, 0); 769 ret = platform_get_irq(pdev, 0);
747 if (jzdma->irq < 0) { 770 if (ret < 0) {
748 dev_err(dev, "failed to get IRQ: %d\n", ret); 771 dev_err(dev, "failed to get IRQ: %d\n", ret);
749 return jzdma->irq; 772 return ret;
750 } 773 }
751 774
752 ret = devm_request_irq(dev, jzdma->irq, jz4780_dma_irq_handler, 0, 775 jzdma->irq = ret;
753 dev_name(dev), jzdma); 776
777 ret = request_irq(jzdma->irq, jz4780_dma_irq_handler, 0, dev_name(dev),
778 jzdma);
754 if (ret) { 779 if (ret) {
755 dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq); 780 dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq);
756 return -EINVAL; 781 return ret;
757 } 782 }
758 783
759 jzdma->clk = devm_clk_get(dev, NULL); 784 jzdma->clk = devm_clk_get(dev, NULL);
760 if (IS_ERR(jzdma->clk)) { 785 if (IS_ERR(jzdma->clk)) {
761 dev_err(dev, "failed to get clock\n"); 786 dev_err(dev, "failed to get clock\n");
762 return PTR_ERR(jzdma->clk); 787 ret = PTR_ERR(jzdma->clk);
788 goto err_free_irq;
763 } 789 }
764 790
765 clk_prepare_enable(jzdma->clk); 791 clk_prepare_enable(jzdma->clk);
@@ -775,13 +801,13 @@ static int jz4780_dma_probe(struct platform_device *pdev)
775 dma_cap_set(DMA_CYCLIC, dd->cap_mask); 801 dma_cap_set(DMA_CYCLIC, dd->cap_mask);
776 802
777 dd->dev = dev; 803 dd->dev = dev;
778 dd->copy_align = 2; /* 2^2 = 4 byte alignment */ 804 dd->copy_align = DMAENGINE_ALIGN_4_BYTES;
779 dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources; 805 dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources;
780 dd->device_free_chan_resources = jz4780_dma_free_chan_resources; 806 dd->device_free_chan_resources = jz4780_dma_free_chan_resources;
781 dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg; 807 dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg;
782 dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic; 808 dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic;
783 dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy; 809 dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy;
784 dd->device_config = jz4780_dma_slave_config; 810 dd->device_config = jz4780_dma_config;
785 dd->device_terminate_all = jz4780_dma_terminate_all; 811 dd->device_terminate_all = jz4780_dma_terminate_all;
786 dd->device_tx_status = jz4780_dma_tx_status; 812 dd->device_tx_status = jz4780_dma_tx_status;
787 dd->device_issue_pending = jz4780_dma_issue_pending; 813 dd->device_issue_pending = jz4780_dma_issue_pending;
@@ -790,7 +816,6 @@ static int jz4780_dma_probe(struct platform_device *pdev)
790 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 816 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
791 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 817 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
792 818
793
794 /* 819 /*
795 * Enable DMA controller, mark all channels as not programmable. 820 * Enable DMA controller, mark all channels as not programmable.
796 * Also set the FMSC bit - it increases MSC performance, so it makes 821 * Also set the FMSC bit - it increases MSC performance, so it makes
@@ -832,15 +857,24 @@ err_unregister_dev:
832 857
833err_disable_clk: 858err_disable_clk:
834 clk_disable_unprepare(jzdma->clk); 859 clk_disable_unprepare(jzdma->clk);
860
861err_free_irq:
862 free_irq(jzdma->irq, jzdma);
835 return ret; 863 return ret;
836} 864}
837 865
838static int jz4780_dma_remove(struct platform_device *pdev) 866static int jz4780_dma_remove(struct platform_device *pdev)
839{ 867{
840 struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); 868 struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev);
869 int i;
841 870
842 of_dma_controller_free(pdev->dev.of_node); 871 of_dma_controller_free(pdev->dev.of_node);
843 devm_free_irq(&pdev->dev, jzdma->irq, jzdma); 872
873 free_irq(jzdma->irq, jzdma);
874
875 for (i = 0; i < JZ_DMA_NR_CHANNELS; i++)
876 tasklet_kill(&jzdma->chan[i].vchan.task);
877
844 dma_async_device_unregister(&jzdma->dma_device); 878 dma_async_device_unregister(&jzdma->dma_device);
845 return 0; 879 return 0;
846} 880}
diff --git a/drivers/dma/dw/Kconfig b/drivers/dma/dw/Kconfig
index 36e02f0f645e..e00c9b022964 100644
--- a/drivers/dma/dw/Kconfig
+++ b/drivers/dma/dw/Kconfig
@@ -6,6 +6,9 @@ config DW_DMAC_CORE
6 tristate 6 tristate
7 select DMA_ENGINE 7 select DMA_ENGINE
8 8
9config DW_DMAC_BIG_ENDIAN_IO
10 bool
11
9config DW_DMAC 12config DW_DMAC
10 tristate "Synopsys DesignWare AHB DMA platform driver" 13 tristate "Synopsys DesignWare AHB DMA platform driver"
11 select DW_DMAC_CORE 14 select DW_DMAC_CORE
@@ -23,6 +26,3 @@ config DW_DMAC_PCI
23 Support the Synopsys DesignWare AHB DMA controller on the 26 Support the Synopsys DesignWare AHB DMA controller on the
24 platfroms that enumerate it as a PCI device. For example, 27 platfroms that enumerate it as a PCI device. For example,
25 Intel Medfield has integrated this GPDMA controller. 28 Intel Medfield has integrated this GPDMA controller.
26
27config DW_DMAC_BIG_ENDIAN_IO
28 bool
diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c
index 88853af69489..3e5d4f193005 100644
--- a/drivers/dma/edma.c
+++ b/drivers/dma/edma.c
@@ -1000,7 +1000,7 @@ static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
1000 * code using dma memcpy must make sure alignment of 1000 * code using dma memcpy must make sure alignment of
1001 * length is at dma->copy_align boundary. 1001 * length is at dma->copy_align boundary.
1002 */ 1002 */
1003 dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES; 1003 dma->copy_align = DMAENGINE_ALIGN_4_BYTES;
1004 1004
1005 INIT_LIST_HEAD(&dma->channels); 1005 INIT_LIST_HEAD(&dma->channels);
1006} 1006}
diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c
index f42f71e37e73..7669c7dd1e34 100644
--- a/drivers/dma/hsu/hsu.c
+++ b/drivers/dma/hsu/hsu.c
@@ -99,21 +99,13 @@ static void hsu_dma_chan_start(struct hsu_dma_chan *hsuc)
99 99
100static void hsu_dma_stop_channel(struct hsu_dma_chan *hsuc) 100static void hsu_dma_stop_channel(struct hsu_dma_chan *hsuc)
101{ 101{
102 unsigned long flags;
103
104 spin_lock_irqsave(&hsuc->lock, flags);
105 hsu_chan_disable(hsuc); 102 hsu_chan_disable(hsuc);
106 hsu_chan_writel(hsuc, HSU_CH_DCR, 0); 103 hsu_chan_writel(hsuc, HSU_CH_DCR, 0);
107 spin_unlock_irqrestore(&hsuc->lock, flags);
108} 104}
109 105
110static void hsu_dma_start_channel(struct hsu_dma_chan *hsuc) 106static void hsu_dma_start_channel(struct hsu_dma_chan *hsuc)
111{ 107{
112 unsigned long flags;
113
114 spin_lock_irqsave(&hsuc->lock, flags);
115 hsu_dma_chan_start(hsuc); 108 hsu_dma_chan_start(hsuc);
116 spin_unlock_irqrestore(&hsuc->lock, flags);
117} 109}
118 110
119static void hsu_dma_start_transfer(struct hsu_dma_chan *hsuc) 111static void hsu_dma_start_transfer(struct hsu_dma_chan *hsuc)
@@ -139,9 +131,9 @@ static u32 hsu_dma_chan_get_sr(struct hsu_dma_chan *hsuc)
139 unsigned long flags; 131 unsigned long flags;
140 u32 sr; 132 u32 sr;
141 133
142 spin_lock_irqsave(&hsuc->lock, flags); 134 spin_lock_irqsave(&hsuc->vchan.lock, flags);
143 sr = hsu_chan_readl(hsuc, HSU_CH_SR); 135 sr = hsu_chan_readl(hsuc, HSU_CH_SR);
144 spin_unlock_irqrestore(&hsuc->lock, flags); 136 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
145 137
146 return sr; 138 return sr;
147} 139}
@@ -273,14 +265,11 @@ static size_t hsu_dma_active_desc_size(struct hsu_dma_chan *hsuc)
273 struct hsu_dma_desc *desc = hsuc->desc; 265 struct hsu_dma_desc *desc = hsuc->desc;
274 size_t bytes = hsu_dma_desc_size(desc); 266 size_t bytes = hsu_dma_desc_size(desc);
275 int i; 267 int i;
276 unsigned long flags;
277 268
278 spin_lock_irqsave(&hsuc->lock, flags);
279 i = desc->active % HSU_DMA_CHAN_NR_DESC; 269 i = desc->active % HSU_DMA_CHAN_NR_DESC;
280 do { 270 do {
281 bytes += hsu_chan_readl(hsuc, HSU_CH_DxTSR(i)); 271 bytes += hsu_chan_readl(hsuc, HSU_CH_DxTSR(i));
282 } while (--i >= 0); 272 } while (--i >= 0);
283 spin_unlock_irqrestore(&hsuc->lock, flags);
284 273
285 return bytes; 274 return bytes;
286} 275}
@@ -327,24 +316,6 @@ static int hsu_dma_slave_config(struct dma_chan *chan,
327 return 0; 316 return 0;
328} 317}
329 318
330static void hsu_dma_chan_deactivate(struct hsu_dma_chan *hsuc)
331{
332 unsigned long flags;
333
334 spin_lock_irqsave(&hsuc->lock, flags);
335 hsu_chan_disable(hsuc);
336 spin_unlock_irqrestore(&hsuc->lock, flags);
337}
338
339static void hsu_dma_chan_activate(struct hsu_dma_chan *hsuc)
340{
341 unsigned long flags;
342
343 spin_lock_irqsave(&hsuc->lock, flags);
344 hsu_chan_enable(hsuc);
345 spin_unlock_irqrestore(&hsuc->lock, flags);
346}
347
348static int hsu_dma_pause(struct dma_chan *chan) 319static int hsu_dma_pause(struct dma_chan *chan)
349{ 320{
350 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan); 321 struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan);
@@ -352,7 +323,7 @@ static int hsu_dma_pause(struct dma_chan *chan)
352 323
353 spin_lock_irqsave(&hsuc->vchan.lock, flags); 324 spin_lock_irqsave(&hsuc->vchan.lock, flags);
354 if (hsuc->desc && hsuc->desc->status == DMA_IN_PROGRESS) { 325 if (hsuc->desc && hsuc->desc->status == DMA_IN_PROGRESS) {
355 hsu_dma_chan_deactivate(hsuc); 326 hsu_chan_disable(hsuc);
356 hsuc->desc->status = DMA_PAUSED; 327 hsuc->desc->status = DMA_PAUSED;
357 } 328 }
358 spin_unlock_irqrestore(&hsuc->vchan.lock, flags); 329 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
@@ -368,7 +339,7 @@ static int hsu_dma_resume(struct dma_chan *chan)
368 spin_lock_irqsave(&hsuc->vchan.lock, flags); 339 spin_lock_irqsave(&hsuc->vchan.lock, flags);
369 if (hsuc->desc && hsuc->desc->status == DMA_PAUSED) { 340 if (hsuc->desc && hsuc->desc->status == DMA_PAUSED) {
370 hsuc->desc->status = DMA_IN_PROGRESS; 341 hsuc->desc->status = DMA_IN_PROGRESS;
371 hsu_dma_chan_activate(hsuc); 342 hsu_chan_enable(hsuc);
372 } 343 }
373 spin_unlock_irqrestore(&hsuc->vchan.lock, flags); 344 spin_unlock_irqrestore(&hsuc->vchan.lock, flags);
374 345
@@ -441,8 +412,6 @@ int hsu_dma_probe(struct hsu_dma_chip *chip)
441 412
442 hsuc->direction = (i & 0x1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; 413 hsuc->direction = (i & 0x1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;
443 hsuc->reg = addr + i * HSU_DMA_CHAN_LENGTH; 414 hsuc->reg = addr + i * HSU_DMA_CHAN_LENGTH;
444
445 spin_lock_init(&hsuc->lock);
446 } 415 }
447 416
448 dma_cap_set(DMA_SLAVE, hsu->dma.cap_mask); 417 dma_cap_set(DMA_SLAVE, hsu->dma.cap_mask);
diff --git a/drivers/dma/hsu/hsu.h b/drivers/dma/hsu/hsu.h
index 0275233cf550..eeb9fff66967 100644
--- a/drivers/dma/hsu/hsu.h
+++ b/drivers/dma/hsu/hsu.h
@@ -78,7 +78,6 @@ struct hsu_dma_chan {
78 struct virt_dma_chan vchan; 78 struct virt_dma_chan vchan;
79 79
80 void __iomem *reg; 80 void __iomem *reg;
81 spinlock_t lock;
82 81
83 /* hardware configuration */ 82 /* hardware configuration */
84 enum dma_transfer_direction direction; 83 enum dma_transfer_direction direction;
diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c
index 865501fcc67d..48d85f8b95fe 100644
--- a/drivers/dma/imx-dma.c
+++ b/drivers/dma/imx-dma.c
@@ -1083,8 +1083,12 @@ static int __init imxdma_probe(struct platform_device *pdev)
1083 if (IS_ERR(imxdma->dma_ahb)) 1083 if (IS_ERR(imxdma->dma_ahb))
1084 return PTR_ERR(imxdma->dma_ahb); 1084 return PTR_ERR(imxdma->dma_ahb);
1085 1085
1086 clk_prepare_enable(imxdma->dma_ipg); 1086 ret = clk_prepare_enable(imxdma->dma_ipg);
1087 clk_prepare_enable(imxdma->dma_ahb); 1087 if (ret)
1088 return ret;
1089 ret = clk_prepare_enable(imxdma->dma_ahb);
1090 if (ret)
1091 goto disable_dma_ipg_clk;
1088 1092
1089 /* reset DMA module */ 1093 /* reset DMA module */
1090 imx_dmav1_writel(imxdma, DCR_DRST, DMA_DCR); 1094 imx_dmav1_writel(imxdma, DCR_DRST, DMA_DCR);
@@ -1094,20 +1098,20 @@ static int __init imxdma_probe(struct platform_device *pdev)
1094 dma_irq_handler, 0, "DMA", imxdma); 1098 dma_irq_handler, 0, "DMA", imxdma);
1095 if (ret) { 1099 if (ret) {
1096 dev_warn(imxdma->dev, "Can't register IRQ for DMA\n"); 1100 dev_warn(imxdma->dev, "Can't register IRQ for DMA\n");
1097 goto err; 1101 goto disable_dma_ahb_clk;
1098 } 1102 }
1099 1103
1100 irq_err = platform_get_irq(pdev, 1); 1104 irq_err = platform_get_irq(pdev, 1);
1101 if (irq_err < 0) { 1105 if (irq_err < 0) {
1102 ret = irq_err; 1106 ret = irq_err;
1103 goto err; 1107 goto disable_dma_ahb_clk;
1104 } 1108 }
1105 1109
1106 ret = devm_request_irq(&pdev->dev, irq_err, 1110 ret = devm_request_irq(&pdev->dev, irq_err,
1107 imxdma_err_handler, 0, "DMA", imxdma); 1111 imxdma_err_handler, 0, "DMA", imxdma);
1108 if (ret) { 1112 if (ret) {
1109 dev_warn(imxdma->dev, "Can't register ERRIRQ for DMA\n"); 1113 dev_warn(imxdma->dev, "Can't register ERRIRQ for DMA\n");
1110 goto err; 1114 goto disable_dma_ahb_clk;
1111 } 1115 }
1112 } 1116 }
1113 1117
@@ -1144,7 +1148,7 @@ static int __init imxdma_probe(struct platform_device *pdev)
1144 dev_warn(imxdma->dev, "Can't register IRQ %d " 1148 dev_warn(imxdma->dev, "Can't register IRQ %d "
1145 "for DMA channel %d\n", 1149 "for DMA channel %d\n",
1146 irq + i, i); 1150 irq + i, i);
1147 goto err; 1151 goto disable_dma_ahb_clk;
1148 } 1152 }
1149 init_timer(&imxdmac->watchdog); 1153 init_timer(&imxdmac->watchdog);
1150 imxdmac->watchdog.function = &imxdma_watchdog; 1154 imxdmac->watchdog.function = &imxdma_watchdog;
@@ -1183,14 +1187,14 @@ static int __init imxdma_probe(struct platform_device *pdev)
1183 1187
1184 platform_set_drvdata(pdev, imxdma); 1188 platform_set_drvdata(pdev, imxdma);
1185 1189
1186 imxdma->dma_device.copy_align = 2; /* 2^2 = 4 bytes alignment */ 1190 imxdma->dma_device.copy_align = DMAENGINE_ALIGN_4_BYTES;
1187 imxdma->dma_device.dev->dma_parms = &imxdma->dma_parms; 1191 imxdma->dma_device.dev->dma_parms = &imxdma->dma_parms;
1188 dma_set_max_seg_size(imxdma->dma_device.dev, 0xffffff); 1192 dma_set_max_seg_size(imxdma->dma_device.dev, 0xffffff);
1189 1193
1190 ret = dma_async_device_register(&imxdma->dma_device); 1194 ret = dma_async_device_register(&imxdma->dma_device);
1191 if (ret) { 1195 if (ret) {
1192 dev_err(&pdev->dev, "unable to register\n"); 1196 dev_err(&pdev->dev, "unable to register\n");
1193 goto err; 1197 goto disable_dma_ahb_clk;
1194 } 1198 }
1195 1199
1196 if (pdev->dev.of_node) { 1200 if (pdev->dev.of_node) {
@@ -1206,9 +1210,10 @@ static int __init imxdma_probe(struct platform_device *pdev)
1206 1210
1207err_of_dma_controller: 1211err_of_dma_controller:
1208 dma_async_device_unregister(&imxdma->dma_device); 1212 dma_async_device_unregister(&imxdma->dma_device);
1209err: 1213disable_dma_ahb_clk:
1210 clk_disable_unprepare(imxdma->dma_ipg);
1211 clk_disable_unprepare(imxdma->dma_ahb); 1214 clk_disable_unprepare(imxdma->dma_ahb);
1215disable_dma_ipg_clk:
1216 clk_disable_unprepare(imxdma->dma_ipg);
1212 return ret; 1217 return ret;
1213} 1218}
1214 1219
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
index 77b6aab04f47..9d375bc7590a 100644
--- a/drivers/dma/imx-sdma.c
+++ b/drivers/dma/imx-sdma.c
@@ -35,12 +35,16 @@
35#include <linux/platform_device.h> 35#include <linux/platform_device.h>
36#include <linux/dmaengine.h> 36#include <linux/dmaengine.h>
37#include <linux/of.h> 37#include <linux/of.h>
38#include <linux/of_address.h>
38#include <linux/of_device.h> 39#include <linux/of_device.h>
39#include <linux/of_dma.h> 40#include <linux/of_dma.h>
40 41
41#include <asm/irq.h> 42#include <asm/irq.h>
42#include <linux/platform_data/dma-imx-sdma.h> 43#include <linux/platform_data/dma-imx-sdma.h>
43#include <linux/platform_data/dma-imx.h> 44#include <linux/platform_data/dma-imx.h>
45#include <linux/regmap.h>
46#include <linux/mfd/syscon.h>
47#include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
44 48
45#include "dmaengine.h" 49#include "dmaengine.h"
46 50
@@ -124,6 +128,56 @@
124#define CHANGE_ENDIANNESS 0x80 128#define CHANGE_ENDIANNESS 0x80
125 129
126/* 130/*
131 * p_2_p watermark_level description
132 * Bits Name Description
133 * 0-7 Lower WML Lower watermark level
134 * 8 PS 1: Pad Swallowing
135 * 0: No Pad Swallowing
136 * 9 PA 1: Pad Adding
137 * 0: No Pad Adding
138 * 10 SPDIF If this bit is set both source
139 * and destination are on SPBA
140 * 11 Source Bit(SP) 1: Source on SPBA
141 * 0: Source on AIPS
142 * 12 Destination Bit(DP) 1: Destination on SPBA
143 * 0: Destination on AIPS
144 * 13-15 --------- MUST BE 0
145 * 16-23 Higher WML HWML
146 * 24-27 N Total number of samples after
147 * which Pad adding/Swallowing
148 * must be done. It must be odd.
149 * 28 Lower WML Event(LWE) SDMA events reg to check for
150 * LWML event mask
151 * 0: LWE in EVENTS register
152 * 1: LWE in EVENTS2 register
153 * 29 Higher WML Event(HWE) SDMA events reg to check for
154 * HWML event mask
155 * 0: HWE in EVENTS register
156 * 1: HWE in EVENTS2 register
157 * 30 --------- MUST BE 0
158 * 31 CONT 1: Amount of samples to be
159 * transferred is unknown and
160 * script will keep on
161 * transferring samples as long as
162 * both events are detected and
163 * script must be manually stopped
164 * by the application
165 * 0: The amount of samples to be
166 * transferred is equal to the
167 * count field of mode word
168 */
169#define SDMA_WATERMARK_LEVEL_LWML 0xFF
170#define SDMA_WATERMARK_LEVEL_PS BIT(8)
171#define SDMA_WATERMARK_LEVEL_PA BIT(9)
172#define SDMA_WATERMARK_LEVEL_SPDIF BIT(10)
173#define SDMA_WATERMARK_LEVEL_SP BIT(11)
174#define SDMA_WATERMARK_LEVEL_DP BIT(12)
175#define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16)
176#define SDMA_WATERMARK_LEVEL_LWE BIT(28)
177#define SDMA_WATERMARK_LEVEL_HWE BIT(29)
178#define SDMA_WATERMARK_LEVEL_CONT BIT(31)
179
180/*
127 * Mode/Count of data node descriptors - IPCv2 181 * Mode/Count of data node descriptors - IPCv2
128 */ 182 */
129struct sdma_mode_count { 183struct sdma_mode_count {
@@ -259,8 +313,9 @@ struct sdma_channel {
259 struct sdma_buffer_descriptor *bd; 313 struct sdma_buffer_descriptor *bd;
260 dma_addr_t bd_phys; 314 dma_addr_t bd_phys;
261 unsigned int pc_from_device, pc_to_device; 315 unsigned int pc_from_device, pc_to_device;
316 unsigned int device_to_device;
262 unsigned long flags; 317 unsigned long flags;
263 dma_addr_t per_address; 318 dma_addr_t per_address, per_address2;
264 unsigned long event_mask[2]; 319 unsigned long event_mask[2];
265 unsigned long watermark_level; 320 unsigned long watermark_level;
266 u32 shp_addr, per_addr; 321 u32 shp_addr, per_addr;
@@ -328,6 +383,8 @@ struct sdma_engine {
328 u32 script_number; 383 u32 script_number;
329 struct sdma_script_start_addrs *script_addrs; 384 struct sdma_script_start_addrs *script_addrs;
330 const struct sdma_driver_data *drvdata; 385 const struct sdma_driver_data *drvdata;
386 u32 spba_start_addr;
387 u32 spba_end_addr;
331}; 388};
332 389
333static struct sdma_driver_data sdma_imx31 = { 390static struct sdma_driver_data sdma_imx31 = {
@@ -705,6 +762,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac,
705 762
706 sdmac->pc_from_device = 0; 763 sdmac->pc_from_device = 0;
707 sdmac->pc_to_device = 0; 764 sdmac->pc_to_device = 0;
765 sdmac->device_to_device = 0;
708 766
709 switch (peripheral_type) { 767 switch (peripheral_type) {
710 case IMX_DMATYPE_MEMORY: 768 case IMX_DMATYPE_MEMORY:
@@ -780,6 +838,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac,
780 838
781 sdmac->pc_from_device = per_2_emi; 839 sdmac->pc_from_device = per_2_emi;
782 sdmac->pc_to_device = emi_2_per; 840 sdmac->pc_to_device = emi_2_per;
841 sdmac->device_to_device = per_2_per;
783} 842}
784 843
785static int sdma_load_context(struct sdma_channel *sdmac) 844static int sdma_load_context(struct sdma_channel *sdmac)
@@ -792,11 +851,12 @@ static int sdma_load_context(struct sdma_channel *sdmac)
792 int ret; 851 int ret;
793 unsigned long flags; 852 unsigned long flags;
794 853
795 if (sdmac->direction == DMA_DEV_TO_MEM) { 854 if (sdmac->direction == DMA_DEV_TO_MEM)
796 load_address = sdmac->pc_from_device; 855 load_address = sdmac->pc_from_device;
797 } else { 856 else if (sdmac->direction == DMA_DEV_TO_DEV)
857 load_address = sdmac->device_to_device;
858 else
798 load_address = sdmac->pc_to_device; 859 load_address = sdmac->pc_to_device;
799 }
800 860
801 if (load_address < 0) 861 if (load_address < 0)
802 return load_address; 862 return load_address;
@@ -851,6 +911,46 @@ static int sdma_disable_channel(struct dma_chan *chan)
851 return 0; 911 return 0;
852} 912}
853 913
914static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)
915{
916 struct sdma_engine *sdma = sdmac->sdma;
917
918 int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;
919 int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;
920
921 set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]);
922 set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]);
923
924 if (sdmac->event_id0 > 31)
925 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;
926
927 if (sdmac->event_id1 > 31)
928 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;
929
930 /*
931 * If LWML(src_maxburst) > HWML(dst_maxburst), we need
932 * swap LWML and HWML of INFO(A.3.2.5.1), also need swap
933 * r0(event_mask[1]) and r1(event_mask[0]).
934 */
935 if (lwml > hwml) {
936 sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |
937 SDMA_WATERMARK_LEVEL_HWML);
938 sdmac->watermark_level |= hwml;
939 sdmac->watermark_level |= lwml << 16;
940 swap(sdmac->event_mask[0], sdmac->event_mask[1]);
941 }
942
943 if (sdmac->per_address2 >= sdma->spba_start_addr &&
944 sdmac->per_address2 <= sdma->spba_end_addr)
945 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;
946
947 if (sdmac->per_address >= sdma->spba_start_addr &&
948 sdmac->per_address <= sdma->spba_end_addr)
949 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;
950
951 sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;
952}
953
854static int sdma_config_channel(struct dma_chan *chan) 954static int sdma_config_channel(struct dma_chan *chan)
855{ 955{
856 struct sdma_channel *sdmac = to_sdma_chan(chan); 956 struct sdma_channel *sdmac = to_sdma_chan(chan);
@@ -869,6 +969,12 @@ static int sdma_config_channel(struct dma_chan *chan)
869 sdma_event_enable(sdmac, sdmac->event_id0); 969 sdma_event_enable(sdmac, sdmac->event_id0);
870 } 970 }
871 971
972 if (sdmac->event_id1) {
973 if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)
974 return -EINVAL;
975 sdma_event_enable(sdmac, sdmac->event_id1);
976 }
977
872 switch (sdmac->peripheral_type) { 978 switch (sdmac->peripheral_type) {
873 case IMX_DMATYPE_DSP: 979 case IMX_DMATYPE_DSP:
874 sdma_config_ownership(sdmac, false, true, true); 980 sdma_config_ownership(sdmac, false, true, true);
@@ -887,19 +993,17 @@ static int sdma_config_channel(struct dma_chan *chan)
887 (sdmac->peripheral_type != IMX_DMATYPE_DSP)) { 993 (sdmac->peripheral_type != IMX_DMATYPE_DSP)) {
888 /* Handle multiple event channels differently */ 994 /* Handle multiple event channels differently */
889 if (sdmac->event_id1) { 995 if (sdmac->event_id1) {
890 sdmac->event_mask[1] = BIT(sdmac->event_id1 % 32); 996 if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||
891 if (sdmac->event_id1 > 31) 997 sdmac->peripheral_type == IMX_DMATYPE_ASRC)
892 __set_bit(31, &sdmac->watermark_level); 998 sdma_set_watermarklevel_for_p2p(sdmac);
893 sdmac->event_mask[0] = BIT(sdmac->event_id0 % 32); 999 } else
894 if (sdmac->event_id0 > 31)
895 __set_bit(30, &sdmac->watermark_level);
896 } else {
897 __set_bit(sdmac->event_id0, sdmac->event_mask); 1000 __set_bit(sdmac->event_id0, sdmac->event_mask);
898 } 1001
899 /* Watermark Level */ 1002 /* Watermark Level */
900 sdmac->watermark_level |= sdmac->watermark_level; 1003 sdmac->watermark_level |= sdmac->watermark_level;
901 /* Address */ 1004 /* Address */
902 sdmac->shp_addr = sdmac->per_address; 1005 sdmac->shp_addr = sdmac->per_address;
1006 sdmac->per_addr = sdmac->per_address2;
903 } else { 1007 } else {
904 sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */ 1008 sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */
905 } 1009 }
@@ -987,17 +1091,22 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan)
987 1091
988 sdmac->peripheral_type = data->peripheral_type; 1092 sdmac->peripheral_type = data->peripheral_type;
989 sdmac->event_id0 = data->dma_request; 1093 sdmac->event_id0 = data->dma_request;
1094 sdmac->event_id1 = data->dma_request2;
990 1095
991 clk_enable(sdmac->sdma->clk_ipg); 1096 ret = clk_enable(sdmac->sdma->clk_ipg);
992 clk_enable(sdmac->sdma->clk_ahb); 1097 if (ret)
1098 return ret;
1099 ret = clk_enable(sdmac->sdma->clk_ahb);
1100 if (ret)
1101 goto disable_clk_ipg;
993 1102
994 ret = sdma_request_channel(sdmac); 1103 ret = sdma_request_channel(sdmac);
995 if (ret) 1104 if (ret)
996 return ret; 1105 goto disable_clk_ahb;
997 1106
998 ret = sdma_set_channel_priority(sdmac, prio); 1107 ret = sdma_set_channel_priority(sdmac, prio);
999 if (ret) 1108 if (ret)
1000 return ret; 1109 goto disable_clk_ahb;
1001 1110
1002 dma_async_tx_descriptor_init(&sdmac->desc, chan); 1111 dma_async_tx_descriptor_init(&sdmac->desc, chan);
1003 sdmac->desc.tx_submit = sdma_tx_submit; 1112 sdmac->desc.tx_submit = sdma_tx_submit;
@@ -1005,6 +1114,12 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan)
1005 sdmac->desc.flags = DMA_CTRL_ACK; 1114 sdmac->desc.flags = DMA_CTRL_ACK;
1006 1115
1007 return 0; 1116 return 0;
1117
1118disable_clk_ahb:
1119 clk_disable(sdmac->sdma->clk_ahb);
1120disable_clk_ipg:
1121 clk_disable(sdmac->sdma->clk_ipg);
1122 return ret;
1008} 1123}
1009 1124
1010static void sdma_free_chan_resources(struct dma_chan *chan) 1125static void sdma_free_chan_resources(struct dma_chan *chan)
@@ -1221,6 +1336,14 @@ static int sdma_config(struct dma_chan *chan,
1221 sdmac->watermark_level = dmaengine_cfg->src_maxburst * 1336 sdmac->watermark_level = dmaengine_cfg->src_maxburst *
1222 dmaengine_cfg->src_addr_width; 1337 dmaengine_cfg->src_addr_width;
1223 sdmac->word_size = dmaengine_cfg->src_addr_width; 1338 sdmac->word_size = dmaengine_cfg->src_addr_width;
1339 } else if (dmaengine_cfg->direction == DMA_DEV_TO_DEV) {
1340 sdmac->per_address2 = dmaengine_cfg->src_addr;
1341 sdmac->per_address = dmaengine_cfg->dst_addr;
1342 sdmac->watermark_level = dmaengine_cfg->src_maxburst &
1343 SDMA_WATERMARK_LEVEL_LWML;
1344 sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &
1345 SDMA_WATERMARK_LEVEL_HWML;
1346 sdmac->word_size = dmaengine_cfg->dst_addr_width;
1224 } else { 1347 } else {
1225 sdmac->per_address = dmaengine_cfg->dst_addr; 1348 sdmac->per_address = dmaengine_cfg->dst_addr;
1226 sdmac->watermark_level = dmaengine_cfg->dst_maxburst * 1349 sdmac->watermark_level = dmaengine_cfg->dst_maxburst *
@@ -1337,6 +1460,72 @@ err_firmware:
1337 release_firmware(fw); 1460 release_firmware(fw);
1338} 1461}
1339 1462
1463#define EVENT_REMAP_CELLS 3
1464
1465static int __init sdma_event_remap(struct sdma_engine *sdma)
1466{
1467 struct device_node *np = sdma->dev->of_node;
1468 struct device_node *gpr_np = of_parse_phandle(np, "gpr", 0);
1469 struct property *event_remap;
1470 struct regmap *gpr;
1471 char propname[] = "fsl,sdma-event-remap";
1472 u32 reg, val, shift, num_map, i;
1473 int ret = 0;
1474
1475 if (IS_ERR(np) || IS_ERR(gpr_np))
1476 goto out;
1477
1478 event_remap = of_find_property(np, propname, NULL);
1479 num_map = event_remap ? (event_remap->length / sizeof(u32)) : 0;
1480 if (!num_map) {
1481 dev_warn(sdma->dev, "no event needs to be remapped\n");
1482 goto out;
1483 } else if (num_map % EVENT_REMAP_CELLS) {
1484 dev_err(sdma->dev, "the property %s must modulo %d\n",
1485 propname, EVENT_REMAP_CELLS);
1486 ret = -EINVAL;
1487 goto out;
1488 }
1489
1490 gpr = syscon_node_to_regmap(gpr_np);
1491 if (IS_ERR(gpr)) {
1492 dev_err(sdma->dev, "failed to get gpr regmap\n");
1493 ret = PTR_ERR(gpr);
1494 goto out;
1495 }
1496
1497 for (i = 0; i < num_map; i += EVENT_REMAP_CELLS) {
1498 ret = of_property_read_u32_index(np, propname, i, &reg);
1499 if (ret) {
1500 dev_err(sdma->dev, "failed to read property %s index %d\n",
1501 propname, i);
1502 goto out;
1503 }
1504
1505 ret = of_property_read_u32_index(np, propname, i + 1, &shift);
1506 if (ret) {
1507 dev_err(sdma->dev, "failed to read property %s index %d\n",
1508 propname, i + 1);
1509 goto out;
1510 }
1511
1512 ret = of_property_read_u32_index(np, propname, i + 2, &val);
1513 if (ret) {
1514 dev_err(sdma->dev, "failed to read property %s index %d\n",
1515 propname, i + 2);
1516 goto out;
1517 }
1518
1519 regmap_update_bits(gpr, reg, BIT(shift), val << shift);
1520 }
1521
1522out:
1523 if (!IS_ERR(gpr_np))
1524 of_node_put(gpr_np);
1525
1526 return ret;
1527}
1528
1340static int sdma_get_firmware(struct sdma_engine *sdma, 1529static int sdma_get_firmware(struct sdma_engine *sdma,
1341 const char *fw_name) 1530 const char *fw_name)
1342{ 1531{
@@ -1354,8 +1543,12 @@ static int sdma_init(struct sdma_engine *sdma)
1354 int i, ret; 1543 int i, ret;
1355 dma_addr_t ccb_phys; 1544 dma_addr_t ccb_phys;
1356 1545
1357 clk_enable(sdma->clk_ipg); 1546 ret = clk_enable(sdma->clk_ipg);
1358 clk_enable(sdma->clk_ahb); 1547 if (ret)
1548 return ret;
1549 ret = clk_enable(sdma->clk_ahb);
1550 if (ret)
1551 goto disable_clk_ipg;
1359 1552
1360 /* Be sure SDMA has not started yet */ 1553 /* Be sure SDMA has not started yet */
1361 writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); 1554 writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);
@@ -1411,8 +1604,9 @@ static int sdma_init(struct sdma_engine *sdma)
1411 return 0; 1604 return 0;
1412 1605
1413err_dma_alloc: 1606err_dma_alloc:
1414 clk_disable(sdma->clk_ipg);
1415 clk_disable(sdma->clk_ahb); 1607 clk_disable(sdma->clk_ahb);
1608disable_clk_ipg:
1609 clk_disable(sdma->clk_ipg);
1416 dev_err(sdma->dev, "initialisation failed with %d\n", ret); 1610 dev_err(sdma->dev, "initialisation failed with %d\n", ret);
1417 return ret; 1611 return ret;
1418} 1612}
@@ -1444,6 +1638,14 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec,
1444 data.dma_request = dma_spec->args[0]; 1638 data.dma_request = dma_spec->args[0];
1445 data.peripheral_type = dma_spec->args[1]; 1639 data.peripheral_type = dma_spec->args[1];
1446 data.priority = dma_spec->args[2]; 1640 data.priority = dma_spec->args[2];
1641 /*
1642 * init dma_request2 to zero, which is not used by the dts.
1643 * For P2P, dma_request2 is init from dma_request_channel(),
1644 * chan->private will point to the imx_dma_data, and in
1645 * device_alloc_chan_resources(), imx_dma_data.dma_request2 will
1646 * be set to sdmac->event_id1.
1647 */
1648 data.dma_request2 = 0;
1447 1649
1448 return dma_request_channel(mask, sdma_filter_fn, &data); 1650 return dma_request_channel(mask, sdma_filter_fn, &data);
1449} 1651}
@@ -1453,10 +1655,12 @@ static int sdma_probe(struct platform_device *pdev)
1453 const struct of_device_id *of_id = 1655 const struct of_device_id *of_id =
1454 of_match_device(sdma_dt_ids, &pdev->dev); 1656 of_match_device(sdma_dt_ids, &pdev->dev);
1455 struct device_node *np = pdev->dev.of_node; 1657 struct device_node *np = pdev->dev.of_node;
1658 struct device_node *spba_bus;
1456 const char *fw_name; 1659 const char *fw_name;
1457 int ret; 1660 int ret;
1458 int irq; 1661 int irq;
1459 struct resource *iores; 1662 struct resource *iores;
1663 struct resource spba_res;
1460 struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev); 1664 struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev);
1461 int i; 1665 int i;
1462 struct sdma_engine *sdma; 1666 struct sdma_engine *sdma;
@@ -1551,6 +1755,10 @@ static int sdma_probe(struct platform_device *pdev)
1551 if (ret) 1755 if (ret)
1552 goto err_init; 1756 goto err_init;
1553 1757
1758 ret = sdma_event_remap(sdma);
1759 if (ret)
1760 goto err_init;
1761
1554 if (sdma->drvdata->script_addrs) 1762 if (sdma->drvdata->script_addrs)
1555 sdma_add_scripts(sdma, sdma->drvdata->script_addrs); 1763 sdma_add_scripts(sdma, sdma->drvdata->script_addrs);
1556 if (pdata && pdata->script_addrs) 1764 if (pdata && pdata->script_addrs)
@@ -1608,6 +1816,14 @@ static int sdma_probe(struct platform_device *pdev)
1608 dev_err(&pdev->dev, "failed to register controller\n"); 1816 dev_err(&pdev->dev, "failed to register controller\n");
1609 goto err_register; 1817 goto err_register;
1610 } 1818 }
1819
1820 spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus");
1821 ret = of_address_to_resource(spba_bus, 0, &spba_res);
1822 if (!ret) {
1823 sdma->spba_start_addr = spba_res.start;
1824 sdma->spba_end_addr = spba_res.end;
1825 }
1826 of_node_put(spba_bus);
1611 } 1827 }
1612 1828
1613 dev_info(sdma->dev, "initialized\n"); 1829 dev_info(sdma->dev, "initialized\n");
diff --git a/drivers/dma/ioat/Makefile b/drivers/dma/ioat/Makefile
index 0ff7270af25b..cf5fedbe2b75 100644
--- a/drivers/dma/ioat/Makefile
+++ b/drivers/dma/ioat/Makefile
@@ -1,2 +1,2 @@
1obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o 1obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
2ioatdma-y := pci.o dma.o dma_v2.o dma_v3.o dca.o 2ioatdma-y := init.o dma.o prep.o dca.o sysfs.o
diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c
index ea1e107ae884..2cb7c308d5c7 100644
--- a/drivers/dma/ioat/dca.c
+++ b/drivers/dma/ioat/dca.c
@@ -31,7 +31,6 @@
31 31
32#include "dma.h" 32#include "dma.h"
33#include "registers.h" 33#include "registers.h"
34#include "dma_v2.h"
35 34
36/* 35/*
37 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6 36 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
@@ -71,14 +70,6 @@ static inline int dca2_tag_map_valid(u8 *tag_map)
71#define APICID_BIT(x) (DCA_TAG_MAP_VALID | (x)) 70#define APICID_BIT(x) (DCA_TAG_MAP_VALID | (x))
72#define IOAT_TAG_MAP_LEN 8 71#define IOAT_TAG_MAP_LEN 8
73 72
74static u8 ioat_tag_map_BNB[IOAT_TAG_MAP_LEN] = {
75 1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), };
76static u8 ioat_tag_map_SCNB[IOAT_TAG_MAP_LEN] = {
77 1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), };
78static u8 ioat_tag_map_CNB[IOAT_TAG_MAP_LEN] = {
79 1, APICID_BIT(1), APICID_BIT(3), APICID_BIT(4), APICID_BIT(2), };
80static u8 ioat_tag_map_UNISYS[IOAT_TAG_MAP_LEN] = { 0 };
81
82/* pack PCI B/D/F into a u16 */ 73/* pack PCI B/D/F into a u16 */
83static inline u16 dcaid_from_pcidev(struct pci_dev *pci) 74static inline u16 dcaid_from_pcidev(struct pci_dev *pci)
84{ 75{
@@ -126,96 +117,6 @@ struct ioat_dca_priv {
126 struct ioat_dca_slot req_slots[0]; 117 struct ioat_dca_slot req_slots[0];
127}; 118};
128 119
129/* 5000 series chipset DCA Port Requester ID Table Entry Format
130 * [15:8] PCI-Express Bus Number
131 * [7:3] PCI-Express Device Number
132 * [2:0] PCI-Express Function Number
133 *
134 * 5000 series chipset DCA control register format
135 * [7:1] Reserved (0)
136 * [0] Ignore Function Number
137 */
138
139static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev)
140{
141 struct ioat_dca_priv *ioatdca = dca_priv(dca);
142 struct pci_dev *pdev;
143 int i;
144 u16 id;
145
146 /* This implementation only supports PCI-Express */
147 if (!dev_is_pci(dev))
148 return -ENODEV;
149 pdev = to_pci_dev(dev);
150 id = dcaid_from_pcidev(pdev);
151
152 if (ioatdca->requester_count == ioatdca->max_requesters)
153 return -ENODEV;
154
155 for (i = 0; i < ioatdca->max_requesters; i++) {
156 if (ioatdca->req_slots[i].pdev == NULL) {
157 /* found an empty slot */
158 ioatdca->requester_count++;
159 ioatdca->req_slots[i].pdev = pdev;
160 ioatdca->req_slots[i].rid = id;
161 writew(id, ioatdca->dca_base + (i * 4));
162 /* make sure the ignore function bit is off */
163 writeb(0, ioatdca->dca_base + (i * 4) + 2);
164 return i;
165 }
166 }
167 /* Error, ioatdma->requester_count is out of whack */
168 return -EFAULT;
169}
170
171static int ioat_dca_remove_requester(struct dca_provider *dca,
172 struct device *dev)
173{
174 struct ioat_dca_priv *ioatdca = dca_priv(dca);
175 struct pci_dev *pdev;
176 int i;
177
178 /* This implementation only supports PCI-Express */
179 if (!dev_is_pci(dev))
180 return -ENODEV;
181 pdev = to_pci_dev(dev);
182
183 for (i = 0; i < ioatdca->max_requesters; i++) {
184 if (ioatdca->req_slots[i].pdev == pdev) {
185 writew(0, ioatdca->dca_base + (i * 4));
186 ioatdca->req_slots[i].pdev = NULL;
187 ioatdca->req_slots[i].rid = 0;
188 ioatdca->requester_count--;
189 return i;
190 }
191 }
192 return -ENODEV;
193}
194
195static u8 ioat_dca_get_tag(struct dca_provider *dca,
196 struct device *dev,
197 int cpu)
198{
199 struct ioat_dca_priv *ioatdca = dca_priv(dca);
200 int i, apic_id, bit, value;
201 u8 entry, tag;
202
203 tag = 0;
204 apic_id = cpu_physical_id(cpu);
205
206 for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
207 entry = ioatdca->tag_map[i];
208 if (entry & DCA_TAG_MAP_VALID) {
209 bit = entry & ~DCA_TAG_MAP_VALID;
210 value = (apic_id & (1 << bit)) ? 1 : 0;
211 } else {
212 value = entry ? 1 : 0;
213 }
214 tag |= (value << i);
215 }
216 return tag;
217}
218
219static int ioat_dca_dev_managed(struct dca_provider *dca, 120static int ioat_dca_dev_managed(struct dca_provider *dca,
220 struct device *dev) 121 struct device *dev)
221{ 122{
@@ -231,260 +132,7 @@ static int ioat_dca_dev_managed(struct dca_provider *dca,
231 return 0; 132 return 0;
232} 133}
233 134
234static struct dca_ops ioat_dca_ops = { 135static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev)
235 .add_requester = ioat_dca_add_requester,
236 .remove_requester = ioat_dca_remove_requester,
237 .get_tag = ioat_dca_get_tag,
238 .dev_managed = ioat_dca_dev_managed,
239};
240
241
242struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
243{
244 struct dca_provider *dca;
245 struct ioat_dca_priv *ioatdca;
246 u8 *tag_map = NULL;
247 int i;
248 int err;
249 u8 version;
250 u8 max_requesters;
251
252 if (!system_has_dca_enabled(pdev))
253 return NULL;
254
255 /* I/OAT v1 systems must have a known tag_map to support DCA */
256 switch (pdev->vendor) {
257 case PCI_VENDOR_ID_INTEL:
258 switch (pdev->device) {
259 case PCI_DEVICE_ID_INTEL_IOAT:
260 tag_map = ioat_tag_map_BNB;
261 break;
262 case PCI_DEVICE_ID_INTEL_IOAT_CNB:
263 tag_map = ioat_tag_map_CNB;
264 break;
265 case PCI_DEVICE_ID_INTEL_IOAT_SCNB:
266 tag_map = ioat_tag_map_SCNB;
267 break;
268 }
269 break;
270 case PCI_VENDOR_ID_UNISYS:
271 switch (pdev->device) {
272 case PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR:
273 tag_map = ioat_tag_map_UNISYS;
274 break;
275 }
276 break;
277 }
278 if (tag_map == NULL)
279 return NULL;
280
281 version = readb(iobase + IOAT_VER_OFFSET);
282 if (version == IOAT_VER_3_0)
283 max_requesters = IOAT3_DCA_MAX_REQ;
284 else
285 max_requesters = IOAT_DCA_MAX_REQ;
286
287 dca = alloc_dca_provider(&ioat_dca_ops,
288 sizeof(*ioatdca) +
289 (sizeof(struct ioat_dca_slot) * max_requesters));
290 if (!dca)
291 return NULL;
292
293 ioatdca = dca_priv(dca);
294 ioatdca->max_requesters = max_requesters;
295 ioatdca->dca_base = iobase + 0x54;
296
297 /* copy over the APIC ID to DCA tag mapping */
298 for (i = 0; i < IOAT_TAG_MAP_LEN; i++)
299 ioatdca->tag_map[i] = tag_map[i];
300
301 err = register_dca_provider(dca, &pdev->dev);
302 if (err) {
303 free_dca_provider(dca);
304 return NULL;
305 }
306
307 return dca;
308}
309
310
311static int ioat2_dca_add_requester(struct dca_provider *dca, struct device *dev)
312{
313 struct ioat_dca_priv *ioatdca = dca_priv(dca);
314 struct pci_dev *pdev;
315 int i;
316 u16 id;
317 u16 global_req_table;
318
319 /* This implementation only supports PCI-Express */
320 if (!dev_is_pci(dev))
321 return -ENODEV;
322 pdev = to_pci_dev(dev);
323 id = dcaid_from_pcidev(pdev);
324
325 if (ioatdca->requester_count == ioatdca->max_requesters)
326 return -ENODEV;
327
328 for (i = 0; i < ioatdca->max_requesters; i++) {
329 if (ioatdca->req_slots[i].pdev == NULL) {
330 /* found an empty slot */
331 ioatdca->requester_count++;
332 ioatdca->req_slots[i].pdev = pdev;
333 ioatdca->req_slots[i].rid = id;
334 global_req_table =
335 readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET);
336 writel(id | IOAT_DCA_GREQID_VALID,
337 ioatdca->iobase + global_req_table + (i * 4));
338 return i;
339 }
340 }
341 /* Error, ioatdma->requester_count is out of whack */
342 return -EFAULT;
343}
344
345static int ioat2_dca_remove_requester(struct dca_provider *dca,
346 struct device *dev)
347{
348 struct ioat_dca_priv *ioatdca = dca_priv(dca);
349 struct pci_dev *pdev;
350 int i;
351 u16 global_req_table;
352
353 /* This implementation only supports PCI-Express */
354 if (!dev_is_pci(dev))
355 return -ENODEV;
356 pdev = to_pci_dev(dev);
357
358 for (i = 0; i < ioatdca->max_requesters; i++) {
359 if (ioatdca->req_slots[i].pdev == pdev) {
360 global_req_table =
361 readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET);
362 writel(0, ioatdca->iobase + global_req_table + (i * 4));
363 ioatdca->req_slots[i].pdev = NULL;
364 ioatdca->req_slots[i].rid = 0;
365 ioatdca->requester_count--;
366 return i;
367 }
368 }
369 return -ENODEV;
370}
371
372static u8 ioat2_dca_get_tag(struct dca_provider *dca,
373 struct device *dev,
374 int cpu)
375{
376 u8 tag;
377
378 tag = ioat_dca_get_tag(dca, dev, cpu);
379 tag = (~tag) & 0x1F;
380 return tag;
381}
382
383static struct dca_ops ioat2_dca_ops = {
384 .add_requester = ioat2_dca_add_requester,
385 .remove_requester = ioat2_dca_remove_requester,
386 .get_tag = ioat2_dca_get_tag,
387 .dev_managed = ioat_dca_dev_managed,
388};
389
390static int ioat2_dca_count_dca_slots(void __iomem *iobase, u16 dca_offset)
391{
392 int slots = 0;
393 u32 req;
394 u16 global_req_table;
395
396 global_req_table = readw(iobase + dca_offset + IOAT_DCA_GREQID_OFFSET);
397 if (global_req_table == 0)
398 return 0;
399 do {
400 req = readl(iobase + global_req_table + (slots * sizeof(u32)));
401 slots++;
402 } while ((req & IOAT_DCA_GREQID_LASTID) == 0);
403
404 return slots;
405}
406
407struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
408{
409 struct dca_provider *dca;
410 struct ioat_dca_priv *ioatdca;
411 int slots;
412 int i;
413 int err;
414 u32 tag_map;
415 u16 dca_offset;
416 u16 csi_fsb_control;
417 u16 pcie_control;
418 u8 bit;
419
420 if (!system_has_dca_enabled(pdev))
421 return NULL;
422
423 dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
424 if (dca_offset == 0)
425 return NULL;
426
427 slots = ioat2_dca_count_dca_slots(iobase, dca_offset);
428 if (slots == 0)
429 return NULL;
430
431 dca = alloc_dca_provider(&ioat2_dca_ops,
432 sizeof(*ioatdca)
433 + (sizeof(struct ioat_dca_slot) * slots));
434 if (!dca)
435 return NULL;
436
437 ioatdca = dca_priv(dca);
438 ioatdca->iobase = iobase;
439 ioatdca->dca_base = iobase + dca_offset;
440 ioatdca->max_requesters = slots;
441
442 /* some bios might not know to turn these on */
443 csi_fsb_control = readw(ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET);
444 if ((csi_fsb_control & IOAT_FSB_CAP_ENABLE_PREFETCH) == 0) {
445 csi_fsb_control |= IOAT_FSB_CAP_ENABLE_PREFETCH;
446 writew(csi_fsb_control,
447 ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET);
448 }
449 pcie_control = readw(ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET);
450 if ((pcie_control & IOAT_PCI_CAP_ENABLE_MEMWR) == 0) {
451 pcie_control |= IOAT_PCI_CAP_ENABLE_MEMWR;
452 writew(pcie_control,
453 ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET);
454 }
455
456
457 /* TODO version, compatibility and configuration checks */
458
459 /* copy out the APIC to DCA tag map */
460 tag_map = readl(ioatdca->dca_base + IOAT_APICID_TAG_MAP_OFFSET);
461 for (i = 0; i < 5; i++) {
462 bit = (tag_map >> (4 * i)) & 0x0f;
463 if (bit < 8)
464 ioatdca->tag_map[i] = bit | DCA_TAG_MAP_VALID;
465 else
466 ioatdca->tag_map[i] = 0;
467 }
468
469 if (!dca2_tag_map_valid(ioatdca->tag_map)) {
470 WARN_TAINT_ONCE(1, TAINT_FIRMWARE_WORKAROUND,
471 "%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n",
472 dev_driver_string(&pdev->dev),
473 dev_name(&pdev->dev));
474 free_dca_provider(dca);
475 return NULL;
476 }
477
478 err = register_dca_provider(dca, &pdev->dev);
479 if (err) {
480 free_dca_provider(dca);
481 return NULL;
482 }
483
484 return dca;
485}
486
487static int ioat3_dca_add_requester(struct dca_provider *dca, struct device *dev)
488{ 136{
489 struct ioat_dca_priv *ioatdca = dca_priv(dca); 137 struct ioat_dca_priv *ioatdca = dca_priv(dca);
490 struct pci_dev *pdev; 138 struct pci_dev *pdev;
@@ -518,7 +166,7 @@ static int ioat3_dca_add_requester(struct dca_provider *dca, struct device *dev)
518 return -EFAULT; 166 return -EFAULT;
519} 167}
520 168
521static int ioat3_dca_remove_requester(struct dca_provider *dca, 169static int ioat_dca_remove_requester(struct dca_provider *dca,
522 struct device *dev) 170 struct device *dev)
523{ 171{
524 struct ioat_dca_priv *ioatdca = dca_priv(dca); 172 struct ioat_dca_priv *ioatdca = dca_priv(dca);
@@ -545,7 +193,7 @@ static int ioat3_dca_remove_requester(struct dca_provider *dca,
545 return -ENODEV; 193 return -ENODEV;
546} 194}
547 195
548static u8 ioat3_dca_get_tag(struct dca_provider *dca, 196static u8 ioat_dca_get_tag(struct dca_provider *dca,
549 struct device *dev, 197 struct device *dev,
550 int cpu) 198 int cpu)
551{ 199{
@@ -576,14 +224,14 @@ static u8 ioat3_dca_get_tag(struct dca_provider *dca,
576 return tag; 224 return tag;
577} 225}
578 226
579static struct dca_ops ioat3_dca_ops = { 227static struct dca_ops ioat_dca_ops = {
580 .add_requester = ioat3_dca_add_requester, 228 .add_requester = ioat_dca_add_requester,
581 .remove_requester = ioat3_dca_remove_requester, 229 .remove_requester = ioat_dca_remove_requester,
582 .get_tag = ioat3_dca_get_tag, 230 .get_tag = ioat_dca_get_tag,
583 .dev_managed = ioat_dca_dev_managed, 231 .dev_managed = ioat_dca_dev_managed,
584}; 232};
585 233
586static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset) 234static int ioat_dca_count_dca_slots(void *iobase, u16 dca_offset)
587{ 235{
588 int slots = 0; 236 int slots = 0;
589 u32 req; 237 u32 req;
@@ -618,7 +266,7 @@ static inline int dca3_tag_map_invalid(u8 *tag_map)
618 (tag_map[4] == DCA_TAG_MAP_VALID)); 266 (tag_map[4] == DCA_TAG_MAP_VALID));
619} 267}
620 268
621struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase) 269struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
622{ 270{
623 struct dca_provider *dca; 271 struct dca_provider *dca;
624 struct ioat_dca_priv *ioatdca; 272 struct ioat_dca_priv *ioatdca;
@@ -645,11 +293,11 @@ struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
645 if (dca_offset == 0) 293 if (dca_offset == 0)
646 return NULL; 294 return NULL;
647 295
648 slots = ioat3_dca_count_dca_slots(iobase, dca_offset); 296 slots = ioat_dca_count_dca_slots(iobase, dca_offset);
649 if (slots == 0) 297 if (slots == 0)
650 return NULL; 298 return NULL;
651 299
652 dca = alloc_dca_provider(&ioat3_dca_ops, 300 dca = alloc_dca_provider(&ioat_dca_ops,
653 sizeof(*ioatdca) 301 sizeof(*ioatdca)
654 + (sizeof(struct ioat_dca_slot) * slots)); 302 + (sizeof(struct ioat_dca_slot) * slots));
655 if (!dca) 303 if (!dca)
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index ee0aa9f4ccfa..f66b7e640610 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -1,6 +1,6 @@
1/* 1/*
2 * Intel I/OAT DMA Linux driver 2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation. 3 * Copyright(c) 2004 - 2015 Intel Corporation.
4 * 4 *
5 * This program is free software; you can redistribute it and/or modify it 5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License, 6 * under the terms and conditions of the GNU General Public License,
@@ -31,31 +31,23 @@
31#include <linux/dma-mapping.h> 31#include <linux/dma-mapping.h>
32#include <linux/workqueue.h> 32#include <linux/workqueue.h>
33#include <linux/prefetch.h> 33#include <linux/prefetch.h>
34#include <linux/i7300_idle.h>
35#include "dma.h" 34#include "dma.h"
36#include "registers.h" 35#include "registers.h"
37#include "hw.h" 36#include "hw.h"
38 37
39#include "../dmaengine.h" 38#include "../dmaengine.h"
40 39
41int ioat_pending_level = 4; 40static void ioat_eh(struct ioatdma_chan *ioat_chan);
42module_param(ioat_pending_level, int, 0644);
43MODULE_PARM_DESC(ioat_pending_level,
44 "high-water mark for pushing ioat descriptors (default: 4)");
45
46/* internal functions */
47static void ioat1_cleanup(struct ioat_dma_chan *ioat);
48static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat);
49 41
50/** 42/**
51 * ioat_dma_do_interrupt - handler used for single vector interrupt mode 43 * ioat_dma_do_interrupt - handler used for single vector interrupt mode
52 * @irq: interrupt id 44 * @irq: interrupt id
53 * @data: interrupt data 45 * @data: interrupt data
54 */ 46 */
55static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) 47irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
56{ 48{
57 struct ioatdma_device *instance = data; 49 struct ioatdma_device *instance = data;
58 struct ioat_chan_common *chan; 50 struct ioatdma_chan *ioat_chan;
59 unsigned long attnstatus; 51 unsigned long attnstatus;
60 int bit; 52 int bit;
61 u8 intrctrl; 53 u8 intrctrl;
@@ -72,9 +64,9 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
72 64
73 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); 65 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
74 for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { 66 for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) {
75 chan = ioat_chan_by_index(instance, bit); 67 ioat_chan = ioat_chan_by_index(instance, bit);
76 if (test_bit(IOAT_RUN, &chan->state)) 68 if (test_bit(IOAT_RUN, &ioat_chan->state))
77 tasklet_schedule(&chan->cleanup_task); 69 tasklet_schedule(&ioat_chan->cleanup_task);
78 } 70 }
79 71
80 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); 72 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
@@ -86,1161 +78,912 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
86 * @irq: interrupt id 78 * @irq: interrupt id
87 * @data: interrupt data 79 * @data: interrupt data
88 */ 80 */
89static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) 81irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
90{ 82{
91 struct ioat_chan_common *chan = data; 83 struct ioatdma_chan *ioat_chan = data;
92 84
93 if (test_bit(IOAT_RUN, &chan->state)) 85 if (test_bit(IOAT_RUN, &ioat_chan->state))
94 tasklet_schedule(&chan->cleanup_task); 86 tasklet_schedule(&ioat_chan->cleanup_task);
95 87
96 return IRQ_HANDLED; 88 return IRQ_HANDLED;
97} 89}
98 90
99/* common channel initialization */ 91void ioat_stop(struct ioatdma_chan *ioat_chan)
100void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx)
101{ 92{
102 struct dma_device *dma = &device->common; 93 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
103 struct dma_chan *c = &chan->common; 94 struct pci_dev *pdev = ioat_dma->pdev;
104 unsigned long data = (unsigned long) c; 95 int chan_id = chan_num(ioat_chan);
105 96 struct msix_entry *msix;
106 chan->device = device; 97
107 chan->reg_base = device->reg_base + (0x80 * (idx + 1)); 98 /* 1/ stop irq from firing tasklets
108 spin_lock_init(&chan->cleanup_lock); 99 * 2/ stop the tasklet from re-arming irqs
109 chan->common.device = dma; 100 */
110 dma_cookie_init(&chan->common); 101 clear_bit(IOAT_RUN, &ioat_chan->state);
111 list_add_tail(&chan->common.device_node, &dma->channels); 102
112 device->idx[idx] = chan; 103 /* flush inflight interrupts */
113 init_timer(&chan->timer); 104 switch (ioat_dma->irq_mode) {
114 chan->timer.function = device->timer_fn; 105 case IOAT_MSIX:
115 chan->timer.data = data; 106 msix = &ioat_dma->msix_entries[chan_id];
116 tasklet_init(&chan->cleanup_task, device->cleanup_fn, data); 107 synchronize_irq(msix->vector);
108 break;
109 case IOAT_MSI:
110 case IOAT_INTX:
111 synchronize_irq(pdev->irq);
112 break;
113 default:
114 break;
115 }
116
117 /* flush inflight timers */
118 del_timer_sync(&ioat_chan->timer);
119
120 /* flush inflight tasklet runs */
121 tasklet_kill(&ioat_chan->cleanup_task);
122
123 /* final cleanup now that everything is quiesced and can't re-arm */
124 ioat_cleanup_event((unsigned long)&ioat_chan->dma_chan);
117} 125}
118 126
119/** 127static void __ioat_issue_pending(struct ioatdma_chan *ioat_chan)
120 * ioat1_dma_enumerate_channels - find and initialize the device's channels
121 * @device: the device to be enumerated
122 */
123static int ioat1_enumerate_channels(struct ioatdma_device *device)
124{ 128{
125 u8 xfercap_scale; 129 ioat_chan->dmacount += ioat_ring_pending(ioat_chan);
126 u32 xfercap; 130 ioat_chan->issued = ioat_chan->head;
127 int i; 131 writew(ioat_chan->dmacount,
128 struct ioat_dma_chan *ioat; 132 ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
129 struct device *dev = &device->pdev->dev; 133 dev_dbg(to_dev(ioat_chan),
130 struct dma_device *dma = &device->common; 134 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
131 135 __func__, ioat_chan->head, ioat_chan->tail,
132 INIT_LIST_HEAD(&dma->channels); 136 ioat_chan->issued, ioat_chan->dmacount);
133 dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET); 137}
134 dma->chancnt &= 0x1f; /* bits [4:0] valid */ 138
135 if (dma->chancnt > ARRAY_SIZE(device->idx)) { 139void ioat_issue_pending(struct dma_chan *c)
136 dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n", 140{
137 dma->chancnt, ARRAY_SIZE(device->idx)); 141 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
138 dma->chancnt = ARRAY_SIZE(device->idx);
139 }
140 xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
141 xfercap_scale &= 0x1f; /* bits [4:0] valid */
142 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
143 dev_dbg(dev, "%s: xfercap = %d\n", __func__, xfercap);
144
145#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
146 if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
147 dma->chancnt--;
148#endif
149 for (i = 0; i < dma->chancnt; i++) {
150 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
151 if (!ioat)
152 break;
153 142
154 ioat_init_channel(device, &ioat->base, i); 143 if (ioat_ring_pending(ioat_chan)) {
155 ioat->xfercap = xfercap; 144 spin_lock_bh(&ioat_chan->prep_lock);
156 spin_lock_init(&ioat->desc_lock); 145 __ioat_issue_pending(ioat_chan);
157 INIT_LIST_HEAD(&ioat->free_desc); 146 spin_unlock_bh(&ioat_chan->prep_lock);
158 INIT_LIST_HEAD(&ioat->used_desc);
159 } 147 }
160 dma->chancnt = i;
161 return i;
162} 148}
163 149
164/** 150/**
165 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended 151 * ioat_update_pending - log pending descriptors
166 * descriptors to hw 152 * @ioat: ioat+ channel
167 * @chan: DMA channel handle 153 *
154 * Check if the number of unsubmitted descriptors has exceeded the
155 * watermark. Called with prep_lock held
168 */ 156 */
169static inline void 157static void ioat_update_pending(struct ioatdma_chan *ioat_chan)
170__ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
171{ 158{
172 void __iomem *reg_base = ioat->base.reg_base; 159 if (ioat_ring_pending(ioat_chan) > ioat_pending_level)
173 160 __ioat_issue_pending(ioat_chan);
174 dev_dbg(to_dev(&ioat->base), "%s: pending: %d\n",
175 __func__, ioat->pending);
176 ioat->pending = 0;
177 writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET);
178} 161}
179 162
180static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan) 163static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
181{ 164{
182 struct ioat_dma_chan *ioat = to_ioat_chan(chan); 165 struct ioat_ring_ent *desc;
166 struct ioat_dma_descriptor *hw;
183 167
184 if (ioat->pending > 0) { 168 if (ioat_ring_space(ioat_chan) < 1) {
185 spin_lock_bh(&ioat->desc_lock); 169 dev_err(to_dev(ioat_chan),
186 __ioat1_dma_memcpy_issue_pending(ioat); 170 "Unable to start null desc - ring full\n");
187 spin_unlock_bh(&ioat->desc_lock); 171 return;
188 } 172 }
173
174 dev_dbg(to_dev(ioat_chan),
175 "%s: head: %#x tail: %#x issued: %#x\n",
176 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
177 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);
178
179 hw = desc->hw;
180 hw->ctl = 0;
181 hw->ctl_f.null = 1;
182 hw->ctl_f.int_en = 1;
183 hw->ctl_f.compl_write = 1;
184 /* set size to non-zero value (channel returns error when size is 0) */
185 hw->size = NULL_DESC_BUFFER_SIZE;
186 hw->src_addr = 0;
187 hw->dst_addr = 0;
188 async_tx_ack(&desc->txd);
189 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
190 dump_desc_dbg(ioat_chan, desc);
191 /* make sure descriptors are written before we submit */
192 wmb();
193 ioat_chan->head += 1;
194 __ioat_issue_pending(ioat_chan);
189} 195}
190 196
191/** 197void ioat_start_null_desc(struct ioatdma_chan *ioat_chan)
192 * ioat1_reset_channel - restart a channel
193 * @ioat: IOAT DMA channel handle
194 */
195static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
196{ 198{
197 struct ioat_chan_common *chan = &ioat->base; 199 spin_lock_bh(&ioat_chan->prep_lock);
198 void __iomem *reg_base = chan->reg_base; 200 __ioat_start_null_desc(ioat_chan);
199 u32 chansts, chanerr; 201 spin_unlock_bh(&ioat_chan->prep_lock);
200 202}
201 dev_warn(to_dev(chan), "reset\n"); 203
202 chanerr = readl(reg_base + IOAT_CHANERR_OFFSET); 204static void __ioat_restart_chan(struct ioatdma_chan *ioat_chan)
203 chansts = *chan->completion & IOAT_CHANSTS_STATUS; 205{
204 if (chanerr) { 206 /* set the tail to be re-issued */
205 dev_err(to_dev(chan), 207 ioat_chan->issued = ioat_chan->tail;
206 "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n", 208 ioat_chan->dmacount = 0;
207 chan_num(chan), chansts, chanerr); 209 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
208 writel(chanerr, reg_base + IOAT_CHANERR_OFFSET); 210
211 dev_dbg(to_dev(ioat_chan),
212 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
213 __func__, ioat_chan->head, ioat_chan->tail,
214 ioat_chan->issued, ioat_chan->dmacount);
215
216 if (ioat_ring_pending(ioat_chan)) {
217 struct ioat_ring_ent *desc;
218
219 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
220 ioat_set_chainaddr(ioat_chan, desc->txd.phys);
221 __ioat_issue_pending(ioat_chan);
222 } else
223 __ioat_start_null_desc(ioat_chan);
224}
225
226static int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo)
227{
228 unsigned long end = jiffies + tmo;
229 int err = 0;
230 u32 status;
231
232 status = ioat_chansts(ioat_chan);
233 if (is_ioat_active(status) || is_ioat_idle(status))
234 ioat_suspend(ioat_chan);
235 while (is_ioat_active(status) || is_ioat_idle(status)) {
236 if (tmo && time_after(jiffies, end)) {
237 err = -ETIMEDOUT;
238 break;
239 }
240 status = ioat_chansts(ioat_chan);
241 cpu_relax();
209 } 242 }
210 243
211 /* 244 return err;
212 * whack it upside the head with a reset 245}
213 * and wait for things to settle out.
214 * force the pending count to a really big negative
215 * to make sure no one forces an issue_pending
216 * while we're waiting.
217 */
218 246
219 ioat->pending = INT_MIN; 247static int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo)
220 writeb(IOAT_CHANCMD_RESET, 248{
221 reg_base + IOAT_CHANCMD_OFFSET(chan->device->version)); 249 unsigned long end = jiffies + tmo;
222 set_bit(IOAT_RESET_PENDING, &chan->state); 250 int err = 0;
223 mod_timer(&chan->timer, jiffies + RESET_DELAY); 251
252 ioat_reset(ioat_chan);
253 while (ioat_reset_pending(ioat_chan)) {
254 if (end && time_after(jiffies, end)) {
255 err = -ETIMEDOUT;
256 break;
257 }
258 cpu_relax();
259 }
260
261 return err;
224} 262}
225 263
226static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx) 264static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
265 __releases(&ioat_chan->prep_lock)
227{ 266{
228 struct dma_chan *c = tx->chan; 267 struct dma_chan *c = tx->chan;
229 struct ioat_dma_chan *ioat = to_ioat_chan(c); 268 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
230 struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
231 struct ioat_chan_common *chan = &ioat->base;
232 struct ioat_desc_sw *first;
233 struct ioat_desc_sw *chain_tail;
234 dma_cookie_t cookie; 269 dma_cookie_t cookie;
235 270
236 spin_lock_bh(&ioat->desc_lock);
237 /* cookie incr and addition to used_list must be atomic */
238 cookie = dma_cookie_assign(tx); 271 cookie = dma_cookie_assign(tx);
239 dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie); 272 dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie);
273
274 if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
275 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
240 276
241 /* write address into NextDescriptor field of last desc in chain */ 277 /* make descriptor updates visible before advancing ioat->head,
242 first = to_ioat_desc(desc->tx_list.next); 278 * this is purposefully not smp_wmb() since we are also
243 chain_tail = to_ioat_desc(ioat->used_desc.prev); 279 * publishing the descriptor updates to a dma device
244 /* make descriptor updates globally visible before chaining */ 280 */
245 wmb(); 281 wmb();
246 chain_tail->hw->next = first->txd.phys;
247 list_splice_tail_init(&desc->tx_list, &ioat->used_desc);
248 dump_desc_dbg(ioat, chain_tail);
249 dump_desc_dbg(ioat, first);
250 282
251 if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state)) 283 ioat_chan->head += ioat_chan->produce;
252 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
253 284
254 ioat->active += desc->hw->tx_cnt; 285 ioat_update_pending(ioat_chan);
255 ioat->pending += desc->hw->tx_cnt; 286 spin_unlock_bh(&ioat_chan->prep_lock);
256 if (ioat->pending >= ioat_pending_level)
257 __ioat1_dma_memcpy_issue_pending(ioat);
258 spin_unlock_bh(&ioat->desc_lock);
259 287
260 return cookie; 288 return cookie;
261} 289}
262 290
263/** 291static struct ioat_ring_ent *
264 * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair 292ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
265 * @ioat: the channel supplying the memory pool for the descriptors
266 * @flags: allocation flags
267 */
268static struct ioat_desc_sw *
269ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
270{ 293{
271 struct ioat_dma_descriptor *desc; 294 struct ioat_dma_descriptor *hw;
272 struct ioat_desc_sw *desc_sw; 295 struct ioat_ring_ent *desc;
273 struct ioatdma_device *ioatdma_device; 296 struct ioatdma_device *ioat_dma;
274 dma_addr_t phys; 297 dma_addr_t phys;
275 298
276 ioatdma_device = ioat->base.device; 299 ioat_dma = to_ioatdma_device(chan->device);
277 desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys); 300 hw = pci_pool_alloc(ioat_dma->dma_pool, flags, &phys);
278 if (unlikely(!desc)) 301 if (!hw)
279 return NULL; 302 return NULL;
303 memset(hw, 0, sizeof(*hw));
280 304
281 desc_sw = kzalloc(sizeof(*desc_sw), flags); 305 desc = kmem_cache_zalloc(ioat_cache, flags);
282 if (unlikely(!desc_sw)) { 306 if (!desc) {
283 pci_pool_free(ioatdma_device->dma_pool, desc, phys); 307 pci_pool_free(ioat_dma->dma_pool, hw, phys);
284 return NULL; 308 return NULL;
285 } 309 }
286 310
287 memset(desc, 0, sizeof(*desc)); 311 dma_async_tx_descriptor_init(&desc->txd, chan);
312 desc->txd.tx_submit = ioat_tx_submit_unlock;
313 desc->hw = hw;
314 desc->txd.phys = phys;
315 return desc;
316}
288 317
289 INIT_LIST_HEAD(&desc_sw->tx_list); 318void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
290 dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common); 319{
291 desc_sw->txd.tx_submit = ioat1_tx_submit; 320 struct ioatdma_device *ioat_dma;
292 desc_sw->hw = desc;
293 desc_sw->txd.phys = phys;
294 set_desc_id(desc_sw, -1);
295 321
296 return desc_sw; 322 ioat_dma = to_ioatdma_device(chan->device);
323 pci_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys);
324 kmem_cache_free(ioat_cache, desc);
297} 325}
298 326
299static int ioat_initial_desc_count = 256; 327struct ioat_ring_ent **
300module_param(ioat_initial_desc_count, int, 0644); 328ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
301MODULE_PARM_DESC(ioat_initial_desc_count,
302 "ioat1: initial descriptors per channel (default: 256)");
303/**
304 * ioat1_dma_alloc_chan_resources - returns the number of allocated descriptors
305 * @chan: the channel to be filled out
306 */
307static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)
308{ 329{
309 struct ioat_dma_chan *ioat = to_ioat_chan(c); 330 struct ioat_ring_ent **ring;
310 struct ioat_chan_common *chan = &ioat->base; 331 int descs = 1 << order;
311 struct ioat_desc_sw *desc;
312 u32 chanerr;
313 int i; 332 int i;
314 LIST_HEAD(tmp_list);
315
316 /* have we already been set up? */
317 if (!list_empty(&ioat->free_desc))
318 return ioat->desccount;
319 333
320 /* Setup register to interrupt and write completion status on error */ 334 if (order > ioat_get_max_alloc_order())
321 writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET); 335 return NULL;
322 336
323 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); 337 /* allocate the array to hold the software ring */
324 if (chanerr) { 338 ring = kcalloc(descs, sizeof(*ring), flags);
325 dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr); 339 if (!ring)
326 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); 340 return NULL;
341 for (i = 0; i < descs; i++) {
342 ring[i] = ioat_alloc_ring_ent(c, flags);
343 if (!ring[i]) {
344 while (i--)
345 ioat_free_ring_ent(ring[i], c);
346 kfree(ring);
347 return NULL;
348 }
349 set_desc_id(ring[i], i);
327 } 350 }
328 351
329 /* Allocate descriptors */ 352 /* link descs */
330 for (i = 0; i < ioat_initial_desc_count; i++) { 353 for (i = 0; i < descs-1; i++) {
331 desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL); 354 struct ioat_ring_ent *next = ring[i+1];
332 if (!desc) { 355 struct ioat_dma_descriptor *hw = ring[i]->hw;
333 dev_err(to_dev(chan), "Only %d initial descriptors\n", i); 356
334 break; 357 hw->next = next->txd.phys;
335 }
336 set_desc_id(desc, i);
337 list_add_tail(&desc->node, &tmp_list);
338 } 358 }
339 spin_lock_bh(&ioat->desc_lock); 359 ring[i]->hw->next = ring[0]->txd.phys;
340 ioat->desccount = i; 360
341 list_splice(&tmp_list, &ioat->free_desc); 361 return ring;
342 spin_unlock_bh(&ioat->desc_lock);
343
344 /* allocate a completion writeback area */
345 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
346 chan->completion = pci_pool_alloc(chan->device->completion_pool,
347 GFP_KERNEL, &chan->completion_dma);
348 memset(chan->completion, 0, sizeof(*chan->completion));
349 writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
350 chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
351 writel(((u64) chan->completion_dma) >> 32,
352 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
353
354 set_bit(IOAT_RUN, &chan->state);
355 ioat1_dma_start_null_desc(ioat); /* give chain to dma device */
356 dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
357 __func__, ioat->desccount);
358 return ioat->desccount;
359} 362}
360 363
361void ioat_stop(struct ioat_chan_common *chan) 364static bool reshape_ring(struct ioatdma_chan *ioat_chan, int order)
362{ 365{
363 struct ioatdma_device *device = chan->device; 366 /* reshape differs from normal ring allocation in that we want
364 struct pci_dev *pdev = device->pdev; 367 * to allocate a new software ring while only
365 int chan_id = chan_num(chan); 368 * extending/truncating the hardware ring
366 struct msix_entry *msix; 369 */
370 struct dma_chan *c = &ioat_chan->dma_chan;
371 const u32 curr_size = ioat_ring_size(ioat_chan);
372 const u16 active = ioat_ring_active(ioat_chan);
373 const u32 new_size = 1 << order;
374 struct ioat_ring_ent **ring;
375 u32 i;
376
377 if (order > ioat_get_max_alloc_order())
378 return false;
367 379
368 /* 1/ stop irq from firing tasklets 380 /* double check that we have at least 1 free descriptor */
369 * 2/ stop the tasklet from re-arming irqs 381 if (active == curr_size)
382 return false;
383
384 /* when shrinking, verify that we can hold the current active
385 * set in the new ring
370 */ 386 */
371 clear_bit(IOAT_RUN, &chan->state); 387 if (active >= new_size)
388 return false;
372 389
373 /* flush inflight interrupts */ 390 /* allocate the array to hold the software ring */
374 switch (device->irq_mode) { 391 ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);
375 case IOAT_MSIX: 392 if (!ring)
376 msix = &device->msix_entries[chan_id]; 393 return false;
377 synchronize_irq(msix->vector);
378 break;
379 case IOAT_MSI:
380 case IOAT_INTX:
381 synchronize_irq(pdev->irq);
382 break;
383 default:
384 break;
385 }
386 394
387 /* flush inflight timers */ 395 /* allocate/trim descriptors as needed */
388 del_timer_sync(&chan->timer); 396 if (new_size > curr_size) {
397 /* copy current descriptors to the new ring */
398 for (i = 0; i < curr_size; i++) {
399 u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);
400 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
389 401
390 /* flush inflight tasklet runs */ 402 ring[new_idx] = ioat_chan->ring[curr_idx];
391 tasklet_kill(&chan->cleanup_task); 403 set_desc_id(ring[new_idx], new_idx);
404 }
392 405
393 /* final cleanup now that everything is quiesced and can't re-arm */ 406 /* add new descriptors to the ring */
394 device->cleanup_fn((unsigned long) &chan->common); 407 for (i = curr_size; i < new_size; i++) {
395} 408 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
396 409
397/** 410 ring[new_idx] = ioat_alloc_ring_ent(c, GFP_NOWAIT);
398 * ioat1_dma_free_chan_resources - release all the descriptors 411 if (!ring[new_idx]) {
399 * @chan: the channel to be cleaned 412 while (i--) {
400 */ 413 u16 new_idx = (ioat_chan->tail+i) &
401static void ioat1_dma_free_chan_resources(struct dma_chan *c) 414 (new_size-1);
402{ 415
403 struct ioat_dma_chan *ioat = to_ioat_chan(c); 416 ioat_free_ring_ent(ring[new_idx], c);
404 struct ioat_chan_common *chan = &ioat->base; 417 }
405 struct ioatdma_device *ioatdma_device = chan->device; 418 kfree(ring);
406 struct ioat_desc_sw *desc, *_desc; 419 return false;
407 int in_use_descs = 0; 420 }
408 421 set_desc_id(ring[new_idx], new_idx);
409 /* Before freeing channel resources first check 422 }
410 * if they have been previously allocated for this channel.
411 */
412 if (ioat->desccount == 0)
413 return;
414 423
415 ioat_stop(chan); 424 /* hw link new descriptors */
425 for (i = curr_size-1; i < new_size; i++) {
426 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
427 struct ioat_ring_ent *next =
428 ring[(new_idx+1) & (new_size-1)];
429 struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
416 430
417 /* Delay 100ms after reset to allow internal DMA logic to quiesce 431 hw->next = next->txd.phys;
418 * before removing DMA descriptor resources. 432 }
419 */ 433 } else {
420 writeb(IOAT_CHANCMD_RESET, 434 struct ioat_dma_descriptor *hw;
421 chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version)); 435 struct ioat_ring_ent *next;
422 mdelay(100); 436
423 437 /* copy current descriptors to the new ring, dropping the
424 spin_lock_bh(&ioat->desc_lock); 438 * removed descriptors
425 list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) { 439 */
426 dev_dbg(to_dev(chan), "%s: freeing %d from used list\n", 440 for (i = 0; i < new_size; i++) {
427 __func__, desc_id(desc)); 441 u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1);
428 dump_desc_dbg(ioat, desc); 442 u16 new_idx = (ioat_chan->tail+i) & (new_size-1);
429 in_use_descs++; 443
430 list_del(&desc->node); 444 ring[new_idx] = ioat_chan->ring[curr_idx];
431 pci_pool_free(ioatdma_device->dma_pool, desc->hw, 445 set_desc_id(ring[new_idx], new_idx);
432 desc->txd.phys); 446 }
433 kfree(desc); 447
434 } 448 /* free deleted descriptors */
435 list_for_each_entry_safe(desc, _desc, 449 for (i = new_size; i < curr_size; i++) {
436 &ioat->free_desc, node) { 450 struct ioat_ring_ent *ent;
437 list_del(&desc->node); 451
438 pci_pool_free(ioatdma_device->dma_pool, desc->hw, 452 ent = ioat_get_ring_ent(ioat_chan, ioat_chan->tail+i);
439 desc->txd.phys); 453 ioat_free_ring_ent(ent, c);
440 kfree(desc); 454 }
455
456 /* fix up hardware ring */
457 hw = ring[(ioat_chan->tail+new_size-1) & (new_size-1)]->hw;
458 next = ring[(ioat_chan->tail+new_size) & (new_size-1)];
459 hw->next = next->txd.phys;
441 } 460 }
442 spin_unlock_bh(&ioat->desc_lock);
443 461
444 pci_pool_free(ioatdma_device->completion_pool, 462 dev_dbg(to_dev(ioat_chan), "%s: allocated %d descriptors\n",
445 chan->completion, 463 __func__, new_size);
446 chan->completion_dma);
447 464
448 /* one is ok since we left it on there on purpose */ 465 kfree(ioat_chan->ring);
449 if (in_use_descs > 1) 466 ioat_chan->ring = ring;
450 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n", 467 ioat_chan->alloc_order = order;
451 in_use_descs - 1);
452 468
453 chan->last_completion = 0; 469 return true;
454 chan->completion_dma = 0;
455 ioat->pending = 0;
456 ioat->desccount = 0;
457} 470}
458 471
459/** 472/**
460 * ioat1_dma_get_next_descriptor - return the next available descriptor 473 * ioat_check_space_lock - verify space and grab ring producer lock
461 * @ioat: IOAT DMA channel handle 474 * @ioat: ioat,3 channel (ring) to operate on
462 * 475 * @num_descs: allocation length
463 * Gets the next descriptor from the chain, and must be called with the
464 * channel's desc_lock held. Allocates more descriptors if the channel
465 * has run out.
466 */ 476 */
467static struct ioat_desc_sw * 477int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs)
468ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat) 478 __acquires(&ioat_chan->prep_lock)
469{ 479{
470 struct ioat_desc_sw *new; 480 bool retry;
471 481
472 if (!list_empty(&ioat->free_desc)) { 482 retry:
473 new = to_ioat_desc(ioat->free_desc.next); 483 spin_lock_bh(&ioat_chan->prep_lock);
474 list_del(&new->node); 484 /* never allow the last descriptor to be consumed, we need at
475 } else { 485 * least one free at all times to allow for on-the-fly ring
476 /* try to get another desc */ 486 * resizing.
477 new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC); 487 */
478 if (!new) { 488 if (likely(ioat_ring_space(ioat_chan) > num_descs)) {
479 dev_err(to_dev(&ioat->base), "alloc failed\n"); 489 dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n",
480 return NULL; 490 __func__, num_descs, ioat_chan->head,
481 } 491 ioat_chan->tail, ioat_chan->issued);
492 ioat_chan->produce = num_descs;
493 return 0; /* with ioat->prep_lock held */
494 }
495 retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
496 spin_unlock_bh(&ioat_chan->prep_lock);
497
498 /* is another cpu already trying to expand the ring? */
499 if (retry)
500 goto retry;
501
502 spin_lock_bh(&ioat_chan->cleanup_lock);
503 spin_lock_bh(&ioat_chan->prep_lock);
504 retry = reshape_ring(ioat_chan, ioat_chan->alloc_order + 1);
505 clear_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state);
506 spin_unlock_bh(&ioat_chan->prep_lock);
507 spin_unlock_bh(&ioat_chan->cleanup_lock);
508
509 /* if we were able to expand the ring retry the allocation */
510 if (retry)
511 goto retry;
512
513 dev_dbg_ratelimited(to_dev(ioat_chan),
514 "%s: ring full! num_descs: %d (%x:%x:%x)\n",
515 __func__, num_descs, ioat_chan->head,
516 ioat_chan->tail, ioat_chan->issued);
517
518 /* progress reclaim in the allocation failure case we may be
519 * called under bh_disabled so we need to trigger the timer
520 * event directly
521 */
522 if (time_is_before_jiffies(ioat_chan->timer.expires)
523 && timer_pending(&ioat_chan->timer)) {
524 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
525 ioat_timer_event((unsigned long)ioat_chan);
482 } 526 }
483 dev_dbg(to_dev(&ioat->base), "%s: allocated: %d\n", 527
484 __func__, desc_id(new)); 528 return -ENOMEM;
485 prefetch(new->hw);
486 return new;
487} 529}
488 530
489static struct dma_async_tx_descriptor * 531static bool desc_has_ext(struct ioat_ring_ent *desc)
490ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
491 dma_addr_t dma_src, size_t len, unsigned long flags)
492{ 532{
493 struct ioat_dma_chan *ioat = to_ioat_chan(c); 533 struct ioat_dma_descriptor *hw = desc->hw;
494 struct ioat_desc_sw *desc;
495 size_t copy;
496 LIST_HEAD(chain);
497 dma_addr_t src = dma_src;
498 dma_addr_t dest = dma_dest;
499 size_t total_len = len;
500 struct ioat_dma_descriptor *hw = NULL;
501 int tx_cnt = 0;
502
503 spin_lock_bh(&ioat->desc_lock);
504 desc = ioat1_dma_get_next_descriptor(ioat);
505 do {
506 if (!desc)
507 break;
508
509 tx_cnt++;
510 copy = min_t(size_t, len, ioat->xfercap);
511 534
512 hw = desc->hw; 535 if (hw->ctl_f.op == IOAT_OP_XOR ||
513 hw->size = copy; 536 hw->ctl_f.op == IOAT_OP_XOR_VAL) {
514 hw->ctl = 0; 537 struct ioat_xor_descriptor *xor = desc->xor;
515 hw->src_addr = src;
516 hw->dst_addr = dest;
517 538
518 list_add_tail(&desc->node, &chain); 539 if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
540 return true;
541 } else if (hw->ctl_f.op == IOAT_OP_PQ ||
542 hw->ctl_f.op == IOAT_OP_PQ_VAL) {
543 struct ioat_pq_descriptor *pq = desc->pq;
519 544
520 len -= copy; 545 if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
521 dest += copy; 546 return true;
522 src += copy;
523 if (len) {
524 struct ioat_desc_sw *next;
525
526 async_tx_ack(&desc->txd);
527 next = ioat1_dma_get_next_descriptor(ioat);
528 hw->next = next ? next->txd.phys : 0;
529 dump_desc_dbg(ioat, desc);
530 desc = next;
531 } else
532 hw->next = 0;
533 } while (len);
534
535 if (!desc) {
536 struct ioat_chan_common *chan = &ioat->base;
537
538 dev_err(to_dev(chan),
539 "chan%d - get_next_desc failed\n", chan_num(chan));
540 list_splice(&chain, &ioat->free_desc);
541 spin_unlock_bh(&ioat->desc_lock);
542 return NULL;
543 } 547 }
544 spin_unlock_bh(&ioat->desc_lock);
545 548
546 desc->txd.flags = flags; 549 return false;
547 desc->len = total_len;
548 list_splice(&chain, &desc->tx_list);
549 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
550 hw->ctl_f.compl_write = 1;
551 hw->tx_cnt = tx_cnt;
552 dump_desc_dbg(ioat, desc);
553
554 return &desc->txd;
555} 550}
556 551
557static void ioat1_cleanup_event(unsigned long data) 552static void
553ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed)
558{ 554{
559 struct ioat_dma_chan *ioat = to_ioat_chan((void *) data); 555 if (!sed)
560 struct ioat_chan_common *chan = &ioat->base;
561
562 ioat1_cleanup(ioat);
563 if (!test_bit(IOAT_RUN, &chan->state))
564 return; 556 return;
565 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); 557
558 dma_pool_free(ioat_dma->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma);
559 kmem_cache_free(ioat_sed_cache, sed);
566} 560}
567 561
568dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) 562static u64 ioat_get_current_completion(struct ioatdma_chan *ioat_chan)
569{ 563{
570 dma_addr_t phys_complete; 564 u64 phys_complete;
571 u64 completion; 565 u64 completion;
572 566
573 completion = *chan->completion; 567 completion = *ioat_chan->completion;
574 phys_complete = ioat_chansts_to_addr(completion); 568 phys_complete = ioat_chansts_to_addr(completion);
575 569
576 dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__, 570 dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__,
577 (unsigned long long) phys_complete); 571 (unsigned long long) phys_complete);
578 572
579 if (is_ioat_halted(completion)) {
580 u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
581 dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
582 chanerr);
583
584 /* TODO do something to salvage the situation */
585 }
586
587 return phys_complete; 573 return phys_complete;
588} 574}
589 575
590bool ioat_cleanup_preamble(struct ioat_chan_common *chan, 576static bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan,
591 dma_addr_t *phys_complete) 577 u64 *phys_complete)
592{ 578{
593 *phys_complete = ioat_get_current_completion(chan); 579 *phys_complete = ioat_get_current_completion(ioat_chan);
594 if (*phys_complete == chan->last_completion) 580 if (*phys_complete == ioat_chan->last_completion)
595 return false; 581 return false;
596 clear_bit(IOAT_COMPLETION_ACK, &chan->state); 582
597 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 583 clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
584 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
598 585
599 return true; 586 return true;
600} 587}
601 588
602static void __cleanup(struct ioat_dma_chan *ioat, dma_addr_t phys_complete) 589static void
590desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc)
603{ 591{
604 struct ioat_chan_common *chan = &ioat->base; 592 struct ioat_dma_descriptor *hw = desc->hw;
605 struct list_head *_desc, *n;
606 struct dma_async_tx_descriptor *tx;
607 593
608 dev_dbg(to_dev(chan), "%s: phys_complete: %llx\n", 594 switch (hw->ctl_f.op) {
609 __func__, (unsigned long long) phys_complete); 595 case IOAT_OP_PQ_VAL:
610 list_for_each_safe(_desc, n, &ioat->used_desc) { 596 case IOAT_OP_PQ_VAL_16S:
611 struct ioat_desc_sw *desc; 597 {
598 struct ioat_pq_descriptor *pq = desc->pq;
612 599
613 prefetch(n); 600 /* check if there's error written */
614 desc = list_entry(_desc, typeof(*desc), node); 601 if (!pq->dwbes_f.wbes)
615 tx = &desc->txd; 602 return;
616 /*
617 * Incoming DMA requests may use multiple descriptors,
618 * due to exceeding xfercap, perhaps. If so, only the
619 * last one will have a cookie, and require unmapping.
620 */
621 dump_desc_dbg(ioat, desc);
622 if (tx->cookie) {
623 dma_cookie_complete(tx);
624 dma_descriptor_unmap(tx);
625 ioat->active -= desc->hw->tx_cnt;
626 if (tx->callback) {
627 tx->callback(tx->callback_param);
628 tx->callback = NULL;
629 }
630 }
631 603
632 if (tx->phys != phys_complete) { 604 /* need to set a chanerr var for checking to clear later */
633 /*
634 * a completed entry, but not the last, so clean
635 * up if the client is done with the descriptor
636 */
637 if (async_tx_test_ack(tx))
638 list_move_tail(&desc->node, &ioat->free_desc);
639 } else {
640 /*
641 * last used desc. Do not remove, so we can
642 * append from it.
643 */
644
645 /* if nothing else is pending, cancel the
646 * completion timeout
647 */
648 if (n == &ioat->used_desc) {
649 dev_dbg(to_dev(chan),
650 "%s cancel completion timeout\n",
651 __func__);
652 clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
653 }
654 605
655 /* TODO check status bits? */ 606 if (pq->dwbes_f.p_val_err)
656 break; 607 *desc->result |= SUM_CHECK_P_RESULT;
657 } 608
658 } 609 if (pq->dwbes_f.q_val_err)
610 *desc->result |= SUM_CHECK_Q_RESULT;
659 611
660 chan->last_completion = phys_complete; 612 return;
613 }
614 default:
615 return;
616 }
661} 617}
662 618
663/** 619/**
664 * ioat1_cleanup - cleanup up finished descriptors 620 * __cleanup - reclaim used descriptors
665 * @chan: ioat channel to be cleaned up 621 * @ioat: channel (ring) to clean
666 *
667 * To prevent lock contention we defer cleanup when the locks are
668 * contended with a terminal timeout that forces cleanup and catches
669 * completion notification errors.
670 */ 622 */
671static void ioat1_cleanup(struct ioat_dma_chan *ioat) 623static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
672{ 624{
673 struct ioat_chan_common *chan = &ioat->base; 625 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
674 dma_addr_t phys_complete; 626 struct ioat_ring_ent *desc;
675 627 bool seen_current = false;
676 prefetch(chan->completion); 628 int idx = ioat_chan->tail, i;
677 629 u16 active;
678 if (!spin_trylock_bh(&chan->cleanup_lock))
679 return;
680 630
681 if (!ioat_cleanup_preamble(chan, &phys_complete)) { 631 dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n",
682 spin_unlock_bh(&chan->cleanup_lock); 632 __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued);
683 return;
684 }
685 633
686 if (!spin_trylock_bh(&ioat->desc_lock)) { 634 /*
687 spin_unlock_bh(&chan->cleanup_lock); 635 * At restart of the channel, the completion address and the
636 * channel status will be 0 due to starting a new chain. Since
637 * it's new chain and the first descriptor "fails", there is
638 * nothing to clean up. We do not want to reap the entire submitted
639 * chain due to this 0 address value and then BUG.
640 */
641 if (!phys_complete)
688 return; 642 return;
689 }
690 643
691 __cleanup(ioat, phys_complete); 644 active = ioat_ring_active(ioat_chan);
645 for (i = 0; i < active && !seen_current; i++) {
646 struct dma_async_tx_descriptor *tx;
692 647
693 spin_unlock_bh(&ioat->desc_lock); 648 smp_read_barrier_depends();
694 spin_unlock_bh(&chan->cleanup_lock); 649 prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
695} 650 desc = ioat_get_ring_ent(ioat_chan, idx + i);
696 651 dump_desc_dbg(ioat_chan, desc);
697static void ioat1_timer_event(unsigned long data)
698{
699 struct ioat_dma_chan *ioat = to_ioat_chan((void *) data);
700 struct ioat_chan_common *chan = &ioat->base;
701 652
702 dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state); 653 /* set err stat if we are using dwbes */
654 if (ioat_dma->cap & IOAT_CAP_DWBES)
655 desc_get_errstat(ioat_chan, desc);
703 656
704 spin_lock_bh(&chan->cleanup_lock); 657 tx = &desc->txd;
705 if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) { 658 if (tx->cookie) {
706 struct ioat_desc_sw *desc; 659 dma_cookie_complete(tx);
707 660 dma_descriptor_unmap(tx);
708 spin_lock_bh(&ioat->desc_lock); 661 if (tx->callback) {
662 tx->callback(tx->callback_param);
663 tx->callback = NULL;
664 }
665 }
709 666
710 /* restart active descriptors */ 667 if (tx->phys == phys_complete)
711 desc = to_ioat_desc(ioat->used_desc.prev); 668 seen_current = true;
712 ioat_set_chainaddr(ioat, desc->txd.phys);
713 ioat_start(chan);
714 669
715 ioat->pending = 0; 670 /* skip extended descriptors */
716 set_bit(IOAT_COMPLETION_PENDING, &chan->state); 671 if (desc_has_ext(desc)) {
717 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 672 BUG_ON(i + 1 >= active);
718 spin_unlock_bh(&ioat->desc_lock); 673 i++;
719 } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) { 674 }
720 dma_addr_t phys_complete;
721 675
722 spin_lock_bh(&ioat->desc_lock); 676 /* cleanup super extended descriptors */
723 /* if we haven't made progress and we have already 677 if (desc->sed) {
724 * acknowledged a pending completion once, then be more 678 ioat_free_sed(ioat_dma, desc->sed);
725 * forceful with a restart 679 desc->sed = NULL;
726 */
727 if (ioat_cleanup_preamble(chan, &phys_complete))
728 __cleanup(ioat, phys_complete);
729 else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
730 ioat1_reset_channel(ioat);
731 else {
732 u64 status = ioat_chansts(chan);
733
734 /* manually update the last completion address */
735 if (ioat_chansts_to_addr(status) != 0)
736 *chan->completion = status;
737
738 set_bit(IOAT_COMPLETION_ACK, &chan->state);
739 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
740 } 680 }
741 spin_unlock_bh(&ioat->desc_lock);
742 } 681 }
743 spin_unlock_bh(&chan->cleanup_lock);
744}
745
746enum dma_status
747ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
748 struct dma_tx_state *txstate)
749{
750 struct ioat_chan_common *chan = to_chan_common(c);
751 struct ioatdma_device *device = chan->device;
752 enum dma_status ret;
753 682
754 ret = dma_cookie_status(c, cookie, txstate); 683 /* finish all descriptor reads before incrementing tail */
755 if (ret == DMA_COMPLETE) 684 smp_mb();
756 return ret; 685 ioat_chan->tail = idx + i;
686 /* no active descs have written a completion? */
687 BUG_ON(active && !seen_current);
688 ioat_chan->last_completion = phys_complete;
757 689
758 device->cleanup_fn((unsigned long) c); 690 if (active - i == 0) {
691 dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n",
692 __func__);
693 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
694 }
759 695
760 return dma_cookie_status(c, cookie, txstate); 696 /* 5 microsecond delay per pending descriptor */
697 writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK),
698 ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET);
761} 699}
762 700
763static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat) 701static void ioat_cleanup(struct ioatdma_chan *ioat_chan)
764{ 702{
765 struct ioat_chan_common *chan = &ioat->base; 703 u64 phys_complete;
766 struct ioat_desc_sw *desc;
767 struct ioat_dma_descriptor *hw;
768 704
769 spin_lock_bh(&ioat->desc_lock); 705 spin_lock_bh(&ioat_chan->cleanup_lock);
770 706
771 desc = ioat1_dma_get_next_descriptor(ioat); 707 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
708 __cleanup(ioat_chan, phys_complete);
772 709
773 if (!desc) { 710 if (is_ioat_halted(*ioat_chan->completion)) {
774 dev_err(to_dev(chan), 711 u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
775 "Unable to start null desc - get next desc failed\n");
776 spin_unlock_bh(&ioat->desc_lock);
777 return;
778 }
779 712
780 hw = desc->hw; 713 if (chanerr & IOAT_CHANERR_HANDLE_MASK) {
781 hw->ctl = 0; 714 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
782 hw->ctl_f.null = 1; 715 ioat_eh(ioat_chan);
783 hw->ctl_f.int_en = 1; 716 }
784 hw->ctl_f.compl_write = 1; 717 }
785 /* set size to non-zero value (channel returns error when size is 0) */
786 hw->size = NULL_DESC_BUFFER_SIZE;
787 hw->src_addr = 0;
788 hw->dst_addr = 0;
789 async_tx_ack(&desc->txd);
790 hw->next = 0;
791 list_add_tail(&desc->node, &ioat->used_desc);
792 dump_desc_dbg(ioat, desc);
793 718
794 ioat_set_chainaddr(ioat, desc->txd.phys); 719 spin_unlock_bh(&ioat_chan->cleanup_lock);
795 ioat_start(chan);
796 spin_unlock_bh(&ioat->desc_lock);
797} 720}
798 721
799/* 722void ioat_cleanup_event(unsigned long data)
800 * Perform a IOAT transaction to verify the HW works. 723{
801 */ 724 struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data);
802#define IOAT_TEST_SIZE 2000 725
726 ioat_cleanup(ioat_chan);
727 if (!test_bit(IOAT_RUN, &ioat_chan->state))
728 return;
729 writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
730}
803 731
804static void ioat_dma_test_callback(void *dma_async_param) 732static void ioat_restart_channel(struct ioatdma_chan *ioat_chan)
805{ 733{
806 struct completion *cmp = dma_async_param; 734 u64 phys_complete;
735
736 ioat_quiesce(ioat_chan, 0);
737 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
738 __cleanup(ioat_chan, phys_complete);
807 739
808 complete(cmp); 740 __ioat_restart_chan(ioat_chan);
809} 741}
810 742
811/** 743static void ioat_eh(struct ioatdma_chan *ioat_chan)
812 * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
813 * @device: device to be tested
814 */
815int ioat_dma_self_test(struct ioatdma_device *device)
816{ 744{
817 int i; 745 struct pci_dev *pdev = to_pdev(ioat_chan);
818 u8 *src; 746 struct ioat_dma_descriptor *hw;
819 u8 *dest;
820 struct dma_device *dma = &device->common;
821 struct device *dev = &device->pdev->dev;
822 struct dma_chan *dma_chan;
823 struct dma_async_tx_descriptor *tx; 747 struct dma_async_tx_descriptor *tx;
824 dma_addr_t dma_dest, dma_src; 748 u64 phys_complete;
825 dma_cookie_t cookie; 749 struct ioat_ring_ent *desc;
826 int err = 0; 750 u32 err_handled = 0;
827 struct completion cmp; 751 u32 chanerr_int;
828 unsigned long tmo; 752 u32 chanerr;
829 unsigned long flags;
830
831 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
832 if (!src)
833 return -ENOMEM;
834 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
835 if (!dest) {
836 kfree(src);
837 return -ENOMEM;
838 }
839 753
840 /* Fill in src buffer */ 754 /* cleanup so tail points to descriptor that caused the error */
841 for (i = 0; i < IOAT_TEST_SIZE; i++) 755 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
842 src[i] = (u8)i; 756 __cleanup(ioat_chan, phys_complete);
843
844 /* Start copy, using first DMA channel */
845 dma_chan = container_of(dma->channels.next, struct dma_chan,
846 device_node);
847 if (dma->device_alloc_chan_resources(dma_chan) < 1) {
848 dev_err(dev, "selftest cannot allocate chan resource\n");
849 err = -ENODEV;
850 goto out;
851 }
852 757
853 dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); 758 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
854 if (dma_mapping_error(dev, dma_src)) { 759 pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int);
855 dev_err(dev, "mapping src buffer failed\n");
856 goto free_resources;
857 }
858 dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
859 if (dma_mapping_error(dev, dma_dest)) {
860 dev_err(dev, "mapping dest buffer failed\n");
861 goto unmap_src;
862 }
863 flags = DMA_PREP_INTERRUPT;
864 tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
865 IOAT_TEST_SIZE, flags);
866 if (!tx) {
867 dev_err(dev, "Self-test prep failed, disabling\n");
868 err = -ENODEV;
869 goto unmap_dma;
870 }
871 760
872 async_tx_ack(tx); 761 dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n",
873 init_completion(&cmp); 762 __func__, chanerr, chanerr_int);
874 tx->callback = ioat_dma_test_callback;
875 tx->callback_param = &cmp;
876 cookie = tx->tx_submit(tx);
877 if (cookie < 0) {
878 dev_err(dev, "Self-test setup failed, disabling\n");
879 err = -ENODEV;
880 goto unmap_dma;
881 }
882 dma->device_issue_pending(dma_chan);
883 763
884 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); 764 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail);
765 hw = desc->hw;
766 dump_desc_dbg(ioat_chan, desc);
885 767
886 if (tmo == 0 || 768 switch (hw->ctl_f.op) {
887 dma->device_tx_status(dma_chan, cookie, NULL) 769 case IOAT_OP_XOR_VAL:
888 != DMA_COMPLETE) { 770 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
889 dev_err(dev, "Self-test copy timed out, disabling\n"); 771 *desc->result |= SUM_CHECK_P_RESULT;
890 err = -ENODEV; 772 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
891 goto unmap_dma; 773 }
892 } 774 break;
893 if (memcmp(src, dest, IOAT_TEST_SIZE)) { 775 case IOAT_OP_PQ_VAL:
894 dev_err(dev, "Self-test copy failed compare, disabling\n"); 776 case IOAT_OP_PQ_VAL_16S:
895 err = -ENODEV; 777 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
896 goto free_resources; 778 *desc->result |= SUM_CHECK_P_RESULT;
779 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
780 }
781 if (chanerr & IOAT_CHANERR_XOR_Q_ERR) {
782 *desc->result |= SUM_CHECK_Q_RESULT;
783 err_handled |= IOAT_CHANERR_XOR_Q_ERR;
784 }
785 break;
897 } 786 }
898 787
899unmap_dma: 788 /* fault on unhandled error or spurious halt */
900 dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); 789 if (chanerr ^ err_handled || chanerr == 0) {
901unmap_src: 790 dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n",
902 dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE); 791 __func__, chanerr, err_handled);
903free_resources: 792 BUG();
904 dma->device_free_chan_resources(dma_chan); 793 } else { /* cleanup the faulty descriptor */
905out: 794 tx = &desc->txd;
906 kfree(src); 795 if (tx->cookie) {
907 kfree(dest); 796 dma_cookie_complete(tx);
908 return err; 797 dma_descriptor_unmap(tx);
909} 798 if (tx->callback) {
910 799 tx->callback(tx->callback_param);
911static char ioat_interrupt_style[32] = "msix"; 800 tx->callback = NULL;
912module_param_string(ioat_interrupt_style, ioat_interrupt_style,
913 sizeof(ioat_interrupt_style), 0644);
914MODULE_PARM_DESC(ioat_interrupt_style,
915 "set ioat interrupt style: msix (default), msi, intx");
916
917/**
918 * ioat_dma_setup_interrupts - setup interrupt handler
919 * @device: ioat device
920 */
921int ioat_dma_setup_interrupts(struct ioatdma_device *device)
922{
923 struct ioat_chan_common *chan;
924 struct pci_dev *pdev = device->pdev;
925 struct device *dev = &pdev->dev;
926 struct msix_entry *msix;
927 int i, j, msixcnt;
928 int err = -EINVAL;
929 u8 intrctrl = 0;
930
931 if (!strcmp(ioat_interrupt_style, "msix"))
932 goto msix;
933 if (!strcmp(ioat_interrupt_style, "msi"))
934 goto msi;
935 if (!strcmp(ioat_interrupt_style, "intx"))
936 goto intx;
937 dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
938 goto err_no_irq;
939
940msix:
941 /* The number of MSI-X vectors should equal the number of channels */
942 msixcnt = device->common.chancnt;
943 for (i = 0; i < msixcnt; i++)
944 device->msix_entries[i].entry = i;
945
946 err = pci_enable_msix_exact(pdev, device->msix_entries, msixcnt);
947 if (err)
948 goto msi;
949
950 for (i = 0; i < msixcnt; i++) {
951 msix = &device->msix_entries[i];
952 chan = ioat_chan_by_index(device, i);
953 err = devm_request_irq(dev, msix->vector,
954 ioat_dma_do_interrupt_msix, 0,
955 "ioat-msix", chan);
956 if (err) {
957 for (j = 0; j < i; j++) {
958 msix = &device->msix_entries[j];
959 chan = ioat_chan_by_index(device, j);
960 devm_free_irq(dev, msix->vector, chan);
961 } 801 }
962 goto msi;
963 } 802 }
964 } 803 }
965 intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
966 device->irq_mode = IOAT_MSIX;
967 goto done;
968 804
969msi: 805 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
970 err = pci_enable_msi(pdev); 806 pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int);
971 if (err)
972 goto intx;
973 807
974 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0, 808 /* mark faulting descriptor as complete */
975 "ioat-msi", device); 809 *ioat_chan->completion = desc->txd.phys;
976 if (err) {
977 pci_disable_msi(pdev);
978 goto intx;
979 }
980 device->irq_mode = IOAT_MSI;
981 goto done;
982
983intx:
984 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
985 IRQF_SHARED, "ioat-intx", device);
986 if (err)
987 goto err_no_irq;
988
989 device->irq_mode = IOAT_INTX;
990done:
991 if (device->intr_quirk)
992 device->intr_quirk(device);
993 intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
994 writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
995 return 0;
996
997err_no_irq:
998 /* Disable all interrupt generation */
999 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1000 device->irq_mode = IOAT_NOIRQ;
1001 dev_err(dev, "no usable interrupts\n");
1002 return err;
1003}
1004EXPORT_SYMBOL(ioat_dma_setup_interrupts);
1005 810
1006static void ioat_disable_interrupts(struct ioatdma_device *device) 811 spin_lock_bh(&ioat_chan->prep_lock);
1007{ 812 ioat_restart_channel(ioat_chan);
1008 /* Disable all interrupt generation */ 813 spin_unlock_bh(&ioat_chan->prep_lock);
1009 writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1010} 814}
1011 815
1012int ioat_probe(struct ioatdma_device *device) 816static void check_active(struct ioatdma_chan *ioat_chan)
1013{ 817{
1014 int err = -ENODEV; 818 if (ioat_ring_active(ioat_chan)) {
1015 struct dma_device *dma = &device->common; 819 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
1016 struct pci_dev *pdev = device->pdev; 820 return;
1017 struct device *dev = &pdev->dev;
1018
1019 /* DMA coherent memory pool for DMA descriptor allocations */
1020 device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
1021 sizeof(struct ioat_dma_descriptor),
1022 64, 0);
1023 if (!device->dma_pool) {
1024 err = -ENOMEM;
1025 goto err_dma_pool;
1026 }
1027
1028 device->completion_pool = pci_pool_create("completion_pool", pdev,
1029 sizeof(u64), SMP_CACHE_BYTES,
1030 SMP_CACHE_BYTES);
1031
1032 if (!device->completion_pool) {
1033 err = -ENOMEM;
1034 goto err_completion_pool;
1035 } 821 }
1036 822
1037 device->enumerate_channels(device); 823 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state))
1038 824 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
1039 dma_cap_set(DMA_MEMCPY, dma->cap_mask); 825 else if (ioat_chan->alloc_order > ioat_get_alloc_order()) {
1040 dma->dev = &pdev->dev; 826 /* if the ring is idle, empty, and oversized try to step
827 * down the size
828 */
829 reshape_ring(ioat_chan, ioat_chan->alloc_order - 1);
1041 830
1042 if (!dma->chancnt) { 831 /* keep shrinking until we get back to our minimum
1043 dev_err(dev, "channel enumeration error\n"); 832 * default size
1044 goto err_setup_interrupts; 833 */
834 if (ioat_chan->alloc_order > ioat_get_alloc_order())
835 mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT);
1045 } 836 }
1046 837
1047 err = ioat_dma_setup_interrupts(device); 838}
1048 if (err)
1049 goto err_setup_interrupts;
1050 839
1051 err = device->self_test(device); 840void ioat_timer_event(unsigned long data)
1052 if (err) 841{
1053 goto err_self_test; 842 struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data);
843 dma_addr_t phys_complete;
844 u64 status;
1054 845
1055 return 0; 846 status = ioat_chansts(ioat_chan);
1056 847
1057err_self_test: 848 /* when halted due to errors check for channel
1058 ioat_disable_interrupts(device); 849 * programming errors before advancing the completion state
1059err_setup_interrupts: 850 */
1060 pci_pool_destroy(device->completion_pool); 851 if (is_ioat_halted(status)) {
1061err_completion_pool: 852 u32 chanerr;
1062 pci_pool_destroy(device->dma_pool);
1063err_dma_pool:
1064 return err;
1065}
1066 853
1067int ioat_register(struct ioatdma_device *device) 854 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
1068{ 855 dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n",
1069 int err = dma_async_device_register(&device->common); 856 __func__, chanerr);
857 if (test_bit(IOAT_RUN, &ioat_chan->state))
858 BUG_ON(is_ioat_bug(chanerr));
859 else /* we never got off the ground */
860 return;
861 }
1070 862
1071 if (err) { 863 /* if we haven't made progress and we have already
1072 ioat_disable_interrupts(device); 864 * acknowledged a pending completion once, then be more
1073 pci_pool_destroy(device->completion_pool); 865 * forceful with a restart
1074 pci_pool_destroy(device->dma_pool); 866 */
867 spin_lock_bh(&ioat_chan->cleanup_lock);
868 if (ioat_cleanup_preamble(ioat_chan, &phys_complete))
869 __cleanup(ioat_chan, phys_complete);
870 else if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) {
871 spin_lock_bh(&ioat_chan->prep_lock);
872 ioat_restart_channel(ioat_chan);
873 spin_unlock_bh(&ioat_chan->prep_lock);
874 spin_unlock_bh(&ioat_chan->cleanup_lock);
875 return;
876 } else {
877 set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state);
878 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
1075 } 879 }
1076 880
1077 return err;
1078}
1079 881
1080/* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */ 882 if (ioat_ring_active(ioat_chan))
1081static void ioat1_intr_quirk(struct ioatdma_device *device) 883 mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT);
1082{ 884 else {
1083 struct pci_dev *pdev = device->pdev; 885 spin_lock_bh(&ioat_chan->prep_lock);
1084 u32 dmactrl; 886 check_active(ioat_chan);
1085 887 spin_unlock_bh(&ioat_chan->prep_lock);
1086 pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl); 888 }
1087 if (pdev->msi_enabled) 889 spin_unlock_bh(&ioat_chan->cleanup_lock);
1088 dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
1089 else
1090 dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
1091 pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
1092} 890}
1093 891
1094static ssize_t ring_size_show(struct dma_chan *c, char *page) 892enum dma_status
893ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie,
894 struct dma_tx_state *txstate)
1095{ 895{
1096 struct ioat_dma_chan *ioat = to_ioat_chan(c); 896 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
897 enum dma_status ret;
1097 898
1098 return sprintf(page, "%d\n", ioat->desccount); 899 ret = dma_cookie_status(c, cookie, txstate);
1099} 900 if (ret == DMA_COMPLETE)
1100static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); 901 return ret;
1101 902
1102static ssize_t ring_active_show(struct dma_chan *c, char *page) 903 ioat_cleanup(ioat_chan);
1103{
1104 struct ioat_dma_chan *ioat = to_ioat_chan(c);
1105 904
1106 return sprintf(page, "%d\n", ioat->active); 905 return dma_cookie_status(c, cookie, txstate);
1107} 906}
1108static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
1109 907
1110static ssize_t cap_show(struct dma_chan *c, char *page) 908static int ioat_irq_reinit(struct ioatdma_device *ioat_dma)
1111{ 909{
1112 struct dma_device *dma = c->device; 910 struct pci_dev *pdev = ioat_dma->pdev;
911 int irq = pdev->irq, i;
1113 912
1114 return sprintf(page, "copy%s%s%s%s%s\n", 913 if (!is_bwd_ioat(pdev))
1115 dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "", 914 return 0;
1116 dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "",
1117 dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "",
1118 dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "",
1119 dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : "");
1120 915
1121} 916 switch (ioat_dma->irq_mode) {
1122struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap); 917 case IOAT_MSIX:
1123 918 for (i = 0; i < ioat_dma->dma_dev.chancnt; i++) {
1124static ssize_t version_show(struct dma_chan *c, char *page) 919 struct msix_entry *msix = &ioat_dma->msix_entries[i];
1125{ 920 struct ioatdma_chan *ioat_chan;
1126 struct dma_device *dma = c->device;
1127 struct ioatdma_device *device = to_ioatdma_device(dma);
1128 921
1129 return sprintf(page, "%d.%d\n", 922 ioat_chan = ioat_chan_by_index(ioat_dma, i);
1130 device->version >> 4, device->version & 0xf); 923 devm_free_irq(&pdev->dev, msix->vector, ioat_chan);
1131} 924 }
1132struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version);
1133
1134static struct attribute *ioat1_attrs[] = {
1135 &ring_size_attr.attr,
1136 &ring_active_attr.attr,
1137 &ioat_cap_attr.attr,
1138 &ioat_version_attr.attr,
1139 NULL,
1140};
1141
1142static ssize_t
1143ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
1144{
1145 struct ioat_sysfs_entry *entry;
1146 struct ioat_chan_common *chan;
1147 925
1148 entry = container_of(attr, struct ioat_sysfs_entry, attr); 926 pci_disable_msix(pdev);
1149 chan = container_of(kobj, struct ioat_chan_common, kobj); 927 break;
928 case IOAT_MSI:
929 pci_disable_msi(pdev);
930 /* fall through */
931 case IOAT_INTX:
932 devm_free_irq(&pdev->dev, irq, ioat_dma);
933 break;
934 default:
935 return 0;
936 }
937 ioat_dma->irq_mode = IOAT_NOIRQ;
1150 938
1151 if (!entry->show) 939 return ioat_dma_setup_interrupts(ioat_dma);
1152 return -EIO;
1153 return entry->show(&chan->common, page);
1154} 940}
1155 941
1156const struct sysfs_ops ioat_sysfs_ops = { 942int ioat_reset_hw(struct ioatdma_chan *ioat_chan)
1157 .show = ioat_attr_show,
1158};
1159
1160static struct kobj_type ioat1_ktype = {
1161 .sysfs_ops = &ioat_sysfs_ops,
1162 .default_attrs = ioat1_attrs,
1163};
1164
1165void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type)
1166{ 943{
1167 struct dma_device *dma = &device->common; 944 /* throw away whatever the channel was doing and get it
1168 struct dma_chan *c; 945 * initialized, with ioat3 specific workarounds
946 */
947 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
948 struct pci_dev *pdev = ioat_dma->pdev;
949 u32 chanerr;
950 u16 dev_id;
951 int err;
952
953 ioat_quiesce(ioat_chan, msecs_to_jiffies(100));
1169 954
1170 list_for_each_entry(c, &dma->channels, device_node) { 955 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
1171 struct ioat_chan_common *chan = to_chan_common(c); 956 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
1172 struct kobject *parent = &c->dev->device.kobj;
1173 int err;
1174 957
1175 err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata"); 958 if (ioat_dma->version < IOAT_VER_3_3) {
959 /* clear any pending errors */
960 err = pci_read_config_dword(pdev,
961 IOAT_PCI_CHANERR_INT_OFFSET, &chanerr);
1176 if (err) { 962 if (err) {
1177 dev_warn(to_dev(chan), 963 dev_err(&pdev->dev,
1178 "sysfs init error (%d), continuing...\n", err); 964 "channel error register unreachable\n");
1179 kobject_put(&chan->kobj); 965 return err;
1180 set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state);
1181 } 966 }
1182 } 967 pci_write_config_dword(pdev,
1183} 968 IOAT_PCI_CHANERR_INT_OFFSET, chanerr);
1184 969
1185void ioat_kobject_del(struct ioatdma_device *device) 970 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1186{ 971 * (workaround for spurious config parity error after restart)
1187 struct dma_device *dma = &device->common; 972 */
1188 struct dma_chan *c; 973 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1189 974 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
1190 list_for_each_entry(c, &dma->channels, device_node) { 975 pci_write_config_dword(pdev,
1191 struct ioat_chan_common *chan = to_chan_common(c); 976 IOAT_PCI_DMAUNCERRSTS_OFFSET,
1192 977 0x10);
1193 if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) {
1194 kobject_del(&chan->kobj);
1195 kobject_put(&chan->kobj);
1196 } 978 }
1197 } 979 }
1198}
1199 980
1200int ioat1_dma_probe(struct ioatdma_device *device, int dca) 981 err = ioat_reset_sync(ioat_chan, msecs_to_jiffies(200));
1201{ 982 if (!err)
1202 struct pci_dev *pdev = device->pdev; 983 err = ioat_irq_reinit(ioat_dma);
1203 struct dma_device *dma;
1204 int err;
1205 984
1206 device->intr_quirk = ioat1_intr_quirk;
1207 device->enumerate_channels = ioat1_enumerate_channels;
1208 device->self_test = ioat_dma_self_test;
1209 device->timer_fn = ioat1_timer_event;
1210 device->cleanup_fn = ioat1_cleanup_event;
1211 dma = &device->common;
1212 dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1213 dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1214 dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources;
1215 dma->device_free_chan_resources = ioat1_dma_free_chan_resources;
1216 dma->device_tx_status = ioat_dma_tx_status;
1217
1218 err = ioat_probe(device);
1219 if (err)
1220 return err;
1221 err = ioat_register(device);
1222 if (err) 985 if (err)
1223 return err; 986 dev_err(&pdev->dev, "Failed to reset: %d\n", err);
1224 ioat_kobject_add(device, &ioat1_ktype);
1225
1226 if (dca)
1227 device->dca = ioat_dca_init(pdev, device->reg_base);
1228 987
1229 return err; 988 return err;
1230} 989}
1231
1232void ioat_dma_remove(struct ioatdma_device *device)
1233{
1234 struct dma_device *dma = &device->common;
1235
1236 ioat_disable_interrupts(device);
1237
1238 ioat_kobject_del(device);
1239
1240 dma_async_device_unregister(dma);
1241
1242 pci_pool_destroy(device->dma_pool);
1243 pci_pool_destroy(device->completion_pool);
1244
1245 INIT_LIST_HEAD(&dma->channels);
1246}
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
index 30f5c7eede16..1bc084986646 100644
--- a/drivers/dma/ioat/dma.h
+++ b/drivers/dma/ioat/dma.h
@@ -18,26 +18,32 @@
18#define IOATDMA_H 18#define IOATDMA_H
19 19
20#include <linux/dmaengine.h> 20#include <linux/dmaengine.h>
21#include "hw.h"
22#include "registers.h"
23#include <linux/init.h> 21#include <linux/init.h>
24#include <linux/dmapool.h> 22#include <linux/dmapool.h>
25#include <linux/cache.h> 23#include <linux/cache.h>
26#include <linux/pci_ids.h> 24#include <linux/pci_ids.h>
27#include <net/tcp.h> 25#include <linux/circ_buf.h>
26#include <linux/interrupt.h>
27#include "registers.h"
28#include "hw.h"
28 29
29#define IOAT_DMA_VERSION "4.00" 30#define IOAT_DMA_VERSION "4.00"
30 31
31#define IOAT_LOW_COMPLETION_MASK 0xffffffc0
32#define IOAT_DMA_DCA_ANY_CPU ~0 32#define IOAT_DMA_DCA_ANY_CPU ~0
33 33
34#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common) 34#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, dma_dev)
35#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) 35#define to_dev(ioat_chan) (&(ioat_chan)->ioat_dma->pdev->dev)
36#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, txd) 36#define to_pdev(ioat_chan) ((ioat_chan)->ioat_dma->pdev)
37#define to_dev(ioat_chan) (&(ioat_chan)->device->pdev->dev) 37
38#define to_pdev(ioat_chan) ((ioat_chan)->device->pdev) 38#define chan_num(ch) ((int)((ch)->reg_base - (ch)->ioat_dma->reg_base) / 0x80)
39 39
40#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80) 40/* ioat hardware assumes at least two sources for raid operations */
41#define src_cnt_to_sw(x) ((x) + 2)
42#define src_cnt_to_hw(x) ((x) - 2)
43#define ndest_to_sw(x) ((x) + 1)
44#define ndest_to_hw(x) ((x) - 1)
45#define src16_cnt_to_sw(x) ((x) + 9)
46#define src16_cnt_to_hw(x) ((x) - 9)
41 47
42/* 48/*
43 * workaround for IOAT ver.3.0 null descriptor issue 49 * workaround for IOAT ver.3.0 null descriptor issue
@@ -57,19 +63,15 @@ enum ioat_irq_mode {
57 * @pdev: PCI-Express device 63 * @pdev: PCI-Express device
58 * @reg_base: MMIO register space base address 64 * @reg_base: MMIO register space base address
59 * @dma_pool: for allocating DMA descriptors 65 * @dma_pool: for allocating DMA descriptors
60 * @common: embedded struct dma_device 66 * @completion_pool: DMA buffers for completion ops
67 * @sed_hw_pool: DMA super descriptor pools
68 * @dma_dev: embedded struct dma_device
61 * @version: version of ioatdma device 69 * @version: version of ioatdma device
62 * @msix_entries: irq handlers 70 * @msix_entries: irq handlers
63 * @idx: per channel data 71 * @idx: per channel data
64 * @dca: direct cache access context 72 * @dca: direct cache access context
65 * @intr_quirk: interrupt setup quirk (for ioat_v1 devices) 73 * @irq_mode: interrupt mode (INTX, MSI, MSIX)
66 * @enumerate_channels: hw version specific channel enumeration 74 * @cap: read DMA capabilities register
67 * @reset_hw: hw version specific channel (re)initialization
68 * @cleanup_fn: select between the v2 and v3 cleanup routines
69 * @timer_fn: select between the v2 and v3 timer watchdog routines
70 * @self_test: hardware version specific self test for each supported op type
71 *
72 * Note: the v3 cleanup routine supports raid operations
73 */ 75 */
74struct ioatdma_device { 76struct ioatdma_device {
75 struct pci_dev *pdev; 77 struct pci_dev *pdev;
@@ -78,28 +80,21 @@ struct ioatdma_device {
78 struct pci_pool *completion_pool; 80 struct pci_pool *completion_pool;
79#define MAX_SED_POOLS 5 81#define MAX_SED_POOLS 5
80 struct dma_pool *sed_hw_pool[MAX_SED_POOLS]; 82 struct dma_pool *sed_hw_pool[MAX_SED_POOLS];
81 struct dma_device common; 83 struct dma_device dma_dev;
82 u8 version; 84 u8 version;
83 struct msix_entry msix_entries[4]; 85 struct msix_entry msix_entries[4];
84 struct ioat_chan_common *idx[4]; 86 struct ioatdma_chan *idx[4];
85 struct dca_provider *dca; 87 struct dca_provider *dca;
86 enum ioat_irq_mode irq_mode; 88 enum ioat_irq_mode irq_mode;
87 u32 cap; 89 u32 cap;
88 void (*intr_quirk)(struct ioatdma_device *device);
89 int (*enumerate_channels)(struct ioatdma_device *device);
90 int (*reset_hw)(struct ioat_chan_common *chan);
91 void (*cleanup_fn)(unsigned long data);
92 void (*timer_fn)(unsigned long data);
93 int (*self_test)(struct ioatdma_device *device);
94}; 90};
95 91
96struct ioat_chan_common { 92struct ioatdma_chan {
97 struct dma_chan common; 93 struct dma_chan dma_chan;
98 void __iomem *reg_base; 94 void __iomem *reg_base;
99 dma_addr_t last_completion; 95 dma_addr_t last_completion;
100 spinlock_t cleanup_lock; 96 spinlock_t cleanup_lock;
101 unsigned long state; 97 unsigned long state;
102 #define IOAT_COMPLETION_PENDING 0
103 #define IOAT_COMPLETION_ACK 1 98 #define IOAT_COMPLETION_ACK 1
104 #define IOAT_RESET_PENDING 2 99 #define IOAT_RESET_PENDING 2
105 #define IOAT_KOBJ_INIT_FAIL 3 100 #define IOAT_KOBJ_INIT_FAIL 3
@@ -110,11 +105,32 @@ struct ioat_chan_common {
110 #define COMPLETION_TIMEOUT msecs_to_jiffies(100) 105 #define COMPLETION_TIMEOUT msecs_to_jiffies(100)
111 #define IDLE_TIMEOUT msecs_to_jiffies(2000) 106 #define IDLE_TIMEOUT msecs_to_jiffies(2000)
112 #define RESET_DELAY msecs_to_jiffies(100) 107 #define RESET_DELAY msecs_to_jiffies(100)
113 struct ioatdma_device *device; 108 struct ioatdma_device *ioat_dma;
114 dma_addr_t completion_dma; 109 dma_addr_t completion_dma;
115 u64 *completion; 110 u64 *completion;
116 struct tasklet_struct cleanup_task; 111 struct tasklet_struct cleanup_task;
117 struct kobject kobj; 112 struct kobject kobj;
113
114/* ioat v2 / v3 channel attributes
115 * @xfercap_log; log2 of channel max transfer length (for fast division)
116 * @head: allocated index
117 * @issued: hardware notification point
118 * @tail: cleanup index
119 * @dmacount: identical to 'head' except for occasionally resetting to zero
120 * @alloc_order: log2 of the number of allocated descriptors
121 * @produce: number of descriptors to produce at submit time
122 * @ring: software ring buffer implementation of hardware ring
123 * @prep_lock: serializes descriptor preparation (producers)
124 */
125 size_t xfercap_log;
126 u16 head;
127 u16 issued;
128 u16 tail;
129 u16 dmacount;
130 u16 alloc_order;
131 u16 produce;
132 struct ioat_ring_ent **ring;
133 spinlock_t prep_lock;
118}; 134};
119 135
120struct ioat_sysfs_entry { 136struct ioat_sysfs_entry {
@@ -123,28 +139,11 @@ struct ioat_sysfs_entry {
123}; 139};
124 140
125/** 141/**
126 * struct ioat_dma_chan - internal representation of a DMA channel
127 */
128struct ioat_dma_chan {
129 struct ioat_chan_common base;
130
131 size_t xfercap; /* XFERCAP register value expanded out */
132
133 spinlock_t desc_lock;
134 struct list_head free_desc;
135 struct list_head used_desc;
136
137 int pending;
138 u16 desccount;
139 u16 active;
140};
141
142/**
143 * struct ioat_sed_ent - wrapper around super extended hardware descriptor 142 * struct ioat_sed_ent - wrapper around super extended hardware descriptor
144 * @hw: hardware SED 143 * @hw: hardware SED
145 * @sed_dma: dma address for the SED 144 * @dma: dma address for the SED
146 * @list: list member
147 * @parent: point to the dma descriptor that's the parent 145 * @parent: point to the dma descriptor that's the parent
146 * @hw_pool: descriptor pool index
148 */ 147 */
149struct ioat_sed_ent { 148struct ioat_sed_ent {
150 struct ioat_sed_raw_descriptor *hw; 149 struct ioat_sed_raw_descriptor *hw;
@@ -153,39 +152,57 @@ struct ioat_sed_ent {
153 unsigned int hw_pool; 152 unsigned int hw_pool;
154}; 153};
155 154
156static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c)
157{
158 return container_of(c, struct ioat_chan_common, common);
159}
160
161static inline struct ioat_dma_chan *to_ioat_chan(struct dma_chan *c)
162{
163 struct ioat_chan_common *chan = to_chan_common(c);
164
165 return container_of(chan, struct ioat_dma_chan, base);
166}
167
168/* wrapper around hardware descriptor format + additional software fields */
169
170/** 155/**
171 * struct ioat_desc_sw - wrapper around hardware descriptor 156 * struct ioat_ring_ent - wrapper around hardware descriptor
172 * @hw: hardware DMA descriptor (for memcpy) 157 * @hw: hardware DMA descriptor (for memcpy)
173 * @node: this descriptor will either be on the free list, 158 * @xor: hardware xor descriptor
174 * or attached to a transaction list (tx_list) 159 * @xor_ex: hardware xor extension descriptor
160 * @pq: hardware pq descriptor
161 * @pq_ex: hardware pq extension descriptor
162 * @pqu: hardware pq update descriptor
163 * @raw: hardware raw (un-typed) descriptor
175 * @txd: the generic software descriptor for all engines 164 * @txd: the generic software descriptor for all engines
165 * @len: total transaction length for unmap
166 * @result: asynchronous result of validate operations
176 * @id: identifier for debug 167 * @id: identifier for debug
168 * @sed: pointer to super extended descriptor sw desc
177 */ 169 */
178struct ioat_desc_sw { 170
179 struct ioat_dma_descriptor *hw; 171struct ioat_ring_ent {
180 struct list_head node; 172 union {
173 struct ioat_dma_descriptor *hw;
174 struct ioat_xor_descriptor *xor;
175 struct ioat_xor_ext_descriptor *xor_ex;
176 struct ioat_pq_descriptor *pq;
177 struct ioat_pq_ext_descriptor *pq_ex;
178 struct ioat_pq_update_descriptor *pqu;
179 struct ioat_raw_descriptor *raw;
180 };
181 size_t len; 181 size_t len;
182 struct list_head tx_list;
183 struct dma_async_tx_descriptor txd; 182 struct dma_async_tx_descriptor txd;
183 enum sum_check_flags *result;
184 #ifdef DEBUG 184 #ifdef DEBUG
185 int id; 185 int id;
186 #endif 186 #endif
187 struct ioat_sed_ent *sed;
187}; 188};
188 189
190extern const struct sysfs_ops ioat_sysfs_ops;
191extern struct ioat_sysfs_entry ioat_version_attr;
192extern struct ioat_sysfs_entry ioat_cap_attr;
193extern int ioat_pending_level;
194extern int ioat_ring_alloc_order;
195extern struct kobj_type ioat_ktype;
196extern struct kmem_cache *ioat_cache;
197extern int ioat_ring_max_alloc_order;
198extern struct kmem_cache *ioat_sed_cache;
199
200static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c)
201{
202 return container_of(c, struct ioatdma_chan, dma_chan);
203}
204
205/* wrapper around hardware descriptor format + additional software fields */
189#ifdef DEBUG 206#ifdef DEBUG
190#define set_desc_id(desc, i) ((desc)->id = (i)) 207#define set_desc_id(desc, i) ((desc)->id = (i))
191#define desc_id(desc) ((desc)->id) 208#define desc_id(desc) ((desc)->id)
@@ -195,10 +212,10 @@ struct ioat_desc_sw {
195#endif 212#endif
196 213
197static inline void 214static inline void
198__dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw, 215__dump_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_dma_descriptor *hw,
199 struct dma_async_tx_descriptor *tx, int id) 216 struct dma_async_tx_descriptor *tx, int id)
200{ 217{
201 struct device *dev = to_dev(chan); 218 struct device *dev = to_dev(ioat_chan);
202 219
203 dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x" 220 dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x"
204 " ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id, 221 " ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id,
@@ -208,25 +225,25 @@ __dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw,
208} 225}
209 226
210#define dump_desc_dbg(c, d) \ 227#define dump_desc_dbg(c, d) \
211 ({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; }) 228 ({ if (d) __dump_desc_dbg(c, d->hw, &d->txd, desc_id(d)); 0; })
212 229
213static inline struct ioat_chan_common * 230static inline struct ioatdma_chan *
214ioat_chan_by_index(struct ioatdma_device *device, int index) 231ioat_chan_by_index(struct ioatdma_device *ioat_dma, int index)
215{ 232{
216 return device->idx[index]; 233 return ioat_dma->idx[index];
217} 234}
218 235
219static inline u64 ioat_chansts_32(struct ioat_chan_common *chan) 236static inline u64 ioat_chansts_32(struct ioatdma_chan *ioat_chan)
220{ 237{
221 u8 ver = chan->device->version; 238 u8 ver = ioat_chan->ioat_dma->version;
222 u64 status; 239 u64 status;
223 u32 status_lo; 240 u32 status_lo;
224 241
225 /* We need to read the low address first as this causes the 242 /* We need to read the low address first as this causes the
226 * chipset to latch the upper bits for the subsequent read 243 * chipset to latch the upper bits for the subsequent read
227 */ 244 */
228 status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver)); 245 status_lo = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
229 status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver)); 246 status = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
230 status <<= 32; 247 status <<= 32;
231 status |= status_lo; 248 status |= status_lo;
232 249
@@ -235,16 +252,16 @@ static inline u64 ioat_chansts_32(struct ioat_chan_common *chan)
235 252
236#if BITS_PER_LONG == 64 253#if BITS_PER_LONG == 64
237 254
238static inline u64 ioat_chansts(struct ioat_chan_common *chan) 255static inline u64 ioat_chansts(struct ioatdma_chan *ioat_chan)
239{ 256{
240 u8 ver = chan->device->version; 257 u8 ver = ioat_chan->ioat_dma->version;
241 u64 status; 258 u64 status;
242 259
243 /* With IOAT v3.3 the status register is 64bit. */ 260 /* With IOAT v3.3 the status register is 64bit. */
244 if (ver >= IOAT_VER_3_3) 261 if (ver >= IOAT_VER_3_3)
245 status = readq(chan->reg_base + IOAT_CHANSTS_OFFSET(ver)); 262 status = readq(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET(ver));
246 else 263 else
247 status = ioat_chansts_32(chan); 264 status = ioat_chansts_32(ioat_chan);
248 265
249 return status; 266 return status;
250} 267}
@@ -253,56 +270,41 @@ static inline u64 ioat_chansts(struct ioat_chan_common *chan)
253#define ioat_chansts ioat_chansts_32 270#define ioat_chansts ioat_chansts_32
254#endif 271#endif
255 272
256static inline void ioat_start(struct ioat_chan_common *chan)
257{
258 u8 ver = chan->device->version;
259
260 writeb(IOAT_CHANCMD_START, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
261}
262
263static inline u64 ioat_chansts_to_addr(u64 status) 273static inline u64 ioat_chansts_to_addr(u64 status)
264{ 274{
265 return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; 275 return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
266} 276}
267 277
268static inline u32 ioat_chanerr(struct ioat_chan_common *chan) 278static inline u32 ioat_chanerr(struct ioatdma_chan *ioat_chan)
269{ 279{
270 return readl(chan->reg_base + IOAT_CHANERR_OFFSET); 280 return readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
271} 281}
272 282
273static inline void ioat_suspend(struct ioat_chan_common *chan) 283static inline void ioat_suspend(struct ioatdma_chan *ioat_chan)
274{ 284{
275 u8 ver = chan->device->version; 285 u8 ver = ioat_chan->ioat_dma->version;
276 286
277 writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); 287 writeb(IOAT_CHANCMD_SUSPEND,
288 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
278} 289}
279 290
280static inline void ioat_reset(struct ioat_chan_common *chan) 291static inline void ioat_reset(struct ioatdma_chan *ioat_chan)
281{ 292{
282 u8 ver = chan->device->version; 293 u8 ver = ioat_chan->ioat_dma->version;
283 294
284 writeb(IOAT_CHANCMD_RESET, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); 295 writeb(IOAT_CHANCMD_RESET,
296 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
285} 297}
286 298
287static inline bool ioat_reset_pending(struct ioat_chan_common *chan) 299static inline bool ioat_reset_pending(struct ioatdma_chan *ioat_chan)
288{ 300{
289 u8 ver = chan->device->version; 301 u8 ver = ioat_chan->ioat_dma->version;
290 u8 cmd; 302 u8 cmd;
291 303
292 cmd = readb(chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); 304 cmd = readb(ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
293 return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET; 305 return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET;
294} 306}
295 307
296static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr)
297{
298 struct ioat_chan_common *chan = &ioat->base;
299
300 writel(addr & 0x00000000FFFFFFFF,
301 chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
302 writel(addr >> 32,
303 chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
304}
305
306static inline bool is_ioat_active(unsigned long status) 308static inline bool is_ioat_active(unsigned long status)
307{ 309{
308 return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE); 310 return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
@@ -329,24 +331,111 @@ static inline bool is_ioat_bug(unsigned long err)
329 return !!err; 331 return !!err;
330} 332}
331 333
332int ioat_probe(struct ioatdma_device *device); 334#define IOAT_MAX_ORDER 16
333int ioat_register(struct ioatdma_device *device); 335#define ioat_get_alloc_order() \
334int ioat1_dma_probe(struct ioatdma_device *dev, int dca); 336 (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
335int ioat_dma_self_test(struct ioatdma_device *device); 337#define ioat_get_max_alloc_order() \
336void ioat_dma_remove(struct ioatdma_device *device); 338 (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
339
340static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan)
341{
342 return 1 << ioat_chan->alloc_order;
343}
344
345/* count of descriptors in flight with the engine */
346static inline u16 ioat_ring_active(struct ioatdma_chan *ioat_chan)
347{
348 return CIRC_CNT(ioat_chan->head, ioat_chan->tail,
349 ioat_ring_size(ioat_chan));
350}
351
352/* count of descriptors pending submission to hardware */
353static inline u16 ioat_ring_pending(struct ioatdma_chan *ioat_chan)
354{
355 return CIRC_CNT(ioat_chan->head, ioat_chan->issued,
356 ioat_ring_size(ioat_chan));
357}
358
359static inline u32 ioat_ring_space(struct ioatdma_chan *ioat_chan)
360{
361 return ioat_ring_size(ioat_chan) - ioat_ring_active(ioat_chan);
362}
363
364static inline u16
365ioat_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len)
366{
367 u16 num_descs = len >> ioat_chan->xfercap_log;
368
369 num_descs += !!(len & ((1 << ioat_chan->xfercap_log) - 1));
370 return num_descs;
371}
372
373static inline struct ioat_ring_ent *
374ioat_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx)
375{
376 return ioat_chan->ring[idx & (ioat_ring_size(ioat_chan) - 1)];
377}
378
379static inline void
380ioat_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr)
381{
382 writel(addr & 0x00000000FFFFFFFF,
383 ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
384 writel(addr >> 32,
385 ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
386}
387
388/* IOAT Prep functions */
389struct dma_async_tx_descriptor *
390ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
391 dma_addr_t dma_src, size_t len, unsigned long flags);
392struct dma_async_tx_descriptor *
393ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags);
394struct dma_async_tx_descriptor *
395ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
396 unsigned int src_cnt, size_t len, unsigned long flags);
397struct dma_async_tx_descriptor *
398ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
399 unsigned int src_cnt, size_t len,
400 enum sum_check_flags *result, unsigned long flags);
401struct dma_async_tx_descriptor *
402ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
403 unsigned int src_cnt, const unsigned char *scf, size_t len,
404 unsigned long flags);
405struct dma_async_tx_descriptor *
406ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
407 unsigned int src_cnt, const unsigned char *scf, size_t len,
408 enum sum_check_flags *pqres, unsigned long flags);
409struct dma_async_tx_descriptor *
410ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
411 unsigned int src_cnt, size_t len, unsigned long flags);
412struct dma_async_tx_descriptor *
413ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
414 unsigned int src_cnt, size_t len,
415 enum sum_check_flags *result, unsigned long flags);
416
417/* IOAT Operation functions */
418irqreturn_t ioat_dma_do_interrupt(int irq, void *data);
419irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data);
420struct ioat_ring_ent **
421ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags);
422void ioat_start_null_desc(struct ioatdma_chan *ioat_chan);
423void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan);
424int ioat_reset_hw(struct ioatdma_chan *ioat_chan);
425enum dma_status
426ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie,
427 struct dma_tx_state *txstate);
428void ioat_cleanup_event(unsigned long data);
429void ioat_timer_event(unsigned long data);
430int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs);
431void ioat_issue_pending(struct dma_chan *chan);
432void ioat_timer_event(unsigned long data);
433
434/* IOAT Init functions */
435bool is_bwd_ioat(struct pci_dev *pdev);
337struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); 436struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
338dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan); 437void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type);
339void ioat_init_channel(struct ioatdma_device *device, 438void ioat_kobject_del(struct ioatdma_device *ioat_dma);
340 struct ioat_chan_common *chan, int idx); 439int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma);
341enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, 440void ioat_stop(struct ioatdma_chan *ioat_chan);
342 struct dma_tx_state *txstate);
343bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
344 dma_addr_t *phys_complete);
345void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type);
346void ioat_kobject_del(struct ioatdma_device *device);
347int ioat_dma_setup_interrupts(struct ioatdma_device *device);
348void ioat_stop(struct ioat_chan_common *chan);
349extern const struct sysfs_ops ioat_sysfs_ops;
350extern struct ioat_sysfs_entry ioat_version_attr;
351extern struct ioat_sysfs_entry ioat_cap_attr;
352#endif /* IOATDMA_H */ 441#endif /* IOATDMA_H */
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
deleted file mode 100644
index 69c7dfcad023..000000000000
--- a/drivers/dma/ioat/dma_v2.c
+++ /dev/null
@@ -1,916 +0,0 @@
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2009 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * The full GNU General Public License is included in this distribution in
15 * the file called "COPYING".
16 *
17 */
18
19/*
20 * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
21 * does asynchronous data movement and checksumming operations.
22 */
23
24#include <linux/init.h>
25#include <linux/module.h>
26#include <linux/slab.h>
27#include <linux/pci.h>
28#include <linux/interrupt.h>
29#include <linux/dmaengine.h>
30#include <linux/delay.h>
31#include <linux/dma-mapping.h>
32#include <linux/workqueue.h>
33#include <linux/prefetch.h>
34#include <linux/i7300_idle.h>
35#include "dma.h"
36#include "dma_v2.h"
37#include "registers.h"
38#include "hw.h"
39
40#include "../dmaengine.h"
41
42int ioat_ring_alloc_order = 8;
43module_param(ioat_ring_alloc_order, int, 0644);
44MODULE_PARM_DESC(ioat_ring_alloc_order,
45 "ioat2+: allocate 2^n descriptors per channel"
46 " (default: 8 max: 16)");
47static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER;
48module_param(ioat_ring_max_alloc_order, int, 0644);
49MODULE_PARM_DESC(ioat_ring_max_alloc_order,
50 "ioat2+: upper limit for ring size (default: 16)");
51
52void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
53{
54 struct ioat_chan_common *chan = &ioat->base;
55
56 ioat->dmacount += ioat2_ring_pending(ioat);
57 ioat->issued = ioat->head;
58 writew(ioat->dmacount, chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
59 dev_dbg(to_dev(chan),
60 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
61 __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
62}
63
64void ioat2_issue_pending(struct dma_chan *c)
65{
66 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
67
68 if (ioat2_ring_pending(ioat)) {
69 spin_lock_bh(&ioat->prep_lock);
70 __ioat2_issue_pending(ioat);
71 spin_unlock_bh(&ioat->prep_lock);
72 }
73}
74
75/**
76 * ioat2_update_pending - log pending descriptors
77 * @ioat: ioat2+ channel
78 *
79 * Check if the number of unsubmitted descriptors has exceeded the
80 * watermark. Called with prep_lock held
81 */
82static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
83{
84 if (ioat2_ring_pending(ioat) > ioat_pending_level)
85 __ioat2_issue_pending(ioat);
86}
87
88static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
89{
90 struct ioat_ring_ent *desc;
91 struct ioat_dma_descriptor *hw;
92
93 if (ioat2_ring_space(ioat) < 1) {
94 dev_err(to_dev(&ioat->base),
95 "Unable to start null desc - ring full\n");
96 return;
97 }
98
99 dev_dbg(to_dev(&ioat->base), "%s: head: %#x tail: %#x issued: %#x\n",
100 __func__, ioat->head, ioat->tail, ioat->issued);
101 desc = ioat2_get_ring_ent(ioat, ioat->head);
102
103 hw = desc->hw;
104 hw->ctl = 0;
105 hw->ctl_f.null = 1;
106 hw->ctl_f.int_en = 1;
107 hw->ctl_f.compl_write = 1;
108 /* set size to non-zero value (channel returns error when size is 0) */
109 hw->size = NULL_DESC_BUFFER_SIZE;
110 hw->src_addr = 0;
111 hw->dst_addr = 0;
112 async_tx_ack(&desc->txd);
113 ioat2_set_chainaddr(ioat, desc->txd.phys);
114 dump_desc_dbg(ioat, desc);
115 wmb();
116 ioat->head += 1;
117 __ioat2_issue_pending(ioat);
118}
119
120static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
121{
122 spin_lock_bh(&ioat->prep_lock);
123 __ioat2_start_null_desc(ioat);
124 spin_unlock_bh(&ioat->prep_lock);
125}
126
127static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)
128{
129 struct ioat_chan_common *chan = &ioat->base;
130 struct dma_async_tx_descriptor *tx;
131 struct ioat_ring_ent *desc;
132 bool seen_current = false;
133 u16 active;
134 int idx = ioat->tail, i;
135
136 dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
137 __func__, ioat->head, ioat->tail, ioat->issued);
138
139 active = ioat2_ring_active(ioat);
140 for (i = 0; i < active && !seen_current; i++) {
141 smp_read_barrier_depends();
142 prefetch(ioat2_get_ring_ent(ioat, idx + i + 1));
143 desc = ioat2_get_ring_ent(ioat, idx + i);
144 tx = &desc->txd;
145 dump_desc_dbg(ioat, desc);
146 if (tx->cookie) {
147 dma_descriptor_unmap(tx);
148 dma_cookie_complete(tx);
149 if (tx->callback) {
150 tx->callback(tx->callback_param);
151 tx->callback = NULL;
152 }
153 }
154
155 if (tx->phys == phys_complete)
156 seen_current = true;
157 }
158 smp_mb(); /* finish all descriptor reads before incrementing tail */
159 ioat->tail = idx + i;
160 BUG_ON(active && !seen_current); /* no active descs have written a completion? */
161
162 chan->last_completion = phys_complete;
163 if (active - i == 0) {
164 dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
165 __func__);
166 clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
167 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
168 }
169}
170
171/**
172 * ioat2_cleanup - clean finished descriptors (advance tail pointer)
173 * @chan: ioat channel to be cleaned up
174 */
175static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
176{
177 struct ioat_chan_common *chan = &ioat->base;
178 dma_addr_t phys_complete;
179
180 spin_lock_bh(&chan->cleanup_lock);
181 if (ioat_cleanup_preamble(chan, &phys_complete))
182 __cleanup(ioat, phys_complete);
183 spin_unlock_bh(&chan->cleanup_lock);
184}
185
186void ioat2_cleanup_event(unsigned long data)
187{
188 struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
189 struct ioat_chan_common *chan = &ioat->base;
190
191 ioat2_cleanup(ioat);
192 if (!test_bit(IOAT_RUN, &chan->state))
193 return;
194 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
195}
196
197void __ioat2_restart_chan(struct ioat2_dma_chan *ioat)
198{
199 struct ioat_chan_common *chan = &ioat->base;
200
201 /* set the tail to be re-issued */
202 ioat->issued = ioat->tail;
203 ioat->dmacount = 0;
204 set_bit(IOAT_COMPLETION_PENDING, &chan->state);
205 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
206
207 dev_dbg(to_dev(chan),
208 "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
209 __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
210
211 if (ioat2_ring_pending(ioat)) {
212 struct ioat_ring_ent *desc;
213
214 desc = ioat2_get_ring_ent(ioat, ioat->tail);
215 ioat2_set_chainaddr(ioat, desc->txd.phys);
216 __ioat2_issue_pending(ioat);
217 } else
218 __ioat2_start_null_desc(ioat);
219}
220
221int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo)
222{
223 unsigned long end = jiffies + tmo;
224 int err = 0;
225 u32 status;
226
227 status = ioat_chansts(chan);
228 if (is_ioat_active(status) || is_ioat_idle(status))
229 ioat_suspend(chan);
230 while (is_ioat_active(status) || is_ioat_idle(status)) {
231 if (tmo && time_after(jiffies, end)) {
232 err = -ETIMEDOUT;
233 break;
234 }
235 status = ioat_chansts(chan);
236 cpu_relax();
237 }
238
239 return err;
240}
241
242int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo)
243{
244 unsigned long end = jiffies + tmo;
245 int err = 0;
246
247 ioat_reset(chan);
248 while (ioat_reset_pending(chan)) {
249 if (end && time_after(jiffies, end)) {
250 err = -ETIMEDOUT;
251 break;
252 }
253 cpu_relax();
254 }
255
256 return err;
257}
258
259static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
260{
261 struct ioat_chan_common *chan = &ioat->base;
262 dma_addr_t phys_complete;
263
264 ioat2_quiesce(chan, 0);
265 if (ioat_cleanup_preamble(chan, &phys_complete))
266 __cleanup(ioat, phys_complete);
267
268 __ioat2_restart_chan(ioat);
269}
270
271static void check_active(struct ioat2_dma_chan *ioat)
272{
273 struct ioat_chan_common *chan = &ioat->base;
274
275 if (ioat2_ring_active(ioat)) {
276 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
277 return;
278 }
279
280 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &chan->state))
281 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
282 else if (ioat->alloc_order > ioat_get_alloc_order()) {
283 /* if the ring is idle, empty, and oversized try to step
284 * down the size
285 */
286 reshape_ring(ioat, ioat->alloc_order - 1);
287
288 /* keep shrinking until we get back to our minimum
289 * default size
290 */
291 if (ioat->alloc_order > ioat_get_alloc_order())
292 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
293 }
294
295}
296
297void ioat2_timer_event(unsigned long data)
298{
299 struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
300 struct ioat_chan_common *chan = &ioat->base;
301 dma_addr_t phys_complete;
302 u64 status;
303
304 status = ioat_chansts(chan);
305
306 /* when halted due to errors check for channel
307 * programming errors before advancing the completion state
308 */
309 if (is_ioat_halted(status)) {
310 u32 chanerr;
311
312 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
313 dev_err(to_dev(chan), "%s: Channel halted (%x)\n",
314 __func__, chanerr);
315 if (test_bit(IOAT_RUN, &chan->state))
316 BUG_ON(is_ioat_bug(chanerr));
317 else /* we never got off the ground */
318 return;
319 }
320
321 /* if we haven't made progress and we have already
322 * acknowledged a pending completion once, then be more
323 * forceful with a restart
324 */
325 spin_lock_bh(&chan->cleanup_lock);
326 if (ioat_cleanup_preamble(chan, &phys_complete))
327 __cleanup(ioat, phys_complete);
328 else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) {
329 spin_lock_bh(&ioat->prep_lock);
330 ioat2_restart_channel(ioat);
331 spin_unlock_bh(&ioat->prep_lock);
332 spin_unlock_bh(&chan->cleanup_lock);
333 return;
334 } else {
335 set_bit(IOAT_COMPLETION_ACK, &chan->state);
336 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
337 }
338
339
340 if (ioat2_ring_active(ioat))
341 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
342 else {
343 spin_lock_bh(&ioat->prep_lock);
344 check_active(ioat);
345 spin_unlock_bh(&ioat->prep_lock);
346 }
347 spin_unlock_bh(&chan->cleanup_lock);
348}
349
350static int ioat2_reset_hw(struct ioat_chan_common *chan)
351{
352 /* throw away whatever the channel was doing and get it initialized */
353 u32 chanerr;
354
355 ioat2_quiesce(chan, msecs_to_jiffies(100));
356
357 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
358 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
359
360 return ioat2_reset_sync(chan, msecs_to_jiffies(200));
361}
362
363/**
364 * ioat2_enumerate_channels - find and initialize the device's channels
365 * @device: the device to be enumerated
366 */
367int ioat2_enumerate_channels(struct ioatdma_device *device)
368{
369 struct ioat2_dma_chan *ioat;
370 struct device *dev = &device->pdev->dev;
371 struct dma_device *dma = &device->common;
372 u8 xfercap_log;
373 int i;
374
375 INIT_LIST_HEAD(&dma->channels);
376 dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
377 dma->chancnt &= 0x1f; /* bits [4:0] valid */
378 if (dma->chancnt > ARRAY_SIZE(device->idx)) {
379 dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
380 dma->chancnt, ARRAY_SIZE(device->idx));
381 dma->chancnt = ARRAY_SIZE(device->idx);
382 }
383 xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
384 xfercap_log &= 0x1f; /* bits [4:0] valid */
385 if (xfercap_log == 0)
386 return 0;
387 dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
388
389 /* FIXME which i/oat version is i7300? */
390#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
391 if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
392 dma->chancnt--;
393#endif
394 for (i = 0; i < dma->chancnt; i++) {
395 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
396 if (!ioat)
397 break;
398
399 ioat_init_channel(device, &ioat->base, i);
400 ioat->xfercap_log = xfercap_log;
401 spin_lock_init(&ioat->prep_lock);
402 if (device->reset_hw(&ioat->base)) {
403 i = 0;
404 break;
405 }
406 }
407 dma->chancnt = i;
408 return i;
409}
410
411static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
412{
413 struct dma_chan *c = tx->chan;
414 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
415 struct ioat_chan_common *chan = &ioat->base;
416 dma_cookie_t cookie;
417
418 cookie = dma_cookie_assign(tx);
419 dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
420
421 if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &chan->state))
422 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
423
424 /* make descriptor updates visible before advancing ioat->head,
425 * this is purposefully not smp_wmb() since we are also
426 * publishing the descriptor updates to a dma device
427 */
428 wmb();
429
430 ioat->head += ioat->produce;
431
432 ioat2_update_pending(ioat);
433 spin_unlock_bh(&ioat->prep_lock);
434
435 return cookie;
436}
437
438static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
439{
440 struct ioat_dma_descriptor *hw;
441 struct ioat_ring_ent *desc;
442 struct ioatdma_device *dma;
443 dma_addr_t phys;
444
445 dma = to_ioatdma_device(chan->device);
446 hw = pci_pool_alloc(dma->dma_pool, flags, &phys);
447 if (!hw)
448 return NULL;
449 memset(hw, 0, sizeof(*hw));
450
451 desc = kmem_cache_zalloc(ioat2_cache, flags);
452 if (!desc) {
453 pci_pool_free(dma->dma_pool, hw, phys);
454 return NULL;
455 }
456
457 dma_async_tx_descriptor_init(&desc->txd, chan);
458 desc->txd.tx_submit = ioat2_tx_submit_unlock;
459 desc->hw = hw;
460 desc->txd.phys = phys;
461 return desc;
462}
463
464static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
465{
466 struct ioatdma_device *dma;
467
468 dma = to_ioatdma_device(chan->device);
469 pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
470 kmem_cache_free(ioat2_cache, desc);
471}
472
473static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
474{
475 struct ioat_ring_ent **ring;
476 int descs = 1 << order;
477 int i;
478
479 if (order > ioat_get_max_alloc_order())
480 return NULL;
481
482 /* allocate the array to hold the software ring */
483 ring = kcalloc(descs, sizeof(*ring), flags);
484 if (!ring)
485 return NULL;
486 for (i = 0; i < descs; i++) {
487 ring[i] = ioat2_alloc_ring_ent(c, flags);
488 if (!ring[i]) {
489 while (i--)
490 ioat2_free_ring_ent(ring[i], c);
491 kfree(ring);
492 return NULL;
493 }
494 set_desc_id(ring[i], i);
495 }
496
497 /* link descs */
498 for (i = 0; i < descs-1; i++) {
499 struct ioat_ring_ent *next = ring[i+1];
500 struct ioat_dma_descriptor *hw = ring[i]->hw;
501
502 hw->next = next->txd.phys;
503 }
504 ring[i]->hw->next = ring[0]->txd.phys;
505
506 return ring;
507}
508
509void ioat2_free_chan_resources(struct dma_chan *c);
510
511/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
512 * @chan: channel to be initialized
513 */
514int ioat2_alloc_chan_resources(struct dma_chan *c)
515{
516 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
517 struct ioat_chan_common *chan = &ioat->base;
518 struct ioat_ring_ent **ring;
519 u64 status;
520 int order;
521 int i = 0;
522
523 /* have we already been set up? */
524 if (ioat->ring)
525 return 1 << ioat->alloc_order;
526
527 /* Setup register to interrupt and write completion status on error */
528 writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
529
530 /* allocate a completion writeback area */
531 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
532 chan->completion = pci_pool_alloc(chan->device->completion_pool,
533 GFP_KERNEL, &chan->completion_dma);
534 if (!chan->completion)
535 return -ENOMEM;
536
537 memset(chan->completion, 0, sizeof(*chan->completion));
538 writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
539 chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
540 writel(((u64) chan->completion_dma) >> 32,
541 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
542
543 order = ioat_get_alloc_order();
544 ring = ioat2_alloc_ring(c, order, GFP_KERNEL);
545 if (!ring)
546 return -ENOMEM;
547
548 spin_lock_bh(&chan->cleanup_lock);
549 spin_lock_bh(&ioat->prep_lock);
550 ioat->ring = ring;
551 ioat->head = 0;
552 ioat->issued = 0;
553 ioat->tail = 0;
554 ioat->alloc_order = order;
555 set_bit(IOAT_RUN, &chan->state);
556 spin_unlock_bh(&ioat->prep_lock);
557 spin_unlock_bh(&chan->cleanup_lock);
558
559 ioat2_start_null_desc(ioat);
560
561 /* check that we got off the ground */
562 do {
563 udelay(1);
564 status = ioat_chansts(chan);
565 } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status));
566
567 if (is_ioat_active(status) || is_ioat_idle(status)) {
568 return 1 << ioat->alloc_order;
569 } else {
570 u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
571
572 dev_WARN(to_dev(chan),
573 "failed to start channel chanerr: %#x\n", chanerr);
574 ioat2_free_chan_resources(c);
575 return -EFAULT;
576 }
577}
578
579bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
580{
581 /* reshape differs from normal ring allocation in that we want
582 * to allocate a new software ring while only
583 * extending/truncating the hardware ring
584 */
585 struct ioat_chan_common *chan = &ioat->base;
586 struct dma_chan *c = &chan->common;
587 const u32 curr_size = ioat2_ring_size(ioat);
588 const u16 active = ioat2_ring_active(ioat);
589 const u32 new_size = 1 << order;
590 struct ioat_ring_ent **ring;
591 u16 i;
592
593 if (order > ioat_get_max_alloc_order())
594 return false;
595
596 /* double check that we have at least 1 free descriptor */
597 if (active == curr_size)
598 return false;
599
600 /* when shrinking, verify that we can hold the current active
601 * set in the new ring
602 */
603 if (active >= new_size)
604 return false;
605
606 /* allocate the array to hold the software ring */
607 ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);
608 if (!ring)
609 return false;
610
611 /* allocate/trim descriptors as needed */
612 if (new_size > curr_size) {
613 /* copy current descriptors to the new ring */
614 for (i = 0; i < curr_size; i++) {
615 u16 curr_idx = (ioat->tail+i) & (curr_size-1);
616 u16 new_idx = (ioat->tail+i) & (new_size-1);
617
618 ring[new_idx] = ioat->ring[curr_idx];
619 set_desc_id(ring[new_idx], new_idx);
620 }
621
622 /* add new descriptors to the ring */
623 for (i = curr_size; i < new_size; i++) {
624 u16 new_idx = (ioat->tail+i) & (new_size-1);
625
626 ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT);
627 if (!ring[new_idx]) {
628 while (i--) {
629 u16 new_idx = (ioat->tail+i) & (new_size-1);
630
631 ioat2_free_ring_ent(ring[new_idx], c);
632 }
633 kfree(ring);
634 return false;
635 }
636 set_desc_id(ring[new_idx], new_idx);
637 }
638
639 /* hw link new descriptors */
640 for (i = curr_size-1; i < new_size; i++) {
641 u16 new_idx = (ioat->tail+i) & (new_size-1);
642 struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)];
643 struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
644
645 hw->next = next->txd.phys;
646 }
647 } else {
648 struct ioat_dma_descriptor *hw;
649 struct ioat_ring_ent *next;
650
651 /* copy current descriptors to the new ring, dropping the
652 * removed descriptors
653 */
654 for (i = 0; i < new_size; i++) {
655 u16 curr_idx = (ioat->tail+i) & (curr_size-1);
656 u16 new_idx = (ioat->tail+i) & (new_size-1);
657
658 ring[new_idx] = ioat->ring[curr_idx];
659 set_desc_id(ring[new_idx], new_idx);
660 }
661
662 /* free deleted descriptors */
663 for (i = new_size; i < curr_size; i++) {
664 struct ioat_ring_ent *ent;
665
666 ent = ioat2_get_ring_ent(ioat, ioat->tail+i);
667 ioat2_free_ring_ent(ent, c);
668 }
669
670 /* fix up hardware ring */
671 hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->hw;
672 next = ring[(ioat->tail+new_size) & (new_size-1)];
673 hw->next = next->txd.phys;
674 }
675
676 dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
677 __func__, new_size);
678
679 kfree(ioat->ring);
680 ioat->ring = ring;
681 ioat->alloc_order = order;
682
683 return true;
684}
685
686/**
687 * ioat2_check_space_lock - verify space and grab ring producer lock
688 * @ioat: ioat2,3 channel (ring) to operate on
689 * @num_descs: allocation length
690 */
691int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs)
692{
693 struct ioat_chan_common *chan = &ioat->base;
694 bool retry;
695
696 retry:
697 spin_lock_bh(&ioat->prep_lock);
698 /* never allow the last descriptor to be consumed, we need at
699 * least one free at all times to allow for on-the-fly ring
700 * resizing.
701 */
702 if (likely(ioat2_ring_space(ioat) > num_descs)) {
703 dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n",
704 __func__, num_descs, ioat->head, ioat->tail, ioat->issued);
705 ioat->produce = num_descs;
706 return 0; /* with ioat->prep_lock held */
707 }
708 retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &chan->state);
709 spin_unlock_bh(&ioat->prep_lock);
710
711 /* is another cpu already trying to expand the ring? */
712 if (retry)
713 goto retry;
714
715 spin_lock_bh(&chan->cleanup_lock);
716 spin_lock_bh(&ioat->prep_lock);
717 retry = reshape_ring(ioat, ioat->alloc_order + 1);
718 clear_bit(IOAT_RESHAPE_PENDING, &chan->state);
719 spin_unlock_bh(&ioat->prep_lock);
720 spin_unlock_bh(&chan->cleanup_lock);
721
722 /* if we were able to expand the ring retry the allocation */
723 if (retry)
724 goto retry;
725
726 if (printk_ratelimit())
727 dev_dbg(to_dev(chan), "%s: ring full! num_descs: %d (%x:%x:%x)\n",
728 __func__, num_descs, ioat->head, ioat->tail, ioat->issued);
729
730 /* progress reclaim in the allocation failure case we may be
731 * called under bh_disabled so we need to trigger the timer
732 * event directly
733 */
734 if (time_is_before_jiffies(chan->timer.expires)
735 && timer_pending(&chan->timer)) {
736 struct ioatdma_device *device = chan->device;
737
738 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
739 device->timer_fn((unsigned long) &chan->common);
740 }
741
742 return -ENOMEM;
743}
744
745struct dma_async_tx_descriptor *
746ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
747 dma_addr_t dma_src, size_t len, unsigned long flags)
748{
749 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
750 struct ioat_dma_descriptor *hw;
751 struct ioat_ring_ent *desc;
752 dma_addr_t dst = dma_dest;
753 dma_addr_t src = dma_src;
754 size_t total_len = len;
755 int num_descs, idx, i;
756
757 num_descs = ioat2_xferlen_to_descs(ioat, len);
758 if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs) == 0)
759 idx = ioat->head;
760 else
761 return NULL;
762 i = 0;
763 do {
764 size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log);
765
766 desc = ioat2_get_ring_ent(ioat, idx + i);
767 hw = desc->hw;
768
769 hw->size = copy;
770 hw->ctl = 0;
771 hw->src_addr = src;
772 hw->dst_addr = dst;
773
774 len -= copy;
775 dst += copy;
776 src += copy;
777 dump_desc_dbg(ioat, desc);
778 } while (++i < num_descs);
779
780 desc->txd.flags = flags;
781 desc->len = total_len;
782 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
783 hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
784 hw->ctl_f.compl_write = 1;
785 dump_desc_dbg(ioat, desc);
786 /* we leave the channel locked to ensure in order submission */
787
788 return &desc->txd;
789}
790
791/**
792 * ioat2_free_chan_resources - release all the descriptors
793 * @chan: the channel to be cleaned
794 */
795void ioat2_free_chan_resources(struct dma_chan *c)
796{
797 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
798 struct ioat_chan_common *chan = &ioat->base;
799 struct ioatdma_device *device = chan->device;
800 struct ioat_ring_ent *desc;
801 const u16 total_descs = 1 << ioat->alloc_order;
802 int descs;
803 int i;
804
805 /* Before freeing channel resources first check
806 * if they have been previously allocated for this channel.
807 */
808 if (!ioat->ring)
809 return;
810
811 ioat_stop(chan);
812 device->reset_hw(chan);
813
814 spin_lock_bh(&chan->cleanup_lock);
815 spin_lock_bh(&ioat->prep_lock);
816 descs = ioat2_ring_space(ioat);
817 dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs);
818 for (i = 0; i < descs; i++) {
819 desc = ioat2_get_ring_ent(ioat, ioat->head + i);
820 ioat2_free_ring_ent(desc, c);
821 }
822
823 if (descs < total_descs)
824 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
825 total_descs - descs);
826
827 for (i = 0; i < total_descs - descs; i++) {
828 desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
829 dump_desc_dbg(ioat, desc);
830 ioat2_free_ring_ent(desc, c);
831 }
832
833 kfree(ioat->ring);
834 ioat->ring = NULL;
835 ioat->alloc_order = 0;
836 pci_pool_free(device->completion_pool, chan->completion,
837 chan->completion_dma);
838 spin_unlock_bh(&ioat->prep_lock);
839 spin_unlock_bh(&chan->cleanup_lock);
840
841 chan->last_completion = 0;
842 chan->completion_dma = 0;
843 ioat->dmacount = 0;
844}
845
846static ssize_t ring_size_show(struct dma_chan *c, char *page)
847{
848 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
849
850 return sprintf(page, "%d\n", (1 << ioat->alloc_order) & ~1);
851}
852static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
853
854static ssize_t ring_active_show(struct dma_chan *c, char *page)
855{
856 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
857
858 /* ...taken outside the lock, no need to be precise */
859 return sprintf(page, "%d\n", ioat2_ring_active(ioat));
860}
861static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
862
863static struct attribute *ioat2_attrs[] = {
864 &ring_size_attr.attr,
865 &ring_active_attr.attr,
866 &ioat_cap_attr.attr,
867 &ioat_version_attr.attr,
868 NULL,
869};
870
871struct kobj_type ioat2_ktype = {
872 .sysfs_ops = &ioat_sysfs_ops,
873 .default_attrs = ioat2_attrs,
874};
875
876int ioat2_dma_probe(struct ioatdma_device *device, int dca)
877{
878 struct pci_dev *pdev = device->pdev;
879 struct dma_device *dma;
880 struct dma_chan *c;
881 struct ioat_chan_common *chan;
882 int err;
883
884 device->enumerate_channels = ioat2_enumerate_channels;
885 device->reset_hw = ioat2_reset_hw;
886 device->cleanup_fn = ioat2_cleanup_event;
887 device->timer_fn = ioat2_timer_event;
888 device->self_test = ioat_dma_self_test;
889 dma = &device->common;
890 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
891 dma->device_issue_pending = ioat2_issue_pending;
892 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
893 dma->device_free_chan_resources = ioat2_free_chan_resources;
894 dma->device_tx_status = ioat_dma_tx_status;
895
896 err = ioat_probe(device);
897 if (err)
898 return err;
899
900 list_for_each_entry(c, &dma->channels, device_node) {
901 chan = to_chan_common(c);
902 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
903 chan->reg_base + IOAT_DCACTRL_OFFSET);
904 }
905
906 err = ioat_register(device);
907 if (err)
908 return err;
909
910 ioat_kobject_add(device, &ioat2_ktype);
911
912 if (dca)
913 device->dca = ioat2_dca_init(pdev, device->reg_base);
914
915 return err;
916}
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
deleted file mode 100644
index bf24ebe874b0..000000000000
--- a/drivers/dma/ioat/dma_v2.h
+++ /dev/null
@@ -1,175 +0,0 @@
1/*
2 * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * The full GNU General Public License is included in this distribution in the
15 * file called COPYING.
16 */
17#ifndef IOATDMA_V2_H
18#define IOATDMA_V2_H
19
20#include <linux/dmaengine.h>
21#include <linux/circ_buf.h>
22#include "dma.h"
23#include "hw.h"
24
25
26extern int ioat_pending_level;
27extern int ioat_ring_alloc_order;
28
29/*
30 * workaround for IOAT ver.3.0 null descriptor issue
31 * (channel returns error when size is 0)
32 */
33#define NULL_DESC_BUFFER_SIZE 1
34
35#define IOAT_MAX_ORDER 16
36#define ioat_get_alloc_order() \
37 (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
38#define ioat_get_max_alloc_order() \
39 (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
40
41/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes
42 * @base: common ioat channel parameters
43 * @xfercap_log; log2 of channel max transfer length (for fast division)
44 * @head: allocated index
45 * @issued: hardware notification point
46 * @tail: cleanup index
47 * @dmacount: identical to 'head' except for occasionally resetting to zero
48 * @alloc_order: log2 of the number of allocated descriptors
49 * @produce: number of descriptors to produce at submit time
50 * @ring: software ring buffer implementation of hardware ring
51 * @prep_lock: serializes descriptor preparation (producers)
52 */
53struct ioat2_dma_chan {
54 struct ioat_chan_common base;
55 size_t xfercap_log;
56 u16 head;
57 u16 issued;
58 u16 tail;
59 u16 dmacount;
60 u16 alloc_order;
61 u16 produce;
62 struct ioat_ring_ent **ring;
63 spinlock_t prep_lock;
64};
65
66static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c)
67{
68 struct ioat_chan_common *chan = to_chan_common(c);
69
70 return container_of(chan, struct ioat2_dma_chan, base);
71}
72
73static inline u32 ioat2_ring_size(struct ioat2_dma_chan *ioat)
74{
75 return 1 << ioat->alloc_order;
76}
77
78/* count of descriptors in flight with the engine */
79static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat)
80{
81 return CIRC_CNT(ioat->head, ioat->tail, ioat2_ring_size(ioat));
82}
83
84/* count of descriptors pending submission to hardware */
85static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat)
86{
87 return CIRC_CNT(ioat->head, ioat->issued, ioat2_ring_size(ioat));
88}
89
90static inline u32 ioat2_ring_space(struct ioat2_dma_chan *ioat)
91{
92 return ioat2_ring_size(ioat) - ioat2_ring_active(ioat);
93}
94
95static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len)
96{
97 u16 num_descs = len >> ioat->xfercap_log;
98
99 num_descs += !!(len & ((1 << ioat->xfercap_log) - 1));
100 return num_descs;
101}
102
103/**
104 * struct ioat_ring_ent - wrapper around hardware descriptor
105 * @hw: hardware DMA descriptor (for memcpy)
106 * @fill: hardware fill descriptor
107 * @xor: hardware xor descriptor
108 * @xor_ex: hardware xor extension descriptor
109 * @pq: hardware pq descriptor
110 * @pq_ex: hardware pq extension descriptor
111 * @pqu: hardware pq update descriptor
112 * @raw: hardware raw (un-typed) descriptor
113 * @txd: the generic software descriptor for all engines
114 * @len: total transaction length for unmap
115 * @result: asynchronous result of validate operations
116 * @id: identifier for debug
117 */
118
119struct ioat_ring_ent {
120 union {
121 struct ioat_dma_descriptor *hw;
122 struct ioat_xor_descriptor *xor;
123 struct ioat_xor_ext_descriptor *xor_ex;
124 struct ioat_pq_descriptor *pq;
125 struct ioat_pq_ext_descriptor *pq_ex;
126 struct ioat_pq_update_descriptor *pqu;
127 struct ioat_raw_descriptor *raw;
128 };
129 size_t len;
130 struct dma_async_tx_descriptor txd;
131 enum sum_check_flags *result;
132 #ifdef DEBUG
133 int id;
134 #endif
135 struct ioat_sed_ent *sed;
136};
137
138static inline struct ioat_ring_ent *
139ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
140{
141 return ioat->ring[idx & (ioat2_ring_size(ioat) - 1)];
142}
143
144static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
145{
146 struct ioat_chan_common *chan = &ioat->base;
147
148 writel(addr & 0x00000000FFFFFFFF,
149 chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
150 writel(addr >> 32,
151 chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
152}
153
154int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
155int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
156struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
157struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
158int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs);
159int ioat2_enumerate_channels(struct ioatdma_device *device);
160struct dma_async_tx_descriptor *
161ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
162 dma_addr_t dma_src, size_t len, unsigned long flags);
163void ioat2_issue_pending(struct dma_chan *chan);
164int ioat2_alloc_chan_resources(struct dma_chan *c);
165void ioat2_free_chan_resources(struct dma_chan *c);
166void __ioat2_restart_chan(struct ioat2_dma_chan *ioat);
167bool reshape_ring(struct ioat2_dma_chan *ioat, int order);
168void __ioat2_issue_pending(struct ioat2_dma_chan *ioat);
169void ioat2_cleanup_event(unsigned long data);
170void ioat2_timer_event(unsigned long data);
171int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo);
172int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo);
173extern struct kobj_type ioat2_ktype;
174extern struct kmem_cache *ioat2_cache;
175#endif /* IOATDMA_V2_H */
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c
deleted file mode 100644
index 64790a45ef5d..000000000000
--- a/drivers/dma/ioat/dma_v3.c
+++ /dev/null
@@ -1,1717 +0,0 @@
1/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
8 *
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms and conditions of the GNU General Public License,
11 * version 2, as published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 * BSD LICENSE
22 *
23 * Copyright(c) 2004-2009 Intel Corporation. All rights reserved.
24 *
25 * Redistribution and use in source and binary forms, with or without
26 * modification, are permitted provided that the following conditions are met:
27 *
28 * * Redistributions of source code must retain the above copyright
29 * notice, this list of conditions and the following disclaimer.
30 * * Redistributions in binary form must reproduce the above copyright
31 * notice, this list of conditions and the following disclaimer in
32 * the documentation and/or other materials provided with the
33 * distribution.
34 * * Neither the name of Intel Corporation nor the names of its
35 * contributors may be used to endorse or promote products derived
36 * from this software without specific prior written permission.
37 *
38 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
39 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
40 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
41 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
42 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
43 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
44 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
45 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
46 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
47 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
48 * POSSIBILITY OF SUCH DAMAGE.
49 */
50
51/*
52 * Support routines for v3+ hardware
53 */
54#include <linux/module.h>
55#include <linux/pci.h>
56#include <linux/gfp.h>
57#include <linux/dmaengine.h>
58#include <linux/dma-mapping.h>
59#include <linux/prefetch.h>
60#include "../dmaengine.h"
61#include "registers.h"
62#include "hw.h"
63#include "dma.h"
64#include "dma_v2.h"
65
66extern struct kmem_cache *ioat3_sed_cache;
67
68/* ioat hardware assumes at least two sources for raid operations */
69#define src_cnt_to_sw(x) ((x) + 2)
70#define src_cnt_to_hw(x) ((x) - 2)
71#define ndest_to_sw(x) ((x) + 1)
72#define ndest_to_hw(x) ((x) - 1)
73#define src16_cnt_to_sw(x) ((x) + 9)
74#define src16_cnt_to_hw(x) ((x) - 9)
75
76/* provide a lookup table for setting the source address in the base or
77 * extended descriptor of an xor or pq descriptor
78 */
79static const u8 xor_idx_to_desc = 0xe0;
80static const u8 xor_idx_to_field[] = { 1, 4, 5, 6, 7, 0, 1, 2 };
81static const u8 pq_idx_to_desc = 0xf8;
82static const u8 pq16_idx_to_desc[] = { 0, 0, 1, 1, 1, 1, 1, 1, 1,
83 2, 2, 2, 2, 2, 2, 2 };
84static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 };
85static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7,
86 0, 1, 2, 3, 4, 5, 6 };
87
88static void ioat3_eh(struct ioat2_dma_chan *ioat);
89
90static void xor_set_src(struct ioat_raw_descriptor *descs[2],
91 dma_addr_t addr, u32 offset, int idx)
92{
93 struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
94
95 raw->field[xor_idx_to_field[idx]] = addr + offset;
96}
97
98static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx)
99{
100 struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
101
102 return raw->field[pq_idx_to_field[idx]];
103}
104
105static dma_addr_t pq16_get_src(struct ioat_raw_descriptor *desc[3], int idx)
106{
107 struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]];
108
109 return raw->field[pq16_idx_to_field[idx]];
110}
111
112static void pq_set_src(struct ioat_raw_descriptor *descs[2],
113 dma_addr_t addr, u32 offset, u8 coef, int idx)
114{
115 struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0];
116 struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
117
118 raw->field[pq_idx_to_field[idx]] = addr + offset;
119 pq->coef[idx] = coef;
120}
121
122static bool is_jf_ioat(struct pci_dev *pdev)
123{
124 switch (pdev->device) {
125 case PCI_DEVICE_ID_INTEL_IOAT_JSF0:
126 case PCI_DEVICE_ID_INTEL_IOAT_JSF1:
127 case PCI_DEVICE_ID_INTEL_IOAT_JSF2:
128 case PCI_DEVICE_ID_INTEL_IOAT_JSF3:
129 case PCI_DEVICE_ID_INTEL_IOAT_JSF4:
130 case PCI_DEVICE_ID_INTEL_IOAT_JSF5:
131 case PCI_DEVICE_ID_INTEL_IOAT_JSF6:
132 case PCI_DEVICE_ID_INTEL_IOAT_JSF7:
133 case PCI_DEVICE_ID_INTEL_IOAT_JSF8:
134 case PCI_DEVICE_ID_INTEL_IOAT_JSF9:
135 return true;
136 default:
137 return false;
138 }
139}
140
141static bool is_snb_ioat(struct pci_dev *pdev)
142{
143 switch (pdev->device) {
144 case PCI_DEVICE_ID_INTEL_IOAT_SNB0:
145 case PCI_DEVICE_ID_INTEL_IOAT_SNB1:
146 case PCI_DEVICE_ID_INTEL_IOAT_SNB2:
147 case PCI_DEVICE_ID_INTEL_IOAT_SNB3:
148 case PCI_DEVICE_ID_INTEL_IOAT_SNB4:
149 case PCI_DEVICE_ID_INTEL_IOAT_SNB5:
150 case PCI_DEVICE_ID_INTEL_IOAT_SNB6:
151 case PCI_DEVICE_ID_INTEL_IOAT_SNB7:
152 case PCI_DEVICE_ID_INTEL_IOAT_SNB8:
153 case PCI_DEVICE_ID_INTEL_IOAT_SNB9:
154 return true;
155 default:
156 return false;
157 }
158}
159
160static bool is_ivb_ioat(struct pci_dev *pdev)
161{
162 switch (pdev->device) {
163 case PCI_DEVICE_ID_INTEL_IOAT_IVB0:
164 case PCI_DEVICE_ID_INTEL_IOAT_IVB1:
165 case PCI_DEVICE_ID_INTEL_IOAT_IVB2:
166 case PCI_DEVICE_ID_INTEL_IOAT_IVB3:
167 case PCI_DEVICE_ID_INTEL_IOAT_IVB4:
168 case PCI_DEVICE_ID_INTEL_IOAT_IVB5:
169 case PCI_DEVICE_ID_INTEL_IOAT_IVB6:
170 case PCI_DEVICE_ID_INTEL_IOAT_IVB7:
171 case PCI_DEVICE_ID_INTEL_IOAT_IVB8:
172 case PCI_DEVICE_ID_INTEL_IOAT_IVB9:
173 return true;
174 default:
175 return false;
176 }
177
178}
179
180static bool is_hsw_ioat(struct pci_dev *pdev)
181{
182 switch (pdev->device) {
183 case PCI_DEVICE_ID_INTEL_IOAT_HSW0:
184 case PCI_DEVICE_ID_INTEL_IOAT_HSW1:
185 case PCI_DEVICE_ID_INTEL_IOAT_HSW2:
186 case PCI_DEVICE_ID_INTEL_IOAT_HSW3:
187 case PCI_DEVICE_ID_INTEL_IOAT_HSW4:
188 case PCI_DEVICE_ID_INTEL_IOAT_HSW5:
189 case PCI_DEVICE_ID_INTEL_IOAT_HSW6:
190 case PCI_DEVICE_ID_INTEL_IOAT_HSW7:
191 case PCI_DEVICE_ID_INTEL_IOAT_HSW8:
192 case PCI_DEVICE_ID_INTEL_IOAT_HSW9:
193 return true;
194 default:
195 return false;
196 }
197
198}
199
200static bool is_xeon_cb32(struct pci_dev *pdev)
201{
202 return is_jf_ioat(pdev) || is_snb_ioat(pdev) || is_ivb_ioat(pdev) ||
203 is_hsw_ioat(pdev);
204}
205
206static bool is_bwd_ioat(struct pci_dev *pdev)
207{
208 switch (pdev->device) {
209 case PCI_DEVICE_ID_INTEL_IOAT_BWD0:
210 case PCI_DEVICE_ID_INTEL_IOAT_BWD1:
211 case PCI_DEVICE_ID_INTEL_IOAT_BWD2:
212 case PCI_DEVICE_ID_INTEL_IOAT_BWD3:
213 /* even though not Atom, BDX-DE has same DMA silicon */
214 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0:
215 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1:
216 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2:
217 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3:
218 return true;
219 default:
220 return false;
221 }
222}
223
224static bool is_bwd_noraid(struct pci_dev *pdev)
225{
226 switch (pdev->device) {
227 case PCI_DEVICE_ID_INTEL_IOAT_BWD2:
228 case PCI_DEVICE_ID_INTEL_IOAT_BWD3:
229 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0:
230 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1:
231 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2:
232 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3:
233 return true;
234 default:
235 return false;
236 }
237
238}
239
240static void pq16_set_src(struct ioat_raw_descriptor *desc[3],
241 dma_addr_t addr, u32 offset, u8 coef, unsigned idx)
242{
243 struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *)desc[0];
244 struct ioat_pq16a_descriptor *pq16 =
245 (struct ioat_pq16a_descriptor *)desc[1];
246 struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]];
247
248 raw->field[pq16_idx_to_field[idx]] = addr + offset;
249
250 if (idx < 8)
251 pq->coef[idx] = coef;
252 else
253 pq16->coef[idx - 8] = coef;
254}
255
256static struct ioat_sed_ent *
257ioat3_alloc_sed(struct ioatdma_device *device, unsigned int hw_pool)
258{
259 struct ioat_sed_ent *sed;
260 gfp_t flags = __GFP_ZERO | GFP_ATOMIC;
261
262 sed = kmem_cache_alloc(ioat3_sed_cache, flags);
263 if (!sed)
264 return NULL;
265
266 sed->hw_pool = hw_pool;
267 sed->hw = dma_pool_alloc(device->sed_hw_pool[hw_pool],
268 flags, &sed->dma);
269 if (!sed->hw) {
270 kmem_cache_free(ioat3_sed_cache, sed);
271 return NULL;
272 }
273
274 return sed;
275}
276
277static void ioat3_free_sed(struct ioatdma_device *device, struct ioat_sed_ent *sed)
278{
279 if (!sed)
280 return;
281
282 dma_pool_free(device->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma);
283 kmem_cache_free(ioat3_sed_cache, sed);
284}
285
286static bool desc_has_ext(struct ioat_ring_ent *desc)
287{
288 struct ioat_dma_descriptor *hw = desc->hw;
289
290 if (hw->ctl_f.op == IOAT_OP_XOR ||
291 hw->ctl_f.op == IOAT_OP_XOR_VAL) {
292 struct ioat_xor_descriptor *xor = desc->xor;
293
294 if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
295 return true;
296 } else if (hw->ctl_f.op == IOAT_OP_PQ ||
297 hw->ctl_f.op == IOAT_OP_PQ_VAL) {
298 struct ioat_pq_descriptor *pq = desc->pq;
299
300 if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
301 return true;
302 }
303
304 return false;
305}
306
307static u64 ioat3_get_current_completion(struct ioat_chan_common *chan)
308{
309 u64 phys_complete;
310 u64 completion;
311
312 completion = *chan->completion;
313 phys_complete = ioat_chansts_to_addr(completion);
314
315 dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
316 (unsigned long long) phys_complete);
317
318 return phys_complete;
319}
320
321static bool ioat3_cleanup_preamble(struct ioat_chan_common *chan,
322 u64 *phys_complete)
323{
324 *phys_complete = ioat3_get_current_completion(chan);
325 if (*phys_complete == chan->last_completion)
326 return false;
327
328 clear_bit(IOAT_COMPLETION_ACK, &chan->state);
329 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
330
331 return true;
332}
333
334static void
335desc_get_errstat(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc)
336{
337 struct ioat_dma_descriptor *hw = desc->hw;
338
339 switch (hw->ctl_f.op) {
340 case IOAT_OP_PQ_VAL:
341 case IOAT_OP_PQ_VAL_16S:
342 {
343 struct ioat_pq_descriptor *pq = desc->pq;
344
345 /* check if there's error written */
346 if (!pq->dwbes_f.wbes)
347 return;
348
349 /* need to set a chanerr var for checking to clear later */
350
351 if (pq->dwbes_f.p_val_err)
352 *desc->result |= SUM_CHECK_P_RESULT;
353
354 if (pq->dwbes_f.q_val_err)
355 *desc->result |= SUM_CHECK_Q_RESULT;
356
357 return;
358 }
359 default:
360 return;
361 }
362}
363
364/**
365 * __cleanup - reclaim used descriptors
366 * @ioat: channel (ring) to clean
367 *
368 * The difference from the dma_v2.c __cleanup() is that this routine
369 * handles extended descriptors and dma-unmapping raid operations.
370 */
371static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)
372{
373 struct ioat_chan_common *chan = &ioat->base;
374 struct ioatdma_device *device = chan->device;
375 struct ioat_ring_ent *desc;
376 bool seen_current = false;
377 int idx = ioat->tail, i;
378 u16 active;
379
380 dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
381 __func__, ioat->head, ioat->tail, ioat->issued);
382
383 /*
384 * At restart of the channel, the completion address and the
385 * channel status will be 0 due to starting a new chain. Since
386 * it's new chain and the first descriptor "fails", there is
387 * nothing to clean up. We do not want to reap the entire submitted
388 * chain due to this 0 address value and then BUG.
389 */
390 if (!phys_complete)
391 return;
392
393 active = ioat2_ring_active(ioat);
394 for (i = 0; i < active && !seen_current; i++) {
395 struct dma_async_tx_descriptor *tx;
396
397 smp_read_barrier_depends();
398 prefetch(ioat2_get_ring_ent(ioat, idx + i + 1));
399 desc = ioat2_get_ring_ent(ioat, idx + i);
400 dump_desc_dbg(ioat, desc);
401
402 /* set err stat if we are using dwbes */
403 if (device->cap & IOAT_CAP_DWBES)
404 desc_get_errstat(ioat, desc);
405
406 tx = &desc->txd;
407 if (tx->cookie) {
408 dma_cookie_complete(tx);
409 dma_descriptor_unmap(tx);
410 if (tx->callback) {
411 tx->callback(tx->callback_param);
412 tx->callback = NULL;
413 }
414 }
415
416 if (tx->phys == phys_complete)
417 seen_current = true;
418
419 /* skip extended descriptors */
420 if (desc_has_ext(desc)) {
421 BUG_ON(i + 1 >= active);
422 i++;
423 }
424
425 /* cleanup super extended descriptors */
426 if (desc->sed) {
427 ioat3_free_sed(device, desc->sed);
428 desc->sed = NULL;
429 }
430 }
431 smp_mb(); /* finish all descriptor reads before incrementing tail */
432 ioat->tail = idx + i;
433 BUG_ON(active && !seen_current); /* no active descs have written a completion? */
434 chan->last_completion = phys_complete;
435
436 if (active - i == 0) {
437 dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
438 __func__);
439 clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
440 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
441 }
442 /* 5 microsecond delay per pending descriptor */
443 writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK),
444 chan->device->reg_base + IOAT_INTRDELAY_OFFSET);
445}
446
447static void ioat3_cleanup(struct ioat2_dma_chan *ioat)
448{
449 struct ioat_chan_common *chan = &ioat->base;
450 u64 phys_complete;
451
452 spin_lock_bh(&chan->cleanup_lock);
453
454 if (ioat3_cleanup_preamble(chan, &phys_complete))
455 __cleanup(ioat, phys_complete);
456
457 if (is_ioat_halted(*chan->completion)) {
458 u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
459
460 if (chanerr & IOAT_CHANERR_HANDLE_MASK) {
461 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
462 ioat3_eh(ioat);
463 }
464 }
465
466 spin_unlock_bh(&chan->cleanup_lock);
467}
468
469static void ioat3_cleanup_event(unsigned long data)
470{
471 struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
472 struct ioat_chan_common *chan = &ioat->base;
473
474 ioat3_cleanup(ioat);
475 if (!test_bit(IOAT_RUN, &chan->state))
476 return;
477 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
478}
479
480static void ioat3_restart_channel(struct ioat2_dma_chan *ioat)
481{
482 struct ioat_chan_common *chan = &ioat->base;
483 u64 phys_complete;
484
485 ioat2_quiesce(chan, 0);
486 if (ioat3_cleanup_preamble(chan, &phys_complete))
487 __cleanup(ioat, phys_complete);
488
489 __ioat2_restart_chan(ioat);
490}
491
492static void ioat3_eh(struct ioat2_dma_chan *ioat)
493{
494 struct ioat_chan_common *chan = &ioat->base;
495 struct pci_dev *pdev = to_pdev(chan);
496 struct ioat_dma_descriptor *hw;
497 struct dma_async_tx_descriptor *tx;
498 u64 phys_complete;
499 struct ioat_ring_ent *desc;
500 u32 err_handled = 0;
501 u32 chanerr_int;
502 u32 chanerr;
503
504 /* cleanup so tail points to descriptor that caused the error */
505 if (ioat3_cleanup_preamble(chan, &phys_complete))
506 __cleanup(ioat, phys_complete);
507
508 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
509 pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int);
510
511 dev_dbg(to_dev(chan), "%s: error = %x:%x\n",
512 __func__, chanerr, chanerr_int);
513
514 desc = ioat2_get_ring_ent(ioat, ioat->tail);
515 hw = desc->hw;
516 dump_desc_dbg(ioat, desc);
517
518 switch (hw->ctl_f.op) {
519 case IOAT_OP_XOR_VAL:
520 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
521 *desc->result |= SUM_CHECK_P_RESULT;
522 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
523 }
524 break;
525 case IOAT_OP_PQ_VAL:
526 case IOAT_OP_PQ_VAL_16S:
527 if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) {
528 *desc->result |= SUM_CHECK_P_RESULT;
529 err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR;
530 }
531 if (chanerr & IOAT_CHANERR_XOR_Q_ERR) {
532 *desc->result |= SUM_CHECK_Q_RESULT;
533 err_handled |= IOAT_CHANERR_XOR_Q_ERR;
534 }
535 break;
536 }
537
538 /* fault on unhandled error or spurious halt */
539 if (chanerr ^ err_handled || chanerr == 0) {
540 dev_err(to_dev(chan), "%s: fatal error (%x:%x)\n",
541 __func__, chanerr, err_handled);
542 BUG();
543 } else { /* cleanup the faulty descriptor */
544 tx = &desc->txd;
545 if (tx->cookie) {
546 dma_cookie_complete(tx);
547 dma_descriptor_unmap(tx);
548 if (tx->callback) {
549 tx->callback(tx->callback_param);
550 tx->callback = NULL;
551 }
552 }
553 }
554
555 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
556 pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int);
557
558 /* mark faulting descriptor as complete */
559 *chan->completion = desc->txd.phys;
560
561 spin_lock_bh(&ioat->prep_lock);
562 ioat3_restart_channel(ioat);
563 spin_unlock_bh(&ioat->prep_lock);
564}
565
566static void check_active(struct ioat2_dma_chan *ioat)
567{
568 struct ioat_chan_common *chan = &ioat->base;
569
570 if (ioat2_ring_active(ioat)) {
571 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
572 return;
573 }
574
575 if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &chan->state))
576 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
577 else if (ioat->alloc_order > ioat_get_alloc_order()) {
578 /* if the ring is idle, empty, and oversized try to step
579 * down the size
580 */
581 reshape_ring(ioat, ioat->alloc_order - 1);
582
583 /* keep shrinking until we get back to our minimum
584 * default size
585 */
586 if (ioat->alloc_order > ioat_get_alloc_order())
587 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
588 }
589
590}
591
592static void ioat3_timer_event(unsigned long data)
593{
594 struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data);
595 struct ioat_chan_common *chan = &ioat->base;
596 dma_addr_t phys_complete;
597 u64 status;
598
599 status = ioat_chansts(chan);
600
601 /* when halted due to errors check for channel
602 * programming errors before advancing the completion state
603 */
604 if (is_ioat_halted(status)) {
605 u32 chanerr;
606
607 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
608 dev_err(to_dev(chan), "%s: Channel halted (%x)\n",
609 __func__, chanerr);
610 if (test_bit(IOAT_RUN, &chan->state))
611 BUG_ON(is_ioat_bug(chanerr));
612 else /* we never got off the ground */
613 return;
614 }
615
616 /* if we haven't made progress and we have already
617 * acknowledged a pending completion once, then be more
618 * forceful with a restart
619 */
620 spin_lock_bh(&chan->cleanup_lock);
621 if (ioat_cleanup_preamble(chan, &phys_complete))
622 __cleanup(ioat, phys_complete);
623 else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) {
624 spin_lock_bh(&ioat->prep_lock);
625 ioat3_restart_channel(ioat);
626 spin_unlock_bh(&ioat->prep_lock);
627 spin_unlock_bh(&chan->cleanup_lock);
628 return;
629 } else {
630 set_bit(IOAT_COMPLETION_ACK, &chan->state);
631 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
632 }
633
634
635 if (ioat2_ring_active(ioat))
636 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
637 else {
638 spin_lock_bh(&ioat->prep_lock);
639 check_active(ioat);
640 spin_unlock_bh(&ioat->prep_lock);
641 }
642 spin_unlock_bh(&chan->cleanup_lock);
643}
644
645static enum dma_status
646ioat3_tx_status(struct dma_chan *c, dma_cookie_t cookie,
647 struct dma_tx_state *txstate)
648{
649 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
650 enum dma_status ret;
651
652 ret = dma_cookie_status(c, cookie, txstate);
653 if (ret == DMA_COMPLETE)
654 return ret;
655
656 ioat3_cleanup(ioat);
657
658 return dma_cookie_status(c, cookie, txstate);
659}
660
661static struct dma_async_tx_descriptor *
662__ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
663 dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt,
664 size_t len, unsigned long flags)
665{
666 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
667 struct ioat_ring_ent *compl_desc;
668 struct ioat_ring_ent *desc;
669 struct ioat_ring_ent *ext;
670 size_t total_len = len;
671 struct ioat_xor_descriptor *xor;
672 struct ioat_xor_ext_descriptor *xor_ex = NULL;
673 struct ioat_dma_descriptor *hw;
674 int num_descs, with_ext, idx, i;
675 u32 offset = 0;
676 u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR;
677
678 BUG_ON(src_cnt < 2);
679
680 num_descs = ioat2_xferlen_to_descs(ioat, len);
681 /* we need 2x the number of descriptors to cover greater than 5
682 * sources
683 */
684 if (src_cnt > 5) {
685 with_ext = 1;
686 num_descs *= 2;
687 } else
688 with_ext = 0;
689
690 /* completion writes from the raid engine may pass completion
691 * writes from the legacy engine, so we need one extra null
692 * (legacy) descriptor to ensure all completion writes arrive in
693 * order.
694 */
695 if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs+1) == 0)
696 idx = ioat->head;
697 else
698 return NULL;
699 i = 0;
700 do {
701 struct ioat_raw_descriptor *descs[2];
702 size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
703 int s;
704
705 desc = ioat2_get_ring_ent(ioat, idx + i);
706 xor = desc->xor;
707
708 /* save a branch by unconditionally retrieving the
709 * extended descriptor xor_set_src() knows to not write
710 * to it in the single descriptor case
711 */
712 ext = ioat2_get_ring_ent(ioat, idx + i + 1);
713 xor_ex = ext->xor_ex;
714
715 descs[0] = (struct ioat_raw_descriptor *) xor;
716 descs[1] = (struct ioat_raw_descriptor *) xor_ex;
717 for (s = 0; s < src_cnt; s++)
718 xor_set_src(descs, src[s], offset, s);
719 xor->size = xfer_size;
720 xor->dst_addr = dest + offset;
721 xor->ctl = 0;
722 xor->ctl_f.op = op;
723 xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt);
724
725 len -= xfer_size;
726 offset += xfer_size;
727 dump_desc_dbg(ioat, desc);
728 } while ((i += 1 + with_ext) < num_descs);
729
730 /* last xor descriptor carries the unmap parameters and fence bit */
731 desc->txd.flags = flags;
732 desc->len = total_len;
733 if (result)
734 desc->result = result;
735 xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
736
737 /* completion descriptor carries interrupt bit */
738 compl_desc = ioat2_get_ring_ent(ioat, idx + i);
739 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
740 hw = compl_desc->hw;
741 hw->ctl = 0;
742 hw->ctl_f.null = 1;
743 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
744 hw->ctl_f.compl_write = 1;
745 hw->size = NULL_DESC_BUFFER_SIZE;
746 dump_desc_dbg(ioat, compl_desc);
747
748 /* we leave the channel locked to ensure in order submission */
749 return &compl_desc->txd;
750}
751
752static struct dma_async_tx_descriptor *
753ioat3_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
754 unsigned int src_cnt, size_t len, unsigned long flags)
755{
756 return __ioat3_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags);
757}
758
759static struct dma_async_tx_descriptor *
760ioat3_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
761 unsigned int src_cnt, size_t len,
762 enum sum_check_flags *result, unsigned long flags)
763{
764 /* the cleanup routine only sets bits on validate failure, it
765 * does not clear bits on validate success... so clear it here
766 */
767 *result = 0;
768
769 return __ioat3_prep_xor_lock(chan, result, src[0], &src[1],
770 src_cnt - 1, len, flags);
771}
772
773static void
774dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext)
775{
776 struct device *dev = to_dev(&ioat->base);
777 struct ioat_pq_descriptor *pq = desc->pq;
778 struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL;
779 struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex };
780 int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
781 int i;
782
783 dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
784 " sz: %#10.8x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'"
785 " src_cnt: %d)\n",
786 desc_id(desc), (unsigned long long) desc->txd.phys,
787 (unsigned long long) (pq_ex ? pq_ex->next : pq->next),
788 desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op, pq->ctl_f.int_en,
789 pq->ctl_f.compl_write,
790 pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
791 pq->ctl_f.src_cnt);
792 for (i = 0; i < src_cnt; i++)
793 dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
794 (unsigned long long) pq_get_src(descs, i), pq->coef[i]);
795 dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
796 dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
797 dev_dbg(dev, "\tNEXT: %#llx\n", pq->next);
798}
799
800static void dump_pq16_desc_dbg(struct ioat2_dma_chan *ioat,
801 struct ioat_ring_ent *desc)
802{
803 struct device *dev = to_dev(&ioat->base);
804 struct ioat_pq_descriptor *pq = desc->pq;
805 struct ioat_raw_descriptor *descs[] = { (void *)pq,
806 (void *)pq,
807 (void *)pq };
808 int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt);
809 int i;
810
811 if (desc->sed) {
812 descs[1] = (void *)desc->sed->hw;
813 descs[2] = (void *)desc->sed->hw + 64;
814 }
815
816 dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
817 " sz: %#x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'"
818 " src_cnt: %d)\n",
819 desc_id(desc), (unsigned long long) desc->txd.phys,
820 (unsigned long long) pq->next,
821 desc->txd.flags, pq->size, pq->ctl,
822 pq->ctl_f.op, pq->ctl_f.int_en,
823 pq->ctl_f.compl_write,
824 pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
825 pq->ctl_f.src_cnt);
826 for (i = 0; i < src_cnt; i++) {
827 dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
828 (unsigned long long) pq16_get_src(descs, i),
829 pq->coef[i]);
830 }
831 dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
832 dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
833}
834
835static struct dma_async_tx_descriptor *
836__ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
837 const dma_addr_t *dst, const dma_addr_t *src,
838 unsigned int src_cnt, const unsigned char *scf,
839 size_t len, unsigned long flags)
840{
841 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
842 struct ioat_chan_common *chan = &ioat->base;
843 struct ioatdma_device *device = chan->device;
844 struct ioat_ring_ent *compl_desc;
845 struct ioat_ring_ent *desc;
846 struct ioat_ring_ent *ext;
847 size_t total_len = len;
848 struct ioat_pq_descriptor *pq;
849 struct ioat_pq_ext_descriptor *pq_ex = NULL;
850 struct ioat_dma_descriptor *hw;
851 u32 offset = 0;
852 u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ;
853 int i, s, idx, with_ext, num_descs;
854 int cb32 = (device->version < IOAT_VER_3_3) ? 1 : 0;
855
856 dev_dbg(to_dev(chan), "%s\n", __func__);
857 /* the engine requires at least two sources (we provide
858 * at least 1 implied source in the DMA_PREP_CONTINUE case)
859 */
860 BUG_ON(src_cnt + dmaf_continue(flags) < 2);
861
862 num_descs = ioat2_xferlen_to_descs(ioat, len);
863 /* we need 2x the number of descriptors to cover greater than 3
864 * sources (we need 1 extra source in the q-only continuation
865 * case and 3 extra sources in the p+q continuation case.
866 */
867 if (src_cnt + dmaf_p_disabled_continue(flags) > 3 ||
868 (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) {
869 with_ext = 1;
870 num_descs *= 2;
871 } else
872 with_ext = 0;
873
874 /* completion writes from the raid engine may pass completion
875 * writes from the legacy engine, so we need one extra null
876 * (legacy) descriptor to ensure all completion writes arrive in
877 * order.
878 */
879 if (likely(num_descs) &&
880 ioat2_check_space_lock(ioat, num_descs + cb32) == 0)
881 idx = ioat->head;
882 else
883 return NULL;
884 i = 0;
885 do {
886 struct ioat_raw_descriptor *descs[2];
887 size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
888
889 desc = ioat2_get_ring_ent(ioat, idx + i);
890 pq = desc->pq;
891
892 /* save a branch by unconditionally retrieving the
893 * extended descriptor pq_set_src() knows to not write
894 * to it in the single descriptor case
895 */
896 ext = ioat2_get_ring_ent(ioat, idx + i + with_ext);
897 pq_ex = ext->pq_ex;
898
899 descs[0] = (struct ioat_raw_descriptor *) pq;
900 descs[1] = (struct ioat_raw_descriptor *) pq_ex;
901
902 for (s = 0; s < src_cnt; s++)
903 pq_set_src(descs, src[s], offset, scf[s], s);
904
905 /* see the comment for dma_maxpq in include/linux/dmaengine.h */
906 if (dmaf_p_disabled_continue(flags))
907 pq_set_src(descs, dst[1], offset, 1, s++);
908 else if (dmaf_continue(flags)) {
909 pq_set_src(descs, dst[0], offset, 0, s++);
910 pq_set_src(descs, dst[1], offset, 1, s++);
911 pq_set_src(descs, dst[1], offset, 0, s++);
912 }
913 pq->size = xfer_size;
914 pq->p_addr = dst[0] + offset;
915 pq->q_addr = dst[1] + offset;
916 pq->ctl = 0;
917 pq->ctl_f.op = op;
918 /* we turn on descriptor write back error status */
919 if (device->cap & IOAT_CAP_DWBES)
920 pq->ctl_f.wb_en = result ? 1 : 0;
921 pq->ctl_f.src_cnt = src_cnt_to_hw(s);
922 pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
923 pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
924
925 len -= xfer_size;
926 offset += xfer_size;
927 } while ((i += 1 + with_ext) < num_descs);
928
929 /* last pq descriptor carries the unmap parameters and fence bit */
930 desc->txd.flags = flags;
931 desc->len = total_len;
932 if (result)
933 desc->result = result;
934 pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
935 dump_pq_desc_dbg(ioat, desc, ext);
936
937 if (!cb32) {
938 pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
939 pq->ctl_f.compl_write = 1;
940 compl_desc = desc;
941 } else {
942 /* completion descriptor carries interrupt bit */
943 compl_desc = ioat2_get_ring_ent(ioat, idx + i);
944 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
945 hw = compl_desc->hw;
946 hw->ctl = 0;
947 hw->ctl_f.null = 1;
948 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
949 hw->ctl_f.compl_write = 1;
950 hw->size = NULL_DESC_BUFFER_SIZE;
951 dump_desc_dbg(ioat, compl_desc);
952 }
953
954
955 /* we leave the channel locked to ensure in order submission */
956 return &compl_desc->txd;
957}
958
959static struct dma_async_tx_descriptor *
960__ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
961 const dma_addr_t *dst, const dma_addr_t *src,
962 unsigned int src_cnt, const unsigned char *scf,
963 size_t len, unsigned long flags)
964{
965 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
966 struct ioat_chan_common *chan = &ioat->base;
967 struct ioatdma_device *device = chan->device;
968 struct ioat_ring_ent *desc;
969 size_t total_len = len;
970 struct ioat_pq_descriptor *pq;
971 u32 offset = 0;
972 u8 op;
973 int i, s, idx, num_descs;
974
975 /* this function is only called with 9-16 sources */
976 op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S;
977
978 dev_dbg(to_dev(chan), "%s\n", __func__);
979
980 num_descs = ioat2_xferlen_to_descs(ioat, len);
981
982 /*
983 * 16 source pq is only available on cb3.3 and has no completion
984 * write hw bug.
985 */
986 if (num_descs && ioat2_check_space_lock(ioat, num_descs) == 0)
987 idx = ioat->head;
988 else
989 return NULL;
990
991 i = 0;
992
993 do {
994 struct ioat_raw_descriptor *descs[4];
995 size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
996
997 desc = ioat2_get_ring_ent(ioat, idx + i);
998 pq = desc->pq;
999
1000 descs[0] = (struct ioat_raw_descriptor *) pq;
1001
1002 desc->sed = ioat3_alloc_sed(device, (src_cnt-2) >> 3);
1003 if (!desc->sed) {
1004 dev_err(to_dev(chan),
1005 "%s: no free sed entries\n", __func__);
1006 return NULL;
1007 }
1008
1009 pq->sed_addr = desc->sed->dma;
1010 desc->sed->parent = desc;
1011
1012 descs[1] = (struct ioat_raw_descriptor *)desc->sed->hw;
1013 descs[2] = (void *)descs[1] + 64;
1014
1015 for (s = 0; s < src_cnt; s++)
1016 pq16_set_src(descs, src[s], offset, scf[s], s);
1017
1018 /* see the comment for dma_maxpq in include/linux/dmaengine.h */
1019 if (dmaf_p_disabled_continue(flags))
1020 pq16_set_src(descs, dst[1], offset, 1, s++);
1021 else if (dmaf_continue(flags)) {
1022 pq16_set_src(descs, dst[0], offset, 0, s++);
1023 pq16_set_src(descs, dst[1], offset, 1, s++);
1024 pq16_set_src(descs, dst[1], offset, 0, s++);
1025 }
1026
1027 pq->size = xfer_size;
1028 pq->p_addr = dst[0] + offset;
1029 pq->q_addr = dst[1] + offset;
1030 pq->ctl = 0;
1031 pq->ctl_f.op = op;
1032 pq->ctl_f.src_cnt = src16_cnt_to_hw(s);
1033 /* we turn on descriptor write back error status */
1034 if (device->cap & IOAT_CAP_DWBES)
1035 pq->ctl_f.wb_en = result ? 1 : 0;
1036 pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
1037 pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
1038
1039 len -= xfer_size;
1040 offset += xfer_size;
1041 } while (++i < num_descs);
1042
1043 /* last pq descriptor carries the unmap parameters and fence bit */
1044 desc->txd.flags = flags;
1045 desc->len = total_len;
1046 if (result)
1047 desc->result = result;
1048 pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
1049
1050 /* with cb3.3 we should be able to do completion w/o a null desc */
1051 pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
1052 pq->ctl_f.compl_write = 1;
1053
1054 dump_pq16_desc_dbg(ioat, desc);
1055
1056 /* we leave the channel locked to ensure in order submission */
1057 return &desc->txd;
1058}
1059
1060static int src_cnt_flags(unsigned int src_cnt, unsigned long flags)
1061{
1062 if (dmaf_p_disabled_continue(flags))
1063 return src_cnt + 1;
1064 else if (dmaf_continue(flags))
1065 return src_cnt + 3;
1066 else
1067 return src_cnt;
1068}
1069
1070static struct dma_async_tx_descriptor *
1071ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
1072 unsigned int src_cnt, const unsigned char *scf, size_t len,
1073 unsigned long flags)
1074{
1075 /* specify valid address for disabled result */
1076 if (flags & DMA_PREP_PQ_DISABLE_P)
1077 dst[0] = dst[1];
1078 if (flags & DMA_PREP_PQ_DISABLE_Q)
1079 dst[1] = dst[0];
1080
1081 /* handle the single source multiply case from the raid6
1082 * recovery path
1083 */
1084 if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) {
1085 dma_addr_t single_source[2];
1086 unsigned char single_source_coef[2];
1087
1088 BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q);
1089 single_source[0] = src[0];
1090 single_source[1] = src[0];
1091 single_source_coef[0] = scf[0];
1092 single_source_coef[1] = 0;
1093
1094 return src_cnt_flags(src_cnt, flags) > 8 ?
1095 __ioat3_prep_pq16_lock(chan, NULL, dst, single_source,
1096 2, single_source_coef, len,
1097 flags) :
1098 __ioat3_prep_pq_lock(chan, NULL, dst, single_source, 2,
1099 single_source_coef, len, flags);
1100
1101 } else {
1102 return src_cnt_flags(src_cnt, flags) > 8 ?
1103 __ioat3_prep_pq16_lock(chan, NULL, dst, src, src_cnt,
1104 scf, len, flags) :
1105 __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt,
1106 scf, len, flags);
1107 }
1108}
1109
1110static struct dma_async_tx_descriptor *
1111ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
1112 unsigned int src_cnt, const unsigned char *scf, size_t len,
1113 enum sum_check_flags *pqres, unsigned long flags)
1114{
1115 /* specify valid address for disabled result */
1116 if (flags & DMA_PREP_PQ_DISABLE_P)
1117 pq[0] = pq[1];
1118 if (flags & DMA_PREP_PQ_DISABLE_Q)
1119 pq[1] = pq[0];
1120
1121 /* the cleanup routine only sets bits on validate failure, it
1122 * does not clear bits on validate success... so clear it here
1123 */
1124 *pqres = 0;
1125
1126 return src_cnt_flags(src_cnt, flags) > 8 ?
1127 __ioat3_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len,
1128 flags) :
1129 __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
1130 flags);
1131}
1132
1133static struct dma_async_tx_descriptor *
1134ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
1135 unsigned int src_cnt, size_t len, unsigned long flags)
1136{
1137 unsigned char scf[src_cnt];
1138 dma_addr_t pq[2];
1139
1140 memset(scf, 0, src_cnt);
1141 pq[0] = dst;
1142 flags |= DMA_PREP_PQ_DISABLE_Q;
1143 pq[1] = dst; /* specify valid address for disabled result */
1144
1145 return src_cnt_flags(src_cnt, flags) > 8 ?
1146 __ioat3_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len,
1147 flags) :
1148 __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
1149 flags);
1150}
1151
1152static struct dma_async_tx_descriptor *
1153ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
1154 unsigned int src_cnt, size_t len,
1155 enum sum_check_flags *result, unsigned long flags)
1156{
1157 unsigned char scf[src_cnt];
1158 dma_addr_t pq[2];
1159
1160 /* the cleanup routine only sets bits on validate failure, it
1161 * does not clear bits on validate success... so clear it here
1162 */
1163 *result = 0;
1164
1165 memset(scf, 0, src_cnt);
1166 pq[0] = src[0];
1167 flags |= DMA_PREP_PQ_DISABLE_Q;
1168 pq[1] = pq[0]; /* specify valid address for disabled result */
1169
1170 return src_cnt_flags(src_cnt, flags) > 8 ?
1171 __ioat3_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1,
1172 scf, len, flags) :
1173 __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1,
1174 scf, len, flags);
1175}
1176
1177static struct dma_async_tx_descriptor *
1178ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
1179{
1180 struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
1181 struct ioat_ring_ent *desc;
1182 struct ioat_dma_descriptor *hw;
1183
1184 if (ioat2_check_space_lock(ioat, 1) == 0)
1185 desc = ioat2_get_ring_ent(ioat, ioat->head);
1186 else
1187 return NULL;
1188
1189 hw = desc->hw;
1190 hw->ctl = 0;
1191 hw->ctl_f.null = 1;
1192 hw->ctl_f.int_en = 1;
1193 hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
1194 hw->ctl_f.compl_write = 1;
1195 hw->size = NULL_DESC_BUFFER_SIZE;
1196 hw->src_addr = 0;
1197 hw->dst_addr = 0;
1198
1199 desc->txd.flags = flags;
1200 desc->len = 1;
1201
1202 dump_desc_dbg(ioat, desc);
1203
1204 /* we leave the channel locked to ensure in order submission */
1205 return &desc->txd;
1206}
1207
1208static void ioat3_dma_test_callback(void *dma_async_param)
1209{
1210 struct completion *cmp = dma_async_param;
1211
1212 complete(cmp);
1213}
1214
1215#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */
1216static int ioat_xor_val_self_test(struct ioatdma_device *device)
1217{
1218 int i, src_idx;
1219 struct page *dest;
1220 struct page *xor_srcs[IOAT_NUM_SRC_TEST];
1221 struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1];
1222 dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1];
1223 dma_addr_t dest_dma;
1224 struct dma_async_tx_descriptor *tx;
1225 struct dma_chan *dma_chan;
1226 dma_cookie_t cookie;
1227 u8 cmp_byte = 0;
1228 u32 cmp_word;
1229 u32 xor_val_result;
1230 int err = 0;
1231 struct completion cmp;
1232 unsigned long tmo;
1233 struct device *dev = &device->pdev->dev;
1234 struct dma_device *dma = &device->common;
1235 u8 op = 0;
1236
1237 dev_dbg(dev, "%s\n", __func__);
1238
1239 if (!dma_has_cap(DMA_XOR, dma->cap_mask))
1240 return 0;
1241
1242 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
1243 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
1244 if (!xor_srcs[src_idx]) {
1245 while (src_idx--)
1246 __free_page(xor_srcs[src_idx]);
1247 return -ENOMEM;
1248 }
1249 }
1250
1251 dest = alloc_page(GFP_KERNEL);
1252 if (!dest) {
1253 while (src_idx--)
1254 __free_page(xor_srcs[src_idx]);
1255 return -ENOMEM;
1256 }
1257
1258 /* Fill in src buffers */
1259 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
1260 u8 *ptr = page_address(xor_srcs[src_idx]);
1261 for (i = 0; i < PAGE_SIZE; i++)
1262 ptr[i] = (1 << src_idx);
1263 }
1264
1265 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++)
1266 cmp_byte ^= (u8) (1 << src_idx);
1267
1268 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
1269 (cmp_byte << 8) | cmp_byte;
1270
1271 memset(page_address(dest), 0, PAGE_SIZE);
1272
1273 dma_chan = container_of(dma->channels.next, struct dma_chan,
1274 device_node);
1275 if (dma->device_alloc_chan_resources(dma_chan) < 1) {
1276 err = -ENODEV;
1277 goto out;
1278 }
1279
1280 /* test xor */
1281 op = IOAT_OP_XOR;
1282
1283 dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE);
1284 if (dma_mapping_error(dev, dest_dma))
1285 goto dma_unmap;
1286
1287 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
1288 dma_srcs[i] = DMA_ERROR_CODE;
1289 for (i = 0; i < IOAT_NUM_SRC_TEST; i++) {
1290 dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE,
1291 DMA_TO_DEVICE);
1292 if (dma_mapping_error(dev, dma_srcs[i]))
1293 goto dma_unmap;
1294 }
1295 tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
1296 IOAT_NUM_SRC_TEST, PAGE_SIZE,
1297 DMA_PREP_INTERRUPT);
1298
1299 if (!tx) {
1300 dev_err(dev, "Self-test xor prep failed\n");
1301 err = -ENODEV;
1302 goto dma_unmap;
1303 }
1304
1305 async_tx_ack(tx);
1306 init_completion(&cmp);
1307 tx->callback = ioat3_dma_test_callback;
1308 tx->callback_param = &cmp;
1309 cookie = tx->tx_submit(tx);
1310 if (cookie < 0) {
1311 dev_err(dev, "Self-test xor setup failed\n");
1312 err = -ENODEV;
1313 goto dma_unmap;
1314 }
1315 dma->device_issue_pending(dma_chan);
1316
1317 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1318
1319 if (tmo == 0 ||
1320 dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1321 dev_err(dev, "Self-test xor timed out\n");
1322 err = -ENODEV;
1323 goto dma_unmap;
1324 }
1325
1326 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
1327 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
1328
1329 dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
1330 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
1331 u32 *ptr = page_address(dest);
1332 if (ptr[i] != cmp_word) {
1333 dev_err(dev, "Self-test xor failed compare\n");
1334 err = -ENODEV;
1335 goto free_resources;
1336 }
1337 }
1338 dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
1339
1340 dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
1341
1342 /* skip validate if the capability is not present */
1343 if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
1344 goto free_resources;
1345
1346 op = IOAT_OP_XOR_VAL;
1347
1348 /* validate the sources with the destintation page */
1349 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
1350 xor_val_srcs[i] = xor_srcs[i];
1351 xor_val_srcs[i] = dest;
1352
1353 xor_val_result = 1;
1354
1355 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
1356 dma_srcs[i] = DMA_ERROR_CODE;
1357 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) {
1358 dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
1359 DMA_TO_DEVICE);
1360 if (dma_mapping_error(dev, dma_srcs[i]))
1361 goto dma_unmap;
1362 }
1363 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
1364 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
1365 &xor_val_result, DMA_PREP_INTERRUPT);
1366 if (!tx) {
1367 dev_err(dev, "Self-test zero prep failed\n");
1368 err = -ENODEV;
1369 goto dma_unmap;
1370 }
1371
1372 async_tx_ack(tx);
1373 init_completion(&cmp);
1374 tx->callback = ioat3_dma_test_callback;
1375 tx->callback_param = &cmp;
1376 cookie = tx->tx_submit(tx);
1377 if (cookie < 0) {
1378 dev_err(dev, "Self-test zero setup failed\n");
1379 err = -ENODEV;
1380 goto dma_unmap;
1381 }
1382 dma->device_issue_pending(dma_chan);
1383
1384 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1385
1386 if (tmo == 0 ||
1387 dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1388 dev_err(dev, "Self-test validate timed out\n");
1389 err = -ENODEV;
1390 goto dma_unmap;
1391 }
1392
1393 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
1394 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
1395
1396 if (xor_val_result != 0) {
1397 dev_err(dev, "Self-test validate failed compare\n");
1398 err = -ENODEV;
1399 goto free_resources;
1400 }
1401
1402 memset(page_address(dest), 0, PAGE_SIZE);
1403
1404 /* test for non-zero parity sum */
1405 op = IOAT_OP_XOR_VAL;
1406
1407 xor_val_result = 0;
1408 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
1409 dma_srcs[i] = DMA_ERROR_CODE;
1410 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) {
1411 dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
1412 DMA_TO_DEVICE);
1413 if (dma_mapping_error(dev, dma_srcs[i]))
1414 goto dma_unmap;
1415 }
1416 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
1417 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
1418 &xor_val_result, DMA_PREP_INTERRUPT);
1419 if (!tx) {
1420 dev_err(dev, "Self-test 2nd zero prep failed\n");
1421 err = -ENODEV;
1422 goto dma_unmap;
1423 }
1424
1425 async_tx_ack(tx);
1426 init_completion(&cmp);
1427 tx->callback = ioat3_dma_test_callback;
1428 tx->callback_param = &cmp;
1429 cookie = tx->tx_submit(tx);
1430 if (cookie < 0) {
1431 dev_err(dev, "Self-test 2nd zero setup failed\n");
1432 err = -ENODEV;
1433 goto dma_unmap;
1434 }
1435 dma->device_issue_pending(dma_chan);
1436
1437 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1438
1439 if (tmo == 0 ||
1440 dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1441 dev_err(dev, "Self-test 2nd validate timed out\n");
1442 err = -ENODEV;
1443 goto dma_unmap;
1444 }
1445
1446 if (xor_val_result != SUM_CHECK_P_RESULT) {
1447 dev_err(dev, "Self-test validate failed compare\n");
1448 err = -ENODEV;
1449 goto dma_unmap;
1450 }
1451
1452 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
1453 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
1454
1455 goto free_resources;
1456dma_unmap:
1457 if (op == IOAT_OP_XOR) {
1458 if (dest_dma != DMA_ERROR_CODE)
1459 dma_unmap_page(dev, dest_dma, PAGE_SIZE,
1460 DMA_FROM_DEVICE);
1461 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
1462 if (dma_srcs[i] != DMA_ERROR_CODE)
1463 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE,
1464 DMA_TO_DEVICE);
1465 } else if (op == IOAT_OP_XOR_VAL) {
1466 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
1467 if (dma_srcs[i] != DMA_ERROR_CODE)
1468 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE,
1469 DMA_TO_DEVICE);
1470 }
1471free_resources:
1472 dma->device_free_chan_resources(dma_chan);
1473out:
1474 src_idx = IOAT_NUM_SRC_TEST;
1475 while (src_idx--)
1476 __free_page(xor_srcs[src_idx]);
1477 __free_page(dest);
1478 return err;
1479}
1480
1481static int ioat3_dma_self_test(struct ioatdma_device *device)
1482{
1483 int rc = ioat_dma_self_test(device);
1484
1485 if (rc)
1486 return rc;
1487
1488 rc = ioat_xor_val_self_test(device);
1489 if (rc)
1490 return rc;
1491
1492 return 0;
1493}
1494
1495static int ioat3_irq_reinit(struct ioatdma_device *device)
1496{
1497 struct pci_dev *pdev = device->pdev;
1498 int irq = pdev->irq, i;
1499
1500 if (!is_bwd_ioat(pdev))
1501 return 0;
1502
1503 switch (device->irq_mode) {
1504 case IOAT_MSIX:
1505 for (i = 0; i < device->common.chancnt; i++) {
1506 struct msix_entry *msix = &device->msix_entries[i];
1507 struct ioat_chan_common *chan;
1508
1509 chan = ioat_chan_by_index(device, i);
1510 devm_free_irq(&pdev->dev, msix->vector, chan);
1511 }
1512
1513 pci_disable_msix(pdev);
1514 break;
1515 case IOAT_MSI:
1516 pci_disable_msi(pdev);
1517 /* fall through */
1518 case IOAT_INTX:
1519 devm_free_irq(&pdev->dev, irq, device);
1520 break;
1521 default:
1522 return 0;
1523 }
1524 device->irq_mode = IOAT_NOIRQ;
1525
1526 return ioat_dma_setup_interrupts(device);
1527}
1528
1529static int ioat3_reset_hw(struct ioat_chan_common *chan)
1530{
1531 /* throw away whatever the channel was doing and get it
1532 * initialized, with ioat3 specific workarounds
1533 */
1534 struct ioatdma_device *device = chan->device;
1535 struct pci_dev *pdev = device->pdev;
1536 u32 chanerr;
1537 u16 dev_id;
1538 int err;
1539
1540 ioat2_quiesce(chan, msecs_to_jiffies(100));
1541
1542 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
1543 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
1544
1545 if (device->version < IOAT_VER_3_3) {
1546 /* clear any pending errors */
1547 err = pci_read_config_dword(pdev,
1548 IOAT_PCI_CHANERR_INT_OFFSET, &chanerr);
1549 if (err) {
1550 dev_err(&pdev->dev,
1551 "channel error register unreachable\n");
1552 return err;
1553 }
1554 pci_write_config_dword(pdev,
1555 IOAT_PCI_CHANERR_INT_OFFSET, chanerr);
1556
1557 /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
1558 * (workaround for spurious config parity error after restart)
1559 */
1560 pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id);
1561 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
1562 pci_write_config_dword(pdev,
1563 IOAT_PCI_DMAUNCERRSTS_OFFSET,
1564 0x10);
1565 }
1566 }
1567
1568 err = ioat2_reset_sync(chan, msecs_to_jiffies(200));
1569 if (!err)
1570 err = ioat3_irq_reinit(device);
1571
1572 if (err)
1573 dev_err(&pdev->dev, "Failed to reset: %d\n", err);
1574
1575 return err;
1576}
1577
1578static void ioat3_intr_quirk(struct ioatdma_device *device)
1579{
1580 struct dma_device *dma;
1581 struct dma_chan *c;
1582 struct ioat_chan_common *chan;
1583 u32 errmask;
1584
1585 dma = &device->common;
1586
1587 /*
1588 * if we have descriptor write back error status, we mask the
1589 * error interrupts
1590 */
1591 if (device->cap & IOAT_CAP_DWBES) {
1592 list_for_each_entry(c, &dma->channels, device_node) {
1593 chan = to_chan_common(c);
1594 errmask = readl(chan->reg_base +
1595 IOAT_CHANERR_MASK_OFFSET);
1596 errmask |= IOAT_CHANERR_XOR_P_OR_CRC_ERR |
1597 IOAT_CHANERR_XOR_Q_ERR;
1598 writel(errmask, chan->reg_base +
1599 IOAT_CHANERR_MASK_OFFSET);
1600 }
1601 }
1602}
1603
1604int ioat3_dma_probe(struct ioatdma_device *device, int dca)
1605{
1606 struct pci_dev *pdev = device->pdev;
1607 int dca_en = system_has_dca_enabled(pdev);
1608 struct dma_device *dma;
1609 struct dma_chan *c;
1610 struct ioat_chan_common *chan;
1611 bool is_raid_device = false;
1612 int err;
1613
1614 device->enumerate_channels = ioat2_enumerate_channels;
1615 device->reset_hw = ioat3_reset_hw;
1616 device->self_test = ioat3_dma_self_test;
1617 device->intr_quirk = ioat3_intr_quirk;
1618 dma = &device->common;
1619 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
1620 dma->device_issue_pending = ioat2_issue_pending;
1621 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
1622 dma->device_free_chan_resources = ioat2_free_chan_resources;
1623
1624 dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
1625 dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock;
1626
1627 device->cap = readl(device->reg_base + IOAT_DMA_CAP_OFFSET);
1628
1629 if (is_xeon_cb32(pdev) || is_bwd_noraid(pdev))
1630 device->cap &= ~(IOAT_CAP_XOR | IOAT_CAP_PQ | IOAT_CAP_RAID16SS);
1631
1632 /* dca is incompatible with raid operations */
1633 if (dca_en && (device->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ)))
1634 device->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ);
1635
1636 if (device->cap & IOAT_CAP_XOR) {
1637 is_raid_device = true;
1638 dma->max_xor = 8;
1639
1640 dma_cap_set(DMA_XOR, dma->cap_mask);
1641 dma->device_prep_dma_xor = ioat3_prep_xor;
1642
1643 dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
1644 dma->device_prep_dma_xor_val = ioat3_prep_xor_val;
1645 }
1646
1647 if (device->cap & IOAT_CAP_PQ) {
1648 is_raid_device = true;
1649
1650 dma->device_prep_dma_pq = ioat3_prep_pq;
1651 dma->device_prep_dma_pq_val = ioat3_prep_pq_val;
1652 dma_cap_set(DMA_PQ, dma->cap_mask);
1653 dma_cap_set(DMA_PQ_VAL, dma->cap_mask);
1654
1655 if (device->cap & IOAT_CAP_RAID16SS) {
1656 dma_set_maxpq(dma, 16, 0);
1657 } else {
1658 dma_set_maxpq(dma, 8, 0);
1659 }
1660
1661 if (!(device->cap & IOAT_CAP_XOR)) {
1662 dma->device_prep_dma_xor = ioat3_prep_pqxor;
1663 dma->device_prep_dma_xor_val = ioat3_prep_pqxor_val;
1664 dma_cap_set(DMA_XOR, dma->cap_mask);
1665 dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
1666
1667 if (device->cap & IOAT_CAP_RAID16SS) {
1668 dma->max_xor = 16;
1669 } else {
1670 dma->max_xor = 8;
1671 }
1672 }
1673 }
1674
1675 dma->device_tx_status = ioat3_tx_status;
1676 device->cleanup_fn = ioat3_cleanup_event;
1677 device->timer_fn = ioat3_timer_event;
1678
1679 /* starting with CB3.3 super extended descriptors are supported */
1680 if (device->cap & IOAT_CAP_RAID16SS) {
1681 char pool_name[14];
1682 int i;
1683
1684 for (i = 0; i < MAX_SED_POOLS; i++) {
1685 snprintf(pool_name, 14, "ioat_hw%d_sed", i);
1686
1687 /* allocate SED DMA pool */
1688 device->sed_hw_pool[i] = dmam_pool_create(pool_name,
1689 &pdev->dev,
1690 SED_SIZE * (i + 1), 64, 0);
1691 if (!device->sed_hw_pool[i])
1692 return -ENOMEM;
1693
1694 }
1695 }
1696
1697 err = ioat_probe(device);
1698 if (err)
1699 return err;
1700
1701 list_for_each_entry(c, &dma->channels, device_node) {
1702 chan = to_chan_common(c);
1703 writel(IOAT_DMA_DCA_ANY_CPU,
1704 chan->reg_base + IOAT_DCACTRL_OFFSET);
1705 }
1706
1707 err = ioat_register(device);
1708 if (err)
1709 return err;
1710
1711 ioat_kobject_add(device, &ioat2_ktype);
1712
1713 if (dca)
1714 device->dca = ioat3_dca_init(pdev, device->reg_base);
1715
1716 return 0;
1717}
diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h
index a3e731edce57..690e3b4f8202 100644
--- a/drivers/dma/ioat/hw.h
+++ b/drivers/dma/ioat/hw.h
@@ -21,11 +21,6 @@
21#define IOAT_MMIO_BAR 0 21#define IOAT_MMIO_BAR 0
22 22
23/* CB device ID's */ 23/* CB device ID's */
24#define IOAT_PCI_DID_5000 0x1A38
25#define IOAT_PCI_DID_CNB 0x360B
26#define IOAT_PCI_DID_SCNB 0x65FF
27#define IOAT_PCI_DID_SNB 0x402F
28
29#define PCI_DEVICE_ID_INTEL_IOAT_IVB0 0x0e20 24#define PCI_DEVICE_ID_INTEL_IOAT_IVB0 0x0e20
30#define PCI_DEVICE_ID_INTEL_IOAT_IVB1 0x0e21 25#define PCI_DEVICE_ID_INTEL_IOAT_IVB1 0x0e21
31#define PCI_DEVICE_ID_INTEL_IOAT_IVB2 0x0e22 26#define PCI_DEVICE_ID_INTEL_IOAT_IVB2 0x0e22
@@ -58,6 +53,17 @@
58#define PCI_DEVICE_ID_INTEL_IOAT_BDXDE2 0x6f52 53#define PCI_DEVICE_ID_INTEL_IOAT_BDXDE2 0x6f52
59#define PCI_DEVICE_ID_INTEL_IOAT_BDXDE3 0x6f53 54#define PCI_DEVICE_ID_INTEL_IOAT_BDXDE3 0x6f53
60 55
56#define PCI_DEVICE_ID_INTEL_IOAT_BDX0 0x6f20
57#define PCI_DEVICE_ID_INTEL_IOAT_BDX1 0x6f21
58#define PCI_DEVICE_ID_INTEL_IOAT_BDX2 0x6f22
59#define PCI_DEVICE_ID_INTEL_IOAT_BDX3 0x6f23
60#define PCI_DEVICE_ID_INTEL_IOAT_BDX4 0x6f24
61#define PCI_DEVICE_ID_INTEL_IOAT_BDX5 0x6f25
62#define PCI_DEVICE_ID_INTEL_IOAT_BDX6 0x6f26
63#define PCI_DEVICE_ID_INTEL_IOAT_BDX7 0x6f27
64#define PCI_DEVICE_ID_INTEL_IOAT_BDX8 0x6f2e
65#define PCI_DEVICE_ID_INTEL_IOAT_BDX9 0x6f2f
66
61#define IOAT_VER_1_2 0x12 /* Version 1.2 */ 67#define IOAT_VER_1_2 0x12 /* Version 1.2 */
62#define IOAT_VER_2_0 0x20 /* Version 2.0 */ 68#define IOAT_VER_2_0 0x20 /* Version 2.0 */
63#define IOAT_VER_3_0 0x30 /* Version 3.0 */ 69#define IOAT_VER_3_0 0x30 /* Version 3.0 */
diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c
new file mode 100644
index 000000000000..1c3c9b0abf4e
--- /dev/null
+++ b/drivers/dma/ioat/init.c
@@ -0,0 +1,1314 @@
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2015 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * The full GNU General Public License is included in this distribution in
15 * the file called "COPYING".
16 *
17 */
18
19#include <linux/init.h>
20#include <linux/module.h>
21#include <linux/slab.h>
22#include <linux/pci.h>
23#include <linux/interrupt.h>
24#include <linux/dmaengine.h>
25#include <linux/delay.h>
26#include <linux/dma-mapping.h>
27#include <linux/workqueue.h>
28#include <linux/prefetch.h>
29#include <linux/dca.h>
30#include "dma.h"
31#include "registers.h"
32#include "hw.h"
33
34#include "../dmaengine.h"
35
36MODULE_VERSION(IOAT_DMA_VERSION);
37MODULE_LICENSE("Dual BSD/GPL");
38MODULE_AUTHOR("Intel Corporation");
39
40static struct pci_device_id ioat_pci_tbl[] = {
41 /* I/OAT v3 platforms */
42 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
43 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
44 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },
45 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) },
46 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) },
47 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) },
48 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) },
49 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) },
50
51 /* I/OAT v3.2 platforms */
52 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) },
53 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) },
54 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) },
55 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) },
56 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) },
57 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) },
58 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) },
59 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) },
60 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) },
61 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) },
62
63 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0) },
64 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1) },
65 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2) },
66 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3) },
67 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4) },
68 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5) },
69 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6) },
70 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7) },
71 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB8) },
72 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB9) },
73
74 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0) },
75 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1) },
76 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2) },
77 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3) },
78 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4) },
79 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5) },
80 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6) },
81 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7) },
82 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB8) },
83 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB9) },
84
85 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0) },
86 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1) },
87 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2) },
88 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3) },
89 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4) },
90 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5) },
91 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6) },
92 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7) },
93 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW8) },
94 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW9) },
95
96 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX0) },
97 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX1) },
98 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX2) },
99 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX3) },
100 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX4) },
101 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX5) },
102 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX6) },
103 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX7) },
104 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX8) },
105 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX9) },
106
107 /* I/OAT v3.3 platforms */
108 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) },
109 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) },
110 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2) },
111 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3) },
112
113 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0) },
114 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1) },
115 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2) },
116 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3) },
117
118 { 0, }
119};
120MODULE_DEVICE_TABLE(pci, ioat_pci_tbl);
121
122static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
123static void ioat_remove(struct pci_dev *pdev);
124static void
125ioat_init_channel(struct ioatdma_device *ioat_dma,
126 struct ioatdma_chan *ioat_chan, int idx);
127static void ioat_intr_quirk(struct ioatdma_device *ioat_dma);
128static int ioat_enumerate_channels(struct ioatdma_device *ioat_dma);
129static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma);
130
131static int ioat_dca_enabled = 1;
132module_param(ioat_dca_enabled, int, 0644);
133MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
134int ioat_pending_level = 4;
135module_param(ioat_pending_level, int, 0644);
136MODULE_PARM_DESC(ioat_pending_level,
137 "high-water mark for pushing ioat descriptors (default: 4)");
138int ioat_ring_alloc_order = 8;
139module_param(ioat_ring_alloc_order, int, 0644);
140MODULE_PARM_DESC(ioat_ring_alloc_order,
141 "ioat+: allocate 2^n descriptors per channel (default: 8 max: 16)");
142int ioat_ring_max_alloc_order = IOAT_MAX_ORDER;
143module_param(ioat_ring_max_alloc_order, int, 0644);
144MODULE_PARM_DESC(ioat_ring_max_alloc_order,
145 "ioat+: upper limit for ring size (default: 16)");
146static char ioat_interrupt_style[32] = "msix";
147module_param_string(ioat_interrupt_style, ioat_interrupt_style,
148 sizeof(ioat_interrupt_style), 0644);
149MODULE_PARM_DESC(ioat_interrupt_style,
150 "set ioat interrupt style: msix (default), msi, intx");
151
152struct kmem_cache *ioat_cache;
153struct kmem_cache *ioat_sed_cache;
154
155static bool is_jf_ioat(struct pci_dev *pdev)
156{
157 switch (pdev->device) {
158 case PCI_DEVICE_ID_INTEL_IOAT_JSF0:
159 case PCI_DEVICE_ID_INTEL_IOAT_JSF1:
160 case PCI_DEVICE_ID_INTEL_IOAT_JSF2:
161 case PCI_DEVICE_ID_INTEL_IOAT_JSF3:
162 case PCI_DEVICE_ID_INTEL_IOAT_JSF4:
163 case PCI_DEVICE_ID_INTEL_IOAT_JSF5:
164 case PCI_DEVICE_ID_INTEL_IOAT_JSF6:
165 case PCI_DEVICE_ID_INTEL_IOAT_JSF7:
166 case PCI_DEVICE_ID_INTEL_IOAT_JSF8:
167 case PCI_DEVICE_ID_INTEL_IOAT_JSF9:
168 return true;
169 default:
170 return false;
171 }
172}
173
174static bool is_snb_ioat(struct pci_dev *pdev)
175{
176 switch (pdev->device) {
177 case PCI_DEVICE_ID_INTEL_IOAT_SNB0:
178 case PCI_DEVICE_ID_INTEL_IOAT_SNB1:
179 case PCI_DEVICE_ID_INTEL_IOAT_SNB2:
180 case PCI_DEVICE_ID_INTEL_IOAT_SNB3:
181 case PCI_DEVICE_ID_INTEL_IOAT_SNB4:
182 case PCI_DEVICE_ID_INTEL_IOAT_SNB5:
183 case PCI_DEVICE_ID_INTEL_IOAT_SNB6:
184 case PCI_DEVICE_ID_INTEL_IOAT_SNB7:
185 case PCI_DEVICE_ID_INTEL_IOAT_SNB8:
186 case PCI_DEVICE_ID_INTEL_IOAT_SNB9:
187 return true;
188 default:
189 return false;
190 }
191}
192
193static bool is_ivb_ioat(struct pci_dev *pdev)
194{
195 switch (pdev->device) {
196 case PCI_DEVICE_ID_INTEL_IOAT_IVB0:
197 case PCI_DEVICE_ID_INTEL_IOAT_IVB1:
198 case PCI_DEVICE_ID_INTEL_IOAT_IVB2:
199 case PCI_DEVICE_ID_INTEL_IOAT_IVB3:
200 case PCI_DEVICE_ID_INTEL_IOAT_IVB4:
201 case PCI_DEVICE_ID_INTEL_IOAT_IVB5:
202 case PCI_DEVICE_ID_INTEL_IOAT_IVB6:
203 case PCI_DEVICE_ID_INTEL_IOAT_IVB7:
204 case PCI_DEVICE_ID_INTEL_IOAT_IVB8:
205 case PCI_DEVICE_ID_INTEL_IOAT_IVB9:
206 return true;
207 default:
208 return false;
209 }
210
211}
212
213static bool is_hsw_ioat(struct pci_dev *pdev)
214{
215 switch (pdev->device) {
216 case PCI_DEVICE_ID_INTEL_IOAT_HSW0:
217 case PCI_DEVICE_ID_INTEL_IOAT_HSW1:
218 case PCI_DEVICE_ID_INTEL_IOAT_HSW2:
219 case PCI_DEVICE_ID_INTEL_IOAT_HSW3:
220 case PCI_DEVICE_ID_INTEL_IOAT_HSW4:
221 case PCI_DEVICE_ID_INTEL_IOAT_HSW5:
222 case PCI_DEVICE_ID_INTEL_IOAT_HSW6:
223 case PCI_DEVICE_ID_INTEL_IOAT_HSW7:
224 case PCI_DEVICE_ID_INTEL_IOAT_HSW8:
225 case PCI_DEVICE_ID_INTEL_IOAT_HSW9:
226 return true;
227 default:
228 return false;
229 }
230
231}
232
233static bool is_bdx_ioat(struct pci_dev *pdev)
234{
235 switch (pdev->device) {
236 case PCI_DEVICE_ID_INTEL_IOAT_BDX0:
237 case PCI_DEVICE_ID_INTEL_IOAT_BDX1:
238 case PCI_DEVICE_ID_INTEL_IOAT_BDX2:
239 case PCI_DEVICE_ID_INTEL_IOAT_BDX3:
240 case PCI_DEVICE_ID_INTEL_IOAT_BDX4:
241 case PCI_DEVICE_ID_INTEL_IOAT_BDX5:
242 case PCI_DEVICE_ID_INTEL_IOAT_BDX6:
243 case PCI_DEVICE_ID_INTEL_IOAT_BDX7:
244 case PCI_DEVICE_ID_INTEL_IOAT_BDX8:
245 case PCI_DEVICE_ID_INTEL_IOAT_BDX9:
246 return true;
247 default:
248 return false;
249 }
250}
251
252static bool is_xeon_cb32(struct pci_dev *pdev)
253{
254 return is_jf_ioat(pdev) || is_snb_ioat(pdev) || is_ivb_ioat(pdev) ||
255 is_hsw_ioat(pdev) || is_bdx_ioat(pdev);
256}
257
258bool is_bwd_ioat(struct pci_dev *pdev)
259{
260 switch (pdev->device) {
261 case PCI_DEVICE_ID_INTEL_IOAT_BWD0:
262 case PCI_DEVICE_ID_INTEL_IOAT_BWD1:
263 case PCI_DEVICE_ID_INTEL_IOAT_BWD2:
264 case PCI_DEVICE_ID_INTEL_IOAT_BWD3:
265 /* even though not Atom, BDX-DE has same DMA silicon */
266 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0:
267 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1:
268 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2:
269 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3:
270 return true;
271 default:
272 return false;
273 }
274}
275
276static bool is_bwd_noraid(struct pci_dev *pdev)
277{
278 switch (pdev->device) {
279 case PCI_DEVICE_ID_INTEL_IOAT_BWD2:
280 case PCI_DEVICE_ID_INTEL_IOAT_BWD3:
281 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0:
282 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1:
283 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2:
284 case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3:
285 return true;
286 default:
287 return false;
288 }
289
290}
291
292/*
293 * Perform a IOAT transaction to verify the HW works.
294 */
295#define IOAT_TEST_SIZE 2000
296
297static void ioat_dma_test_callback(void *dma_async_param)
298{
299 struct completion *cmp = dma_async_param;
300
301 complete(cmp);
302}
303
304/**
305 * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
306 * @ioat_dma: dma device to be tested
307 */
308static int ioat_dma_self_test(struct ioatdma_device *ioat_dma)
309{
310 int i;
311 u8 *src;
312 u8 *dest;
313 struct dma_device *dma = &ioat_dma->dma_dev;
314 struct device *dev = &ioat_dma->pdev->dev;
315 struct dma_chan *dma_chan;
316 struct dma_async_tx_descriptor *tx;
317 dma_addr_t dma_dest, dma_src;
318 dma_cookie_t cookie;
319 int err = 0;
320 struct completion cmp;
321 unsigned long tmo;
322 unsigned long flags;
323
324 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
325 if (!src)
326 return -ENOMEM;
327 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
328 if (!dest) {
329 kfree(src);
330 return -ENOMEM;
331 }
332
333 /* Fill in src buffer */
334 for (i = 0; i < IOAT_TEST_SIZE; i++)
335 src[i] = (u8)i;
336
337 /* Start copy, using first DMA channel */
338 dma_chan = container_of(dma->channels.next, struct dma_chan,
339 device_node);
340 if (dma->device_alloc_chan_resources(dma_chan) < 1) {
341 dev_err(dev, "selftest cannot allocate chan resource\n");
342 err = -ENODEV;
343 goto out;
344 }
345
346 dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
347 if (dma_mapping_error(dev, dma_src)) {
348 dev_err(dev, "mapping src buffer failed\n");
349 goto free_resources;
350 }
351 dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
352 if (dma_mapping_error(dev, dma_dest)) {
353 dev_err(dev, "mapping dest buffer failed\n");
354 goto unmap_src;
355 }
356 flags = DMA_PREP_INTERRUPT;
357 tx = ioat_dma->dma_dev.device_prep_dma_memcpy(dma_chan, dma_dest,
358 dma_src, IOAT_TEST_SIZE,
359 flags);
360 if (!tx) {
361 dev_err(dev, "Self-test prep failed, disabling\n");
362 err = -ENODEV;
363 goto unmap_dma;
364 }
365
366 async_tx_ack(tx);
367 init_completion(&cmp);
368 tx->callback = ioat_dma_test_callback;
369 tx->callback_param = &cmp;
370 cookie = tx->tx_submit(tx);
371 if (cookie < 0) {
372 dev_err(dev, "Self-test setup failed, disabling\n");
373 err = -ENODEV;
374 goto unmap_dma;
375 }
376 dma->device_issue_pending(dma_chan);
377
378 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
379
380 if (tmo == 0 ||
381 dma->device_tx_status(dma_chan, cookie, NULL)
382 != DMA_COMPLETE) {
383 dev_err(dev, "Self-test copy timed out, disabling\n");
384 err = -ENODEV;
385 goto unmap_dma;
386 }
387 if (memcmp(src, dest, IOAT_TEST_SIZE)) {
388 dev_err(dev, "Self-test copy failed compare, disabling\n");
389 err = -ENODEV;
390 goto free_resources;
391 }
392
393unmap_dma:
394 dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
395unmap_src:
396 dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
397free_resources:
398 dma->device_free_chan_resources(dma_chan);
399out:
400 kfree(src);
401 kfree(dest);
402 return err;
403}
404
405/**
406 * ioat_dma_setup_interrupts - setup interrupt handler
407 * @ioat_dma: ioat dma device
408 */
409int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma)
410{
411 struct ioatdma_chan *ioat_chan;
412 struct pci_dev *pdev = ioat_dma->pdev;
413 struct device *dev = &pdev->dev;
414 struct msix_entry *msix;
415 int i, j, msixcnt;
416 int err = -EINVAL;
417 u8 intrctrl = 0;
418
419 if (!strcmp(ioat_interrupt_style, "msix"))
420 goto msix;
421 if (!strcmp(ioat_interrupt_style, "msi"))
422 goto msi;
423 if (!strcmp(ioat_interrupt_style, "intx"))
424 goto intx;
425 dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
426 goto err_no_irq;
427
428msix:
429 /* The number of MSI-X vectors should equal the number of channels */
430 msixcnt = ioat_dma->dma_dev.chancnt;
431 for (i = 0; i < msixcnt; i++)
432 ioat_dma->msix_entries[i].entry = i;
433
434 err = pci_enable_msix_exact(pdev, ioat_dma->msix_entries, msixcnt);
435 if (err)
436 goto msi;
437
438 for (i = 0; i < msixcnt; i++) {
439 msix = &ioat_dma->msix_entries[i];
440 ioat_chan = ioat_chan_by_index(ioat_dma, i);
441 err = devm_request_irq(dev, msix->vector,
442 ioat_dma_do_interrupt_msix, 0,
443 "ioat-msix", ioat_chan);
444 if (err) {
445 for (j = 0; j < i; j++) {
446 msix = &ioat_dma->msix_entries[j];
447 ioat_chan = ioat_chan_by_index(ioat_dma, j);
448 devm_free_irq(dev, msix->vector, ioat_chan);
449 }
450 goto msi;
451 }
452 }
453 intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
454 ioat_dma->irq_mode = IOAT_MSIX;
455 goto done;
456
457msi:
458 err = pci_enable_msi(pdev);
459 if (err)
460 goto intx;
461
462 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
463 "ioat-msi", ioat_dma);
464 if (err) {
465 pci_disable_msi(pdev);
466 goto intx;
467 }
468 ioat_dma->irq_mode = IOAT_MSI;
469 goto done;
470
471intx:
472 err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
473 IRQF_SHARED, "ioat-intx", ioat_dma);
474 if (err)
475 goto err_no_irq;
476
477 ioat_dma->irq_mode = IOAT_INTX;
478done:
479 if (is_bwd_ioat(pdev))
480 ioat_intr_quirk(ioat_dma);
481 intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
482 writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
483 return 0;
484
485err_no_irq:
486 /* Disable all interrupt generation */
487 writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
488 ioat_dma->irq_mode = IOAT_NOIRQ;
489 dev_err(dev, "no usable interrupts\n");
490 return err;
491}
492
493static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma)
494{
495 /* Disable all interrupt generation */
496 writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET);
497}
498
499static int ioat_probe(struct ioatdma_device *ioat_dma)
500{
501 int err = -ENODEV;
502 struct dma_device *dma = &ioat_dma->dma_dev;
503 struct pci_dev *pdev = ioat_dma->pdev;
504 struct device *dev = &pdev->dev;
505
506 /* DMA coherent memory pool for DMA descriptor allocations */
507 ioat_dma->dma_pool = pci_pool_create("dma_desc_pool", pdev,
508 sizeof(struct ioat_dma_descriptor),
509 64, 0);
510 if (!ioat_dma->dma_pool) {
511 err = -ENOMEM;
512 goto err_dma_pool;
513 }
514
515 ioat_dma->completion_pool = pci_pool_create("completion_pool", pdev,
516 sizeof(u64),
517 SMP_CACHE_BYTES,
518 SMP_CACHE_BYTES);
519
520 if (!ioat_dma->completion_pool) {
521 err = -ENOMEM;
522 goto err_completion_pool;
523 }
524
525 ioat_enumerate_channels(ioat_dma);
526
527 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
528 dma->dev = &pdev->dev;
529
530 if (!dma->chancnt) {
531 dev_err(dev, "channel enumeration error\n");
532 goto err_setup_interrupts;
533 }
534
535 err = ioat_dma_setup_interrupts(ioat_dma);
536 if (err)
537 goto err_setup_interrupts;
538
539 err = ioat3_dma_self_test(ioat_dma);
540 if (err)
541 goto err_self_test;
542
543 return 0;
544
545err_self_test:
546 ioat_disable_interrupts(ioat_dma);
547err_setup_interrupts:
548 pci_pool_destroy(ioat_dma->completion_pool);
549err_completion_pool:
550 pci_pool_destroy(ioat_dma->dma_pool);
551err_dma_pool:
552 return err;
553}
554
555static int ioat_register(struct ioatdma_device *ioat_dma)
556{
557 int err = dma_async_device_register(&ioat_dma->dma_dev);
558
559 if (err) {
560 ioat_disable_interrupts(ioat_dma);
561 pci_pool_destroy(ioat_dma->completion_pool);
562 pci_pool_destroy(ioat_dma->dma_pool);
563 }
564
565 return err;
566}
567
568static void ioat_dma_remove(struct ioatdma_device *ioat_dma)
569{
570 struct dma_device *dma = &ioat_dma->dma_dev;
571
572 ioat_disable_interrupts(ioat_dma);
573
574 ioat_kobject_del(ioat_dma);
575
576 dma_async_device_unregister(dma);
577
578 pci_pool_destroy(ioat_dma->dma_pool);
579 pci_pool_destroy(ioat_dma->completion_pool);
580
581 INIT_LIST_HEAD(&dma->channels);
582}
583
584/**
585 * ioat_enumerate_channels - find and initialize the device's channels
586 * @ioat_dma: the ioat dma device to be enumerated
587 */
588static int ioat_enumerate_channels(struct ioatdma_device *ioat_dma)
589{
590 struct ioatdma_chan *ioat_chan;
591 struct device *dev = &ioat_dma->pdev->dev;
592 struct dma_device *dma = &ioat_dma->dma_dev;
593 u8 xfercap_log;
594 int i;
595
596 INIT_LIST_HEAD(&dma->channels);
597 dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET);
598 dma->chancnt &= 0x1f; /* bits [4:0] valid */
599 if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) {
600 dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
601 dma->chancnt, ARRAY_SIZE(ioat_dma->idx));
602 dma->chancnt = ARRAY_SIZE(ioat_dma->idx);
603 }
604 xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET);
605 xfercap_log &= 0x1f; /* bits [4:0] valid */
606 if (xfercap_log == 0)
607 return 0;
608 dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
609
610 for (i = 0; i < dma->chancnt; i++) {
611 ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
612 if (!ioat_chan)
613 break;
614
615 ioat_init_channel(ioat_dma, ioat_chan, i);
616 ioat_chan->xfercap_log = xfercap_log;
617 spin_lock_init(&ioat_chan->prep_lock);
618 if (ioat_reset_hw(ioat_chan)) {
619 i = 0;
620 break;
621 }
622 }
623 dma->chancnt = i;
624 return i;
625}
626
627/**
628 * ioat_free_chan_resources - release all the descriptors
629 * @chan: the channel to be cleaned
630 */
631static void ioat_free_chan_resources(struct dma_chan *c)
632{
633 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
634 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
635 struct ioat_ring_ent *desc;
636 const int total_descs = 1 << ioat_chan->alloc_order;
637 int descs;
638 int i;
639
640 /* Before freeing channel resources first check
641 * if they have been previously allocated for this channel.
642 */
643 if (!ioat_chan->ring)
644 return;
645
646 ioat_stop(ioat_chan);
647 ioat_reset_hw(ioat_chan);
648
649 spin_lock_bh(&ioat_chan->cleanup_lock);
650 spin_lock_bh(&ioat_chan->prep_lock);
651 descs = ioat_ring_space(ioat_chan);
652 dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs);
653 for (i = 0; i < descs; i++) {
654 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head + i);
655 ioat_free_ring_ent(desc, c);
656 }
657
658 if (descs < total_descs)
659 dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n",
660 total_descs - descs);
661
662 for (i = 0; i < total_descs - descs; i++) {
663 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail + i);
664 dump_desc_dbg(ioat_chan, desc);
665 ioat_free_ring_ent(desc, c);
666 }
667
668 kfree(ioat_chan->ring);
669 ioat_chan->ring = NULL;
670 ioat_chan->alloc_order = 0;
671 pci_pool_free(ioat_dma->completion_pool, ioat_chan->completion,
672 ioat_chan->completion_dma);
673 spin_unlock_bh(&ioat_chan->prep_lock);
674 spin_unlock_bh(&ioat_chan->cleanup_lock);
675
676 ioat_chan->last_completion = 0;
677 ioat_chan->completion_dma = 0;
678 ioat_chan->dmacount = 0;
679}
680
681/* ioat_alloc_chan_resources - allocate/initialize ioat descriptor ring
682 * @chan: channel to be initialized
683 */
684static int ioat_alloc_chan_resources(struct dma_chan *c)
685{
686 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
687 struct ioat_ring_ent **ring;
688 u64 status;
689 int order;
690 int i = 0;
691 u32 chanerr;
692
693 /* have we already been set up? */
694 if (ioat_chan->ring)
695 return 1 << ioat_chan->alloc_order;
696
697 /* Setup register to interrupt and write completion status on error */
698 writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
699
700 /* allocate a completion writeback area */
701 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
702 ioat_chan->completion =
703 pci_pool_alloc(ioat_chan->ioat_dma->completion_pool,
704 GFP_KERNEL, &ioat_chan->completion_dma);
705 if (!ioat_chan->completion)
706 return -ENOMEM;
707
708 memset(ioat_chan->completion, 0, sizeof(*ioat_chan->completion));
709 writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF,
710 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
711 writel(((u64)ioat_chan->completion_dma) >> 32,
712 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
713
714 order = ioat_get_alloc_order();
715 ring = ioat_alloc_ring(c, order, GFP_KERNEL);
716 if (!ring)
717 return -ENOMEM;
718
719 spin_lock_bh(&ioat_chan->cleanup_lock);
720 spin_lock_bh(&ioat_chan->prep_lock);
721 ioat_chan->ring = ring;
722 ioat_chan->head = 0;
723 ioat_chan->issued = 0;
724 ioat_chan->tail = 0;
725 ioat_chan->alloc_order = order;
726 set_bit(IOAT_RUN, &ioat_chan->state);
727 spin_unlock_bh(&ioat_chan->prep_lock);
728 spin_unlock_bh(&ioat_chan->cleanup_lock);
729
730 ioat_start_null_desc(ioat_chan);
731
732 /* check that we got off the ground */
733 do {
734 udelay(1);
735 status = ioat_chansts(ioat_chan);
736 } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status));
737
738 if (is_ioat_active(status) || is_ioat_idle(status))
739 return 1 << ioat_chan->alloc_order;
740
741 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
742
743 dev_WARN(to_dev(ioat_chan),
744 "failed to start channel chanerr: %#x\n", chanerr);
745 ioat_free_chan_resources(c);
746 return -EFAULT;
747}
748
749/* common channel initialization */
750static void
751ioat_init_channel(struct ioatdma_device *ioat_dma,
752 struct ioatdma_chan *ioat_chan, int idx)
753{
754 struct dma_device *dma = &ioat_dma->dma_dev;
755 struct dma_chan *c = &ioat_chan->dma_chan;
756 unsigned long data = (unsigned long) c;
757
758 ioat_chan->ioat_dma = ioat_dma;
759 ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1));
760 spin_lock_init(&ioat_chan->cleanup_lock);
761 ioat_chan->dma_chan.device = dma;
762 dma_cookie_init(&ioat_chan->dma_chan);
763 list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels);
764 ioat_dma->idx[idx] = ioat_chan;
765 init_timer(&ioat_chan->timer);
766 ioat_chan->timer.function = ioat_timer_event;
767 ioat_chan->timer.data = data;
768 tasklet_init(&ioat_chan->cleanup_task, ioat_cleanup_event, data);
769}
770
771#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */
772static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma)
773{
774 int i, src_idx;
775 struct page *dest;
776 struct page *xor_srcs[IOAT_NUM_SRC_TEST];
777 struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1];
778 dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1];
779 dma_addr_t dest_dma;
780 struct dma_async_tx_descriptor *tx;
781 struct dma_chan *dma_chan;
782 dma_cookie_t cookie;
783 u8 cmp_byte = 0;
784 u32 cmp_word;
785 u32 xor_val_result;
786 int err = 0;
787 struct completion cmp;
788 unsigned long tmo;
789 struct device *dev = &ioat_dma->pdev->dev;
790 struct dma_device *dma = &ioat_dma->dma_dev;
791 u8 op = 0;
792
793 dev_dbg(dev, "%s\n", __func__);
794
795 if (!dma_has_cap(DMA_XOR, dma->cap_mask))
796 return 0;
797
798 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
799 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
800 if (!xor_srcs[src_idx]) {
801 while (src_idx--)
802 __free_page(xor_srcs[src_idx]);
803 return -ENOMEM;
804 }
805 }
806
807 dest = alloc_page(GFP_KERNEL);
808 if (!dest) {
809 while (src_idx--)
810 __free_page(xor_srcs[src_idx]);
811 return -ENOMEM;
812 }
813
814 /* Fill in src buffers */
815 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
816 u8 *ptr = page_address(xor_srcs[src_idx]);
817
818 for (i = 0; i < PAGE_SIZE; i++)
819 ptr[i] = (1 << src_idx);
820 }
821
822 for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++)
823 cmp_byte ^= (u8) (1 << src_idx);
824
825 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
826 (cmp_byte << 8) | cmp_byte;
827
828 memset(page_address(dest), 0, PAGE_SIZE);
829
830 dma_chan = container_of(dma->channels.next, struct dma_chan,
831 device_node);
832 if (dma->device_alloc_chan_resources(dma_chan) < 1) {
833 err = -ENODEV;
834 goto out;
835 }
836
837 /* test xor */
838 op = IOAT_OP_XOR;
839
840 dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE);
841 if (dma_mapping_error(dev, dest_dma))
842 goto dma_unmap;
843
844 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
845 dma_srcs[i] = DMA_ERROR_CODE;
846 for (i = 0; i < IOAT_NUM_SRC_TEST; i++) {
847 dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE,
848 DMA_TO_DEVICE);
849 if (dma_mapping_error(dev, dma_srcs[i]))
850 goto dma_unmap;
851 }
852 tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
853 IOAT_NUM_SRC_TEST, PAGE_SIZE,
854 DMA_PREP_INTERRUPT);
855
856 if (!tx) {
857 dev_err(dev, "Self-test xor prep failed\n");
858 err = -ENODEV;
859 goto dma_unmap;
860 }
861
862 async_tx_ack(tx);
863 init_completion(&cmp);
864 tx->callback = ioat_dma_test_callback;
865 tx->callback_param = &cmp;
866 cookie = tx->tx_submit(tx);
867 if (cookie < 0) {
868 dev_err(dev, "Self-test xor setup failed\n");
869 err = -ENODEV;
870 goto dma_unmap;
871 }
872 dma->device_issue_pending(dma_chan);
873
874 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
875
876 if (tmo == 0 ||
877 dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
878 dev_err(dev, "Self-test xor timed out\n");
879 err = -ENODEV;
880 goto dma_unmap;
881 }
882
883 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
884 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
885
886 dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
887 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
888 u32 *ptr = page_address(dest);
889
890 if (ptr[i] != cmp_word) {
891 dev_err(dev, "Self-test xor failed compare\n");
892 err = -ENODEV;
893 goto free_resources;
894 }
895 }
896 dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
897
898 dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
899
900 /* skip validate if the capability is not present */
901 if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
902 goto free_resources;
903
904 op = IOAT_OP_XOR_VAL;
905
906 /* validate the sources with the destintation page */
907 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
908 xor_val_srcs[i] = xor_srcs[i];
909 xor_val_srcs[i] = dest;
910
911 xor_val_result = 1;
912
913 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
914 dma_srcs[i] = DMA_ERROR_CODE;
915 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) {
916 dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
917 DMA_TO_DEVICE);
918 if (dma_mapping_error(dev, dma_srcs[i]))
919 goto dma_unmap;
920 }
921 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
922 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
923 &xor_val_result, DMA_PREP_INTERRUPT);
924 if (!tx) {
925 dev_err(dev, "Self-test zero prep failed\n");
926 err = -ENODEV;
927 goto dma_unmap;
928 }
929
930 async_tx_ack(tx);
931 init_completion(&cmp);
932 tx->callback = ioat_dma_test_callback;
933 tx->callback_param = &cmp;
934 cookie = tx->tx_submit(tx);
935 if (cookie < 0) {
936 dev_err(dev, "Self-test zero setup failed\n");
937 err = -ENODEV;
938 goto dma_unmap;
939 }
940 dma->device_issue_pending(dma_chan);
941
942 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
943
944 if (tmo == 0 ||
945 dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
946 dev_err(dev, "Self-test validate timed out\n");
947 err = -ENODEV;
948 goto dma_unmap;
949 }
950
951 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
952 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
953
954 if (xor_val_result != 0) {
955 dev_err(dev, "Self-test validate failed compare\n");
956 err = -ENODEV;
957 goto free_resources;
958 }
959
960 memset(page_address(dest), 0, PAGE_SIZE);
961
962 /* test for non-zero parity sum */
963 op = IOAT_OP_XOR_VAL;
964
965 xor_val_result = 0;
966 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
967 dma_srcs[i] = DMA_ERROR_CODE;
968 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) {
969 dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
970 DMA_TO_DEVICE);
971 if (dma_mapping_error(dev, dma_srcs[i]))
972 goto dma_unmap;
973 }
974 tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
975 IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
976 &xor_val_result, DMA_PREP_INTERRUPT);
977 if (!tx) {
978 dev_err(dev, "Self-test 2nd zero prep failed\n");
979 err = -ENODEV;
980 goto dma_unmap;
981 }
982
983 async_tx_ack(tx);
984 init_completion(&cmp);
985 tx->callback = ioat_dma_test_callback;
986 tx->callback_param = &cmp;
987 cookie = tx->tx_submit(tx);
988 if (cookie < 0) {
989 dev_err(dev, "Self-test 2nd zero setup failed\n");
990 err = -ENODEV;
991 goto dma_unmap;
992 }
993 dma->device_issue_pending(dma_chan);
994
995 tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
996
997 if (tmo == 0 ||
998 dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
999 dev_err(dev, "Self-test 2nd validate timed out\n");
1000 err = -ENODEV;
1001 goto dma_unmap;
1002 }
1003
1004 if (xor_val_result != SUM_CHECK_P_RESULT) {
1005 dev_err(dev, "Self-test validate failed compare\n");
1006 err = -ENODEV;
1007 goto dma_unmap;
1008 }
1009
1010 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
1011 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE);
1012
1013 goto free_resources;
1014dma_unmap:
1015 if (op == IOAT_OP_XOR) {
1016 if (dest_dma != DMA_ERROR_CODE)
1017 dma_unmap_page(dev, dest_dma, PAGE_SIZE,
1018 DMA_FROM_DEVICE);
1019 for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
1020 if (dma_srcs[i] != DMA_ERROR_CODE)
1021 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE,
1022 DMA_TO_DEVICE);
1023 } else if (op == IOAT_OP_XOR_VAL) {
1024 for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
1025 if (dma_srcs[i] != DMA_ERROR_CODE)
1026 dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE,
1027 DMA_TO_DEVICE);
1028 }
1029free_resources:
1030 dma->device_free_chan_resources(dma_chan);
1031out:
1032 src_idx = IOAT_NUM_SRC_TEST;
1033 while (src_idx--)
1034 __free_page(xor_srcs[src_idx]);
1035 __free_page(dest);
1036 return err;
1037}
1038
1039static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma)
1040{
1041 int rc;
1042
1043 rc = ioat_dma_self_test(ioat_dma);
1044 if (rc)
1045 return rc;
1046
1047 rc = ioat_xor_val_self_test(ioat_dma);
1048
1049 return rc;
1050}
1051
1052static void ioat_intr_quirk(struct ioatdma_device *ioat_dma)
1053{
1054 struct dma_device *dma;
1055 struct dma_chan *c;
1056 struct ioatdma_chan *ioat_chan;
1057 u32 errmask;
1058
1059 dma = &ioat_dma->dma_dev;
1060
1061 /*
1062 * if we have descriptor write back error status, we mask the
1063 * error interrupts
1064 */
1065 if (ioat_dma->cap & IOAT_CAP_DWBES) {
1066 list_for_each_entry(c, &dma->channels, device_node) {
1067 ioat_chan = to_ioat_chan(c);
1068 errmask = readl(ioat_chan->reg_base +
1069 IOAT_CHANERR_MASK_OFFSET);
1070 errmask |= IOAT_CHANERR_XOR_P_OR_CRC_ERR |
1071 IOAT_CHANERR_XOR_Q_ERR;
1072 writel(errmask, ioat_chan->reg_base +
1073 IOAT_CHANERR_MASK_OFFSET);
1074 }
1075 }
1076}
1077
1078static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca)
1079{
1080 struct pci_dev *pdev = ioat_dma->pdev;
1081 int dca_en = system_has_dca_enabled(pdev);
1082 struct dma_device *dma;
1083 struct dma_chan *c;
1084 struct ioatdma_chan *ioat_chan;
1085 bool is_raid_device = false;
1086 int err;
1087
1088 dma = &ioat_dma->dma_dev;
1089 dma->device_prep_dma_memcpy = ioat_dma_prep_memcpy_lock;
1090 dma->device_issue_pending = ioat_issue_pending;
1091 dma->device_alloc_chan_resources = ioat_alloc_chan_resources;
1092 dma->device_free_chan_resources = ioat_free_chan_resources;
1093
1094 dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
1095 dma->device_prep_dma_interrupt = ioat_prep_interrupt_lock;
1096
1097 ioat_dma->cap = readl(ioat_dma->reg_base + IOAT_DMA_CAP_OFFSET);
1098
1099 if (is_xeon_cb32(pdev) || is_bwd_noraid(pdev))
1100 ioat_dma->cap &=
1101 ~(IOAT_CAP_XOR | IOAT_CAP_PQ | IOAT_CAP_RAID16SS);
1102
1103 /* dca is incompatible with raid operations */
1104 if (dca_en && (ioat_dma->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ)))
1105 ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ);
1106
1107 if (ioat_dma->cap & IOAT_CAP_XOR) {
1108 is_raid_device = true;
1109 dma->max_xor = 8;
1110
1111 dma_cap_set(DMA_XOR, dma->cap_mask);
1112 dma->device_prep_dma_xor = ioat_prep_xor;
1113
1114 dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
1115 dma->device_prep_dma_xor_val = ioat_prep_xor_val;
1116 }
1117
1118 if (ioat_dma->cap & IOAT_CAP_PQ) {
1119 is_raid_device = true;
1120
1121 dma->device_prep_dma_pq = ioat_prep_pq;
1122 dma->device_prep_dma_pq_val = ioat_prep_pq_val;
1123 dma_cap_set(DMA_PQ, dma->cap_mask);
1124 dma_cap_set(DMA_PQ_VAL, dma->cap_mask);
1125
1126 if (ioat_dma->cap & IOAT_CAP_RAID16SS)
1127 dma_set_maxpq(dma, 16, 0);
1128 else
1129 dma_set_maxpq(dma, 8, 0);
1130
1131 if (!(ioat_dma->cap & IOAT_CAP_XOR)) {
1132 dma->device_prep_dma_xor = ioat_prep_pqxor;
1133 dma->device_prep_dma_xor_val = ioat_prep_pqxor_val;
1134 dma_cap_set(DMA_XOR, dma->cap_mask);
1135 dma_cap_set(DMA_XOR_VAL, dma->cap_mask);
1136
1137 if (ioat_dma->cap & IOAT_CAP_RAID16SS)
1138 dma->max_xor = 16;
1139 else
1140 dma->max_xor = 8;
1141 }
1142 }
1143
1144 dma->device_tx_status = ioat_tx_status;
1145
1146 /* starting with CB3.3 super extended descriptors are supported */
1147 if (ioat_dma->cap & IOAT_CAP_RAID16SS) {
1148 char pool_name[14];
1149 int i;
1150
1151 for (i = 0; i < MAX_SED_POOLS; i++) {
1152 snprintf(pool_name, 14, "ioat_hw%d_sed", i);
1153
1154 /* allocate SED DMA pool */
1155 ioat_dma->sed_hw_pool[i] = dmam_pool_create(pool_name,
1156 &pdev->dev,
1157 SED_SIZE * (i + 1), 64, 0);
1158 if (!ioat_dma->sed_hw_pool[i])
1159 return -ENOMEM;
1160
1161 }
1162 }
1163
1164 if (!(ioat_dma->cap & (IOAT_CAP_XOR | IOAT_CAP_PQ)))
1165 dma_cap_set(DMA_PRIVATE, dma->cap_mask);
1166
1167 err = ioat_probe(ioat_dma);
1168 if (err)
1169 return err;
1170
1171 list_for_each_entry(c, &dma->channels, device_node) {
1172 ioat_chan = to_ioat_chan(c);
1173 writel(IOAT_DMA_DCA_ANY_CPU,
1174 ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
1175 }
1176
1177 err = ioat_register(ioat_dma);
1178 if (err)
1179 return err;
1180
1181 ioat_kobject_add(ioat_dma, &ioat_ktype);
1182
1183 if (dca)
1184 ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base);
1185
1186 return 0;
1187}
1188
1189#define DRV_NAME "ioatdma"
1190
1191static struct pci_driver ioat_pci_driver = {
1192 .name = DRV_NAME,
1193 .id_table = ioat_pci_tbl,
1194 .probe = ioat_pci_probe,
1195 .remove = ioat_remove,
1196};
1197
1198static struct ioatdma_device *
1199alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase)
1200{
1201 struct device *dev = &pdev->dev;
1202 struct ioatdma_device *d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL);
1203
1204 if (!d)
1205 return NULL;
1206 d->pdev = pdev;
1207 d->reg_base = iobase;
1208 return d;
1209}
1210
1211static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1212{
1213 void __iomem * const *iomap;
1214 struct device *dev = &pdev->dev;
1215 struct ioatdma_device *device;
1216 int err;
1217
1218 err = pcim_enable_device(pdev);
1219 if (err)
1220 return err;
1221
1222 err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME);
1223 if (err)
1224 return err;
1225 iomap = pcim_iomap_table(pdev);
1226 if (!iomap)
1227 return -ENOMEM;
1228
1229 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1230 if (err)
1231 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1232 if (err)
1233 return err;
1234
1235 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1236 if (err)
1237 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1238 if (err)
1239 return err;
1240
1241 device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]);
1242 if (!device)
1243 return -ENOMEM;
1244 pci_set_master(pdev);
1245 pci_set_drvdata(pdev, device);
1246
1247 device->version = readb(device->reg_base + IOAT_VER_OFFSET);
1248 if (device->version >= IOAT_VER_3_0)
1249 err = ioat3_dma_probe(device, ioat_dca_enabled);
1250 else
1251 return -ENODEV;
1252
1253 if (err) {
1254 dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n");
1255 return -ENODEV;
1256 }
1257
1258 return 0;
1259}
1260
1261static void ioat_remove(struct pci_dev *pdev)
1262{
1263 struct ioatdma_device *device = pci_get_drvdata(pdev);
1264
1265 if (!device)
1266 return;
1267
1268 dev_err(&pdev->dev, "Removing dma and dca services\n");
1269 if (device->dca) {
1270 unregister_dca_provider(device->dca, &pdev->dev);
1271 free_dca_provider(device->dca);
1272 device->dca = NULL;
1273 }
1274 ioat_dma_remove(device);
1275}
1276
1277static int __init ioat_init_module(void)
1278{
1279 int err = -ENOMEM;
1280
1281 pr_info("%s: Intel(R) QuickData Technology Driver %s\n",
1282 DRV_NAME, IOAT_DMA_VERSION);
1283
1284 ioat_cache = kmem_cache_create("ioat", sizeof(struct ioat_ring_ent),
1285 0, SLAB_HWCACHE_ALIGN, NULL);
1286 if (!ioat_cache)
1287 return -ENOMEM;
1288
1289 ioat_sed_cache = KMEM_CACHE(ioat_sed_ent, 0);
1290 if (!ioat_sed_cache)
1291 goto err_ioat_cache;
1292
1293 err = pci_register_driver(&ioat_pci_driver);
1294 if (err)
1295 goto err_ioat3_cache;
1296
1297 return 0;
1298
1299 err_ioat3_cache:
1300 kmem_cache_destroy(ioat_sed_cache);
1301
1302 err_ioat_cache:
1303 kmem_cache_destroy(ioat_cache);
1304
1305 return err;
1306}
1307module_init(ioat_init_module);
1308
1309static void __exit ioat_exit_module(void)
1310{
1311 pci_unregister_driver(&ioat_pci_driver);
1312 kmem_cache_destroy(ioat_cache);
1313}
1314module_exit(ioat_exit_module);
diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c
deleted file mode 100644
index 76f0dc688a19..000000000000
--- a/drivers/dma/ioat/pci.c
+++ /dev/null
@@ -1,258 +0,0 @@
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2007 - 2009 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * The full GNU General Public License is included in this distribution in
15 * the file called "COPYING".
16 *
17 */
18
19/*
20 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
21 * copy operations.
22 */
23
24#include <linux/init.h>
25#include <linux/module.h>
26#include <linux/pci.h>
27#include <linux/interrupt.h>
28#include <linux/dca.h>
29#include <linux/slab.h>
30#include "dma.h"
31#include "dma_v2.h"
32#include "registers.h"
33#include "hw.h"
34
35MODULE_VERSION(IOAT_DMA_VERSION);
36MODULE_LICENSE("Dual BSD/GPL");
37MODULE_AUTHOR("Intel Corporation");
38
39static struct pci_device_id ioat_pci_tbl[] = {
40 /* I/OAT v1 platforms */
41 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
42 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB) },
43 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) },
44 { PCI_VDEVICE(UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) },
45
46 /* I/OAT v2 platforms */
47 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) },
48
49 /* I/OAT v3 platforms */
50 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
51 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
52 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },
53 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) },
54 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) },
55 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) },
56 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) },
57 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) },
58
59 /* I/OAT v3.2 platforms */
60 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) },
61 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) },
62 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) },
63 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) },
64 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) },
65 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) },
66 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) },
67 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) },
68 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) },
69 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) },
70
71 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0) },
72 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1) },
73 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2) },
74 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3) },
75 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4) },
76 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5) },
77 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6) },
78 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7) },
79 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB8) },
80 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB9) },
81
82 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0) },
83 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1) },
84 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2) },
85 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3) },
86 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4) },
87 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5) },
88 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6) },
89 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7) },
90 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB8) },
91 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB9) },
92
93 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0) },
94 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1) },
95 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2) },
96 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3) },
97 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4) },
98 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5) },
99 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6) },
100 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7) },
101 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW8) },
102 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW9) },
103
104 /* I/OAT v3.3 platforms */
105 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) },
106 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) },
107 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2) },
108 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3) },
109
110 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0) },
111 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1) },
112 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2) },
113 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3) },
114
115 { 0, }
116};
117MODULE_DEVICE_TABLE(pci, ioat_pci_tbl);
118
119static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
120static void ioat_remove(struct pci_dev *pdev);
121
122static int ioat_dca_enabled = 1;
123module_param(ioat_dca_enabled, int, 0644);
124MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
125
126struct kmem_cache *ioat2_cache;
127struct kmem_cache *ioat3_sed_cache;
128
129#define DRV_NAME "ioatdma"
130
131static struct pci_driver ioat_pci_driver = {
132 .name = DRV_NAME,
133 .id_table = ioat_pci_tbl,
134 .probe = ioat_pci_probe,
135 .remove = ioat_remove,
136};
137
138static struct ioatdma_device *
139alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase)
140{
141 struct device *dev = &pdev->dev;
142 struct ioatdma_device *d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL);
143
144 if (!d)
145 return NULL;
146 d->pdev = pdev;
147 d->reg_base = iobase;
148 return d;
149}
150
151static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
152{
153 void __iomem * const *iomap;
154 struct device *dev = &pdev->dev;
155 struct ioatdma_device *device;
156 int err;
157
158 err = pcim_enable_device(pdev);
159 if (err)
160 return err;
161
162 err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME);
163 if (err)
164 return err;
165 iomap = pcim_iomap_table(pdev);
166 if (!iomap)
167 return -ENOMEM;
168
169 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
170 if (err)
171 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
172 if (err)
173 return err;
174
175 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
176 if (err)
177 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
178 if (err)
179 return err;
180
181 device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]);
182 if (!device)
183 return -ENOMEM;
184 pci_set_master(pdev);
185 pci_set_drvdata(pdev, device);
186
187 device->version = readb(device->reg_base + IOAT_VER_OFFSET);
188 if (device->version == IOAT_VER_1_2)
189 err = ioat1_dma_probe(device, ioat_dca_enabled);
190 else if (device->version == IOAT_VER_2_0)
191 err = ioat2_dma_probe(device, ioat_dca_enabled);
192 else if (device->version >= IOAT_VER_3_0)
193 err = ioat3_dma_probe(device, ioat_dca_enabled);
194 else
195 return -ENODEV;
196
197 if (err) {
198 dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n");
199 return -ENODEV;
200 }
201
202 return 0;
203}
204
205static void ioat_remove(struct pci_dev *pdev)
206{
207 struct ioatdma_device *device = pci_get_drvdata(pdev);
208
209 if (!device)
210 return;
211
212 dev_err(&pdev->dev, "Removing dma and dca services\n");
213 if (device->dca) {
214 unregister_dca_provider(device->dca, &pdev->dev);
215 free_dca_provider(device->dca);
216 device->dca = NULL;
217 }
218 ioat_dma_remove(device);
219}
220
221static int __init ioat_init_module(void)
222{
223 int err = -ENOMEM;
224
225 pr_info("%s: Intel(R) QuickData Technology Driver %s\n",
226 DRV_NAME, IOAT_DMA_VERSION);
227
228 ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent),
229 0, SLAB_HWCACHE_ALIGN, NULL);
230 if (!ioat2_cache)
231 return -ENOMEM;
232
233 ioat3_sed_cache = KMEM_CACHE(ioat_sed_ent, 0);
234 if (!ioat3_sed_cache)
235 goto err_ioat2_cache;
236
237 err = pci_register_driver(&ioat_pci_driver);
238 if (err)
239 goto err_ioat3_cache;
240
241 return 0;
242
243 err_ioat3_cache:
244 kmem_cache_destroy(ioat3_sed_cache);
245
246 err_ioat2_cache:
247 kmem_cache_destroy(ioat2_cache);
248
249 return err;
250}
251module_init(ioat_init_module);
252
253static void __exit ioat_exit_module(void)
254{
255 pci_unregister_driver(&ioat_pci_driver);
256 kmem_cache_destroy(ioat2_cache);
257}
258module_exit(ioat_exit_module);
diff --git a/drivers/dma/ioat/prep.c b/drivers/dma/ioat/prep.c
new file mode 100644
index 000000000000..ad4fb41cd23b
--- /dev/null
+++ b/drivers/dma/ioat/prep.c
@@ -0,0 +1,715 @@
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2015 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * The full GNU General Public License is included in this distribution in
15 * the file called "COPYING".
16 *
17 */
18#include <linux/module.h>
19#include <linux/pci.h>
20#include <linux/gfp.h>
21#include <linux/dmaengine.h>
22#include <linux/dma-mapping.h>
23#include <linux/prefetch.h>
24#include "../dmaengine.h"
25#include "registers.h"
26#include "hw.h"
27#include "dma.h"
28
29#define MAX_SCF 1024
30
31/* provide a lookup table for setting the source address in the base or
32 * extended descriptor of an xor or pq descriptor
33 */
34static const u8 xor_idx_to_desc = 0xe0;
35static const u8 xor_idx_to_field[] = { 1, 4, 5, 6, 7, 0, 1, 2 };
36static const u8 pq_idx_to_desc = 0xf8;
37static const u8 pq16_idx_to_desc[] = { 0, 0, 1, 1, 1, 1, 1, 1, 1,
38 2, 2, 2, 2, 2, 2, 2 };
39static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 };
40static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7,
41 0, 1, 2, 3, 4, 5, 6 };
42
43static void xor_set_src(struct ioat_raw_descriptor *descs[2],
44 dma_addr_t addr, u32 offset, int idx)
45{
46 struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
47
48 raw->field[xor_idx_to_field[idx]] = addr + offset;
49}
50
51static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx)
52{
53 struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
54
55 return raw->field[pq_idx_to_field[idx]];
56}
57
58static dma_addr_t pq16_get_src(struct ioat_raw_descriptor *desc[3], int idx)
59{
60 struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]];
61
62 return raw->field[pq16_idx_to_field[idx]];
63}
64
65static void pq_set_src(struct ioat_raw_descriptor *descs[2],
66 dma_addr_t addr, u32 offset, u8 coef, int idx)
67{
68 struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0];
69 struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
70
71 raw->field[pq_idx_to_field[idx]] = addr + offset;
72 pq->coef[idx] = coef;
73}
74
75static void pq16_set_src(struct ioat_raw_descriptor *desc[3],
76 dma_addr_t addr, u32 offset, u8 coef, unsigned idx)
77{
78 struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *)desc[0];
79 struct ioat_pq16a_descriptor *pq16 =
80 (struct ioat_pq16a_descriptor *)desc[1];
81 struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]];
82
83 raw->field[pq16_idx_to_field[idx]] = addr + offset;
84
85 if (idx < 8)
86 pq->coef[idx] = coef;
87 else
88 pq16->coef[idx - 8] = coef;
89}
90
91static struct ioat_sed_ent *
92ioat3_alloc_sed(struct ioatdma_device *ioat_dma, unsigned int hw_pool)
93{
94 struct ioat_sed_ent *sed;
95 gfp_t flags = __GFP_ZERO | GFP_ATOMIC;
96
97 sed = kmem_cache_alloc(ioat_sed_cache, flags);
98 if (!sed)
99 return NULL;
100
101 sed->hw_pool = hw_pool;
102 sed->hw = dma_pool_alloc(ioat_dma->sed_hw_pool[hw_pool],
103 flags, &sed->dma);
104 if (!sed->hw) {
105 kmem_cache_free(ioat_sed_cache, sed);
106 return NULL;
107 }
108
109 return sed;
110}
111
112struct dma_async_tx_descriptor *
113ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
114 dma_addr_t dma_src, size_t len, unsigned long flags)
115{
116 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
117 struct ioat_dma_descriptor *hw;
118 struct ioat_ring_ent *desc;
119 dma_addr_t dst = dma_dest;
120 dma_addr_t src = dma_src;
121 size_t total_len = len;
122 int num_descs, idx, i;
123
124 num_descs = ioat_xferlen_to_descs(ioat_chan, len);
125 if (likely(num_descs) &&
126 ioat_check_space_lock(ioat_chan, num_descs) == 0)
127 idx = ioat_chan->head;
128 else
129 return NULL;
130 i = 0;
131 do {
132 size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log);
133
134 desc = ioat_get_ring_ent(ioat_chan, idx + i);
135 hw = desc->hw;
136
137 hw->size = copy;
138 hw->ctl = 0;
139 hw->src_addr = src;
140 hw->dst_addr = dst;
141
142 len -= copy;
143 dst += copy;
144 src += copy;
145 dump_desc_dbg(ioat_chan, desc);
146 } while (++i < num_descs);
147
148 desc->txd.flags = flags;
149 desc->len = total_len;
150 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
151 hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
152 hw->ctl_f.compl_write = 1;
153 dump_desc_dbg(ioat_chan, desc);
154 /* we leave the channel locked to ensure in order submission */
155
156 return &desc->txd;
157}
158
159
160static struct dma_async_tx_descriptor *
161__ioat_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
162 dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt,
163 size_t len, unsigned long flags)
164{
165 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
166 struct ioat_ring_ent *compl_desc;
167 struct ioat_ring_ent *desc;
168 struct ioat_ring_ent *ext;
169 size_t total_len = len;
170 struct ioat_xor_descriptor *xor;
171 struct ioat_xor_ext_descriptor *xor_ex = NULL;
172 struct ioat_dma_descriptor *hw;
173 int num_descs, with_ext, idx, i;
174 u32 offset = 0;
175 u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR;
176
177 BUG_ON(src_cnt < 2);
178
179 num_descs = ioat_xferlen_to_descs(ioat_chan, len);
180 /* we need 2x the number of descriptors to cover greater than 5
181 * sources
182 */
183 if (src_cnt > 5) {
184 with_ext = 1;
185 num_descs *= 2;
186 } else
187 with_ext = 0;
188
189 /* completion writes from the raid engine may pass completion
190 * writes from the legacy engine, so we need one extra null
191 * (legacy) descriptor to ensure all completion writes arrive in
192 * order.
193 */
194 if (likely(num_descs) &&
195 ioat_check_space_lock(ioat_chan, num_descs+1) == 0)
196 idx = ioat_chan->head;
197 else
198 return NULL;
199 i = 0;
200 do {
201 struct ioat_raw_descriptor *descs[2];
202 size_t xfer_size = min_t(size_t,
203 len, 1 << ioat_chan->xfercap_log);
204 int s;
205
206 desc = ioat_get_ring_ent(ioat_chan, idx + i);
207 xor = desc->xor;
208
209 /* save a branch by unconditionally retrieving the
210 * extended descriptor xor_set_src() knows to not write
211 * to it in the single descriptor case
212 */
213 ext = ioat_get_ring_ent(ioat_chan, idx + i + 1);
214 xor_ex = ext->xor_ex;
215
216 descs[0] = (struct ioat_raw_descriptor *) xor;
217 descs[1] = (struct ioat_raw_descriptor *) xor_ex;
218 for (s = 0; s < src_cnt; s++)
219 xor_set_src(descs, src[s], offset, s);
220 xor->size = xfer_size;
221 xor->dst_addr = dest + offset;
222 xor->ctl = 0;
223 xor->ctl_f.op = op;
224 xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt);
225
226 len -= xfer_size;
227 offset += xfer_size;
228 dump_desc_dbg(ioat_chan, desc);
229 } while ((i += 1 + with_ext) < num_descs);
230
231 /* last xor descriptor carries the unmap parameters and fence bit */
232 desc->txd.flags = flags;
233 desc->len = total_len;
234 if (result)
235 desc->result = result;
236 xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
237
238 /* completion descriptor carries interrupt bit */
239 compl_desc = ioat_get_ring_ent(ioat_chan, idx + i);
240 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
241 hw = compl_desc->hw;
242 hw->ctl = 0;
243 hw->ctl_f.null = 1;
244 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
245 hw->ctl_f.compl_write = 1;
246 hw->size = NULL_DESC_BUFFER_SIZE;
247 dump_desc_dbg(ioat_chan, compl_desc);
248
249 /* we leave the channel locked to ensure in order submission */
250 return &compl_desc->txd;
251}
252
253struct dma_async_tx_descriptor *
254ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
255 unsigned int src_cnt, size_t len, unsigned long flags)
256{
257 return __ioat_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags);
258}
259
260struct dma_async_tx_descriptor *
261ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
262 unsigned int src_cnt, size_t len,
263 enum sum_check_flags *result, unsigned long flags)
264{
265 /* the cleanup routine only sets bits on validate failure, it
266 * does not clear bits on validate success... so clear it here
267 */
268 *result = 0;
269
270 return __ioat_prep_xor_lock(chan, result, src[0], &src[1],
271 src_cnt - 1, len, flags);
272}
273
274static void
275dump_pq_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc,
276 struct ioat_ring_ent *ext)
277{
278 struct device *dev = to_dev(ioat_chan);
279 struct ioat_pq_descriptor *pq = desc->pq;
280 struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL;
281 struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex };
282 int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
283 int i;
284
285 dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
286 " sz: %#10.8x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'"
287 " src_cnt: %d)\n",
288 desc_id(desc), (unsigned long long) desc->txd.phys,
289 (unsigned long long) (pq_ex ? pq_ex->next : pq->next),
290 desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op,
291 pq->ctl_f.int_en, pq->ctl_f.compl_write,
292 pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
293 pq->ctl_f.src_cnt);
294 for (i = 0; i < src_cnt; i++)
295 dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
296 (unsigned long long) pq_get_src(descs, i), pq->coef[i]);
297 dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
298 dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
299 dev_dbg(dev, "\tNEXT: %#llx\n", pq->next);
300}
301
302static void dump_pq16_desc_dbg(struct ioatdma_chan *ioat_chan,
303 struct ioat_ring_ent *desc)
304{
305 struct device *dev = to_dev(ioat_chan);
306 struct ioat_pq_descriptor *pq = desc->pq;
307 struct ioat_raw_descriptor *descs[] = { (void *)pq,
308 (void *)pq,
309 (void *)pq };
310 int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt);
311 int i;
312
313 if (desc->sed) {
314 descs[1] = (void *)desc->sed->hw;
315 descs[2] = (void *)desc->sed->hw + 64;
316 }
317
318 dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
319 " sz: %#x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'"
320 " src_cnt: %d)\n",
321 desc_id(desc), (unsigned long long) desc->txd.phys,
322 (unsigned long long) pq->next,
323 desc->txd.flags, pq->size, pq->ctl,
324 pq->ctl_f.op, pq->ctl_f.int_en,
325 pq->ctl_f.compl_write,
326 pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
327 pq->ctl_f.src_cnt);
328 for (i = 0; i < src_cnt; i++) {
329 dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
330 (unsigned long long) pq16_get_src(descs, i),
331 pq->coef[i]);
332 }
333 dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
334 dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
335}
336
337static struct dma_async_tx_descriptor *
338__ioat_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
339 const dma_addr_t *dst, const dma_addr_t *src,
340 unsigned int src_cnt, const unsigned char *scf,
341 size_t len, unsigned long flags)
342{
343 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
344 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
345 struct ioat_ring_ent *compl_desc;
346 struct ioat_ring_ent *desc;
347 struct ioat_ring_ent *ext;
348 size_t total_len = len;
349 struct ioat_pq_descriptor *pq;
350 struct ioat_pq_ext_descriptor *pq_ex = NULL;
351 struct ioat_dma_descriptor *hw;
352 u32 offset = 0;
353 u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ;
354 int i, s, idx, with_ext, num_descs;
355 int cb32 = (ioat_dma->version < IOAT_VER_3_3) ? 1 : 0;
356
357 dev_dbg(to_dev(ioat_chan), "%s\n", __func__);
358 /* the engine requires at least two sources (we provide
359 * at least 1 implied source in the DMA_PREP_CONTINUE case)
360 */
361 BUG_ON(src_cnt + dmaf_continue(flags) < 2);
362
363 num_descs = ioat_xferlen_to_descs(ioat_chan, len);
364 /* we need 2x the number of descriptors to cover greater than 3
365 * sources (we need 1 extra source in the q-only continuation
366 * case and 3 extra sources in the p+q continuation case.
367 */
368 if (src_cnt + dmaf_p_disabled_continue(flags) > 3 ||
369 (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) {
370 with_ext = 1;
371 num_descs *= 2;
372 } else
373 with_ext = 0;
374
375 /* completion writes from the raid engine may pass completion
376 * writes from the legacy engine, so we need one extra null
377 * (legacy) descriptor to ensure all completion writes arrive in
378 * order.
379 */
380 if (likely(num_descs) &&
381 ioat_check_space_lock(ioat_chan, num_descs + cb32) == 0)
382 idx = ioat_chan->head;
383 else
384 return NULL;
385 i = 0;
386 do {
387 struct ioat_raw_descriptor *descs[2];
388 size_t xfer_size = min_t(size_t, len,
389 1 << ioat_chan->xfercap_log);
390
391 desc = ioat_get_ring_ent(ioat_chan, idx + i);
392 pq = desc->pq;
393
394 /* save a branch by unconditionally retrieving the
395 * extended descriptor pq_set_src() knows to not write
396 * to it in the single descriptor case
397 */
398 ext = ioat_get_ring_ent(ioat_chan, idx + i + with_ext);
399 pq_ex = ext->pq_ex;
400
401 descs[0] = (struct ioat_raw_descriptor *) pq;
402 descs[1] = (struct ioat_raw_descriptor *) pq_ex;
403
404 for (s = 0; s < src_cnt; s++)
405 pq_set_src(descs, src[s], offset, scf[s], s);
406
407 /* see the comment for dma_maxpq in include/linux/dmaengine.h */
408 if (dmaf_p_disabled_continue(flags))
409 pq_set_src(descs, dst[1], offset, 1, s++);
410 else if (dmaf_continue(flags)) {
411 pq_set_src(descs, dst[0], offset, 0, s++);
412 pq_set_src(descs, dst[1], offset, 1, s++);
413 pq_set_src(descs, dst[1], offset, 0, s++);
414 }
415 pq->size = xfer_size;
416 pq->p_addr = dst[0] + offset;
417 pq->q_addr = dst[1] + offset;
418 pq->ctl = 0;
419 pq->ctl_f.op = op;
420 /* we turn on descriptor write back error status */
421 if (ioat_dma->cap & IOAT_CAP_DWBES)
422 pq->ctl_f.wb_en = result ? 1 : 0;
423 pq->ctl_f.src_cnt = src_cnt_to_hw(s);
424 pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
425 pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
426
427 len -= xfer_size;
428 offset += xfer_size;
429 } while ((i += 1 + with_ext) < num_descs);
430
431 /* last pq descriptor carries the unmap parameters and fence bit */
432 desc->txd.flags = flags;
433 desc->len = total_len;
434 if (result)
435 desc->result = result;
436 pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
437 dump_pq_desc_dbg(ioat_chan, desc, ext);
438
439 if (!cb32) {
440 pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
441 pq->ctl_f.compl_write = 1;
442 compl_desc = desc;
443 } else {
444 /* completion descriptor carries interrupt bit */
445 compl_desc = ioat_get_ring_ent(ioat_chan, idx + i);
446 compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
447 hw = compl_desc->hw;
448 hw->ctl = 0;
449 hw->ctl_f.null = 1;
450 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
451 hw->ctl_f.compl_write = 1;
452 hw->size = NULL_DESC_BUFFER_SIZE;
453 dump_desc_dbg(ioat_chan, compl_desc);
454 }
455
456
457 /* we leave the channel locked to ensure in order submission */
458 return &compl_desc->txd;
459}
460
461static struct dma_async_tx_descriptor *
462__ioat_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result,
463 const dma_addr_t *dst, const dma_addr_t *src,
464 unsigned int src_cnt, const unsigned char *scf,
465 size_t len, unsigned long flags)
466{
467 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
468 struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;
469 struct ioat_ring_ent *desc;
470 size_t total_len = len;
471 struct ioat_pq_descriptor *pq;
472 u32 offset = 0;
473 u8 op;
474 int i, s, idx, num_descs;
475
476 /* this function is only called with 9-16 sources */
477 op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S;
478
479 dev_dbg(to_dev(ioat_chan), "%s\n", __func__);
480
481 num_descs = ioat_xferlen_to_descs(ioat_chan, len);
482
483 /*
484 * 16 source pq is only available on cb3.3 and has no completion
485 * write hw bug.
486 */
487 if (num_descs && ioat_check_space_lock(ioat_chan, num_descs) == 0)
488 idx = ioat_chan->head;
489 else
490 return NULL;
491
492 i = 0;
493
494 do {
495 struct ioat_raw_descriptor *descs[4];
496 size_t xfer_size = min_t(size_t, len,
497 1 << ioat_chan->xfercap_log);
498
499 desc = ioat_get_ring_ent(ioat_chan, idx + i);
500 pq = desc->pq;
501
502 descs[0] = (struct ioat_raw_descriptor *) pq;
503
504 desc->sed = ioat3_alloc_sed(ioat_dma, (src_cnt-2) >> 3);
505 if (!desc->sed) {
506 dev_err(to_dev(ioat_chan),
507 "%s: no free sed entries\n", __func__);
508 return NULL;
509 }
510
511 pq->sed_addr = desc->sed->dma;
512 desc->sed->parent = desc;
513
514 descs[1] = (struct ioat_raw_descriptor *)desc->sed->hw;
515 descs[2] = (void *)descs[1] + 64;
516
517 for (s = 0; s < src_cnt; s++)
518 pq16_set_src(descs, src[s], offset, scf[s], s);
519
520 /* see the comment for dma_maxpq in include/linux/dmaengine.h */
521 if (dmaf_p_disabled_continue(flags))
522 pq16_set_src(descs, dst[1], offset, 1, s++);
523 else if (dmaf_continue(flags)) {
524 pq16_set_src(descs, dst[0], offset, 0, s++);
525 pq16_set_src(descs, dst[1], offset, 1, s++);
526 pq16_set_src(descs, dst[1], offset, 0, s++);
527 }
528
529 pq->size = xfer_size;
530 pq->p_addr = dst[0] + offset;
531 pq->q_addr = dst[1] + offset;
532 pq->ctl = 0;
533 pq->ctl_f.op = op;
534 pq->ctl_f.src_cnt = src16_cnt_to_hw(s);
535 /* we turn on descriptor write back error status */
536 if (ioat_dma->cap & IOAT_CAP_DWBES)
537 pq->ctl_f.wb_en = result ? 1 : 0;
538 pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
539 pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
540
541 len -= xfer_size;
542 offset += xfer_size;
543 } while (++i < num_descs);
544
545 /* last pq descriptor carries the unmap parameters and fence bit */
546 desc->txd.flags = flags;
547 desc->len = total_len;
548 if (result)
549 desc->result = result;
550 pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
551
552 /* with cb3.3 we should be able to do completion w/o a null desc */
553 pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
554 pq->ctl_f.compl_write = 1;
555
556 dump_pq16_desc_dbg(ioat_chan, desc);
557
558 /* we leave the channel locked to ensure in order submission */
559 return &desc->txd;
560}
561
562static int src_cnt_flags(unsigned int src_cnt, unsigned long flags)
563{
564 if (dmaf_p_disabled_continue(flags))
565 return src_cnt + 1;
566 else if (dmaf_continue(flags))
567 return src_cnt + 3;
568 else
569 return src_cnt;
570}
571
572struct dma_async_tx_descriptor *
573ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
574 unsigned int src_cnt, const unsigned char *scf, size_t len,
575 unsigned long flags)
576{
577 /* specify valid address for disabled result */
578 if (flags & DMA_PREP_PQ_DISABLE_P)
579 dst[0] = dst[1];
580 if (flags & DMA_PREP_PQ_DISABLE_Q)
581 dst[1] = dst[0];
582
583 /* handle the single source multiply case from the raid6
584 * recovery path
585 */
586 if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) {
587 dma_addr_t single_source[2];
588 unsigned char single_source_coef[2];
589
590 BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q);
591 single_source[0] = src[0];
592 single_source[1] = src[0];
593 single_source_coef[0] = scf[0];
594 single_source_coef[1] = 0;
595
596 return src_cnt_flags(src_cnt, flags) > 8 ?
597 __ioat_prep_pq16_lock(chan, NULL, dst, single_source,
598 2, single_source_coef, len,
599 flags) :
600 __ioat_prep_pq_lock(chan, NULL, dst, single_source, 2,
601 single_source_coef, len, flags);
602
603 } else {
604 return src_cnt_flags(src_cnt, flags) > 8 ?
605 __ioat_prep_pq16_lock(chan, NULL, dst, src, src_cnt,
606 scf, len, flags) :
607 __ioat_prep_pq_lock(chan, NULL, dst, src, src_cnt,
608 scf, len, flags);
609 }
610}
611
612struct dma_async_tx_descriptor *
613ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
614 unsigned int src_cnt, const unsigned char *scf, size_t len,
615 enum sum_check_flags *pqres, unsigned long flags)
616{
617 /* specify valid address for disabled result */
618 if (flags & DMA_PREP_PQ_DISABLE_P)
619 pq[0] = pq[1];
620 if (flags & DMA_PREP_PQ_DISABLE_Q)
621 pq[1] = pq[0];
622
623 /* the cleanup routine only sets bits on validate failure, it
624 * does not clear bits on validate success... so clear it here
625 */
626 *pqres = 0;
627
628 return src_cnt_flags(src_cnt, flags) > 8 ?
629 __ioat_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len,
630 flags) :
631 __ioat_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
632 flags);
633}
634
635struct dma_async_tx_descriptor *
636ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
637 unsigned int src_cnt, size_t len, unsigned long flags)
638{
639 unsigned char scf[MAX_SCF];
640 dma_addr_t pq[2];
641
642 if (src_cnt > MAX_SCF)
643 return NULL;
644
645 memset(scf, 0, src_cnt);
646 pq[0] = dst;
647 flags |= DMA_PREP_PQ_DISABLE_Q;
648 pq[1] = dst; /* specify valid address for disabled result */
649
650 return src_cnt_flags(src_cnt, flags) > 8 ?
651 __ioat_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len,
652 flags) :
653 __ioat_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
654 flags);
655}
656
657struct dma_async_tx_descriptor *
658ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
659 unsigned int src_cnt, size_t len,
660 enum sum_check_flags *result, unsigned long flags)
661{
662 unsigned char scf[MAX_SCF];
663 dma_addr_t pq[2];
664
665 if (src_cnt > MAX_SCF)
666 return NULL;
667
668 /* the cleanup routine only sets bits on validate failure, it
669 * does not clear bits on validate success... so clear it here
670 */
671 *result = 0;
672
673 memset(scf, 0, src_cnt);
674 pq[0] = src[0];
675 flags |= DMA_PREP_PQ_DISABLE_Q;
676 pq[1] = pq[0]; /* specify valid address for disabled result */
677
678 return src_cnt_flags(src_cnt, flags) > 8 ?
679 __ioat_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1,
680 scf, len, flags) :
681 __ioat_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1,
682 scf, len, flags);
683}
684
685struct dma_async_tx_descriptor *
686ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
687{
688 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
689 struct ioat_ring_ent *desc;
690 struct ioat_dma_descriptor *hw;
691
692 if (ioat_check_space_lock(ioat_chan, 1) == 0)
693 desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);
694 else
695 return NULL;
696
697 hw = desc->hw;
698 hw->ctl = 0;
699 hw->ctl_f.null = 1;
700 hw->ctl_f.int_en = 1;
701 hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
702 hw->ctl_f.compl_write = 1;
703 hw->size = NULL_DESC_BUFFER_SIZE;
704 hw->src_addr = 0;
705 hw->dst_addr = 0;
706
707 desc->txd.flags = flags;
708 desc->len = 1;
709
710 dump_desc_dbg(ioat_chan, desc);
711
712 /* we leave the channel locked to ensure in order submission */
713 return &desc->txd;
714}
715
diff --git a/drivers/dma/ioat/sysfs.c b/drivers/dma/ioat/sysfs.c
new file mode 100644
index 000000000000..cb4a857ee21b
--- /dev/null
+++ b/drivers/dma/ioat/sysfs.c
@@ -0,0 +1,135 @@
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2015 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * The full GNU General Public License is included in this distribution in
15 * the file called "COPYING".
16 *
17 */
18
19#include <linux/init.h>
20#include <linux/module.h>
21#include <linux/dmaengine.h>
22#include <linux/pci.h>
23#include "dma.h"
24#include "registers.h"
25#include "hw.h"
26
27#include "../dmaengine.h"
28
29static ssize_t cap_show(struct dma_chan *c, char *page)
30{
31 struct dma_device *dma = c->device;
32
33 return sprintf(page, "copy%s%s%s%s%s\n",
34 dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "",
35 dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "",
36 dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "",
37 dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "",
38 dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : "");
39
40}
41struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap);
42
43static ssize_t version_show(struct dma_chan *c, char *page)
44{
45 struct dma_device *dma = c->device;
46 struct ioatdma_device *ioat_dma = to_ioatdma_device(dma);
47
48 return sprintf(page, "%d.%d\n",
49 ioat_dma->version >> 4, ioat_dma->version & 0xf);
50}
51struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version);
52
53static ssize_t
54ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
55{
56 struct ioat_sysfs_entry *entry;
57 struct ioatdma_chan *ioat_chan;
58
59 entry = container_of(attr, struct ioat_sysfs_entry, attr);
60 ioat_chan = container_of(kobj, struct ioatdma_chan, kobj);
61
62 if (!entry->show)
63 return -EIO;
64 return entry->show(&ioat_chan->dma_chan, page);
65}
66
67const struct sysfs_ops ioat_sysfs_ops = {
68 .show = ioat_attr_show,
69};
70
71void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type)
72{
73 struct dma_device *dma = &ioat_dma->dma_dev;
74 struct dma_chan *c;
75
76 list_for_each_entry(c, &dma->channels, device_node) {
77 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
78 struct kobject *parent = &c->dev->device.kobj;
79 int err;
80
81 err = kobject_init_and_add(&ioat_chan->kobj, type,
82 parent, "quickdata");
83 if (err) {
84 dev_warn(to_dev(ioat_chan),
85 "sysfs init error (%d), continuing...\n", err);
86 kobject_put(&ioat_chan->kobj);
87 set_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state);
88 }
89 }
90}
91
92void ioat_kobject_del(struct ioatdma_device *ioat_dma)
93{
94 struct dma_device *dma = &ioat_dma->dma_dev;
95 struct dma_chan *c;
96
97 list_for_each_entry(c, &dma->channels, device_node) {
98 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
99
100 if (!test_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state)) {
101 kobject_del(&ioat_chan->kobj);
102 kobject_put(&ioat_chan->kobj);
103 }
104 }
105}
106
107static ssize_t ring_size_show(struct dma_chan *c, char *page)
108{
109 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
110
111 return sprintf(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1);
112}
113static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
114
115static ssize_t ring_active_show(struct dma_chan *c, char *page)
116{
117 struct ioatdma_chan *ioat_chan = to_ioat_chan(c);
118
119 /* ...taken outside the lock, no need to be precise */
120 return sprintf(page, "%d\n", ioat_ring_active(ioat_chan));
121}
122static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
123
124static struct attribute *ioat_attrs[] = {
125 &ring_size_attr.attr,
126 &ring_active_attr.attr,
127 &ioat_cap_attr.attr,
128 &ioat_version_attr.attr,
129 NULL,
130};
131
132struct kobj_type ioat_ktype = {
133 .sysfs_ops = &ioat_sysfs_ops,
134 .default_attrs = ioat_attrs,
135};
diff --git a/drivers/dma/ipu/ipu_irq.c b/drivers/dma/ipu/ipu_irq.c
index 2e284a4438bc..4768a829253a 100644
--- a/drivers/dma/ipu/ipu_irq.c
+++ b/drivers/dma/ipu/ipu_irq.c
@@ -265,10 +265,10 @@ int ipu_irq_unmap(unsigned int source)
265 return ret; 265 return ret;
266} 266}
267 267
268/* Chained IRQ handler for IPU error interrupt */ 268/* Chained IRQ handler for IPU function and error interrupt */
269static void ipu_irq_err(unsigned int irq, struct irq_desc *desc) 269static void ipu_irq_handler(unsigned int __irq, struct irq_desc *desc)
270{ 270{
271 struct ipu *ipu = irq_get_handler_data(irq); 271 struct ipu *ipu = irq_desc_get_handler_data(desc);
272 u32 status; 272 u32 status;
273 int i, line; 273 int i, line;
274 274
@@ -286,43 +286,7 @@ static void ipu_irq_err(unsigned int irq, struct irq_desc *desc)
286 raw_spin_unlock(&bank_lock); 286 raw_spin_unlock(&bank_lock);
287 while ((line = ffs(status))) { 287 while ((line = ffs(status))) {
288 struct ipu_irq_map *map; 288 struct ipu_irq_map *map;
289 289 unsigned int irq = NO_IRQ;
290 line--;
291 status &= ~(1UL << line);
292
293 raw_spin_lock(&bank_lock);
294 map = src2map(32 * i + line);
295 if (map)
296 irq = map->irq;
297 raw_spin_unlock(&bank_lock);
298
299 if (!map) {
300 pr_err("IPU: Interrupt on unmapped source %u bank %d\n",
301 line, i);
302 continue;
303 }
304 generic_handle_irq(irq);
305 }
306 }
307}
308
309/* Chained IRQ handler for IPU function interrupt */
310static void ipu_irq_fn(unsigned int irq, struct irq_desc *desc)
311{
312 struct ipu *ipu = irq_desc_get_handler_data(desc);
313 u32 status;
314 int i, line;
315
316 for (i = 0; i < IPU_IRQ_NR_FN_BANKS; i++) {
317 struct ipu_irq_bank *bank = irq_bank + i;
318
319 raw_spin_lock(&bank_lock);
320 status = ipu_read_reg(ipu, bank->status);
321 /* Not clearing all interrupts, see above */
322 status &= ipu_read_reg(ipu, bank->control);
323 raw_spin_unlock(&bank_lock);
324 while ((line = ffs(status))) {
325 struct ipu_irq_map *map;
326 290
327 line--; 291 line--;
328 status &= ~(1UL << line); 292 status &= ~(1UL << line);
@@ -377,16 +341,12 @@ int __init ipu_irq_attach_irq(struct ipu *ipu, struct platform_device *dev)
377 irq_map[i].irq = irq; 341 irq_map[i].irq = irq;
378 irq_map[i].source = -EINVAL; 342 irq_map[i].source = -EINVAL;
379 irq_set_handler(irq, handle_level_irq); 343 irq_set_handler(irq, handle_level_irq);
380#ifdef CONFIG_ARM 344 irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE);
381 set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
382#endif
383 } 345 }
384 346
385 irq_set_handler_data(ipu->irq_fn, ipu); 347 irq_set_chained_handler_and_data(ipu->irq_fn, ipu_irq_handler, ipu);
386 irq_set_chained_handler(ipu->irq_fn, ipu_irq_fn);
387 348
388 irq_set_handler_data(ipu->irq_err, ipu); 349 irq_set_chained_handler_and_data(ipu->irq_err, ipu_irq_handler, ipu);
389 irq_set_chained_handler(ipu->irq_err, ipu_irq_err);
390 350
391 ipu->irq_base = irq_base; 351 ipu->irq_base = irq_base;
392 352
@@ -399,16 +359,12 @@ void ipu_irq_detach_irq(struct ipu *ipu, struct platform_device *dev)
399 359
400 irq_base = ipu->irq_base; 360 irq_base = ipu->irq_base;
401 361
402 irq_set_chained_handler(ipu->irq_fn, NULL); 362 irq_set_chained_handler_and_data(ipu->irq_fn, NULL, NULL);
403 irq_set_handler_data(ipu->irq_fn, NULL);
404 363
405 irq_set_chained_handler(ipu->irq_err, NULL); 364 irq_set_chained_handler_and_data(ipu->irq_err, NULL, NULL);
406 irq_set_handler_data(ipu->irq_err, NULL);
407 365
408 for (irq = irq_base; irq < irq_base + CONFIG_MX3_IPU_IRQS; irq++) { 366 for (irq = irq_base; irq < irq_base + CONFIG_MX3_IPU_IRQS; irq++) {
409#ifdef CONFIG_ARM 367 irq_set_status_flags(irq, IRQ_NOREQUEST);
410 set_irq_flags(irq, 0);
411#endif
412 irq_set_chip(irq, NULL); 368 irq_set_chip(irq, NULL);
413 irq_set_chip_data(irq, NULL); 369 irq_set_chip_data(irq, NULL);
414 } 370 }
diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c
index 647e362f01fd..1ba2fd73852d 100644
--- a/drivers/dma/k3dma.c
+++ b/drivers/dma/k3dma.c
@@ -24,7 +24,6 @@
24#include "virt-dma.h" 24#include "virt-dma.h"
25 25
26#define DRIVER_NAME "k3-dma" 26#define DRIVER_NAME "k3-dma"
27#define DMA_ALIGN 3
28#define DMA_MAX_SIZE 0x1ffc 27#define DMA_MAX_SIZE 0x1ffc
29 28
30#define INT_STAT 0x00 29#define INT_STAT 0x00
@@ -732,7 +731,7 @@ static int k3_dma_probe(struct platform_device *op)
732 d->slave.device_pause = k3_dma_transfer_pause; 731 d->slave.device_pause = k3_dma_transfer_pause;
733 d->slave.device_resume = k3_dma_transfer_resume; 732 d->slave.device_resume = k3_dma_transfer_resume;
734 d->slave.device_terminate_all = k3_dma_terminate_all; 733 d->slave.device_terminate_all = k3_dma_terminate_all;
735 d->slave.copy_align = DMA_ALIGN; 734 d->slave.copy_align = DMAENGINE_ALIGN_8_BYTES;
736 735
737 /* init virtual channel */ 736 /* init virtual channel */
738 d->chans = devm_kzalloc(&op->dev, 737 d->chans = devm_kzalloc(&op->dev,
diff --git a/drivers/dma/lpc18xx-dmamux.c b/drivers/dma/lpc18xx-dmamux.c
new file mode 100644
index 000000000000..761f32687055
--- /dev/null
+++ b/drivers/dma/lpc18xx-dmamux.c
@@ -0,0 +1,183 @@
1/*
2 * DMA Router driver for LPC18xx/43xx DMA MUX
3 *
4 * Copyright (C) 2015 Joachim Eastwood <manabian@gmail.com>
5 *
6 * Based on TI DMA Crossbar driver by:
7 * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com
8 * Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
13 *
14 */
15
16#include <linux/err.h>
17#include <linux/init.h>
18#include <linux/mfd/syscon.h>
19#include <linux/of_device.h>
20#include <linux/of_dma.h>
21#include <linux/regmap.h>
22#include <linux/spinlock.h>
23
24/* CREG register offset and macros for mux manipulation */
25#define LPC18XX_CREG_DMAMUX 0x11c
26#define LPC18XX_DMAMUX_VAL(v, n) ((v) << (n * 2))
27#define LPC18XX_DMAMUX_MASK(n) (0x3 << (n * 2))
28#define LPC18XX_DMAMUX_MAX_VAL 0x3
29
30struct lpc18xx_dmamux {
31 u32 value;
32 bool busy;
33};
34
35struct lpc18xx_dmamux_data {
36 struct dma_router dmarouter;
37 struct lpc18xx_dmamux *muxes;
38 u32 dma_master_requests;
39 u32 dma_mux_requests;
40 struct regmap *reg;
41 spinlock_t lock;
42};
43
44static void lpc18xx_dmamux_free(struct device *dev, void *route_data)
45{
46 struct lpc18xx_dmamux_data *dmamux = dev_get_drvdata(dev);
47 struct lpc18xx_dmamux *mux = route_data;
48 unsigned long flags;
49
50 spin_lock_irqsave(&dmamux->lock, flags);
51 mux->busy = false;
52 spin_unlock_irqrestore(&dmamux->lock, flags);
53}
54
55static void *lpc18xx_dmamux_reserve(struct of_phandle_args *dma_spec,
56 struct of_dma *ofdma)
57{
58 struct platform_device *pdev = of_find_device_by_node(ofdma->of_node);
59 struct lpc18xx_dmamux_data *dmamux = platform_get_drvdata(pdev);
60 unsigned long flags;
61 unsigned mux;
62
63 if (dma_spec->args_count != 3) {
64 dev_err(&pdev->dev, "invalid number of dma mux args\n");
65 return ERR_PTR(-EINVAL);
66 }
67
68 mux = dma_spec->args[0];
69 if (mux >= dmamux->dma_master_requests) {
70 dev_err(&pdev->dev, "invalid mux number: %d\n",
71 dma_spec->args[0]);
72 return ERR_PTR(-EINVAL);
73 }
74
75 if (dma_spec->args[1] > LPC18XX_DMAMUX_MAX_VAL) {
76 dev_err(&pdev->dev, "invalid dma mux value: %d\n",
77 dma_spec->args[1]);
78 return ERR_PTR(-EINVAL);
79 }
80
81 /* The of_node_put() will be done in the core for the node */
82 dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0);
83 if (!dma_spec->np) {
84 dev_err(&pdev->dev, "can't get dma master\n");
85 return ERR_PTR(-EINVAL);
86 }
87
88 spin_lock_irqsave(&dmamux->lock, flags);
89 if (dmamux->muxes[mux].busy) {
90 spin_unlock_irqrestore(&dmamux->lock, flags);
91 dev_err(&pdev->dev, "dma request %u busy with %u.%u\n",
92 mux, mux, dmamux->muxes[mux].value);
93 of_node_put(dma_spec->np);
94 return ERR_PTR(-EBUSY);
95 }
96
97 dmamux->muxes[mux].busy = true;
98 dmamux->muxes[mux].value = dma_spec->args[1];
99
100 regmap_update_bits(dmamux->reg, LPC18XX_CREG_DMAMUX,
101 LPC18XX_DMAMUX_MASK(mux),
102 LPC18XX_DMAMUX_VAL(dmamux->muxes[mux].value, mux));
103 spin_unlock_irqrestore(&dmamux->lock, flags);
104
105 dma_spec->args[1] = dma_spec->args[2];
106 dma_spec->args_count = 2;
107
108 dev_dbg(&pdev->dev, "mapping dmamux %u.%u to dma request %u\n", mux,
109 dmamux->muxes[mux].value, mux);
110
111 return &dmamux->muxes[mux];
112}
113
114static int lpc18xx_dmamux_probe(struct platform_device *pdev)
115{
116 struct device_node *dma_np, *np = pdev->dev.of_node;
117 struct lpc18xx_dmamux_data *dmamux;
118 int ret;
119
120 dmamux = devm_kzalloc(&pdev->dev, sizeof(*dmamux), GFP_KERNEL);
121 if (!dmamux)
122 return -ENOMEM;
123
124 dmamux->reg = syscon_regmap_lookup_by_compatible("nxp,lpc1850-creg");
125 if (IS_ERR(dmamux->reg)) {
126 dev_err(&pdev->dev, "syscon lookup failed\n");
127 return PTR_ERR(dmamux->reg);
128 }
129
130 ret = of_property_read_u32(np, "dma-requests",
131 &dmamux->dma_mux_requests);
132 if (ret) {
133 dev_err(&pdev->dev, "missing dma-requests property\n");
134 return ret;
135 }
136
137 dma_np = of_parse_phandle(np, "dma-masters", 0);
138 if (!dma_np) {
139 dev_err(&pdev->dev, "can't get dma master\n");
140 return -ENODEV;
141 }
142
143 ret = of_property_read_u32(dma_np, "dma-requests",
144 &dmamux->dma_master_requests);
145 of_node_put(dma_np);
146 if (ret) {
147 dev_err(&pdev->dev, "missing master dma-requests property\n");
148 return ret;
149 }
150
151 dmamux->muxes = devm_kcalloc(&pdev->dev, dmamux->dma_master_requests,
152 sizeof(struct lpc18xx_dmamux),
153 GFP_KERNEL);
154 if (!dmamux->muxes)
155 return -ENOMEM;
156
157 spin_lock_init(&dmamux->lock);
158 platform_set_drvdata(pdev, dmamux);
159 dmamux->dmarouter.dev = &pdev->dev;
160 dmamux->dmarouter.route_free = lpc18xx_dmamux_free;
161
162 return of_dma_router_register(np, lpc18xx_dmamux_reserve,
163 &dmamux->dmarouter);
164}
165
166static const struct of_device_id lpc18xx_dmamux_match[] = {
167 { .compatible = "nxp,lpc1850-dmamux" },
168 {},
169};
170
171static struct platform_driver lpc18xx_dmamux_driver = {
172 .probe = lpc18xx_dmamux_probe,
173 .driver = {
174 .name = "lpc18xx-dmamux",
175 .of_match_table = lpc18xx_dmamux_match,
176 },
177};
178
179static int __init lpc18xx_dmamux_init(void)
180{
181 return platform_driver_register(&lpc18xx_dmamux_driver);
182}
183arch_initcall(lpc18xx_dmamux_init);
diff --git a/drivers/dma/mic_x100_dma.h b/drivers/dma/mic_x100_dma.h
index f663b0bdd11d..d89982034e68 100644
--- a/drivers/dma/mic_x100_dma.h
+++ b/drivers/dma/mic_x100_dma.h
@@ -39,7 +39,7 @@
39 */ 39 */
40#define MIC_DMA_MAX_NUM_CHAN 8 40#define MIC_DMA_MAX_NUM_CHAN 8
41#define MIC_DMA_NUM_CHAN 4 41#define MIC_DMA_NUM_CHAN 4
42#define MIC_DMA_ALIGN_SHIFT 6 42#define MIC_DMA_ALIGN_SHIFT DMAENGINE_ALIGN_64_BYTES
43#define MIC_DMA_ALIGN_BYTES (1 << MIC_DMA_ALIGN_SHIFT) 43#define MIC_DMA_ALIGN_BYTES (1 << MIC_DMA_ALIGN_SHIFT)
44#define MIC_DMA_DESC_RX_SIZE (128 * 1024 - 4) 44#define MIC_DMA_DESC_RX_SIZE (128 * 1024 - 4)
45 45
diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c
index 462a0229a743..e39457f13d4d 100644
--- a/drivers/dma/mmp_pdma.c
+++ b/drivers/dma/mmp_pdma.c
@@ -72,7 +72,6 @@
72#define DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */ 72#define DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */
73#define DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */ 73#define DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */
74 74
75#define PDMA_ALIGNMENT 3
76#define PDMA_MAX_DESC_BYTES DCMD_LENGTH 75#define PDMA_MAX_DESC_BYTES DCMD_LENGTH
77 76
78struct mmp_pdma_desc_hw { 77struct mmp_pdma_desc_hw {
@@ -1071,7 +1070,7 @@ static int mmp_pdma_probe(struct platform_device *op)
1071 pdev->device.device_issue_pending = mmp_pdma_issue_pending; 1070 pdev->device.device_issue_pending = mmp_pdma_issue_pending;
1072 pdev->device.device_config = mmp_pdma_config; 1071 pdev->device.device_config = mmp_pdma_config;
1073 pdev->device.device_terminate_all = mmp_pdma_terminate_all; 1072 pdev->device.device_terminate_all = mmp_pdma_terminate_all;
1074 pdev->device.copy_align = PDMA_ALIGNMENT; 1073 pdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES;
1075 pdev->device.src_addr_widths = widths; 1074 pdev->device.src_addr_widths = widths;
1076 pdev->device.dst_addr_widths = widths; 1075 pdev->device.dst_addr_widths = widths;
1077 pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 1076 pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c
index e683761e0f8f..3df0422607d5 100644
--- a/drivers/dma/mmp_tdma.c
+++ b/drivers/dma/mmp_tdma.c
@@ -100,7 +100,6 @@ enum mmp_tdma_type {
100 PXA910_SQU, 100 PXA910_SQU,
101}; 101};
102 102
103#define TDMA_ALIGNMENT 3
104#define TDMA_MAX_XFER_BYTES SZ_64K 103#define TDMA_MAX_XFER_BYTES SZ_64K
105 104
106struct mmp_tdma_chan { 105struct mmp_tdma_chan {
@@ -695,7 +694,7 @@ static int mmp_tdma_probe(struct platform_device *pdev)
695 tdev->device.device_pause = mmp_tdma_pause_chan; 694 tdev->device.device_pause = mmp_tdma_pause_chan;
696 tdev->device.device_resume = mmp_tdma_resume_chan; 695 tdev->device.device_resume = mmp_tdma_resume_chan;
697 tdev->device.device_terminate_all = mmp_tdma_terminate_all; 696 tdev->device.device_terminate_all = mmp_tdma_terminate_all;
698 tdev->device.copy_align = TDMA_ALIGNMENT; 697 tdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES;
699 698
700 dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 699 dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
701 platform_set_drvdata(pdev, tdev); 700 platform_set_drvdata(pdev, tdev);
diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c
index f1325f62563e..1c2de9a834a9 100644
--- a/drivers/dma/mv_xor.c
+++ b/drivers/dma/mv_xor.c
@@ -13,7 +13,6 @@
13 */ 13 */
14 14
15#include <linux/init.h> 15#include <linux/init.h>
16#include <linux/module.h>
17#include <linux/slab.h> 16#include <linux/slab.h>
18#include <linux/delay.h> 17#include <linux/delay.h>
19#include <linux/dma-mapping.h> 18#include <linux/dma-mapping.h>
@@ -26,6 +25,7 @@
26#include <linux/of.h> 25#include <linux/of.h>
27#include <linux/of_irq.h> 26#include <linux/of_irq.h>
28#include <linux/irqdomain.h> 27#include <linux/irqdomain.h>
28#include <linux/cpumask.h>
29#include <linux/platform_data/dma-mv_xor.h> 29#include <linux/platform_data/dma-mv_xor.h>
30 30
31#include "dmaengine.h" 31#include "dmaengine.h"
@@ -1126,7 +1126,8 @@ static const struct of_device_id mv_xor_dt_ids[] = {
1126 { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_MODE_IN_DESC }, 1126 { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_MODE_IN_DESC },
1127 {}, 1127 {},
1128}; 1128};
1129MODULE_DEVICE_TABLE(of, mv_xor_dt_ids); 1129
1130static unsigned int mv_xor_engine_count;
1130 1131
1131static int mv_xor_probe(struct platform_device *pdev) 1132static int mv_xor_probe(struct platform_device *pdev)
1132{ 1133{
@@ -1134,6 +1135,7 @@ static int mv_xor_probe(struct platform_device *pdev)
1134 struct mv_xor_device *xordev; 1135 struct mv_xor_device *xordev;
1135 struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev); 1136 struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);
1136 struct resource *res; 1137 struct resource *res;
1138 unsigned int max_engines, max_channels;
1137 int i, ret; 1139 int i, ret;
1138 int op_in_desc; 1140 int op_in_desc;
1139 1141
@@ -1177,6 +1179,21 @@ static int mv_xor_probe(struct platform_device *pdev)
1177 if (!IS_ERR(xordev->clk)) 1179 if (!IS_ERR(xordev->clk))
1178 clk_prepare_enable(xordev->clk); 1180 clk_prepare_enable(xordev->clk);
1179 1181
1182 /*
1183 * We don't want to have more than one channel per CPU in
1184 * order for async_tx to perform well. So we limit the number
1185 * of engines and channels so that we take into account this
1186 * constraint. Note that we also want to use channels from
1187 * separate engines when possible.
1188 */
1189 max_engines = num_present_cpus();
1190 max_channels = min_t(unsigned int,
1191 MV_XOR_MAX_CHANNELS,
1192 DIV_ROUND_UP(num_present_cpus(), 2));
1193
1194 if (mv_xor_engine_count >= max_engines)
1195 return 0;
1196
1180 if (pdev->dev.of_node) { 1197 if (pdev->dev.of_node) {
1181 struct device_node *np; 1198 struct device_node *np;
1182 int i = 0; 1199 int i = 0;
@@ -1190,13 +1207,13 @@ static int mv_xor_probe(struct platform_device *pdev)
1190 int irq; 1207 int irq;
1191 op_in_desc = (int)of_id->data; 1208 op_in_desc = (int)of_id->data;
1192 1209
1210 if (i >= max_channels)
1211 continue;
1212
1193 dma_cap_zero(cap_mask); 1213 dma_cap_zero(cap_mask);
1194 if (of_property_read_bool(np, "dmacap,memcpy")) 1214 dma_cap_set(DMA_MEMCPY, cap_mask);
1195 dma_cap_set(DMA_MEMCPY, cap_mask); 1215 dma_cap_set(DMA_XOR, cap_mask);
1196 if (of_property_read_bool(np, "dmacap,xor")) 1216 dma_cap_set(DMA_INTERRUPT, cap_mask);
1197 dma_cap_set(DMA_XOR, cap_mask);
1198 if (of_property_read_bool(np, "dmacap,interrupt"))
1199 dma_cap_set(DMA_INTERRUPT, cap_mask);
1200 1217
1201 irq = irq_of_parse_and_map(np, 0); 1218 irq = irq_of_parse_and_map(np, 0);
1202 if (!irq) { 1219 if (!irq) {
@@ -1216,7 +1233,7 @@ static int mv_xor_probe(struct platform_device *pdev)
1216 i++; 1233 i++;
1217 } 1234 }
1218 } else if (pdata && pdata->channels) { 1235 } else if (pdata && pdata->channels) {
1219 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) { 1236 for (i = 0; i < max_channels; i++) {
1220 struct mv_xor_channel_data *cd; 1237 struct mv_xor_channel_data *cd;
1221 struct mv_xor_chan *chan; 1238 struct mv_xor_chan *chan;
1222 int irq; 1239 int irq;
@@ -1263,27 +1280,8 @@ err_channel_add:
1263 return ret; 1280 return ret;
1264} 1281}
1265 1282
1266static int mv_xor_remove(struct platform_device *pdev)
1267{
1268 struct mv_xor_device *xordev = platform_get_drvdata(pdev);
1269 int i;
1270
1271 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1272 if (xordev->channels[i])
1273 mv_xor_channel_remove(xordev->channels[i]);
1274 }
1275
1276 if (!IS_ERR(xordev->clk)) {
1277 clk_disable_unprepare(xordev->clk);
1278 clk_put(xordev->clk);
1279 }
1280
1281 return 0;
1282}
1283
1284static struct platform_driver mv_xor_driver = { 1283static struct platform_driver mv_xor_driver = {
1285 .probe = mv_xor_probe, 1284 .probe = mv_xor_probe,
1286 .remove = mv_xor_remove,
1287 .driver = { 1285 .driver = {
1288 .name = MV_XOR_NAME, 1286 .name = MV_XOR_NAME,
1289 .of_match_table = of_match_ptr(mv_xor_dt_ids), 1287 .of_match_table = of_match_ptr(mv_xor_dt_ids),
@@ -1295,19 +1293,10 @@ static int __init mv_xor_init(void)
1295{ 1293{
1296 return platform_driver_register(&mv_xor_driver); 1294 return platform_driver_register(&mv_xor_driver);
1297} 1295}
1298module_init(mv_xor_init); 1296device_initcall(mv_xor_init);
1299
1300/* it's currently unsafe to unload this module */
1301#if 0
1302static void __exit mv_xor_exit(void)
1303{
1304 platform_driver_unregister(&mv_xor_driver);
1305 return;
1306}
1307
1308module_exit(mv_xor_exit);
1309#endif
1310 1297
1298/*
1311MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>"); 1299MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1312MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine"); 1300MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1313MODULE_LICENSE("GPL"); 1301MODULE_LICENSE("GPL");
1302*/
diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c
index b859792dde95..113605f6fe20 100644
--- a/drivers/dma/pch_dma.c
+++ b/drivers/dma/pch_dma.c
@@ -11,10 +11,6 @@
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details. 13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 */ 14 */
19 15
20#include <linux/dmaengine.h> 16#include <linux/dmaengine.h>
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index ecab4ea059b4..17ee758b419f 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -1198,6 +1198,9 @@ static inline int _loop(unsigned dry_run, u8 buf[],
1198 unsigned lcnt0, lcnt1, ljmp0, ljmp1; 1198 unsigned lcnt0, lcnt1, ljmp0, ljmp1;
1199 struct _arg_LPEND lpend; 1199 struct _arg_LPEND lpend;
1200 1200
1201 if (*bursts == 1)
1202 return _bursts(dry_run, buf, pxs, 1);
1203
1201 /* Max iterations possible in DMALP is 256 */ 1204 /* Max iterations possible in DMALP is 256 */
1202 if (*bursts >= 256*256) { 1205 if (*bursts >= 256*256) {
1203 lcnt1 = 256; 1206 lcnt1 = 256;
diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c
index ddcbbf5cd9e9..5cb61ce01036 100644
--- a/drivers/dma/pxa_dma.c
+++ b/drivers/dma/pxa_dma.c
@@ -184,19 +184,18 @@ static unsigned int pxad_drcmr(unsigned int line)
184 184
185static int dbg_show_requester_chan(struct seq_file *s, void *p) 185static int dbg_show_requester_chan(struct seq_file *s, void *p)
186{ 186{
187 int pos = 0;
188 struct pxad_phy *phy = s->private; 187 struct pxad_phy *phy = s->private;
189 int i; 188 int i;
190 u32 drcmr; 189 u32 drcmr;
191 190
192 pos += seq_printf(s, "DMA channel %d requester :\n", phy->idx); 191 seq_printf(s, "DMA channel %d requester :\n", phy->idx);
193 for (i = 0; i < 70; i++) { 192 for (i = 0; i < 70; i++) {
194 drcmr = readl_relaxed(phy->base + pxad_drcmr(i)); 193 drcmr = readl_relaxed(phy->base + pxad_drcmr(i));
195 if ((drcmr & DRCMR_CHLNUM) == phy->idx) 194 if ((drcmr & DRCMR_CHLNUM) == phy->idx)
196 pos += seq_printf(s, "\tRequester %d (MAPVLD=%d)\n", i, 195 seq_printf(s, "\tRequester %d (MAPVLD=%d)\n", i,
197 !!(drcmr & DRCMR_MAPVLD)); 196 !!(drcmr & DRCMR_MAPVLD));
198 } 197 }
199 return pos; 198 return 0;
200} 199}
201 200
202static inline int dbg_burst_from_dcmd(u32 dcmd) 201static inline int dbg_burst_from_dcmd(u32 dcmd)
@@ -906,21 +905,21 @@ static void pxad_get_config(struct pxad_chan *chan,
906 enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; 905 enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED;
907 906
908 *dcmd = 0; 907 *dcmd = 0;
909 if (chan->cfg.direction == DMA_DEV_TO_MEM) { 908 if (dir == DMA_DEV_TO_MEM) {
910 maxburst = chan->cfg.src_maxburst; 909 maxburst = chan->cfg.src_maxburst;
911 width = chan->cfg.src_addr_width; 910 width = chan->cfg.src_addr_width;
912 dev_addr = chan->cfg.src_addr; 911 dev_addr = chan->cfg.src_addr;
913 *dev_src = dev_addr; 912 *dev_src = dev_addr;
914 *dcmd |= PXA_DCMD_INCTRGADDR | PXA_DCMD_FLOWSRC; 913 *dcmd |= PXA_DCMD_INCTRGADDR | PXA_DCMD_FLOWSRC;
915 } 914 }
916 if (chan->cfg.direction == DMA_MEM_TO_DEV) { 915 if (dir == DMA_MEM_TO_DEV) {
917 maxburst = chan->cfg.dst_maxburst; 916 maxburst = chan->cfg.dst_maxburst;
918 width = chan->cfg.dst_addr_width; 917 width = chan->cfg.dst_addr_width;
919 dev_addr = chan->cfg.dst_addr; 918 dev_addr = chan->cfg.dst_addr;
920 *dev_dst = dev_addr; 919 *dev_dst = dev_addr;
921 *dcmd |= PXA_DCMD_INCSRCADDR | PXA_DCMD_FLOWTRG; 920 *dcmd |= PXA_DCMD_INCSRCADDR | PXA_DCMD_FLOWTRG;
922 } 921 }
923 if (chan->cfg.direction == DMA_MEM_TO_MEM) 922 if (dir == DMA_MEM_TO_MEM)
924 *dcmd |= PXA_DCMD_BURST32 | PXA_DCMD_INCTRGADDR | 923 *dcmd |= PXA_DCMD_BURST32 | PXA_DCMD_INCTRGADDR |
925 PXA_DCMD_INCSRCADDR; 924 PXA_DCMD_INCSRCADDR;
926 925
diff --git a/drivers/dma/sh/Kconfig b/drivers/dma/sh/Kconfig
index 0f371524a4d9..9fda65af841e 100644
--- a/drivers/dma/sh/Kconfig
+++ b/drivers/dma/sh/Kconfig
@@ -39,18 +39,6 @@ config SH_DMAE_R8A73A4
39 39
40endif 40endif
41 41
42config SUDMAC
43 tristate "Renesas SUDMAC support"
44 depends on SH_DMAE_BASE
45 help
46 Enable support for the Renesas SUDMAC controllers.
47
48config RCAR_HPB_DMAE
49 tristate "Renesas R-Car HPB DMAC support"
50 depends on SH_DMAE_BASE
51 help
52 Enable support for the Renesas R-Car series DMA controllers.
53
54config RCAR_DMAC 42config RCAR_DMAC
55 tristate "Renesas R-Car Gen2 DMA Controller" 43 tristate "Renesas R-Car Gen2 DMA Controller"
56 depends on ARCH_SHMOBILE || COMPILE_TEST 44 depends on ARCH_SHMOBILE || COMPILE_TEST
@@ -59,6 +47,12 @@ config RCAR_DMAC
59 This driver supports the general purpose DMA controller found in the 47 This driver supports the general purpose DMA controller found in the
60 Renesas R-Car second generation SoCs. 48 Renesas R-Car second generation SoCs.
61 49
50config RCAR_HPB_DMAE
51 tristate "Renesas R-Car HPB DMAC support"
52 depends on SH_DMAE_BASE
53 help
54 Enable support for the Renesas R-Car series DMA controllers.
55
62config RENESAS_USB_DMAC 56config RENESAS_USB_DMAC
63 tristate "Renesas USB-DMA Controller" 57 tristate "Renesas USB-DMA Controller"
64 depends on ARCH_SHMOBILE || COMPILE_TEST 58 depends on ARCH_SHMOBILE || COMPILE_TEST
@@ -67,3 +61,9 @@ config RENESAS_USB_DMAC
67 help 61 help
68 This driver supports the USB-DMA controller found in the Renesas 62 This driver supports the USB-DMA controller found in the Renesas
69 SoCs. 63 SoCs.
64
65config SUDMAC
66 tristate "Renesas SUDMAC support"
67 depends on SH_DMAE_BASE
68 help
69 Enable support for the Renesas SUDMAC controllers.
diff --git a/drivers/dma/sh/Makefile b/drivers/dma/sh/Makefile
index b8a598066ce2..0133e4658196 100644
--- a/drivers/dma/sh/Makefile
+++ b/drivers/dma/sh/Makefile
@@ -13,7 +13,7 @@ shdma-$(CONFIG_SH_DMAE_R8A73A4) += shdma-r8a73a4.o
13shdma-objs := $(shdma-y) 13shdma-objs := $(shdma-y)
14obj-$(CONFIG_SH_DMAE) += shdma.o 14obj-$(CONFIG_SH_DMAE) += shdma.o
15 15
16obj-$(CONFIG_SUDMAC) += sudmac.o
17obj-$(CONFIG_RCAR_HPB_DMAE) += rcar-hpbdma.o
18obj-$(CONFIG_RCAR_DMAC) += rcar-dmac.o 16obj-$(CONFIG_RCAR_DMAC) += rcar-dmac.o
17obj-$(CONFIG_RCAR_HPB_DMAE) += rcar-hpbdma.o
19obj-$(CONFIG_RENESAS_USB_DMAC) += usb-dmac.o 18obj-$(CONFIG_RENESAS_USB_DMAC) += usb-dmac.o
19obj-$(CONFIG_SUDMAC) += sudmac.o
diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c
index 8c5186cc9f63..7d5598d874e1 100644
--- a/drivers/dma/sirf-dma.c
+++ b/drivers/dma/sirf-dma.c
@@ -455,6 +455,7 @@ static int sirfsoc_dma_terminate_all(struct dma_chan *chan)
455 switch (sdma->type) { 455 switch (sdma->type) {
456 case SIRFSOC_DMA_VER_A7V1: 456 case SIRFSOC_DMA_VER_A7V1:
457 writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_INT_EN_CLR); 457 writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_INT_EN_CLR);
458 writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_INT);
458 writel_relaxed((1 << cid) | 1 << (cid + 16), 459 writel_relaxed((1 << cid) | 1 << (cid + 16),
459 sdma->base + 460 sdma->base +
460 SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7); 461 SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7);
@@ -462,6 +463,8 @@ static int sirfsoc_dma_terminate_all(struct dma_chan *chan)
462 break; 463 break;
463 case SIRFSOC_DMA_VER_A7V2: 464 case SIRFSOC_DMA_VER_A7V2:
464 writel_relaxed(0, sdma->base + SIRFSOC_DMA_INT_EN_ATLAS7); 465 writel_relaxed(0, sdma->base + SIRFSOC_DMA_INT_EN_ATLAS7);
466 writel_relaxed(SIRFSOC_DMA_INT_ALL_ATLAS7,
467 sdma->base + SIRFSOC_DMA_INT_ATLAS7);
465 writel_relaxed(0, sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); 468 writel_relaxed(0, sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7);
466 writel_relaxed(0, sdma->base + SIRFSOC_DMA_VALID_ATLAS7); 469 writel_relaxed(0, sdma->base + SIRFSOC_DMA_VALID_ATLAS7);
467 break; 470 break;
diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c
index 3c10f034d4b9..750d1b313684 100644
--- a/drivers/dma/ste_dma40.c
+++ b/drivers/dma/ste_dma40.c
@@ -2853,7 +2853,7 @@ static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
2853 * This controller can only access address at even 2853 * This controller can only access address at even
2854 * 32bit boundaries, i.e. 2^2 2854 * 32bit boundaries, i.e. 2^2
2855 */ 2855 */
2856 dev->copy_align = 2; 2856 dev->copy_align = DMAENGINE_ALIGN_4_BYTES;
2857 } 2857 }
2858 2858
2859 if (dma_has_cap(DMA_SG, dev->cap_mask)) 2859 if (dma_has_cap(DMA_SG, dev->cap_mask))
diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c
new file mode 100644
index 000000000000..a1a500d96ff2
--- /dev/null
+++ b/drivers/dma/sun4i-dma.c
@@ -0,0 +1,1288 @@
1/*
2 * Copyright (C) 2014 Emilio López
3 * Emilio López <emilio@elopez.com.ar>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 */
10
11#include <linux/bitmap.h>
12#include <linux/bitops.h>
13#include <linux/clk.h>
14#include <linux/dmaengine.h>
15#include <linux/dmapool.h>
16#include <linux/interrupt.h>
17#include <linux/module.h>
18#include <linux/of_dma.h>
19#include <linux/platform_device.h>
20#include <linux/slab.h>
21#include <linux/spinlock.h>
22
23#include "virt-dma.h"
24
25/** Common macros to normal and dedicated DMA registers **/
26
27#define SUN4I_DMA_CFG_LOADING BIT(31)
28#define SUN4I_DMA_CFG_DST_DATA_WIDTH(width) ((width) << 25)
29#define SUN4I_DMA_CFG_DST_BURST_LENGTH(len) ((len) << 23)
30#define SUN4I_DMA_CFG_DST_ADDR_MODE(mode) ((mode) << 21)
31#define SUN4I_DMA_CFG_DST_DRQ_TYPE(type) ((type) << 16)
32#define SUN4I_DMA_CFG_SRC_DATA_WIDTH(width) ((width) << 9)
33#define SUN4I_DMA_CFG_SRC_BURST_LENGTH(len) ((len) << 7)
34#define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode) ((mode) << 5)
35#define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type) (type)
36
37/** Normal DMA register values **/
38
39/* Normal DMA source/destination data request type values */
40#define SUN4I_NDMA_DRQ_TYPE_SDRAM 0x16
41#define SUN4I_NDMA_DRQ_TYPE_LIMIT (0x1F + 1)
42
43/** Normal DMA register layout **/
44
45/* Dedicated DMA source/destination address mode values */
46#define SUN4I_NDMA_ADDR_MODE_LINEAR 0
47#define SUN4I_NDMA_ADDR_MODE_IO 1
48
49/* Normal DMA configuration register layout */
50#define SUN4I_NDMA_CFG_CONT_MODE BIT(30)
51#define SUN4I_NDMA_CFG_WAIT_STATE(n) ((n) << 27)
52#define SUN4I_NDMA_CFG_DST_NON_SECURE BIT(22)
53#define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15)
54#define SUN4I_NDMA_CFG_SRC_NON_SECURE BIT(6)
55
56/** Dedicated DMA register values **/
57
58/* Dedicated DMA source/destination address mode values */
59#define SUN4I_DDMA_ADDR_MODE_LINEAR 0
60#define SUN4I_DDMA_ADDR_MODE_IO 1
61#define SUN4I_DDMA_ADDR_MODE_HORIZONTAL_PAGE 2
62#define SUN4I_DDMA_ADDR_MODE_VERTICAL_PAGE 3
63
64/* Dedicated DMA source/destination data request type values */
65#define SUN4I_DDMA_DRQ_TYPE_SDRAM 0x1
66#define SUN4I_DDMA_DRQ_TYPE_LIMIT (0x1F + 1)
67
68/** Dedicated DMA register layout **/
69
70/* Dedicated DMA configuration register layout */
71#define SUN4I_DDMA_CFG_BUSY BIT(30)
72#define SUN4I_DDMA_CFG_CONT_MODE BIT(29)
73#define SUN4I_DDMA_CFG_DST_NON_SECURE BIT(28)
74#define SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15)
75#define SUN4I_DDMA_CFG_SRC_NON_SECURE BIT(12)
76
77/* Dedicated DMA parameter register layout */
78#define SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(n) (((n) - 1) << 24)
79#define SUN4I_DDMA_PARA_DST_WAIT_CYCLES(n) (((n) - 1) << 16)
80#define SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(n) (((n) - 1) << 8)
81#define SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(n) (((n) - 1) << 0)
82
83/** DMA register offsets **/
84
85/* General register offsets */
86#define SUN4I_DMA_IRQ_ENABLE_REG 0x0
87#define SUN4I_DMA_IRQ_PENDING_STATUS_REG 0x4
88
89/* Normal DMA register offsets */
90#define SUN4I_NDMA_CHANNEL_REG_BASE(n) (0x100 + (n) * 0x20)
91#define SUN4I_NDMA_CFG_REG 0x0
92#define SUN4I_NDMA_SRC_ADDR_REG 0x4
93#define SUN4I_NDMA_DST_ADDR_REG 0x8
94#define SUN4I_NDMA_BYTE_COUNT_REG 0xC
95
96/* Dedicated DMA register offsets */
97#define SUN4I_DDMA_CHANNEL_REG_BASE(n) (0x300 + (n) * 0x20)
98#define SUN4I_DDMA_CFG_REG 0x0
99#define SUN4I_DDMA_SRC_ADDR_REG 0x4
100#define SUN4I_DDMA_DST_ADDR_REG 0x8
101#define SUN4I_DDMA_BYTE_COUNT_REG 0xC
102#define SUN4I_DDMA_PARA_REG 0x18
103
104/** DMA Driver **/
105
106/*
107 * Normal DMA has 8 channels, and Dedicated DMA has another 8, so
108 * that's 16 channels. As for endpoints, there's 29 and 21
109 * respectively. Given that the Normal DMA endpoints (other than
110 * SDRAM) can be used as tx/rx, we need 78 vchans in total
111 */
112#define SUN4I_NDMA_NR_MAX_CHANNELS 8
113#define SUN4I_DDMA_NR_MAX_CHANNELS 8
114#define SUN4I_DMA_NR_MAX_CHANNELS \
115 (SUN4I_NDMA_NR_MAX_CHANNELS + SUN4I_DDMA_NR_MAX_CHANNELS)
116#define SUN4I_NDMA_NR_MAX_VCHANS (29 * 2 - 1)
117#define SUN4I_DDMA_NR_MAX_VCHANS 21
118#define SUN4I_DMA_NR_MAX_VCHANS \
119 (SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS)
120
121/* This set of SUN4I_DDMA timing parameters were found experimentally while
122 * working with the SPI driver and seem to make it behave correctly */
123#define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \
124 (SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(1) | \
125 SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(1) | \
126 SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) | \
127 SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2))
128
129struct sun4i_dma_pchan {
130 /* Register base of channel */
131 void __iomem *base;
132 /* vchan currently being serviced */
133 struct sun4i_dma_vchan *vchan;
134 /* Is this a dedicated pchan? */
135 int is_dedicated;
136};
137
138struct sun4i_dma_vchan {
139 struct virt_dma_chan vc;
140 struct dma_slave_config cfg;
141 struct sun4i_dma_pchan *pchan;
142 struct sun4i_dma_promise *processing;
143 struct sun4i_dma_contract *contract;
144 u8 endpoint;
145 int is_dedicated;
146};
147
148struct sun4i_dma_promise {
149 u32 cfg;
150 u32 para;
151 dma_addr_t src;
152 dma_addr_t dst;
153 size_t len;
154 struct list_head list;
155};
156
157/* A contract is a set of promises */
158struct sun4i_dma_contract {
159 struct virt_dma_desc vd;
160 struct list_head demands;
161 struct list_head completed_demands;
162 int is_cyclic;
163};
164
165struct sun4i_dma_dev {
166 DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS);
167 struct dma_device slave;
168 struct sun4i_dma_pchan *pchans;
169 struct sun4i_dma_vchan *vchans;
170 void __iomem *base;
171 struct clk *clk;
172 int irq;
173 spinlock_t lock;
174};
175
176static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev)
177{
178 return container_of(dev, struct sun4i_dma_dev, slave);
179}
180
181static struct sun4i_dma_vchan *to_sun4i_dma_vchan(struct dma_chan *chan)
182{
183 return container_of(chan, struct sun4i_dma_vchan, vc.chan);
184}
185
186static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd)
187{
188 return container_of(vd, struct sun4i_dma_contract, vd);
189}
190
191static struct device *chan2dev(struct dma_chan *chan)
192{
193 return &chan->dev->device;
194}
195
196static int convert_burst(u32 maxburst)
197{
198 if (maxburst > 8)
199 return -EINVAL;
200
201 /* 1 -> 0, 4 -> 1, 8 -> 2 */
202 return (maxburst >> 2);
203}
204
205static int convert_buswidth(enum dma_slave_buswidth addr_width)
206{
207 if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES)
208 return -EINVAL;
209
210 /* 8 (1 byte) -> 0, 16 (2 bytes) -> 1, 32 (4 bytes) -> 2 */
211 return (addr_width >> 1);
212}
213
214static void sun4i_dma_free_chan_resources(struct dma_chan *chan)
215{
216 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
217
218 vchan_free_chan_resources(&vchan->vc);
219}
220
221static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv,
222 struct sun4i_dma_vchan *vchan)
223{
224 struct sun4i_dma_pchan *pchan = NULL, *pchans = priv->pchans;
225 unsigned long flags;
226 int i, max;
227
228 /*
229 * pchans 0-SUN4I_NDMA_NR_MAX_CHANNELS are normal, and
230 * SUN4I_NDMA_NR_MAX_CHANNELS+ are dedicated ones
231 */
232 if (vchan->is_dedicated) {
233 i = SUN4I_NDMA_NR_MAX_CHANNELS;
234 max = SUN4I_DMA_NR_MAX_CHANNELS;
235 } else {
236 i = 0;
237 max = SUN4I_NDMA_NR_MAX_CHANNELS;
238 }
239
240 spin_lock_irqsave(&priv->lock, flags);
241 for_each_clear_bit_from(i, &priv->pchans_used, max) {
242 pchan = &pchans[i];
243 pchan->vchan = vchan;
244 set_bit(i, priv->pchans_used);
245 break;
246 }
247 spin_unlock_irqrestore(&priv->lock, flags);
248
249 return pchan;
250}
251
252static void release_pchan(struct sun4i_dma_dev *priv,
253 struct sun4i_dma_pchan *pchan)
254{
255 unsigned long flags;
256 int nr = pchan - priv->pchans;
257
258 spin_lock_irqsave(&priv->lock, flags);
259
260 pchan->vchan = NULL;
261 clear_bit(nr, priv->pchans_used);
262
263 spin_unlock_irqrestore(&priv->lock, flags);
264}
265
266static void configure_pchan(struct sun4i_dma_pchan *pchan,
267 struct sun4i_dma_promise *d)
268{
269 /*
270 * Configure addresses and misc parameters depending on type
271 * SUN4I_DDMA has an extra field with timing parameters
272 */
273 if (pchan->is_dedicated) {
274 writel_relaxed(d->src, pchan->base + SUN4I_DDMA_SRC_ADDR_REG);
275 writel_relaxed(d->dst, pchan->base + SUN4I_DDMA_DST_ADDR_REG);
276 writel_relaxed(d->len, pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
277 writel_relaxed(d->para, pchan->base + SUN4I_DDMA_PARA_REG);
278 writel_relaxed(d->cfg, pchan->base + SUN4I_DDMA_CFG_REG);
279 } else {
280 writel_relaxed(d->src, pchan->base + SUN4I_NDMA_SRC_ADDR_REG);
281 writel_relaxed(d->dst, pchan->base + SUN4I_NDMA_DST_ADDR_REG);
282 writel_relaxed(d->len, pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
283 writel_relaxed(d->cfg, pchan->base + SUN4I_NDMA_CFG_REG);
284 }
285}
286
287static void set_pchan_interrupt(struct sun4i_dma_dev *priv,
288 struct sun4i_dma_pchan *pchan,
289 int half, int end)
290{
291 u32 reg;
292 int pchan_number = pchan - priv->pchans;
293 unsigned long flags;
294
295 spin_lock_irqsave(&priv->lock, flags);
296
297 reg = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
298
299 if (half)
300 reg |= BIT(pchan_number * 2);
301 else
302 reg &= ~BIT(pchan_number * 2);
303
304 if (end)
305 reg |= BIT(pchan_number * 2 + 1);
306 else
307 reg &= ~BIT(pchan_number * 2 + 1);
308
309 writel_relaxed(reg, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
310
311 spin_unlock_irqrestore(&priv->lock, flags);
312}
313
314/**
315 * Execute pending operations on a vchan
316 *
317 * When given a vchan, this function will try to acquire a suitable
318 * pchan and, if successful, will configure it to fulfill a promise
319 * from the next pending contract.
320 *
321 * This function must be called with &vchan->vc.lock held.
322 */
323static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
324 struct sun4i_dma_vchan *vchan)
325{
326 struct sun4i_dma_promise *promise = NULL;
327 struct sun4i_dma_contract *contract = NULL;
328 struct sun4i_dma_pchan *pchan;
329 struct virt_dma_desc *vd;
330 int ret;
331
332 lockdep_assert_held(&vchan->vc.lock);
333
334 /* We need a pchan to do anything, so secure one if available */
335 pchan = find_and_use_pchan(priv, vchan);
336 if (!pchan)
337 return -EBUSY;
338
339 /*
340 * Channel endpoints must not be repeated, so if this vchan
341 * has already submitted some work, we can't do anything else
342 */
343 if (vchan->processing) {
344 dev_dbg(chan2dev(&vchan->vc.chan),
345 "processing something to this endpoint already\n");
346 ret = -EBUSY;
347 goto release_pchan;
348 }
349
350 do {
351 /* Figure out which contract we're working with today */
352 vd = vchan_next_desc(&vchan->vc);
353 if (!vd) {
354 dev_dbg(chan2dev(&vchan->vc.chan),
355 "No pending contract found");
356 ret = 0;
357 goto release_pchan;
358 }
359
360 contract = to_sun4i_dma_contract(vd);
361 if (list_empty(&contract->demands)) {
362 /* The contract has been completed so mark it as such */
363 list_del(&contract->vd.node);
364 vchan_cookie_complete(&contract->vd);
365 dev_dbg(chan2dev(&vchan->vc.chan),
366 "Empty contract found and marked complete");
367 }
368 } while (list_empty(&contract->demands));
369
370 /* Now find out what we need to do */
371 promise = list_first_entry(&contract->demands,
372 struct sun4i_dma_promise, list);
373 vchan->processing = promise;
374
375 /* ... and make it reality */
376 if (promise) {
377 vchan->contract = contract;
378 vchan->pchan = pchan;
379 set_pchan_interrupt(priv, pchan, contract->is_cyclic, 1);
380 configure_pchan(pchan, promise);
381 }
382
383 return 0;
384
385release_pchan:
386 release_pchan(priv, pchan);
387 return ret;
388}
389
390static int sanitize_config(struct dma_slave_config *sconfig,
391 enum dma_transfer_direction direction)
392{
393 switch (direction) {
394 case DMA_MEM_TO_DEV:
395 if ((sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
396 !sconfig->dst_maxburst)
397 return -EINVAL;
398
399 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
400 sconfig->src_addr_width = sconfig->dst_addr_width;
401
402 if (!sconfig->src_maxburst)
403 sconfig->src_maxburst = sconfig->dst_maxburst;
404
405 break;
406
407 case DMA_DEV_TO_MEM:
408 if ((sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
409 !sconfig->src_maxburst)
410 return -EINVAL;
411
412 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
413 sconfig->dst_addr_width = sconfig->src_addr_width;
414
415 if (!sconfig->dst_maxburst)
416 sconfig->dst_maxburst = sconfig->src_maxburst;
417
418 break;
419 default:
420 return 0;
421 }
422
423 return 0;
424}
425
426/**
427 * Generate a promise, to be used in a normal DMA contract.
428 *
429 * A NDMA promise contains all the information required to program the
430 * normal part of the DMA Engine and get data copied. A non-executed
431 * promise will live in the demands list on a contract. Once it has been
432 * completed, it will be moved to the completed demands list for later freeing.
433 * All linked promises will be freed when the corresponding contract is freed
434 */
435static struct sun4i_dma_promise *
436generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
437 size_t len, struct dma_slave_config *sconfig,
438 enum dma_transfer_direction direction)
439{
440 struct sun4i_dma_promise *promise;
441 int ret;
442
443 ret = sanitize_config(sconfig, direction);
444 if (ret)
445 return NULL;
446
447 promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
448 if (!promise)
449 return NULL;
450
451 promise->src = src;
452 promise->dst = dest;
453 promise->len = len;
454 promise->cfg = SUN4I_DMA_CFG_LOADING |
455 SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN;
456
457 dev_dbg(chan2dev(chan),
458 "src burst %d, dst burst %d, src buswidth %d, dst buswidth %d",
459 sconfig->src_maxburst, sconfig->dst_maxburst,
460 sconfig->src_addr_width, sconfig->dst_addr_width);
461
462 /* Source burst */
463 ret = convert_burst(sconfig->src_maxburst);
464 if (IS_ERR_VALUE(ret))
465 goto fail;
466 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
467
468 /* Destination burst */
469 ret = convert_burst(sconfig->dst_maxburst);
470 if (IS_ERR_VALUE(ret))
471 goto fail;
472 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
473
474 /* Source bus width */
475 ret = convert_buswidth(sconfig->src_addr_width);
476 if (IS_ERR_VALUE(ret))
477 goto fail;
478 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
479
480 /* Destination bus width */
481 ret = convert_buswidth(sconfig->dst_addr_width);
482 if (IS_ERR_VALUE(ret))
483 goto fail;
484 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
485
486 return promise;
487
488fail:
489 kfree(promise);
490 return NULL;
491}
492
493/**
494 * Generate a promise, to be used in a dedicated DMA contract.
495 *
496 * A DDMA promise contains all the information required to program the
497 * Dedicated part of the DMA Engine and get data copied. A non-executed
498 * promise will live in the demands list on a contract. Once it has been
499 * completed, it will be moved to the completed demands list for later freeing.
500 * All linked promises will be freed when the corresponding contract is freed
501 */
502static struct sun4i_dma_promise *
503generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
504 size_t len, struct dma_slave_config *sconfig)
505{
506 struct sun4i_dma_promise *promise;
507 int ret;
508
509 promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
510 if (!promise)
511 return NULL;
512
513 promise->src = src;
514 promise->dst = dest;
515 promise->len = len;
516 promise->cfg = SUN4I_DMA_CFG_LOADING |
517 SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN;
518
519 /* Source burst */
520 ret = convert_burst(sconfig->src_maxburst);
521 if (IS_ERR_VALUE(ret))
522 goto fail;
523 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
524
525 /* Destination burst */
526 ret = convert_burst(sconfig->dst_maxburst);
527 if (IS_ERR_VALUE(ret))
528 goto fail;
529 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
530
531 /* Source bus width */
532 ret = convert_buswidth(sconfig->src_addr_width);
533 if (IS_ERR_VALUE(ret))
534 goto fail;
535 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
536
537 /* Destination bus width */
538 ret = convert_buswidth(sconfig->dst_addr_width);
539 if (IS_ERR_VALUE(ret))
540 goto fail;
541 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
542
543 return promise;
544
545fail:
546 kfree(promise);
547 return NULL;
548}
549
550/**
551 * Generate a contract
552 *
553 * Contracts function as DMA descriptors. As our hardware does not support
554 * linked lists, we need to implement SG via software. We use a contract
555 * to hold all the pieces of the request and process them serially one
556 * after another. Each piece is represented as a promise.
557 */
558static struct sun4i_dma_contract *generate_dma_contract(void)
559{
560 struct sun4i_dma_contract *contract;
561
562 contract = kzalloc(sizeof(*contract), GFP_NOWAIT);
563 if (!contract)
564 return NULL;
565
566 INIT_LIST_HEAD(&contract->demands);
567 INIT_LIST_HEAD(&contract->completed_demands);
568
569 return contract;
570}
571
572/**
573 * Get next promise on a cyclic transfer
574 *
575 * Cyclic contracts contain a series of promises which are executed on a
576 * loop. This function returns the next promise from a cyclic contract,
577 * so it can be programmed into the hardware.
578 */
579static struct sun4i_dma_promise *
580get_next_cyclic_promise(struct sun4i_dma_contract *contract)
581{
582 struct sun4i_dma_promise *promise;
583
584 promise = list_first_entry_or_null(&contract->demands,
585 struct sun4i_dma_promise, list);
586 if (!promise) {
587 list_splice_init(&contract->completed_demands,
588 &contract->demands);
589 promise = list_first_entry(&contract->demands,
590 struct sun4i_dma_promise, list);
591 }
592
593 return promise;
594}
595
596/**
597 * Free a contract and all its associated promises
598 */
599static void sun4i_dma_free_contract(struct virt_dma_desc *vd)
600{
601 struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd);
602 struct sun4i_dma_promise *promise;
603
604 /* Free all the demands and completed demands */
605 list_for_each_entry(promise, &contract->demands, list)
606 kfree(promise);
607
608 list_for_each_entry(promise, &contract->completed_demands, list)
609 kfree(promise);
610
611 kfree(contract);
612}
613
614static struct dma_async_tx_descriptor *
615sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
616 dma_addr_t src, size_t len, unsigned long flags)
617{
618 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
619 struct dma_slave_config *sconfig = &vchan->cfg;
620 struct sun4i_dma_promise *promise;
621 struct sun4i_dma_contract *contract;
622
623 contract = generate_dma_contract();
624 if (!contract)
625 return NULL;
626
627 /*
628 * We can only do the copy to bus aligned addresses, so
629 * choose the best one so we get decent performance. We also
630 * maximize the burst size for this same reason.
631 */
632 sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
633 sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
634 sconfig->src_maxburst = 8;
635 sconfig->dst_maxburst = 8;
636
637 if (vchan->is_dedicated)
638 promise = generate_ddma_promise(chan, src, dest, len, sconfig);
639 else
640 promise = generate_ndma_promise(chan, src, dest, len, sconfig,
641 DMA_MEM_TO_MEM);
642
643 if (!promise) {
644 kfree(contract);
645 return NULL;
646 }
647
648 /* Configure memcpy mode */
649 if (vchan->is_dedicated) {
650 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) |
651 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM);
652 } else {
653 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) |
654 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM);
655 }
656
657 /* Fill the contract with our only promise */
658 list_add_tail(&promise->list, &contract->demands);
659
660 /* And add it to the vchan */
661 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
662}
663
664static struct dma_async_tx_descriptor *
665sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
666 size_t period_len, enum dma_transfer_direction dir,
667 unsigned long flags)
668{
669 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
670 struct dma_slave_config *sconfig = &vchan->cfg;
671 struct sun4i_dma_promise *promise;
672 struct sun4i_dma_contract *contract;
673 dma_addr_t src, dest;
674 u32 endpoints;
675 int nr_periods, offset, plength, i;
676
677 if (!is_slave_direction(dir)) {
678 dev_err(chan2dev(chan), "Invalid DMA direction\n");
679 return NULL;
680 }
681
682 if (vchan->is_dedicated) {
683 /*
684 * As we are using this just for audio data, we need to use
685 * normal DMA. There is nothing stopping us from supporting
686 * dedicated DMA here as well, so if a client comes up and
687 * requires it, it will be simple to implement it.
688 */
689 dev_err(chan2dev(chan),
690 "Cyclic transfers are only supported on Normal DMA\n");
691 return NULL;
692 }
693
694 contract = generate_dma_contract();
695 if (!contract)
696 return NULL;
697
698 contract->is_cyclic = 1;
699
700 /* Figure out the endpoints and the address we need */
701 if (dir == DMA_MEM_TO_DEV) {
702 src = buf;
703 dest = sconfig->dst_addr;
704 endpoints = SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) |
705 SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
706 SUN4I_DMA_CFG_DST_ADDR_MODE(SUN4I_NDMA_ADDR_MODE_IO);
707 } else {
708 src = sconfig->src_addr;
709 dest = buf;
710 endpoints = SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
711 SUN4I_DMA_CFG_SRC_ADDR_MODE(SUN4I_NDMA_ADDR_MODE_IO) |
712 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM);
713 }
714
715 /*
716 * We will be using half done interrupts to make two periods
717 * out of a promise, so we need to program the DMA engine less
718 * often
719 */
720
721 /*
722 * The engine can interrupt on half-transfer, so we can use
723 * this feature to program the engine half as often as if we
724 * didn't use it (keep in mind the hardware doesn't support
725 * linked lists).
726 *
727 * Say you have a set of periods (| marks the start/end, I for
728 * interrupt, P for programming the engine to do a new
729 * transfer), the easy but slow way would be to do
730 *
731 * |---|---|---|---| (periods / promises)
732 * P I,P I,P I,P I
733 *
734 * Using half transfer interrupts you can do
735 *
736 * |-------|-------| (promises as configured on hw)
737 * |---|---|---|---| (periods)
738 * P I I,P I I
739 *
740 * Which requires half the engine programming for the same
741 * functionality.
742 */
743 nr_periods = DIV_ROUND_UP(len / period_len, 2);
744 for (i = 0; i < nr_periods; i++) {
745 /* Calculate the offset in the buffer and the length needed */
746 offset = i * period_len * 2;
747 plength = min((len - offset), (period_len * 2));
748 if (dir == DMA_MEM_TO_DEV)
749 src = buf + offset;
750 else
751 dest = buf + offset;
752
753 /* Make the promise */
754 promise = generate_ndma_promise(chan, src, dest,
755 plength, sconfig, dir);
756 if (!promise) {
757 /* TODO: should we free everything? */
758 return NULL;
759 }
760 promise->cfg |= endpoints;
761
762 /* Then add it to the contract */
763 list_add_tail(&promise->list, &contract->demands);
764 }
765
766 /* And add it to the vchan */
767 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
768}
769
770static struct dma_async_tx_descriptor *
771sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
772 unsigned int sg_len, enum dma_transfer_direction dir,
773 unsigned long flags, void *context)
774{
775 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
776 struct dma_slave_config *sconfig = &vchan->cfg;
777 struct sun4i_dma_promise *promise;
778 struct sun4i_dma_contract *contract;
779 u8 ram_type, io_mode, linear_mode;
780 struct scatterlist *sg;
781 dma_addr_t srcaddr, dstaddr;
782 u32 endpoints, para;
783 int i;
784
785 if (!sgl)
786 return NULL;
787
788 if (!is_slave_direction(dir)) {
789 dev_err(chan2dev(chan), "Invalid DMA direction\n");
790 return NULL;
791 }
792
793 contract = generate_dma_contract();
794 if (!contract)
795 return NULL;
796
797 if (vchan->is_dedicated) {
798 io_mode = SUN4I_DDMA_ADDR_MODE_IO;
799 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
800 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
801 } else {
802 io_mode = SUN4I_NDMA_ADDR_MODE_IO;
803 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
804 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
805 }
806
807 if (dir == DMA_MEM_TO_DEV)
808 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
809 SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
810 SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
811 SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
812 else
813 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
814 SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
815 SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
816 SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
817
818 for_each_sg(sgl, sg, sg_len, i) {
819 /* Figure out addresses */
820 if (dir == DMA_MEM_TO_DEV) {
821 srcaddr = sg_dma_address(sg);
822 dstaddr = sconfig->dst_addr;
823 } else {
824 srcaddr = sconfig->src_addr;
825 dstaddr = sg_dma_address(sg);
826 }
827
828 /*
829 * These are the magic DMA engine timings that keep SPI going.
830 * I haven't seen any interface on DMAEngine to configure
831 * timings, and so far they seem to work for everything we
832 * support, so I've kept them here. I don't know if other
833 * devices need different timings because, as usual, we only
834 * have the "para" bitfield meanings, but no comment on what
835 * the values should be when doing a certain operation :|
836 */
837 para = SUN4I_DDMA_MAGIC_SPI_PARAMETERS;
838
839 /* And make a suitable promise */
840 if (vchan->is_dedicated)
841 promise = generate_ddma_promise(chan, srcaddr, dstaddr,
842 sg_dma_len(sg),
843 sconfig);
844 else
845 promise = generate_ndma_promise(chan, srcaddr, dstaddr,
846 sg_dma_len(sg),
847 sconfig, dir);
848
849 if (!promise)
850 return NULL; /* TODO: should we free everything? */
851
852 promise->cfg |= endpoints;
853 promise->para = para;
854
855 /* Then add it to the contract */
856 list_add_tail(&promise->list, &contract->demands);
857 }
858
859 /*
860 * Once we've got all the promises ready, add the contract
861 * to the pending list on the vchan
862 */
863 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
864}
865
866static int sun4i_dma_terminate_all(struct dma_chan *chan)
867{
868 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
869 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
870 struct sun4i_dma_pchan *pchan = vchan->pchan;
871 LIST_HEAD(head);
872 unsigned long flags;
873
874 spin_lock_irqsave(&vchan->vc.lock, flags);
875 vchan_get_all_descriptors(&vchan->vc, &head);
876 spin_unlock_irqrestore(&vchan->vc.lock, flags);
877
878 /*
879 * Clearing the configuration register will halt the pchan. Interrupts
880 * may still trigger, so don't forget to disable them.
881 */
882 if (pchan) {
883 if (pchan->is_dedicated)
884 writel(0, pchan->base + SUN4I_DDMA_CFG_REG);
885 else
886 writel(0, pchan->base + SUN4I_NDMA_CFG_REG);
887 set_pchan_interrupt(priv, pchan, 0, 0);
888 release_pchan(priv, pchan);
889 }
890
891 spin_lock_irqsave(&vchan->vc.lock, flags);
892 vchan_dma_desc_free_list(&vchan->vc, &head);
893 /* Clear these so the vchan is usable again */
894 vchan->processing = NULL;
895 vchan->pchan = NULL;
896 spin_unlock_irqrestore(&vchan->vc.lock, flags);
897
898 return 0;
899}
900
901static int sun4i_dma_config(struct dma_chan *chan,
902 struct dma_slave_config *config)
903{
904 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
905
906 memcpy(&vchan->cfg, config, sizeof(*config));
907
908 return 0;
909}
910
911static struct dma_chan *sun4i_dma_of_xlate(struct of_phandle_args *dma_spec,
912 struct of_dma *ofdma)
913{
914 struct sun4i_dma_dev *priv = ofdma->of_dma_data;
915 struct sun4i_dma_vchan *vchan;
916 struct dma_chan *chan;
917 u8 is_dedicated = dma_spec->args[0];
918 u8 endpoint = dma_spec->args[1];
919
920 /* Check if type is Normal or Dedicated */
921 if (is_dedicated != 0 && is_dedicated != 1)
922 return NULL;
923
924 /* Make sure the endpoint looks sane */
925 if ((is_dedicated && endpoint >= SUN4I_DDMA_DRQ_TYPE_LIMIT) ||
926 (!is_dedicated && endpoint >= SUN4I_NDMA_DRQ_TYPE_LIMIT))
927 return NULL;
928
929 chan = dma_get_any_slave_channel(&priv->slave);
930 if (!chan)
931 return NULL;
932
933 /* Assign the endpoint to the vchan */
934 vchan = to_sun4i_dma_vchan(chan);
935 vchan->is_dedicated = is_dedicated;
936 vchan->endpoint = endpoint;
937
938 return chan;
939}
940
941static enum dma_status sun4i_dma_tx_status(struct dma_chan *chan,
942 dma_cookie_t cookie,
943 struct dma_tx_state *state)
944{
945 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
946 struct sun4i_dma_pchan *pchan = vchan->pchan;
947 struct sun4i_dma_contract *contract;
948 struct sun4i_dma_promise *promise;
949 struct virt_dma_desc *vd;
950 unsigned long flags;
951 enum dma_status ret;
952 size_t bytes = 0;
953
954 ret = dma_cookie_status(chan, cookie, state);
955 if (!state || (ret == DMA_COMPLETE))
956 return ret;
957
958 spin_lock_irqsave(&vchan->vc.lock, flags);
959 vd = vchan_find_desc(&vchan->vc, cookie);
960 if (!vd)
961 goto exit;
962 contract = to_sun4i_dma_contract(vd);
963
964 list_for_each_entry(promise, &contract->demands, list)
965 bytes += promise->len;
966
967 /*
968 * The hardware is configured to return the remaining byte
969 * quantity. If possible, replace the first listed element's
970 * full size with the actual remaining amount
971 */
972 promise = list_first_entry_or_null(&contract->demands,
973 struct sun4i_dma_promise, list);
974 if (promise && pchan) {
975 bytes -= promise->len;
976 if (pchan->is_dedicated)
977 bytes += readl(pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
978 else
979 bytes += readl(pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
980 }
981
982exit:
983
984 dma_set_residue(state, bytes);
985 spin_unlock_irqrestore(&vchan->vc.lock, flags);
986
987 return ret;
988}
989
990static void sun4i_dma_issue_pending(struct dma_chan *chan)
991{
992 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
993 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
994 unsigned long flags;
995
996 spin_lock_irqsave(&vchan->vc.lock, flags);
997
998 /*
999 * If there are pending transactions for this vchan, push one of
1000 * them into the engine to get the ball rolling.
1001 */
1002 if (vchan_issue_pending(&vchan->vc))
1003 __execute_vchan_pending(priv, vchan);
1004
1005 spin_unlock_irqrestore(&vchan->vc.lock, flags);
1006}
1007
1008static irqreturn_t sun4i_dma_interrupt(int irq, void *dev_id)
1009{
1010 struct sun4i_dma_dev *priv = dev_id;
1011 struct sun4i_dma_pchan *pchans = priv->pchans, *pchan;
1012 struct sun4i_dma_vchan *vchan;
1013 struct sun4i_dma_contract *contract;
1014 struct sun4i_dma_promise *promise;
1015 unsigned long pendirq, irqs, disableirqs;
1016 int bit, i, free_room, allow_mitigation = 1;
1017
1018 pendirq = readl_relaxed(priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1019
1020handle_pending:
1021
1022 disableirqs = 0;
1023 free_room = 0;
1024
1025 for_each_set_bit(bit, &pendirq, 32) {
1026 pchan = &pchans[bit >> 1];
1027 vchan = pchan->vchan;
1028 if (!vchan) /* a terminated channel may still interrupt */
1029 continue;
1030 contract = vchan->contract;
1031
1032 /*
1033 * Disable the IRQ and free the pchan if it's an end
1034 * interrupt (odd bit)
1035 */
1036 if (bit & 1) {
1037 spin_lock(&vchan->vc.lock);
1038
1039 /*
1040 * Move the promise into the completed list now that
1041 * we're done with it
1042 */
1043 list_del(&vchan->processing->list);
1044 list_add_tail(&vchan->processing->list,
1045 &contract->completed_demands);
1046
1047 /*
1048 * Cyclic DMA transfers are special:
1049 * - There's always something we can dispatch
1050 * - We need to run the callback
1051 * - Latency is very important, as this is used by audio
1052 * We therefore just cycle through the list and dispatch
1053 * whatever we have here, reusing the pchan. There's
1054 * no need to run the thread after this.
1055 *
1056 * For non-cyclic transfers we need to look around,
1057 * so we can program some more work, or notify the
1058 * client that their transfers have been completed.
1059 */
1060 if (contract->is_cyclic) {
1061 promise = get_next_cyclic_promise(contract);
1062 vchan->processing = promise;
1063 configure_pchan(pchan, promise);
1064 vchan_cyclic_callback(&contract->vd);
1065 } else {
1066 vchan->processing = NULL;
1067 vchan->pchan = NULL;
1068
1069 free_room = 1;
1070 disableirqs |= BIT(bit);
1071 release_pchan(priv, pchan);
1072 }
1073
1074 spin_unlock(&vchan->vc.lock);
1075 } else {
1076 /* Half done interrupt */
1077 if (contract->is_cyclic)
1078 vchan_cyclic_callback(&contract->vd);
1079 else
1080 disableirqs |= BIT(bit);
1081 }
1082 }
1083
1084 /* Disable the IRQs for events we handled */
1085 spin_lock(&priv->lock);
1086 irqs = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1087 writel_relaxed(irqs & ~disableirqs,
1088 priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1089 spin_unlock(&priv->lock);
1090
1091 /* Writing 1 to the pending field will clear the pending interrupt */
1092 writel_relaxed(pendirq, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1093
1094 /*
1095 * If a pchan was freed, we may be able to schedule something else,
1096 * so have a look around
1097 */
1098 if (free_room) {
1099 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1100 vchan = &priv->vchans[i];
1101 spin_lock(&vchan->vc.lock);
1102 __execute_vchan_pending(priv, vchan);
1103 spin_unlock(&vchan->vc.lock);
1104 }
1105 }
1106
1107 /*
1108 * Handle newer interrupts if some showed up, but only do it once
1109 * to avoid a too long a loop
1110 */
1111 if (allow_mitigation) {
1112 pendirq = readl_relaxed(priv->base +
1113 SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1114 if (pendirq) {
1115 allow_mitigation = 0;
1116 goto handle_pending;
1117 }
1118 }
1119
1120 return IRQ_HANDLED;
1121}
1122
1123static int sun4i_dma_probe(struct platform_device *pdev)
1124{
1125 struct sun4i_dma_dev *priv;
1126 struct resource *res;
1127 int i, j, ret;
1128
1129 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1130 if (!priv)
1131 return -ENOMEM;
1132
1133 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1134 priv->base = devm_ioremap_resource(&pdev->dev, res);
1135 if (IS_ERR(priv->base))
1136 return PTR_ERR(priv->base);
1137
1138 priv->irq = platform_get_irq(pdev, 0);
1139 if (priv->irq < 0) {
1140 dev_err(&pdev->dev, "Cannot claim IRQ\n");
1141 return priv->irq;
1142 }
1143
1144 priv->clk = devm_clk_get(&pdev->dev, NULL);
1145 if (IS_ERR(priv->clk)) {
1146 dev_err(&pdev->dev, "No clock specified\n");
1147 return PTR_ERR(priv->clk);
1148 }
1149
1150 platform_set_drvdata(pdev, priv);
1151 spin_lock_init(&priv->lock);
1152
1153 dma_cap_zero(priv->slave.cap_mask);
1154 dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask);
1155 dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask);
1156 dma_cap_set(DMA_CYCLIC, priv->slave.cap_mask);
1157 dma_cap_set(DMA_SLAVE, priv->slave.cap_mask);
1158
1159 INIT_LIST_HEAD(&priv->slave.channels);
1160 priv->slave.device_free_chan_resources = sun4i_dma_free_chan_resources;
1161 priv->slave.device_tx_status = sun4i_dma_tx_status;
1162 priv->slave.device_issue_pending = sun4i_dma_issue_pending;
1163 priv->slave.device_prep_slave_sg = sun4i_dma_prep_slave_sg;
1164 priv->slave.device_prep_dma_memcpy = sun4i_dma_prep_dma_memcpy;
1165 priv->slave.device_prep_dma_cyclic = sun4i_dma_prep_dma_cyclic;
1166 priv->slave.device_config = sun4i_dma_config;
1167 priv->slave.device_terminate_all = sun4i_dma_terminate_all;
1168 priv->slave.copy_align = 2;
1169 priv->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1170 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1171 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1172 priv->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1173 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1174 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1175 priv->slave.directions = BIT(DMA_DEV_TO_MEM) |
1176 BIT(DMA_MEM_TO_DEV);
1177 priv->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1178
1179 priv->slave.dev = &pdev->dev;
1180
1181 priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS,
1182 sizeof(struct sun4i_dma_pchan), GFP_KERNEL);
1183 priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS,
1184 sizeof(struct sun4i_dma_vchan), GFP_KERNEL);
1185 if (!priv->vchans || !priv->pchans)
1186 return -ENOMEM;
1187
1188 /*
1189 * [0..SUN4I_NDMA_NR_MAX_CHANNELS) are normal pchans, and
1190 * [SUN4I_NDMA_NR_MAX_CHANNELS..SUN4I_DMA_NR_MAX_CHANNELS) are
1191 * dedicated ones
1192 */
1193 for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++)
1194 priv->pchans[i].base = priv->base +
1195 SUN4I_NDMA_CHANNEL_REG_BASE(i);
1196
1197 for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) {
1198 priv->pchans[i].base = priv->base +
1199 SUN4I_DDMA_CHANNEL_REG_BASE(j);
1200 priv->pchans[i].is_dedicated = 1;
1201 }
1202
1203 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1204 struct sun4i_dma_vchan *vchan = &priv->vchans[i];
1205
1206 spin_lock_init(&vchan->vc.lock);
1207 vchan->vc.desc_free = sun4i_dma_free_contract;
1208 vchan_init(&vchan->vc, &priv->slave);
1209 }
1210
1211 ret = clk_prepare_enable(priv->clk);
1212 if (ret) {
1213 dev_err(&pdev->dev, "Couldn't enable the clock\n");
1214 return ret;
1215 }
1216
1217 /*
1218 * Make sure the IRQs are all disabled and accounted for. The bootloader
1219 * likes to leave these dirty
1220 */
1221 writel(0, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1222 writel(0xFFFFFFFF, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1223
1224 ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt,
1225 0, dev_name(&pdev->dev), priv);
1226 if (ret) {
1227 dev_err(&pdev->dev, "Cannot request IRQ\n");
1228 goto err_clk_disable;
1229 }
1230
1231 ret = dma_async_device_register(&priv->slave);
1232 if (ret) {
1233 dev_warn(&pdev->dev, "Failed to register DMA engine device\n");
1234 goto err_clk_disable;
1235 }
1236
1237 ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate,
1238 priv);
1239 if (ret) {
1240 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1241 goto err_dma_unregister;
1242 }
1243
1244 dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n");
1245
1246 return 0;
1247
1248err_dma_unregister:
1249 dma_async_device_unregister(&priv->slave);
1250err_clk_disable:
1251 clk_disable_unprepare(priv->clk);
1252 return ret;
1253}
1254
1255static int sun4i_dma_remove(struct platform_device *pdev)
1256{
1257 struct sun4i_dma_dev *priv = platform_get_drvdata(pdev);
1258
1259 /* Disable IRQ so no more work is scheduled */
1260 disable_irq(priv->irq);
1261
1262 of_dma_controller_free(pdev->dev.of_node);
1263 dma_async_device_unregister(&priv->slave);
1264
1265 clk_disable_unprepare(priv->clk);
1266
1267 return 0;
1268}
1269
1270static const struct of_device_id sun4i_dma_match[] = {
1271 { .compatible = "allwinner,sun4i-a10-dma" },
1272 { /* sentinel */ },
1273};
1274
1275static struct platform_driver sun4i_dma_driver = {
1276 .probe = sun4i_dma_probe,
1277 .remove = sun4i_dma_remove,
1278 .driver = {
1279 .name = "sun4i-dma",
1280 .of_match_table = sun4i_dma_match,
1281 },
1282};
1283
1284module_platform_driver(sun4i_dma_driver);
1285
1286MODULE_DESCRIPTION("Allwinner A10 Dedicated DMA Controller Driver");
1287MODULE_AUTHOR("Emilio López <emilio@elopez.com.ar>");
1288MODULE_LICENSE("GPL");
diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c
index 842ff97c2cfb..73e0be6e2100 100644
--- a/drivers/dma/sun6i-dma.c
+++ b/drivers/dma/sun6i-dma.c
@@ -969,7 +969,7 @@ static int sun6i_dma_probe(struct platform_device *pdev)
969 sdc->slave.device_issue_pending = sun6i_dma_issue_pending; 969 sdc->slave.device_issue_pending = sun6i_dma_issue_pending;
970 sdc->slave.device_prep_slave_sg = sun6i_dma_prep_slave_sg; 970 sdc->slave.device_prep_slave_sg = sun6i_dma_prep_slave_sg;
971 sdc->slave.device_prep_dma_memcpy = sun6i_dma_prep_dma_memcpy; 971 sdc->slave.device_prep_dma_memcpy = sun6i_dma_prep_dma_memcpy;
972 sdc->slave.copy_align = 4; 972 sdc->slave.copy_align = DMAENGINE_ALIGN_4_BYTES;
973 sdc->slave.device_config = sun6i_dma_config; 973 sdc->slave.device_config = sun6i_dma_config;
974 sdc->slave.device_pause = sun6i_dma_pause; 974 sdc->slave.device_pause = sun6i_dma_pause;
975 sdc->slave.device_resume = sun6i_dma_resume; 975 sdc->slave.device_resume = sun6i_dma_resume;
diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c
index eaf585e8286b..c8f79dcaaee8 100644
--- a/drivers/dma/tegra20-apb-dma.c
+++ b/drivers/dma/tegra20-apb-dma.c
@@ -155,7 +155,6 @@ struct tegra_dma_sg_req {
155 int req_len; 155 int req_len;
156 bool configured; 156 bool configured;
157 bool last_sg; 157 bool last_sg;
158 bool half_done;
159 struct list_head node; 158 struct list_head node;
160 struct tegra_dma_desc *dma_desc; 159 struct tegra_dma_desc *dma_desc;
161}; 160};
@@ -188,7 +187,7 @@ struct tegra_dma_channel {
188 bool config_init; 187 bool config_init;
189 int id; 188 int id;
190 int irq; 189 int irq;
191 unsigned long chan_base_offset; 190 void __iomem *chan_addr;
192 spinlock_t lock; 191 spinlock_t lock;
193 bool busy; 192 bool busy;
194 struct tegra_dma *tdma; 193 struct tegra_dma *tdma;
@@ -203,8 +202,6 @@ struct tegra_dma_channel {
203 /* ISR handler and tasklet for bottom half of isr handling */ 202 /* ISR handler and tasklet for bottom half of isr handling */
204 dma_isr_handler isr_handler; 203 dma_isr_handler isr_handler;
205 struct tasklet_struct tasklet; 204 struct tasklet_struct tasklet;
206 dma_async_tx_callback callback;
207 void *callback_param;
208 205
209 /* Channel-slave specific configuration */ 206 /* Channel-slave specific configuration */
210 unsigned int slave_id; 207 unsigned int slave_id;
@@ -222,6 +219,13 @@ struct tegra_dma {
222 void __iomem *base_addr; 219 void __iomem *base_addr;
223 const struct tegra_dma_chip_data *chip_data; 220 const struct tegra_dma_chip_data *chip_data;
224 221
222 /*
223 * Counter for managing global pausing of the DMA controller.
224 * Only applicable for devices that don't support individual
225 * channel pausing.
226 */
227 u32 global_pause_count;
228
225 /* Some register need to be cache before suspend */ 229 /* Some register need to be cache before suspend */
226 u32 reg_gen; 230 u32 reg_gen;
227 231
@@ -242,12 +246,12 @@ static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg)
242static inline void tdc_write(struct tegra_dma_channel *tdc, 246static inline void tdc_write(struct tegra_dma_channel *tdc,
243 u32 reg, u32 val) 247 u32 reg, u32 val)
244{ 248{
245 writel(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg); 249 writel(val, tdc->chan_addr + reg);
246} 250}
247 251
248static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg) 252static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg)
249{ 253{
250 return readl(tdc->tdma->base_addr + tdc->chan_base_offset + reg); 254 return readl(tdc->chan_addr + reg);
251} 255}
252 256
253static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc) 257static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc)
@@ -361,16 +365,32 @@ static void tegra_dma_global_pause(struct tegra_dma_channel *tdc,
361 struct tegra_dma *tdma = tdc->tdma; 365 struct tegra_dma *tdma = tdc->tdma;
362 366
363 spin_lock(&tdma->global_lock); 367 spin_lock(&tdma->global_lock);
364 tdma_write(tdma, TEGRA_APBDMA_GENERAL, 0); 368
365 if (wait_for_burst_complete) 369 if (tdc->tdma->global_pause_count == 0) {
366 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); 370 tdma_write(tdma, TEGRA_APBDMA_GENERAL, 0);
371 if (wait_for_burst_complete)
372 udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME);
373 }
374
375 tdc->tdma->global_pause_count++;
376
377 spin_unlock(&tdma->global_lock);
367} 378}
368 379
369static void tegra_dma_global_resume(struct tegra_dma_channel *tdc) 380static void tegra_dma_global_resume(struct tegra_dma_channel *tdc)
370{ 381{
371 struct tegra_dma *tdma = tdc->tdma; 382 struct tegra_dma *tdma = tdc->tdma;
372 383
373 tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); 384 spin_lock(&tdma->global_lock);
385
386 if (WARN_ON(tdc->tdma->global_pause_count == 0))
387 goto out;
388
389 if (--tdc->tdma->global_pause_count == 0)
390 tdma_write(tdma, TEGRA_APBDMA_GENERAL,
391 TEGRA_APBDMA_GENERAL_ENABLE);
392
393out:
374 spin_unlock(&tdma->global_lock); 394 spin_unlock(&tdma->global_lock);
375} 395}
376 396
@@ -601,7 +621,6 @@ static void handle_once_dma_done(struct tegra_dma_channel *tdc,
601 return; 621 return;
602 622
603 tdc_start_head_req(tdc); 623 tdc_start_head_req(tdc);
604 return;
605} 624}
606 625
607static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, 626static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc,
@@ -628,7 +647,6 @@ static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc,
628 if (!st) 647 if (!st)
629 dma_desc->dma_status = DMA_ERROR; 648 dma_desc->dma_status = DMA_ERROR;
630 } 649 }
631 return;
632} 650}
633 651
634static void tegra_dma_tasklet(unsigned long data) 652static void tegra_dma_tasklet(unsigned long data)
@@ -720,7 +738,6 @@ static void tegra_dma_issue_pending(struct dma_chan *dc)
720 } 738 }
721end: 739end:
722 spin_unlock_irqrestore(&tdc->lock, flags); 740 spin_unlock_irqrestore(&tdc->lock, flags);
723 return;
724} 741}
725 742
726static int tegra_dma_terminate_all(struct dma_chan *dc) 743static int tegra_dma_terminate_all(struct dma_chan *dc)
@@ -932,7 +949,6 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg(
932 struct tegra_dma_sg_req *sg_req = NULL; 949 struct tegra_dma_sg_req *sg_req = NULL;
933 u32 burst_size; 950 u32 burst_size;
934 enum dma_slave_buswidth slave_bw; 951 enum dma_slave_buswidth slave_bw;
935 int ret;
936 952
937 if (!tdc->config_init) { 953 if (!tdc->config_init) {
938 dev_err(tdc2dev(tdc), "dma channel is not configured\n"); 954 dev_err(tdc2dev(tdc), "dma channel is not configured\n");
@@ -943,9 +959,8 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg(
943 return NULL; 959 return NULL;
944 } 960 }
945 961
946 ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, 962 if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,
947 &burst_size, &slave_bw); 963 &burst_size, &slave_bw) < 0)
948 if (ret < 0)
949 return NULL; 964 return NULL;
950 965
951 INIT_LIST_HEAD(&req_list); 966 INIT_LIST_HEAD(&req_list);
@@ -1048,7 +1063,6 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic(
1048 dma_addr_t mem = buf_addr; 1063 dma_addr_t mem = buf_addr;
1049 u32 burst_size; 1064 u32 burst_size;
1050 enum dma_slave_buswidth slave_bw; 1065 enum dma_slave_buswidth slave_bw;
1051 int ret;
1052 1066
1053 if (!buf_len || !period_len) { 1067 if (!buf_len || !period_len) {
1054 dev_err(tdc2dev(tdc), "Invalid buffer/period len\n"); 1068 dev_err(tdc2dev(tdc), "Invalid buffer/period len\n");
@@ -1087,12 +1101,10 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic(
1087 return NULL; 1101 return NULL;
1088 } 1102 }
1089 1103
1090 ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, 1104 if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr,
1091 &burst_size, &slave_bw); 1105 &burst_size, &slave_bw) < 0)
1092 if (ret < 0)
1093 return NULL; 1106 return NULL;
1094 1107
1095
1096 ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB; 1108 ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB;
1097 ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE << 1109 ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE <<
1098 TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT; 1110 TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT;
@@ -1136,7 +1148,6 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic(
1136 sg_req->ch_regs.apb_seq = apb_seq; 1148 sg_req->ch_regs.apb_seq = apb_seq;
1137 sg_req->ch_regs.ahb_seq = ahb_seq; 1149 sg_req->ch_regs.ahb_seq = ahb_seq;
1138 sg_req->configured = false; 1150 sg_req->configured = false;
1139 sg_req->half_done = false;
1140 sg_req->last_sg = false; 1151 sg_req->last_sg = false;
1141 sg_req->dma_desc = dma_desc; 1152 sg_req->dma_desc = dma_desc;
1142 sg_req->req_len = len; 1153 sg_req->req_len = len;
@@ -1377,8 +1388,9 @@ static int tegra_dma_probe(struct platform_device *pdev)
1377 for (i = 0; i < cdata->nr_channels; i++) { 1388 for (i = 0; i < cdata->nr_channels; i++) {
1378 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1389 struct tegra_dma_channel *tdc = &tdma->channels[i];
1379 1390
1380 tdc->chan_base_offset = TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET + 1391 tdc->chan_addr = tdma->base_addr +
1381 i * cdata->channel_reg_size; 1392 TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET +
1393 (i * cdata->channel_reg_size);
1382 1394
1383 res = platform_get_resource(pdev, IORESOURCE_IRQ, i); 1395 res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
1384 if (!res) { 1396 if (!res) {
@@ -1418,6 +1430,7 @@ static int tegra_dma_probe(struct platform_device *pdev)
1418 dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); 1430 dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
1419 dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask); 1431 dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);
1420 1432
1433 tdma->global_pause_count = 0;
1421 tdma->dma_dev.dev = &pdev->dev; 1434 tdma->dma_dev.dev = &pdev->dev;
1422 tdma->dma_dev.device_alloc_chan_resources = 1435 tdma->dma_dev.device_alloc_chan_resources =
1423 tegra_dma_alloc_chan_resources; 1436 tegra_dma_alloc_chan_resources;
diff --git a/drivers/dma/ti-dma-crossbar.c b/drivers/dma/ti-dma-crossbar.c
index 24f5ca2356bf..5cce8c9d0026 100644
--- a/drivers/dma/ti-dma-crossbar.c
+++ b/drivers/dma/ti-dma-crossbar.c
@@ -20,16 +20,19 @@
20#define TI_XBAR_OUTPUTS 127 20#define TI_XBAR_OUTPUTS 127
21#define TI_XBAR_INPUTS 256 21#define TI_XBAR_INPUTS 256
22 22
23static DEFINE_IDR(map_idr); 23#define TI_XBAR_EDMA_OFFSET 0
24#define TI_XBAR_SDMA_OFFSET 1
24 25
25struct ti_dma_xbar_data { 26struct ti_dma_xbar_data {
26 void __iomem *iomem; 27 void __iomem *iomem;
27 28
28 struct dma_router dmarouter; 29 struct dma_router dmarouter;
30 struct idr map_idr;
29 31
30 u16 safe_val; /* Value to rest the crossbar lines */ 32 u16 safe_val; /* Value to rest the crossbar lines */
31 u32 xbar_requests; /* number of DMA requests connected to XBAR */ 33 u32 xbar_requests; /* number of DMA requests connected to XBAR */
32 u32 dma_requests; /* number of DMA requests forwarded to DMA */ 34 u32 dma_requests; /* number of DMA requests forwarded to DMA */
35 u32 dma_offset;
33}; 36};
34 37
35struct ti_dma_xbar_map { 38struct ti_dma_xbar_map {
@@ -51,7 +54,7 @@ static void ti_dma_xbar_free(struct device *dev, void *route_data)
51 map->xbar_in, map->xbar_out); 54 map->xbar_in, map->xbar_out);
52 55
53 ti_dma_xbar_write(xbar->iomem, map->xbar_out, xbar->safe_val); 56 ti_dma_xbar_write(xbar->iomem, map->xbar_out, xbar->safe_val);
54 idr_remove(&map_idr, map->xbar_out); 57 idr_remove(&xbar->map_idr, map->xbar_out);
55 kfree(map); 58 kfree(map);
56} 59}
57 60
@@ -81,12 +84,11 @@ static void *ti_dma_xbar_route_allocate(struct of_phandle_args *dma_spec,
81 return ERR_PTR(-ENOMEM); 84 return ERR_PTR(-ENOMEM);
82 } 85 }
83 86
84 map->xbar_out = idr_alloc(&map_idr, NULL, 0, xbar->dma_requests, 87 map->xbar_out = idr_alloc(&xbar->map_idr, NULL, 0, xbar->dma_requests,
85 GFP_KERNEL); 88 GFP_KERNEL);
86 map->xbar_in = (u16)dma_spec->args[0]; 89 map->xbar_in = (u16)dma_spec->args[0];
87 90
88 /* The DMA request is 1 based in sDMA */ 91 dma_spec->args[0] = map->xbar_out + xbar->dma_offset;
89 dma_spec->args[0] = map->xbar_out + 1;
90 92
91 dev_dbg(&pdev->dev, "Mapping XBAR%u to DMA%d\n", 93 dev_dbg(&pdev->dev, "Mapping XBAR%u to DMA%d\n",
92 map->xbar_in, map->xbar_out); 94 map->xbar_in, map->xbar_out);
@@ -96,9 +98,22 @@ static void *ti_dma_xbar_route_allocate(struct of_phandle_args *dma_spec,
96 return map; 98 return map;
97} 99}
98 100
101static const struct of_device_id ti_dma_master_match[] = {
102 {
103 .compatible = "ti,omap4430-sdma",
104 .data = (void *)TI_XBAR_SDMA_OFFSET,
105 },
106 {
107 .compatible = "ti,edma3",
108 .data = (void *)TI_XBAR_EDMA_OFFSET,
109 },
110 {},
111};
112
99static int ti_dma_xbar_probe(struct platform_device *pdev) 113static int ti_dma_xbar_probe(struct platform_device *pdev)
100{ 114{
101 struct device_node *node = pdev->dev.of_node; 115 struct device_node *node = pdev->dev.of_node;
116 const struct of_device_id *match;
102 struct device_node *dma_node; 117 struct device_node *dma_node;
103 struct ti_dma_xbar_data *xbar; 118 struct ti_dma_xbar_data *xbar;
104 struct resource *res; 119 struct resource *res;
@@ -113,12 +128,20 @@ static int ti_dma_xbar_probe(struct platform_device *pdev)
113 if (!xbar) 128 if (!xbar)
114 return -ENOMEM; 129 return -ENOMEM;
115 130
131 idr_init(&xbar->map_idr);
132
116 dma_node = of_parse_phandle(node, "dma-masters", 0); 133 dma_node = of_parse_phandle(node, "dma-masters", 0);
117 if (!dma_node) { 134 if (!dma_node) {
118 dev_err(&pdev->dev, "Can't get DMA master node\n"); 135 dev_err(&pdev->dev, "Can't get DMA master node\n");
119 return -ENODEV; 136 return -ENODEV;
120 } 137 }
121 138
139 match = of_match_node(ti_dma_master_match, dma_node);
140 if (!match) {
141 dev_err(&pdev->dev, "DMA master is not supported\n");
142 return -EINVAL;
143 }
144
122 if (of_property_read_u32(dma_node, "dma-requests", 145 if (of_property_read_u32(dma_node, "dma-requests",
123 &xbar->dma_requests)) { 146 &xbar->dma_requests)) {
124 dev_info(&pdev->dev, 147 dev_info(&pdev->dev,
@@ -139,17 +162,15 @@ static int ti_dma_xbar_probe(struct platform_device *pdev)
139 xbar->safe_val = (u16)safe_val; 162 xbar->safe_val = (u16)safe_val;
140 163
141 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 164 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
142 if (!res)
143 return -ENODEV;
144
145 iomem = devm_ioremap_resource(&pdev->dev, res); 165 iomem = devm_ioremap_resource(&pdev->dev, res);
146 if (!iomem) 166 if (IS_ERR(iomem))
147 return -ENOMEM; 167 return PTR_ERR(iomem);
148 168
149 xbar->iomem = iomem; 169 xbar->iomem = iomem;
150 170
151 xbar->dmarouter.dev = &pdev->dev; 171 xbar->dmarouter.dev = &pdev->dev;
152 xbar->dmarouter.route_free = ti_dma_xbar_free; 172 xbar->dmarouter.route_free = ti_dma_xbar_free;
173 xbar->dma_offset = (u32)match->data;
153 174
154 platform_set_drvdata(pdev, xbar); 175 platform_set_drvdata(pdev, xbar);
155 176
diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c
index c4c3d93fdd1b..559cd4073698 100644
--- a/drivers/dma/timb_dma.c
+++ b/drivers/dma/timb_dma.c
@@ -10,10 +10,6 @@
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details. 12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
17 */ 13 */
18 14
19/* Supports: 15/* Supports:
diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c
index dff22ab01851..b23e8d52d126 100644
--- a/drivers/dma/xgene-dma.c
+++ b/drivers/dma/xgene-dma.c
@@ -21,6 +21,7 @@
21 * NOTE: PM support is currently not available. 21 * NOTE: PM support is currently not available.
22 */ 22 */
23 23
24#include <linux/acpi.h>
24#include <linux/clk.h> 25#include <linux/clk.h>
25#include <linux/delay.h> 26#include <linux/delay.h>
26#include <linux/dma-mapping.h> 27#include <linux/dma-mapping.h>
@@ -151,7 +152,6 @@
151#define XGENE_DMA_PQ_CHANNEL 1 152#define XGENE_DMA_PQ_CHANNEL 1
152#define XGENE_DMA_MAX_BYTE_CNT 0x4000 /* 16 KB */ 153#define XGENE_DMA_MAX_BYTE_CNT 0x4000 /* 16 KB */
153#define XGENE_DMA_MAX_64B_DESC_BYTE_CNT 0x14000 /* 80 KB */ 154#define XGENE_DMA_MAX_64B_DESC_BYTE_CNT 0x14000 /* 80 KB */
154#define XGENE_DMA_XOR_ALIGNMENT 6 /* 64 Bytes */
155#define XGENE_DMA_MAX_XOR_SRC 5 155#define XGENE_DMA_MAX_XOR_SRC 5
156#define XGENE_DMA_16K_BUFFER_LEN_CODE 0x0 156#define XGENE_DMA_16K_BUFFER_LEN_CODE 0x0
157#define XGENE_DMA_INVALID_LEN_CODE 0x7800000000000000ULL 157#define XGENE_DMA_INVALID_LEN_CODE 0x7800000000000000ULL
@@ -764,12 +764,17 @@ static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan)
764 struct xgene_dma_ring *ring = &chan->rx_ring; 764 struct xgene_dma_ring *ring = &chan->rx_ring;
765 struct xgene_dma_desc_sw *desc_sw, *_desc_sw; 765 struct xgene_dma_desc_sw *desc_sw, *_desc_sw;
766 struct xgene_dma_desc_hw *desc_hw; 766 struct xgene_dma_desc_hw *desc_hw;
767 struct list_head ld_completed;
767 u8 status; 768 u8 status;
768 769
770 INIT_LIST_HEAD(&ld_completed);
771
772 spin_lock_bh(&chan->lock);
773
769 /* Clean already completed and acked descriptors */ 774 /* Clean already completed and acked descriptors */
770 xgene_dma_clean_completed_descriptor(chan); 775 xgene_dma_clean_completed_descriptor(chan);
771 776
772 /* Run the callback for each descriptor, in order */ 777 /* Move all completed descriptors to ld completed queue, in order */
773 list_for_each_entry_safe(desc_sw, _desc_sw, &chan->ld_running, node) { 778 list_for_each_entry_safe(desc_sw, _desc_sw, &chan->ld_running, node) {
774 /* Get subsequent hw descriptor from DMA rx ring */ 779 /* Get subsequent hw descriptor from DMA rx ring */
775 desc_hw = &ring->desc_hw[ring->head]; 780 desc_hw = &ring->desc_hw[ring->head];
@@ -812,15 +817,17 @@ static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan)
812 /* Mark this hw descriptor as processed */ 817 /* Mark this hw descriptor as processed */
813 desc_hw->m0 = cpu_to_le64(XGENE_DMA_DESC_EMPTY_SIGNATURE); 818 desc_hw->m0 = cpu_to_le64(XGENE_DMA_DESC_EMPTY_SIGNATURE);
814 819
815 xgene_dma_run_tx_complete_actions(chan, desc_sw);
816
817 xgene_dma_clean_running_descriptor(chan, desc_sw);
818
819 /* 820 /*
820 * Decrement the pending transaction count 821 * Decrement the pending transaction count
821 * as we have processed one 822 * as we have processed one
822 */ 823 */
823 chan->pending--; 824 chan->pending--;
825
826 /*
827 * Delete this node from ld running queue and append it to
828 * ld completed queue for further processing
829 */
830 list_move_tail(&desc_sw->node, &ld_completed);
824 } 831 }
825 832
826 /* 833 /*
@@ -829,6 +836,14 @@ static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan)
829 * ahead and free the descriptors below. 836 * ahead and free the descriptors below.
830 */ 837 */
831 xgene_chan_xfer_ld_pending(chan); 838 xgene_chan_xfer_ld_pending(chan);
839
840 spin_unlock_bh(&chan->lock);
841
842 /* Run the callback for each descriptor, in order */
843 list_for_each_entry_safe(desc_sw, _desc_sw, &ld_completed, node) {
844 xgene_dma_run_tx_complete_actions(chan, desc_sw);
845 xgene_dma_clean_running_descriptor(chan, desc_sw);
846 }
832} 847}
833 848
834static int xgene_dma_alloc_chan_resources(struct dma_chan *dchan) 849static int xgene_dma_alloc_chan_resources(struct dma_chan *dchan)
@@ -877,11 +892,11 @@ static void xgene_dma_free_chan_resources(struct dma_chan *dchan)
877 if (!chan->desc_pool) 892 if (!chan->desc_pool)
878 return; 893 return;
879 894
880 spin_lock_bh(&chan->lock);
881
882 /* Process all running descriptor */ 895 /* Process all running descriptor */
883 xgene_dma_cleanup_descriptors(chan); 896 xgene_dma_cleanup_descriptors(chan);
884 897
898 spin_lock_bh(&chan->lock);
899
885 /* Clean all link descriptor queues */ 900 /* Clean all link descriptor queues */
886 xgene_dma_free_desc_list(chan, &chan->ld_pending); 901 xgene_dma_free_desc_list(chan, &chan->ld_pending);
887 xgene_dma_free_desc_list(chan, &chan->ld_running); 902 xgene_dma_free_desc_list(chan, &chan->ld_running);
@@ -1201,15 +1216,11 @@ static void xgene_dma_tasklet_cb(unsigned long data)
1201{ 1216{
1202 struct xgene_dma_chan *chan = (struct xgene_dma_chan *)data; 1217 struct xgene_dma_chan *chan = (struct xgene_dma_chan *)data;
1203 1218
1204 spin_lock_bh(&chan->lock);
1205
1206 /* Run all cleanup for descriptors which have been completed */ 1219 /* Run all cleanup for descriptors which have been completed */
1207 xgene_dma_cleanup_descriptors(chan); 1220 xgene_dma_cleanup_descriptors(chan);
1208 1221
1209 /* Re-enable DMA channel IRQ */ 1222 /* Re-enable DMA channel IRQ */
1210 enable_irq(chan->rx_irq); 1223 enable_irq(chan->rx_irq);
1211
1212 spin_unlock_bh(&chan->lock);
1213} 1224}
1214 1225
1215static irqreturn_t xgene_dma_chan_ring_isr(int irq, void *id) 1226static irqreturn_t xgene_dma_chan_ring_isr(int irq, void *id)
@@ -1741,13 +1752,13 @@ static void xgene_dma_set_caps(struct xgene_dma_chan *chan,
1741 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1752 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1742 dma_dev->device_prep_dma_xor = xgene_dma_prep_xor; 1753 dma_dev->device_prep_dma_xor = xgene_dma_prep_xor;
1743 dma_dev->max_xor = XGENE_DMA_MAX_XOR_SRC; 1754 dma_dev->max_xor = XGENE_DMA_MAX_XOR_SRC;
1744 dma_dev->xor_align = XGENE_DMA_XOR_ALIGNMENT; 1755 dma_dev->xor_align = DMAENGINE_ALIGN_64_BYTES;
1745 } 1756 }
1746 1757
1747 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 1758 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1748 dma_dev->device_prep_dma_pq = xgene_dma_prep_pq; 1759 dma_dev->device_prep_dma_pq = xgene_dma_prep_pq;
1749 dma_dev->max_pq = XGENE_DMA_MAX_XOR_SRC; 1760 dma_dev->max_pq = XGENE_DMA_MAX_XOR_SRC;
1750 dma_dev->pq_align = XGENE_DMA_XOR_ALIGNMENT; 1761 dma_dev->pq_align = DMAENGINE_ALIGN_64_BYTES;
1751 } 1762 }
1752} 1763}
1753 1764
@@ -1944,16 +1955,18 @@ static int xgene_dma_probe(struct platform_device *pdev)
1944 return ret; 1955 return ret;
1945 1956
1946 pdma->clk = devm_clk_get(&pdev->dev, NULL); 1957 pdma->clk = devm_clk_get(&pdev->dev, NULL);
1947 if (IS_ERR(pdma->clk)) { 1958 if (IS_ERR(pdma->clk) && !ACPI_COMPANION(&pdev->dev)) {
1948 dev_err(&pdev->dev, "Failed to get clk\n"); 1959 dev_err(&pdev->dev, "Failed to get clk\n");
1949 return PTR_ERR(pdma->clk); 1960 return PTR_ERR(pdma->clk);
1950 } 1961 }
1951 1962
1952 /* Enable clk before accessing registers */ 1963 /* Enable clk before accessing registers */
1953 ret = clk_prepare_enable(pdma->clk); 1964 if (!IS_ERR(pdma->clk)) {
1954 if (ret) { 1965 ret = clk_prepare_enable(pdma->clk);
1955 dev_err(&pdev->dev, "Failed to enable clk %d\n", ret); 1966 if (ret) {
1956 return ret; 1967 dev_err(&pdev->dev, "Failed to enable clk %d\n", ret);
1968 return ret;
1969 }
1957 } 1970 }
1958 1971
1959 /* Remove DMA RAM out of shutdown */ 1972 /* Remove DMA RAM out of shutdown */
@@ -1998,7 +2011,8 @@ err_request_irq:
1998 2011
1999err_dma_mask: 2012err_dma_mask:
2000err_clk_enable: 2013err_clk_enable:
2001 clk_disable_unprepare(pdma->clk); 2014 if (!IS_ERR(pdma->clk))
2015 clk_disable_unprepare(pdma->clk);
2002 2016
2003 return ret; 2017 return ret;
2004} 2018}
@@ -2022,11 +2036,20 @@ static int xgene_dma_remove(struct platform_device *pdev)
2022 xgene_dma_delete_chan_rings(chan); 2036 xgene_dma_delete_chan_rings(chan);
2023 } 2037 }
2024 2038
2025 clk_disable_unprepare(pdma->clk); 2039 if (!IS_ERR(pdma->clk))
2040 clk_disable_unprepare(pdma->clk);
2026 2041
2027 return 0; 2042 return 0;
2028} 2043}
2029 2044
2045#ifdef CONFIG_ACPI
2046static const struct acpi_device_id xgene_dma_acpi_match_ptr[] = {
2047 {"APMC0D43", 0},
2048 {},
2049};
2050MODULE_DEVICE_TABLE(acpi, xgene_dma_acpi_match_ptr);
2051#endif
2052
2030static const struct of_device_id xgene_dma_of_match_ptr[] = { 2053static const struct of_device_id xgene_dma_of_match_ptr[] = {
2031 {.compatible = "apm,xgene-storm-dma",}, 2054 {.compatible = "apm,xgene-storm-dma",},
2032 {}, 2055 {},
@@ -2039,6 +2062,7 @@ static struct platform_driver xgene_dma_driver = {
2039 .driver = { 2062 .driver = {
2040 .name = "X-Gene-DMA", 2063 .name = "X-Gene-DMA",
2041 .of_match_table = xgene_dma_of_match_ptr, 2064 .of_match_table = xgene_dma_of_match_ptr,
2065 .acpi_match_table = ACPI_PTR(xgene_dma_acpi_match_ptr),
2042 }, 2066 },
2043}; 2067};
2044 2068
diff --git a/drivers/dma/zx296702_dma.c b/drivers/dma/zx296702_dma.c
new file mode 100644
index 000000000000..39915a6b7986
--- /dev/null
+++ b/drivers/dma/zx296702_dma.c
@@ -0,0 +1,951 @@
1/*
2 * Copyright 2015 Linaro.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8#include <linux/sched.h>
9#include <linux/device.h>
10#include <linux/dmaengine.h>
11#include <linux/dma-mapping.h>
12#include <linux/dmapool.h>
13#include <linux/init.h>
14#include <linux/interrupt.h>
15#include <linux/kernel.h>
16#include <linux/module.h>
17#include <linux/platform_device.h>
18#include <linux/slab.h>
19#include <linux/spinlock.h>
20#include <linux/of_device.h>
21#include <linux/of.h>
22#include <linux/clk.h>
23#include <linux/of_dma.h>
24
25#include "virt-dma.h"
26
27#define DRIVER_NAME "zx-dma"
28#define DMA_ALIGN 4
29#define DMA_MAX_SIZE (0x10000 - PAGE_SIZE)
30#define LLI_BLOCK_SIZE (4 * PAGE_SIZE)
31
32#define REG_ZX_SRC_ADDR 0x00
33#define REG_ZX_DST_ADDR 0x04
34#define REG_ZX_TX_X_COUNT 0x08
35#define REG_ZX_TX_ZY_COUNT 0x0c
36#define REG_ZX_SRC_ZY_STEP 0x10
37#define REG_ZX_DST_ZY_STEP 0x14
38#define REG_ZX_LLI_ADDR 0x1c
39#define REG_ZX_CTRL 0x20
40#define REG_ZX_TC_IRQ 0x800
41#define REG_ZX_SRC_ERR_IRQ 0x804
42#define REG_ZX_DST_ERR_IRQ 0x808
43#define REG_ZX_CFG_ERR_IRQ 0x80c
44#define REG_ZX_TC_IRQ_RAW 0x810
45#define REG_ZX_SRC_ERR_IRQ_RAW 0x814
46#define REG_ZX_DST_ERR_IRQ_RAW 0x818
47#define REG_ZX_CFG_ERR_IRQ_RAW 0x81c
48#define REG_ZX_STATUS 0x820
49#define REG_ZX_DMA_GRP_PRIO 0x824
50#define REG_ZX_DMA_ARB 0x828
51
52#define ZX_FORCE_CLOSE BIT(31)
53#define ZX_DST_BURST_WIDTH(x) (((x) & 0x7) << 13)
54#define ZX_MAX_BURST_LEN 16
55#define ZX_SRC_BURST_LEN(x) (((x) & 0xf) << 9)
56#define ZX_SRC_BURST_WIDTH(x) (((x) & 0x7) << 6)
57#define ZX_IRQ_ENABLE_ALL (3 << 4)
58#define ZX_DST_FIFO_MODE BIT(3)
59#define ZX_SRC_FIFO_MODE BIT(2)
60#define ZX_SOFT_REQ BIT(1)
61#define ZX_CH_ENABLE BIT(0)
62
63#define ZX_DMA_BUSWIDTHS \
64 (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
65 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
66 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
67 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
68 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
69
70enum zx_dma_burst_width {
71 ZX_DMA_WIDTH_8BIT = 0,
72 ZX_DMA_WIDTH_16BIT = 1,
73 ZX_DMA_WIDTH_32BIT = 2,
74 ZX_DMA_WIDTH_64BIT = 3,
75};
76
77struct zx_desc_hw {
78 u32 saddr;
79 u32 daddr;
80 u32 src_x;
81 u32 src_zy;
82 u32 src_zy_step;
83 u32 dst_zy_step;
84 u32 reserved1;
85 u32 lli;
86 u32 ctr;
87 u32 reserved[7]; /* pack as hardware registers region size */
88} __aligned(32);
89
90struct zx_dma_desc_sw {
91 struct virt_dma_desc vd;
92 dma_addr_t desc_hw_lli;
93 size_t desc_num;
94 size_t size;
95 struct zx_desc_hw *desc_hw;
96};
97
98struct zx_dma_phy;
99
100struct zx_dma_chan {
101 struct dma_slave_config slave_cfg;
102 int id; /* Request phy chan id */
103 u32 ccfg;
104 u32 cyclic;
105 struct virt_dma_chan vc;
106 struct zx_dma_phy *phy;
107 struct list_head node;
108 dma_addr_t dev_addr;
109 enum dma_status status;
110};
111
112struct zx_dma_phy {
113 u32 idx;
114 void __iomem *base;
115 struct zx_dma_chan *vchan;
116 struct zx_dma_desc_sw *ds_run;
117 struct zx_dma_desc_sw *ds_done;
118};
119
120struct zx_dma_dev {
121 struct dma_device slave;
122 void __iomem *base;
123 spinlock_t lock; /* lock for ch and phy */
124 struct list_head chan_pending;
125 struct zx_dma_phy *phy;
126 struct zx_dma_chan *chans;
127 struct clk *clk;
128 struct dma_pool *pool;
129 u32 dma_channels;
130 u32 dma_requests;
131 int irq;
132};
133
134#define to_zx_dma(dmadev) container_of(dmadev, struct zx_dma_dev, slave)
135
136static struct zx_dma_chan *to_zx_chan(struct dma_chan *chan)
137{
138 return container_of(chan, struct zx_dma_chan, vc.chan);
139}
140
141static void zx_dma_terminate_chan(struct zx_dma_phy *phy, struct zx_dma_dev *d)
142{
143 u32 val = 0;
144
145 val = readl_relaxed(phy->base + REG_ZX_CTRL);
146 val &= ~ZX_CH_ENABLE;
147 val |= ZX_FORCE_CLOSE;
148 writel_relaxed(val, phy->base + REG_ZX_CTRL);
149
150 val = 0x1 << phy->idx;
151 writel_relaxed(val, d->base + REG_ZX_TC_IRQ_RAW);
152 writel_relaxed(val, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
153 writel_relaxed(val, d->base + REG_ZX_DST_ERR_IRQ_RAW);
154 writel_relaxed(val, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
155}
156
157static void zx_dma_set_desc(struct zx_dma_phy *phy, struct zx_desc_hw *hw)
158{
159 writel_relaxed(hw->saddr, phy->base + REG_ZX_SRC_ADDR);
160 writel_relaxed(hw->daddr, phy->base + REG_ZX_DST_ADDR);
161 writel_relaxed(hw->src_x, phy->base + REG_ZX_TX_X_COUNT);
162 writel_relaxed(0, phy->base + REG_ZX_TX_ZY_COUNT);
163 writel_relaxed(0, phy->base + REG_ZX_SRC_ZY_STEP);
164 writel_relaxed(0, phy->base + REG_ZX_DST_ZY_STEP);
165 writel_relaxed(hw->lli, phy->base + REG_ZX_LLI_ADDR);
166 writel_relaxed(hw->ctr, phy->base + REG_ZX_CTRL);
167}
168
169static u32 zx_dma_get_curr_lli(struct zx_dma_phy *phy)
170{
171 return readl_relaxed(phy->base + REG_ZX_LLI_ADDR);
172}
173
174static u32 zx_dma_get_chan_stat(struct zx_dma_dev *d)
175{
176 return readl_relaxed(d->base + REG_ZX_STATUS);
177}
178
179static void zx_dma_init_state(struct zx_dma_dev *d)
180{
181 /* set same priority */
182 writel_relaxed(0x0, d->base + REG_ZX_DMA_ARB);
183 /* clear all irq */
184 writel_relaxed(0xffffffff, d->base + REG_ZX_TC_IRQ_RAW);
185 writel_relaxed(0xffffffff, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
186 writel_relaxed(0xffffffff, d->base + REG_ZX_DST_ERR_IRQ_RAW);
187 writel_relaxed(0xffffffff, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
188}
189
190static int zx_dma_start_txd(struct zx_dma_chan *c)
191{
192 struct zx_dma_dev *d = to_zx_dma(c->vc.chan.device);
193 struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
194
195 if (!c->phy)
196 return -EAGAIN;
197
198 if (BIT(c->phy->idx) & zx_dma_get_chan_stat(d))
199 return -EAGAIN;
200
201 if (vd) {
202 struct zx_dma_desc_sw *ds =
203 container_of(vd, struct zx_dma_desc_sw, vd);
204 /*
205 * fetch and remove request from vc->desc_issued
206 * so vc->desc_issued only contains desc pending
207 */
208 list_del(&ds->vd.node);
209 c->phy->ds_run = ds;
210 c->phy->ds_done = NULL;
211 /* start dma */
212 zx_dma_set_desc(c->phy, ds->desc_hw);
213 return 0;
214 }
215 c->phy->ds_done = NULL;
216 c->phy->ds_run = NULL;
217 return -EAGAIN;
218}
219
220static void zx_dma_task(struct zx_dma_dev *d)
221{
222 struct zx_dma_phy *p;
223 struct zx_dma_chan *c, *cn;
224 unsigned pch, pch_alloc = 0;
225 unsigned long flags;
226
227 /* check new dma request of running channel in vc->desc_issued */
228 list_for_each_entry_safe(c, cn, &d->slave.channels,
229 vc.chan.device_node) {
230 spin_lock_irqsave(&c->vc.lock, flags);
231 p = c->phy;
232 if (p && p->ds_done && zx_dma_start_txd(c)) {
233 /* No current txd associated with this channel */
234 dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx);
235 /* Mark this channel free */
236 c->phy = NULL;
237 p->vchan = NULL;
238 }
239 spin_unlock_irqrestore(&c->vc.lock, flags);
240 }
241
242 /* check new channel request in d->chan_pending */
243 spin_lock_irqsave(&d->lock, flags);
244 while (!list_empty(&d->chan_pending)) {
245 c = list_first_entry(&d->chan_pending,
246 struct zx_dma_chan, node);
247 p = &d->phy[c->id];
248 if (!p->vchan) {
249 /* remove from d->chan_pending */
250 list_del_init(&c->node);
251 pch_alloc |= 1 << c->id;
252 /* Mark this channel allocated */
253 p->vchan = c;
254 c->phy = p;
255 } else {
256 dev_dbg(d->slave.dev, "pchan %u: busy!\n", c->id);
257 }
258 }
259 spin_unlock_irqrestore(&d->lock, flags);
260
261 for (pch = 0; pch < d->dma_channels; pch++) {
262 if (pch_alloc & (1 << pch)) {
263 p = &d->phy[pch];
264 c = p->vchan;
265 if (c) {
266 spin_lock_irqsave(&c->vc.lock, flags);
267 zx_dma_start_txd(c);
268 spin_unlock_irqrestore(&c->vc.lock, flags);
269 }
270 }
271 }
272}
273
274static irqreturn_t zx_dma_int_handler(int irq, void *dev_id)
275{
276 struct zx_dma_dev *d = (struct zx_dma_dev *)dev_id;
277 struct zx_dma_phy *p;
278 struct zx_dma_chan *c;
279 u32 tc = readl_relaxed(d->base + REG_ZX_TC_IRQ);
280 u32 serr = readl_relaxed(d->base + REG_ZX_SRC_ERR_IRQ);
281 u32 derr = readl_relaxed(d->base + REG_ZX_DST_ERR_IRQ);
282 u32 cfg = readl_relaxed(d->base + REG_ZX_CFG_ERR_IRQ);
283 u32 i, irq_chan = 0, task = 0;
284
285 while (tc) {
286 i = __ffs(tc);
287 tc &= ~BIT(i);
288 p = &d->phy[i];
289 c = p->vchan;
290 if (c) {
291 unsigned long flags;
292
293 spin_lock_irqsave(&c->vc.lock, flags);
294 if (c->cyclic) {
295 vchan_cyclic_callback(&p->ds_run->vd);
296 } else {
297 vchan_cookie_complete(&p->ds_run->vd);
298 p->ds_done = p->ds_run;
299 task = 1;
300 }
301 spin_unlock_irqrestore(&c->vc.lock, flags);
302 irq_chan |= BIT(i);
303 }
304 }
305
306 if (serr || derr || cfg)
307 dev_warn(d->slave.dev, "DMA ERR src 0x%x, dst 0x%x, cfg 0x%x\n",
308 serr, derr, cfg);
309
310 writel_relaxed(irq_chan, d->base + REG_ZX_TC_IRQ_RAW);
311 writel_relaxed(serr, d->base + REG_ZX_SRC_ERR_IRQ_RAW);
312 writel_relaxed(derr, d->base + REG_ZX_DST_ERR_IRQ_RAW);
313 writel_relaxed(cfg, d->base + REG_ZX_CFG_ERR_IRQ_RAW);
314
315 if (task)
316 zx_dma_task(d);
317 return IRQ_HANDLED;
318}
319
320static void zx_dma_free_chan_resources(struct dma_chan *chan)
321{
322 struct zx_dma_chan *c = to_zx_chan(chan);
323 struct zx_dma_dev *d = to_zx_dma(chan->device);
324 unsigned long flags;
325
326 spin_lock_irqsave(&d->lock, flags);
327 list_del_init(&c->node);
328 spin_unlock_irqrestore(&d->lock, flags);
329
330 vchan_free_chan_resources(&c->vc);
331 c->ccfg = 0;
332}
333
334static enum dma_status zx_dma_tx_status(struct dma_chan *chan,
335 dma_cookie_t cookie,
336 struct dma_tx_state *state)
337{
338 struct zx_dma_chan *c = to_zx_chan(chan);
339 struct zx_dma_phy *p;
340 struct virt_dma_desc *vd;
341 unsigned long flags;
342 enum dma_status ret;
343 size_t bytes = 0;
344
345 ret = dma_cookie_status(&c->vc.chan, cookie, state);
346 if (ret == DMA_COMPLETE || !state)
347 return ret;
348
349 spin_lock_irqsave(&c->vc.lock, flags);
350 p = c->phy;
351 ret = c->status;
352
353 /*
354 * If the cookie is on our issue queue, then the residue is
355 * its total size.
356 */
357 vd = vchan_find_desc(&c->vc, cookie);
358 if (vd) {
359 bytes = container_of(vd, struct zx_dma_desc_sw, vd)->size;
360 } else if ((!p) || (!p->ds_run)) {
361 bytes = 0;
362 } else {
363 struct zx_dma_desc_sw *ds = p->ds_run;
364 u32 clli = 0, index = 0;
365
366 bytes = 0;
367 clli = zx_dma_get_curr_lli(p);
368 index = (clli - ds->desc_hw_lli) / sizeof(struct zx_desc_hw);
369 for (; index < ds->desc_num; index++) {
370 bytes += ds->desc_hw[index].src_x;
371 /* end of lli */
372 if (!ds->desc_hw[index].lli)
373 break;
374 }
375 }
376 spin_unlock_irqrestore(&c->vc.lock, flags);
377 dma_set_residue(state, bytes);
378 return ret;
379}
380
381static void zx_dma_issue_pending(struct dma_chan *chan)
382{
383 struct zx_dma_chan *c = to_zx_chan(chan);
384 struct zx_dma_dev *d = to_zx_dma(chan->device);
385 unsigned long flags;
386 int issue = 0;
387
388 spin_lock_irqsave(&c->vc.lock, flags);
389 /* add request to vc->desc_issued */
390 if (vchan_issue_pending(&c->vc)) {
391 spin_lock(&d->lock);
392 if (!c->phy && list_empty(&c->node)) {
393 /* if new channel, add chan_pending */
394 list_add_tail(&c->node, &d->chan_pending);
395 issue = 1;
396 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
397 }
398 spin_unlock(&d->lock);
399 } else {
400 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
401 }
402 spin_unlock_irqrestore(&c->vc.lock, flags);
403
404 if (issue)
405 zx_dma_task(d);
406}
407
408static void zx_dma_fill_desc(struct zx_dma_desc_sw *ds, dma_addr_t dst,
409 dma_addr_t src, size_t len, u32 num, u32 ccfg)
410{
411 if ((num + 1) < ds->desc_num)
412 ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) *
413 sizeof(struct zx_desc_hw);
414 ds->desc_hw[num].saddr = src;
415 ds->desc_hw[num].daddr = dst;
416 ds->desc_hw[num].src_x = len;
417 ds->desc_hw[num].ctr = ccfg;
418}
419
420static struct zx_dma_desc_sw *zx_alloc_desc_resource(int num,
421 struct dma_chan *chan)
422{
423 struct zx_dma_chan *c = to_zx_chan(chan);
424 struct zx_dma_desc_sw *ds;
425 struct zx_dma_dev *d = to_zx_dma(chan->device);
426 int lli_limit = LLI_BLOCK_SIZE / sizeof(struct zx_desc_hw);
427
428 if (num > lli_limit) {
429 dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n",
430 &c->vc, num, lli_limit);
431 return NULL;
432 }
433
434 ds = kzalloc(sizeof(*ds), GFP_ATOMIC);
435 if (!ds)
436 return NULL;
437
438 ds->desc_hw = dma_pool_alloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli);
439 if (!ds->desc_hw) {
440 dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc);
441 kfree(ds);
442 return NULL;
443 }
444 memset(ds->desc_hw, sizeof(struct zx_desc_hw) * num, 0);
445 ds->desc_num = num;
446 return ds;
447}
448
449static enum zx_dma_burst_width zx_dma_burst_width(enum dma_slave_buswidth width)
450{
451 switch (width) {
452 case DMA_SLAVE_BUSWIDTH_1_BYTE:
453 case DMA_SLAVE_BUSWIDTH_2_BYTES:
454 case DMA_SLAVE_BUSWIDTH_4_BYTES:
455 case DMA_SLAVE_BUSWIDTH_8_BYTES:
456 return ffs(width) - 1;
457 default:
458 return ZX_DMA_WIDTH_32BIT;
459 }
460}
461
462static int zx_pre_config(struct zx_dma_chan *c, enum dma_transfer_direction dir)
463{
464 struct dma_slave_config *cfg = &c->slave_cfg;
465 enum zx_dma_burst_width src_width;
466 enum zx_dma_burst_width dst_width;
467 u32 maxburst = 0;
468
469 switch (dir) {
470 case DMA_MEM_TO_MEM:
471 c->ccfg = ZX_CH_ENABLE | ZX_SOFT_REQ
472 | ZX_SRC_BURST_LEN(ZX_MAX_BURST_LEN - 1)
473 | ZX_SRC_BURST_WIDTH(ZX_DMA_WIDTH_32BIT)
474 | ZX_DST_BURST_WIDTH(ZX_DMA_WIDTH_32BIT);
475 break;
476 case DMA_MEM_TO_DEV:
477 c->dev_addr = cfg->dst_addr;
478 /* dst len is calculated from src width, len and dst width.
479 * We need make sure dst len not exceed MAX LEN.
480 * Trailing single transaction that does not fill a full
481 * burst also require identical src/dst data width.
482 */
483 dst_width = zx_dma_burst_width(cfg->dst_addr_width);
484 maxburst = cfg->dst_maxburst;
485 maxburst = maxburst < ZX_MAX_BURST_LEN ?
486 maxburst : ZX_MAX_BURST_LEN;
487 c->ccfg = ZX_DST_FIFO_MODE | ZX_CH_ENABLE
488 | ZX_SRC_BURST_LEN(maxburst - 1)
489 | ZX_SRC_BURST_WIDTH(dst_width)
490 | ZX_DST_BURST_WIDTH(dst_width);
491 break;
492 case DMA_DEV_TO_MEM:
493 c->dev_addr = cfg->src_addr;
494 src_width = zx_dma_burst_width(cfg->src_addr_width);
495 maxburst = cfg->src_maxburst;
496 maxburst = maxburst < ZX_MAX_BURST_LEN ?
497 maxburst : ZX_MAX_BURST_LEN;
498 c->ccfg = ZX_SRC_FIFO_MODE | ZX_CH_ENABLE
499 | ZX_SRC_BURST_LEN(maxburst - 1)
500 | ZX_SRC_BURST_WIDTH(src_width)
501 | ZX_DST_BURST_WIDTH(src_width);
502 break;
503 default:
504 return -EINVAL;
505 }
506 return 0;
507}
508
509static struct dma_async_tx_descriptor *zx_dma_prep_memcpy(
510 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
511 size_t len, unsigned long flags)
512{
513 struct zx_dma_chan *c = to_zx_chan(chan);
514 struct zx_dma_desc_sw *ds;
515 size_t copy = 0;
516 int num = 0;
517
518 if (!len)
519 return NULL;
520
521 if (zx_pre_config(c, DMA_MEM_TO_MEM))
522 return NULL;
523
524 num = DIV_ROUND_UP(len, DMA_MAX_SIZE);
525
526 ds = zx_alloc_desc_resource(num, chan);
527 if (!ds)
528 return NULL;
529
530 ds->size = len;
531 num = 0;
532
533 do {
534 copy = min_t(size_t, len, DMA_MAX_SIZE);
535 zx_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg);
536
537 src += copy;
538 dst += copy;
539 len -= copy;
540 } while (len);
541
542 c->cyclic = 0;
543 ds->desc_hw[num - 1].lli = 0; /* end of link */
544 ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL;
545 return vchan_tx_prep(&c->vc, &ds->vd, flags);
546}
547
548static struct dma_async_tx_descriptor *zx_dma_prep_slave_sg(
549 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen,
550 enum dma_transfer_direction dir, unsigned long flags, void *context)
551{
552 struct zx_dma_chan *c = to_zx_chan(chan);
553 struct zx_dma_desc_sw *ds;
554 size_t len, avail, total = 0;
555 struct scatterlist *sg;
556 dma_addr_t addr, src = 0, dst = 0;
557 int num = sglen, i;
558
559 if (!sgl)
560 return NULL;
561
562 if (zx_pre_config(c, dir))
563 return NULL;
564
565 for_each_sg(sgl, sg, sglen, i) {
566 avail = sg_dma_len(sg);
567 if (avail > DMA_MAX_SIZE)
568 num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1;
569 }
570
571 ds = zx_alloc_desc_resource(num, chan);
572 if (!ds)
573 return NULL;
574
575 c->cyclic = 0;
576 num = 0;
577 for_each_sg(sgl, sg, sglen, i) {
578 addr = sg_dma_address(sg);
579 avail = sg_dma_len(sg);
580 total += avail;
581
582 do {
583 len = min_t(size_t, avail, DMA_MAX_SIZE);
584
585 if (dir == DMA_MEM_TO_DEV) {
586 src = addr;
587 dst = c->dev_addr;
588 } else if (dir == DMA_DEV_TO_MEM) {
589 src = c->dev_addr;
590 dst = addr;
591 }
592
593 zx_dma_fill_desc(ds, dst, src, len, num++, c->ccfg);
594
595 addr += len;
596 avail -= len;
597 } while (avail);
598 }
599
600 ds->desc_hw[num - 1].lli = 0; /* end of link */
601 ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL;
602 ds->size = total;
603 return vchan_tx_prep(&c->vc, &ds->vd, flags);
604}
605
606static struct dma_async_tx_descriptor *zx_dma_prep_dma_cyclic(
607 struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
608 size_t period_len, enum dma_transfer_direction dir,
609 unsigned long flags)
610{
611 struct zx_dma_chan *c = to_zx_chan(chan);
612 struct zx_dma_desc_sw *ds;
613 dma_addr_t src = 0, dst = 0;
614 int num_periods = buf_len / period_len;
615 int buf = 0, num = 0;
616
617 if (period_len > DMA_MAX_SIZE) {
618 dev_err(chan->device->dev, "maximum period size exceeded\n");
619 return NULL;
620 }
621
622 if (zx_pre_config(c, dir))
623 return NULL;
624
625 ds = zx_alloc_desc_resource(num_periods, chan);
626 if (!ds)
627 return NULL;
628 c->cyclic = 1;
629
630 while (buf < buf_len) {
631 if (dir == DMA_MEM_TO_DEV) {
632 src = dma_addr;
633 dst = c->dev_addr;
634 } else if (dir == DMA_DEV_TO_MEM) {
635 src = c->dev_addr;
636 dst = dma_addr;
637 }
638 zx_dma_fill_desc(ds, dst, src, period_len, num++,
639 c->ccfg | ZX_IRQ_ENABLE_ALL);
640 dma_addr += period_len;
641 buf += period_len;
642 }
643
644 ds->desc_hw[num - 1].lli = ds->desc_hw_lli;
645 ds->size = buf_len;
646 return vchan_tx_prep(&c->vc, &ds->vd, flags);
647}
648
649static int zx_dma_config(struct dma_chan *chan,
650 struct dma_slave_config *cfg)
651{
652 struct zx_dma_chan *c = to_zx_chan(chan);
653
654 if (!cfg)
655 return -EINVAL;
656
657 memcpy(&c->slave_cfg, cfg, sizeof(*cfg));
658
659 return 0;
660}
661
662static int zx_dma_terminate_all(struct dma_chan *chan)
663{
664 struct zx_dma_chan *c = to_zx_chan(chan);
665 struct zx_dma_dev *d = to_zx_dma(chan->device);
666 struct zx_dma_phy *p = c->phy;
667 unsigned long flags;
668 LIST_HEAD(head);
669
670 dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
671
672 /* Prevent this channel being scheduled */
673 spin_lock(&d->lock);
674 list_del_init(&c->node);
675 spin_unlock(&d->lock);
676
677 /* Clear the tx descriptor lists */
678 spin_lock_irqsave(&c->vc.lock, flags);
679 vchan_get_all_descriptors(&c->vc, &head);
680 if (p) {
681 /* vchan is assigned to a pchan - stop the channel */
682 zx_dma_terminate_chan(p, d);
683 c->phy = NULL;
684 p->vchan = NULL;
685 p->ds_run = NULL;
686 p->ds_done = NULL;
687 }
688 spin_unlock_irqrestore(&c->vc.lock, flags);
689 vchan_dma_desc_free_list(&c->vc, &head);
690
691 return 0;
692}
693
694static int zx_dma_transfer_pause(struct dma_chan *chan)
695{
696 struct zx_dma_chan *c = to_zx_chan(chan);
697 u32 val = 0;
698
699 val = readl_relaxed(c->phy->base + REG_ZX_CTRL);
700 val &= ~ZX_CH_ENABLE;
701 writel_relaxed(val, c->phy->base + REG_ZX_CTRL);
702
703 return 0;
704}
705
706static int zx_dma_transfer_resume(struct dma_chan *chan)
707{
708 struct zx_dma_chan *c = to_zx_chan(chan);
709 u32 val = 0;
710
711 val = readl_relaxed(c->phy->base + REG_ZX_CTRL);
712 val |= ZX_CH_ENABLE;
713 writel_relaxed(val, c->phy->base + REG_ZX_CTRL);
714
715 return 0;
716}
717
718static void zx_dma_free_desc(struct virt_dma_desc *vd)
719{
720 struct zx_dma_desc_sw *ds =
721 container_of(vd, struct zx_dma_desc_sw, vd);
722 struct zx_dma_dev *d = to_zx_dma(vd->tx.chan->device);
723
724 dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli);
725 kfree(ds);
726}
727
728static const struct of_device_id zx6702_dma_dt_ids[] = {
729 { .compatible = "zte,zx296702-dma", },
730 {}
731};
732MODULE_DEVICE_TABLE(of, zx6702_dma_dt_ids);
733
734static struct dma_chan *zx_of_dma_simple_xlate(struct of_phandle_args *dma_spec,
735 struct of_dma *ofdma)
736{
737 struct zx_dma_dev *d = ofdma->of_dma_data;
738 unsigned int request = dma_spec->args[0];
739 struct dma_chan *chan;
740 struct zx_dma_chan *c;
741
742 if (request > d->dma_requests)
743 return NULL;
744
745 chan = dma_get_any_slave_channel(&d->slave);
746 if (!chan) {
747 dev_err(d->slave.dev, "get channel fail in %s.\n", __func__);
748 return NULL;
749 }
750 c = to_zx_chan(chan);
751 c->id = request;
752 dev_info(d->slave.dev, "zx_dma: pchan %u: alloc vchan %p\n",
753 c->id, &c->vc);
754 return chan;
755}
756
757static int zx_dma_probe(struct platform_device *op)
758{
759 struct zx_dma_dev *d;
760 struct resource *iores;
761 int i, ret = 0;
762
763 iores = platform_get_resource(op, IORESOURCE_MEM, 0);
764 if (!iores)
765 return -EINVAL;
766
767 d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL);
768 if (!d)
769 return -ENOMEM;
770
771 d->base = devm_ioremap_resource(&op->dev, iores);
772 if (IS_ERR(d->base))
773 return PTR_ERR(d->base);
774
775 of_property_read_u32((&op->dev)->of_node,
776 "dma-channels", &d->dma_channels);
777 of_property_read_u32((&op->dev)->of_node,
778 "dma-requests", &d->dma_requests);
779 if (!d->dma_requests || !d->dma_channels)
780 return -EINVAL;
781
782 d->clk = devm_clk_get(&op->dev, NULL);
783 if (IS_ERR(d->clk)) {
784 dev_err(&op->dev, "no dma clk\n");
785 return PTR_ERR(d->clk);
786 }
787
788 d->irq = platform_get_irq(op, 0);
789 ret = devm_request_irq(&op->dev, d->irq, zx_dma_int_handler,
790 0, DRIVER_NAME, d);
791 if (ret)
792 return ret;
793
794 /* A DMA memory pool for LLIs, align on 32-byte boundary */
795 d->pool = dmam_pool_create(DRIVER_NAME, &op->dev,
796 LLI_BLOCK_SIZE, 32, 0);
797 if (!d->pool)
798 return -ENOMEM;
799
800 /* init phy channel */
801 d->phy = devm_kzalloc(&op->dev,
802 d->dma_channels * sizeof(struct zx_dma_phy), GFP_KERNEL);
803 if (!d->phy)
804 return -ENOMEM;
805
806 for (i = 0; i < d->dma_channels; i++) {
807 struct zx_dma_phy *p = &d->phy[i];
808
809 p->idx = i;
810 p->base = d->base + i * 0x40;
811 }
812
813 INIT_LIST_HEAD(&d->slave.channels);
814 dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
815 dma_cap_set(DMA_MEMCPY, d->slave.cap_mask);
816 dma_cap_set(DMA_PRIVATE, d->slave.cap_mask);
817 d->slave.dev = &op->dev;
818 d->slave.device_free_chan_resources = zx_dma_free_chan_resources;
819 d->slave.device_tx_status = zx_dma_tx_status;
820 d->slave.device_prep_dma_memcpy = zx_dma_prep_memcpy;
821 d->slave.device_prep_slave_sg = zx_dma_prep_slave_sg;
822 d->slave.device_prep_dma_cyclic = zx_dma_prep_dma_cyclic;
823 d->slave.device_issue_pending = zx_dma_issue_pending;
824 d->slave.device_config = zx_dma_config;
825 d->slave.device_terminate_all = zx_dma_terminate_all;
826 d->slave.device_pause = zx_dma_transfer_pause;
827 d->slave.device_resume = zx_dma_transfer_resume;
828 d->slave.copy_align = DMA_ALIGN;
829 d->slave.src_addr_widths = ZX_DMA_BUSWIDTHS;
830 d->slave.dst_addr_widths = ZX_DMA_BUSWIDTHS;
831 d->slave.directions = BIT(DMA_MEM_TO_MEM) | BIT(DMA_MEM_TO_DEV)
832 | BIT(DMA_DEV_TO_MEM);
833 d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
834
835 /* init virtual channel */
836 d->chans = devm_kzalloc(&op->dev,
837 d->dma_requests * sizeof(struct zx_dma_chan), GFP_KERNEL);
838 if (!d->chans)
839 return -ENOMEM;
840
841 for (i = 0; i < d->dma_requests; i++) {
842 struct zx_dma_chan *c = &d->chans[i];
843
844 c->status = DMA_IN_PROGRESS;
845 INIT_LIST_HEAD(&c->node);
846 c->vc.desc_free = zx_dma_free_desc;
847 vchan_init(&c->vc, &d->slave);
848 }
849
850 /* Enable clock before accessing registers */
851 ret = clk_prepare_enable(d->clk);
852 if (ret < 0) {
853 dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret);
854 goto zx_dma_out;
855 }
856
857 zx_dma_init_state(d);
858
859 spin_lock_init(&d->lock);
860 INIT_LIST_HEAD(&d->chan_pending);
861 platform_set_drvdata(op, d);
862
863 ret = dma_async_device_register(&d->slave);
864 if (ret)
865 goto clk_dis;
866
867 ret = of_dma_controller_register((&op->dev)->of_node,
868 zx_of_dma_simple_xlate, d);
869 if (ret)
870 goto of_dma_register_fail;
871
872 dev_info(&op->dev, "initialized\n");
873 return 0;
874
875of_dma_register_fail:
876 dma_async_device_unregister(&d->slave);
877clk_dis:
878 clk_disable_unprepare(d->clk);
879zx_dma_out:
880 return ret;
881}
882
883static int zx_dma_remove(struct platform_device *op)
884{
885 struct zx_dma_chan *c, *cn;
886 struct zx_dma_dev *d = platform_get_drvdata(op);
887
888 /* explictly free the irq */
889 devm_free_irq(&op->dev, d->irq, d);
890
891 dma_async_device_unregister(&d->slave);
892 of_dma_controller_free((&op->dev)->of_node);
893
894 list_for_each_entry_safe(c, cn, &d->slave.channels,
895 vc.chan.device_node) {
896 list_del(&c->vc.chan.device_node);
897 }
898 clk_disable_unprepare(d->clk);
899 dmam_pool_destroy(d->pool);
900
901 return 0;
902}
903
904#ifdef CONFIG_PM_SLEEP
905static int zx_dma_suspend_dev(struct device *dev)
906{
907 struct zx_dma_dev *d = dev_get_drvdata(dev);
908 u32 stat = 0;
909
910 stat = zx_dma_get_chan_stat(d);
911 if (stat) {
912 dev_warn(d->slave.dev,
913 "chan %d is running fail to suspend\n", stat);
914 return -1;
915 }
916 clk_disable_unprepare(d->clk);
917 return 0;
918}
919
920static int zx_dma_resume_dev(struct device *dev)
921{
922 struct zx_dma_dev *d = dev_get_drvdata(dev);
923 int ret = 0;
924
925 ret = clk_prepare_enable(d->clk);
926 if (ret < 0) {
927 dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret);
928 return ret;
929 }
930 zx_dma_init_state(d);
931 return 0;
932}
933#endif
934
935static SIMPLE_DEV_PM_OPS(zx_dma_pmops, zx_dma_suspend_dev, zx_dma_resume_dev);
936
937static struct platform_driver zx_pdma_driver = {
938 .driver = {
939 .name = DRIVER_NAME,
940 .pm = &zx_dma_pmops,
941 .of_match_table = zx6702_dma_dt_ids,
942 },
943 .probe = zx_dma_probe,
944 .remove = zx_dma_remove,
945};
946
947module_platform_driver(zx_pdma_driver);
948
949MODULE_DESCRIPTION("ZTE ZX296702 DMA Driver");
950MODULE_AUTHOR("Jun Nie jun.nie@linaro.org");
951MODULE_LICENSE("GPL v2");
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-15 04:41:39 -0500