aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSebastian Andrzej Siewior <bigeasy@linutronix.de>2013-06-20 06:13:04 -0400
committerFelipe Balbi <balbi@ti.com>2013-08-09 10:40:16 -0400
commit9b3452d1fa3c017d3664ff9e6a601daa6e0576eb (patch)
treeac2fdd71c45975f2430d094ef138ccd0a2d835c0
parent97238b35d5bbb5d5312d83c30a429824b777619f (diff)
usb: musb dma: add cppi41 dma driver
This driver is currently used by musb' cppi41 couter part. I may merge both dma engine user of musb at some point but not just yet. The driver seems to work in RX/TX mode in host mode, tested on mass storage. I increaed the size of the TX / RX transfers and waited for the core code to cancel a transfers and it seems to recover. v2..3: - use mall transfers on RX side and check data toggle. - use rndis mode on tx side so we haveon interrupt for 4096 transfers. - remove custom "transferred" hack and use dmaengine_tx_status() to compute the total amount of data that has been transferred. - cancel transfers and reclaim descriptors v1..v2: - RX path added - dma mode 0 & 1 is working - device tree nodes re-created. Cc: Vinod Koul <vinod.koul@intel.com> Cc: Dan Williams <djbw@fb.com> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Signed-off-by: Felipe Balbi <balbi@ti.com>
Diffstat
-rw-r--r--arch/arm/boot/dts/am335x-evm.dts4
-rw-r--r--arch/arm/boot/dts/am33xx.dtsi61
-rw-r--r--drivers/dma/Kconfig8
-rw-r--r--drivers/dma/Makefile1
-rw-r--r--drivers/dma/cppi41.c1048
-rw-r--r--drivers/usb/musb/Kconfig4
-rw-r--r--drivers/usb/musb/Makefile1
-rw-r--r--drivers/usb/musb/musb_cppi41.c552
-rw-r--r--drivers/usb/musb/musb_dma.h2
9 files changed, 1680 insertions, 1 deletions
diff --git a/arch/arm/boot/dts/am335x-evm.dts b/arch/arm/boot/dts/am335x-evm.dts
index b2987e03b19f..c26c16cace3c 100644
--- a/arch/arm/boot/dts/am335x-evm.dts
+++ b/arch/arm/boot/dts/am335x-evm.dts
@@ -193,6 +193,10 @@
193 usb@47401800 { 193 usb@47401800 {
194 status = "okay"; 194 status = "okay";
195 }; 195 };
196
197 dma@07402000 {
198 status = "okay";
199 };
196 }; 200 };
197 201
198 i2c1: i2c@4802a000 { 202 i2c1: i2c@4802a000 {
diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi
index e1e773cd2556..24d63095ab83 100644
--- a/arch/arm/boot/dts/am33xx.dtsi
+++ b/arch/arm/boot/dts/am33xx.dtsi
@@ -383,6 +383,29 @@
383 port-mode = <3>; 383 port-mode = <3>;
384 power = <250>; 384 power = <250>;
385 phys = <&usb0_phy>; 385 phys = <&usb0_phy>;
386
387 dmas = <&cppi41dma 0 0 &cppi41dma 1 0
388 &cppi41dma 2 0 &cppi41dma 3 0
389 &cppi41dma 4 0 &cppi41dma 5 0
390 &cppi41dma 6 0 &cppi41dma 7 0
391 &cppi41dma 8 0 &cppi41dma 9 0
392 &cppi41dma 10 0 &cppi41dma 11 0
393 &cppi41dma 12 0 &cppi41dma 13 0
394 &cppi41dma 14 0 &cppi41dma 0 1
395 &cppi41dma 1 1 &cppi41dma 2 1
396 &cppi41dma 3 1 &cppi41dma 4 1
397 &cppi41dma 5 1 &cppi41dma 6 1
398 &cppi41dma 7 1 &cppi41dma 8 1
399 &cppi41dma 9 1 &cppi41dma 10 1
400 &cppi41dma 11 1 &cppi41dma 12 1
401 &cppi41dma 13 1 &cppi41dma 14 1>;
402 dma-names =
403 "rx1", "rx2", "rx3", "rx4", "rx5", "rx6", "rx7",
404 "rx8", "rx9", "rx10", "rx11", "rx12", "rx13",
405 "rx14", "rx15",
406 "tx1", "tx2", "tx3", "tx4", "tx5", "tx6", "tx7",
407 "tx8", "tx9", "tx10", "tx11", "tx12", "tx13",
408 "tx14", "tx15";
386 }; 409 };
387 }; 410 };
388 411
@@ -415,8 +438,46 @@
415 port-mode = <3>; 438 port-mode = <3>;
416 power = <250>; 439 power = <250>;
417 phys = <&usb1_phy>; 440 phys = <&usb1_phy>;
441
442 dmas = <&cppi41dma 15 0 &cppi41dma 16 0
443 &cppi41dma 17 0 &cppi41dma 18 0
444 &cppi41dma 19 0 &cppi41dma 20 0
445 &cppi41dma 21 0 &cppi41dma 22 0
446 &cppi41dma 23 0 &cppi41dma 24 0
447 &cppi41dma 25 0 &cppi41dma 26 0
448 &cppi41dma 27 0 &cppi41dma 28 0
449 &cppi41dma 29 0 &cppi41dma 15 1
450 &cppi41dma 16 1 &cppi41dma 17 1
451 &cppi41dma 18 1 &cppi41dma 19 1
452 &cppi41dma 20 1 &cppi41dma 21 1
453 &cppi41dma 22 1 &cppi41dma 23 1
454 &cppi41dma 24 1 &cppi41dma 25 1
455 &cppi41dma 26 1 &cppi41dma 27 1
456 &cppi41dma 28 1 &cppi41dma 29 1>;
457 dma-names =
458 "rx1", "rx2", "rx3", "rx4", "rx5", "rx6", "rx7",
459 "rx8", "rx9", "rx10", "rx11", "rx12", "rx13",
460 "rx14", "rx15",
461 "tx1", "tx2", "tx3", "tx4", "tx5", "tx6", "tx7",
462 "tx8", "tx9", "tx10", "tx11", "tx12", "tx13",
463 "tx14", "tx15";
418 }; 464 };
419 }; 465 };
466
467 cppi41dma: dma@07402000 {
468 compatible = "ti,am3359-cppi41";
469 reg = <0x47400000 0x1000
470 0x47402000 0x1000
471 0x47403000 0x1000
472 0x47404000 0x4000>;
473 reg-names = "glue controller scheduler queuemgr";
474 interrupts = <17>;
475 interrupt-names = "glue";
476 #dma-cells = <2>;
477 #dma-channels = <30>;
478 #dma-requests = <256>;
479 status = "disabled";
480 };
420 }; 481 };
421 482
422 epwmss0: epwmss@48300000 { 483 epwmss0: epwmss@48300000 {
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 6825957c97fb..77bc480117b7 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -287,6 +287,14 @@ config DMA_OMAP
287 select DMA_ENGINE 287 select DMA_ENGINE
288 select DMA_VIRTUAL_CHANNELS 288 select DMA_VIRTUAL_CHANNELS
289 289
290config TI_CPPI41
291 tristate "AM33xx CPPI41 DMA support"
292 depends on ARCH_OMAP
293 select DMA_ENGINE
294 help
295 The Communications Port Programming Interface (CPPI) 4.1 DMA engine
296 is currently used by the USB driver on AM335x platforms.
297
290config MMP_PDMA 298config MMP_PDMA
291 bool "MMP PDMA support" 299 bool "MMP PDMA support"
292 depends on (ARCH_MMP || ARCH_PXA) 300 depends on (ARCH_MMP || ARCH_PXA)
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index 5e0f2ef85614..6d62ec30c4bc 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -39,3 +39,4 @@ obj-$(CONFIG_MMP_TDMA) += mmp_tdma.o
39obj-$(CONFIG_DMA_OMAP) += omap-dma.o 39obj-$(CONFIG_DMA_OMAP) += omap-dma.o
40obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o 40obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o
41obj-$(CONFIG_DMA_JZ4740) += dma-jz4740.o 41obj-$(CONFIG_DMA_JZ4740) += dma-jz4740.o
42obj-$(CONFIG_TI_CPPI41) += cppi41.o
diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c
new file mode 100644
index 000000000000..5dcebca37760
--- /dev/null
+++ b/drivers/dma/cppi41.c
@@ -0,0 +1,1048 @@
1#include <linux/dmaengine.h>
2#include <linux/dma-mapping.h>
3#include <linux/platform_device.h>
4#include <linux/module.h>
5#include <linux/of.h>
6#include <linux/slab.h>
7#include <linux/of_dma.h>
8#include <linux/of_irq.h>
9#include <linux/dmapool.h>
10#include <linux/interrupt.h>
11#include <linux/of_address.h>
12#include "dmaengine.h"
13
14#define DESC_TYPE 27
15#define DESC_TYPE_HOST 0x10
16#define DESC_TYPE_TEARD 0x13
17
18#define TD_DESC_IS_RX (1 << 16)
19#define TD_DESC_DMA_NUM 10
20
21#define DESC_LENGTH_BITS_NUM 21
22
23#define DESC_TYPE_USB (5 << 26)
24#define DESC_PD_COMPLETE (1 << 31)
25
26/* DMA engine */
27#define DMA_TDFDQ 4
28#define DMA_TXGCR(x) (0x800 + (x) * 0x20)
29#define DMA_RXGCR(x) (0x808 + (x) * 0x20)
30#define RXHPCRA0 4
31
32#define GCR_CHAN_ENABLE (1 << 31)
33#define GCR_TEARDOWN (1 << 30)
34#define GCR_STARV_RETRY (1 << 24)
35#define GCR_DESC_TYPE_HOST (1 << 14)
36
37/* DMA scheduler */
38#define DMA_SCHED_CTRL 0
39#define DMA_SCHED_CTRL_EN (1 << 31)
40#define DMA_SCHED_WORD(x) ((x) * 4 + 0x800)
41
42#define SCHED_ENTRY0_CHAN(x) ((x) << 0)
43#define SCHED_ENTRY0_IS_RX (1 << 7)
44
45#define SCHED_ENTRY1_CHAN(x) ((x) << 8)
46#define SCHED_ENTRY1_IS_RX (1 << 15)
47
48#define SCHED_ENTRY2_CHAN(x) ((x) << 16)
49#define SCHED_ENTRY2_IS_RX (1 << 23)
50
51#define SCHED_ENTRY3_CHAN(x) ((x) << 24)
52#define SCHED_ENTRY3_IS_RX (1 << 31)
53
54/* Queue manager */
55/* 4 KiB of memory for descriptors, 2 for each endpoint */
56#define ALLOC_DECS_NUM 128
57#define DESCS_AREAS 1
58#define TOTAL_DESCS_NUM (ALLOC_DECS_NUM * DESCS_AREAS)
59#define QMGR_SCRATCH_SIZE (TOTAL_DESCS_NUM * 4)
60
61#define QMGR_LRAM0_BASE 0x80
62#define QMGR_LRAM_SIZE 0x84
63#define QMGR_LRAM1_BASE 0x88
64#define QMGR_MEMBASE(x) (0x1000 + (x) * 0x10)
65#define QMGR_MEMCTRL(x) (0x1004 + (x) * 0x10)
66#define QMGR_MEMCTRL_IDX_SH 16
67#define QMGR_MEMCTRL_DESC_SH 8
68
69#define QMGR_NUM_PEND 5
70#define QMGR_PEND(x) (0x90 + (x) * 4)
71
72#define QMGR_PENDING_SLOT_Q(x) (x / 32)
73#define QMGR_PENDING_BIT_Q(x) (x % 32)
74
75#define QMGR_QUEUE_A(n) (0x2000 + (n) * 0x10)
76#define QMGR_QUEUE_B(n) (0x2004 + (n) * 0x10)
77#define QMGR_QUEUE_C(n) (0x2008 + (n) * 0x10)
78#define QMGR_QUEUE_D(n) (0x200c + (n) * 0x10)
79
80/* Glue layer specific */
81/* USBSS / USB AM335x */
82#define USBSS_IRQ_STATUS 0x28
83#define USBSS_IRQ_ENABLER 0x2c
84#define USBSS_IRQ_CLEARR 0x30
85
86#define USBSS_IRQ_PD_COMP (1 << 2)
87
88struct cppi41_channel {
89 struct dma_chan chan;
90 struct dma_async_tx_descriptor txd;
91 struct cppi41_dd *cdd;
92 struct cppi41_desc *desc;
93 dma_addr_t desc_phys;
94 void __iomem *gcr_reg;
95 int is_tx;
96 u32 residue;
97
98 unsigned int q_num;
99 unsigned int q_comp_num;
100 unsigned int port_num;
101
102 unsigned td_retry;
103 unsigned td_queued:1;
104 unsigned td_seen:1;
105 unsigned td_desc_seen:1;
106};
107
108struct cppi41_desc {
109 u32 pd0;
110 u32 pd1;
111 u32 pd2;
112 u32 pd3;
113 u32 pd4;
114 u32 pd5;
115 u32 pd6;
116 u32 pd7;
117} __aligned(32);
118
119struct chan_queues {
120 u16 submit;
121 u16 complete;
122};
123
124struct cppi41_dd {
125 struct dma_device ddev;
126
127 void *qmgr_scratch;
128 dma_addr_t scratch_phys;
129
130 struct cppi41_desc *cd;
131 dma_addr_t descs_phys;
132 u32 first_td_desc;
133 struct cppi41_channel *chan_busy[ALLOC_DECS_NUM];
134
135 void __iomem *usbss_mem;
136 void __iomem *ctrl_mem;
137 void __iomem *sched_mem;
138 void __iomem *qmgr_mem;
139 unsigned int irq;
140 const struct chan_queues *queues_rx;
141 const struct chan_queues *queues_tx;
142 struct chan_queues td_queue;
143};
144
145#define FIST_COMPLETION_QUEUE 93
146static struct chan_queues usb_queues_tx[] = {
147 /* USB0 ENDP 1 */
148 [ 0] = { .submit = 32, .complete = 93},
149 [ 1] = { .submit = 34, .complete = 94},
150 [ 2] = { .submit = 36, .complete = 95},
151 [ 3] = { .submit = 38, .complete = 96},
152 [ 4] = { .submit = 40, .complete = 97},
153 [ 5] = { .submit = 42, .complete = 98},
154 [ 6] = { .submit = 44, .complete = 99},
155 [ 7] = { .submit = 46, .complete = 100},
156 [ 8] = { .submit = 48, .complete = 101},
157 [ 9] = { .submit = 50, .complete = 102},
158 [10] = { .submit = 52, .complete = 103},
159 [11] = { .submit = 54, .complete = 104},
160 [12] = { .submit = 56, .complete = 105},
161 [13] = { .submit = 58, .complete = 106},
162 [14] = { .submit = 60, .complete = 107},
163
164 /* USB1 ENDP1 */
165 [15] = { .submit = 62, .complete = 125},
166 [16] = { .submit = 64, .complete = 126},
167 [17] = { .submit = 66, .complete = 127},
168 [18] = { .submit = 68, .complete = 128},
169 [19] = { .submit = 70, .complete = 129},
170 [20] = { .submit = 72, .complete = 130},
171 [21] = { .submit = 74, .complete = 131},
172 [22] = { .submit = 76, .complete = 132},
173 [23] = { .submit = 78, .complete = 133},
174 [24] = { .submit = 80, .complete = 134},
175 [25] = { .submit = 82, .complete = 135},
176 [26] = { .submit = 84, .complete = 136},
177 [27] = { .submit = 86, .complete = 137},
178 [28] = { .submit = 88, .complete = 138},
179 [29] = { .submit = 90, .complete = 139},
180};
181
182static const struct chan_queues usb_queues_rx[] = {
183 /* USB0 ENDP 1 */
184 [ 0] = { .submit = 1, .complete = 109},
185 [ 1] = { .submit = 2, .complete = 110},
186 [ 2] = { .submit = 3, .complete = 111},
187 [ 3] = { .submit = 4, .complete = 112},
188 [ 4] = { .submit = 5, .complete = 113},
189 [ 5] = { .submit = 6, .complete = 114},
190 [ 6] = { .submit = 7, .complete = 115},
191 [ 7] = { .submit = 8, .complete = 116},
192 [ 8] = { .submit = 9, .complete = 117},
193 [ 9] = { .submit = 10, .complete = 118},
194 [10] = { .submit = 11, .complete = 119},
195 [11] = { .submit = 12, .complete = 120},
196 [12] = { .submit = 13, .complete = 121},
197 [13] = { .submit = 14, .complete = 122},
198 [14] = { .submit = 15, .complete = 123},
199
200 /* USB1 ENDP 1 */
201 [15] = { .submit = 16, .complete = 141},
202 [16] = { .submit = 17, .complete = 142},
203 [17] = { .submit = 18, .complete = 143},
204 [18] = { .submit = 19, .complete = 144},
205 [19] = { .submit = 20, .complete = 145},
206 [20] = { .submit = 21, .complete = 146},
207 [21] = { .submit = 22, .complete = 147},
208 [22] = { .submit = 23, .complete = 148},
209 [23] = { .submit = 24, .complete = 149},
210 [24] = { .submit = 25, .complete = 150},
211 [25] = { .submit = 26, .complete = 151},
212 [26] = { .submit = 27, .complete = 152},
213 [27] = { .submit = 28, .complete = 153},
214 [28] = { .submit = 29, .complete = 154},
215 [29] = { .submit = 30, .complete = 155},
216};
217
218struct cppi_glue_infos {
219 irqreturn_t (*isr)(int irq, void *data);
220 const struct chan_queues *queues_rx;
221 const struct chan_queues *queues_tx;
222 struct chan_queues td_queue;
223};
224
225static struct cppi41_channel *to_cpp41_chan(struct dma_chan *c)
226{
227 return container_of(c, struct cppi41_channel, chan);
228}
229
230static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc)
231{
232 struct cppi41_channel *c;
233 u32 descs_size;
234 u32 desc_num;
235
236 descs_size = sizeof(struct cppi41_desc) * ALLOC_DECS_NUM;
237
238 if (!((desc >= cdd->descs_phys) &&
239 (desc < (cdd->descs_phys + descs_size)))) {
240 return NULL;
241 }
242
243 desc_num = (desc - cdd->descs_phys) / sizeof(struct cppi41_desc);
244 BUG_ON(desc_num > ALLOC_DECS_NUM);
245 c = cdd->chan_busy[desc_num];
246 cdd->chan_busy[desc_num] = NULL;
247 return c;
248}
249
250static void cppi_writel(u32 val, void *__iomem *mem)
251{
252 __raw_writel(val, mem);
253}
254
255static u32 cppi_readl(void *__iomem *mem)
256{
257 return __raw_readl(mem);
258}
259
260static u32 pd_trans_len(u32 val)
261{
262 return val & ((1 << (DESC_LENGTH_BITS_NUM + 1)) - 1);
263}
264
265static irqreturn_t cppi41_irq(int irq, void *data)
266{
267 struct cppi41_dd *cdd = data;
268 struct cppi41_channel *c;
269 u32 status;
270 int i;
271
272 status = cppi_readl(cdd->usbss_mem + USBSS_IRQ_STATUS);
273 if (!(status & USBSS_IRQ_PD_COMP))
274 return IRQ_NONE;
275 cppi_writel(status, cdd->usbss_mem + USBSS_IRQ_STATUS);
276
277 for (i = QMGR_PENDING_SLOT_Q(FIST_COMPLETION_QUEUE); i < QMGR_NUM_PEND;
278 i++) {
279 u32 val;
280 u32 q_num;
281
282 val = cppi_readl(cdd->qmgr_mem + QMGR_PEND(i));
283 if (i == QMGR_PENDING_SLOT_Q(FIST_COMPLETION_QUEUE) && val) {
284 u32 mask;
285 /* set corresponding bit for completetion Q 93 */
286 mask = 1 << QMGR_PENDING_BIT_Q(FIST_COMPLETION_QUEUE);
287 /* not set all bits for queues less than Q 93 */
288 mask--;
289 /* now invert and keep only Q 93+ set */
290 val &= ~mask;
291 }
292
293 if (val)
294 __iormb();
295
296 while (val) {
297 u32 desc;
298
299 q_num = __fls(val);
300 val &= ~(1 << q_num);
301 q_num += 32 * i;
302 desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(q_num));
303 desc &= ~0x1f;
304 c = desc_to_chan(cdd, desc);
305 if (WARN_ON(!c)) {
306 pr_err("%s() q %d desc %08x\n", __func__,
307 q_num, desc);
308 continue;
309 }
310 c->residue = pd_trans_len(c->desc->pd6) -
311 pd_trans_len(c->desc->pd0);
312
313 dma_cookie_complete(&c->txd);
314 c->txd.callback(c->txd.callback_param);
315 }
316 }
317 return IRQ_HANDLED;
318}
319
320static dma_cookie_t cppi41_tx_submit(struct dma_async_tx_descriptor *tx)
321{
322 dma_cookie_t cookie;
323
324 cookie = dma_cookie_assign(tx);
325
326 return cookie;
327}
328
329static int cppi41_dma_alloc_chan_resources(struct dma_chan *chan)
330{
331 struct cppi41_channel *c = to_cpp41_chan(chan);
332
333 dma_cookie_init(chan);
334 dma_async_tx_descriptor_init(&c->txd, chan);
335 c->txd.tx_submit = cppi41_tx_submit;
336
337 if (!c->is_tx)
338 cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0);
339
340 return 0;
341}
342
343static void cppi41_dma_free_chan_resources(struct dma_chan *chan)
344{
345}
346
347static enum dma_status cppi41_dma_tx_status(struct dma_chan *chan,
348 dma_cookie_t cookie, struct dma_tx_state *txstate)
349{
350 struct cppi41_channel *c = to_cpp41_chan(chan);
351 enum dma_status ret;
352
353 /* lock */
354 ret = dma_cookie_status(chan, cookie, txstate);
355 if (txstate && ret == DMA_SUCCESS)
356 txstate->residue = c->residue;
357 /* unlock */
358
359 return ret;
360}
361
362static void push_desc_queue(struct cppi41_channel *c)
363{
364 struct cppi41_dd *cdd = c->cdd;
365 u32 desc_num;
366 u32 desc_phys;
367 u32 reg;
368
369 desc_phys = lower_32_bits(c->desc_phys);
370 desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
371 WARN_ON(cdd->chan_busy[desc_num]);
372 cdd->chan_busy[desc_num] = c;
373
374 reg = (sizeof(struct cppi41_desc) - 24) / 4;
375 reg |= desc_phys;
376 cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));
377}
378
379static void cppi41_dma_issue_pending(struct dma_chan *chan)
380{
381 struct cppi41_channel *c = to_cpp41_chan(chan);
382 u32 reg;
383
384 c->residue = 0;
385
386 reg = GCR_CHAN_ENABLE;
387 if (!c->is_tx) {
388 reg |= GCR_STARV_RETRY;
389 reg |= GCR_DESC_TYPE_HOST;
390 reg |= c->q_comp_num;
391 }
392
393 cppi_writel(reg, c->gcr_reg);
394
395 /*
396 * We don't use writel() but __raw_writel() so we have to make sure
397 * that the DMA descriptor in coherent memory made to the main memory
398 * before starting the dma engine.
399 */
400 __iowmb();
401 push_desc_queue(c);
402}
403
404static u32 get_host_pd0(u32 length)
405{
406 u32 reg;
407
408 reg = DESC_TYPE_HOST << DESC_TYPE;
409 reg |= length;
410
411 return reg;
412}
413
414static u32 get_host_pd1(struct cppi41_channel *c)
415{
416 u32 reg;
417
418 reg = 0;
419
420 return reg;
421}
422
423static u32 get_host_pd2(struct cppi41_channel *c)
424{
425 u32 reg;
426
427 reg = DESC_TYPE_USB;
428 reg |= c->q_comp_num;
429
430 return reg;
431}
432
433static u32 get_host_pd3(u32 length)
434{
435 u32 reg;
436
437 /* PD3 = packet size */
438 reg = length;
439
440 return reg;
441}
442
443static u32 get_host_pd6(u32 length)
444{
445 u32 reg;
446
447 /* PD6 buffer size */
448 reg = DESC_PD_COMPLETE;
449 reg |= length;
450
451 return reg;
452}
453
454static u32 get_host_pd4_or_7(u32 addr)
455{
456 u32 reg;
457
458 reg = addr;
459
460 return reg;
461}
462
463static u32 get_host_pd5(void)
464{
465 u32 reg;
466
467 reg = 0;
468
469 return reg;
470}
471
472static struct dma_async_tx_descriptor *cppi41_dma_prep_slave_sg(
473 struct dma_chan *chan, struct scatterlist *sgl, unsigned sg_len,
474 enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
475{
476 struct cppi41_channel *c = to_cpp41_chan(chan);
477 struct cppi41_desc *d;
478 struct scatterlist *sg;
479 unsigned int i;
480 unsigned int num;
481
482 num = 0;
483 d = c->desc;
484 for_each_sg(sgl, sg, sg_len, i) {
485 u32 addr;
486 u32 len;
487
488 /* We need to use more than one desc once musb supports sg */
489 BUG_ON(num > 0);
490 addr = lower_32_bits(sg_dma_address(sg));
491 len = sg_dma_len(sg);
492
493 d->pd0 = get_host_pd0(len);
494 d->pd1 = get_host_pd1(c);
495 d->pd2 = get_host_pd2(c);
496 d->pd3 = get_host_pd3(len);
497 d->pd4 = get_host_pd4_or_7(addr);
498 d->pd5 = get_host_pd5();
499 d->pd6 = get_host_pd6(len);
500 d->pd7 = get_host_pd4_or_7(addr);
501
502 d++;
503 }
504
505 return &c->txd;
506}
507
508static int cpp41_cfg_chan(struct cppi41_channel *c,
509 struct dma_slave_config *cfg)
510{
511 return 0;
512}
513
514static void cppi41_compute_td_desc(struct cppi41_desc *d)
515{
516 d->pd0 = DESC_TYPE_TEARD << DESC_TYPE;
517}
518
519static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num)
520{
521 u32 desc;
522
523 desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num));
524 desc &= ~0x1f;
525 return desc;
526}
527
528static int cppi41_tear_down_chan(struct cppi41_channel *c)
529{
530 struct cppi41_dd *cdd = c->cdd;
531 struct cppi41_desc *td;
532 u32 reg;
533 u32 desc_phys;
534 u32 td_desc_phys;
535
536 td = cdd->cd;
537 td += cdd->first_td_desc;
538
539 td_desc_phys = cdd->descs_phys;
540 td_desc_phys += cdd->first_td_desc * sizeof(struct cppi41_desc);
541
542 if (!c->td_queued) {
543 cppi41_compute_td_desc(td);
544 __iowmb();
545
546 reg = (sizeof(struct cppi41_desc) - 24) / 4;
547 reg |= td_desc_phys;
548 cppi_writel(reg, cdd->qmgr_mem +
549 QMGR_QUEUE_D(cdd->td_queue.submit));
550
551 reg = GCR_CHAN_ENABLE;
552 if (!c->is_tx) {
553 reg |= GCR_STARV_RETRY;
554 reg |= GCR_DESC_TYPE_HOST;
555 reg |= c->q_comp_num;
556 }
557 reg |= GCR_TEARDOWN;
558 cppi_writel(reg, c->gcr_reg);
559 c->td_queued = 1;
560 c->td_retry = 100;
561 }
562
563 if (!c->td_seen) {
564 unsigned td_comp_queue;
565
566 if (c->is_tx)
567 td_comp_queue = cdd->td_queue.complete;
568 else
569 td_comp_queue = c->q_comp_num;
570
571 desc_phys = cppi41_pop_desc(cdd, td_comp_queue);
572 if (desc_phys) {
573 __iormb();
574
575 if (desc_phys == td_desc_phys) {
576 u32 pd0;
577 pd0 = td->pd0;
578 WARN_ON((pd0 >> DESC_TYPE) != DESC_TYPE_TEARD);
579 WARN_ON(!c->is_tx && !(pd0 & TD_DESC_IS_RX));
580 WARN_ON((pd0 & 0x1f) != c->port_num);
581 } else {
582 __WARN();
583 }
584 c->td_seen = 1;
585 }
586 }
587 if (!c->td_desc_seen) {
588 desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);
589 if (desc_phys) {
590 __iormb();
591 WARN_ON(c->desc_phys != desc_phys);
592 c->td_desc_seen = 1;
593 }
594 }
595 c->td_retry--;
596 /*
597 * If the TX descriptor / channel is in use, the caller needs to poke
598 * his TD bit multiple times. After that he hardware releases the
599 * transfer descriptor followed by TD descriptor. Waiting seems not to
600 * cause any difference.
601 * RX seems to be thrown out right away. However once the TearDown
602 * descriptor gets through we are done. If we have seens the transfer
603 * descriptor before the TD we fetch it from enqueue, it has to be
604 * there waiting for us.
605 */
606 if (!c->td_seen && c->td_retry)
607 return -EAGAIN;
608
609 WARN_ON(!c->td_retry);
610 if (!c->td_desc_seen) {
611 desc_phys = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));
612 WARN_ON(!desc_phys);
613 }
614
615 c->td_queued = 0;
616 c->td_seen = 0;
617 c->td_desc_seen = 0;
618 cppi_writel(0, c->gcr_reg);
619 return 0;
620}
621
622static int cppi41_stop_chan(struct dma_chan *chan)
623{
624 struct cppi41_channel *c = to_cpp41_chan(chan);
625 struct cppi41_dd *cdd = c->cdd;
626 u32 desc_num;
627 u32 desc_phys;
628 int ret;
629
630 ret = cppi41_tear_down_chan(c);
631 if (ret)
632 return ret;
633
634 desc_phys = lower_32_bits(c->desc_phys);
635 desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);
636 WARN_ON(!cdd->chan_busy[desc_num]);
637 cdd->chan_busy[desc_num] = NULL;
638
639 return 0;
640}
641
642static int cppi41_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
643 unsigned long arg)
644{
645 struct cppi41_channel *c = to_cpp41_chan(chan);
646 int ret;
647
648 switch (cmd) {
649 case DMA_SLAVE_CONFIG:
650 ret = cpp41_cfg_chan(c, (struct dma_slave_config *) arg);
651 break;
652
653 case DMA_TERMINATE_ALL:
654 ret = cppi41_stop_chan(chan);
655 break;
656
657 default:
658 ret = -ENXIO;
659 break;
660 }
661 return ret;
662}
663
664static void cleanup_chans(struct cppi41_dd *cdd)
665{
666 while (!list_empty(&cdd->ddev.channels)) {
667 struct cppi41_channel *cchan;
668
669 cchan = list_first_entry(&cdd->ddev.channels,
670 struct cppi41_channel, chan.device_node);
671 list_del(&cchan->chan.device_node);
672 kfree(cchan);
673 }
674}
675
676static int cppi41_add_chans(struct platform_device *pdev, struct cppi41_dd *cdd)
677{
678 struct cppi41_channel *cchan;
679 int i;
680 int ret;
681 u32 n_chans;
682
683 ret = of_property_read_u32(pdev->dev.of_node, "#dma-channels",
684 &n_chans);
685 if (ret)
686 return ret;
687 /*
688 * The channels can only be used as TX or as RX. So we add twice
689 * that much dma channels because USB can only do RX or TX.
690 */
691 n_chans *= 2;
692
693 for (i = 0; i < n_chans; i++) {
694 cchan = kzalloc(sizeof(*cchan), GFP_KERNEL);
695 if (!cchan)
696 goto err;
697
698 cchan->cdd = cdd;
699 if (i & 1) {
700 cchan->gcr_reg = cdd->ctrl_mem + DMA_TXGCR(i >> 1);
701 cchan->is_tx = 1;
702 } else {
703 cchan->gcr_reg = cdd->ctrl_mem + DMA_RXGCR(i >> 1);
704 cchan->is_tx = 0;
705 }
706 cchan->port_num = i >> 1;
707 cchan->desc = &cdd->cd[i];
708 cchan->desc_phys = cdd->descs_phys;
709 cchan->desc_phys += i * sizeof(struct cppi41_desc);
710 cchan->chan.device = &cdd->ddev;
711 list_add_tail(&cchan->chan.device_node, &cdd->ddev.channels);
712 }
713 cdd->first_td_desc = n_chans;
714
715 return 0;
716err:
717 cleanup_chans(cdd);
718 return -ENOMEM;
719}
720
721static void purge_descs(struct platform_device *pdev, struct cppi41_dd *cdd)
722{
723 unsigned int mem_decs;
724 int i;
725
726 mem_decs = ALLOC_DECS_NUM * sizeof(struct cppi41_desc);
727
728 for (i = 0; i < DESCS_AREAS; i++) {
729
730 cppi_writel(0, cdd->qmgr_mem + QMGR_MEMBASE(i));
731 cppi_writel(0, cdd->qmgr_mem + QMGR_MEMCTRL(i));
732
733 dma_free_coherent(&pdev->dev, mem_decs, cdd->cd,
734 cdd->descs_phys);
735 }
736}
737
738static void disable_sched(struct cppi41_dd *cdd)
739{
740 cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
741}
742
743static void deinit_cpii41(struct platform_device *pdev, struct cppi41_dd *cdd)
744{
745 disable_sched(cdd);
746
747 purge_descs(pdev, cdd);
748
749 cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
750 cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);
751 dma_free_coherent(&pdev->dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch,
752 cdd->scratch_phys);
753}
754
755static int init_descs(struct platform_device *pdev, struct cppi41_dd *cdd)
756{
757 unsigned int desc_size;
758 unsigned int mem_decs;
759 int i;
760 u32 reg;
761 u32 idx;
762
763 BUILD_BUG_ON(sizeof(struct cppi41_desc) &
764 (sizeof(struct cppi41_desc) - 1));
765 BUILD_BUG_ON(sizeof(struct cppi41_desc) < 32);
766 BUILD_BUG_ON(ALLOC_DECS_NUM < 32);
767
768 desc_size = sizeof(struct cppi41_desc);
769 mem_decs = ALLOC_DECS_NUM * desc_size;
770
771 idx = 0;
772 for (i = 0; i < DESCS_AREAS; i++) {
773
774 reg = idx << QMGR_MEMCTRL_IDX_SH;
775 reg |= (ilog2(desc_size) - 5) << QMGR_MEMCTRL_DESC_SH;
776 reg |= ilog2(ALLOC_DECS_NUM) - 5;
777
778 BUILD_BUG_ON(DESCS_AREAS != 1);
779 cdd->cd = dma_alloc_coherent(&pdev->dev, mem_decs,
780 &cdd->descs_phys, GFP_KERNEL);
781 if (!cdd->cd)
782 return -ENOMEM;
783
784 cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));
785 cppi_writel(reg, cdd->qmgr_mem + QMGR_MEMCTRL(i));
786
787 idx += ALLOC_DECS_NUM;
788 }
789 return 0;
790}
791
792static void init_sched(struct cppi41_dd *cdd)
793{
794 unsigned ch;
795 unsigned word;
796 u32 reg;
797
798 word = 0;
799 cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);
800 for (ch = 0; ch < 15 * 2; ch += 2) {
801
802 reg = SCHED_ENTRY0_CHAN(ch);
803 reg |= SCHED_ENTRY1_CHAN(ch) | SCHED_ENTRY1_IS_RX;
804
805 reg |= SCHED_ENTRY2_CHAN(ch + 1);
806 reg |= SCHED_ENTRY3_CHAN(ch + 1) | SCHED_ENTRY3_IS_RX;
807 cppi_writel(reg, cdd->sched_mem + DMA_SCHED_WORD(word));
808 word++;
809 }
810 reg = 15 * 2 * 2 - 1;
811 reg |= DMA_SCHED_CTRL_EN;
812 cppi_writel(reg, cdd->sched_mem + DMA_SCHED_CTRL);
813}
814
815static int init_cppi41(struct platform_device *pdev, struct cppi41_dd *cdd)
816{
817 int ret;
818
819 BUILD_BUG_ON(QMGR_SCRATCH_SIZE > ((1 << 14) - 1));
820 cdd->qmgr_scratch = dma_alloc_coherent(&pdev->dev, QMGR_SCRATCH_SIZE,
821 &cdd->scratch_phys, GFP_KERNEL);
822 if (!cdd->qmgr_scratch)
823 return -ENOMEM;
824
825 cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);
826 cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE);
827 cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);
828
829 ret = init_descs(pdev, cdd);
830 if (ret)
831 goto err_td;
832
833 cppi_writel(cdd->td_queue.submit, cdd->ctrl_mem + DMA_TDFDQ);
834 init_sched(cdd);
835 return 0;
836err_td:
837 deinit_cpii41(pdev, cdd);
838 return ret;
839}
840
841static struct platform_driver cpp41_dma_driver;
842/*
843 * The param format is:
844 * X Y
845 * X: Port
846 * Y: 0 = RX else TX
847 */
848#define INFO_PORT 0
849#define INFO_IS_TX 1
850
851static bool cpp41_dma_filter_fn(struct dma_chan *chan, void *param)
852{
853 struct cppi41_channel *cchan;
854 struct cppi41_dd *cdd;
855 const struct chan_queues *queues;
856 u32 *num = param;
857
858 if (chan->device->dev->driver != &cpp41_dma_driver.driver)
859 return false;
860
861 cchan = to_cpp41_chan(chan);
862
863 if (cchan->port_num != num[INFO_PORT])
864 return false;
865
866 if (cchan->is_tx && !num[INFO_IS_TX])
867 return false;
868 cdd = cchan->cdd;
869 if (cchan->is_tx)
870 queues = cdd->queues_tx;
871 else
872 queues = cdd->queues_rx;
873
874 BUILD_BUG_ON(ARRAY_SIZE(usb_queues_rx) != ARRAY_SIZE(usb_queues_tx));
875 if (WARN_ON(cchan->port_num > ARRAY_SIZE(usb_queues_rx)))
876 return false;
877
878 cchan->q_num = queues[cchan->port_num].submit;
879 cchan->q_comp_num = queues[cchan->port_num].complete;
880 return true;
881}
882
883static struct of_dma_filter_info cpp41_dma_info = {
884 .filter_fn = cpp41_dma_filter_fn,
885};
886
887static struct dma_chan *cppi41_dma_xlate(struct of_phandle_args *dma_spec,
888 struct of_dma *ofdma)
889{
890 int count = dma_spec->args_count;
891 struct of_dma_filter_info *info = ofdma->of_dma_data;
892
893 if (!info || !info->filter_fn)
894 return NULL;
895
896 if (count != 2)
897 return NULL;
898
899 return dma_request_channel(info->dma_cap, info->filter_fn,
900 &dma_spec->args[0]);
901}
902
903static const struct cppi_glue_infos usb_infos = {
904 .isr = cppi41_irq,
905 .queues_rx = usb_queues_rx,
906 .queues_tx = usb_queues_tx,
907 .td_queue = { .submit = 31, .complete = 0 },
908};
909
910static const struct of_device_id cppi41_dma_ids[] = {
911 { .compatible = "ti,am3359-cppi41", .data = &usb_infos},
912 {},
913};
914MODULE_DEVICE_TABLE(of, cppi41_dma_ids);
915
916static const struct cppi_glue_infos *get_glue_info(struct platform_device *pdev)
917{
918 const struct of_device_id *of_id;
919
920 of_id = of_match_node(cppi41_dma_ids, pdev->dev.of_node);
921 if (!of_id)
922 return NULL;
923 return of_id->data;
924}
925
926static int cppi41_dma_probe(struct platform_device *pdev)
927{
928 struct cppi41_dd *cdd;
929 const struct cppi_glue_infos *glue_info;
930 int irq;
931 int ret;
932
933 glue_info = get_glue_info(pdev);
934 if (!glue_info)
935 return -EINVAL;
936
937 cdd = kzalloc(sizeof(*cdd), GFP_KERNEL);
938 if (!cdd)
939 return -ENOMEM;
940
941 dma_cap_set(DMA_SLAVE, cdd->ddev.cap_mask);
942 cdd->ddev.device_alloc_chan_resources = cppi41_dma_alloc_chan_resources;
943 cdd->ddev.device_free_chan_resources = cppi41_dma_free_chan_resources;
944 cdd->ddev.device_tx_status = cppi41_dma_tx_status;
945 cdd->ddev.device_issue_pending = cppi41_dma_issue_pending;
946 cdd->ddev.device_prep_slave_sg = cppi41_dma_prep_slave_sg;
947 cdd->ddev.device_control = cppi41_dma_control;
948 cdd->ddev.dev = &pdev->dev;
949 INIT_LIST_HEAD(&cdd->ddev.channels);
950 cpp41_dma_info.dma_cap = cdd->ddev.cap_mask;
951
952 cdd->usbss_mem = of_iomap(pdev->dev.of_node, 0);
953 cdd->ctrl_mem = of_iomap(pdev->dev.of_node, 1);
954 cdd->sched_mem = of_iomap(pdev->dev.of_node, 2);
955 cdd->qmgr_mem = of_iomap(pdev->dev.of_node, 3);
956
957 if (!cdd->usbss_mem || !cdd->ctrl_mem || !cdd->sched_mem ||
958 !cdd->qmgr_mem) {
959 ret = -ENXIO;
960 goto err_remap;
961 }
962
963 cdd->queues_rx = glue_info->queues_rx;
964 cdd->queues_tx = glue_info->queues_tx;
965 cdd->td_queue = glue_info->td_queue;
966
967 ret = init_cppi41(pdev, cdd);
968 if (ret)
969 goto err_init_cppi;
970
971 ret = cppi41_add_chans(pdev, cdd);
972 if (ret)
973 goto err_chans;
974
975 irq = irq_of_parse_and_map(pdev->dev.of_node, 0);
976 if (!irq)
977 goto err_irq;
978
979 cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER);
980
981 ret = request_irq(irq, glue_info->isr, IRQF_SHARED,
982 dev_name(&pdev->dev), cdd);
983 if (ret)
984 goto err_irq;
985 cdd->irq = irq;
986
987 ret = dma_async_device_register(&cdd->ddev);
988 if (ret)
989 goto err_dma_reg;
990
991 ret = of_dma_controller_register(pdev->dev.of_node,
992 cppi41_dma_xlate, &cpp41_dma_info);
993 if (ret)
994 goto err_of;
995
996 platform_set_drvdata(pdev, cdd);
997 return 0;
998err_of:
999 dma_async_device_unregister(&cdd->ddev);
1000err_dma_reg:
1001 free_irq(irq, cdd);
1002err_irq:
1003 cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR);
1004 cleanup_chans(cdd);
1005err_chans:
1006 deinit_cpii41(pdev, cdd);
1007err_init_cppi:
1008 iounmap(cdd->usbss_mem);
1009 iounmap(cdd->ctrl_mem);
1010 iounmap(cdd->sched_mem);
1011 iounmap(cdd->qmgr_mem);
1012err_remap:
1013 kfree(cdd);
1014 return ret;
1015}
1016
1017static int cppi41_dma_remove(struct platform_device *pdev)
1018{
1019 struct cppi41_dd *cdd = platform_get_drvdata(pdev);
1020
1021 of_dma_controller_free(pdev->dev.of_node);
1022 dma_async_device_unregister(&cdd->ddev);
1023
1024 cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR);
1025 free_irq(cdd->irq, cdd);
1026 cleanup_chans(cdd);
1027 deinit_cpii41(pdev, cdd);
1028 iounmap(cdd->usbss_mem);
1029 iounmap(cdd->ctrl_mem);
1030 iounmap(cdd->sched_mem);
1031 iounmap(cdd->qmgr_mem);
1032 kfree(cdd);
1033 return 0;
1034}
1035
1036static struct platform_driver cpp41_dma_driver = {
1037 .probe = cppi41_dma_probe,
1038 .remove = cppi41_dma_remove,
1039 .driver = {
1040 .name = "cppi41-dma-engine",
1041 .owner = THIS_MODULE,
1042 .of_match_table = of_match_ptr(cppi41_dma_ids),
1043 },
1044};
1045
1046module_platform_driver(cpp41_dma_driver);
1047MODULE_LICENSE("GPL");
1048MODULE_AUTHOR("Sebastian Andrzej Siewior <bigeasy@linutronix.de>");
diff --git a/drivers/usb/musb/Kconfig b/drivers/usb/musb/Kconfig
index b7257ae038fd..04658d7666e9 100644
--- a/drivers/usb/musb/Kconfig
+++ b/drivers/usb/musb/Kconfig
@@ -129,6 +129,10 @@ config USB_TI_CPPI_DMA
129 help 129 help
130 Enable DMA transfers when TI CPPI DMA is available. 130 Enable DMA transfers when TI CPPI DMA is available.
131 131
132config USB_TI_CPPI41_DMA
133 bool 'TI CPPI 4.1 (AM335x)'
134 depends on ARCH_OMAP
135
132config USB_TUSB_OMAP_DMA 136config USB_TUSB_OMAP_DMA
133 bool 'TUSB 6010' 137 bool 'TUSB 6010'
134 depends on USB_MUSB_TUSB6010 138 depends on USB_MUSB_TUSB6010
diff --git a/drivers/usb/musb/Makefile b/drivers/usb/musb/Makefile
index 52f552c1ba2b..c5ea5c6dc169 100644
--- a/drivers/usb/musb/Makefile
+++ b/drivers/usb/musb/Makefile
@@ -32,3 +32,4 @@ musb_hdrc-$(CONFIG_USB_INVENTRA_DMA) += musbhsdma.o
32musb_hdrc-$(CONFIG_USB_TI_CPPI_DMA) += cppi_dma.o 32musb_hdrc-$(CONFIG_USB_TI_CPPI_DMA) += cppi_dma.o
33musb_hdrc-$(CONFIG_USB_TUSB_OMAP_DMA) += tusb6010_omap.o 33musb_hdrc-$(CONFIG_USB_TUSB_OMAP_DMA) += tusb6010_omap.o
34musb_hdrc-$(CONFIG_USB_UX500_DMA) += ux500_dma.o 34musb_hdrc-$(CONFIG_USB_UX500_DMA) += ux500_dma.o
35musb_hdrc-$(CONFIG_USB_TI_CPPI41_DMA) += musb_cppi41.o
diff --git a/drivers/usb/musb/musb_cppi41.c b/drivers/usb/musb/musb_cppi41.c
new file mode 100644
index 000000000000..a74259ea0306
--- /dev/null
+++ b/drivers/usb/musb/musb_cppi41.c
@@ -0,0 +1,552 @@
1#include <linux/device.h>
2#include <linux/dma-mapping.h>
3#include <linux/dmaengine.h>
4#include <linux/sizes.h>
5#include <linux/platform_device.h>
6#include <linux/of.h>
7
8#include "musb_core.h"
9
10#define RNDIS_REG(x) (0x80 + ((x - 1) * 4))
11
12#define EP_MODE_AUTOREG_NONE 0
13#define EP_MODE_AUTOREG_ALL_NEOP 1
14#define EP_MODE_AUTOREG_ALWAYS 3
15
16#define EP_MODE_DMA_TRANSPARENT 0
17#define EP_MODE_DMA_RNDIS 1
18#define EP_MODE_DMA_GEN_RNDIS 3
19
20#define USB_CTRL_TX_MODE 0x70
21#define USB_CTRL_RX_MODE 0x74
22#define USB_CTRL_AUTOREQ 0xd0
23#define USB_TDOWN 0xd8
24
25struct cppi41_dma_channel {
26 struct dma_channel channel;
27 struct cppi41_dma_controller *controller;
28 struct musb_hw_ep *hw_ep;
29 struct dma_chan *dc;
30 dma_cookie_t cookie;
31 u8 port_num;
32 u8 is_tx;
33 u8 is_allocated;
34 u8 usb_toggle;
35
36 dma_addr_t buf_addr;
37 u32 total_len;
38 u32 prog_len;
39 u32 transferred;
40 u32 packet_sz;
41};
42
43#define MUSB_DMA_NUM_CHANNELS 15
44
45struct cppi41_dma_controller {
46 struct dma_controller controller;
47 struct cppi41_dma_channel rx_channel[MUSB_DMA_NUM_CHANNELS];
48 struct cppi41_dma_channel tx_channel[MUSB_DMA_NUM_CHANNELS];
49 struct musb *musb;
50 u32 rx_mode;
51 u32 tx_mode;
52 u32 auto_req;
53};
54
55static void save_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
56{
57 u16 csr;
58 u8 toggle;
59
60 if (cppi41_channel->is_tx)
61 return;
62 if (!is_host_active(cppi41_channel->controller->musb))
63 return;
64
65 csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR);
66 toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
67
68 cppi41_channel->usb_toggle = toggle;
69}
70
71static void update_rx_toggle(struct cppi41_dma_channel *cppi41_channel)
72{
73 u16 csr;
74 u8 toggle;
75
76 if (cppi41_channel->is_tx)
77 return;
78 if (!is_host_active(cppi41_channel->controller->musb))
79 return;
80
81 csr = musb_readw(cppi41_channel->hw_ep->regs, MUSB_RXCSR);
82 toggle = csr & MUSB_RXCSR_H_DATATOGGLE ? 1 : 0;
83
84 /*
85 * AM335x Advisory 1.0.13: Due to internal synchronisation error the
86 * data toggle may reset from DATA1 to DATA0 during receiving data from
87 * more than one endpoint.
88 */
89 if (!toggle && toggle == cppi41_channel->usb_toggle) {
90 csr |= MUSB_RXCSR_H_DATATOGGLE | MUSB_RXCSR_H_WR_DATATOGGLE;
91 musb_writew(cppi41_channel->hw_ep->regs, MUSB_RXCSR, csr);
92 dev_dbg(cppi41_channel->controller->musb->controller,
93 "Restoring DATA1 toggle.\n");
94 }
95
96 cppi41_channel->usb_toggle = toggle;
97}
98
99static void cppi41_dma_callback(void *private_data)
100{
101 struct dma_channel *channel = private_data;
102 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
103 struct musb_hw_ep *hw_ep = cppi41_channel->hw_ep;
104 struct musb *musb = hw_ep->musb;
105 unsigned long flags;
106 struct dma_tx_state txstate;
107 u32 transferred;
108
109 spin_lock_irqsave(&musb->lock, flags);
110
111 dmaengine_tx_status(cppi41_channel->dc, cppi41_channel->cookie,
112 &txstate);
113 transferred = cppi41_channel->prog_len - txstate.residue;
114 cppi41_channel->transferred += transferred;
115
116 dev_dbg(musb->controller, "DMA transfer done on hw_ep=%d bytes=%d/%d\n",
117 hw_ep->epnum, cppi41_channel->transferred,
118 cppi41_channel->total_len);
119
120 update_rx_toggle(cppi41_channel);
121
122 if (cppi41_channel->transferred == cppi41_channel->total_len ||
123 transferred < cppi41_channel->packet_sz) {
124
125 /* done, complete */
126 cppi41_channel->channel.actual_len =
127 cppi41_channel->transferred;
128 cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
129 musb_dma_completion(musb, hw_ep->epnum, cppi41_channel->is_tx);
130 } else {
131 /* next iteration, reload */
132 struct dma_chan *dc = cppi41_channel->dc;
133 struct dma_async_tx_descriptor *dma_desc;
134 enum dma_transfer_direction direction;
135 u16 csr;
136 u32 remain_bytes;
137 void __iomem *epio = cppi41_channel->hw_ep->regs;
138
139 cppi41_channel->buf_addr += cppi41_channel->packet_sz;
140
141 remain_bytes = cppi41_channel->total_len;
142 remain_bytes -= cppi41_channel->transferred;
143 remain_bytes = min(remain_bytes, cppi41_channel->packet_sz);
144 cppi41_channel->prog_len = remain_bytes;
145
146 direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV
147 : DMA_DEV_TO_MEM;
148 dma_desc = dmaengine_prep_slave_single(dc,
149 cppi41_channel->buf_addr,
150 remain_bytes,
151 direction,
152 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
153 if (WARN_ON(!dma_desc))
154 return;
155
156 dma_desc->callback = cppi41_dma_callback;
157 dma_desc->callback_param = channel;
158 cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
159 dma_async_issue_pending(dc);
160
161 if (!cppi41_channel->is_tx) {
162 csr = musb_readw(epio, MUSB_RXCSR);
163 csr |= MUSB_RXCSR_H_REQPKT;
164 musb_writew(epio, MUSB_RXCSR, csr);
165 }
166 }
167 spin_unlock_irqrestore(&musb->lock, flags);
168}
169
170static u32 update_ep_mode(unsigned ep, unsigned mode, u32 old)
171{
172 unsigned shift;
173
174 shift = (ep - 1) * 2;
175 old &= ~(3 << shift);
176 old |= mode << shift;
177 return old;
178}
179
180static void cppi41_set_dma_mode(struct cppi41_dma_channel *cppi41_channel,
181 unsigned mode)
182{
183 struct cppi41_dma_controller *controller = cppi41_channel->controller;
184 u32 port;
185 u32 new_mode;
186 u32 old_mode;
187
188 if (cppi41_channel->is_tx)
189 old_mode = controller->tx_mode;
190 else
191 old_mode = controller->rx_mode;
192 port = cppi41_channel->port_num;
193 new_mode = update_ep_mode(port, mode, old_mode);
194
195 if (new_mode == old_mode)
196 return;
197 if (cppi41_channel->is_tx) {
198 controller->tx_mode = new_mode;
199 musb_writel(controller->musb->ctrl_base, USB_CTRL_TX_MODE,
200 new_mode);
201 } else {
202 controller->rx_mode = new_mode;
203 musb_writel(controller->musb->ctrl_base, USB_CTRL_RX_MODE,
204 new_mode);
205 }
206}
207
208static void cppi41_set_autoreq_mode(struct cppi41_dma_channel *cppi41_channel,
209 unsigned mode)
210{
211 struct cppi41_dma_controller *controller = cppi41_channel->controller;
212 u32 port;
213 u32 new_mode;
214 u32 old_mode;
215
216 old_mode = controller->auto_req;
217 port = cppi41_channel->port_num;
218 new_mode = update_ep_mode(port, mode, old_mode);
219
220 if (new_mode == old_mode)
221 return;
222 controller->auto_req = new_mode;
223 musb_writel(controller->musb->ctrl_base, USB_CTRL_AUTOREQ, new_mode);
224}
225
226static bool cppi41_configure_channel(struct dma_channel *channel,
227 u16 packet_sz, u8 mode,
228 dma_addr_t dma_addr, u32 len)
229{
230 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
231 struct dma_chan *dc = cppi41_channel->dc;
232 struct dma_async_tx_descriptor *dma_desc;
233 enum dma_transfer_direction direction;
234 struct musb *musb = cppi41_channel->controller->musb;
235 unsigned use_gen_rndis = 0;
236
237 dev_dbg(musb->controller,
238 "configure ep%d/%x packet_sz=%d, mode=%d, dma_addr=0x%llx, len=%d is_tx=%d\n",
239 cppi41_channel->port_num, RNDIS_REG(cppi41_channel->port_num),
240 packet_sz, mode, (unsigned long long) dma_addr,
241 len, cppi41_channel->is_tx);
242
243 cppi41_channel->buf_addr = dma_addr;
244 cppi41_channel->total_len = len;
245 cppi41_channel->transferred = 0;
246 cppi41_channel->packet_sz = packet_sz;
247
248 /*
249 * Due to AM335x' Advisory 1.0.13 we are not allowed to transfer more
250 * than max packet size at a time.
251 */
252 if (cppi41_channel->is_tx)
253 use_gen_rndis = 1;
254
255 if (use_gen_rndis) {
256 /* RNDIS mode */
257 if (len > packet_sz) {
258 musb_writel(musb->ctrl_base,
259 RNDIS_REG(cppi41_channel->port_num), len);
260 /* gen rndis */
261 cppi41_set_dma_mode(cppi41_channel,
262 EP_MODE_DMA_GEN_RNDIS);
263
264 /* auto req */
265 cppi41_set_autoreq_mode(cppi41_channel,
266 EP_MODE_AUTOREG_ALL_NEOP);
267 } else {
268 musb_writel(musb->ctrl_base,
269 RNDIS_REG(cppi41_channel->port_num), 0);
270 cppi41_set_dma_mode(cppi41_channel,
271 EP_MODE_DMA_TRANSPARENT);
272 cppi41_set_autoreq_mode(cppi41_channel,
273 EP_MODE_AUTOREG_NONE);
274 }
275 } else {
276 /* fallback mode */
277 cppi41_set_dma_mode(cppi41_channel, EP_MODE_DMA_TRANSPARENT);
278 cppi41_set_autoreq_mode(cppi41_channel, EP_MODE_AUTOREG_NONE);
279 len = min_t(u32, packet_sz, len);
280 }
281 cppi41_channel->prog_len = len;
282 direction = cppi41_channel->is_tx ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
283 dma_desc = dmaengine_prep_slave_single(dc, dma_addr, len, direction,
284 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
285 if (!dma_desc)
286 return false;
287
288 dma_desc->callback = cppi41_dma_callback;
289 dma_desc->callback_param = channel;
290 cppi41_channel->cookie = dma_desc->tx_submit(dma_desc);
291
292 save_rx_toggle(cppi41_channel);
293 dma_async_issue_pending(dc);
294 return true;
295}
296
297static struct dma_channel *cppi41_dma_channel_allocate(struct dma_controller *c,
298 struct musb_hw_ep *hw_ep, u8 is_tx)
299{
300 struct cppi41_dma_controller *controller = container_of(c,
301 struct cppi41_dma_controller, controller);
302 struct cppi41_dma_channel *cppi41_channel = NULL;
303 u8 ch_num = hw_ep->epnum - 1;
304
305 if (ch_num >= MUSB_DMA_NUM_CHANNELS)
306 return NULL;
307
308 if (is_tx)
309 cppi41_channel = &controller->tx_channel[ch_num];
310 else
311 cppi41_channel = &controller->rx_channel[ch_num];
312
313 if (!cppi41_channel->dc)
314 return NULL;
315
316 if (cppi41_channel->is_allocated)
317 return NULL;
318
319 cppi41_channel->hw_ep = hw_ep;
320 cppi41_channel->is_allocated = 1;
321
322 return &cppi41_channel->channel;
323}
324
325static void cppi41_dma_channel_release(struct dma_channel *channel)
326{
327 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
328
329 if (cppi41_channel->is_allocated) {
330 cppi41_channel->is_allocated = 0;
331 channel->status = MUSB_DMA_STATUS_FREE;
332 channel->actual_len = 0;
333 }
334}
335
336static int cppi41_dma_channel_program(struct dma_channel *channel,
337 u16 packet_sz, u8 mode,
338 dma_addr_t dma_addr, u32 len)
339{
340 int ret;
341
342 BUG_ON(channel->status == MUSB_DMA_STATUS_UNKNOWN ||
343 channel->status == MUSB_DMA_STATUS_BUSY);
344
345 channel->status = MUSB_DMA_STATUS_BUSY;
346 channel->actual_len = 0;
347 ret = cppi41_configure_channel(channel, packet_sz, mode, dma_addr, len);
348 if (!ret)
349 channel->status = MUSB_DMA_STATUS_FREE;
350
351 return ret;
352}
353
354static int cppi41_is_compatible(struct dma_channel *channel, u16 maxpacket,
355 void *buf, u32 length)
356{
357 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
358 struct cppi41_dma_controller *controller = cppi41_channel->controller;
359 struct musb *musb = controller->musb;
360
361 if (is_host_active(musb)) {
362 WARN_ON(1);
363 return 1;
364 }
365 return 0;
366}
367
368static int cppi41_dma_channel_abort(struct dma_channel *channel)
369{
370 struct cppi41_dma_channel *cppi41_channel = channel->private_data;
371 struct cppi41_dma_controller *controller = cppi41_channel->controller;
372 struct musb *musb = controller->musb;
373 void __iomem *epio = cppi41_channel->hw_ep->regs;
374 int tdbit;
375 int ret;
376 unsigned is_tx;
377 u16 csr;
378
379 is_tx = cppi41_channel->is_tx;
380 dev_dbg(musb->controller, "abort channel=%d, is_tx=%d\n",
381 cppi41_channel->port_num, is_tx);
382
383 if (cppi41_channel->channel.status == MUSB_DMA_STATUS_FREE)
384 return 0;
385
386 if (is_tx) {
387 csr = musb_readw(epio, MUSB_TXCSR);
388 csr &= ~MUSB_TXCSR_DMAENAB;
389 musb_writew(epio, MUSB_TXCSR, csr);
390 } else {
391 csr = musb_readw(epio, MUSB_RXCSR);
392 csr &= ~(MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_DMAENAB);
393 musb_writew(epio, MUSB_RXCSR, csr);
394
395 csr = musb_readw(epio, MUSB_RXCSR);
396 if (csr & MUSB_RXCSR_RXPKTRDY) {
397 csr |= MUSB_RXCSR_FLUSHFIFO;
398 musb_writew(epio, MUSB_RXCSR, csr);
399 musb_writew(epio, MUSB_RXCSR, csr);
400 }
401 }
402
403 tdbit = 1 << cppi41_channel->port_num;
404 if (is_tx)
405 tdbit <<= 16;
406
407 do {
408 musb_writel(musb->ctrl_base, USB_TDOWN, tdbit);
409 ret = dmaengine_terminate_all(cppi41_channel->dc);
410 } while (ret == -EAGAIN);
411
412 musb_writel(musb->ctrl_base, USB_TDOWN, tdbit);
413
414 if (is_tx) {
415 csr = musb_readw(epio, MUSB_TXCSR);
416 if (csr & MUSB_TXCSR_TXPKTRDY) {
417 csr |= MUSB_TXCSR_FLUSHFIFO;
418 musb_writew(epio, MUSB_TXCSR, csr);
419 }
420 }
421
422 cppi41_channel->channel.status = MUSB_DMA_STATUS_FREE;
423 return 0;
424}
425
426static void cppi41_release_all_dma_chans(struct cppi41_dma_controller *ctrl)
427{
428 struct dma_chan *dc;
429 int i;
430
431 for (i = 0; i < MUSB_DMA_NUM_CHANNELS; i++) {
432 dc = ctrl->tx_channel[i].dc;
433 if (dc)
434 dma_release_channel(dc);
435 dc = ctrl->rx_channel[i].dc;
436 if (dc)
437 dma_release_channel(dc);
438 }
439}
440
441static void cppi41_dma_controller_stop(struct cppi41_dma_controller *controller)
442{
443 cppi41_release_all_dma_chans(controller);
444}
445
446static int cppi41_dma_controller_start(struct cppi41_dma_controller *controller)
447{
448 struct musb *musb = controller->musb;
449 struct device *dev = musb->controller;
450 struct device_node *np = dev->of_node;
451 struct cppi41_dma_channel *cppi41_channel;
452 int count;
453 int i;
454 int ret;
455
456 count = of_property_count_strings(np, "dma-names");
457 if (count < 0)
458 return count;
459
460 for (i = 0; i < count; i++) {
461 struct dma_chan *dc;
462 struct dma_channel *musb_dma;
463 const char *str;
464 unsigned is_tx;
465 unsigned int port;
466
467 ret = of_property_read_string_index(np, "dma-names", i, &str);
468 if (ret)
469 goto err;
470 if (!strncmp(str, "tx", 2))
471 is_tx = 1;
472 else if (!strncmp(str, "rx", 2))
473 is_tx = 0;
474 else {
475 dev_err(dev, "Wrong dmatype %s\n", str);
476 goto err;
477 }
478 ret = kstrtouint(str + 2, 0, &port);
479 if (ret)
480 goto err;
481
482 if (port > MUSB_DMA_NUM_CHANNELS || !port)
483 goto err;
484 if (is_tx)
485 cppi41_channel = &controller->tx_channel[port - 1];
486 else
487 cppi41_channel = &controller->rx_channel[port - 1];
488
489 cppi41_channel->controller = controller;
490 cppi41_channel->port_num = port;
491 cppi41_channel->is_tx = is_tx;
492
493 musb_dma = &cppi41_channel->channel;
494 musb_dma->private_data = cppi41_channel;
495 musb_dma->status = MUSB_DMA_STATUS_FREE;
496 musb_dma->max_len = SZ_4M;
497
498 dc = dma_request_slave_channel(dev, str);
499 if (!dc) {
500 dev_err(dev, "Falied to request %s.\n", str);
501 goto err;
502 }
503 cppi41_channel->dc = dc;
504 }
505 return 0;
506err:
507 cppi41_release_all_dma_chans(controller);
508 return -EINVAL;
509}
510
511void dma_controller_destroy(struct dma_controller *c)
512{
513 struct cppi41_dma_controller *controller = container_of(c,
514 struct cppi41_dma_controller, controller);
515
516 cppi41_dma_controller_stop(controller);
517 kfree(controller);
518}
519
520struct dma_controller *dma_controller_create(struct musb *musb,
521 void __iomem *base)
522{
523 struct cppi41_dma_controller *controller;
524 int ret;
525
526 if (!musb->controller->of_node) {
527 dev_err(musb->controller, "Need DT for the DMA engine.\n");
528 return NULL;
529 }
530
531 controller = kzalloc(sizeof(*controller), GFP_KERNEL);
532 if (!controller)
533 goto kzalloc_fail;
534
535 controller->musb = musb;
536
537 controller->controller.channel_alloc = cppi41_dma_channel_allocate;
538 controller->controller.channel_release = cppi41_dma_channel_release;
539 controller->controller.channel_program = cppi41_dma_channel_program;
540 controller->controller.channel_abort = cppi41_dma_channel_abort;
541 controller->controller.is_compatible = cppi41_is_compatible;
542
543 ret = cppi41_dma_controller_start(controller);
544 if (ret)
545 goto plat_get_fail;
546 return &controller->controller;
547
548plat_get_fail:
549 kfree(controller);
550kzalloc_fail:
551 return NULL;
552}
diff --git a/drivers/usb/musb/musb_dma.h b/drivers/usb/musb/musb_dma.h
index c8e67fde2156..1345a4ff041a 100644
--- a/drivers/usb/musb/musb_dma.h
+++ b/drivers/usb/musb/musb_dma.h
@@ -68,7 +68,7 @@ struct musb_hw_ep;
68#define is_dma_capable() (1) 68#define is_dma_capable() (1)
69#endif 69#endif
70 70
71#ifdef CONFIG_USB_TI_CPPI_DMA 71#if defined(CONFIG_USB_TI_CPPI_DMA) || defined(CONFIG_USB_TI_CPPI41_DMA)
72#define is_cppi_enabled() 1 72#define is_cppi_enabled() 1
73#else 73#else
74#define is_cppi_enabled() 0 74#define is_cppi_enabled() 0
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-30 09:29:24 -0400