aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm/common/Kconfig3
-rw-r--r--arch/arm/common/Makefile1
-rw-r--r--arch/arm/common/pl330.c1959
-rw-r--r--arch/arm/include/asm/hardware/pl330.h217
-rw-r--r--drivers/dma/Kconfig1
-rw-r--r--drivers/dma/pl330.c2114
-rw-r--r--include/linux/amba/pl330.h1
7 files changed, 2113 insertions, 2183 deletions
diff --git a/arch/arm/common/Kconfig b/arch/arm/common/Kconfig
index 81a933eb090..a43c758ff0d 100644
--- a/arch/arm/common/Kconfig
+++ b/arch/arm/common/Kconfig
@@ -24,9 +24,6 @@ config ARM_VIC_NR
24config ICST 24config ICST
25 bool 25 bool
26 26
27config PL330
28 bool
29
30config SA1111 27config SA1111
31 bool 28 bool
32 select DMABOUNCE if !ARCH_PXA 29 select DMABOUNCE if !ARCH_PXA
diff --git a/arch/arm/common/Makefile b/arch/arm/common/Makefile
index 6ea9b6f3607..107d6d97c6e 100644
--- a/arch/arm/common/Makefile
+++ b/arch/arm/common/Makefile
@@ -5,7 +5,6 @@
5obj-$(CONFIG_ARM_GIC) += gic.o 5obj-$(CONFIG_ARM_GIC) += gic.o
6obj-$(CONFIG_ARM_VIC) += vic.o 6obj-$(CONFIG_ARM_VIC) += vic.o
7obj-$(CONFIG_ICST) += icst.o 7obj-$(CONFIG_ICST) += icst.o
8obj-$(CONFIG_PL330) += pl330.o
9obj-$(CONFIG_SA1111) += sa1111.o 8obj-$(CONFIG_SA1111) += sa1111.o
10obj-$(CONFIG_PCI_HOST_VIA82C505) += via82c505.o 9obj-$(CONFIG_PCI_HOST_VIA82C505) += via82c505.o
11obj-$(CONFIG_DMABOUNCE) += dmabounce.o 10obj-$(CONFIG_DMABOUNCE) += dmabounce.o
diff --git a/arch/arm/common/pl330.c b/arch/arm/common/pl330.c
deleted file mode 100644
index d8e44a43047..00000000000
--- a/arch/arm/common/pl330.c
+++ /dev/null
@@ -1,1959 +0,0 @@
1/* linux/arch/arm/common/pl330.c
2 *
3 * Copyright (C) 2010 Samsung Electronics Co Ltd.
4 * Jaswinder Singh <jassi.brar@samsung.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 */
20
21#include <linux/kernel.h>
22#include <linux/init.h>
23#include <linux/slab.h>
24#include <linux/module.h>
25#include <linux/string.h>
26#include <linux/io.h>
27#include <linux/delay.h>
28#include <linux/interrupt.h>
29#include <linux/dma-mapping.h>
30
31#include <asm/hardware/pl330.h>
32
33/* Register and Bit field Definitions */
34#define DS 0x0
35#define DS_ST_STOP 0x0
36#define DS_ST_EXEC 0x1
37#define DS_ST_CMISS 0x2
38#define DS_ST_UPDTPC 0x3
39#define DS_ST_WFE 0x4
40#define DS_ST_ATBRR 0x5
41#define DS_ST_QBUSY 0x6
42#define DS_ST_WFP 0x7
43#define DS_ST_KILL 0x8
44#define DS_ST_CMPLT 0x9
45#define DS_ST_FLTCMP 0xe
46#define DS_ST_FAULT 0xf
47
48#define DPC 0x4
49#define INTEN 0x20
50#define ES 0x24
51#define INTSTATUS 0x28
52#define INTCLR 0x2c
53#define FSM 0x30
54#define FSC 0x34
55#define FTM 0x38
56
57#define _FTC 0x40
58#define FTC(n) (_FTC + (n)*0x4)
59
60#define _CS 0x100
61#define CS(n) (_CS + (n)*0x8)
62#define CS_CNS (1 << 21)
63
64#define _CPC 0x104
65#define CPC(n) (_CPC + (n)*0x8)
66
67#define _SA 0x400
68#define SA(n) (_SA + (n)*0x20)
69
70#define _DA 0x404
71#define DA(n) (_DA + (n)*0x20)
72
73#define _CC 0x408
74#define CC(n) (_CC + (n)*0x20)
75
76#define CC_SRCINC (1 << 0)
77#define CC_DSTINC (1 << 14)
78#define CC_SRCPRI (1 << 8)
79#define CC_DSTPRI (1 << 22)
80#define CC_SRCNS (1 << 9)
81#define CC_DSTNS (1 << 23)
82#define CC_SRCIA (1 << 10)
83#define CC_DSTIA (1 << 24)
84#define CC_SRCBRSTLEN_SHFT 4
85#define CC_DSTBRSTLEN_SHFT 18
86#define CC_SRCBRSTSIZE_SHFT 1
87#define CC_DSTBRSTSIZE_SHFT 15
88#define CC_SRCCCTRL_SHFT 11
89#define CC_SRCCCTRL_MASK 0x7
90#define CC_DSTCCTRL_SHFT 25
91#define CC_DRCCCTRL_MASK 0x7
92#define CC_SWAP_SHFT 28
93
94#define _LC0 0x40c
95#define LC0(n) (_LC0 + (n)*0x20)
96
97#define _LC1 0x410
98#define LC1(n) (_LC1 + (n)*0x20)
99
100#define DBGSTATUS 0xd00
101#define DBG_BUSY (1 << 0)
102
103#define DBGCMD 0xd04
104#define DBGINST0 0xd08
105#define DBGINST1 0xd0c
106
107#define CR0 0xe00
108#define CR1 0xe04
109#define CR2 0xe08
110#define CR3 0xe0c
111#define CR4 0xe10
112#define CRD 0xe14
113
114#define PERIPH_ID 0xfe0
115#define PCELL_ID 0xff0
116
117#define CR0_PERIPH_REQ_SET (1 << 0)
118#define CR0_BOOT_EN_SET (1 << 1)
119#define CR0_BOOT_MAN_NS (1 << 2)
120#define CR0_NUM_CHANS_SHIFT 4
121#define CR0_NUM_CHANS_MASK 0x7
122#define CR0_NUM_PERIPH_SHIFT 12
123#define CR0_NUM_PERIPH_MASK 0x1f
124#define CR0_NUM_EVENTS_SHIFT 17
125#define CR0_NUM_EVENTS_MASK 0x1f
126
127#define CR1_ICACHE_LEN_SHIFT 0
128#define CR1_ICACHE_LEN_MASK 0x7
129#define CR1_NUM_ICACHELINES_SHIFT 4
130#define CR1_NUM_ICACHELINES_MASK 0xf
131
132#define CRD_DATA_WIDTH_SHIFT 0
133#define CRD_DATA_WIDTH_MASK 0x7
134#define CRD_WR_CAP_SHIFT 4
135#define CRD_WR_CAP_MASK 0x7
136#define CRD_WR_Q_DEP_SHIFT 8
137#define CRD_WR_Q_DEP_MASK 0xf
138#define CRD_RD_CAP_SHIFT 12
139#define CRD_RD_CAP_MASK 0x7
140#define CRD_RD_Q_DEP_SHIFT 16
141#define CRD_RD_Q_DEP_MASK 0xf
142#define CRD_DATA_BUFF_SHIFT 20
143#define CRD_DATA_BUFF_MASK 0x3ff
144
145#define PART 0x330
146#define DESIGNER 0x41
147#define REVISION 0x0
148#define INTEG_CFG 0x0
149#define PERIPH_ID_VAL ((PART << 0) | (DESIGNER << 12))
150
151#define PCELL_ID_VAL 0xb105f00d
152
153#define PL330_STATE_STOPPED (1 << 0)
154#define PL330_STATE_EXECUTING (1 << 1)
155#define PL330_STATE_WFE (1 << 2)
156#define PL330_STATE_FAULTING (1 << 3)
157#define PL330_STATE_COMPLETING (1 << 4)
158#define PL330_STATE_WFP (1 << 5)
159#define PL330_STATE_KILLING (1 << 6)
160#define PL330_STATE_FAULT_COMPLETING (1 << 7)
161#define PL330_STATE_CACHEMISS (1 << 8)
162#define PL330_STATE_UPDTPC (1 << 9)
163#define PL330_STATE_ATBARRIER (1 << 10)
164#define PL330_STATE_QUEUEBUSY (1 << 11)
165#define PL330_STATE_INVALID (1 << 15)
166
167#define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \
168 | PL330_STATE_WFE | PL330_STATE_FAULTING)
169
170#define CMD_DMAADDH 0x54
171#define CMD_DMAEND 0x00
172#define CMD_DMAFLUSHP 0x35
173#define CMD_DMAGO 0xa0
174#define CMD_DMALD 0x04
175#define CMD_DMALDP 0x25
176#define CMD_DMALP 0x20
177#define CMD_DMALPEND 0x28
178#define CMD_DMAKILL 0x01
179#define CMD_DMAMOV 0xbc
180#define CMD_DMANOP 0x18
181#define CMD_DMARMB 0x12
182#define CMD_DMASEV 0x34
183#define CMD_DMAST 0x08
184#define CMD_DMASTP 0x29
185#define CMD_DMASTZ 0x0c
186#define CMD_DMAWFE 0x36
187#define CMD_DMAWFP 0x30
188#define CMD_DMAWMB 0x13
189
190#define SZ_DMAADDH 3
191#define SZ_DMAEND 1
192#define SZ_DMAFLUSHP 2
193#define SZ_DMALD 1
194#define SZ_DMALDP 2
195#define SZ_DMALP 2
196#define SZ_DMALPEND 2
197#define SZ_DMAKILL 1
198#define SZ_DMAMOV 6
199#define SZ_DMANOP 1
200#define SZ_DMARMB 1
201#define SZ_DMASEV 2
202#define SZ_DMAST 1
203#define SZ_DMASTP 2
204#define SZ_DMASTZ 1
205#define SZ_DMAWFE 2
206#define SZ_DMAWFP 2
207#define SZ_DMAWMB 1
208#define SZ_DMAGO 6
209
210#define BRST_LEN(ccr) ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1)
211#define BRST_SIZE(ccr) (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7))
212
213#define BYTE_TO_BURST(b, ccr) ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr))
214#define BURST_TO_BYTE(c, ccr) ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr))
215
216/*
217 * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req
218 * at 1byte/burst for P<->M and M<->M respectively.
219 * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req
220 * should be enough for P<->M and M<->M respectively.
221 */
222#define MCODE_BUFF_PER_REQ 256
223
224/* If the _pl330_req is available to the client */
225#define IS_FREE(req) (*((u8 *)((req)->mc_cpu)) == CMD_DMAEND)
226
227/* Use this _only_ to wait on transient states */
228#define UNTIL(t, s) while (!(_state(t) & (s))) cpu_relax();
229
230#ifdef PL330_DEBUG_MCGEN
231static unsigned cmd_line;
232#define PL330_DBGCMD_DUMP(off, x...) do { \
233 printk("%x:", cmd_line); \
234 printk(x); \
235 cmd_line += off; \
236 } while (0)
237#define PL330_DBGMC_START(addr) (cmd_line = addr)
238#else
239#define PL330_DBGCMD_DUMP(off, x...) do {} while (0)
240#define PL330_DBGMC_START(addr) do {} while (0)
241#endif
242
243struct _xfer_spec {
244 u32 ccr;
245 struct pl330_req *r;
246 struct pl330_xfer *x;
247};
248
249enum dmamov_dst {
250 SAR = 0,
251 CCR,
252 DAR,
253};
254
255enum pl330_dst {
256 SRC = 0,
257 DST,
258};
259
260enum pl330_cond {
261 SINGLE,
262 BURST,
263 ALWAYS,
264};
265
266struct _pl330_req {
267 u32 mc_bus;
268 void *mc_cpu;
269 /* Number of bytes taken to setup MC for the req */
270 u32 mc_len;
271 struct pl330_req *r;
272 /* Hook to attach to DMAC's list of reqs with due callback */
273 struct list_head rqd;
274};
275
276/* ToBeDone for tasklet */
277struct _pl330_tbd {
278 bool reset_dmac;
279 bool reset_mngr;
280 u8 reset_chan;
281};
282
283/* A DMAC Thread */
284struct pl330_thread {
285 u8 id;
286 int ev;
287 /* If the channel is not yet acquired by any client */
288 bool free;
289 /* Parent DMAC */
290 struct pl330_dmac *dmac;
291 /* Only two at a time */
292 struct _pl330_req req[2];
293 /* Index of the last enqueued request */
294 unsigned lstenq;
295 /* Index of the last submitted request or -1 if the DMA is stopped */
296 int req_running;
297};
298
299enum pl330_dmac_state {
300 UNINIT,
301 INIT,
302 DYING,
303};
304
305/* A DMAC */
306struct pl330_dmac {
307 spinlock_t lock;
308 /* Holds list of reqs with due callbacks */
309 struct list_head req_done;
310 /* Pointer to platform specific stuff */
311 struct pl330_info *pinfo;
312 /* Maximum possible events/irqs */
313 int events[32];
314 /* BUS address of MicroCode buffer */
315 u32 mcode_bus;
316 /* CPU address of MicroCode buffer */
317 void *mcode_cpu;
318 /* List of all Channel threads */
319 struct pl330_thread *channels;
320 /* Pointer to the MANAGER thread */
321 struct pl330_thread *manager;
322 /* To handle bad news in interrupt */
323 struct tasklet_struct tasks;
324 struct _pl330_tbd dmac_tbd;
325 /* State of DMAC operation */
326 enum pl330_dmac_state state;
327};
328
329static inline void _callback(struct pl330_req *r, enum pl330_op_err err)
330{
331 if (r && r->xfer_cb)
332 r->xfer_cb(r->token, err);
333}
334
335static inline bool _queue_empty(struct pl330_thread *thrd)
336{
337 return (IS_FREE(&thrd->req[0]) && IS_FREE(&thrd->req[1]))
338 ? true : false;
339}
340
341static inline bool _queue_full(struct pl330_thread *thrd)
342{
343 return (IS_FREE(&thrd->req[0]) || IS_FREE(&thrd->req[1]))
344 ? false : true;
345}
346
347static inline bool is_manager(struct pl330_thread *thrd)
348{
349 struct pl330_dmac *pl330 = thrd->dmac;
350
351 /* MANAGER is indexed at the end */
352 if (thrd->id == pl330->pinfo->pcfg.num_chan)
353 return true;
354 else
355 return false;
356}
357
358/* If manager of the thread is in Non-Secure mode */
359static inline bool _manager_ns(struct pl330_thread *thrd)
360{
361 struct pl330_dmac *pl330 = thrd->dmac;
362
363 return (pl330->pinfo->pcfg.mode & DMAC_MODE_NS) ? true : false;
364}
365
366static inline u32 get_id(struct pl330_info *pi, u32 off)
367{
368 void __iomem *regs = pi->base;
369 u32 id = 0;
370
371 id |= (readb(regs + off + 0x0) << 0);
372 id |= (readb(regs + off + 0x4) << 8);
373 id |= (readb(regs + off + 0x8) << 16);
374 id |= (readb(regs + off + 0xc) << 24);
375
376 return id;
377}
378
379static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[],
380 enum pl330_dst da, u16 val)
381{
382 if (dry_run)
383 return SZ_DMAADDH;
384
385 buf[0] = CMD_DMAADDH;
386 buf[0] |= (da << 1);
387 *((u16 *)&buf[1]) = val;
388
389 PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n",
390 da == 1 ? "DA" : "SA", val);
391
392 return SZ_DMAADDH;
393}
394
395static inline u32 _emit_END(unsigned dry_run, u8 buf[])
396{
397 if (dry_run)
398 return SZ_DMAEND;
399
400 buf[0] = CMD_DMAEND;
401
402 PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n");
403
404 return SZ_DMAEND;
405}
406
407static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri)
408{
409 if (dry_run)
410 return SZ_DMAFLUSHP;
411
412 buf[0] = CMD_DMAFLUSHP;
413
414 peri &= 0x1f;
415 peri <<= 3;
416 buf[1] = peri;
417
418 PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3);
419
420 return SZ_DMAFLUSHP;
421}
422
423static inline u32 _emit_LD(unsigned dry_run, u8 buf[], enum pl330_cond cond)
424{
425 if (dry_run)
426 return SZ_DMALD;
427
428 buf[0] = CMD_DMALD;
429
430 if (cond == SINGLE)
431 buf[0] |= (0 << 1) | (1 << 0);
432 else if (cond == BURST)
433 buf[0] |= (1 << 1) | (1 << 0);
434
435 PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n",
436 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
437
438 return SZ_DMALD;
439}
440
441static inline u32 _emit_LDP(unsigned dry_run, u8 buf[],
442 enum pl330_cond cond, u8 peri)
443{
444 if (dry_run)
445 return SZ_DMALDP;
446
447 buf[0] = CMD_DMALDP;
448
449 if (cond == BURST)
450 buf[0] |= (1 << 1);
451
452 peri &= 0x1f;
453 peri <<= 3;
454 buf[1] = peri;
455
456 PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n",
457 cond == SINGLE ? 'S' : 'B', peri >> 3);
458
459 return SZ_DMALDP;
460}
461
462static inline u32 _emit_LP(unsigned dry_run, u8 buf[],
463 unsigned loop, u8 cnt)
464{
465 if (dry_run)
466 return SZ_DMALP;
467
468 buf[0] = CMD_DMALP;
469
470 if (loop)
471 buf[0] |= (1 << 1);
472
473 cnt--; /* DMAC increments by 1 internally */
474 buf[1] = cnt;
475
476 PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt);
477
478 return SZ_DMALP;
479}
480
481struct _arg_LPEND {
482 enum pl330_cond cond;
483 bool forever;
484 unsigned loop;
485 u8 bjump;
486};
487
488static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[],
489 const struct _arg_LPEND *arg)
490{
491 enum pl330_cond cond = arg->cond;
492 bool forever = arg->forever;
493 unsigned loop = arg->loop;
494 u8 bjump = arg->bjump;
495
496 if (dry_run)
497 return SZ_DMALPEND;
498
499 buf[0] = CMD_DMALPEND;
500
501 if (loop)
502 buf[0] |= (1 << 2);
503
504 if (!forever)
505 buf[0] |= (1 << 4);
506
507 if (cond == SINGLE)
508 buf[0] |= (0 << 1) | (1 << 0);
509 else if (cond == BURST)
510 buf[0] |= (1 << 1) | (1 << 0);
511
512 buf[1] = bjump;
513
514 PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n",
515 forever ? "FE" : "END",
516 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'),
517 loop ? '1' : '0',
518 bjump);
519
520 return SZ_DMALPEND;
521}
522
523static inline u32 _emit_KILL(unsigned dry_run, u8 buf[])
524{
525 if (dry_run)
526 return SZ_DMAKILL;
527
528 buf[0] = CMD_DMAKILL;
529
530 return SZ_DMAKILL;
531}
532
533static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],
534 enum dmamov_dst dst, u32 val)
535{
536 if (dry_run)
537 return SZ_DMAMOV;
538
539 buf[0] = CMD_DMAMOV;
540 buf[1] = dst;
541 *((u32 *)&buf[2]) = val;
542
543 PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",
544 dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);
545
546 return SZ_DMAMOV;
547}
548
549static inline u32 _emit_NOP(unsigned dry_run, u8 buf[])
550{
551 if (dry_run)
552 return SZ_DMANOP;
553
554 buf[0] = CMD_DMANOP;
555
556 PL330_DBGCMD_DUMP(SZ_DMANOP, "\tDMANOP\n");
557
558 return SZ_DMANOP;
559}
560
561static inline u32 _emit_RMB(unsigned dry_run, u8 buf[])
562{
563 if (dry_run)
564 return SZ_DMARMB;
565
566 buf[0] = CMD_DMARMB;
567
568 PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n");
569
570 return SZ_DMARMB;
571}
572
573static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev)
574{
575 if (dry_run)
576 return SZ_DMASEV;
577
578 buf[0] = CMD_DMASEV;
579
580 ev &= 0x1f;
581 ev <<= 3;
582 buf[1] = ev;
583
584 PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3);
585
586 return SZ_DMASEV;
587}
588
589static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond)
590{
591 if (dry_run)
592 return SZ_DMAST;
593
594 buf[0] = CMD_DMAST;
595
596 if (cond == SINGLE)
597 buf[0] |= (0 << 1) | (1 << 0);
598 else if (cond == BURST)
599 buf[0] |= (1 << 1) | (1 << 0);
600
601 PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n",
602 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
603
604 return SZ_DMAST;
605}
606
607static inline u32 _emit_STP(unsigned dry_run, u8 buf[],
608 enum pl330_cond cond, u8 peri)
609{
610 if (dry_run)
611 return SZ_DMASTP;
612
613 buf[0] = CMD_DMASTP;
614
615 if (cond == BURST)
616 buf[0] |= (1 << 1);
617
618 peri &= 0x1f;
619 peri <<= 3;
620 buf[1] = peri;
621
622 PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n",
623 cond == SINGLE ? 'S' : 'B', peri >> 3);
624
625 return SZ_DMASTP;
626}
627
628static inline u32 _emit_STZ(unsigned dry_run, u8 buf[])
629{
630 if (dry_run)
631 return SZ_DMASTZ;
632
633 buf[0] = CMD_DMASTZ;
634
635 PL330_DBGCMD_DUMP(SZ_DMASTZ, "\tDMASTZ\n");
636
637 return SZ_DMASTZ;
638}
639
640static inline u32 _emit_WFE(unsigned dry_run, u8 buf[], u8 ev,
641 unsigned invalidate)
642{
643 if (dry_run)
644 return SZ_DMAWFE;
645
646 buf[0] = CMD_DMAWFE;
647
648 ev &= 0x1f;
649 ev <<= 3;
650 buf[1] = ev;
651
652 if (invalidate)
653 buf[1] |= (1 << 1);
654
655 PL330_DBGCMD_DUMP(SZ_DMAWFE, "\tDMAWFE %u%s\n",
656 ev >> 3, invalidate ? ", I" : "");
657
658 return SZ_DMAWFE;
659}
660
661static inline u32 _emit_WFP(unsigned dry_run, u8 buf[],
662 enum pl330_cond cond, u8 peri)
663{
664 if (dry_run)
665 return SZ_DMAWFP;
666
667 buf[0] = CMD_DMAWFP;
668
669 if (cond == SINGLE)
670 buf[0] |= (0 << 1) | (0 << 0);
671 else if (cond == BURST)
672 buf[0] |= (1 << 1) | (0 << 0);
673 else
674 buf[0] |= (0 << 1) | (1 << 0);
675
676 peri &= 0x1f;
677 peri <<= 3;
678 buf[1] = peri;
679
680 PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n",
681 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3);
682
683 return SZ_DMAWFP;
684}
685
686static inline u32 _emit_WMB(unsigned dry_run, u8 buf[])
687{
688 if (dry_run)
689 return SZ_DMAWMB;
690
691 buf[0] = CMD_DMAWMB;
692
693 PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n");
694
695 return SZ_DMAWMB;
696}
697
698struct _arg_GO {
699 u8 chan;
700 u32 addr;
701 unsigned ns;
702};
703
704static inline u32 _emit_GO(unsigned dry_run, u8 buf[],
705 const struct _arg_GO *arg)
706{
707 u8 chan = arg->chan;
708 u32 addr = arg->addr;
709 unsigned ns = arg->ns;
710
711 if (dry_run)
712 return SZ_DMAGO;
713
714 buf[0] = CMD_DMAGO;
715 buf[0] |= (ns << 1);
716
717 buf[1] = chan & 0x7;
718
719 *((u32 *)&buf[2]) = addr;
720
721 return SZ_DMAGO;
722}
723
724#define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t)
725
726/* Returns Time-Out */
727static bool _until_dmac_idle(struct pl330_thread *thrd)
728{
729 void __iomem *regs = thrd->dmac->pinfo->base;
730 unsigned long loops = msecs_to_loops(5);
731
732 do {
733 /* Until Manager is Idle */
734 if (!(readl(regs + DBGSTATUS) & DBG_BUSY))
735 break;
736
737 cpu_relax();
738 } while (--loops);
739
740 if (!loops)
741 return true;
742
743 return false;
744}
745
746static inline void _execute_DBGINSN(struct pl330_thread *thrd,
747 u8 insn[], bool as_manager)
748{
749 void __iomem *regs = thrd->dmac->pinfo->base;
750 u32 val;
751
752 val = (insn[0] << 16) | (insn[1] << 24);
753 if (!as_manager) {
754 val |= (1 << 0);
755 val |= (thrd->id << 8); /* Channel Number */
756 }
757 writel(val, regs + DBGINST0);
758
759 val = *((u32 *)&insn[2]);
760 writel(val, regs + DBGINST1);
761
762 /* If timed out due to halted state-machine */
763 if (_until_dmac_idle(thrd)) {
764 dev_err(thrd->dmac->pinfo->dev, "DMAC halted!\n");
765 return;
766 }
767
768 /* Get going */
769 writel(0, regs + DBGCMD);
770}
771
772/*
773 * Mark a _pl330_req as free.
774 * We do it by writing DMAEND as the first instruction
775 * because no valid request is going to have DMAEND as
776 * its first instruction to execute.
777 */
778static void mark_free(struct pl330_thread *thrd, int idx)
779{
780 struct _pl330_req *req = &thrd->req[idx];
781
782 _emit_END(0, req->mc_cpu);
783 req->mc_len = 0;
784
785 thrd->req_running = -1;
786}
787
788static inline u32 _state(struct pl330_thread *thrd)
789{
790 void __iomem *regs = thrd->dmac->pinfo->base;
791 u32 val;
792
793 if (is_manager(thrd))
794 val = readl(regs + DS) & 0xf;
795 else
796 val = readl(regs + CS(thrd->id)) & 0xf;
797
798 switch (val) {
799 case DS_ST_STOP:
800 return PL330_STATE_STOPPED;
801 case DS_ST_EXEC:
802 return PL330_STATE_EXECUTING;
803 case DS_ST_CMISS:
804 return PL330_STATE_CACHEMISS;
805 case DS_ST_UPDTPC:
806 return PL330_STATE_UPDTPC;
807 case DS_ST_WFE:
808 return PL330_STATE_WFE;
809 case DS_ST_FAULT:
810 return PL330_STATE_FAULTING;
811 case DS_ST_ATBRR:
812 if (is_manager(thrd))
813 return PL330_STATE_INVALID;
814 else
815 return PL330_STATE_ATBARRIER;
816 case DS_ST_QBUSY:
817 if (is_manager(thrd))
818 return PL330_STATE_INVALID;
819 else
820 return PL330_STATE_QUEUEBUSY;
821 case DS_ST_WFP:
822 if (is_manager(thrd))
823 return PL330_STATE_INVALID;
824 else
825 return PL330_STATE_WFP;
826 case DS_ST_KILL:
827 if (is_manager(thrd))
828 return PL330_STATE_INVALID;
829 else
830 return PL330_STATE_KILLING;
831 case DS_ST_CMPLT:
832 if (is_manager(thrd))
833 return PL330_STATE_INVALID;
834 else
835 return PL330_STATE_COMPLETING;
836 case DS_ST_FLTCMP:
837 if (is_manager(thrd))
838 return PL330_STATE_INVALID;
839 else
840 return PL330_STATE_FAULT_COMPLETING;
841 default:
842 return PL330_STATE_INVALID;
843 }
844}
845
846static void _stop(struct pl330_thread *thrd)
847{
848 void __iomem *regs = thrd->dmac->pinfo->base;
849 u8 insn[6] = {0, 0, 0, 0, 0, 0};
850
851 if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
852 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
853
854 /* Return if nothing needs to be done */
855 if (_state(thrd) == PL330_STATE_COMPLETING
856 || _state(thrd) == PL330_STATE_KILLING
857 || _state(thrd) == PL330_STATE_STOPPED)
858 return;
859
860 _emit_KILL(0, insn);
861
862 /* Stop generating interrupts for SEV */
863 writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN);
864
865 _execute_DBGINSN(thrd, insn, is_manager(thrd));
866}
867
868/* Start doing req 'idx' of thread 'thrd' */
869static bool _trigger(struct pl330_thread *thrd)
870{
871 void __iomem *regs = thrd->dmac->pinfo->base;
872 struct _pl330_req *req;
873 struct pl330_req *r;
874 struct _arg_GO go;
875 unsigned ns;
876 u8 insn[6] = {0, 0, 0, 0, 0, 0};
877 int idx;
878
879 /* Return if already ACTIVE */
880 if (_state(thrd) != PL330_STATE_STOPPED)
881 return true;
882
883 idx = 1 - thrd->lstenq;
884 if (!IS_FREE(&thrd->req[idx]))
885 req = &thrd->req[idx];
886 else {
887 idx = thrd->lstenq;
888 if (!IS_FREE(&thrd->req[idx]))
889 req = &thrd->req[idx];
890 else
891 req = NULL;
892 }
893
894 /* Return if no request */
895 if (!req || !req->r)
896 return true;
897
898 r = req->r;
899
900 if (r->cfg)
901 ns = r->cfg->nonsecure ? 1 : 0;
902 else if (readl(regs + CS(thrd->id)) & CS_CNS)
903 ns = 1;
904 else
905 ns = 0;
906
907 /* See 'Abort Sources' point-4 at Page 2-25 */
908 if (_manager_ns(thrd) && !ns)
909 dev_info(thrd->dmac->pinfo->dev, "%s:%d Recipe for ABORT!\n",
910 __func__, __LINE__);
911
912 go.chan = thrd->id;
913 go.addr = req->mc_bus;
914 go.ns = ns;
915 _emit_GO(0, insn, &go);
916
917 /* Set to generate interrupts for SEV */
918 writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN);
919
920 /* Only manager can execute GO */
921 _execute_DBGINSN(thrd, insn, true);
922
923 thrd->req_running = idx;
924
925 return true;
926}
927
928static bool _start(struct pl330_thread *thrd)
929{
930 switch (_state(thrd)) {
931 case PL330_STATE_FAULT_COMPLETING:
932 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
933
934 if (_state(thrd) == PL330_STATE_KILLING)
935 UNTIL(thrd, PL330_STATE_STOPPED)
936
937 case PL330_STATE_FAULTING:
938 _stop(thrd);
939
940 case PL330_STATE_KILLING:
941 case PL330_STATE_COMPLETING:
942 UNTIL(thrd, PL330_STATE_STOPPED)
943
944 case PL330_STATE_STOPPED:
945 return _trigger(thrd);
946
947 case PL330_STATE_WFP:
948 case PL330_STATE_QUEUEBUSY:
949 case PL330_STATE_ATBARRIER:
950 case PL330_STATE_UPDTPC:
951 case PL330_STATE_CACHEMISS:
952 case PL330_STATE_EXECUTING:
953 return true;
954
955 case PL330_STATE_WFE: /* For RESUME, nothing yet */
956 default:
957 return false;
958 }
959}
960
961static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
962 const struct _xfer_spec *pxs, int cyc)
963{
964 int off = 0;
965
966 while (cyc--) {
967 off += _emit_LD(dry_run, &buf[off], ALWAYS);
968 off += _emit_RMB(dry_run, &buf[off]);
969 off += _emit_ST(dry_run, &buf[off], ALWAYS);
970 off += _emit_WMB(dry_run, &buf[off]);
971 }
972
973 return off;
974}
975
976static inline int _ldst_devtomem(unsigned dry_run, u8 buf[],
977 const struct _xfer_spec *pxs, int cyc)
978{
979 int off = 0;
980
981 while (cyc--) {
982 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri);
983 off += _emit_LDP(dry_run, &buf[off], SINGLE, pxs->r->peri);
984 off += _emit_ST(dry_run, &buf[off], ALWAYS);
985 off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri);
986 }
987
988 return off;
989}
990
991static inline int _ldst_memtodev(unsigned dry_run, u8 buf[],
992 const struct _xfer_spec *pxs, int cyc)
993{
994 int off = 0;
995
996 while (cyc--) {
997 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri);
998 off += _emit_LD(dry_run, &buf[off], ALWAYS);
999 off += _emit_STP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1000 off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri);
1001 }
1002
1003 return off;
1004}
1005
1006static int _bursts(unsigned dry_run, u8 buf[],
1007 const struct _xfer_spec *pxs, int cyc)
1008{
1009 int off = 0;
1010
1011 switch (pxs->r->rqtype) {
1012 case MEMTODEV:
1013 off += _ldst_memtodev(dry_run, &buf[off], pxs, cyc);
1014 break;
1015 case DEVTOMEM:
1016 off += _ldst_devtomem(dry_run, &buf[off], pxs, cyc);
1017 break;
1018 case MEMTOMEM:
1019 off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
1020 break;
1021 default:
1022 off += 0x40000000; /* Scare off the Client */
1023 break;
1024 }
1025
1026 return off;
1027}
1028
1029/* Returns bytes consumed and updates bursts */
1030static inline int _loop(unsigned dry_run, u8 buf[],
1031 unsigned long *bursts, const struct _xfer_spec *pxs)
1032{
1033 int cyc, cycmax, szlp, szlpend, szbrst, off;
1034 unsigned lcnt0, lcnt1, ljmp0, ljmp1;
1035 struct _arg_LPEND lpend;
1036
1037 /* Max iterations possible in DMALP is 256 */
1038 if (*bursts >= 256*256) {
1039 lcnt1 = 256;
1040 lcnt0 = 256;
1041 cyc = *bursts / lcnt1 / lcnt0;
1042 } else if (*bursts > 256) {
1043 lcnt1 = 256;
1044 lcnt0 = *bursts / lcnt1;
1045 cyc = 1;
1046 } else {
1047 lcnt1 = *bursts;
1048 lcnt0 = 0;
1049 cyc = 1;
1050 }
1051
1052 szlp = _emit_LP(1, buf, 0, 0);
1053 szbrst = _bursts(1, buf, pxs, 1);
1054
1055 lpend.cond = ALWAYS;
1056 lpend.forever = false;
1057 lpend.loop = 0;
1058 lpend.bjump = 0;
1059 szlpend = _emit_LPEND(1, buf, &lpend);
1060
1061 if (lcnt0) {
1062 szlp *= 2;
1063 szlpend *= 2;
1064 }
1065
1066 /*
1067 * Max bursts that we can unroll due to limit on the
1068 * size of backward jump that can be encoded in DMALPEND
1069 * which is 8-bits and hence 255
1070 */
1071 cycmax = (255 - (szlp + szlpend)) / szbrst;
1072
1073 cyc = (cycmax < cyc) ? cycmax : cyc;
1074
1075 off = 0;
1076
1077 if (lcnt0) {
1078 off += _emit_LP(dry_run, &buf[off], 0, lcnt0);
1079 ljmp0 = off;
1080 }
1081
1082 off += _emit_LP(dry_run, &buf[off], 1, lcnt1);
1083 ljmp1 = off;
1084
1085 off += _bursts(dry_run, &buf[off], pxs, cyc);
1086
1087 lpend.cond = ALWAYS;
1088 lpend.forever = false;
1089 lpend.loop = 1;
1090 lpend.bjump = off - ljmp1;
1091 off += _emit_LPEND(dry_run, &buf[off], &lpend);
1092
1093 if (lcnt0) {
1094 lpend.cond = ALWAYS;
1095 lpend.forever = false;
1096 lpend.loop = 0;
1097 lpend.bjump = off - ljmp0;
1098 off += _emit_LPEND(dry_run, &buf[off], &lpend);
1099 }
1100
1101 *bursts = lcnt1 * cyc;
1102 if (lcnt0)
1103 *bursts *= lcnt0;
1104
1105 return off;
1106}
1107
1108static inline int _setup_loops(unsigned dry_run, u8 buf[],
1109 const struct _xfer_spec *pxs)
1110{
1111 struct pl330_xfer *x = pxs->x;
1112 u32 ccr = pxs->ccr;
1113 unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
1114 int off = 0;
1115
1116 while (bursts) {
1117 c = bursts;
1118 off += _loop(dry_run, &buf[off], &c, pxs);
1119 bursts -= c;
1120 }
1121
1122 return off;
1123}
1124
1125static inline int _setup_xfer(unsigned dry_run, u8 buf[],
1126 const struct _xfer_spec *pxs)
1127{
1128 struct pl330_xfer *x = pxs->x;
1129 int off = 0;
1130
1131 /* DMAMOV SAR, x->src_addr */
1132 off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr);
1133 /* DMAMOV DAR, x->dst_addr */
1134 off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr);
1135
1136 /* Setup Loop(s) */
1137 off += _setup_loops(dry_run, &buf[off], pxs);
1138
1139 return off;
1140}
1141
1142/*
1143 * A req is a sequence of one or more xfer units.
1144 * Returns the number of bytes taken to setup the MC for the req.
1145 */
1146static int _setup_req(unsigned dry_run, struct pl330_thread *thrd,
1147 unsigned index, struct _xfer_spec *pxs)
1148{
1149 struct _pl330_req *req = &thrd->req[index];
1150 struct pl330_xfer *x;
1151 u8 *buf = req->mc_cpu;
1152 int off = 0;
1153
1154 PL330_DBGMC_START(req->mc_bus);
1155
1156 /* DMAMOV CCR, ccr */
1157 off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
1158
1159 x = pxs->r->x;
1160 do {
1161 /* Error if xfer length is not aligned at burst size */
1162 if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr)))
1163 return -EINVAL;
1164
1165 pxs->x = x;
1166 off += _setup_xfer(dry_run, &buf[off], pxs);
1167
1168 x = x->next;
1169 } while (x);
1170
1171 /* DMASEV peripheral/event */
1172 off += _emit_SEV(dry_run, &buf[off], thrd->ev);
1173 /* DMAEND */
1174 off += _emit_END(dry_run, &buf[off]);
1175
1176 return off;
1177}
1178
1179static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc)
1180{
1181 u32 ccr = 0;
1182
1183 if (rqc->src_inc)
1184 ccr |= CC_SRCINC;
1185
1186 if (rqc->dst_inc)
1187 ccr |= CC_DSTINC;
1188
1189 /* We set same protection levels for Src and DST for now */
1190 if (rqc->privileged)
1191 ccr |= CC_SRCPRI | CC_DSTPRI;
1192 if (rqc->nonsecure)
1193 ccr |= CC_SRCNS | CC_DSTNS;
1194 if (rqc->insnaccess)
1195 ccr |= CC_SRCIA | CC_DSTIA;
1196
1197 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT);
1198 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT);
1199
1200 ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT);
1201 ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT);
1202
1203 ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT);
1204 ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT);
1205
1206 ccr |= (rqc->swap << CC_SWAP_SHFT);
1207
1208 return ccr;
1209}
1210
1211static inline bool _is_valid(u32 ccr)
1212{
1213 enum pl330_dstcachectrl dcctl;
1214 enum pl330_srccachectrl scctl;
1215
1216 dcctl = (ccr >> CC_DSTCCTRL_SHFT) & CC_DRCCCTRL_MASK;
1217 scctl = (ccr >> CC_SRCCCTRL_SHFT) & CC_SRCCCTRL_MASK;
1218
1219 if (dcctl == DINVALID1 || dcctl == DINVALID2
1220 || scctl == SINVALID1 || scctl == SINVALID2)
1221 return false;
1222 else
1223 return true;
1224}
1225
1226/*
1227 * Submit a list of xfers after which the client wants notification.
1228 * Client is not notified after each xfer unit, just once after all
1229 * xfer units are done or some error occurs.
1230 */
1231int pl330_submit_req(void *ch_id, struct pl330_req *r)
1232{
1233 struct pl330_thread *thrd = ch_id;
1234 struct pl330_dmac *pl330;
1235 struct pl330_info *pi;
1236 struct _xfer_spec xs;
1237 unsigned long flags;
1238 void __iomem *regs;
1239 unsigned idx;
1240 u32 ccr;
1241 int ret = 0;
1242
1243 /* No Req or Unacquired Channel or DMAC */
1244 if (!r || !thrd || thrd->free)
1245 return -EINVAL;
1246
1247 pl330 = thrd->dmac;
1248 pi = pl330->pinfo;
1249 regs = pi->base;
1250
1251 if (pl330->state == DYING
1252 || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
1253 dev_info(thrd->dmac->pinfo->dev, "%s:%d\n",
1254 __func__, __LINE__);
1255 return -EAGAIN;
1256 }
1257
1258 /* If request for non-existing peripheral */
1259 if (r->rqtype != MEMTOMEM && r->peri >= pi->pcfg.num_peri) {
1260 dev_info(thrd->dmac->pinfo->dev,
1261 "%s:%d Invalid peripheral(%u)!\n",
1262 __func__, __LINE__, r->peri);
1263 return -EINVAL;
1264 }
1265
1266 spin_lock_irqsave(&pl330->lock, flags);
1267
1268 if (_queue_full(thrd)) {
1269 ret = -EAGAIN;
1270 goto xfer_exit;
1271 }
1272
1273 /* Prefer Secure Channel */
1274 if (!_manager_ns(thrd))
1275 r->cfg->nonsecure = 0;
1276 else
1277 r->cfg->nonsecure = 1;
1278
1279 /* Use last settings, if not provided */
1280 if (r->cfg)
1281 ccr = _prepare_ccr(r->cfg);
1282 else
1283 ccr = readl(regs + CC(thrd->id));
1284
1285 /* If this req doesn't have valid xfer settings */
1286 if (!_is_valid(ccr)) {
1287 ret = -EINVAL;
1288 dev_info(thrd->dmac->pinfo->dev, "%s:%d Invalid CCR(%x)!\n",
1289 __func__, __LINE__, ccr);
1290 goto xfer_exit;
1291 }
1292
1293 idx = IS_FREE(&thrd->req[0]) ? 0 : 1;
1294
1295 xs.ccr = ccr;
1296 xs.r = r;
1297
1298 /* First dry run to check if req is acceptable */
1299 ret = _setup_req(1, thrd, idx, &xs);
1300 if (ret < 0)
1301 goto xfer_exit;
1302
1303 if (ret > pi->mcbufsz / 2) {
1304 dev_info(thrd->dmac->pinfo->dev,
1305 "%s:%d Trying increasing mcbufsz\n",
1306 __func__, __LINE__);
1307 ret = -ENOMEM;
1308 goto xfer_exit;
1309 }
1310
1311 /* Hook the request */
1312 thrd->lstenq = idx;
1313 thrd->req[idx].mc_len = _setup_req(0, thrd, idx, &xs);
1314 thrd->req[idx].r = r;
1315
1316 ret = 0;
1317
1318xfer_exit:
1319 spin_unlock_irqrestore(&pl330->lock, flags);
1320
1321 return ret;
1322}
1323EXPORT_SYMBOL(pl330_submit_req);
1324
1325static void pl330_dotask(unsigned long data)
1326{
1327 struct pl330_dmac *pl330 = (struct pl330_dmac *) data;
1328 struct pl330_info *pi = pl330->pinfo;
1329 unsigned long flags;
1330 int i;
1331
1332 spin_lock_irqsave(&pl330->lock, flags);
1333
1334 /* The DMAC itself gone nuts */
1335 if (pl330->dmac_tbd.reset_dmac) {
1336 pl330->state = DYING;
1337 /* Reset the manager too */
1338 pl330->dmac_tbd.reset_mngr = true;
1339 /* Clear the reset flag */
1340 pl330->dmac_tbd.reset_dmac = false;
1341 }
1342
1343 if (pl330->dmac_tbd.reset_mngr) {
1344 _stop(pl330->manager);
1345 /* Reset all channels */
1346 pl330->dmac_tbd.reset_chan = (1 << pi->pcfg.num_chan) - 1;
1347 /* Clear the reset flag */
1348 pl330->dmac_tbd.reset_mngr = false;
1349 }
1350
1351 for (i = 0; i < pi->pcfg.num_chan; i++) {
1352
1353 if (pl330->dmac_tbd.reset_chan & (1 << i)) {
1354 struct pl330_thread *thrd = &pl330->channels[i];
1355 void __iomem *regs = pi->base;
1356 enum pl330_op_err err;
1357
1358 _stop(thrd);
1359
1360 if (readl(regs + FSC) & (1 << thrd->id))
1361 err = PL330_ERR_FAIL;
1362 else
1363 err = PL330_ERR_ABORT;
1364
1365 spin_unlock_irqrestore(&pl330->lock, flags);
1366
1367 _callback(thrd->req[1 - thrd->lstenq].r, err);
1368 _callback(thrd->req[thrd->lstenq].r, err);
1369
1370 spin_lock_irqsave(&pl330->lock, flags);
1371
1372 thrd->req[0].r = NULL;
1373 thrd->req[1].r = NULL;
1374 mark_free(thrd, 0);
1375 mark_free(thrd, 1);
1376
1377 /* Clear the reset flag */
1378 pl330->dmac_tbd.reset_chan &= ~(1 << i);
1379 }
1380 }
1381
1382 spin_unlock_irqrestore(&pl330->lock, flags);
1383
1384 return;
1385}
1386
1387/* Returns 1 if state was updated, 0 otherwise */
1388int pl330_update(const struct pl330_info *pi)
1389{
1390 struct _pl330_req *rqdone;
1391 struct pl330_dmac *pl330;
1392 unsigned long flags;
1393 void __iomem *regs;
1394 u32 val;
1395 int id, ev, ret = 0;
1396
1397 if (!pi || !pi->pl330_data)
1398 return 0;
1399
1400 regs = pi->base;
1401 pl330 = pi->pl330_data;
1402
1403 spin_lock_irqsave(&pl330->lock, flags);
1404
1405 val = readl(regs + FSM) & 0x1;
1406 if (val)
1407 pl330->dmac_tbd.reset_mngr = true;
1408 else
1409 pl330->dmac_tbd.reset_mngr = false;
1410
1411 val = readl(regs + FSC) & ((1 << pi->pcfg.num_chan) - 1);
1412 pl330->dmac_tbd.reset_chan |= val;
1413 if (val) {
1414 int i = 0;
1415 while (i < pi->pcfg.num_chan) {
1416 if (val & (1 << i)) {
1417 dev_info(pi->dev,
1418 "Reset Channel-%d\t CS-%x FTC-%x\n",
1419 i, readl(regs + CS(i)),
1420 readl(regs + FTC(i)));
1421 _stop(&pl330->channels[i]);
1422 }
1423 i++;
1424 }
1425 }
1426
1427 /* Check which event happened i.e, thread notified */
1428 val = readl(regs + ES);
1429 if (pi->pcfg.num_events < 32
1430 && val & ~((1 << pi->pcfg.num_events) - 1)) {
1431 pl330->dmac_tbd.reset_dmac = true;
1432 dev_err(pi->dev, "%s:%d Unexpected!\n", __func__, __LINE__);
1433 ret = 1;
1434 goto updt_exit;
1435 }
1436
1437 for (ev = 0; ev < pi->pcfg.num_events; ev++) {
1438 if (val & (1 << ev)) { /* Event occurred */
1439 struct pl330_thread *thrd;
1440 u32 inten = readl(regs + INTEN);
1441 int active;
1442
1443 /* Clear the event */
1444 if (inten & (1 << ev))
1445 writel(1 << ev, regs + INTCLR);
1446
1447 ret = 1;
1448
1449 id = pl330->events[ev];
1450
1451 thrd = &pl330->channels[id];
1452
1453 active = thrd->req_running;
1454 if (active == -1) /* Aborted */
1455 continue;
1456
1457 rqdone = &thrd->req[active];
1458 mark_free(thrd, active);
1459
1460 /* Get going again ASAP */
1461 _start(thrd);
1462
1463 /* For now, just make a list of callbacks to be done */
1464 list_add_tail(&rqdone->rqd, &pl330->req_done);
1465 }
1466 }
1467
1468 /* Now that we are in no hurry, do the callbacks */
1469 while (!list_empty(&pl330->req_done)) {
1470 struct pl330_req *r;
1471
1472 rqdone = container_of(pl330->req_done.next,
1473 struct _pl330_req, rqd);
1474
1475 list_del_init(&rqdone->rqd);
1476
1477 /* Detach the req */
1478 r = rqdone->r;
1479 rqdone->r = NULL;
1480
1481 spin_unlock_irqrestore(&pl330->lock, flags);
1482 _callback(r, PL330_ERR_NONE);
1483 spin_lock_irqsave(&pl330->lock, flags);
1484 }
1485
1486updt_exit:
1487 spin_unlock_irqrestore(&pl330->lock, flags);
1488
1489 if (pl330->dmac_tbd.reset_dmac
1490 || pl330->dmac_tbd.reset_mngr
1491 || pl330->dmac_tbd.reset_chan) {
1492 ret = 1;
1493 tasklet_schedule(&pl330->tasks);
1494 }
1495
1496 return ret;
1497}
1498EXPORT_SYMBOL(pl330_update);
1499
1500int pl330_chan_ctrl(void *ch_id, enum pl330_chan_op op)
1501{
1502 struct pl330_thread *thrd = ch_id;
1503 struct pl330_dmac *pl330;
1504 unsigned long flags;
1505 int ret = 0, active = thrd->req_running;
1506
1507 if (!thrd || thrd->free || thrd->dmac->state == DYING)
1508 return -EINVAL;
1509
1510 pl330 = thrd->dmac;
1511
1512 spin_lock_irqsave(&pl330->lock, flags);
1513
1514 switch (op) {
1515 case PL330_OP_FLUSH:
1516 /* Make sure the channel is stopped */
1517 _stop(thrd);
1518
1519 thrd->req[0].r = NULL;
1520 thrd->req[1].r = NULL;
1521 mark_free(thrd, 0);
1522 mark_free(thrd, 1);
1523 break;
1524
1525 case PL330_OP_ABORT:
1526 /* Make sure the channel is stopped */
1527 _stop(thrd);
1528
1529 /* ABORT is only for the active req */
1530 if (active == -1)
1531 break;
1532
1533 thrd->req[active].r = NULL;
1534 mark_free(thrd, active);
1535
1536 /* Start the next */
1537 case PL330_OP_START:
1538 if ((active == -1) && !_start(thrd))
1539 ret = -EIO;
1540 break;
1541
1542 default:
1543 ret = -EINVAL;
1544 }
1545
1546 spin_unlock_irqrestore(&pl330->lock, flags);
1547 return ret;
1548}
1549EXPORT_SYMBOL(pl330_chan_ctrl);
1550
1551int pl330_chan_status(void *ch_id, struct pl330_chanstatus *pstatus)
1552{
1553 struct pl330_thread *thrd = ch_id;
1554 struct pl330_dmac *pl330;
1555 struct pl330_info *pi;
1556 void __iomem *regs;
1557 int active;
1558 u32 val;
1559
1560 if (!pstatus || !thrd || thrd->free)
1561 return -EINVAL;
1562
1563 pl330 = thrd->dmac;
1564 pi = pl330->pinfo;
1565 regs = pi->base;
1566
1567 /* The client should remove the DMAC and add again */
1568 if (pl330->state == DYING)
1569 pstatus->dmac_halted = true;
1570 else
1571 pstatus->dmac_halted = false;
1572
1573 val = readl(regs + FSC);
1574 if (val & (1 << thrd->id))
1575 pstatus->faulting = true;
1576 else
1577 pstatus->faulting = false;
1578
1579 active = thrd->req_running;
1580
1581 if (active == -1) {
1582 /* Indicate that the thread is not running */
1583 pstatus->top_req = NULL;
1584 pstatus->wait_req = NULL;
1585 } else {
1586 pstatus->top_req = thrd->req[active].r;
1587 pstatus->wait_req = !IS_FREE(&thrd->req[1 - active])
1588 ? thrd->req[1 - active].r : NULL;
1589 }
1590
1591 pstatus->src_addr = readl(regs + SA(thrd->id));
1592 pstatus->dst_addr = readl(regs + DA(thrd->id));
1593
1594 return 0;
1595}
1596EXPORT_SYMBOL(pl330_chan_status);
1597
1598/* Reserve an event */
1599static inline int _alloc_event(struct pl330_thread *thrd)
1600{
1601 struct pl330_dmac *pl330 = thrd->dmac;
1602 struct pl330_info *pi = pl330->pinfo;
1603 int ev;
1604
1605 for (ev = 0; ev < pi->pcfg.num_events; ev++)
1606 if (pl330->events[ev] == -1) {
1607 pl330->events[ev] = thrd->id;
1608 return ev;
1609 }
1610
1611 return -1;
1612}
1613
1614static bool _chan_ns(const struct pl330_info *pi, int i)
1615{
1616 return pi->pcfg.irq_ns & (1 << i);
1617}
1618
1619/* Upon success, returns IdentityToken for the
1620 * allocated channel, NULL otherwise.
1621 */
1622void *pl330_request_channel(const struct pl330_info *pi)
1623{
1624 struct pl330_thread *thrd = NULL;
1625 struct pl330_dmac *pl330;
1626 unsigned long flags;
1627 int chans, i;
1628
1629 if (!pi || !pi->pl330_data)
1630 return NULL;
1631
1632 pl330 = pi->pl330_data;
1633
1634 if (pl330->state == DYING)
1635 return NULL;
1636
1637 chans = pi->pcfg.num_chan;
1638
1639 spin_lock_irqsave(&pl330->lock, flags);
1640
1641 for (i = 0; i < chans; i++) {
1642 thrd = &pl330->channels[i];
1643 if ((thrd->free) && (!_manager_ns(thrd) ||
1644 _chan_ns(pi, i))) {
1645 thrd->ev = _alloc_event(thrd);
1646 if (thrd->ev >= 0) {
1647 thrd->free = false;
1648 thrd->lstenq = 1;
1649 thrd->req[0].r = NULL;
1650 mark_free(thrd, 0);
1651 thrd->req[1].r = NULL;
1652 mark_free(thrd, 1);
1653 break;
1654 }
1655 }
1656 thrd = NULL;
1657 }
1658
1659 spin_unlock_irqrestore(&pl330->lock, flags);
1660
1661 return thrd;
1662}
1663EXPORT_SYMBOL(pl330_request_channel);
1664
1665/* Release an event */
1666static inline void _free_event(struct pl330_thread *thrd, int ev)
1667{
1668 struct pl330_dmac *pl330 = thrd->dmac;
1669 struct pl330_info *pi = pl330->pinfo;
1670
1671 /* If the event is valid and was held by the thread */
1672 if (ev >= 0 && ev < pi->pcfg.num_events
1673 && pl330->events[ev] == thrd->id)
1674 pl330->events[ev] = -1;
1675}
1676
1677void pl330_release_channel(void *ch_id)
1678{
1679 struct pl330_thread *thrd = ch_id;
1680 struct pl330_dmac *pl330;
1681 unsigned long flags;
1682
1683 if (!thrd || thrd->free)
1684 return;
1685
1686 _stop(thrd);
1687
1688 _callback(thrd->req[1 - thrd->lstenq].r, PL330_ERR_ABORT);
1689 _callback(thrd->req[thrd->lstenq].r, PL330_ERR_ABORT);
1690
1691 pl330 = thrd->dmac;
1692
1693 spin_lock_irqsave(&pl330->lock, flags);
1694 _free_event(thrd, thrd->ev);
1695 thrd->free = true;
1696 spin_unlock_irqrestore(&pl330->lock, flags);
1697}
1698EXPORT_SYMBOL(pl330_release_channel);
1699
1700/* Initialize the structure for PL330 configuration, that can be used
1701 * by the client driver the make best use of the DMAC
1702 */
1703static void read_dmac_config(struct pl330_info *pi)
1704{
1705 void __iomem *regs = pi->base;
1706 u32 val;
1707
1708 val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT;
1709 val &= CRD_DATA_WIDTH_MASK;
1710 pi->pcfg.data_bus_width = 8 * (1 << val);
1711
1712 val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT;
1713 val &= CRD_DATA_BUFF_MASK;
1714 pi->pcfg.data_buf_dep = val + 1;
1715
1716 val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT;
1717 val &= CR0_NUM_CHANS_MASK;
1718 val += 1;
1719 pi->pcfg.num_chan = val;
1720
1721 val = readl(regs + CR0);
1722 if (val & CR0_PERIPH_REQ_SET) {
1723 val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK;
1724 val += 1;
1725 pi->pcfg.num_peri = val;
1726 pi->pcfg.peri_ns = readl(regs + CR4);
1727 } else {
1728 pi->pcfg.num_peri = 0;
1729 }
1730
1731 val = readl(regs + CR0);
1732 if (val & CR0_BOOT_MAN_NS)
1733 pi->pcfg.mode |= DMAC_MODE_NS;
1734 else
1735 pi->pcfg.mode &= ~DMAC_MODE_NS;
1736
1737 val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT;
1738 val &= CR0_NUM_EVENTS_MASK;
1739 val += 1;
1740 pi->pcfg.num_events = val;
1741
1742 pi->pcfg.irq_ns = readl(regs + CR3);
1743
1744 pi->pcfg.periph_id = get_id(pi, PERIPH_ID);
1745 pi->pcfg.pcell_id = get_id(pi, PCELL_ID);
1746}
1747
1748static inline void _reset_thread(struct pl330_thread *thrd)
1749{
1750 struct pl330_dmac *pl330 = thrd->dmac;
1751 struct pl330_info *pi = pl330->pinfo;
1752
1753 thrd->req[0].mc_cpu = pl330->mcode_cpu
1754 + (thrd->id * pi->mcbufsz);
1755 thrd->req[0].mc_bus = pl330->mcode_bus
1756 + (thrd->id * pi->mcbufsz);
1757 thrd->req[0].r = NULL;
1758 mark_free(thrd, 0);
1759
1760 thrd->req[1].mc_cpu = thrd->req[0].mc_cpu
1761 + pi->mcbufsz / 2;
1762 thrd->req[1].mc_bus = thrd->req[0].mc_bus
1763 + pi->mcbufsz / 2;
1764 thrd->req[1].r = NULL;
1765 mark_free(thrd, 1);
1766}
1767
1768static int dmac_alloc_threads(struct pl330_dmac *pl330)
1769{
1770 struct pl330_info *pi = pl330->pinfo;
1771 int chans = pi->pcfg.num_chan;
1772 struct pl330_thread *thrd;
1773 int i;
1774
1775 /* Allocate 1 Manager and 'chans' Channel threads */
1776 pl330->channels = kzalloc((1 + chans) * sizeof(*thrd),
1777 GFP_KERNEL);
1778 if (!pl330->channels)
1779 return -ENOMEM;
1780
1781 /* Init Channel threads */
1782 for (i = 0; i < chans; i++) {
1783 thrd = &pl330->channels[i];
1784 thrd->id = i;
1785 thrd->dmac = pl330;
1786 _reset_thread(thrd);
1787 thrd->free = true;
1788 }
1789
1790 /* MANAGER is indexed at the end */
1791 thrd = &pl330->channels[chans];
1792 thrd->id = chans;
1793 thrd->dmac = pl330;
1794 thrd->free = false;
1795 pl330->manager = thrd;
1796
1797 return 0;
1798}
1799
1800static int dmac_alloc_resources(struct pl330_dmac *pl330)
1801{
1802 struct pl330_info *pi = pl330->pinfo;
1803 int chans = pi->pcfg.num_chan;
1804 int ret;
1805
1806 /*
1807 * Alloc MicroCode buffer for 'chans' Channel threads.
1808 * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN)
1809 */
1810 pl330->mcode_cpu = dma_alloc_coherent(pi->dev,
1811 chans * pi->mcbufsz,
1812 &pl330->mcode_bus, GFP_KERNEL);
1813 if (!pl330->mcode_cpu) {
1814 dev_err(pi->dev, "%s:%d Can't allocate memory!\n",
1815 __func__, __LINE__);
1816 return -ENOMEM;
1817 }
1818
1819 ret = dmac_alloc_threads(pl330);
1820 if (ret) {
1821 dev_err(pi->dev, "%s:%d Can't to create channels for DMAC!\n",
1822 __func__, __LINE__);
1823 dma_free_coherent(pi->dev,
1824 chans * pi->mcbufsz,
1825 pl330->mcode_cpu, pl330->mcode_bus);
1826 return ret;
1827 }
1828
1829 return 0;
1830}
1831
1832int pl330_add(struct pl330_info *pi)
1833{
1834 struct pl330_dmac *pl330;
1835 void __iomem *regs;
1836 int i, ret;
1837
1838 if (!pi || !pi->dev)
1839 return -EINVAL;
1840
1841 /* If already added */
1842 if (pi->pl330_data)
1843 return -EINVAL;
1844
1845 /*
1846 * If the SoC can perform reset on the DMAC, then do it
1847 * before reading its configuration.
1848 */
1849 if (pi->dmac_reset)
1850 pi->dmac_reset(pi);
1851
1852 regs = pi->base;
1853
1854 /* Check if we can handle this DMAC */
1855 if ((get_id(pi, PERIPH_ID) & 0xfffff) != PERIPH_ID_VAL
1856 || get_id(pi, PCELL_ID) != PCELL_ID_VAL) {
1857 dev_err(pi->dev, "PERIPH_ID 0x%x, PCELL_ID 0x%x !\n",
1858 get_id(pi, PERIPH_ID), get_id(pi, PCELL_ID));
1859 return -EINVAL;
1860 }
1861
1862 /* Read the configuration of the DMAC */
1863 read_dmac_config(pi);
1864
1865 if (pi->pcfg.num_events == 0) {
1866 dev_err(pi->dev, "%s:%d Can't work without events!\n",
1867 __func__, __LINE__);
1868 return -EINVAL;
1869 }
1870
1871 pl330 = kzalloc(sizeof(*pl330), GFP_KERNEL);
1872 if (!pl330) {
1873 dev_err(pi->dev, "%s:%d Can't allocate memory!\n",
1874 __func__, __LINE__);
1875 return -ENOMEM;
1876 }
1877
1878 /* Assign the info structure and private data */
1879 pl330->pinfo = pi;
1880 pi->pl330_data = pl330;
1881
1882 spin_lock_init(&pl330->lock);
1883
1884 INIT_LIST_HEAD(&pl330->req_done);
1885
1886 /* Use default MC buffer size if not provided */
1887 if (!pi->mcbufsz)
1888 pi->mcbufsz = MCODE_BUFF_PER_REQ * 2;
1889
1890 /* Mark all events as free */
1891 for (i = 0; i < pi->pcfg.num_events; i++)
1892 pl330->events[i] = -1;
1893
1894 /* Allocate resources needed by the DMAC */
1895 ret = dmac_alloc_resources(pl330);
1896 if (ret) {
1897 dev_err(pi->dev, "Unable to create channels for DMAC\n");
1898 kfree(pl330);
1899 return ret;
1900 }
1901
1902 tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330);
1903
1904 pl330->state = INIT;
1905
1906 return 0;
1907}
1908EXPORT_SYMBOL(pl330_add);
1909
1910static int dmac_free_threads(struct pl330_dmac *pl330)
1911{
1912 struct pl330_info *pi = pl330->pinfo;
1913 int chans = pi->pcfg.num_chan;
1914 struct pl330_thread *thrd;
1915 int i;
1916
1917 /* Release Channel threads */
1918 for (i = 0; i < chans; i++) {
1919 thrd = &pl330->channels[i];
1920 pl330_release_channel((void *)thrd);
1921 }
1922
1923 /* Free memory */
1924 kfree(pl330->channels);
1925
1926 return 0;
1927}
1928
1929static void dmac_free_resources(struct pl330_dmac *pl330)
1930{
1931 struct pl330_info *pi = pl330->pinfo;
1932 int chans = pi->pcfg.num_chan;
1933
1934 dmac_free_threads(pl330);
1935
1936 dma_free_coherent(pi->dev, chans * pi->mcbufsz,
1937 pl330->mcode_cpu, pl330->mcode_bus);
1938}
1939
1940void pl330_del(struct pl330_info *pi)
1941{
1942 struct pl330_dmac *pl330;
1943
1944 if (!pi || !pi->pl330_data)
1945 return;
1946
1947 pl330 = pi->pl330_data;
1948
1949 pl330->state = UNINIT;
1950
1951 tasklet_kill(&pl330->tasks);
1952
1953 /* Free DMAC resources */
1954 dmac_free_resources(pl330);
1955
1956 kfree(pl330);
1957 pi->pl330_data = NULL;
1958}
1959EXPORT_SYMBOL(pl330_del);
diff --git a/arch/arm/include/asm/hardware/pl330.h b/arch/arm/include/asm/hardware/pl330.h
deleted file mode 100644
index 575fa8186ca..00000000000
--- a/arch/arm/include/asm/hardware/pl330.h
+++ /dev/null
@@ -1,217 +0,0 @@
1/* linux/include/asm/hardware/pl330.h
2 *
3 * Copyright (C) 2010 Samsung Electronics Co. Ltd.
4 * Jaswinder Singh <jassi.brar@samsung.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 */
20
21#ifndef __PL330_CORE_H
22#define __PL330_CORE_H
23
24#define PL330_MAX_CHAN 8
25#define PL330_MAX_IRQS 32
26#define PL330_MAX_PERI 32
27
28enum pl330_srccachectrl {
29 SCCTRL0 = 0, /* Noncacheable and nonbufferable */
30 SCCTRL1, /* Bufferable only */
31 SCCTRL2, /* Cacheable, but do not allocate */
32 SCCTRL3, /* Cacheable and bufferable, but do not allocate */
33 SINVALID1,
34 SINVALID2,
35 SCCTRL6, /* Cacheable write-through, allocate on reads only */
36 SCCTRL7, /* Cacheable write-back, allocate on reads only */
37};
38
39enum pl330_dstcachectrl {
40 DCCTRL0 = 0, /* Noncacheable and nonbufferable */
41 DCCTRL1, /* Bufferable only */
42 DCCTRL2, /* Cacheable, but do not allocate */
43 DCCTRL3, /* Cacheable and bufferable, but do not allocate */
44 DINVALID1 = 8,
45 DINVALID2,
46 DCCTRL6, /* Cacheable write-through, allocate on writes only */
47 DCCTRL7, /* Cacheable write-back, allocate on writes only */
48};
49
50/* Populated by the PL330 core driver for DMA API driver's info */
51struct pl330_config {
52 u32 periph_id;
53 u32 pcell_id;
54#define DMAC_MODE_NS (1 << 0)
55 unsigned int mode;
56 unsigned int data_bus_width:10; /* In number of bits */
57 unsigned int data_buf_dep:10;
58 unsigned int num_chan:4;
59 unsigned int num_peri:6;
60 u32 peri_ns;
61 unsigned int num_events:6;
62 u32 irq_ns;
63};
64
65/* Handle to the DMAC provided to the PL330 core */
66struct pl330_info {
67 /* Owning device */
68 struct device *dev;
69 /* Size of MicroCode buffers for each channel. */
70 unsigned mcbufsz;
71 /* ioremap'ed address of PL330 registers. */
72 void __iomem *base;
73 /* Client can freely use it. */
74 void *client_data;
75 /* PL330 core data, Client must not touch it. */
76 void *pl330_data;
77 /* Populated by the PL330 core driver during pl330_add */
78 struct pl330_config pcfg;
79 /*
80 * If the DMAC has some reset mechanism, then the
81 * client may want to provide pointer to the method.
82 */
83 void (*dmac_reset)(struct pl330_info *pi);
84};
85
86enum pl330_byteswap {
87 SWAP_NO = 0,
88 SWAP_2,
89 SWAP_4,
90 SWAP_8,
91 SWAP_16,
92};
93
94/**
95 * Request Configuration.
96 * The PL330 core does not modify this and uses the last
97 * working configuration if the request doesn't provide any.
98 *
99 * The Client may want to provide this info only for the
100 * first request and a request with new settings.
101 */
102struct pl330_reqcfg {
103 /* Address Incrementing */
104 unsigned dst_inc:1;
105 unsigned src_inc:1;
106
107 /*
108 * For now, the SRC & DST protection levels
109 * and burst size/length are assumed same.
110 */
111 bool nonsecure;
112 bool privileged;
113 bool insnaccess;
114 unsigned brst_len:5;
115 unsigned brst_size:3; /* in power of 2 */
116
117 enum pl330_dstcachectrl dcctl;
118 enum pl330_srccachectrl scctl;
119 enum pl330_byteswap swap;
120};
121
122/*
123 * One cycle of DMAC operation.
124 * There may be more than one xfer in a request.
125 */
126struct pl330_xfer {
127 u32 src_addr;
128 u32 dst_addr;
129 /* Size to xfer */
130 u32 bytes;
131 /*
132 * Pointer to next xfer in the list.
133 * The last xfer in the req must point to NULL.
134 */
135 struct pl330_xfer *next;
136};
137
138/* The xfer callbacks are made with one of these arguments. */
139enum pl330_op_err {
140 /* The all xfers in the request were success. */
141 PL330_ERR_NONE,
142 /* If req aborted due to global error. */
143 PL330_ERR_ABORT,
144 /* If req failed due to problem with Channel. */
145 PL330_ERR_FAIL,
146};
147
148enum pl330_reqtype {
149 MEMTOMEM,
150 MEMTODEV,
151 DEVTOMEM,
152 DEVTODEV,
153};
154
155/* A request defining Scatter-Gather List ending with NULL xfer. */
156struct pl330_req {
157 enum pl330_reqtype rqtype;
158 /* Index of peripheral for the xfer. */
159 unsigned peri:5;
160 /* Unique token for this xfer, set by the client. */
161 void *token;
162 /* Callback to be called after xfer. */
163 void (*xfer_cb)(void *token, enum pl330_op_err err);
164 /* If NULL, req will be done at last set parameters. */
165 struct pl330_reqcfg *cfg;
166 /* Pointer to first xfer in the request. */
167 struct pl330_xfer *x;
168};
169
170/*
171 * To know the status of the channel and DMAC, the client
172 * provides a pointer to this structure. The PL330 core
173 * fills it with current information.
174 */
175struct pl330_chanstatus {
176 /*
177 * If the DMAC engine halted due to some error,
178 * the client should remove-add DMAC.
179 */
180 bool dmac_halted;
181 /*
182 * If channel is halted due to some error,
183 * the client should ABORT/FLUSH and START the channel.
184 */
185 bool faulting;
186 /* Location of last load */
187 u32 src_addr;
188 /* Location of last store */
189 u32 dst_addr;
190 /*
191 * Pointer to the currently active req, NULL if channel is
192 * inactive, even though the requests may be present.
193 */
194 struct pl330_req *top_req;
195 /* Pointer to req waiting second in the queue if any. */
196 struct pl330_req *wait_req;
197};
198
199enum pl330_chan_op {
200 /* Start the channel */
201 PL330_OP_START,
202 /* Abort the active xfer */
203 PL330_OP_ABORT,
204 /* Stop xfer and flush queue */
205 PL330_OP_FLUSH,
206};
207
208extern int pl330_add(struct pl330_info *);
209extern void pl330_del(struct pl330_info *pi);
210extern int pl330_update(const struct pl330_info *pi);
211extern void pl330_release_channel(void *ch_id);
212extern void *pl330_request_channel(const struct pl330_info *pi);
213extern int pl330_chan_status(void *ch_id, struct pl330_chanstatus *pstatus);
214extern int pl330_chan_ctrl(void *ch_id, enum pl330_chan_op op);
215extern int pl330_submit_req(void *ch_id, struct pl330_req *r);
216
217#endif /* __PL330_CORE_H */
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index f1a274994bb..65c61dba66d 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -201,7 +201,6 @@ config PL330_DMA
201 tristate "DMA API Driver for PL330" 201 tristate "DMA API Driver for PL330"
202 select DMA_ENGINE 202 select DMA_ENGINE
203 depends on ARM_AMBA 203 depends on ARM_AMBA
204 select PL330
205 help 204 help
206 Select if your platform has one or more PL330 DMACs. 205 Select if your platform has one or more PL330 DMACs.
207 You need to provide platform specific settings via 206 You need to provide platform specific settings via
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index 84ebea9bc53..2e351f40fc4 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -1,4 +1,6 @@
1/* linux/drivers/dma/pl330.c 1/*
2 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
3 * http://www.samsung.com
2 * 4 *
3 * Copyright (C) 2010 Samsung Electronics Co. Ltd. 5 * Copyright (C) 2010 Samsung Electronics Co. Ltd.
4 * Jaswinder Singh <jassi.brar@samsung.com> 6 * Jaswinder Singh <jassi.brar@samsung.com>
@@ -9,10 +11,15 @@
9 * (at your option) any later version. 11 * (at your option) any later version.
10 */ 12 */
11 13
14#include <linux/kernel.h>
12#include <linux/io.h> 15#include <linux/io.h>
13#include <linux/init.h> 16#include <linux/init.h>
14#include <linux/slab.h> 17#include <linux/slab.h>
15#include <linux/module.h> 18#include <linux/module.h>
19#include <linux/string.h>
20#include <linux/delay.h>
21#include <linux/interrupt.h>
22#include <linux/dma-mapping.h>
16#include <linux/dmaengine.h> 23#include <linux/dmaengine.h>
17#include <linux/interrupt.h> 24#include <linux/interrupt.h>
18#include <linux/amba/bus.h> 25#include <linux/amba/bus.h>
@@ -21,8 +28,489 @@
21#include <linux/scatterlist.h> 28#include <linux/scatterlist.h>
22#include <linux/of.h> 29#include <linux/of.h>
23 30
31#define PL330_MAX_CHAN 8
32#define PL330_MAX_IRQS 32
33#define PL330_MAX_PERI 32
34
35enum pl330_srccachectrl {
36 SCCTRL0, /* Noncacheable and nonbufferable */
37 SCCTRL1, /* Bufferable only */
38 SCCTRL2, /* Cacheable, but do not allocate */
39 SCCTRL3, /* Cacheable and bufferable, but do not allocate */
40 SINVALID1,
41 SINVALID2,
42 SCCTRL6, /* Cacheable write-through, allocate on reads only */
43 SCCTRL7, /* Cacheable write-back, allocate on reads only */
44};
45
46enum pl330_dstcachectrl {
47 DCCTRL0, /* Noncacheable and nonbufferable */
48 DCCTRL1, /* Bufferable only */
49 DCCTRL2, /* Cacheable, but do not allocate */
50 DCCTRL3, /* Cacheable and bufferable, but do not allocate */
51 DINVALID1 = 8,
52 DINVALID2,
53 DCCTRL6, /* Cacheable write-through, allocate on writes only */
54 DCCTRL7, /* Cacheable write-back, allocate on writes only */
55};
56
57enum pl330_byteswap {
58 SWAP_NO,
59 SWAP_2,
60 SWAP_4,
61 SWAP_8,
62 SWAP_16,
63};
64
65enum pl330_reqtype {
66 MEMTOMEM,
67 MEMTODEV,
68 DEVTOMEM,
69 DEVTODEV,
70};
71
72/* Register and Bit field Definitions */
73#define DS 0x0
74#define DS_ST_STOP 0x0
75#define DS_ST_EXEC 0x1
76#define DS_ST_CMISS 0x2
77#define DS_ST_UPDTPC 0x3
78#define DS_ST_WFE 0x4
79#define DS_ST_ATBRR 0x5
80#define DS_ST_QBUSY 0x6
81#define DS_ST_WFP 0x7
82#define DS_ST_KILL 0x8
83#define DS_ST_CMPLT 0x9
84#define DS_ST_FLTCMP 0xe
85#define DS_ST_FAULT 0xf
86
87#define DPC 0x4
88#define INTEN 0x20
89#define ES 0x24
90#define INTSTATUS 0x28
91#define INTCLR 0x2c
92#define FSM 0x30
93#define FSC 0x34
94#define FTM 0x38
95
96#define _FTC 0x40
97#define FTC(n) (_FTC + (n)*0x4)
98
99#define _CS 0x100
100#define CS(n) (_CS + (n)*0x8)
101#define CS_CNS (1 << 21)
102
103#define _CPC 0x104
104#define CPC(n) (_CPC + (n)*0x8)
105
106#define _SA 0x400
107#define SA(n) (_SA + (n)*0x20)
108
109#define _DA 0x404
110#define DA(n) (_DA + (n)*0x20)
111
112#define _CC 0x408
113#define CC(n) (_CC + (n)*0x20)
114
115#define CC_SRCINC (1 << 0)
116#define CC_DSTINC (1 << 14)
117#define CC_SRCPRI (1 << 8)
118#define CC_DSTPRI (1 << 22)
119#define CC_SRCNS (1 << 9)
120#define CC_DSTNS (1 << 23)
121#define CC_SRCIA (1 << 10)
122#define CC_DSTIA (1 << 24)
123#define CC_SRCBRSTLEN_SHFT 4
124#define CC_DSTBRSTLEN_SHFT 18
125#define CC_SRCBRSTSIZE_SHFT 1
126#define CC_DSTBRSTSIZE_SHFT 15
127#define CC_SRCCCTRL_SHFT 11
128#define CC_SRCCCTRL_MASK 0x7
129#define CC_DSTCCTRL_SHFT 25
130#define CC_DRCCCTRL_MASK 0x7
131#define CC_SWAP_SHFT 28
132
133#define _LC0 0x40c
134#define LC0(n) (_LC0 + (n)*0x20)
135
136#define _LC1 0x410
137#define LC1(n) (_LC1 + (n)*0x20)
138
139#define DBGSTATUS 0xd00
140#define DBG_BUSY (1 << 0)
141
142#define DBGCMD 0xd04
143#define DBGINST0 0xd08
144#define DBGINST1 0xd0c
145
146#define CR0 0xe00
147#define CR1 0xe04
148#define CR2 0xe08
149#define CR3 0xe0c
150#define CR4 0xe10
151#define CRD 0xe14
152
153#define PERIPH_ID 0xfe0
154#define PCELL_ID 0xff0
155
156#define CR0_PERIPH_REQ_SET (1 << 0)
157#define CR0_BOOT_EN_SET (1 << 1)
158#define CR0_BOOT_MAN_NS (1 << 2)
159#define CR0_NUM_CHANS_SHIFT 4
160#define CR0_NUM_CHANS_MASK 0x7
161#define CR0_NUM_PERIPH_SHIFT 12
162#define CR0_NUM_PERIPH_MASK 0x1f
163#define CR0_NUM_EVENTS_SHIFT 17
164#define CR0_NUM_EVENTS_MASK 0x1f
165
166#define CR1_ICACHE_LEN_SHIFT 0
167#define CR1_ICACHE_LEN_MASK 0x7
168#define CR1_NUM_ICACHELINES_SHIFT 4
169#define CR1_NUM_ICACHELINES_MASK 0xf
170
171#define CRD_DATA_WIDTH_SHIFT 0
172#define CRD_DATA_WIDTH_MASK 0x7
173#define CRD_WR_CAP_SHIFT 4
174#define CRD_WR_CAP_MASK 0x7
175#define CRD_WR_Q_DEP_SHIFT 8
176#define CRD_WR_Q_DEP_MASK 0xf
177#define CRD_RD_CAP_SHIFT 12
178#define CRD_RD_CAP_MASK 0x7
179#define CRD_RD_Q_DEP_SHIFT 16
180#define CRD_RD_Q_DEP_MASK 0xf
181#define CRD_DATA_BUFF_SHIFT 20
182#define CRD_DATA_BUFF_MASK 0x3ff
183
184#define PART 0x330
185#define DESIGNER 0x41
186#define REVISION 0x0
187#define INTEG_CFG 0x0
188#define PERIPH_ID_VAL ((PART << 0) | (DESIGNER << 12))
189
190#define PCELL_ID_VAL 0xb105f00d
191
192#define PL330_STATE_STOPPED (1 << 0)
193#define PL330_STATE_EXECUTING (1 << 1)
194#define PL330_STATE_WFE (1 << 2)
195#define PL330_STATE_FAULTING (1 << 3)
196#define PL330_STATE_COMPLETING (1 << 4)
197#define PL330_STATE_WFP (1 << 5)
198#define PL330_STATE_KILLING (1 << 6)
199#define PL330_STATE_FAULT_COMPLETING (1 << 7)
200#define PL330_STATE_CACHEMISS (1 << 8)
201#define PL330_STATE_UPDTPC (1 << 9)
202#define PL330_STATE_ATBARRIER (1 << 10)
203#define PL330_STATE_QUEUEBUSY (1 << 11)
204#define PL330_STATE_INVALID (1 << 15)
205
206#define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \
207 | PL330_STATE_WFE | PL330_STATE_FAULTING)
208
209#define CMD_DMAADDH 0x54
210#define CMD_DMAEND 0x00
211#define CMD_DMAFLUSHP 0x35
212#define CMD_DMAGO 0xa0
213#define CMD_DMALD 0x04
214#define CMD_DMALDP 0x25
215#define CMD_DMALP 0x20
216#define CMD_DMALPEND 0x28
217#define CMD_DMAKILL 0x01
218#define CMD_DMAMOV 0xbc
219#define CMD_DMANOP 0x18
220#define CMD_DMARMB 0x12
221#define CMD_DMASEV 0x34
222#define CMD_DMAST 0x08
223#define CMD_DMASTP 0x29
224#define CMD_DMASTZ 0x0c
225#define CMD_DMAWFE 0x36
226#define CMD_DMAWFP 0x30
227#define CMD_DMAWMB 0x13
228
229#define SZ_DMAADDH 3
230#define SZ_DMAEND 1
231#define SZ_DMAFLUSHP 2
232#define SZ_DMALD 1
233#define SZ_DMALDP 2
234#define SZ_DMALP 2
235#define SZ_DMALPEND 2
236#define SZ_DMAKILL 1
237#define SZ_DMAMOV 6
238#define SZ_DMANOP 1
239#define SZ_DMARMB 1
240#define SZ_DMASEV 2
241#define SZ_DMAST 1
242#define SZ_DMASTP 2
243#define SZ_DMASTZ 1
244#define SZ_DMAWFE 2
245#define SZ_DMAWFP 2
246#define SZ_DMAWMB 1
247#define SZ_DMAGO 6
248
249#define BRST_LEN(ccr) ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1)
250#define BRST_SIZE(ccr) (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7))
251
252#define BYTE_TO_BURST(b, ccr) ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr))
253#define BURST_TO_BYTE(c, ccr) ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr))
254
255/*
256 * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req
257 * at 1byte/burst for P<->M and M<->M respectively.
258 * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req
259 * should be enough for P<->M and M<->M respectively.
260 */
261#define MCODE_BUFF_PER_REQ 256
262
263/* If the _pl330_req is available to the client */
264#define IS_FREE(req) (*((u8 *)((req)->mc_cpu)) == CMD_DMAEND)
265
266/* Use this _only_ to wait on transient states */
267#define UNTIL(t, s) while (!(_state(t) & (s))) cpu_relax();
268
269#ifdef PL330_DEBUG_MCGEN
270static unsigned cmd_line;
271#define PL330_DBGCMD_DUMP(off, x...) do { \
272 printk("%x:", cmd_line); \
273 printk(x); \
274 cmd_line += off; \
275 } while (0)
276#define PL330_DBGMC_START(addr) (cmd_line = addr)
277#else
278#define PL330_DBGCMD_DUMP(off, x...) do {} while (0)
279#define PL330_DBGMC_START(addr) do {} while (0)
280#endif
281
282/* The number of default descriptors */
24#define NR_DEFAULT_DESC 16 283#define NR_DEFAULT_DESC 16
25 284
285/* Populated by the PL330 core driver for DMA API driver's info */
286struct pl330_config {
287 u32 periph_id;
288 u32 pcell_id;
289#define DMAC_MODE_NS (1 << 0)
290 unsigned int mode;
291 unsigned int data_bus_width:10; /* In number of bits */
292 unsigned int data_buf_dep:10;
293 unsigned int num_chan:4;
294 unsigned int num_peri:6;
295 u32 peri_ns;
296 unsigned int num_events:6;
297 u32 irq_ns;
298};
299
300/* Handle to the DMAC provided to the PL330 core */
301struct pl330_info {
302 /* Owning device */
303 struct device *dev;
304 /* Size of MicroCode buffers for each channel. */
305 unsigned mcbufsz;
306 /* ioremap'ed address of PL330 registers. */
307 void __iomem *base;
308 /* Client can freely use it. */
309 void *client_data;
310 /* PL330 core data, Client must not touch it. */
311 void *pl330_data;
312 /* Populated by the PL330 core driver during pl330_add */
313 struct pl330_config pcfg;
314 /*
315 * If the DMAC has some reset mechanism, then the
316 * client may want to provide pointer to the method.
317 */
318 void (*dmac_reset)(struct pl330_info *pi);
319};
320
321/**
322 * Request Configuration.
323 * The PL330 core does not modify this and uses the last
324 * working configuration if the request doesn't provide any.
325 *
326 * The Client may want to provide this info only for the
327 * first request and a request with new settings.
328 */
329struct pl330_reqcfg {
330 /* Address Incrementing */
331 unsigned dst_inc:1;
332 unsigned src_inc:1;
333
334 /*
335 * For now, the SRC & DST protection levels
336 * and burst size/length are assumed same.
337 */
338 bool nonsecure;
339 bool privileged;
340 bool insnaccess;
341 unsigned brst_len:5;
342 unsigned brst_size:3; /* in power of 2 */
343
344 enum pl330_dstcachectrl dcctl;
345 enum pl330_srccachectrl scctl;
346 enum pl330_byteswap swap;
347};
348
349/*
350 * One cycle of DMAC operation.
351 * There may be more than one xfer in a request.
352 */
353struct pl330_xfer {
354 u32 src_addr;
355 u32 dst_addr;
356 /* Size to xfer */
357 u32 bytes;
358 /*
359 * Pointer to next xfer in the list.
360 * The last xfer in the req must point to NULL.
361 */
362 struct pl330_xfer *next;
363};
364
365/* The xfer callbacks are made with one of these arguments. */
366enum pl330_op_err {
367 /* The all xfers in the request were success. */
368 PL330_ERR_NONE,
369 /* If req aborted due to global error. */
370 PL330_ERR_ABORT,
371 /* If req failed due to problem with Channel. */
372 PL330_ERR_FAIL,
373};
374
375/* A request defining Scatter-Gather List ending with NULL xfer. */
376struct pl330_req {
377 enum pl330_reqtype rqtype;
378 /* Index of peripheral for the xfer. */
379 unsigned peri:5;
380 /* Unique token for this xfer, set by the client. */
381 void *token;
382 /* Callback to be called after xfer. */
383 void (*xfer_cb)(void *token, enum pl330_op_err err);
384 /* If NULL, req will be done at last set parameters. */
385 struct pl330_reqcfg *cfg;
386 /* Pointer to first xfer in the request. */
387 struct pl330_xfer *x;
388};
389
390/*
391 * To know the status of the channel and DMAC, the client
392 * provides a pointer to this structure. The PL330 core
393 * fills it with current information.
394 */
395struct pl330_chanstatus {
396 /*
397 * If the DMAC engine halted due to some error,
398 * the client should remove-add DMAC.
399 */
400 bool dmac_halted;
401 /*
402 * If channel is halted due to some error,
403 * the client should ABORT/FLUSH and START the channel.
404 */
405 bool faulting;
406 /* Location of last load */
407 u32 src_addr;
408 /* Location of last store */
409 u32 dst_addr;
410 /*
411 * Pointer to the currently active req, NULL if channel is
412 * inactive, even though the requests may be present.
413 */
414 struct pl330_req *top_req;
415 /* Pointer to req waiting second in the queue if any. */
416 struct pl330_req *wait_req;
417};
418
419enum pl330_chan_op {
420 /* Start the channel */
421 PL330_OP_START,
422 /* Abort the active xfer */
423 PL330_OP_ABORT,
424 /* Stop xfer and flush queue */
425 PL330_OP_FLUSH,
426};
427
428struct _xfer_spec {
429 u32 ccr;
430 struct pl330_req *r;
431 struct pl330_xfer *x;
432};
433
434enum dmamov_dst {
435 SAR = 0,
436 CCR,
437 DAR,
438};
439
440enum pl330_dst {
441 SRC = 0,
442 DST,
443};
444
445enum pl330_cond {
446 SINGLE,
447 BURST,
448 ALWAYS,
449};
450
451struct _pl330_req {
452 u32 mc_bus;
453 void *mc_cpu;
454 /* Number of bytes taken to setup MC for the req */
455 u32 mc_len;
456 struct pl330_req *r;
457 /* Hook to attach to DMAC's list of reqs with due callback */
458 struct list_head rqd;
459};
460
461/* ToBeDone for tasklet */
462struct _pl330_tbd {
463 bool reset_dmac;
464 bool reset_mngr;
465 u8 reset_chan;
466};
467
468/* A DMAC Thread */
469struct pl330_thread {
470 u8 id;
471 int ev;
472 /* If the channel is not yet acquired by any client */
473 bool free;
474 /* Parent DMAC */
475 struct pl330_dmac *dmac;
476 /* Only two at a time */
477 struct _pl330_req req[2];
478 /* Index of the last enqueued request */
479 unsigned lstenq;
480 /* Index of the last submitted request or -1 if the DMA is stopped */
481 int req_running;
482};
483
484enum pl330_dmac_state {
485 UNINIT,
486 INIT,
487 DYING,
488};
489
490/* A DMAC */
491struct pl330_dmac {
492 spinlock_t lock;
493 /* Holds list of reqs with due callbacks */
494 struct list_head req_done;
495 /* Pointer to platform specific stuff */
496 struct pl330_info *pinfo;
497 /* Maximum possible events/irqs */
498 int events[32];
499 /* BUS address of MicroCode buffer */
500 u32 mcode_bus;
501 /* CPU address of MicroCode buffer */
502 void *mcode_cpu;
503 /* List of all Channel threads */
504 struct pl330_thread *channels;
505 /* Pointer to the MANAGER thread */
506 struct pl330_thread *manager;
507 /* To handle bad news in interrupt */
508 struct tasklet_struct tasks;
509 struct _pl330_tbd dmac_tbd;
510 /* State of DMAC operation */
511 enum pl330_dmac_state state;
512};
513
26enum desc_status { 514enum desc_status {
27 /* In the DMAC pool */ 515 /* In the DMAC pool */
28 FREE, 516 FREE,
@@ -117,6 +605,1630 @@ struct dma_pl330_desc {
117 struct dma_pl330_chan *pchan; 605 struct dma_pl330_chan *pchan;
118}; 606};
119 607
608static inline void _callback(struct pl330_req *r, enum pl330_op_err err)
609{
610 if (r && r->xfer_cb)
611 r->xfer_cb(r->token, err);
612}
613
614static inline bool _queue_empty(struct pl330_thread *thrd)
615{
616 return (IS_FREE(&thrd->req[0]) && IS_FREE(&thrd->req[1]))
617 ? true : false;
618}
619
620static inline bool _queue_full(struct pl330_thread *thrd)
621{
622 return (IS_FREE(&thrd->req[0]) || IS_FREE(&thrd->req[1]))
623 ? false : true;
624}
625
626static inline bool is_manager(struct pl330_thread *thrd)
627{
628 struct pl330_dmac *pl330 = thrd->dmac;
629
630 /* MANAGER is indexed at the end */
631 if (thrd->id == pl330->pinfo->pcfg.num_chan)
632 return true;
633 else
634 return false;
635}
636
637/* If manager of the thread is in Non-Secure mode */
638static inline bool _manager_ns(struct pl330_thread *thrd)
639{
640 struct pl330_dmac *pl330 = thrd->dmac;
641
642 return (pl330->pinfo->pcfg.mode & DMAC_MODE_NS) ? true : false;
643}
644
645static inline u32 get_id(struct pl330_info *pi, u32 off)
646{
647 void __iomem *regs = pi->base;
648 u32 id = 0;
649
650 id |= (readb(regs + off + 0x0) << 0);
651 id |= (readb(regs + off + 0x4) << 8);
652 id |= (readb(regs + off + 0x8) << 16);
653 id |= (readb(regs + off + 0xc) << 24);
654
655 return id;
656}
657
658static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[],
659 enum pl330_dst da, u16 val)
660{
661 if (dry_run)
662 return SZ_DMAADDH;
663
664 buf[0] = CMD_DMAADDH;
665 buf[0] |= (da << 1);
666 *((u16 *)&buf[1]) = val;
667
668 PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n",
669 da == 1 ? "DA" : "SA", val);
670
671 return SZ_DMAADDH;
672}
673
674static inline u32 _emit_END(unsigned dry_run, u8 buf[])
675{
676 if (dry_run)
677 return SZ_DMAEND;
678
679 buf[0] = CMD_DMAEND;
680
681 PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n");
682
683 return SZ_DMAEND;
684}
685
686static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri)
687{
688 if (dry_run)
689 return SZ_DMAFLUSHP;
690
691 buf[0] = CMD_DMAFLUSHP;
692
693 peri &= 0x1f;
694 peri <<= 3;
695 buf[1] = peri;
696
697 PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3);
698
699 return SZ_DMAFLUSHP;
700}
701
702static inline u32 _emit_LD(unsigned dry_run, u8 buf[], enum pl330_cond cond)
703{
704 if (dry_run)
705 return SZ_DMALD;
706
707 buf[0] = CMD_DMALD;
708
709 if (cond == SINGLE)
710 buf[0] |= (0 << 1) | (1 << 0);
711 else if (cond == BURST)
712 buf[0] |= (1 << 1) | (1 << 0);
713
714 PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n",
715 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
716
717 return SZ_DMALD;
718}
719
720static inline u32 _emit_LDP(unsigned dry_run, u8 buf[],
721 enum pl330_cond cond, u8 peri)
722{
723 if (dry_run)
724 return SZ_DMALDP;
725
726 buf[0] = CMD_DMALDP;
727
728 if (cond == BURST)
729 buf[0] |= (1 << 1);
730
731 peri &= 0x1f;
732 peri <<= 3;
733 buf[1] = peri;
734
735 PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n",
736 cond == SINGLE ? 'S' : 'B', peri >> 3);
737
738 return SZ_DMALDP;
739}
740
741static inline u32 _emit_LP(unsigned dry_run, u8 buf[],
742 unsigned loop, u8 cnt)
743{
744 if (dry_run)
745 return SZ_DMALP;
746
747 buf[0] = CMD_DMALP;
748
749 if (loop)
750 buf[0] |= (1 << 1);
751
752 cnt--; /* DMAC increments by 1 internally */
753 buf[1] = cnt;
754
755 PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt);
756
757 return SZ_DMALP;
758}
759
760struct _arg_LPEND {
761 enum pl330_cond cond;
762 bool forever;
763 unsigned loop;
764 u8 bjump;
765};
766
767static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[],
768 const struct _arg_LPEND *arg)
769{
770 enum pl330_cond cond = arg->cond;
771 bool forever = arg->forever;
772 unsigned loop = arg->loop;
773 u8 bjump = arg->bjump;
774
775 if (dry_run)
776 return SZ_DMALPEND;
777
778 buf[0] = CMD_DMALPEND;
779
780 if (loop)
781 buf[0] |= (1 << 2);
782
783 if (!forever)
784 buf[0] |= (1 << 4);
785
786 if (cond == SINGLE)
787 buf[0] |= (0 << 1) | (1 << 0);
788 else if (cond == BURST)
789 buf[0] |= (1 << 1) | (1 << 0);
790
791 buf[1] = bjump;
792
793 PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n",
794 forever ? "FE" : "END",
795 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'),
796 loop ? '1' : '0',
797 bjump);
798
799 return SZ_DMALPEND;
800}
801
802static inline u32 _emit_KILL(unsigned dry_run, u8 buf[])
803{
804 if (dry_run)
805 return SZ_DMAKILL;
806
807 buf[0] = CMD_DMAKILL;
808
809 return SZ_DMAKILL;
810}
811
812static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],
813 enum dmamov_dst dst, u32 val)
814{
815 if (dry_run)
816 return SZ_DMAMOV;
817
818 buf[0] = CMD_DMAMOV;
819 buf[1] = dst;
820 *((u32 *)&buf[2]) = val;
821
822 PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",
823 dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);
824
825 return SZ_DMAMOV;
826}
827
828static inline u32 _emit_NOP(unsigned dry_run, u8 buf[])
829{
830 if (dry_run)
831 return SZ_DMANOP;
832
833 buf[0] = CMD_DMANOP;
834
835 PL330_DBGCMD_DUMP(SZ_DMANOP, "\tDMANOP\n");
836
837 return SZ_DMANOP;
838}
839
840static inline u32 _emit_RMB(unsigned dry_run, u8 buf[])
841{
842 if (dry_run)
843 return SZ_DMARMB;
844
845 buf[0] = CMD_DMARMB;
846
847 PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n");
848
849 return SZ_DMARMB;
850}
851
852static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev)
853{
854 if (dry_run)
855 return SZ_DMASEV;
856
857 buf[0] = CMD_DMASEV;
858
859 ev &= 0x1f;
860 ev <<= 3;
861 buf[1] = ev;
862
863 PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3);
864
865 return SZ_DMASEV;
866}
867
868static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond)
869{
870 if (dry_run)
871 return SZ_DMAST;
872
873 buf[0] = CMD_DMAST;
874
875 if (cond == SINGLE)
876 buf[0] |= (0 << 1) | (1 << 0);
877 else if (cond == BURST)
878 buf[0] |= (1 << 1) | (1 << 0);
879
880 PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n",
881 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
882
883 return SZ_DMAST;
884}
885
886static inline u32 _emit_STP(unsigned dry_run, u8 buf[],
887 enum pl330_cond cond, u8 peri)
888{
889 if (dry_run)
890 return SZ_DMASTP;
891
892 buf[0] = CMD_DMASTP;
893
894 if (cond == BURST)
895 buf[0] |= (1 << 1);
896
897 peri &= 0x1f;
898 peri <<= 3;
899 buf[1] = peri;
900
901 PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n",
902 cond == SINGLE ? 'S' : 'B', peri >> 3);
903
904 return SZ_DMASTP;
905}
906
907static inline u32 _emit_STZ(unsigned dry_run, u8 buf[])
908{
909 if (dry_run)
910 return SZ_DMASTZ;
911
912 buf[0] = CMD_DMASTZ;
913
914 PL330_DBGCMD_DUMP(SZ_DMASTZ, "\tDMASTZ\n");
915
916 return SZ_DMASTZ;
917}
918
919static inline u32 _emit_WFE(unsigned dry_run, u8 buf[], u8 ev,
920 unsigned invalidate)
921{
922 if (dry_run)
923 return SZ_DMAWFE;
924
925 buf[0] = CMD_DMAWFE;
926
927 ev &= 0x1f;
928 ev <<= 3;
929 buf[1] = ev;
930
931 if (invalidate)
932 buf[1] |= (1 << 1);
933
934 PL330_DBGCMD_DUMP(SZ_DMAWFE, "\tDMAWFE %u%s\n",
935 ev >> 3, invalidate ? ", I" : "");
936
937 return SZ_DMAWFE;
938}
939
940static inline u32 _emit_WFP(unsigned dry_run, u8 buf[],
941 enum pl330_cond cond, u8 peri)
942{
943 if (dry_run)
944 return SZ_DMAWFP;
945
946 buf[0] = CMD_DMAWFP;
947
948 if (cond == SINGLE)
949 buf[0] |= (0 << 1) | (0 << 0);
950 else if (cond == BURST)
951 buf[0] |= (1 << 1) | (0 << 0);
952 else
953 buf[0] |= (0 << 1) | (1 << 0);
954
955 peri &= 0x1f;
956 peri <<= 3;
957 buf[1] = peri;
958
959 PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n",
960 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3);
961
962 return SZ_DMAWFP;
963}
964
965static inline u32 _emit_WMB(unsigned dry_run, u8 buf[])
966{
967 if (dry_run)
968 return SZ_DMAWMB;
969
970 buf[0] = CMD_DMAWMB;
971
972 PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n");
973
974 return SZ_DMAWMB;
975}
976
977struct _arg_GO {
978 u8 chan;
979 u32 addr;
980 unsigned ns;
981};
982
983static inline u32 _emit_GO(unsigned dry_run, u8 buf[],
984 const struct _arg_GO *arg)
985{
986 u8 chan = arg->chan;
987 u32 addr = arg->addr;
988 unsigned ns = arg->ns;
989
990 if (dry_run)
991 return SZ_DMAGO;
992
993 buf[0] = CMD_DMAGO;
994 buf[0] |= (ns << 1);
995
996 buf[1] = chan & 0x7;
997
998 *((u32 *)&buf[2]) = addr;
999
1000 return SZ_DMAGO;
1001}
1002
1003#define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t)
1004
1005/* Returns Time-Out */
1006static bool _until_dmac_idle(struct pl330_thread *thrd)
1007{
1008 void __iomem *regs = thrd->dmac->pinfo->base;
1009 unsigned long loops = msecs_to_loops(5);
1010
1011 do {
1012 /* Until Manager is Idle */
1013 if (!(readl(regs + DBGSTATUS) & DBG_BUSY))
1014 break;
1015
1016 cpu_relax();
1017 } while (--loops);
1018
1019 if (!loops)
1020 return true;
1021
1022 return false;
1023}
1024
1025static inline void _execute_DBGINSN(struct pl330_thread *thrd,
1026 u8 insn[], bool as_manager)
1027{
1028 void __iomem *regs = thrd->dmac->pinfo->base;
1029 u32 val;
1030
1031 val = (insn[0] << 16) | (insn[1] << 24);
1032 if (!as_manager) {
1033 val |= (1 << 0);
1034 val |= (thrd->id << 8); /* Channel Number */
1035 }
1036 writel(val, regs + DBGINST0);
1037
1038 val = *((u32 *)&insn[2]);
1039 writel(val, regs + DBGINST1);
1040
1041 /* If timed out due to halted state-machine */
1042 if (_until_dmac_idle(thrd)) {
1043 dev_err(thrd->dmac->pinfo->dev, "DMAC halted!\n");
1044 return;
1045 }
1046
1047 /* Get going */
1048 writel(0, regs + DBGCMD);
1049}
1050
1051/*
1052 * Mark a _pl330_req as free.
1053 * We do it by writing DMAEND as the first instruction
1054 * because no valid request is going to have DMAEND as
1055 * its first instruction to execute.
1056 */
1057static void mark_free(struct pl330_thread *thrd, int idx)
1058{
1059 struct _pl330_req *req = &thrd->req[idx];
1060
1061 _emit_END(0, req->mc_cpu);
1062 req->mc_len = 0;
1063
1064 thrd->req_running = -1;
1065}
1066
1067static inline u32 _state(struct pl330_thread *thrd)
1068{
1069 void __iomem *regs = thrd->dmac->pinfo->base;
1070 u32 val;
1071
1072 if (is_manager(thrd))
1073 val = readl(regs + DS) & 0xf;
1074 else
1075 val = readl(regs + CS(thrd->id)) & 0xf;
1076
1077 switch (val) {
1078 case DS_ST_STOP:
1079 return PL330_STATE_STOPPED;
1080 case DS_ST_EXEC:
1081 return PL330_STATE_EXECUTING;
1082 case DS_ST_CMISS:
1083 return PL330_STATE_CACHEMISS;
1084 case DS_ST_UPDTPC:
1085 return PL330_STATE_UPDTPC;
1086 case DS_ST_WFE:
1087 return PL330_STATE_WFE;
1088 case DS_ST_FAULT:
1089 return PL330_STATE_FAULTING;
1090 case DS_ST_ATBRR:
1091 if (is_manager(thrd))
1092 return PL330_STATE_INVALID;
1093 else
1094 return PL330_STATE_ATBARRIER;
1095 case DS_ST_QBUSY:
1096 if (is_manager(thrd))
1097 return PL330_STATE_INVALID;
1098 else
1099 return PL330_STATE_QUEUEBUSY;
1100 case DS_ST_WFP:
1101 if (is_manager(thrd))
1102 return PL330_STATE_INVALID;
1103 else
1104 return PL330_STATE_WFP;
1105 case DS_ST_KILL:
1106 if (is_manager(thrd))
1107 return PL330_STATE_INVALID;
1108 else
1109 return PL330_STATE_KILLING;
1110 case DS_ST_CMPLT:
1111 if (is_manager(thrd))
1112 return PL330_STATE_INVALID;
1113 else
1114 return PL330_STATE_COMPLETING;
1115 case DS_ST_FLTCMP:
1116 if (is_manager(thrd))
1117 return PL330_STATE_INVALID;
1118 else
1119 return PL330_STATE_FAULT_COMPLETING;
1120 default:
1121 return PL330_STATE_INVALID;
1122 }
1123}
1124
1125static void _stop(struct pl330_thread *thrd)
1126{
1127 void __iomem *regs = thrd->dmac->pinfo->base;
1128 u8 insn[6] = {0, 0, 0, 0, 0, 0};
1129
1130 if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
1131 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1132
1133 /* Return if nothing needs to be done */
1134 if (_state(thrd) == PL330_STATE_COMPLETING
1135 || _state(thrd) == PL330_STATE_KILLING
1136 || _state(thrd) == PL330_STATE_STOPPED)
1137 return;
1138
1139 _emit_KILL(0, insn);
1140
1141 /* Stop generating interrupts for SEV */
1142 writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN);
1143
1144 _execute_DBGINSN(thrd, insn, is_manager(thrd));
1145}
1146
1147/* Start doing req 'idx' of thread 'thrd' */
1148static bool _trigger(struct pl330_thread *thrd)
1149{
1150 void __iomem *regs = thrd->dmac->pinfo->base;
1151 struct _pl330_req *req;
1152 struct pl330_req *r;
1153 struct _arg_GO go;
1154 unsigned ns;
1155 u8 insn[6] = {0, 0, 0, 0, 0, 0};
1156 int idx;
1157
1158 /* Return if already ACTIVE */
1159 if (_state(thrd) != PL330_STATE_STOPPED)
1160 return true;
1161
1162 idx = 1 - thrd->lstenq;
1163 if (!IS_FREE(&thrd->req[idx]))
1164 req = &thrd->req[idx];
1165 else {
1166 idx = thrd->lstenq;
1167 if (!IS_FREE(&thrd->req[idx]))
1168 req = &thrd->req[idx];
1169 else
1170 req = NULL;
1171 }
1172
1173 /* Return if no request */
1174 if (!req || !req->r)
1175 return true;
1176
1177 r = req->r;
1178
1179 if (r->cfg)
1180 ns = r->cfg->nonsecure ? 1 : 0;
1181 else if (readl(regs + CS(thrd->id)) & CS_CNS)
1182 ns = 1;
1183 else
1184 ns = 0;
1185
1186 /* See 'Abort Sources' point-4 at Page 2-25 */
1187 if (_manager_ns(thrd) && !ns)
1188 dev_info(thrd->dmac->pinfo->dev, "%s:%d Recipe for ABORT!\n",
1189 __func__, __LINE__);
1190
1191 go.chan = thrd->id;
1192 go.addr = req->mc_bus;
1193 go.ns = ns;
1194 _emit_GO(0, insn, &go);
1195
1196 /* Set to generate interrupts for SEV */
1197 writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN);
1198
1199 /* Only manager can execute GO */
1200 _execute_DBGINSN(thrd, insn, true);
1201
1202 thrd->req_running = idx;
1203
1204 return true;
1205}
1206
1207static bool _start(struct pl330_thread *thrd)
1208{
1209 switch (_state(thrd)) {
1210 case PL330_STATE_FAULT_COMPLETING:
1211 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1212
1213 if (_state(thrd) == PL330_STATE_KILLING)
1214 UNTIL(thrd, PL330_STATE_STOPPED)
1215
1216 case PL330_STATE_FAULTING:
1217 _stop(thrd);
1218
1219 case PL330_STATE_KILLING:
1220 case PL330_STATE_COMPLETING:
1221 UNTIL(thrd, PL330_STATE_STOPPED)
1222
1223 case PL330_STATE_STOPPED:
1224 return _trigger(thrd);
1225
1226 case PL330_STATE_WFP:
1227 case PL330_STATE_QUEUEBUSY:
1228 case PL330_STATE_ATBARRIER:
1229 case PL330_STATE_UPDTPC:
1230 case PL330_STATE_CACHEMISS:
1231 case PL330_STATE_EXECUTING:
1232 return true;
1233
1234 case PL330_STATE_WFE: /* For RESUME, nothing yet */
1235 default:
1236 return false;
1237 }
1238}
1239
1240static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
1241 const struct _xfer_spec *pxs, int cyc)
1242{
1243 int off = 0;
1244
1245 while (cyc--) {
1246 off += _emit_LD(dry_run, &buf[off], ALWAYS);
1247 off += _emit_RMB(dry_run, &buf[off]);
1248 off += _emit_ST(dry_run, &buf[off], ALWAYS);
1249 off += _emit_WMB(dry_run, &buf[off]);
1250 }
1251
1252 return off;
1253}
1254
1255static inline int _ldst_devtomem(unsigned dry_run, u8 buf[],
1256 const struct _xfer_spec *pxs, int cyc)
1257{
1258 int off = 0;
1259
1260 while (cyc--) {
1261 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1262 off += _emit_LDP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1263 off += _emit_ST(dry_run, &buf[off], ALWAYS);
1264 off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri);
1265 }
1266
1267 return off;
1268}
1269
1270static inline int _ldst_memtodev(unsigned dry_run, u8 buf[],
1271 const struct _xfer_spec *pxs, int cyc)
1272{
1273 int off = 0;
1274
1275 while (cyc--) {
1276 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1277 off += _emit_LD(dry_run, &buf[off], ALWAYS);
1278 off += _emit_STP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1279 off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri);
1280 }
1281
1282 return off;
1283}
1284
1285static int _bursts(unsigned dry_run, u8 buf[],
1286 const struct _xfer_spec *pxs, int cyc)
1287{
1288 int off = 0;
1289
1290 switch (pxs->r->rqtype) {
1291 case MEMTODEV:
1292 off += _ldst_memtodev(dry_run, &buf[off], pxs, cyc);
1293 break;
1294 case DEVTOMEM:
1295 off += _ldst_devtomem(dry_run, &buf[off], pxs, cyc);
1296 break;
1297 case MEMTOMEM:
1298 off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
1299 break;
1300 default:
1301 off += 0x40000000; /* Scare off the Client */
1302 break;
1303 }
1304
1305 return off;
1306}
1307
1308/* Returns bytes consumed and updates bursts */
1309static inline int _loop(unsigned dry_run, u8 buf[],
1310 unsigned long *bursts, const struct _xfer_spec *pxs)
1311{
1312 int cyc, cycmax, szlp, szlpend, szbrst, off;
1313 unsigned lcnt0, lcnt1, ljmp0, ljmp1;
1314 struct _arg_LPEND lpend;
1315
1316 /* Max iterations possible in DMALP is 256 */
1317 if (*bursts >= 256*256) {
1318 lcnt1 = 256;
1319 lcnt0 = 256;
1320 cyc = *bursts / lcnt1 / lcnt0;
1321 } else if (*bursts > 256) {
1322 lcnt1 = 256;
1323 lcnt0 = *bursts / lcnt1;
1324 cyc = 1;
1325 } else {
1326 lcnt1 = *bursts;
1327 lcnt0 = 0;
1328 cyc = 1;
1329 }
1330
1331 szlp = _emit_LP(1, buf, 0, 0);
1332 szbrst = _bursts(1, buf, pxs, 1);
1333
1334 lpend.cond = ALWAYS;
1335 lpend.forever = false;
1336 lpend.loop = 0;
1337 lpend.bjump = 0;
1338 szlpend = _emit_LPEND(1, buf, &lpend);
1339
1340 if (lcnt0) {
1341 szlp *= 2;
1342 szlpend *= 2;
1343 }
1344
1345 /*
1346 * Max bursts that we can unroll due to limit on the
1347 * size of backward jump that can be encoded in DMALPEND
1348 * which is 8-bits and hence 255
1349 */
1350 cycmax = (255 - (szlp + szlpend)) / szbrst;
1351
1352 cyc = (cycmax < cyc) ? cycmax : cyc;
1353
1354 off = 0;
1355
1356 if (lcnt0) {
1357 off += _emit_LP(dry_run, &buf[off], 0, lcnt0);
1358 ljmp0 = off;
1359 }
1360
1361 off += _emit_LP(dry_run, &buf[off], 1, lcnt1);
1362 ljmp1 = off;
1363
1364 off += _bursts(dry_run, &buf[off], pxs, cyc);
1365
1366 lpend.cond = ALWAYS;
1367 lpend.forever = false;
1368 lpend.loop = 1;
1369 lpend.bjump = off - ljmp1;
1370 off += _emit_LPEND(dry_run, &buf[off], &lpend);
1371
1372 if (lcnt0) {
1373 lpend.cond = ALWAYS;
1374 lpend.forever = false;
1375 lpend.loop = 0;
1376 lpend.bjump = off - ljmp0;
1377 off += _emit_LPEND(dry_run, &buf[off], &lpend);
1378 }
1379
1380 *bursts = lcnt1 * cyc;
1381 if (lcnt0)
1382 *bursts *= lcnt0;
1383
1384 return off;
1385}
1386
1387static inline int _setup_loops(unsigned dry_run, u8 buf[],
1388 const struct _xfer_spec *pxs)
1389{
1390 struct pl330_xfer *x = pxs->x;
1391 u32 ccr = pxs->ccr;
1392 unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
1393 int off = 0;
1394
1395 while (bursts) {
1396 c = bursts;
1397 off += _loop(dry_run, &buf[off], &c, pxs);
1398 bursts -= c;
1399 }
1400
1401 return off;
1402}
1403
1404static inline int _setup_xfer(unsigned dry_run, u8 buf[],
1405 const struct _xfer_spec *pxs)
1406{
1407 struct pl330_xfer *x = pxs->x;
1408 int off = 0;
1409
1410 /* DMAMOV SAR, x->src_addr */
1411 off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr);
1412 /* DMAMOV DAR, x->dst_addr */
1413 off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr);
1414
1415 /* Setup Loop(s) */
1416 off += _setup_loops(dry_run, &buf[off], pxs);
1417
1418 return off;
1419}
1420
1421/*
1422 * A req is a sequence of one or more xfer units.
1423 * Returns the number of bytes taken to setup the MC for the req.
1424 */
1425static int _setup_req(unsigned dry_run, struct pl330_thread *thrd,
1426 unsigned index, struct _xfer_spec *pxs)
1427{
1428 struct _pl330_req *req = &thrd->req[index];
1429 struct pl330_xfer *x;
1430 u8 *buf = req->mc_cpu;
1431 int off = 0;
1432
1433 PL330_DBGMC_START(req->mc_bus);
1434
1435 /* DMAMOV CCR, ccr */
1436 off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
1437
1438 x = pxs->r->x;
1439 do {
1440 /* Error if xfer length is not aligned at burst size */
1441 if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr)))
1442 return -EINVAL;
1443
1444 pxs->x = x;
1445 off += _setup_xfer(dry_run, &buf[off], pxs);
1446
1447 x = x->next;
1448 } while (x);
1449
1450 /* DMASEV peripheral/event */
1451 off += _emit_SEV(dry_run, &buf[off], thrd->ev);
1452 /* DMAEND */
1453 off += _emit_END(dry_run, &buf[off]);
1454
1455 return off;
1456}
1457
1458static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc)
1459{
1460 u32 ccr = 0;
1461
1462 if (rqc->src_inc)
1463 ccr |= CC_SRCINC;
1464
1465 if (rqc->dst_inc)
1466 ccr |= CC_DSTINC;
1467
1468 /* We set same protection levels for Src and DST for now */
1469 if (rqc->privileged)
1470 ccr |= CC_SRCPRI | CC_DSTPRI;
1471 if (rqc->nonsecure)
1472 ccr |= CC_SRCNS | CC_DSTNS;
1473 if (rqc->insnaccess)
1474 ccr |= CC_SRCIA | CC_DSTIA;
1475
1476 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT);
1477 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT);
1478
1479 ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT);
1480 ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT);
1481
1482 ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT);
1483 ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT);
1484
1485 ccr |= (rqc->swap << CC_SWAP_SHFT);
1486
1487 return ccr;
1488}
1489
1490static inline bool _is_valid(u32 ccr)
1491{
1492 enum pl330_dstcachectrl dcctl;
1493 enum pl330_srccachectrl scctl;
1494
1495 dcctl = (ccr >> CC_DSTCCTRL_SHFT) & CC_DRCCCTRL_MASK;
1496 scctl = (ccr >> CC_SRCCCTRL_SHFT) & CC_SRCCCTRL_MASK;
1497
1498 if (dcctl == DINVALID1 || dcctl == DINVALID2
1499 || scctl == SINVALID1 || scctl == SINVALID2)
1500 return false;
1501 else
1502 return true;
1503}
1504
1505/*
1506 * Submit a list of xfers after which the client wants notification.
1507 * Client is not notified after each xfer unit, just once after all
1508 * xfer units are done or some error occurs.
1509 */
1510static int pl330_submit_req(void *ch_id, struct pl330_req *r)
1511{
1512 struct pl330_thread *thrd = ch_id;
1513 struct pl330_dmac *pl330;
1514 struct pl330_info *pi;
1515 struct _xfer_spec xs;
1516 unsigned long flags;
1517 void __iomem *regs;
1518 unsigned idx;
1519 u32 ccr;
1520 int ret = 0;
1521
1522 /* No Req or Unacquired Channel or DMAC */
1523 if (!r || !thrd || thrd->free)
1524 return -EINVAL;
1525
1526 pl330 = thrd->dmac;
1527 pi = pl330->pinfo;
1528 regs = pi->base;
1529
1530 if (pl330->state == DYING
1531 || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
1532 dev_info(thrd->dmac->pinfo->dev, "%s:%d\n",
1533 __func__, __LINE__);
1534 return -EAGAIN;
1535 }
1536
1537 /* If request for non-existing peripheral */
1538 if (r->rqtype != MEMTOMEM && r->peri >= pi->pcfg.num_peri) {
1539 dev_info(thrd->dmac->pinfo->dev,
1540 "%s:%d Invalid peripheral(%u)!\n",
1541 __func__, __LINE__, r->peri);
1542 return -EINVAL;
1543 }
1544
1545 spin_lock_irqsave(&pl330->lock, flags);
1546
1547 if (_queue_full(thrd)) {
1548 ret = -EAGAIN;
1549 goto xfer_exit;
1550 }
1551
1552 /* Prefer Secure Channel */
1553 if (!_manager_ns(thrd))
1554 r->cfg->nonsecure = 0;
1555 else
1556 r->cfg->nonsecure = 1;
1557
1558 /* Use last settings, if not provided */
1559 if (r->cfg)
1560 ccr = _prepare_ccr(r->cfg);
1561 else
1562 ccr = readl(regs + CC(thrd->id));
1563
1564 /* If this req doesn't have valid xfer settings */
1565 if (!_is_valid(ccr)) {
1566 ret = -EINVAL;
1567 dev_info(thrd->dmac->pinfo->dev, "%s:%d Invalid CCR(%x)!\n",
1568 __func__, __LINE__, ccr);
1569 goto xfer_exit;
1570 }
1571
1572 idx = IS_FREE(&thrd->req[0]) ? 0 : 1;
1573
1574 xs.ccr = ccr;
1575 xs.r = r;
1576
1577 /* First dry run to check if req is acceptable */
1578 ret = _setup_req(1, thrd, idx, &xs);
1579 if (ret < 0)
1580 goto xfer_exit;
1581
1582 if (ret > pi->mcbufsz / 2) {
1583 dev_info(thrd->dmac->pinfo->dev,
1584 "%s:%d Trying increasing mcbufsz\n",
1585 __func__, __LINE__);
1586 ret = -ENOMEM;
1587 goto xfer_exit;
1588 }
1589
1590 /* Hook the request */
1591 thrd->lstenq = idx;
1592 thrd->req[idx].mc_len = _setup_req(0, thrd, idx, &xs);
1593 thrd->req[idx].r = r;
1594
1595 ret = 0;
1596
1597xfer_exit:
1598 spin_unlock_irqrestore(&pl330->lock, flags);
1599
1600 return ret;
1601}
1602
1603static void pl330_dotask(unsigned long data)
1604{
1605 struct pl330_dmac *pl330 = (struct pl330_dmac *) data;
1606 struct pl330_info *pi = pl330->pinfo;
1607 unsigned long flags;
1608 int i;
1609
1610 spin_lock_irqsave(&pl330->lock, flags);
1611
1612 /* The DMAC itself gone nuts */
1613 if (pl330->dmac_tbd.reset_dmac) {
1614 pl330->state = DYING;
1615 /* Reset the manager too */
1616 pl330->dmac_tbd.reset_mngr = true;
1617 /* Clear the reset flag */
1618 pl330->dmac_tbd.reset_dmac = false;
1619 }
1620
1621 if (pl330->dmac_tbd.reset_mngr) {
1622 _stop(pl330->manager);
1623 /* Reset all channels */
1624 pl330->dmac_tbd.reset_chan = (1 << pi->pcfg.num_chan) - 1;
1625 /* Clear the reset flag */
1626 pl330->dmac_tbd.reset_mngr = false;
1627 }
1628
1629 for (i = 0; i < pi->pcfg.num_chan; i++) {
1630
1631 if (pl330->dmac_tbd.reset_chan & (1 << i)) {
1632 struct pl330_thread *thrd = &pl330->channels[i];
1633 void __iomem *regs = pi->base;
1634 enum pl330_op_err err;
1635
1636 _stop(thrd);
1637
1638 if (readl(regs + FSC) & (1 << thrd->id))
1639 err = PL330_ERR_FAIL;
1640 else
1641 err = PL330_ERR_ABORT;
1642
1643 spin_unlock_irqrestore(&pl330->lock, flags);
1644
1645 _callback(thrd->req[1 - thrd->lstenq].r, err);
1646 _callback(thrd->req[thrd->lstenq].r, err);
1647
1648 spin_lock_irqsave(&pl330->lock, flags);
1649
1650 thrd->req[0].r = NULL;
1651 thrd->req[1].r = NULL;
1652 mark_free(thrd, 0);
1653 mark_free(thrd, 1);
1654
1655 /* Clear the reset flag */
1656 pl330->dmac_tbd.reset_chan &= ~(1 << i);
1657 }
1658 }
1659
1660 spin_unlock_irqrestore(&pl330->lock, flags);
1661
1662 return;
1663}
1664
1665/* Returns 1 if state was updated, 0 otherwise */
1666static int pl330_update(const struct pl330_info *pi)
1667{
1668 struct _pl330_req *rqdone;
1669 struct pl330_dmac *pl330;
1670 unsigned long flags;
1671 void __iomem *regs;
1672 u32 val;
1673 int id, ev, ret = 0;
1674
1675 if (!pi || !pi->pl330_data)
1676 return 0;
1677
1678 regs = pi->base;
1679 pl330 = pi->pl330_data;
1680
1681 spin_lock_irqsave(&pl330->lock, flags);
1682
1683 val = readl(regs + FSM) & 0x1;
1684 if (val)
1685 pl330->dmac_tbd.reset_mngr = true;
1686 else
1687 pl330->dmac_tbd.reset_mngr = false;
1688
1689 val = readl(regs + FSC) & ((1 << pi->pcfg.num_chan) - 1);
1690 pl330->dmac_tbd.reset_chan |= val;
1691 if (val) {
1692 int i = 0;
1693 while (i < pi->pcfg.num_chan) {
1694 if (val & (1 << i)) {
1695 dev_info(pi->dev,
1696 "Reset Channel-%d\t CS-%x FTC-%x\n",
1697 i, readl(regs + CS(i)),
1698 readl(regs + FTC(i)));
1699 _stop(&pl330->channels[i]);
1700 }
1701 i++;
1702 }
1703 }
1704
1705 /* Check which event happened i.e, thread notified */
1706 val = readl(regs + ES);
1707 if (pi->pcfg.num_events < 32
1708 && val & ~((1 << pi->pcfg.num_events) - 1)) {
1709 pl330->dmac_tbd.reset_dmac = true;
1710 dev_err(pi->dev, "%s:%d Unexpected!\n", __func__, __LINE__);
1711 ret = 1;
1712 goto updt_exit;
1713 }
1714
1715 for (ev = 0; ev < pi->pcfg.num_events; ev++) {
1716 if (val & (1 << ev)) { /* Event occurred */
1717 struct pl330_thread *thrd;
1718 u32 inten = readl(regs + INTEN);
1719 int active;
1720
1721 /* Clear the event */
1722 if (inten & (1 << ev))
1723 writel(1 << ev, regs + INTCLR);
1724
1725 ret = 1;
1726
1727 id = pl330->events[ev];
1728
1729 thrd = &pl330->channels[id];
1730
1731 active = thrd->req_running;
1732 if (active == -1) /* Aborted */
1733 continue;
1734
1735 rqdone = &thrd->req[active];
1736 mark_free(thrd, active);
1737
1738 /* Get going again ASAP */
1739 _start(thrd);
1740
1741 /* For now, just make a list of callbacks to be done */
1742 list_add_tail(&rqdone->rqd, &pl330->req_done);
1743 }
1744 }
1745
1746 /* Now that we are in no hurry, do the callbacks */
1747 while (!list_empty(&pl330->req_done)) {
1748 struct pl330_req *r;
1749
1750 rqdone = container_of(pl330->req_done.next,
1751 struct _pl330_req, rqd);
1752
1753 list_del_init(&rqdone->rqd);
1754
1755 /* Detach the req */
1756 r = rqdone->r;
1757 rqdone->r = NULL;
1758
1759 spin_unlock_irqrestore(&pl330->lock, flags);
1760 _callback(r, PL330_ERR_NONE);
1761 spin_lock_irqsave(&pl330->lock, flags);
1762 }
1763
1764updt_exit:
1765 spin_unlock_irqrestore(&pl330->lock, flags);
1766
1767 if (pl330->dmac_tbd.reset_dmac
1768 || pl330->dmac_tbd.reset_mngr
1769 || pl330->dmac_tbd.reset_chan) {
1770 ret = 1;
1771 tasklet_schedule(&pl330->tasks);
1772 }
1773
1774 return ret;
1775}
1776
1777static int pl330_chan_ctrl(void *ch_id, enum pl330_chan_op op)
1778{
1779 struct pl330_thread *thrd = ch_id;
1780 struct pl330_dmac *pl330;
1781 unsigned long flags;
1782 int ret = 0, active = thrd->req_running;
1783
1784 if (!thrd || thrd->free || thrd->dmac->state == DYING)
1785 return -EINVAL;
1786
1787 pl330 = thrd->dmac;
1788
1789 spin_lock_irqsave(&pl330->lock, flags);
1790
1791 switch (op) {
1792 case PL330_OP_FLUSH:
1793 /* Make sure the channel is stopped */
1794 _stop(thrd);
1795
1796 thrd->req[0].r = NULL;
1797 thrd->req[1].r = NULL;
1798 mark_free(thrd, 0);
1799 mark_free(thrd, 1);
1800 break;
1801
1802 case PL330_OP_ABORT:
1803 /* Make sure the channel is stopped */
1804 _stop(thrd);
1805
1806 /* ABORT is only for the active req */
1807 if (active == -1)
1808 break;
1809
1810 thrd->req[active].r = NULL;
1811 mark_free(thrd, active);
1812
1813 /* Start the next */
1814 case PL330_OP_START:
1815 if ((active == -1) && !_start(thrd))
1816 ret = -EIO;
1817 break;
1818
1819 default:
1820 ret = -EINVAL;
1821 }
1822
1823 spin_unlock_irqrestore(&pl330->lock, flags);
1824 return ret;
1825}
1826
1827static int pl330_chan_status(void *ch_id, struct pl330_chanstatus *pstatus)
1828{
1829 struct pl330_thread *thrd = ch_id;
1830 struct pl330_dmac *pl330;
1831 struct pl330_info *pi;
1832 void __iomem *regs;
1833 int active;
1834 u32 val;
1835
1836 if (!pstatus || !thrd || thrd->free)
1837 return -EINVAL;
1838
1839 pl330 = thrd->dmac;
1840 pi = pl330->pinfo;
1841 regs = pi->base;
1842
1843 /* The client should remove the DMAC and add again */
1844 if (pl330->state == DYING)
1845 pstatus->dmac_halted = true;
1846 else
1847 pstatus->dmac_halted = false;
1848
1849 val = readl(regs + FSC);
1850 if (val & (1 << thrd->id))
1851 pstatus->faulting = true;
1852 else
1853 pstatus->faulting = false;
1854
1855 active = thrd->req_running;
1856
1857 if (active == -1) {
1858 /* Indicate that the thread is not running */
1859 pstatus->top_req = NULL;
1860 pstatus->wait_req = NULL;
1861 } else {
1862 pstatus->top_req = thrd->req[active].r;
1863 pstatus->wait_req = !IS_FREE(&thrd->req[1 - active])
1864 ? thrd->req[1 - active].r : NULL;
1865 }
1866
1867 pstatus->src_addr = readl(regs + SA(thrd->id));
1868 pstatus->dst_addr = readl(regs + DA(thrd->id));
1869
1870 return 0;
1871}
1872
1873/* Reserve an event */
1874static inline int _alloc_event(struct pl330_thread *thrd)
1875{
1876 struct pl330_dmac *pl330 = thrd->dmac;
1877 struct pl330_info *pi = pl330->pinfo;
1878 int ev;
1879
1880 for (ev = 0; ev < pi->pcfg.num_events; ev++)
1881 if (pl330->events[ev] == -1) {
1882 pl330->events[ev] = thrd->id;
1883 return ev;
1884 }
1885
1886 return -1;
1887}
1888
1889static bool _chan_ns(const struct pl330_info *pi, int i)
1890{
1891 return pi->pcfg.irq_ns & (1 << i);
1892}
1893
1894/* Upon success, returns IdentityToken for the
1895 * allocated channel, NULL otherwise.
1896 */
1897static void *pl330_request_channel(const struct pl330_info *pi)
1898{
1899 struct pl330_thread *thrd = NULL;
1900 struct pl330_dmac *pl330;
1901 unsigned long flags;
1902 int chans, i;
1903
1904 if (!pi || !pi->pl330_data)
1905 return NULL;
1906
1907 pl330 = pi->pl330_data;
1908
1909 if (pl330->state == DYING)
1910 return NULL;
1911
1912 chans = pi->pcfg.num_chan;
1913
1914 spin_lock_irqsave(&pl330->lock, flags);
1915
1916 for (i = 0; i < chans; i++) {
1917 thrd = &pl330->channels[i];
1918 if ((thrd->free) && (!_manager_ns(thrd) ||
1919 _chan_ns(pi, i))) {
1920 thrd->ev = _alloc_event(thrd);
1921 if (thrd->ev >= 0) {
1922 thrd->free = false;
1923 thrd->lstenq = 1;
1924 thrd->req[0].r = NULL;
1925 mark_free(thrd, 0);
1926 thrd->req[1].r = NULL;
1927 mark_free(thrd, 1);
1928 break;
1929 }
1930 }
1931 thrd = NULL;
1932 }
1933
1934 spin_unlock_irqrestore(&pl330->lock, flags);
1935
1936 return thrd;
1937}
1938
1939/* Release an event */
1940static inline void _free_event(struct pl330_thread *thrd, int ev)
1941{
1942 struct pl330_dmac *pl330 = thrd->dmac;
1943 struct pl330_info *pi = pl330->pinfo;
1944
1945 /* If the event is valid and was held by the thread */
1946 if (ev >= 0 && ev < pi->pcfg.num_events
1947 && pl330->events[ev] == thrd->id)
1948 pl330->events[ev] = -1;
1949}
1950
1951static void pl330_release_channel(void *ch_id)
1952{
1953 struct pl330_thread *thrd = ch_id;
1954 struct pl330_dmac *pl330;
1955 unsigned long flags;
1956
1957 if (!thrd || thrd->free)
1958 return;
1959
1960 _stop(thrd);
1961
1962 _callback(thrd->req[1 - thrd->lstenq].r, PL330_ERR_ABORT);
1963 _callback(thrd->req[thrd->lstenq].r, PL330_ERR_ABORT);
1964
1965 pl330 = thrd->dmac;
1966
1967 spin_lock_irqsave(&pl330->lock, flags);
1968 _free_event(thrd, thrd->ev);
1969 thrd->free = true;
1970 spin_unlock_irqrestore(&pl330->lock, flags);
1971}
1972
1973/* Initialize the structure for PL330 configuration, that can be used
1974 * by the client driver the make best use of the DMAC
1975 */
1976static void read_dmac_config(struct pl330_info *pi)
1977{
1978 void __iomem *regs = pi->base;
1979 u32 val;
1980
1981 val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT;
1982 val &= CRD_DATA_WIDTH_MASK;
1983 pi->pcfg.data_bus_width = 8 * (1 << val);
1984
1985 val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT;
1986 val &= CRD_DATA_BUFF_MASK;
1987 pi->pcfg.data_buf_dep = val + 1;
1988
1989 val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT;
1990 val &= CR0_NUM_CHANS_MASK;
1991 val += 1;
1992 pi->pcfg.num_chan = val;
1993
1994 val = readl(regs + CR0);
1995 if (val & CR0_PERIPH_REQ_SET) {
1996 val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK;
1997 val += 1;
1998 pi->pcfg.num_peri = val;
1999 pi->pcfg.peri_ns = readl(regs + CR4);
2000 } else {
2001 pi->pcfg.num_peri = 0;
2002 }
2003
2004 val = readl(regs + CR0);
2005 if (val & CR0_BOOT_MAN_NS)
2006 pi->pcfg.mode |= DMAC_MODE_NS;
2007 else
2008 pi->pcfg.mode &= ~DMAC_MODE_NS;
2009
2010 val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT;
2011 val &= CR0_NUM_EVENTS_MASK;
2012 val += 1;
2013 pi->pcfg.num_events = val;
2014
2015 pi->pcfg.irq_ns = readl(regs + CR3);
2016
2017 pi->pcfg.periph_id = get_id(pi, PERIPH_ID);
2018 pi->pcfg.pcell_id = get_id(pi, PCELL_ID);
2019}
2020
2021static inline void _reset_thread(struct pl330_thread *thrd)
2022{
2023 struct pl330_dmac *pl330 = thrd->dmac;
2024 struct pl330_info *pi = pl330->pinfo;
2025
2026 thrd->req[0].mc_cpu = pl330->mcode_cpu
2027 + (thrd->id * pi->mcbufsz);
2028 thrd->req[0].mc_bus = pl330->mcode_bus
2029 + (thrd->id * pi->mcbufsz);
2030 thrd->req[0].r = NULL;
2031 mark_free(thrd, 0);
2032
2033 thrd->req[1].mc_cpu = thrd->req[0].mc_cpu
2034 + pi->mcbufsz / 2;
2035 thrd->req[1].mc_bus = thrd->req[0].mc_bus
2036 + pi->mcbufsz / 2;
2037 thrd->req[1].r = NULL;
2038 mark_free(thrd, 1);
2039}
2040
2041static int dmac_alloc_threads(struct pl330_dmac *pl330)
2042{
2043 struct pl330_info *pi = pl330->pinfo;
2044 int chans = pi->pcfg.num_chan;
2045 struct pl330_thread *thrd;
2046 int i;
2047
2048 /* Allocate 1 Manager and 'chans' Channel threads */
2049 pl330->channels = kzalloc((1 + chans) * sizeof(*thrd),
2050 GFP_KERNEL);
2051 if (!pl330->channels)
2052 return -ENOMEM;
2053
2054 /* Init Channel threads */
2055 for (i = 0; i < chans; i++) {
2056 thrd = &pl330->channels[i];
2057 thrd->id = i;
2058 thrd->dmac = pl330;
2059 _reset_thread(thrd);
2060 thrd->free = true;
2061 }
2062
2063 /* MANAGER is indexed at the end */
2064 thrd = &pl330->channels[chans];
2065 thrd->id = chans;
2066 thrd->dmac = pl330;
2067 thrd->free = false;
2068 pl330->manager = thrd;
2069
2070 return 0;
2071}
2072
2073static int dmac_alloc_resources(struct pl330_dmac *pl330)
2074{
2075 struct pl330_info *pi = pl330->pinfo;
2076 int chans = pi->pcfg.num_chan;
2077 int ret;
2078
2079 /*
2080 * Alloc MicroCode buffer for 'chans' Channel threads.
2081 * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN)
2082 */
2083 pl330->mcode_cpu = dma_alloc_coherent(pi->dev,
2084 chans * pi->mcbufsz,
2085 &pl330->mcode_bus, GFP_KERNEL);
2086 if (!pl330->mcode_cpu) {
2087 dev_err(pi->dev, "%s:%d Can't allocate memory!\n",
2088 __func__, __LINE__);
2089 return -ENOMEM;
2090 }
2091
2092 ret = dmac_alloc_threads(pl330);
2093 if (ret) {
2094 dev_err(pi->dev, "%s:%d Can't to create channels for DMAC!\n",
2095 __func__, __LINE__);
2096 dma_free_coherent(pi->dev,
2097 chans * pi->mcbufsz,
2098 pl330->mcode_cpu, pl330->mcode_bus);
2099 return ret;
2100 }
2101
2102 return 0;
2103}
2104
2105static int pl330_add(struct pl330_info *pi)
2106{
2107 struct pl330_dmac *pl330;
2108 void __iomem *regs;
2109 int i, ret;
2110
2111 if (!pi || !pi->dev)
2112 return -EINVAL;
2113
2114 /* If already added */
2115 if (pi->pl330_data)
2116 return -EINVAL;
2117
2118 /*
2119 * If the SoC can perform reset on the DMAC, then do it
2120 * before reading its configuration.
2121 */
2122 if (pi->dmac_reset)
2123 pi->dmac_reset(pi);
2124
2125 regs = pi->base;
2126
2127 /* Check if we can handle this DMAC */
2128 if ((get_id(pi, PERIPH_ID) & 0xfffff) != PERIPH_ID_VAL
2129 || get_id(pi, PCELL_ID) != PCELL_ID_VAL) {
2130 dev_err(pi->dev, "PERIPH_ID 0x%x, PCELL_ID 0x%x !\n",
2131 get_id(pi, PERIPH_ID), get_id(pi, PCELL_ID));
2132 return -EINVAL;
2133 }
2134
2135 /* Read the configuration of the DMAC */
2136 read_dmac_config(pi);
2137
2138 if (pi->pcfg.num_events == 0) {
2139 dev_err(pi->dev, "%s:%d Can't work without events!\n",
2140 __func__, __LINE__);
2141 return -EINVAL;
2142 }
2143
2144 pl330 = kzalloc(sizeof(*pl330), GFP_KERNEL);
2145 if (!pl330) {
2146 dev_err(pi->dev, "%s:%d Can't allocate memory!\n",
2147 __func__, __LINE__);
2148 return -ENOMEM;
2149 }
2150
2151 /* Assign the info structure and private data */
2152 pl330->pinfo = pi;
2153 pi->pl330_data = pl330;
2154
2155 spin_lock_init(&pl330->lock);
2156
2157 INIT_LIST_HEAD(&pl330->req_done);
2158
2159 /* Use default MC buffer size if not provided */
2160 if (!pi->mcbufsz)
2161 pi->mcbufsz = MCODE_BUFF_PER_REQ * 2;
2162
2163 /* Mark all events as free */
2164 for (i = 0; i < pi->pcfg.num_events; i++)
2165 pl330->events[i] = -1;
2166
2167 /* Allocate resources needed by the DMAC */
2168 ret = dmac_alloc_resources(pl330);
2169 if (ret) {
2170 dev_err(pi->dev, "Unable to create channels for DMAC\n");
2171 kfree(pl330);
2172 return ret;
2173 }
2174
2175 tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330);
2176
2177 pl330->state = INIT;
2178
2179 return 0;
2180}
2181
2182static int dmac_free_threads(struct pl330_dmac *pl330)
2183{
2184 struct pl330_info *pi = pl330->pinfo;
2185 int chans = pi->pcfg.num_chan;
2186 struct pl330_thread *thrd;
2187 int i;
2188
2189 /* Release Channel threads */
2190 for (i = 0; i < chans; i++) {
2191 thrd = &pl330->channels[i];
2192 pl330_release_channel((void *)thrd);
2193 }
2194
2195 /* Free memory */
2196 kfree(pl330->channels);
2197
2198 return 0;
2199}
2200
2201static void dmac_free_resources(struct pl330_dmac *pl330)
2202{
2203 struct pl330_info *pi = pl330->pinfo;
2204 int chans = pi->pcfg.num_chan;
2205
2206 dmac_free_threads(pl330);
2207
2208 dma_free_coherent(pi->dev, chans * pi->mcbufsz,
2209 pl330->mcode_cpu, pl330->mcode_bus);
2210}
2211
2212static void pl330_del(struct pl330_info *pi)
2213{
2214 struct pl330_dmac *pl330;
2215
2216 if (!pi || !pi->pl330_data)
2217 return;
2218
2219 pl330 = pi->pl330_data;
2220
2221 pl330->state = UNINIT;
2222
2223 tasklet_kill(&pl330->tasks);
2224
2225 /* Free DMAC resources */
2226 dmac_free_resources(pl330);
2227
2228 kfree(pl330);
2229 pi->pl330_data = NULL;
2230}
2231
120/* forward declaration */ 2232/* forward declaration */
121static struct amba_driver pl330_driver; 2233static struct amba_driver pl330_driver;
122 2234
diff --git a/include/linux/amba/pl330.h b/include/linux/amba/pl330.h
index 12e023c19ac..fe93758e840 100644
--- a/include/linux/amba/pl330.h
+++ b/include/linux/amba/pl330.h
@@ -13,7 +13,6 @@
13#define __AMBA_PL330_H_ 13#define __AMBA_PL330_H_
14 14
15#include <linux/dmaengine.h> 15#include <linux/dmaengine.h>
16#include <asm/hardware/pl330.h>
17 16
18struct dma_pl330_platdata { 17struct dma_pl330_platdata {
19 /* 18 /*
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-22 11:37:29 -0400