aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/boot/prom.c
blob: 4bea2f4dcb067412be7610b23308a0d9c996302b (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
/*
 * Copyright (C) Paul Mackerras 1997.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */
#include <stdarg.h>
#include <stddef.h>
#include "string.h"
#include "stdio.h"
#include "prom.h"

int (*prom)(void *);

void *chosen_handle;

void *stdin;
void *stdout;
void *stderr;


int
write(void *handle, void *ptr, int nb)
{
	struct prom_args {
		char *service;
		int nargs;
		int nret;
		void *ihandle;
		void *addr;
		int len;
		int actual;
	} args;

	args.service = "write";
	args.nargs = 3;
	args.nret = 1;
	args.ihandle = handle;
	args.addr = ptr;
	args.len = nb;
	args.actual = -1;
	(*prom)(&args);
	return args.actual;
}

int
read(void *handle, void *ptr, int nb)
{
	struct prom_args {
		char *service;
		int nargs;
		int nret;
		void *ihandle;
		void *addr;
		int len;
		int actual;
	} args;

	args.service = "read";
	args.nargs = 3;
	args.nret = 1;
	args.ihandle = handle;
	args.addr = ptr;
	args.len = nb;
	args.actual = -1;
	(*prom)(&args);
	return args.actual;
}

void
exit()
{
	struct prom_args {
		char *service;
	} args;

	for (;;) {
		args.service = "exit";
		(*prom)(&args);
	}
}

void
pause(void)
{
	struct prom_args {
		char *service;
	} args;

	args.service = "enter";
	(*prom)(&args);
}

void *
finddevice(const char *name)
{
	struct prom_args {
		char *service;
		int nargs;
		int nret;
		const char *devspec;
		void *phandle;
	} args;

	args.service = "finddevice";
	args.nargs = 1;
	args.nret = 1;
	args.devspec = name;
	args.phandle = (void *) -1;
	(*prom)(&args);
	return args.phandle;
}

void *
claim(unsigned long virt, unsigned long size, unsigned long align)
{
	struct prom_args {
		char *service;
		int nargs;
		int nret;
		unsigned int virt;
		unsigned int size;
		unsigned int align;
		void *ret;
	} args;

	args.service = "claim";
	args.nargs = 3;
	args.nret = 1;
	args.virt = virt;
	args.size = size;
	args.align = align;
	(*prom)(&args);
	return args.ret;
}

int
getprop(void *phandle, const char *name, void *buf, int buflen)
{
	struct prom_args {
		char *service;
		int nargs;
		int nret;
		void *phandle;
		const char *name;
		void *buf;
		int buflen;
		int size;
	} args;

	args.service = "getprop";
	args.nargs = 4;
	args.nret = 1;
	args.phandle = phandle;
	args.name = name;
	args.buf = buf;
	args.buflen = buflen;
	args.size = -1;
	(*prom)(&args);
	return args.size;
}

int
putc(int c, void *f)
{
	char ch = c;

	if (c == '\n')
		putc('\r', f);
	return write(f, &ch, 1) == 1? c: -1;
}

int
putchar(int c)
{
	return putc(c, stdout);
}

int
fputs(char *str, void *f)
{
	int n = strlen(str);

	return write(f, str, n) == n? 0: -1;
}

size_t strnlen(const char * s, size_t count)
{
	const char *sc;

	for (sc = s; count-- && *sc != '\0'; ++sc)
		/* nothing */;
	return sc - s;
}

extern unsigned int __div64_32(unsigned long long *dividend,
			       unsigned int divisor);

/* The unnecessary pointer compare is there
 * to check for type safety (n must be 64bit)
 */
# define do_div(n,base) ({						\
	unsigned int __base = (base);					\
	unsigned int __rem;						\
	(void)(((typeof((n)) *)0) == ((unsigned long long *)0));	\
	if (((n) >> 32) == 0) {						\
		__rem = (unsigned int)(n) % __base;			\
		(n) = (unsigned int)(n) / __base;			\
	} else								\
		__rem = __div64_32(&(n), __base);			\
	__rem;								\
 })

static int skip_atoi(const char **s)
{
	int i, c;

	for (i = 0; '0' <= (c = **s) && c <= '9'; ++*s)
		i = i*10 + c - '0';
	return i;
}

#define ZEROPAD	1		/* pad with zero */
#define SIGN	2		/* unsigned/signed long */
#define PLUS	4		/* show plus */
#define SPACE	8		/* space if plus */
#define LEFT	16		/* left justified */
#define SPECIAL	32		/* 0x */
#define LARGE	64		/* use 'ABCDEF' instead of 'abcdef' */

static char * number(char * str, unsigned long long num, int base, int size, int precision, int type)
{
	char c,sign,tmp[66];
	const char *digits="0123456789abcdefghijklmnopqrstuvwxyz";
	int i;

	if (type & LARGE)
		digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ";
	if (type & LEFT)
		type &= ~ZEROPAD;
	if (base < 2 || base > 36)
		return 0;
	c = (type & ZEROPAD) ? '0' : ' ';
	sign = 0;
	if (type & SIGN) {
		if ((signed long long)num < 0) {
			sign = '-';
			num = - (signed long long)num;
			size--;
		} else if (type & PLUS) {
			sign = '+';
			size--;
		} else if (type & SPACE) {
			sign = ' ';
			size--;
		}
	}
	if (type & SPECIAL) {
		if (base == 16)
			size -= 2;
		else if (base == 8)
			size--;
	}
	i = 0;
	if (num == 0)
		tmp[i++]='0';
	else while (num != 0) {
		tmp[i++] = digits[do_div(num, base)];
	}
	if (i > precision)
		precision = i;
	size -= precision;
	if (!(type&(ZEROPAD+LEFT)))
		while(size-->0)
			*str++ = ' ';
	if (sign)
		*str++ = sign;
	if (type & SPECIAL) {
		if (base==8)
			*str++ = '0';
		else if (base==16) {
			*str++ = '0';
			*str++ = digits[33];
		}
	}
	if (!(type & LEFT))
		while (size-- > 0)
			*str++ = c;
	while (i < precision--)
		*str++ = '0';
	while (i-- > 0)
		*str++ = tmp[i];
	while (size-- > 0)
		*str++ = ' ';
	return str;
}

int vsprintf(char *buf, const char *fmt, va_list args)
{
	int len;
	unsigned long long num;
	int i, base;
	char * str;
	const char *s;

	int flags;		/* flags to number() */

	int field_width;	/* width of output field */
	int precision;		/* min. # of digits for integers; max
				   number of chars for from string */
	int qualifier;		/* 'h', 'l', or 'L' for integer fields */
	                        /* 'z' support added 23/7/1999 S.H.    */
				/* 'z' changed to 'Z' --davidm 1/25/99 */

	
	for (str=buf ; *fmt ; ++fmt) {
		if (*fmt != '%') {
			*str++ = *fmt;
			continue;
		}
			
		/* process flags */
		flags = 0;
		repeat:
			++fmt;		/* this also skips first '%' */
			switch (*fmt) {
				case '-': flags |= LEFT; goto repeat;
				case '+': flags |= PLUS; goto repeat;
				case ' ': flags |= SPACE; goto repeat;
				case '#': flags |= SPECIAL; goto repeat;
				case '0': flags |= ZEROPAD; goto repeat;
				}
		
		/* get field width */
		field_width = -1;
		if ('0' <= *fmt && *fmt <= '9')
			field_width = skip_atoi(&fmt);
		else if (*fmt == '*') {
			++fmt;
			/* it's the next argument */
			field_width = va_arg(args, int);
			if (field_width < 0) {
				field_width = -field_width;
				flags |= LEFT;
			}
		}

		/* get the precision */
		precision = -1;
		if (*fmt == '.') {
			++fmt;	
			if ('0' <= *fmt && *fmt <= '9')
				precision = skip_atoi(&fmt);
			else if (*fmt == '*') {
				++fmt;
				/* it's the next argument */
				precision = va_arg(args, int);
			}
			if (precision < 0)
				precision = 0;
		}

		/* get the conversion qualifier */
		qualifier = -1;
		if (*fmt == 'h' || *fmt == 'l' || *fmt == 'L' || *fmt =='Z') {
			qualifier = *fmt;
			++fmt;
		}

		/* default base */
		base = 10;

		switch (*fmt) {
		case 'c':
			if (!(flags & LEFT))
				while (--field_width > 0)
					*str++ = ' ';
			*str++ = (unsigned char) va_arg(args, int);
			while (--field_width > 0)
				*str++ = ' ';
			continue;

		case 's':
			s = va_arg(args, char *);
			if (!s)
				s = "<NULL>";

			len = strnlen(s, precision);

			if (!(flags & LEFT))
				while (len < field_width--)
					*str++ = ' ';
			for (i = 0; i < len; ++i)
				*str++ = *s++;
			while (len < field_width--)
				*str++ = ' ';
			continue;

		case 'p':
			if (field_width == -1) {
				field_width = 2*sizeof(void *);
				flags |= ZEROPAD;
			}
			str = number(str,
				(unsigned long) va_arg(args, void *), 16,
				field_width, precision, flags);
			continue;


		case 'n':
			if (qualifier == 'l') {
				long * ip = va_arg(args, long *);
				*ip = (str - buf);
			} else if (qualifier == 'Z') {
				size_t * ip = va_arg(args, size_t *);
				*ip = (str - buf);
			} else {
				int * ip = va_arg(args, int *);
				*ip = (str - buf);
			}
			continue;

		case '%':
			*str++ = '%';
			continue;

		/* integer number formats - set up the flags and "break" */
		case 'o':
			base = 8;
			break;

		case 'X':
			flags |= LARGE;
		case 'x':
			base = 16;
			break;

		case 'd':
		case 'i':
			flags |= SIGN;
		case 'u':
			break;

		default:
			*str++ = '%';
			if (*fmt)
				*str++ = *fmt;
			else
				--fmt;
			continue;
		}
		if (qualifier == 'l') {
			num = va_arg(args, unsigned long);
			if (flags & SIGN)
				num = (signed long) num;
		} else if (qualifier == 'Z') {
			num = va_arg(args, size_t);
		} else if (qualifier == 'h') {
			num = (unsigned short) va_arg(args, int);
			if (flags & SIGN)
				num = (signed short) num;
		} else {
			num = va_arg(args, unsigned int);
			if (flags & SIGN)
				num = (signed int) num;
		}
		str = number(str, num, base, field_width, precision, flags);
	}
	*str = '\0';
	return str-buf;
}

int sprintf(char * buf, const char *fmt, ...)
{
	va_list args;
	int i;

	va_start(args, fmt);
	i=vsprintf(buf,fmt,args);
	va_end(args);
	return i;
}

static char sprint_buf[1024];

int
printf(const char *fmt, ...)
{
	va_list args;
	int n;

	va_start(args, fmt);
	n = vsprintf(sprint_buf, fmt, args);
	va_end(args);
	write(stdout, sprint_buf, n);
	return n;
}
pan>len); talitos_ptr->ptr = cpu_to_be32(dma_map_single(dev, data, len, dir)); talitos_ptr->j_extent = extent; } /* * unmap bus single (contiguous) h/w descriptor pointer */ static void unmap_single_talitos_ptr(struct device *dev, struct talitos_ptr *talitos_ptr, enum dma_data_direction dir) { dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr), be16_to_cpu(talitos_ptr->len), dir); } static int reset_channel(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; setbits32(priv->reg + TALITOS_CCCR(ch), TALITOS_CCCR_RESET); while ((in_be32(priv->reg + TALITOS_CCCR(ch)) & TALITOS_CCCR_RESET) && --timeout) cpu_relax(); if (timeout == 0) { dev_err(dev, "failed to reset channel %d\n", ch); return -EIO; } /* set done writeback and IRQ */ setbits32(priv->reg + TALITOS_CCCR_LO(ch), TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); return 0; } static int reset_device(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; setbits32(priv->reg + TALITOS_MCR, TALITOS_MCR_SWR); while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR) && --timeout) cpu_relax(); if (timeout == 0) { dev_err(dev, "failed to reset device\n"); return -EIO; } return 0; } /* * Reset and initialize the device */ static int init_device(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); int ch, err; /* * Master reset * errata documentation: warning: certain SEC interrupts * are not fully cleared by writing the MCR:SWR bit, * set bit twice to completely reset */ err = reset_device(dev); if (err) return err; err = reset_device(dev); if (err) return err; /* reset channels */ for (ch = 0; ch < priv->num_channels; ch++) { err = reset_channel(dev, ch); if (err) return err; } /* enable channel done and error interrupts */ setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT); setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); return 0; } /** * talitos_submit - submits a descriptor to the device for processing * @dev: the SEC device to be used * @desc: the descriptor to be processed by the device * @callback: whom to call when processing is complete * @context: a handle for use by caller (optional) * * desc must contain valid dma-mapped (bus physical) address pointers. * callback must check err and feedback in descriptor header * for device processing status. */ static int talitos_submit(struct device *dev, struct talitos_desc *desc, void (*callback)(struct device *dev, struct talitos_desc *desc, void *context, int error), void *context) { struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request; unsigned long flags, ch; int head; /* select done notification */ desc->hdr |= DESC_HDR_DONE_NOTIFY; /* emulate SEC's round-robin channel fifo polling scheme */ ch = atomic_inc_return(&priv->last_chan) & (priv->num_channels - 1); spin_lock_irqsave(&priv->head_lock[ch], flags); if (!atomic_inc_not_zero(&priv->submit_count[ch])) { /* h/w fifo is full */ spin_unlock_irqrestore(&priv->head_lock[ch], flags); return -EAGAIN; } head = priv->head[ch]; request = &priv->fifo[ch][head]; /* map descriptor and save caller data */ request->dma_desc = dma_map_single(dev, desc, sizeof(*desc), DMA_BIDIRECTIONAL); request->callback = callback; request->context = context; /* increment fifo head */ priv->head[ch] = (priv->head[ch] + 1) & (priv->fifo_len - 1); smp_wmb(); request->desc = desc; /* GO! */ wmb(); out_be32(priv->reg + TALITOS_FF_LO(ch), request->dma_desc); spin_unlock_irqrestore(&priv->head_lock[ch], flags); return -EINPROGRESS; } /* * process what was done, notify callback of error if not */ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) { struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request, saved_req; unsigned long flags; int tail, status; spin_lock_irqsave(&priv->tail_lock[ch], flags); tail = priv->tail[ch]; while (priv->fifo[ch][tail].desc) { request = &priv->fifo[ch][tail]; /* descriptors with their done bits set don't get the error */ rmb(); if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; else if (!error) break; else status = error; dma_unmap_single(dev, request->dma_desc, sizeof(struct talitos_desc), DMA_BIDIRECTIONAL); /* copy entries so we can call callback outside lock */ saved_req.desc = request->desc; saved_req.callback = request->callback; saved_req.context = request->context; /* release request entry in fifo */ smp_wmb(); request->desc = NULL; /* increment fifo tail */ priv->tail[ch] = (tail + 1) & (priv->fifo_len - 1); spin_unlock_irqrestore(&priv->tail_lock[ch], flags); atomic_dec(&priv->submit_count[ch]); saved_req.callback(dev, saved_req.desc, saved_req.context, status); /* channel may resume processing in single desc error case */ if (error && !reset_ch && status == error) return; spin_lock_irqsave(&priv->tail_lock[ch], flags); tail = priv->tail[ch]; } spin_unlock_irqrestore(&priv->tail_lock[ch], flags); } /* * process completed requests for channels that have done status */ static void talitos_done(unsigned long data) { struct device *dev = (struct device *)data; struct talitos_private *priv = dev_get_drvdata(dev); int ch; for (ch = 0; ch < priv->num_channels; ch++) flush_channel(dev, ch, 0, 0); } /* * locate current (offending) descriptor */ static struct talitos_desc *current_desc(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); int tail = priv->tail[ch]; dma_addr_t cur_desc; cur_desc = in_be32(priv->reg + TALITOS_CDPR_LO(ch)); while (priv->fifo[ch][tail].dma_desc != cur_desc) { tail = (tail + 1) & (priv->fifo_len - 1); if (tail == priv->tail[ch]) { dev_err(dev, "couldn't locate current descriptor\n"); return NULL; } } return priv->fifo[ch][tail].desc; } /* * user diagnostics; report root cause of error based on execution unit status */ static void report_eu_error(struct device *dev, int ch, struct talitos_desc *desc) { struct talitos_private *priv = dev_get_drvdata(dev); int i; switch (desc->hdr & DESC_HDR_SEL0_MASK) { case DESC_HDR_SEL0_AFEU: dev_err(dev, "AFEUISR 0x%08x_%08x\n", in_be32(priv->reg + TALITOS_AFEUISR), in_be32(priv->reg + TALITOS_AFEUISR_LO)); break; case DESC_HDR_SEL0_DEU: dev_err(dev, "DEUISR 0x%08x_%08x\n", in_be32(priv->reg + TALITOS_DEUISR), in_be32(priv->reg + TALITOS_DEUISR_LO)); break; case DESC_HDR_SEL0_MDEUA: case DESC_HDR_SEL0_MDEUB: dev_err(dev, "MDEUISR 0x%08x_%08x\n", in_be32(priv->reg + TALITOS_MDEUISR), in_be32(priv->reg + TALITOS_MDEUISR_LO)); break; case DESC_HDR_SEL0_RNG: dev_err(dev, "RNGUISR 0x%08x_%08x\n", in_be32(priv->reg + TALITOS_RNGUISR), in_be32(priv->reg + TALITOS_RNGUISR_LO)); break; case DESC_HDR_SEL0_PKEU: dev_err(dev, "PKEUISR 0x%08x_%08x\n", in_be32(priv->reg + TALITOS_PKEUISR), in_be32(priv->reg + TALITOS_PKEUISR_LO)); break; case DESC_HDR_SEL0_AESU: dev_err(dev, "AESUISR 0x%08x_%08x\n", in_be32(priv->reg + TALITOS_AESUISR), in_be32(priv->reg + TALITOS_AESUISR_LO)); break; case DESC_HDR_SEL0_CRCU: dev_err(dev, "CRCUISR 0x%08x_%08x\n", in_be32(priv->reg + TALITOS_CRCUISR), in_be32(priv->reg + TALITOS_CRCUISR_LO)); break; case DESC_HDR_SEL0_KEU: dev_err(dev, "KEUISR 0x%08x_%08x\n", in_be32(priv->reg + TALITOS_KEUISR), in_be32(priv->reg + TALITOS_KEUISR_LO)); break; } switch (desc->hdr & DESC_HDR_SEL1_MASK) { case DESC_HDR_SEL1_MDEUA: case DESC_HDR_SEL1_MDEUB: dev_err(dev, "MDEUISR 0x%08x_%08x\n", in_be32(priv->reg + TALITOS_MDEUISR), in_be32(priv->reg + TALITOS_MDEUISR_LO)); break; case DESC_HDR_SEL1_CRCU: dev_err(dev, "CRCUISR 0x%08x_%08x\n", in_be32(priv->reg + TALITOS_CRCUISR), in_be32(priv->reg + TALITOS_CRCUISR_LO)); break; } for (i = 0; i < 8; i++) dev_err(dev, "DESCBUF 0x%08x_%08x\n", in_be32(priv->reg + TALITOS_DESCBUF(ch) + 8*i), in_be32(priv->reg + TALITOS_DESCBUF_LO(ch) + 8*i)); } /* * recover from error interrupts */ static void talitos_error(unsigned long data) { struct device *dev = (struct device *)data; struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; int ch, error, reset_dev = 0, reset_ch = 0; u32 isr, isr_lo, v, v_lo; isr = in_be32(priv->reg + TALITOS_ISR); isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); for (ch = 0; ch < priv->num_channels; ch++) { /* skip channels without errors */ if (!(isr & (1 << (ch * 2 + 1)))) continue; error = -EINVAL; v = in_be32(priv->reg + TALITOS_CCPSR(ch)); v_lo = in_be32(priv->reg + TALITOS_CCPSR_LO(ch)); if (v_lo & TALITOS_CCPSR_LO_DOF) { dev_err(dev, "double fetch fifo overflow error\n"); error = -EAGAIN; reset_ch = 1; } if (v_lo & TALITOS_CCPSR_LO_SOF) { /* h/w dropped descriptor */ dev_err(dev, "single fetch fifo overflow error\n"); error = -EAGAIN; } if (v_lo & TALITOS_CCPSR_LO_MDTE) dev_err(dev, "master data transfer error\n"); if (v_lo & TALITOS_CCPSR_LO_SGDLZ) dev_err(dev, "s/g data length zero error\n"); if (v_lo & TALITOS_CCPSR_LO_FPZ) dev_err(dev, "fetch pointer zero error\n"); if (v_lo & TALITOS_CCPSR_LO_IDH) dev_err(dev, "illegal descriptor header error\n"); if (v_lo & TALITOS_CCPSR_LO_IEU) dev_err(dev, "invalid execution unit error\n"); if (v_lo & TALITOS_CCPSR_LO_EU) report_eu_error(dev, ch, current_desc(dev, ch)); if (v_lo & TALITOS_CCPSR_LO_GB) dev_err(dev, "gather boundary error\n"); if (v_lo & TALITOS_CCPSR_LO_GRL) dev_err(dev, "gather return/length error\n"); if (v_lo & TALITOS_CCPSR_LO_SB) dev_err(dev, "scatter boundary error\n"); if (v_lo & TALITOS_CCPSR_LO_SRL) dev_err(dev, "scatter return/length error\n"); flush_channel(dev, ch, error, reset_ch); if (reset_ch) { reset_channel(dev, ch); } else { setbits32(priv->reg + TALITOS_CCCR(ch), TALITOS_CCCR_CONT); setbits32(priv->reg + TALITOS_CCCR_LO(ch), 0); while ((in_be32(priv->reg + TALITOS_CCCR(ch)) & TALITOS_CCCR_CONT) && --timeout) cpu_relax(); if (timeout == 0) { dev_err(dev, "failed to restart channel %d\n", ch); reset_dev = 1; } } } if (reset_dev || isr & ~TALITOS_ISR_CHERR || isr_lo) { dev_err(dev, "done overflow, internal time out, or rngu error: " "ISR 0x%08x_%08x\n", isr, isr_lo); /* purge request queues */ for (ch = 0; ch < priv->num_channels; ch++) flush_channel(dev, ch, -EIO, 1); /* reset and reinitialize the device */ init_device(dev); } } static irqreturn_t talitos_interrupt(int irq, void *data) { struct device *dev = data; struct talitos_private *priv = dev_get_drvdata(dev); u32 isr, isr_lo; isr = in_be32(priv->reg + TALITOS_ISR); isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); /* ack */ out_be32(priv->reg + TALITOS_ICR, isr); out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo)) talitos_error((unsigned long)data); else if (likely(isr & TALITOS_ISR_CHDONE)) tasklet_schedule(&priv->done_task); return (isr || isr_lo) ? IRQ_HANDLED : IRQ_NONE; } /* * hwrng */ static int talitos_rng_data_present(struct hwrng *rng, int wait) { struct device *dev = (struct device *)rng->priv; struct talitos_private *priv = dev_get_drvdata(dev); u32 ofl; int i; for (i = 0; i < 20; i++) { ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_OFL; if (ofl || !wait) break; udelay(10); } return !!ofl; } static int talitos_rng_data_read(struct hwrng *rng, u32 *data) { struct device *dev = (struct device *)rng->priv; struct talitos_private *priv = dev_get_drvdata(dev); /* rng fifo requires 64-bit accesses */ *data = in_be32(priv->reg + TALITOS_RNGU_FIFO); *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO); return sizeof(u32); } static int talitos_rng_init(struct hwrng *rng) { struct device *dev = (struct device *)rng->priv; struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR); while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD) && --timeout) cpu_relax(); if (timeout == 0) { dev_err(dev, "failed to reset rng hw\n"); return -ENODEV; } /* start generating */ setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0); return 0; } static int talitos_register_rng(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); priv->rng.name = dev_driver_string(dev), priv->rng.init = talitos_rng_init, priv->rng.data_present = talitos_rng_data_present, priv->rng.data_read = talitos_rng_data_read, priv->rng.priv = (unsigned long)dev; return hwrng_register(&priv->rng); } static void talitos_unregister_rng(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); hwrng_unregister(&priv->rng); } /* * crypto alg */ #define TALITOS_CRA_PRIORITY 3000 #define TALITOS_MAX_KEY_SIZE 64 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ #define MD5_DIGEST_SIZE 16 struct talitos_ctx { struct device *dev; __be32 desc_hdr_template; u8 key[TALITOS_MAX_KEY_SIZE]; u8 iv[TALITOS_MAX_IV_LENGTH]; unsigned int keylen; unsigned int enckeylen; unsigned int authkeylen; unsigned int authsize; }; static int aead_authenc_setauthsize(struct crypto_aead *authenc, unsigned int authsize) { struct talitos_ctx *ctx = crypto_aead_ctx(authenc); ctx->authsize = authsize; return 0; } static int aead_authenc_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { struct talitos_ctx *ctx = crypto_aead_ctx(authenc); struct rtattr *rta = (void *)key; struct crypto_authenc_key_param *param; unsigned int authkeylen; unsigned int enckeylen; if (!RTA_OK(rta, keylen)) goto badkey; if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) goto badkey; if (RTA_PAYLOAD(rta) < sizeof(*param)) goto badkey; param = RTA_DATA(rta); enckeylen = be32_to_cpu(param->enckeylen); key += RTA_ALIGN(rta->rta_len); keylen -= RTA_ALIGN(rta->rta_len); if (keylen < enckeylen) goto badkey; authkeylen = keylen - enckeylen; if (keylen > TALITOS_MAX_KEY_SIZE) goto badkey; memcpy(&ctx->key, key, keylen); ctx->keylen = keylen; ctx->enckeylen = enckeylen; ctx->authkeylen = authkeylen; return 0; badkey: crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } /* * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor * @src_nents: number of segments in input scatterlist * @dst_nents: number of segments in output scatterlist * @dma_len: length of dma mapped link_tbl space * @dma_link_tbl: bus physical address of link_tbl * @desc: h/w descriptor * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) * * if decrypting (with authcheck), or either one of src_nents or dst_nents * is greater than 1, an integrity check value is concatenated to the end * of link_tbl data */ struct ipsec_esp_edesc { int src_nents; int dst_nents; int dma_len; dma_addr_t dma_link_tbl; struct talitos_desc desc; struct talitos_ptr link_tbl[0]; }; static void ipsec_esp_unmap(struct device *dev, struct ipsec_esp_edesc *edesc, struct aead_request *areq) { unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE); dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE); if (areq->src != areq->dst) { dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1, DMA_TO_DEVICE); dma_unmap_sg(dev, areq->dst, edesc->dst_nents ? : 1, DMA_FROM_DEVICE); } else { dma_unmap_sg(dev, areq->src, edesc->src_nents ? : 1, DMA_BIDIRECTIONAL); } if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); } /* * ipsec_esp descriptor callbacks */ static void ipsec_esp_encrypt_done(struct device *dev, struct talitos_desc *desc, void *context, int err) { struct aead_request *areq = context; struct ipsec_esp_edesc *edesc = container_of(desc, struct ipsec_esp_edesc, desc); struct crypto_aead *authenc = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); struct scatterlist *sg; void *icvdata; ipsec_esp_unmap(dev, edesc, areq); /* copy the generated ICV to dst */ if (edesc->dma_len) { icvdata = &edesc->link_tbl[edesc->src_nents + edesc->dst_nents + 2]; sg = sg_last(areq->dst, edesc->dst_nents); memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize, icvdata, ctx->authsize); } kfree(edesc); aead_request_complete(areq, err); } static void ipsec_esp_decrypt_done(struct device *dev, struct talitos_desc *desc, void *context, int err) { struct aead_request *req = context; struct ipsec_esp_edesc *edesc = container_of(desc, struct ipsec_esp_edesc, desc); struct crypto_aead *authenc = crypto_aead_reqtfm(req); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); struct scatterlist *sg; void *icvdata; ipsec_esp_unmap(dev, edesc, req); if (!err) { /* auth check */ if (edesc->dma_len) icvdata = &edesc->link_tbl[edesc->src_nents + edesc->dst_nents + 2]; else icvdata = &edesc->link_tbl[0]; sg = sg_last(req->dst, edesc->dst_nents ? : 1); err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize, ctx->authsize) ? -EBADMSG : 0; } kfree(edesc); aead_request_complete(req, err); } /* * convert scatterlist to SEC h/w link table format * stop at cryptlen bytes */ static int sg_to_link_tbl(struct scatterlist *sg, int sg_count, int cryptlen, struct talitos_ptr *link_tbl_ptr) { int n_sg = sg_count; while (n_sg--) { link_tbl_ptr->ptr = cpu_to_be32(sg_dma_address(sg)); link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg)); link_tbl_ptr->j_extent = 0; link_tbl_ptr++; cryptlen -= sg_dma_len(sg); sg = sg_next(sg); } /* adjust (decrease) last one (or two) entry's len to cryptlen */ link_tbl_ptr--; while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) { /* Empty this entry, and move to previous one */ cryptlen += be16_to_cpu(link_tbl_ptr->len); link_tbl_ptr->len = 0; sg_count--; link_tbl_ptr--; } link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len) + cryptlen); /* tag end of link table */ link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; return sg_count; } /* * fill in and submit ipsec_esp descriptor */ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, u8 *giv, u64 seq, void (*callback) (struct device *dev, struct talitos_desc *desc, void *context, int error)) { struct crypto_aead *aead = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(aead); struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; unsigned int cryptlen = areq->cryptlen; unsigned int authsize = ctx->authsize; unsigned int ivsize; int sg_count, ret; /* hmac key */ map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, 0, DMA_TO_DEVICE); /* hmac data */ map_single_talitos_ptr(dev, &desc->ptr[1], sg_virt(areq->src) - sg_virt(areq->assoc), sg_virt(areq->assoc), 0, DMA_TO_DEVICE); /* cipher iv */ ivsize = crypto_aead_ivsize(aead); map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0, DMA_TO_DEVICE); /* cipher key */ map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, (char *)&ctx->key + ctx->authkeylen, 0, DMA_TO_DEVICE); /* * cipher in * map and adjust cipher len to aead request cryptlen. * extent is bytes of HMAC postpended to ciphertext, * typically 12 for ipsec */ desc->ptr[4].len = cpu_to_be16(cryptlen); desc->ptr[4].j_extent = authsize; if (areq->src == areq->dst) sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1, DMA_BIDIRECTIONAL); else sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ? : 1, DMA_TO_DEVICE); if (sg_count == 1) { desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src)); } else { sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen, &edesc->link_tbl[0]); if (sg_count > 1) { struct talitos_ptr *link_tbl_ptr = &edesc->link_tbl[sg_count-1]; struct scatterlist *sg; struct talitos_private *priv = dev_get_drvdata(dev); desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; desc->ptr[4].ptr = cpu_to_be32(edesc->dma_link_tbl); dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); /* If necessary for this SEC revision, * add a link table entry for ICV. */ if ((priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT) && (edesc->desc.hdr & DESC_HDR_MODE0_ENCRYPT) == 0) { link_tbl_ptr->j_extent = 0; link_tbl_ptr++; link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; link_tbl_ptr->len = cpu_to_be16(authsize); sg = sg_last(areq->src, edesc->src_nents ? : 1); link_tbl_ptr->ptr = cpu_to_be32( (char *)sg_dma_address(sg) + sg->length - authsize); } } else { /* Only one segment now, so no link tbl needed */ desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src)); } } /* cipher out */ desc->ptr[5].len = cpu_to_be16(cryptlen); desc->ptr[5].j_extent = authsize; if (areq->src != areq->dst) { sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1, DMA_FROM_DEVICE); } if (sg_count == 1) { desc->ptr[5].ptr = cpu_to_be32(sg_dma_address(areq->dst)); } else { struct talitos_ptr *link_tbl_ptr = &edesc->link_tbl[edesc->src_nents + 1]; desc->ptr[5].ptr = cpu_to_be32((struct talitos_ptr *) edesc->dma_link_tbl + edesc->src_nents + 1); if (areq->src == areq->dst) { memcpy(link_tbl_ptr, &edesc->link_tbl[0], edesc->src_nents * sizeof(struct talitos_ptr)); } else { sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, link_tbl_ptr); } /* Add an entry to the link table for ICV data */ link_tbl_ptr += sg_count - 1; link_tbl_ptr->j_extent = 0; sg_count++; link_tbl_ptr++; link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; link_tbl_ptr->len = cpu_to_be16(authsize); /* icv data follows link tables */ link_tbl_ptr->ptr = cpu_to_be32((struct talitos_ptr *) edesc->dma_link_tbl + edesc->src_nents + edesc->dst_nents + 2); desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP; dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); } /* iv out */ map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0, DMA_FROM_DEVICE); ret = talitos_submit(dev, desc, callback, areq); if (ret != -EINPROGRESS) { ipsec_esp_unmap(dev, edesc, areq); kfree(edesc); } return ret; } /* * derive number of elements in scatterlist */ static int sg_count(struct scatterlist *sg_list, int nbytes) { struct scatterlist *sg = sg_list; int sg_nents = 0; while (nbytes) { sg_nents++; nbytes -= sg->length; sg = sg_next(sg); } return sg_nents; } /* * allocate and map the ipsec_esp extended descriptor */ static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq, int icv_stashing) { struct crypto_aead *authenc = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); struct ipsec_esp_edesc *edesc; int src_nents, dst_nents, alloc_len, dma_len; gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; if (areq->cryptlen + ctx->authsize > TALITOS_MAX_DATA_LEN) { dev_err(ctx->dev, "cryptlen exceeds h/w max limit\n"); return ERR_PTR(-EINVAL); } src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize); src_nents = (src_nents == 1) ? 0 : src_nents; if (areq->dst == areq->src) { dst_nents = src_nents; } else { dst_nents = sg_count(areq->dst, areq->cryptlen + ctx->authsize); dst_nents = (dst_nents == 1) ? 0 : dst_nents; } /* * allocate space for base edesc plus the link tables, * allowing for two separate entries for ICV and generated ICV (+ 2), * and the ICV data itself */ alloc_len = sizeof(struct ipsec_esp_edesc); if (src_nents || dst_nents) { dma_len = (src_nents + dst_nents + 2) * sizeof(struct talitos_ptr) + ctx->authsize; alloc_len += dma_len; } else { dma_len = 0; alloc_len += icv_stashing ? ctx->authsize : 0; } edesc = kmalloc(alloc_len, GFP_DMA | flags); if (!edesc) { dev_err(ctx->dev, "could not allocate edescriptor\n"); return ERR_PTR(-ENOMEM); } edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; edesc->dma_len = dma_len; edesc->dma_link_tbl = dma_map_single(ctx->dev, &edesc->link_tbl[0], edesc->dma_len, DMA_BIDIRECTIONAL); return edesc; } static int aead_authenc_encrypt(struct aead_request *req) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); struct ipsec_esp_edesc *edesc; /* allocate extended descriptor */ edesc = ipsec_esp_edesc_alloc(req, 0); if (IS_ERR(edesc)) return PTR_ERR(edesc); /* set encrypt */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done); } static int aead_authenc_decrypt(struct aead_request *req) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); unsigned int authsize = ctx->authsize; struct ipsec_esp_edesc *edesc; struct scatterlist *sg; void *icvdata; req->cryptlen -= authsize; /* allocate extended descriptor */ edesc = ipsec_esp_edesc_alloc(req, 1); if (IS_ERR(edesc)) return PTR_ERR(edesc); /* stash incoming ICV for later cmp with ICV generated by the h/w */ if (edesc->dma_len) icvdata = &edesc->link_tbl[edesc->src_nents + edesc->dst_nents + 2]; else icvdata = &edesc->link_tbl[0]; sg = sg_last(req->src, edesc->src_nents ? : 1); memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize, ctx->authsize); /* decrypt */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_done); } static int aead_authenc_givencrypt( struct aead_givcrypt_request *req) { struct aead_request *areq = &req->areq; struct crypto_aead *authenc = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); struct ipsec_esp_edesc *edesc; /* allocate extended descriptor */ edesc = ipsec_esp_edesc_alloc(areq, 0); if (IS_ERR(edesc)) return PTR_ERR(edesc); /* set encrypt */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc)); /* avoid consecutive packets going out with same IV */ *(__be64 *)req->giv ^= cpu_to_be64(req->seq); return ipsec_esp(edesc, areq, req->giv, req->seq, ipsec_esp_encrypt_done); } struct talitos_alg_template { char name[CRYPTO_MAX_ALG_NAME]; char driver_name[CRYPTO_MAX_ALG_NAME]; unsigned int blocksize; struct aead_alg aead; struct device *dev; __be32 desc_hdr_template; }; static struct talitos_alg_template driver_algs[] = { /* single-pass ipsec_esp descriptor */ { .name = "authenc(hmac(sha1),cbc(aes))", .driver_name = "authenc-hmac-sha1-cbc-aes-talitos", .blocksize = AES_BLOCK_SIZE, .aead = { .setkey = aead_authenc_setkey, .setauthsize = aead_authenc_setauthsize, .encrypt = aead_authenc_encrypt, .decrypt = aead_authenc_decrypt, .givencrypt = aead_authenc_givencrypt, .geniv = "<built-in>", .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | DESC_HDR_MODE0_AESU_CBC | DESC_HDR_SEL1_MDEUA | DESC_HDR_MODE1_MDEU_INIT | DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA1_HMAC, }, { .name = "authenc(hmac(sha1),cbc(des3_ede))", .driver_name = "authenc-hmac-sha1-cbc-3des-talitos", .blocksize = DES3_EDE_BLOCK_SIZE, .aead = {