aboutsummaryrefslogtreecommitdiffstats
path: root/arch/powerpc/kernel/align.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/powerpc/kernel/align.c')
-rw-r--r--arch/powerpc/kernel/align.c307
1 files changed, 299 insertions, 8 deletions
diff --git a/arch/powerpc/kernel/align.c b/arch/powerpc/kernel/align.c
index 5c9ff7f5c44e..e06f75daeba3 100644
--- a/arch/powerpc/kernel/align.c
+++ b/arch/powerpc/kernel/align.c
@@ -38,7 +38,7 @@ struct aligninfo {
38/* Bits in the flags field */ 38/* Bits in the flags field */
39#define LD 0 /* load */ 39#define LD 0 /* load */
40#define ST 1 /* store */ 40#define ST 1 /* store */
41#define SE 2 /* sign-extend value */ 41#define SE 2 /* sign-extend value, or FP ld/st as word */
42#define F 4 /* to/from fp regs */ 42#define F 4 /* to/from fp regs */
43#define U 8 /* update index register */ 43#define U 8 /* update index register */
44#define M 0x10 /* multiple load/store */ 44#define M 0x10 /* multiple load/store */
@@ -46,6 +46,8 @@ struct aligninfo {
46#define S 0x40 /* single-precision fp or... */ 46#define S 0x40 /* single-precision fp or... */
47#define SX 0x40 /* ... byte count in XER */ 47#define SX 0x40 /* ... byte count in XER */
48#define HARD 0x80 /* string, stwcx. */ 48#define HARD 0x80 /* string, stwcx. */
49#define E4 0x40 /* SPE endianness is word */
50#define E8 0x80 /* SPE endianness is double word */
49 51
50/* DSISR bits reported for a DCBZ instruction: */ 52/* DSISR bits reported for a DCBZ instruction: */
51#define DCBZ 0x5f /* 8xx/82xx dcbz faults when cache not enabled */ 53#define DCBZ 0x5f /* 8xx/82xx dcbz faults when cache not enabled */
@@ -87,9 +89,9 @@ static struct aligninfo aligninfo[128] = {
87 { 8, LD+F+U }, /* 00 1 1001: lfdu */ 89 { 8, LD+F+U }, /* 00 1 1001: lfdu */
88 { 4, ST+F+S+U }, /* 00 1 1010: stfsu */ 90 { 4, ST+F+S+U }, /* 00 1 1010: stfsu */
89 { 8, ST+F+U }, /* 00 1 1011: stfdu */ 91 { 8, ST+F+U }, /* 00 1 1011: stfdu */
90 INVALID, /* 00 1 1100 */ 92 { 16, LD+F }, /* 00 1 1100: lfdp */
91 INVALID, /* 00 1 1101 */ 93 INVALID, /* 00 1 1101 */
92 INVALID, /* 00 1 1110 */ 94 { 16, ST+F }, /* 00 1 1110: stfdp */
93 INVALID, /* 00 1 1111 */ 95 INVALID, /* 00 1 1111 */
94 { 8, LD }, /* 01 0 0000: ldx */ 96 { 8, LD }, /* 01 0 0000: ldx */
95 INVALID, /* 01 0 0001 */ 97 INVALID, /* 01 0 0001 */
@@ -167,10 +169,10 @@ static struct aligninfo aligninfo[128] = {
167 { 8, LD+F }, /* 11 0 1001: lfdx */ 169 { 8, LD+F }, /* 11 0 1001: lfdx */
168 { 4, ST+F+S }, /* 11 0 1010: stfsx */ 170 { 4, ST+F+S }, /* 11 0 1010: stfsx */
169 { 8, ST+F }, /* 11 0 1011: stfdx */ 171 { 8, ST+F }, /* 11 0 1011: stfdx */
170 INVALID, /* 11 0 1100 */ 172 { 16, LD+F }, /* 11 0 1100: lfdpx */
171 { 8, LD+M }, /* 11 0 1101: lmd */ 173 { 4, LD+F+SE }, /* 11 0 1101: lfiwax */
172 INVALID, /* 11 0 1110 */ 174 { 16, ST+F }, /* 11 0 1110: stfdpx */
173 { 8, ST+M }, /* 11 0 1111: stmd */ 175 { 4, ST+F }, /* 11 0 1111: stfiwx */
174 { 4, LD+U }, /* 11 1 0000: lwzux */ 176 { 4, LD+U }, /* 11 1 0000: lwzux */
175 INVALID, /* 11 1 0001 */ 177 INVALID, /* 11 1 0001 */
176 { 4, ST+U }, /* 11 1 0010: stwux */ 178 { 4, ST+U }, /* 11 1 0010: stwux */
@@ -356,6 +358,284 @@ static int emulate_multiple(struct pt_regs *regs, unsigned char __user *addr,
356 return 1; 358 return 1;
357} 359}
358 360
361/*
362 * Emulate floating-point pair loads and stores.
363 * Only POWER6 has these instructions, and it does true little-endian,
364 * so we don't need the address swizzling.
365 */
366static int emulate_fp_pair(struct pt_regs *regs, unsigned char __user *addr,
367 unsigned int reg, unsigned int flags)
368{
369 char *ptr = (char *) &current->thread.fpr[reg];
370 int i, ret;
371
372 if (!(flags & F))
373 return 0;
374 if (reg & 1)
375 return 0; /* invalid form: FRS/FRT must be even */
376 if (!(flags & SW)) {
377 /* not byte-swapped - easy */
378 if (!(flags & ST))
379 ret = __copy_from_user(ptr, addr, 16);
380 else
381 ret = __copy_to_user(addr, ptr, 16);
382 } else {
383 /* each FPR value is byte-swapped separately */
384 ret = 0;
385 for (i = 0; i < 16; ++i) {
386 if (!(flags & ST))
387 ret |= __get_user(ptr[i^7], addr + i);
388 else
389 ret |= __put_user(ptr[i^7], addr + i);
390 }
391 }
392 if (ret)
393 return -EFAULT;
394 return 1; /* exception handled and fixed up */
395}
396
397#ifdef CONFIG_SPE
398
399static struct aligninfo spe_aligninfo[32] = {
400 { 8, LD+E8 }, /* 0 00 00: evldd[x] */
401 { 8, LD+E4 }, /* 0 00 01: evldw[x] */
402 { 8, LD }, /* 0 00 10: evldh[x] */
403 INVALID, /* 0 00 11 */
404 { 2, LD }, /* 0 01 00: evlhhesplat[x] */
405 INVALID, /* 0 01 01 */
406 { 2, LD }, /* 0 01 10: evlhhousplat[x] */
407 { 2, LD+SE }, /* 0 01 11: evlhhossplat[x] */
408 { 4, LD }, /* 0 10 00: evlwhe[x] */
409 INVALID, /* 0 10 01 */
410 { 4, LD }, /* 0 10 10: evlwhou[x] */
411 { 4, LD+SE }, /* 0 10 11: evlwhos[x] */
412 { 4, LD+E4 }, /* 0 11 00: evlwwsplat[x] */
413 INVALID, /* 0 11 01 */
414 { 4, LD }, /* 0 11 10: evlwhsplat[x] */
415 INVALID, /* 0 11 11 */
416
417 { 8, ST+E8 }, /* 1 00 00: evstdd[x] */
418 { 8, ST+E4 }, /* 1 00 01: evstdw[x] */
419 { 8, ST }, /* 1 00 10: evstdh[x] */
420 INVALID, /* 1 00 11 */
421 INVALID, /* 1 01 00 */
422 INVALID, /* 1 01 01 */
423 INVALID, /* 1 01 10 */
424 INVALID, /* 1 01 11 */
425 { 4, ST }, /* 1 10 00: evstwhe[x] */
426 INVALID, /* 1 10 01 */
427 { 4, ST }, /* 1 10 10: evstwho[x] */
428 INVALID, /* 1 10 11 */
429 { 4, ST+E4 }, /* 1 11 00: evstwwe[x] */
430 INVALID, /* 1 11 01 */
431 { 4, ST+E4 }, /* 1 11 10: evstwwo[x] */
432 INVALID, /* 1 11 11 */
433};
434
435#define EVLDD 0x00
436#define EVLDW 0x01
437#define EVLDH 0x02
438#define EVLHHESPLAT 0x04
439#define EVLHHOUSPLAT 0x06
440#define EVLHHOSSPLAT 0x07
441#define EVLWHE 0x08
442#define EVLWHOU 0x0A
443#define EVLWHOS 0x0B
444#define EVLWWSPLAT 0x0C
445#define EVLWHSPLAT 0x0E
446#define EVSTDD 0x10
447#define EVSTDW 0x11
448#define EVSTDH 0x12
449#define EVSTWHE 0x18
450#define EVSTWHO 0x1A
451#define EVSTWWE 0x1C
452#define EVSTWWO 0x1E
453
454/*
455 * Emulate SPE loads and stores.
456 * Only Book-E has these instructions, and it does true little-endian,
457 * so we don't need the address swizzling.
458 */
459static int emulate_spe(struct pt_regs *regs, unsigned int reg,
460 unsigned int instr)
461{
462 int t, ret;
463 union {
464 u64 ll;
465 u32 w[2];
466 u16 h[4];
467 u8 v[8];
468 } data, temp;
469 unsigned char __user *p, *addr;
470 unsigned long *evr = &current->thread.evr[reg];
471 unsigned int nb, flags;
472
473 instr = (instr >> 1) & 0x1f;
474
475 /* DAR has the operand effective address */
476 addr = (unsigned char __user *)regs->dar;
477
478 nb = spe_aligninfo[instr].len;
479 flags = spe_aligninfo[instr].flags;
480
481 /* Verify the address of the operand */
482 if (unlikely(user_mode(regs) &&
483 !access_ok((flags & ST ? VERIFY_WRITE : VERIFY_READ),
484 addr, nb)))
485 return -EFAULT;
486
487 /* userland only */
488 if (unlikely(!user_mode(regs)))
489 return 0;
490
491 flush_spe_to_thread(current);
492
493 /* If we are loading, get the data from user space, else
494 * get it from register values
495 */
496 if (flags & ST) {
497 data.ll = 0;
498 switch (instr) {
499 case EVSTDD:
500 case EVSTDW:
501 case EVSTDH:
502 data.w[0] = *evr;
503 data.w[1] = regs->gpr[reg];
504 break;
505 case EVSTWHE:
506 data.h[2] = *evr >> 16;
507 data.h[3] = regs->gpr[reg] >> 16;
508 break;
509 case EVSTWHO:
510 data.h[2] = *evr & 0xffff;
511 data.h[3] = regs->gpr[reg] & 0xffff;
512 break;
513 case EVSTWWE:
514 data.w[1] = *evr;
515 break;
516 case EVSTWWO:
517 data.w[1] = regs->gpr[reg];
518 break;
519 default:
520 return -EINVAL;
521 }
522 } else {
523 temp.ll = data.ll = 0;
524 ret = 0;
525 p = addr;
526
527 switch (nb) {
528 case 8:
529 ret |= __get_user_inatomic(temp.v[0], p++);
530 ret |= __get_user_inatomic(temp.v[1], p++);
531 ret |= __get_user_inatomic(temp.v[2], p++);
532 ret |= __get_user_inatomic(temp.v[3], p++);
533 case 4:
534 ret |= __get_user_inatomic(temp.v[4], p++);
535 ret |= __get_user_inatomic(temp.v[5], p++);
536 case 2:
537 ret |= __get_user_inatomic(temp.v[6], p++);
538 ret |= __get_user_inatomic(temp.v[7], p++);
539 if (unlikely(ret))
540 return -EFAULT;
541 }
542
543 switch (instr) {
544 case EVLDD:
545 case EVLDW:
546 case EVLDH:
547 data.ll = temp.ll;
548 break;
549 case EVLHHESPLAT:
550 data.h[0] = temp.h[3];
551 data.h[2] = temp.h[3];
552 break;
553 case EVLHHOUSPLAT:
554 case EVLHHOSSPLAT:
555 data.h[1] = temp.h[3];
556 data.h[3] = temp.h[3];
557 break;
558 case EVLWHE:
559 data.h[0] = temp.h[2];
560 data.h[2] = temp.h[3];
561 break;
562 case EVLWHOU:
563 case EVLWHOS:
564 data.h[1] = temp.h[2];
565 data.h[3] = temp.h[3];
566 break;
567 case EVLWWSPLAT:
568 data.w[0] = temp.w[1];
569 data.w[1] = temp.w[1];
570 break;
571 case EVLWHSPLAT:
572 data.h[0] = temp.h[2];
573 data.h[1] = temp.h[2];
574 data.h[2] = temp.h[3];
575 data.h[3] = temp.h[3];
576 break;
577 default:
578 return -EINVAL;
579 }
580 }
581
582 if (flags & SW) {
583 switch (flags & 0xf0) {
584 case E8:
585 SWAP(data.v[0], data.v[7]);
586 SWAP(data.v[1], data.v[6]);
587 SWAP(data.v[2], data.v[5]);
588 SWAP(data.v[3], data.v[4]);
589 break;
590 case E4:
591
592 SWAP(data.v[0], data.v[3]);
593 SWAP(data.v[1], data.v[2]);
594 SWAP(data.v[4], data.v[7]);
595 SWAP(data.v[5], data.v[6]);
596 break;
597 /* Its half word endian */
598 default:
599 SWAP(data.v[0], data.v[1]);
600 SWAP(data.v[2], data.v[3]);
601 SWAP(data.v[4], data.v[5]);
602 SWAP(data.v[6], data.v[7]);
603 break;
604 }
605 }
606
607 if (flags & SE) {
608 data.w[0] = (s16)data.h[1];
609 data.w[1] = (s16)data.h[3];
610 }
611
612 /* Store result to memory or update registers */
613 if (flags & ST) {
614 ret = 0;
615 p = addr;
616 switch (nb) {
617 case 8:
618 ret |= __put_user_inatomic(data.v[0], p++);
619 ret |= __put_user_inatomic(data.v[1], p++);
620 ret |= __put_user_inatomic(data.v[2], p++);
621 ret |= __put_user_inatomic(data.v[3], p++);
622 case 4:
623 ret |= __put_user_inatomic(data.v[4], p++);
624 ret |= __put_user_inatomic(data.v[5], p++);
625 case 2:
626 ret |= __put_user_inatomic(data.v[6], p++);
627 ret |= __put_user_inatomic(data.v[7], p++);
628 }
629 if (unlikely(ret))
630 return -EFAULT;
631 } else {
632 *evr = data.w[0];
633 regs->gpr[reg] = data.w[1];
634 }
635
636 return 1;
637}
638#endif /* CONFIG_SPE */
359 639
360/* 640/*
361 * Called on alignment exception. Attempts to fixup 641 * Called on alignment exception. Attempts to fixup
@@ -414,6 +694,12 @@ int fix_alignment(struct pt_regs *regs)
414 /* extract the operation and registers from the dsisr */ 694 /* extract the operation and registers from the dsisr */
415 reg = (dsisr >> 5) & 0x1f; /* source/dest register */ 695 reg = (dsisr >> 5) & 0x1f; /* source/dest register */
416 areg = dsisr & 0x1f; /* register to update */ 696 areg = dsisr & 0x1f; /* register to update */
697
698#ifdef CONFIG_SPE
699 if ((instr >> 26) == 0x4)
700 return emulate_spe(regs, reg, instr);
701#endif
702
417 instr = (dsisr >> 10) & 0x7f; 703 instr = (dsisr >> 10) & 0x7f;
418 instr |= (dsisr >> 13) & 0x60; 704 instr |= (dsisr >> 13) & 0x60;
419 705
@@ -471,6 +757,10 @@ int fix_alignment(struct pt_regs *regs)
471 flush_fp_to_thread(current); 757 flush_fp_to_thread(current);
472 } 758 }
473 759
760 /* Special case for 16-byte FP loads and stores */
761 if (nb == 16)
762 return emulate_fp_pair(regs, addr, reg, flags);
763
474 /* If we are loading, get the data from user space, else 764 /* If we are loading, get the data from user space, else
475 * get it from register values 765 * get it from register values
476 */ 766 */
@@ -531,7 +821,8 @@ int fix_alignment(struct pt_regs *regs)
531 * or floating point single precision conversion 821 * or floating point single precision conversion
532 */ 822 */
533 switch (flags & ~(U|SW)) { 823 switch (flags & ~(U|SW)) {
534 case LD+SE: /* sign extend */ 824 case LD+SE: /* sign extending integer loads */
825 case LD+F+SE: /* sign extend for lfiwax */
535 if ( nb == 2 ) 826 if ( nb == 2 )
536 data.ll = data.x16.low16; 827 data.ll = data.x16.low16;
537 else /* nb must be 4 */ 828 else /* nb must be 4 */
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-07 15:03:50 -0500