diff options
Diffstat (limited to 'arch/x86/include/asm/xor_32.h')
-rw-r--r-- | arch/x86/include/asm/xor_32.h | 58 |
1 files changed, 9 insertions, 49 deletions
diff --git a/arch/x86/include/asm/xor_32.h b/arch/x86/include/asm/xor_32.h index 454570891bdc..f79cb7ec0e06 100644 --- a/arch/x86/include/asm/xor_32.h +++ b/arch/x86/include/asm/xor_32.h | |||
@@ -534,38 +534,6 @@ static struct xor_block_template xor_block_p5_mmx = { | |||
534 | * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo) | 534 | * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo) |
535 | */ | 535 | */ |
536 | 536 | ||
537 | #define XMMS_SAVE \ | ||
538 | do { \ | ||
539 | preempt_disable(); \ | ||
540 | cr0 = read_cr0(); \ | ||
541 | clts(); \ | ||
542 | asm volatile( \ | ||
543 | "movups %%xmm0,(%0) ;\n\t" \ | ||
544 | "movups %%xmm1,0x10(%0) ;\n\t" \ | ||
545 | "movups %%xmm2,0x20(%0) ;\n\t" \ | ||
546 | "movups %%xmm3,0x30(%0) ;\n\t" \ | ||
547 | : \ | ||
548 | : "r" (xmm_save) \ | ||
549 | : "memory"); \ | ||
550 | } while (0) | ||
551 | |||
552 | #define XMMS_RESTORE \ | ||
553 | do { \ | ||
554 | asm volatile( \ | ||
555 | "sfence ;\n\t" \ | ||
556 | "movups (%0),%%xmm0 ;\n\t" \ | ||
557 | "movups 0x10(%0),%%xmm1 ;\n\t" \ | ||
558 | "movups 0x20(%0),%%xmm2 ;\n\t" \ | ||
559 | "movups 0x30(%0),%%xmm3 ;\n\t" \ | ||
560 | : \ | ||
561 | : "r" (xmm_save) \ | ||
562 | : "memory"); \ | ||
563 | write_cr0(cr0); \ | ||
564 | preempt_enable(); \ | ||
565 | } while (0) | ||
566 | |||
567 | #define ALIGN16 __attribute__((aligned(16))) | ||
568 | |||
569 | #define OFFS(x) "16*("#x")" | 537 | #define OFFS(x) "16*("#x")" |
570 | #define PF_OFFS(x) "256+16*("#x")" | 538 | #define PF_OFFS(x) "256+16*("#x")" |
571 | #define PF0(x) " prefetchnta "PF_OFFS(x)"(%1) ;\n" | 539 | #define PF0(x) " prefetchnta "PF_OFFS(x)"(%1) ;\n" |
@@ -587,10 +555,8 @@ static void | |||
587 | xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) | 555 | xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) |
588 | { | 556 | { |
589 | unsigned long lines = bytes >> 8; | 557 | unsigned long lines = bytes >> 8; |
590 | char xmm_save[16*4] ALIGN16; | ||
591 | int cr0; | ||
592 | 558 | ||
593 | XMMS_SAVE; | 559 | kernel_fpu_begin(); |
594 | 560 | ||
595 | asm volatile( | 561 | asm volatile( |
596 | #undef BLOCK | 562 | #undef BLOCK |
@@ -633,7 +599,7 @@ xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) | |||
633 | : | 599 | : |
634 | : "memory"); | 600 | : "memory"); |
635 | 601 | ||
636 | XMMS_RESTORE; | 602 | kernel_fpu_end(); |
637 | } | 603 | } |
638 | 604 | ||
639 | static void | 605 | static void |
@@ -641,10 +607,8 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, | |||
641 | unsigned long *p3) | 607 | unsigned long *p3) |
642 | { | 608 | { |
643 | unsigned long lines = bytes >> 8; | 609 | unsigned long lines = bytes >> 8; |
644 | char xmm_save[16*4] ALIGN16; | ||
645 | int cr0; | ||
646 | 610 | ||
647 | XMMS_SAVE; | 611 | kernel_fpu_begin(); |
648 | 612 | ||
649 | asm volatile( | 613 | asm volatile( |
650 | #undef BLOCK | 614 | #undef BLOCK |
@@ -694,7 +658,7 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2, | |||
694 | : | 658 | : |
695 | : "memory" ); | 659 | : "memory" ); |
696 | 660 | ||
697 | XMMS_RESTORE; | 661 | kernel_fpu_end(); |
698 | } | 662 | } |
699 | 663 | ||
700 | static void | 664 | static void |
@@ -702,10 +666,8 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, | |||
702 | unsigned long *p3, unsigned long *p4) | 666 | unsigned long *p3, unsigned long *p4) |
703 | { | 667 | { |
704 | unsigned long lines = bytes >> 8; | 668 | unsigned long lines = bytes >> 8; |
705 | char xmm_save[16*4] ALIGN16; | ||
706 | int cr0; | ||
707 | 669 | ||
708 | XMMS_SAVE; | 670 | kernel_fpu_begin(); |
709 | 671 | ||
710 | asm volatile( | 672 | asm volatile( |
711 | #undef BLOCK | 673 | #undef BLOCK |
@@ -762,7 +724,7 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2, | |||
762 | : | 724 | : |
763 | : "memory" ); | 725 | : "memory" ); |
764 | 726 | ||
765 | XMMS_RESTORE; | 727 | kernel_fpu_end(); |
766 | } | 728 | } |
767 | 729 | ||
768 | static void | 730 | static void |
@@ -770,10 +732,8 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, | |||
770 | unsigned long *p3, unsigned long *p4, unsigned long *p5) | 732 | unsigned long *p3, unsigned long *p4, unsigned long *p5) |
771 | { | 733 | { |
772 | unsigned long lines = bytes >> 8; | 734 | unsigned long lines = bytes >> 8; |
773 | char xmm_save[16*4] ALIGN16; | ||
774 | int cr0; | ||
775 | 735 | ||
776 | XMMS_SAVE; | 736 | kernel_fpu_begin(); |
777 | 737 | ||
778 | /* Make sure GCC forgets anything it knows about p4 or p5, | 738 | /* Make sure GCC forgets anything it knows about p4 or p5, |
779 | such that it won't pass to the asm volatile below a | 739 | such that it won't pass to the asm volatile below a |
@@ -850,7 +810,7 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2, | |||
850 | like assuming they have some legal value. */ | 810 | like assuming they have some legal value. */ |
851 | asm("" : "=r" (p4), "=r" (p5)); | 811 | asm("" : "=r" (p4), "=r" (p5)); |
852 | 812 | ||
853 | XMMS_RESTORE; | 813 | kernel_fpu_end(); |
854 | } | 814 | } |
855 | 815 | ||
856 | static struct xor_block_template xor_block_pIII_sse = { | 816 | static struct xor_block_template xor_block_pIII_sse = { |
@@ -862,7 +822,7 @@ static struct xor_block_template xor_block_pIII_sse = { | |||
862 | }; | 822 | }; |
863 | 823 | ||
864 | /* Also try the AVX routines */ | 824 | /* Also try the AVX routines */ |
865 | #include "xor_avx.h" | 825 | #include <asm/xor_avx.h> |
866 | 826 | ||
867 | /* Also try the generic routines. */ | 827 | /* Also try the generic routines. */ |
868 | #include <asm-generic/xor.h> | 828 | #include <asm-generic/xor.h> |