diff options
Diffstat (limited to 'arch/x86/include/asm/xor_avx.h')
-rw-r--r-- | arch/x86/include/asm/xor_avx.h | 54 |
1 files changed, 12 insertions, 42 deletions
diff --git a/arch/x86/include/asm/xor_avx.h b/arch/x86/include/asm/xor_avx.h index 2510d35f480..7ea79c5fa1f 100644 --- a/arch/x86/include/asm/xor_avx.h +++ b/arch/x86/include/asm/xor_avx.h | |||
@@ -20,32 +20,6 @@ | |||
20 | #include <linux/compiler.h> | 20 | #include <linux/compiler.h> |
21 | #include <asm/i387.h> | 21 | #include <asm/i387.h> |
22 | 22 | ||
23 | #define ALIGN32 __aligned(32) | ||
24 | |||
25 | #define YMM_SAVED_REGS 4 | ||
26 | |||
27 | #define YMMS_SAVE \ | ||
28 | do { \ | ||
29 | preempt_disable(); \ | ||
30 | cr0 = read_cr0(); \ | ||
31 | clts(); \ | ||
32 | asm volatile("vmovaps %%ymm0, %0" : "=m" (ymm_save[0]) : : "memory"); \ | ||
33 | asm volatile("vmovaps %%ymm1, %0" : "=m" (ymm_save[32]) : : "memory"); \ | ||
34 | asm volatile("vmovaps %%ymm2, %0" : "=m" (ymm_save[64]) : : "memory"); \ | ||
35 | asm volatile("vmovaps %%ymm3, %0" : "=m" (ymm_save[96]) : : "memory"); \ | ||
36 | } while (0); | ||
37 | |||
38 | #define YMMS_RESTORE \ | ||
39 | do { \ | ||
40 | asm volatile("sfence" : : : "memory"); \ | ||
41 | asm volatile("vmovaps %0, %%ymm3" : : "m" (ymm_save[96])); \ | ||
42 | asm volatile("vmovaps %0, %%ymm2" : : "m" (ymm_save[64])); \ | ||
43 | asm volatile("vmovaps %0, %%ymm1" : : "m" (ymm_save[32])); \ | ||
44 | asm volatile("vmovaps %0, %%ymm0" : : "m" (ymm_save[0])); \ | ||
45 | write_cr0(cr0); \ | ||
46 | preempt_enable(); \ | ||
47 | } while (0); | ||
48 | |||
49 | #define BLOCK4(i) \ | 23 | #define BLOCK4(i) \ |
50 | BLOCK(32 * i, 0) \ | 24 | BLOCK(32 * i, 0) \ |
51 | BLOCK(32 * (i + 1), 1) \ | 25 | BLOCK(32 * (i + 1), 1) \ |
@@ -60,10 +34,9 @@ do { \ | |||
60 | 34 | ||
61 | static void xor_avx_2(unsigned long bytes, unsigned long *p0, unsigned long *p1) | 35 | static void xor_avx_2(unsigned long bytes, unsigned long *p0, unsigned long *p1) |
62 | { | 36 | { |
63 | unsigned long cr0, lines = bytes >> 9; | 37 | unsigned long lines = bytes >> 9; |
64 | char ymm_save[32 * YMM_SAVED_REGS] ALIGN32; | ||
65 | 38 | ||
66 | YMMS_SAVE | 39 | kernel_fpu_begin(); |
67 | 40 | ||
68 | while (lines--) { | 41 | while (lines--) { |
69 | #undef BLOCK | 42 | #undef BLOCK |
@@ -82,16 +55,15 @@ do { \ | |||
82 | p1 = (unsigned long *)((uintptr_t)p1 + 512); | 55 | p1 = (unsigned long *)((uintptr_t)p1 + 512); |
83 | } | 56 | } |
84 | 57 | ||
85 | YMMS_RESTORE | 58 | kernel_fpu_end(); |
86 | } | 59 | } |
87 | 60 | ||
88 | static void xor_avx_3(unsigned long bytes, unsigned long *p0, unsigned long *p1, | 61 | static void xor_avx_3(unsigned long bytes, unsigned long *p0, unsigned long *p1, |
89 | unsigned long *p2) | 62 | unsigned long *p2) |
90 | { | 63 | { |
91 | unsigned long cr0, lines = bytes >> 9; | 64 | unsigned long lines = bytes >> 9; |
92 | char ymm_save[32 * YMM_SAVED_REGS] ALIGN32; | ||
93 | 65 | ||
94 | YMMS_SAVE | 66 | kernel_fpu_begin(); |
95 | 67 | ||
96 | while (lines--) { | 68 | while (lines--) { |
97 | #undef BLOCK | 69 | #undef BLOCK |
@@ -113,16 +85,15 @@ do { \ | |||
113 | p2 = (unsigned long *)((uintptr_t)p2 + 512); | 85 | p2 = (unsigned long *)((uintptr_t)p2 + 512); |
114 | } | 86 | } |
115 | 87 | ||
116 | YMMS_RESTORE | 88 | kernel_fpu_end(); |
117 | } | 89 | } |
118 | 90 | ||
119 | static void xor_avx_4(unsigned long bytes, unsigned long *p0, unsigned long *p1, | 91 | static void xor_avx_4(unsigned long bytes, unsigned long *p0, unsigned long *p1, |
120 | unsigned long *p2, unsigned long *p3) | 92 | unsigned long *p2, unsigned long *p3) |
121 | { | 93 | { |
122 | unsigned long cr0, lines = bytes >> 9; | 94 | unsigned long lines = bytes >> 9; |
123 | char ymm_save[32 * YMM_SAVED_REGS] ALIGN32; | ||
124 | 95 | ||
125 | YMMS_SAVE | 96 | kernel_fpu_begin(); |
126 | 97 | ||
127 | while (lines--) { | 98 | while (lines--) { |
128 | #undef BLOCK | 99 | #undef BLOCK |
@@ -147,16 +118,15 @@ do { \ | |||
147 | p3 = (unsigned long *)((uintptr_t)p3 + 512); | 118 | p3 = (unsigned long *)((uintptr_t)p3 + 512); |
148 | } | 119 | } |
149 | 120 | ||
150 | YMMS_RESTORE | 121 | kernel_fpu_end(); |
151 | } | 122 | } |
152 | 123 | ||
153 | static void xor_avx_5(unsigned long bytes, unsigned long *p0, unsigned long *p1, | 124 | static void xor_avx_5(unsigned long bytes, unsigned long *p0, unsigned long *p1, |
154 | unsigned long *p2, unsigned long *p3, unsigned long *p4) | 125 | unsigned long *p2, unsigned long *p3, unsigned long *p4) |
155 | { | 126 | { |
156 | unsigned long cr0, lines = bytes >> 9; | 127 | unsigned long lines = bytes >> 9; |
157 | char ymm_save[32 * YMM_SAVED_REGS] ALIGN32; | ||
158 | 128 | ||
159 | YMMS_SAVE | 129 | kernel_fpu_begin(); |
160 | 130 | ||
161 | while (lines--) { | 131 | while (lines--) { |
162 | #undef BLOCK | 132 | #undef BLOCK |
@@ -184,7 +154,7 @@ do { \ | |||
184 | p4 = (unsigned long *)((uintptr_t)p4 + 512); | 154 | p4 = (unsigned long *)((uintptr_t)p4 + 512); |
185 | } | 155 | } |
186 | 156 | ||
187 | YMMS_RESTORE | 157 | kernel_fpu_end(); |
188 | } | 158 | } |
189 | 159 | ||
190 | static struct xor_block_template xor_block_avx = { | 160 | static struct xor_block_template xor_block_avx = { |