aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorYuanhan Liu <yuanhan.liu@linux.intel.com>2012-11-30 16:10:39 -0500
committerNeilBrown <neilb@suse.de>2012-12-13 03:51:03 -0500
commit2c935842bdb46f5f557426feb4d2bdfdad1aa5f9 (patch)
tree3b089f583018fb5c1d58192dbf7ff92e914ddac2
parent7056741fd9fc14a65608549a4657cf5178f05f63 (diff)
lib/raid6: Add AVX2 optimized gen_syndrome functions
Add AVX2 optimized gen_syndrom functions, which is simply based on sse2.c written by hpa. Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com> Reviewed-by: H. Peter Anvin <hpa@zytor.com> Signed-off-by: Jim Kukunas <james.t.kukunas@linux.intel.com> Signed-off-by: NeilBrown <neilb@suse.de>
Diffstat
-rw-r--r--include/linux/raid/pq.h3
-rw-r--r--lib/raid6/Makefile2
-rw-r--r--lib/raid6/algos.c9
-rw-r--r--lib/raid6/avx2.c251
-rw-r--r--lib/raid6/test/Makefile12
5 files changed, 275 insertions, 2 deletions
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index 3156347452b9..8dfaa2ce2e95 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -98,6 +98,9 @@ extern const struct raid6_calls raid6_altivec1;
98extern const struct raid6_calls raid6_altivec2; 98extern const struct raid6_calls raid6_altivec2;
99extern const struct raid6_calls raid6_altivec4; 99extern const struct raid6_calls raid6_altivec4;
100extern const struct raid6_calls raid6_altivec8; 100extern const struct raid6_calls raid6_altivec8;
101extern const struct raid6_calls raid6_avx2x1;
102extern const struct raid6_calls raid6_avx2x2;
103extern const struct raid6_calls raid6_avx2x4;
101 104
102struct raid6_recov_calls { 105struct raid6_recov_calls {
103 void (*data2)(int, size_t, int, int, void **); 106 void (*data2)(int, size_t, int, int, void **);
diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile
index 8c2e22bef661..3430711b9bdf 100644
--- a/lib/raid6/Makefile
+++ b/lib/raid6/Makefile
@@ -2,7 +2,7 @@ obj-$(CONFIG_RAID6_PQ) += raid6_pq.o
2 2
3raid6_pq-y += algos.o recov.o recov_ssse3.o recov_avx2.o tables.o int1.o int2.o int4.o \ 3raid6_pq-y += algos.o recov.o recov_ssse3.o recov_avx2.o tables.o int1.o int2.o int4.o \
4 int8.o int16.o int32.o altivec1.o altivec2.o altivec4.o \ 4 int8.o int16.o int32.o altivec1.o altivec2.o altivec4.o \
5 altivec8.o mmx.o sse1.o sse2.o 5 altivec8.o mmx.o sse1.o sse2.o avx2.o
6hostprogs-y += mktables 6hostprogs-y += mktables
7 7
8quiet_cmd_unroll = UNROLL $@ 8quiet_cmd_unroll = UNROLL $@
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index 8b7f55cadb45..6d7316fe9f30 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -45,11 +45,20 @@ const struct raid6_calls * const raid6_algos[] = {
45 &raid6_sse1x2, 45 &raid6_sse1x2,
46 &raid6_sse2x1, 46 &raid6_sse2x1,
47 &raid6_sse2x2, 47 &raid6_sse2x2,
48#ifdef CONFIG_AS_AVX2
49 &raid6_avx2x1,
50 &raid6_avx2x2,
51#endif
48#endif 52#endif
49#if defined(__x86_64__) && !defined(__arch_um__) 53#if defined(__x86_64__) && !defined(__arch_um__)
50 &raid6_sse2x1, 54 &raid6_sse2x1,
51 &raid6_sse2x2, 55 &raid6_sse2x2,
52 &raid6_sse2x4, 56 &raid6_sse2x4,
57#ifdef CONFIG_AS_AVX2
58 &raid6_avx2x1,
59 &raid6_avx2x2,
60 &raid6_avx2x4,
61#endif
53#endif 62#endif
54#ifdef CONFIG_ALTIVEC 63#ifdef CONFIG_ALTIVEC
55 &raid6_altivec1, 64 &raid6_altivec1,
diff --git a/lib/raid6/avx2.c b/lib/raid6/avx2.c
new file mode 100644
index 000000000000..bc3b1dd436eb
--- /dev/null
+++ b/lib/raid6/avx2.c
@@ -0,0 +1,251 @@
1/* -*- linux-c -*- ------------------------------------------------------- *
2 *
3 * Copyright (C) 2012 Intel Corporation
4 * Author: Yuanhan Liu <yuanhan.liu@linux.intel.com>
5 *
6 * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
7 *
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation, Inc., 53 Temple Place Ste 330,
12 * Boston MA 02111-1307, USA; either version 2 of the License, or
13 * (at your option) any later version; incorporated herein by reference.
14 *
15 * ----------------------------------------------------------------------- */
16
17/*
18 * AVX2 implementation of RAID-6 syndrome functions
19 *
20 */
21
22#ifdef CONFIG_AS_AVX2
23
24#include <linux/raid/pq.h>
25#include "x86.h"
26
27static const struct raid6_avx2_constants {
28 u64 x1d[4];
29} raid6_avx2_constants __aligned(32) = {
30 { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
31 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
32};
33
34static int raid6_have_avx2(void)
35{
36 return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX);
37}
38
39/*
40 * Plain AVX2 implementation
41 */
42static void raid6_avx21_gen_syndrome(int disks, size_t bytes, void **ptrs)
43{
44 u8 **dptr = (u8 **)ptrs;
45 u8 *p, *q;
46 int d, z, z0;
47
48 z0 = disks - 3; /* Highest data disk */
49 p = dptr[z0+1]; /* XOR parity */
50 q = dptr[z0+2]; /* RS syndrome */
51
52 kernel_fpu_begin();
53
54 asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
55 asm volatile("vpxor %ymm3,%ymm3,%ymm3"); /* Zero temp */
56
57 for (d = 0; d < bytes; d += 32) {
58 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
59 asm volatile("vmovdqa %0,%%ymm2" : : "m" (dptr[z0][d]));/* P[0] */
60 asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
61 asm volatile("vmovdqa %ymm2,%ymm4");/* Q[0] */
62 asm volatile("vmovdqa %0,%%ymm6" : : "m" (dptr[z0-1][d]));
63 for (z = z0-2; z >= 0; z--) {
64 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
65 asm volatile("vpcmpgtb %ymm4,%ymm3,%ymm5");
66 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
67 asm volatile("vpand %ymm0,%ymm5,%ymm5");
68 asm volatile("vpxor %ymm5,%ymm4,%ymm4");
69 asm volatile("vpxor %ymm6,%ymm2,%ymm2");
70 asm volatile("vpxor %ymm6,%ymm4,%ymm4");
71 asm volatile("vmovdqa %0,%%ymm6" : : "m" (dptr[z][d]));
72 }
73 asm volatile("vpcmpgtb %ymm4,%ymm3,%ymm5");
74 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
75 asm volatile("vpand %ymm0,%ymm5,%ymm5");
76 asm volatile("vpxor %ymm5,%ymm4,%ymm4");
77 asm volatile("vpxor %ymm6,%ymm2,%ymm2");
78 asm volatile("vpxor %ymm6,%ymm4,%ymm4");
79
80 asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
81 asm volatile("vpxor %ymm2,%ymm2,%ymm2");
82 asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
83 asm volatile("vpxor %ymm4,%ymm4,%ymm4");
84 }
85
86 asm volatile("sfence" : : : "memory");
87 kernel_fpu_end();
88}
89
90const struct raid6_calls raid6_avx2x1 = {
91 raid6_avx21_gen_syndrome,
92 raid6_have_avx2,
93 "avx2x1",
94 1 /* Has cache hints */
95};
96
97/*
98 * Unrolled-by-2 AVX2 implementation
99 */
100static void raid6_avx22_gen_syndrome(int disks, size_t bytes, void **ptrs)
101{
102 u8 **dptr = (u8 **)ptrs;
103 u8 *p, *q;
104 int d, z, z0;
105
106 z0 = disks - 3; /* Highest data disk */
107 p = dptr[z0+1]; /* XOR parity */
108 q = dptr[z0+2]; /* RS syndrome */
109
110 kernel_fpu_begin();
111
112 asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
113 asm volatile("vpxor %ymm1,%ymm1,%ymm1"); /* Zero temp */
114
115 /* We uniformly assume a single prefetch covers at least 32 bytes */
116 for (d = 0; d < bytes; d += 64) {
117 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
118 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d+32]));
119 asm volatile("vmovdqa %0,%%ymm2" : : "m" (dptr[z0][d]));/* P[0] */
120 asm volatile("vmovdqa %0,%%ymm3" : : "m" (dptr[z0][d+32]));/* P[1] */
121 asm volatile("vmovdqa %ymm2,%ymm4"); /* Q[0] */
122 asm volatile("vmovdqa %ymm3,%ymm6"); /* Q[1] */
123 for (z = z0-1; z >= 0; z--) {
124 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
125 asm volatile("prefetchnta %0" : : "m" (dptr[z][d+32]));
126 asm volatile("vpcmpgtb %ymm4,%ymm1,%ymm5");
127 asm volatile("vpcmpgtb %ymm6,%ymm1,%ymm7");
128 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
129 asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
130 asm volatile("vpand %ymm0,%ymm5,%ymm5");
131 asm volatile("vpand %ymm0,%ymm7,%ymm7");
132 asm volatile("vpxor %ymm5,%ymm4,%ymm4");
133 asm volatile("vpxor %ymm7,%ymm6,%ymm6");
134 asm volatile("vmovdqa %0,%%ymm5" : : "m" (dptr[z][d]));
135 asm volatile("vmovdqa %0,%%ymm7" : : "m" (dptr[z][d+32]));
136 asm volatile("vpxor %ymm5,%ymm2,%ymm2");
137 asm volatile("vpxor %ymm7,%ymm3,%ymm3");
138 asm volatile("vpxor %ymm5,%ymm4,%ymm4");
139 asm volatile("vpxor %ymm7,%ymm6,%ymm6");
140 }
141 asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
142 asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));
143 asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
144 asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));
145 }
146
147 asm volatile("sfence" : : : "memory");
148 kernel_fpu_end();
149}
150
151const struct raid6_calls raid6_avx2x2 = {
152 raid6_avx22_gen_syndrome,
153 raid6_have_avx2,
154 "avx2x2",
155 1 /* Has cache hints */
156};
157
158#ifdef CONFIG_X86_64
159
160/*
161 * Unrolled-by-4 AVX2 implementation
162 */
163static void raid6_avx24_gen_syndrome(int disks, size_t bytes, void **ptrs)
164{
165 u8 **dptr = (u8 **)ptrs;
166 u8 *p, *q;
167 int d, z, z0;
168
169 z0 = disks - 3; /* Highest data disk */
170 p = dptr[z0+1]; /* XOR parity */
171 q = dptr[z0+2]; /* RS syndrome */
172
173 kernel_fpu_begin();
174
175 asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
176 asm volatile("vpxor %ymm1,%ymm1,%ymm1"); /* Zero temp */
177 asm volatile("vpxor %ymm2,%ymm2,%ymm2"); /* P[0] */
178 asm volatile("vpxor %ymm3,%ymm3,%ymm3"); /* P[1] */
179 asm volatile("vpxor %ymm4,%ymm4,%ymm4"); /* Q[0] */
180 asm volatile("vpxor %ymm6,%ymm6,%ymm6"); /* Q[1] */
181 asm volatile("vpxor %ymm10,%ymm10,%ymm10"); /* P[2] */
182 asm volatile("vpxor %ymm11,%ymm11,%ymm11"); /* P[3] */
183 asm volatile("vpxor %ymm12,%ymm12,%ymm12"); /* Q[2] */
184 asm volatile("vpxor %ymm14,%ymm14,%ymm14"); /* Q[3] */
185
186 for (d = 0; d < bytes; d += 128) {
187 for (z = z0; z >= 0; z--) {
188 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
189 asm volatile("prefetchnta %0" : : "m" (dptr[z][d+32]));
190 asm volatile("prefetchnta %0" : : "m" (dptr[z][d+64]));
191 asm volatile("prefetchnta %0" : : "m" (dptr[z][d+96]));
192 asm volatile("vpcmpgtb %ymm4,%ymm1,%ymm5");
193 asm volatile("vpcmpgtb %ymm6,%ymm1,%ymm7");
194 asm volatile("vpcmpgtb %ymm12,%ymm1,%ymm13");
195 asm volatile("vpcmpgtb %ymm14,%ymm1,%ymm15");
196 asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
197 asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
198 asm volatile("vpaddb %ymm12,%ymm12,%ymm12");
199 asm volatile("vpaddb %ymm14,%ymm14,%ymm14");
200 asm volatile("vpand %ymm0,%ymm5,%ymm5");
201 asm volatile("vpand %ymm0,%ymm7,%ymm7");
202 asm volatile("vpand %ymm0,%ymm13,%ymm13");
203 asm volatile("vpand %ymm0,%ymm15,%ymm15");
204 asm volatile("vpxor %ymm5,%ymm4,%ymm4");
205 asm volatile("vpxor %ymm7,%ymm6,%ymm6");
206 asm volatile("vpxor %ymm13,%ymm12,%ymm12");
207 asm volatile("vpxor %ymm15,%ymm14,%ymm14");
208 asm volatile("vmovdqa %0,%%ymm5" : : "m" (dptr[z][d]));
209 asm volatile("vmovdqa %0,%%ymm7" : : "m" (dptr[z][d+32]));
210 asm volatile("vmovdqa %0,%%ymm13" : : "m" (dptr[z][d+64]));
211 asm volatile("vmovdqa %0,%%ymm15" : : "m" (dptr[z][d+96]));
212 asm volatile("vpxor %ymm5,%ymm2,%ymm2");
213 asm volatile("vpxor %ymm7,%ymm3,%ymm3");
214 asm volatile("vpxor %ymm13,%ymm10,%ymm10");
215 asm volatile("vpxor %ymm15,%ymm11,%ymm11");
216 asm volatile("vpxor %ymm5,%ymm4,%ymm4");
217 asm volatile("vpxor %ymm7,%ymm6,%ymm6");
218 asm volatile("vpxor %ymm13,%ymm12,%ymm12");
219 asm volatile("vpxor %ymm15,%ymm14,%ymm14");
220 }
221 asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
222 asm volatile("vpxor %ymm2,%ymm2,%ymm2");
223 asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));
224 asm volatile("vpxor %ymm3,%ymm3,%ymm3");
225 asm volatile("vmovntdq %%ymm10,%0" : "=m" (p[d+64]));
226 asm volatile("vpxor %ymm10,%ymm10,%ymm10");
227 asm volatile("vmovntdq %%ymm11,%0" : "=m" (p[d+96]));
228 asm volatile("vpxor %ymm11,%ymm11,%ymm11");
229 asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
230 asm volatile("vpxor %ymm4,%ymm4,%ymm4");
231 asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));
232 asm volatile("vpxor %ymm6,%ymm6,%ymm6");
233 asm volatile("vmovntdq %%ymm12,%0" : "=m" (q[d+64]));
234 asm volatile("vpxor %ymm12,%ymm12,%ymm12");
235 asm volatile("vmovntdq %%ymm14,%0" : "=m" (q[d+96]));
236 asm volatile("vpxor %ymm14,%ymm14,%ymm14");
237 }
238
239 asm volatile("sfence" : : : "memory");
240 kernel_fpu_end();
241}
242
243const struct raid6_calls raid6_avx2x4 = {
244 raid6_avx24_gen_syndrome,
245 raid6_have_avx2,
246 "avx2x4",
247 1 /* Has cache hints */
248};
249#endif
250
251#endif /* CONFIG_AS_AVX2 */
diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile
index d919c98ce266..754cbac0f9f8 100644
--- a/lib/raid6/test/Makefile
+++ b/lib/raid6/test/Makefile
@@ -11,6 +11,16 @@ AWK = awk -f
11AR = ar 11AR = ar
12RANLIB = ranlib 12RANLIB = ranlib
13 13
14ARCH := $(shell uname -m 2>/dev/null | sed -e /s/i.86/i386/)
15ifeq ($(ARCH),i386)
16 CFLAGS += -DCONFIG_X86_32
17endif
18ifeq ($(ARCH),x86_64)
19 CFLAGS += -DCONFIG_X86_64
20endif
21CFLAGS += $(shell echo "vpbroadcastb %xmm0, %ymm1"| gcc -c -x assembler - &&\
22 rm ./-.o && echo -DCONFIG_AS_AVX2=1)
23
14.c.o: 24.c.o:
15 $(CC) $(CFLAGS) -c -o $@ $< 25 $(CC) $(CFLAGS) -c -o $@ $<
16 26
@@ -22,7 +32,7 @@ RANLIB = ranlib
22 32
23all: raid6.a raid6test 33all: raid6.a raid6test
24 34
25raid6.a: int1.o int2.o int4.o int8.o int16.o int32.o mmx.o sse1.o sse2.o \ 35raid6.a: int1.o int2.o int4.o int8.o int16.o int32.o mmx.o sse1.o sse2.o avx2.o \
26 altivec1.o altivec2.o altivec4.o altivec8.o recov.o recov_ssse3.o recov_avx2.o algos.o \ 36 altivec1.o altivec2.o altivec4.o altivec8.o recov.o recov_ssse3.o recov_avx2.o algos.o \
27 tables.o 37 tables.o
28 rm -f $@ 38 rm -f $@
generated by cgit v1.2.2 (git 2.25.0) at 2026-03-04 13:09:19 -0500