aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorWill Deacon <will.deacon@arm.com>2016-07-04 12:44:48 -0400
committerWill Deacon <will.deacon@arm.com>2016-10-19 10:37:29 -0400
commit1e6e57d9b34a9075d5f9e2048ea7b09756590d11 (patch)
treeb5708d14f64b03758cb18b795ef18077fdac9b55
parent1c5b51dfb7b4564008e0cadec5381a69e88b0d21 (diff)
arm64: percpu: rewrite ll/sc loops in assembly
Writing the outer loop of an LL/SC sequence using do {...} while constructs potentially allows the compiler to hoist memory accesses between the STXR and the branch back to the LDXR. On CPUs that do not guarantee forward progress of LL/SC loops when faced with memory accesses to the same ERG (up to 2k) between the failed STXR and the branch back, we may end up livelocking. This patch avoids this issue in our percpu atomics by rewriting the outer loop as part of the LL/SC inline assembly block. Cc: <stable@vger.kernel.org> Fixes: f97fc810798c ("arm64: percpu: Implement this_cpu operations") Reviewed-by: Mark Rutland <mark.rutland@arm.com> Tested-by: Mark Rutland <mark.rutland@arm.com> Signed-off-by: Will Deacon <will.deacon@arm.com>
Diffstat
-rw-r--r--arch/arm64/include/asm/percpu.h120
1 files changed, 56 insertions, 64 deletions
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index 2fee2f59288c..5394c8405e66 100644
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -44,48 +44,44 @@ static inline unsigned long __percpu_##op(void *ptr, \
44 \ 44 \
45 switch (size) { \ 45 switch (size) { \
46 case 1: \ 46 case 1: \
47 do { \ 47 asm ("//__per_cpu_" #op "_1\n" \
48 asm ("//__per_cpu_" #op "_1\n" \ 48 "1: ldxrb %w[ret], %[ptr]\n" \
49 "ldxrb %w[ret], %[ptr]\n" \
50 #asm_op " %w[ret], %w[ret], %w[val]\n" \ 49 #asm_op " %w[ret], %w[ret], %w[val]\n" \
51 "stxrb %w[loop], %w[ret], %[ptr]\n" \ 50 " stxrb %w[loop], %w[ret], %[ptr]\n" \
52 : [loop] "=&r" (loop), [ret] "=&r" (ret), \ 51 " cbnz %w[loop], 1b" \
53 [ptr] "+Q"(*(u8 *)ptr) \ 52 : [loop] "=&r" (loop), [ret] "=&r" (ret), \
54 : [val] "Ir" (val)); \ 53 [ptr] "+Q"(*(u8 *)ptr) \
55 } while (loop); \ 54 : [val] "Ir" (val)); \
56 break; \ 55 break; \
57 case 2: \ 56 case 2: \
58 do { \ 57 asm ("//__per_cpu_" #op "_2\n" \
59 asm ("//__per_cpu_" #op "_2\n" \ 58 "1: ldxrh %w[ret], %[ptr]\n" \
60 "ldxrh %w[ret], %[ptr]\n" \
61 #asm_op " %w[ret], %w[ret], %w[val]\n" \ 59 #asm_op " %w[ret], %w[ret], %w[val]\n" \
62 "stxrh %w[loop], %w[ret], %[ptr]\n" \ 60 " stxrh %w[loop], %w[ret], %[ptr]\n" \
63 : [loop] "=&r" (loop), [ret] "=&r" (ret), \ 61 " cbnz %w[loop], 1b" \
64 [ptr] "+Q"(*(u16 *)ptr) \ 62 : [loop] "=&r" (loop), [ret] "=&r" (ret), \
65 : [val] "Ir" (val)); \ 63 [ptr] "+Q"(*(u16 *)ptr) \
66 } while (loop); \ 64 : [val] "Ir" (val)); \
67 break; \ 65 break; \
68 case 4: \ 66 case 4: \
69 do { \ 67 asm ("//__per_cpu_" #op "_4\n" \
70 asm ("//__per_cpu_" #op "_4\n" \ 68 "1: ldxr %w[ret], %[ptr]\n" \
71 "ldxr %w[ret], %[ptr]\n" \
72 #asm_op " %w[ret], %w[ret], %w[val]\n" \ 69 #asm_op " %w[ret], %w[ret], %w[val]\n" \
73 "stxr %w[loop], %w[ret], %[ptr]\n" \ 70 " stxr %w[loop], %w[ret], %[ptr]\n" \
74 : [loop] "=&r" (loop), [ret] "=&r" (ret), \ 71 " cbnz %w[loop], 1b" \
75 [ptr] "+Q"(*(u32 *)ptr) \ 72 : [loop] "=&r" (loop), [ret] "=&r" (ret), \
76 : [val] "Ir" (val)); \ 73 [ptr] "+Q"(*(u32 *)ptr) \
77 } while (loop); \ 74 : [val] "Ir" (val)); \
78 break; \ 75 break; \
79 case 8: \ 76 case 8: \
80 do { \ 77 asm ("//__per_cpu_" #op "_8\n" \
81 asm ("//__per_cpu_" #op "_8\n" \ 78 "1: ldxr %[ret], %[ptr]\n" \
82 "ldxr %[ret], %[ptr]\n" \
83 #asm_op " %[ret], %[ret], %[val]\n" \ 79 #asm_op " %[ret], %[ret], %[val]\n" \
84 "stxr %w[loop], %[ret], %[ptr]\n" \ 80 " stxr %w[loop], %[ret], %[ptr]\n" \
85 : [loop] "=&r" (loop), [ret] "=&r" (ret), \ 81 " cbnz %w[loop], 1b" \
86 [ptr] "+Q"(*(u64 *)ptr) \ 82 : [loop] "=&r" (loop), [ret] "=&r" (ret), \
87 : [val] "Ir" (val)); \ 83 [ptr] "+Q"(*(u64 *)ptr) \
88 } while (loop); \ 84 : [val] "Ir" (val)); \
89 break; \ 85 break; \
90 default: \ 86 default: \
91 BUILD_BUG(); \ 87 BUILD_BUG(); \
@@ -150,44 +146,40 @@ static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
150 146
151 switch (size) { 147 switch (size) {
152 case 1: 148 case 1:
153 do { 149 asm ("//__percpu_xchg_1\n"
154 asm ("//__percpu_xchg_1\n" 150 "1: ldxrb %w[ret], %[ptr]\n"
155 "ldxrb %w[ret], %[ptr]\n" 151 " stxrb %w[loop], %w[val], %[ptr]\n"
156 "stxrb %w[loop], %w[val], %[ptr]\n" 152 " cbnz %w[loop], 1b"
157 : [loop] "=&r"(loop), [ret] "=&r"(ret), 153 : [loop] "=&r"(loop), [ret] "=&r"(ret),
158 [ptr] "+Q"(*(u8 *)ptr) 154 [ptr] "+Q"(*(u8 *)ptr)
159 : [val] "r" (val)); 155 : [val] "r" (val));
160 } while (loop);
161 break; 156 break;
162 case 2: 157 case 2:
163 do { 158 asm ("//__percpu_xchg_2\n"
164 asm ("//__percpu_xchg_2\n" 159 "1: ldxrh %w[ret], %[ptr]\n"
165 "ldxrh %w[ret], %[ptr]\n" 160 " stxrh %w[loop], %w[val], %[ptr]\n"
166 "stxrh %w[loop], %w[val], %[ptr]\n" 161 " cbnz %w[loop], 1b"
167 : [loop] "=&r"(loop), [ret] "=&r"(ret), 162 : [loop] "=&r"(loop), [ret] "=&r"(ret),
168 [ptr] "+Q"(*(u16 *)ptr) 163 [ptr] "+Q"(*(u16 *)ptr)
169 : [val] "r" (val)); 164 : [val] "r" (val));
170 } while (loop);
171 break; 165 break;
172 case 4: 166 case 4:
173 do { 167 asm ("//__percpu_xchg_4\n"
174 asm ("//__percpu_xchg_4\n" 168 "1: ldxr %w[ret], %[ptr]\n"
175 "ldxr %w[ret], %[ptr]\n" 169 " stxr %w[loop], %w[val], %[ptr]\n"
176 "stxr %w[loop], %w[val], %[ptr]\n" 170 " cbnz %w[loop], 1b"
177 : [loop] "=&r"(loop), [ret] "=&r"(ret), 171 : [loop] "=&r"(loop), [ret] "=&r"(ret),
178 [ptr] "+Q"(*(u32 *)ptr) 172 [ptr] "+Q"(*(u32 *)ptr)
179 : [val] "r" (val)); 173 : [val] "r" (val));
180 } while (loop);
181 break; 174 break;
182 case 8: 175 case 8:
183 do { 176 asm ("//__percpu_xchg_8\n"
184 asm ("//__percpu_xchg_8\n" 177 "1: ldxr %[ret], %[ptr]\n"
185 "ldxr %[ret], %[ptr]\n" 178 " stxr %w[loop], %[val], %[ptr]\n"
186 "stxr %w[loop], %[val], %[ptr]\n" 179 " cbnz %w[loop], 1b"
187 : [loop] "=&r"(loop), [ret] "=&r"(ret), 180 : [loop] "=&r"(loop), [ret] "=&r"(ret),
188 [ptr] "+Q"(*(u64 *)ptr) 181 [ptr] "+Q"(*(u64 *)ptr)
189 : [val] "r" (val)); 182 : [val] "r" (val));
190 } while (loop);
191 break; 183 break;
192 default: 184 default:
193 BUILD_BUG(); 185 BUILD_BUG();
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-23 15:59:34 -0400