aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--tools/testing/selftests/.gitignore1
-rw-r--r--tools/testing/selftests/Makefile39
-rw-r--r--tools/testing/selftests/breakpoints/breakpoint_test.c15
-rw-r--r--tools/testing/selftests/breakpoints/breakpoint_test_arm64.c3
-rw-r--r--tools/testing/selftests/breakpoints/step_after_suspend_test.c8
-rw-r--r--tools/testing/selftests/capabilities/test_execve.c6
-rw-r--r--tools/testing/selftests/drivers/.gitignore1
-rw-r--r--tools/testing/selftests/futex/functional/futex_requeue_pi.c1
-rw-r--r--tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c1
-rw-r--r--tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c1
-rw-r--r--tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c1
-rw-r--r--tools/testing/selftests/futex/functional/futex_wait_timeout.c1
-rw-r--r--tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c1
-rw-r--r--tools/testing/selftests/futex/functional/futex_wait_wouldblock.c1
-rw-r--r--tools/testing/selftests/kselftest.h17
-rwxr-xr-xtools/testing/selftests/kselftest/prefix.pl23
-rw-r--r--tools/testing/selftests/kselftest/runner.sh86
-rw-r--r--tools/testing/selftests/lib.mk76
-rw-r--r--tools/testing/selftests/membarrier/membarrier_test.c1
-rw-r--r--tools/testing/selftests/pidfd/.gitignore1
-rw-r--r--tools/testing/selftests/pidfd/pidfd_test.c1
-rw-r--r--tools/testing/selftests/rseq/Makefile8
-rw-r--r--tools/testing/selftests/rseq/rseq-arm.h132
-rw-r--r--tools/testing/selftests/rseq/rseq-arm64.h74
-rw-r--r--tools/testing/selftests/rseq/rseq-mips.h115
-rw-r--r--tools/testing/selftests/rseq/rseq-ppc.h90
-rw-r--r--tools/testing/selftests/rseq/rseq-s390.h69
-rw-r--r--tools/testing/selftests/rseq/rseq-x86.h264
-rw-r--r--tools/testing/selftests/rseq/rseq.c55
-rw-r--r--tools/testing/selftests/rseq/rseq.h1
-rw-r--r--tools/testing/selftests/sigaltstack/sas.c1
-rw-r--r--tools/testing/selftests/sync/sync_test.c1
32 files changed, 875 insertions, 220 deletions
diff --git a/tools/testing/selftests/.gitignore b/tools/testing/selftests/.gitignore
index 91750352459d..8059ce834247 100644
--- a/tools/testing/selftests/.gitignore
+++ b/tools/testing/selftests/.gitignore
@@ -1,4 +1,3 @@
1kselftest
2gpiogpio-event-mon 1gpiogpio-event-mon
3gpiogpio-hammer 2gpiogpio-hammer
4gpioinclude/ 3gpioinclude/
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index f2ebf8cf4686..9781ca79794a 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -71,6 +71,9 @@ override LDFLAGS =
71override MAKEFLAGS = 71override MAKEFLAGS =
72endif 72endif
73 73
74# Append kselftest to KBUILD_OUTPUT to avoid cluttering
75# KBUILD_OUTPUT with selftest objects and headers installed
76# by selftests Makefile or lib.mk.
74ifneq ($(KBUILD_SRC),) 77ifneq ($(KBUILD_SRC),)
75override LDFLAGS = 78override LDFLAGS =
76endif 79endif
@@ -79,19 +82,13 @@ ifneq ($(O),)
79 BUILD := $(O) 82 BUILD := $(O)
80else 83else
81 ifneq ($(KBUILD_OUTPUT),) 84 ifneq ($(KBUILD_OUTPUT),)
82 BUILD := $(KBUILD_OUTPUT) 85 BUILD := $(KBUILD_OUTPUT)/kselftest
83 else 86 else
84 BUILD := $(shell pwd) 87 BUILD := $(shell pwd)
85 DEFAULT_INSTALL_HDR_PATH := 1 88 DEFAULT_INSTALL_HDR_PATH := 1
86 endif 89 endif
87endif 90endif
88 91
89# KSFT_TAP_LEVEL is used from KSFT framework to prevent nested TAP header
90# printing from tests. Applicable to run_tests case where run_tests adds
91# TAP header prior running tests and when a test program invokes another
92# with system() call. Export it here to cover override RUN_TESTS defines.
93export KSFT_TAP_LEVEL=`echo 1`
94
95# Prepare for headers install 92# Prepare for headers install
96top_srcdir ?= ../../.. 93top_srcdir ?= ../../..
97include $(top_srcdir)/scripts/subarch.include 94include $(top_srcdir)/scripts/subarch.include
@@ -169,14 +166,22 @@ clean_hotplug:
169run_pstore_crash: 166run_pstore_crash:
170 make -C pstore run_crash 167 make -C pstore run_crash
171 168
172INSTALL_PATH ?= install 169# Use $BUILD as the default install root. $BUILD points to the
170# right output location for the following cases:
171# 1. output_dir=kernel_src
172# 2. a separate output directory is specified using O= KBUILD_OUTPUT
173# 3. a separate output directory is specified using KBUILD_OUTPUT
174#
175INSTALL_PATH ?= $(BUILD)/install
173INSTALL_PATH := $(abspath $(INSTALL_PATH)) 176INSTALL_PATH := $(abspath $(INSTALL_PATH))
174ALL_SCRIPT := $(INSTALL_PATH)/run_kselftest.sh 177ALL_SCRIPT := $(INSTALL_PATH)/run_kselftest.sh
175 178
176install: 179install: all
177ifdef INSTALL_PATH 180ifdef INSTALL_PATH
178 @# Ask all targets to install their files 181 @# Ask all targets to install their files
179 mkdir -p $(INSTALL_PATH) 182 mkdir -p $(INSTALL_PATH)/kselftest
183 install -m 744 kselftest/runner.sh $(INSTALL_PATH)/kselftest/
184 install -m 744 kselftest/prefix.pl $(INSTALL_PATH)/kselftest/
180 @for TARGET in $(TARGETS); do \ 185 @for TARGET in $(TARGETS); do \
181 BUILD_TARGET=$$BUILD/$$TARGET; \ 186 BUILD_TARGET=$$BUILD/$$TARGET; \
182 make OUTPUT=$$BUILD_TARGET -C $$TARGET INSTALL_PATH=$(INSTALL_PATH)/$$TARGET install; \ 187 make OUTPUT=$$BUILD_TARGET -C $$TARGET INSTALL_PATH=$(INSTALL_PATH)/$$TARGET install; \
@@ -186,24 +191,20 @@ ifdef INSTALL_PATH
186 echo "#!/bin/sh" > $(ALL_SCRIPT) 191 echo "#!/bin/sh" > $(ALL_SCRIPT)
187 echo "BASE_DIR=\$$(realpath \$$(dirname \$$0))" >> $(ALL_SCRIPT) 192 echo "BASE_DIR=\$$(realpath \$$(dirname \$$0))" >> $(ALL_SCRIPT)
188 echo "cd \$$BASE_DIR" >> $(ALL_SCRIPT) 193 echo "cd \$$BASE_DIR" >> $(ALL_SCRIPT)
194 echo ". ./kselftest/runner.sh" >> $(ALL_SCRIPT)
189 echo "ROOT=\$$PWD" >> $(ALL_SCRIPT) 195 echo "ROOT=\$$PWD" >> $(ALL_SCRIPT)
190 echo "if [ \"\$$1\" = \"--summary\" ]; then" >> $(ALL_SCRIPT) 196 echo "if [ \"\$$1\" = \"--summary\" ]; then" >> $(ALL_SCRIPT)
191 echo " OUTPUT=\$$BASE_DIR/output.log" >> $(ALL_SCRIPT) 197 echo " logfile=\$$BASE_DIR/output.log" >> $(ALL_SCRIPT)
192 echo " cat /dev/null > \$$OUTPUT" >> $(ALL_SCRIPT) 198 echo " cat /dev/null > \$$logfile" >> $(ALL_SCRIPT)
193 echo "else" >> $(ALL_SCRIPT)
194 echo " OUTPUT=/dev/stdout" >> $(ALL_SCRIPT)
195 echo "fi" >> $(ALL_SCRIPT) 199 echo "fi" >> $(ALL_SCRIPT)
196 echo "export KSFT_TAP_LEVEL=1" >> $(ALL_SCRIPT)
197 echo "export skip=4" >> $(ALL_SCRIPT)
198 200
199 for TARGET in $(TARGETS); do \ 201 for TARGET in $(TARGETS); do \
200 BUILD_TARGET=$$BUILD/$$TARGET; \ 202 BUILD_TARGET=$$BUILD/$$TARGET; \
201 echo "echo ; echo TAP version 13" >> $(ALL_SCRIPT); \
202 echo "echo Running tests in $$TARGET" >> $(ALL_SCRIPT); \
203 echo "echo ========================================" >> $(ALL_SCRIPT); \
204 echo "[ -w /dev/kmsg ] && echo \"kselftest: Running tests in $$TARGET\" >> /dev/kmsg" >> $(ALL_SCRIPT); \ 203 echo "[ -w /dev/kmsg ] && echo \"kselftest: Running tests in $$TARGET\" >> /dev/kmsg" >> $(ALL_SCRIPT); \
205 echo "cd $$TARGET" >> $(ALL_SCRIPT); \ 204 echo "cd $$TARGET" >> $(ALL_SCRIPT); \
205 echo -n "run_many" >> $(ALL_SCRIPT); \
206 make -s --no-print-directory OUTPUT=$$BUILD_TARGET -C $$TARGET emit_tests >> $(ALL_SCRIPT); \ 206 make -s --no-print-directory OUTPUT=$$BUILD_TARGET -C $$TARGET emit_tests >> $(ALL_SCRIPT); \
207 echo "" >> $(ALL_SCRIPT); \
207 echo "cd \$$ROOT" >> $(ALL_SCRIPT); \ 208 echo "cd \$$ROOT" >> $(ALL_SCRIPT); \
208 done; 209 done;
209 210
diff --git a/tools/testing/selftests/breakpoints/breakpoint_test.c b/tools/testing/selftests/breakpoints/breakpoint_test.c
index 901b85ea6a59..8f3655e59020 100644
--- a/tools/testing/selftests/breakpoints/breakpoint_test.c
+++ b/tools/testing/selftests/breakpoints/breakpoint_test.c
@@ -21,6 +21,8 @@
21 21
22#include "../kselftest.h" 22#include "../kselftest.h"
23 23
24#define COUNT_ISN_BPS 4
25#define COUNT_WPS 4
24 26
25/* Breakpoint access modes */ 27/* Breakpoint access modes */
26enum { 28enum {
@@ -220,7 +222,7 @@ static void trigger_tests(void)
220 if (!local && !global) 222 if (!local && !global)
221 continue; 223 continue;
222 224
223 for (i = 0; i < 4; i++) { 225 for (i = 0; i < COUNT_ISN_BPS; i++) {
224 dummy_funcs[i](); 226 dummy_funcs[i]();
225 check_trapped(); 227 check_trapped();
226 } 228 }
@@ -292,7 +294,7 @@ static void launch_instruction_breakpoints(char *buf, int local, int global)
292{ 294{
293 int i; 295 int i;
294 296
295 for (i = 0; i < 4; i++) { 297 for (i = 0; i < COUNT_ISN_BPS; i++) {
296 set_breakpoint_addr(dummy_funcs[i], i); 298 set_breakpoint_addr(dummy_funcs[i], i);
297 toggle_breakpoint(i, BP_X, 1, local, global, 1); 299 toggle_breakpoint(i, BP_X, 1, local, global, 1);
298 ptrace(PTRACE_CONT, child_pid, NULL, 0); 300 ptrace(PTRACE_CONT, child_pid, NULL, 0);
@@ -314,7 +316,7 @@ static void launch_watchpoints(char *buf, int mode, int len,
314 else 316 else
315 mode_str = "read"; 317 mode_str = "read";
316 318
317 for (i = 0; i < 4; i++) { 319 for (i = 0; i < COUNT_WPS; i++) {
318 set_breakpoint_addr(&dummy_var[i], i); 320 set_breakpoint_addr(&dummy_var[i], i);
319 toggle_breakpoint(i, mode, len, local, global, 1); 321 toggle_breakpoint(i, mode, len, local, global, 1);
320 ptrace(PTRACE_CONT, child_pid, NULL, 0); 322 ptrace(PTRACE_CONT, child_pid, NULL, 0);
@@ -330,8 +332,15 @@ static void launch_watchpoints(char *buf, int mode, int len,
330static void launch_tests(void) 332static void launch_tests(void)
331{ 333{
332 char buf[1024]; 334 char buf[1024];
335 unsigned int tests = 0;
333 int len, local, global, i; 336 int len, local, global, i;
334 337
338 tests += 3 * COUNT_ISN_BPS;
339 tests += sizeof(long) / 2 * 3 * COUNT_WPS;
340 tests += sizeof(long) / 2 * 3 * COUNT_WPS;
341 tests += 2;
342 ksft_set_plan(tests);
343
335 /* Instruction breakpoints */ 344 /* Instruction breakpoints */
336 for (local = 0; local < 2; local++) { 345 for (local = 0; local < 2; local++) {
337 for (global = 0; global < 2; global++) { 346 for (global = 0; global < 2; global++) {
diff --git a/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c
index 2d95e5adde72..ab59d814341a 100644
--- a/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c
+++ b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c
@@ -118,7 +118,7 @@ static bool set_watchpoint(pid_t pid, int size, int wp)
118 return false; 118 return false;
119} 119}
120 120
121static bool run_test(int wr_size, int wp_size, int wr, int wp) 121static bool arun_test(int wr_size, int wp_size, int wr, int wp)
122{ 122{
123 int status; 123 int status;
124 siginfo_t siginfo; 124 siginfo_t siginfo;
@@ -214,6 +214,7 @@ int main(int argc, char **argv)
214 bool result; 214 bool result;
215 215
216 ksft_print_header(); 216 ksft_print_header();
217 ksft_set_plan(213);
217 218
218 act.sa_handler = sigalrm; 219 act.sa_handler = sigalrm;
219 sigemptyset(&act.sa_mask); 220 sigemptyset(&act.sa_mask);
diff --git a/tools/testing/selftests/breakpoints/step_after_suspend_test.c b/tools/testing/selftests/breakpoints/step_after_suspend_test.c
index f82dcc1f8841..cf868b5e00f7 100644
--- a/tools/testing/selftests/breakpoints/step_after_suspend_test.c
+++ b/tools/testing/selftests/breakpoints/step_after_suspend_test.c
@@ -173,6 +173,7 @@ int main(int argc, char **argv)
173 int opt; 173 int opt;
174 bool do_suspend = true; 174 bool do_suspend = true;
175 bool succeeded = true; 175 bool succeeded = true;
176 unsigned int tests = 0;
176 cpu_set_t available_cpus; 177 cpu_set_t available_cpus;
177 int err; 178 int err;
178 int cpu; 179 int cpu;
@@ -191,6 +192,13 @@ int main(int argc, char **argv)
191 } 192 }
192 } 193 }
193 194
195 for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
196 if (!CPU_ISSET(cpu, &available_cpus))
197 continue;
198 tests++;
199 }
200 ksft_set_plan(tests);
201
194 if (do_suspend) 202 if (do_suspend)
195 suspend(); 203 suspend();
196 204
diff --git a/tools/testing/selftests/capabilities/test_execve.c b/tools/testing/selftests/capabilities/test_execve.c
index 3ab39a61b95b..df0ef02b4036 100644
--- a/tools/testing/selftests/capabilities/test_execve.c
+++ b/tools/testing/selftests/capabilities/test_execve.c
@@ -430,8 +430,6 @@ int main(int argc, char **argv)
430{ 430{
431 char *tmp1, *tmp2, *our_path; 431 char *tmp1, *tmp2, *our_path;
432 432
433 ksft_print_header();
434
435 /* Find our path */ 433 /* Find our path */
436 tmp1 = strdup(argv[0]); 434 tmp1 = strdup(argv[0]);
437 if (!tmp1) 435 if (!tmp1)
@@ -445,6 +443,8 @@ int main(int argc, char **argv)
445 mpid = getpid(); 443 mpid = getpid();
446 444
447 if (fork_wait()) { 445 if (fork_wait()) {
446 ksft_print_header();
447 ksft_set_plan(12);
448 ksft_print_msg("[RUN]\t+++ Tests with uid == 0 +++\n"); 448 ksft_print_msg("[RUN]\t+++ Tests with uid == 0 +++\n");
449 return do_tests(0, our_path); 449 return do_tests(0, our_path);
450 } 450 }
@@ -452,6 +452,8 @@ int main(int argc, char **argv)
452 ksft_print_msg("==================================================\n"); 452 ksft_print_msg("==================================================\n");
453 453
454 if (fork_wait()) { 454 if (fork_wait()) {
455 ksft_print_header();
456 ksft_set_plan(9);
455 ksft_print_msg("[RUN]\t+++ Tests with uid != 0 +++\n"); 457 ksft_print_msg("[RUN]\t+++ Tests with uid != 0 +++\n");
456 return do_tests(1, our_path); 458 return do_tests(1, our_path);
457 } 459 }
diff --git a/tools/testing/selftests/drivers/.gitignore b/tools/testing/selftests/drivers/.gitignore
new file mode 100644
index 000000000000..f6aebcc27b76
--- /dev/null
+++ b/tools/testing/selftests/drivers/.gitignore
@@ -0,0 +1 @@
/dma-buf/udmabuf
diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi.c b/tools/testing/selftests/futex/functional/futex_requeue_pi.c
index 54cd5c414e82..8d20957f7586 100644
--- a/tools/testing/selftests/futex/functional/futex_requeue_pi.c
+++ b/tools/testing/selftests/futex/functional/futex_requeue_pi.c
@@ -395,6 +395,7 @@ int main(int argc, char *argv[])
395 } 395 }
396 396
397 ksft_print_header(); 397 ksft_print_header();
398 ksft_set_plan(1);
398 ksft_print_msg("%s: Test requeue functionality\n", basename(argv[0])); 399 ksft_print_msg("%s: Test requeue functionality\n", basename(argv[0]));
399 ksft_print_msg( 400 ksft_print_msg(
400 "\tArguments: broadcast=%d locked=%d owner=%d timeout=%ldns\n", 401 "\tArguments: broadcast=%d locked=%d owner=%d timeout=%ldns\n",
diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c b/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c
index 08187a16507f..742624c59ba7 100644
--- a/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c
+++ b/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c
@@ -79,6 +79,7 @@ int main(int argc, char *argv[])
79 } 79 }
80 80
81 ksft_print_header(); 81 ksft_print_header();
82 ksft_set_plan(1);
82 ksft_print_msg("%s: Detect mismatched requeue_pi operations\n", 83 ksft_print_msg("%s: Detect mismatched requeue_pi operations\n",
83 basename(argv[0])); 84 basename(argv[0]));
84 85
diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c b/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c
index f0542a344d95..a0f5934707ff 100644
--- a/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c
+++ b/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c
@@ -144,6 +144,7 @@ int main(int argc, char *argv[])
144 } 144 }
145 145
146 ksft_print_header(); 146 ksft_print_header();
147 ksft_set_plan(1);
147 ksft_print_msg("%s: Test signal handling during requeue_pi\n", 148 ksft_print_msg("%s: Test signal handling during requeue_pi\n",
148 basename(argv[0])); 149 basename(argv[0]));
149 ksft_print_msg("\tArguments: <none>\n"); 150 ksft_print_msg("\tArguments: <none>\n");
diff --git a/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c b/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c
index 6216de828093..a458d42ff86e 100644
--- a/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c
+++ b/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c
@@ -98,6 +98,7 @@ int main(int argc, char **argv)
98 } 98 }
99 99
100 ksft_print_header(); 100 ksft_print_header();
101 ksft_set_plan(1);
101 ksft_print_msg( 102 ksft_print_msg(
102 "%s: Test the futex value of private file mappings in FUTEX_WAIT\n", 103 "%s: Test the futex value of private file mappings in FUTEX_WAIT\n",
103 basename(argv[0])); 104 basename(argv[0]));
diff --git a/tools/testing/selftests/futex/functional/futex_wait_timeout.c b/tools/testing/selftests/futex/functional/futex_wait_timeout.c
index bab3dfe1787f..04b95478059c 100644
--- a/tools/testing/selftests/futex/functional/futex_wait_timeout.c
+++ b/tools/testing/selftests/futex/functional/futex_wait_timeout.c
@@ -69,6 +69,7 @@ int main(int argc, char *argv[])
69 } 69 }
70 70
71 ksft_print_header(); 71 ksft_print_header();
72 ksft_set_plan(1);
72 ksft_print_msg("%s: Block on a futex and wait for timeout\n", 73 ksft_print_msg("%s: Block on a futex and wait for timeout\n",
73 basename(argv[0])); 74 basename(argv[0]));
74 ksft_print_msg("\tArguments: timeout=%ldns\n", timeout_ns); 75 ksft_print_msg("\tArguments: timeout=%ldns\n", timeout_ns);
diff --git a/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c b/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c
index 26975322545b..3a1d12a14921 100644
--- a/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c
+++ b/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c
@@ -100,6 +100,7 @@ int main(int argc, char **argv)
100 } 100 }
101 101
102 ksft_print_header(); 102 ksft_print_header();
103 ksft_set_plan(1);
103 ksft_print_msg("%s: Test the uninitialized futex value in FUTEX_WAIT\n", 104 ksft_print_msg("%s: Test the uninitialized futex value in FUTEX_WAIT\n",
104 basename(argv[0])); 105 basename(argv[0]));
105 106
diff --git a/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c b/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c
index da15a63269b4..a34a6bbc30ce 100644
--- a/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c
+++ b/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c
@@ -65,6 +65,7 @@ int main(int argc, char *argv[])
65 } 65 }
66 66
67 ksft_print_header(); 67 ksft_print_header();
68 ksft_set_plan(1);
68 ksft_print_msg("%s: Test the unexpected futex value in FUTEX_WAIT\n", 69 ksft_print_msg("%s: Test the unexpected futex value in FUTEX_WAIT\n",
69 basename(argv[0])); 70 basename(argv[0]));
70 71
diff --git a/tools/testing/selftests/kselftest.h b/tools/testing/selftests/kselftest.h
index 47e1d995c182..ec15c4f6af55 100644
--- a/tools/testing/selftests/kselftest.h
+++ b/tools/testing/selftests/kselftest.h
@@ -33,6 +33,7 @@ struct ksft_count {
33}; 33};
34 34
35static struct ksft_count ksft_cnt; 35static struct ksft_count ksft_cnt;
36static unsigned int ksft_plan;
36 37
37static inline int ksft_test_num(void) 38static inline int ksft_test_num(void)
38{ 39{
@@ -61,13 +62,21 @@ static inline void ksft_print_header(void)
61 printf("TAP version 13\n"); 62 printf("TAP version 13\n");
62} 63}
63 64
65static inline void ksft_set_plan(unsigned int plan)
66{
67 ksft_plan = plan;
68 printf("1..%d\n", ksft_plan);
69}
70
64static inline void ksft_print_cnts(void) 71static inline void ksft_print_cnts(void)
65{ 72{
66 printf("Pass %d Fail %d Xfail %d Xpass %d Skip %d Error %d\n", 73 if (ksft_plan != ksft_test_num())
74 printf("# Planned tests != run tests (%u != %u)\n",
75 ksft_plan, ksft_test_num());
76 printf("# Pass %d Fail %d Xfail %d Xpass %d Skip %d Error %d\n",
67 ksft_cnt.ksft_pass, ksft_cnt.ksft_fail, 77 ksft_cnt.ksft_pass, ksft_cnt.ksft_fail,
68 ksft_cnt.ksft_xfail, ksft_cnt.ksft_xpass, 78 ksft_cnt.ksft_xfail, ksft_cnt.ksft_xpass,
69 ksft_cnt.ksft_xskip, ksft_cnt.ksft_error); 79 ksft_cnt.ksft_xskip, ksft_cnt.ksft_error);
70 printf("1..%d\n", ksft_test_num());
71} 80}
72 81
73static inline void ksft_print_msg(const char *msg, ...) 82static inline void ksft_print_msg(const char *msg, ...)
@@ -111,7 +120,7 @@ static inline void ksft_test_result_skip(const char *msg, ...)
111 ksft_cnt.ksft_xskip++; 120 ksft_cnt.ksft_xskip++;
112 121
113 va_start(args, msg); 122 va_start(args, msg);
114 printf("ok %d # skip ", ksft_test_num()); 123 printf("not ok %d # SKIP ", ksft_test_num());
115 vprintf(msg, args); 124 vprintf(msg, args);
116 va_end(args); 125 va_end(args);
117} 126}
@@ -172,7 +181,7 @@ static inline int ksft_exit_skip(const char *msg, ...)
172 va_list args; 181 va_list args;
173 182
174 va_start(args, msg); 183 va_start(args, msg);
175 printf("1..%d # Skipped: ", ksft_test_num()); 184 printf("not ok %d # SKIP ", 1 + ksft_test_num());
176 vprintf(msg, args); 185 vprintf(msg, args);
177 va_end(args); 186 va_end(args);
178 } else { 187 } else {
diff --git a/tools/testing/selftests/kselftest/prefix.pl b/tools/testing/selftests/kselftest/prefix.pl
new file mode 100755
index 000000000000..ec7e48118183
--- /dev/null
+++ b/tools/testing/selftests/kselftest/prefix.pl
@@ -0,0 +1,23 @@
1#!/usr/bin/perl
2# SPDX-License-Identifier: GPL-2.0
3# Prefix all lines with "# ", unbuffered. Command being piped in may need
4# to have unbuffering forced with "stdbuf -i0 -o0 -e0 $cmd".
5use strict;
6
7binmode STDIN;
8binmode STDOUT;
9
10STDOUT->autoflush(1);
11
12my $needed = 1;
13while (1) {
14 my $char;
15 my $bytes = sysread(STDIN, $char, 1);
16 exit 0 if ($bytes == 0);
17 if ($needed) {
18 print "# ";
19 $needed = 0;
20 }
21 print $char;
22 $needed = 1 if ($char eq "\n");
23}
diff --git a/tools/testing/selftests/kselftest/runner.sh b/tools/testing/selftests/kselftest/runner.sh
new file mode 100644
index 000000000000..eff3ee303d0d
--- /dev/null
+++ b/tools/testing/selftests/kselftest/runner.sh
@@ -0,0 +1,86 @@
1#!/bin/sh
2# SPDX-License-Identifier: GPL-2.0
3#
4# Runs a set of tests in a given subdirectory.
5export skip_rc=4
6export logfile=/dev/stdout
7export per_test_logging=
8
9# There isn't a shell-agnostic way to find the path of a sourced file,
10# so we must rely on BASE_DIR being set to find other tools.
11if [ -z "$BASE_DIR" ]; then
12 echo "Error: BASE_DIR must be set before sourcing." >&2
13 exit 1
14fi
15
16# If Perl is unavailable, we must fall back to line-at-a-time prefixing
17# with sed instead of unbuffered output.
18tap_prefix()
19{
20 if [ ! -x /usr/bin/perl ]; then
21 sed -e 's/^/# /'
22 else
23 "$BASE_DIR"/kselftest/prefix.pl
24 fi
25}
26
27# If stdbuf is unavailable, we must fall back to line-at-a-time piping.
28tap_unbuffer()
29{
30 if ! which stdbuf >/dev/null ; then
31 "$@"
32 else
33 stdbuf -i0 -o0 -e0 "$@"
34 fi
35}
36
37run_one()
38{
39 DIR="$1"
40 TEST="$2"
41 NUM="$3"
42
43 BASENAME_TEST=$(basename $TEST)
44
45 TEST_HDR_MSG="selftests: $DIR: $BASENAME_TEST"
46 echo "# $TEST_HDR_MSG"
47 if [ ! -x "$TEST" ]; then
48 echo -n "# Warning: file $TEST is "
49 if [ ! -e "$TEST" ]; then
50 echo "missing!"
51 else
52 echo "not executable, correct this."
53 fi
54 echo "not ok $test_num $TEST_HDR_MSG"
55 else
56 cd `dirname $TEST` > /dev/null
57 (((((tap_unbuffer ./$BASENAME_TEST 2>&1; echo $? >&3) |
58 tap_prefix >&4) 3>&1) |
59 (read xs; exit $xs)) 4>>"$logfile" &&
60 echo "ok $test_num $TEST_HDR_MSG") ||
61 (if [ $? -eq $skip_rc ]; then \
62 echo "not ok $test_num $TEST_HDR_MSG # SKIP"
63 else
64 echo "not ok $test_num $TEST_HDR_MSG"
65 fi)
66 cd - >/dev/null
67 fi
68}
69
70run_many()
71{
72 echo "TAP version 13"
73 DIR=$(basename "$PWD")
74 test_num=0
75 total=$(echo "$@" | wc -w)
76 echo "1..$total"
77 for TEST in "$@"; do
78 BASENAME_TEST=$(basename $TEST)
79 test_num=$(( test_num + 1 ))
80 if [ -n "$per_test_logging" ]; then
81 logfile="/tmp/$BASENAME_TEST"
82 cat /dev/null > "$logfile"
83 fi
84 run_one "$DIR" "$TEST" "$test_num"
85 done
86}
diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk
index 5979fdc4f36c..077337195783 100644
--- a/tools/testing/selftests/lib.mk
+++ b/tools/testing/selftests/lib.mk
@@ -3,17 +3,12 @@
3CC := $(CROSS_COMPILE)gcc 3CC := $(CROSS_COMPILE)gcc
4 4
5ifeq (0,$(MAKELEVEL)) 5ifeq (0,$(MAKELEVEL))
6 ifneq ($(O),) 6 ifeq ($(OUTPUT),)
7 OUTPUT := $(O) 7 OUTPUT := $(shell pwd)
8 else 8 DEFAULT_INSTALL_HDR_PATH := 1
9 ifneq ($(KBUILD_OUTPUT),)
10 OUTPUT := $(KBUILD_OUTPUT)
11 else
12 OUTPUT := $(shell pwd)
13 DEFAULT_INSTALL_HDR_PATH := 1
14 endif
15 endif 9 endif
16endif 10endif
11selfdir = $(realpath $(dir $(filter %/lib.mk,$(MAKEFILE_LIST))))
17 12
18# The following are built by lib.mk common compile rules. 13# The following are built by lib.mk common compile rules.
19# TEST_CUSTOM_PROGS should be used by tests that require 14# TEST_CUSTOM_PROGS should be used by tests that require
@@ -65,44 +60,13 @@ all: $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED) $(TEST_GEN_FILES)
65endif 60endif
66 61
67.ONESHELL: 62.ONESHELL:
68define RUN_TEST_PRINT_RESULT
69 TEST_HDR_MSG="selftests: "`basename $$PWD`:" $$BASENAME_TEST"; \
70 echo $$TEST_HDR_MSG; \
71 echo "========================================"; \
72 if [ ! -x $$TEST ]; then \
73 echo "$$TEST_HDR_MSG: Warning: file $$BASENAME_TEST is not executable, correct this.";\
74 echo "not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]"; \
75 else \
76 cd `dirname $$TEST` > /dev/null; \
77 if [ "X$(summary)" != "X" ]; then \
78 (./$$BASENAME_TEST > /tmp/$$BASENAME_TEST 2>&1 && \
79 echo "ok 1..$$test_num $$TEST_HDR_MSG [PASS]") || \
80 (if [ $$? -eq $$skip ]; then \
81 echo "not ok 1..$$test_num $$TEST_HDR_MSG [SKIP]"; \
82 else echo "not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]"; \
83 fi;) \
84 else \
85 (./$$BASENAME_TEST && \
86 echo "ok 1..$$test_num $$TEST_HDR_MSG [PASS]") || \
87 (if [ $$? -eq $$skip ]; then \
88 echo "not ok 1..$$test_num $$TEST_HDR_MSG [SKIP]"; \
89 else echo "not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]"; \
90 fi;) \
91 fi; \
92 cd - > /dev/null; \
93 fi;
94endef
95
96define RUN_TESTS 63define RUN_TESTS
97 @export KSFT_TAP_LEVEL=`echo 1`; \ 64 @BASE_DIR="$(selfdir)"; \
98 test_num=`echo 0`; \ 65 . $(selfdir)/kselftest/runner.sh; \
99 skip=`echo 4`; \ 66 if [ "X$(summary)" != "X" ]; then \
100 echo "TAP version 13"; \ 67 per_test_logging=1; \
101 for TEST in $(1); do \ 68 fi; \
102 BASENAME_TEST=`basename $$TEST`; \ 69 run_many $(1)
103 test_num=`echo $$test_num+1 | bc`; \
104 $(call RUN_TEST_PRINT_RESULT,$(TEST),$(BASENAME_TEST),$(test_num),$(skip)) \
105 done;
106endef 70endef
107 71
108run_tests: all 72run_tests: all
@@ -139,24 +103,12 @@ else
139 $(error Error: set INSTALL_PATH to use install) 103 $(error Error: set INSTALL_PATH to use install)
140endif 104endif
141 105
142define EMIT_TESTS 106emit_tests:
143 @test_num=`echo 0`; \
144 for TEST in $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) $(TEST_PROGS); do \ 107 for TEST in $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) $(TEST_PROGS); do \
145 BASENAME_TEST=`basename $$TEST`; \ 108 BASENAME_TEST=`basename $$TEST`; \
146 test_num=`echo $$test_num+1 | bc`; \ 109 echo " \\"; \
147 TEST_HDR_MSG="selftests: "`basename $$PWD`:" $$BASENAME_TEST"; \ 110 echo -n " \"$$BASENAME_TEST\""; \
148 echo "echo $$TEST_HDR_MSG"; \ 111 done; \
149 if [ ! -x $$TEST ]; then \
150 echo "echo \"$$TEST_HDR_MSG: Warning: file $$BASENAME_TEST is not executable, correct this.\""; \
151 echo "echo \"not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]\""; \
152 else
153 echo "(./$$BASENAME_TEST >> \$$OUTPUT 2>&1 && echo \"ok 1..$$test_num $$TEST_HDR_MSG [PASS]\") || (if [ \$$? -eq \$$skip ]; then echo \"not ok 1..$$test_num $$TEST_HDR_MSG [SKIP]\"; else echo \"not ok 1..$$test_num $$TEST_HDR_MSG [FAIL]\"; fi;)"; \
154 fi; \
155 done;
156endef
157
158emit_tests:
159 $(EMIT_TESTS)
160 112
161# define if isn't already. It is undefined in make O= case. 113# define if isn't already. It is undefined in make O= case.
162ifeq ($(RM),) 114ifeq ($(RM),)
diff --git a/tools/testing/selftests/membarrier/membarrier_test.c b/tools/testing/selftests/membarrier/membarrier_test.c
index 6793f8ecc8e7..70b4ddbf126b 100644
--- a/tools/testing/selftests/membarrier/membarrier_test.c
+++ b/tools/testing/selftests/membarrier/membarrier_test.c
@@ -304,6 +304,7 @@ static int test_membarrier_query(void)
304int main(int argc, char **argv) 304int main(int argc, char **argv)
305{ 305{
306 ksft_print_header(); 306 ksft_print_header();
307 ksft_set_plan(13);
307 308
308 test_membarrier_query(); 309 test_membarrier_query();
309 test_membarrier(); 310 test_membarrier();
diff --git a/tools/testing/selftests/pidfd/.gitignore b/tools/testing/selftests/pidfd/.gitignore
new file mode 100644
index 000000000000..822a1e63d045
--- /dev/null
+++ b/tools/testing/selftests/pidfd/.gitignore
@@ -0,0 +1 @@
pidfd_test
diff --git a/tools/testing/selftests/pidfd/pidfd_test.c b/tools/testing/selftests/pidfd/pidfd_test.c
index d59378a93782..5bae1792e3d6 100644
--- a/tools/testing/selftests/pidfd/pidfd_test.c
+++ b/tools/testing/selftests/pidfd/pidfd_test.c
@@ -371,6 +371,7 @@ static int test_pidfd_send_signal_syscall_support(void)
371int main(int argc, char **argv) 371int main(int argc, char **argv)
372{ 372{
373 ksft_print_header(); 373 ksft_print_header();
374 ksft_set_plan(4);
374 375
375 test_pidfd_send_signal_syscall_support(); 376 test_pidfd_send_signal_syscall_support();
376 test_pidfd_send_signal_simple_success(); 377 test_pidfd_send_signal_simple_success();
diff --git a/tools/testing/selftests/rseq/Makefile b/tools/testing/selftests/rseq/Makefile
index c30c52e1d0d2..d6469535630a 100644
--- a/tools/testing/selftests/rseq/Makefile
+++ b/tools/testing/selftests/rseq/Makefile
@@ -1,5 +1,11 @@
1# SPDX-License-Identifier: GPL-2.0+ OR MIT 1# SPDX-License-Identifier: GPL-2.0+ OR MIT
2CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/ -L./ -Wl,-rpath=./ 2
3ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep clang),)
4CLANG_FLAGS += -no-integrated-as
5endif
6
7CFLAGS += -O2 -Wall -g -I./ -I../../../../usr/include/ -L./ -Wl,-rpath=./ \
8 $(CLANG_FLAGS)
3LDLIBS += -lpthread 9LDLIBS += -lpthread
4 10
5# Own dependencies because we only want to build against 1st prerequisite, but 11# Own dependencies because we only want to build against 1st prerequisite, but
diff --git a/tools/testing/selftests/rseq/rseq-arm.h b/tools/testing/selftests/rseq/rseq-arm.h
index 3cea19877227..84f28f147fb6 100644
--- a/tools/testing/selftests/rseq/rseq-arm.h
+++ b/tools/testing/selftests/rseq/rseq-arm.h
@@ -5,7 +5,54 @@
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */ 6 */
7 7
8#define RSEQ_SIG 0x53053053 8/*
9 * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand
10 * value 0x5de3. This traps if user-space reaches this instruction by mistake,
11 * and the uncommon operand ensures the kernel does not move the instruction
12 * pointer to attacker-controlled code on rseq abort.
13 *
14 * The instruction pattern in the A32 instruction set is:
15 *
16 * e7f5def3 udf #24035 ; 0x5de3
17 *
18 * This translates to the following instruction pattern in the T16 instruction
19 * set:
20 *
21 * little endian:
22 * def3 udf #243 ; 0xf3
23 * e7f5 b.n <7f5>
24 *
25 * pre-ARMv6 big endian code:
26 * e7f5 b.n <7f5>
27 * def3 udf #243 ; 0xf3
28 *
29 * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
30 * code and big-endian data. Ensure the RSEQ_SIG data signature matches code
31 * endianness. Prior to ARMv6, -mbig-endian generates big-endian code and data
32 * (which match), so there is no need to reverse the endianness of the data
33 * representation of the signature. However, the choice between BE32 and BE8
34 * is done by the linker, so we cannot know whether code and data endianness
35 * will be mixed before the linker is invoked.
36 */
37
38#define RSEQ_SIG_CODE 0xe7f5def3
39
40#ifndef __ASSEMBLER__
41
42#define RSEQ_SIG_DATA \
43 ({ \
44 int sig; \
45 asm volatile ("b 2f\n\t" \
46 "1: .inst " __rseq_str(RSEQ_SIG_CODE) "\n\t" \
47 "2:\n\t" \
48 "ldr %[sig], 1b\n\t" \
49 : [sig] "=r" (sig)); \
50 sig; \
51 })
52
53#define RSEQ_SIG RSEQ_SIG_DATA
54
55#endif
9 56
10#define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 57#define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
11#define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") 58#define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
@@ -30,18 +77,35 @@ do { \
30#include "rseq-skip.h" 77#include "rseq-skip.h"
31#else /* !RSEQ_SKIP_FASTPATH */ 78#else /* !RSEQ_SKIP_FASTPATH */
32 79
33#define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \ 80#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
34 post_commit_offset, abort_ip) \ 81 post_commit_offset, abort_ip) \
35 ".pushsection __rseq_table, \"aw\"\n\t" \ 82 ".pushsection __rseq_cs, \"aw\"\n\t" \
36 ".balign 32\n\t" \ 83 ".balign 32\n\t" \
84 __rseq_str(label) ":\n\t" \
37 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 85 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
38 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 86 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
87 ".popsection\n\t" \
88 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
89 ".word " __rseq_str(label) "b, 0x0\n\t" \
39 ".popsection\n\t" 90 ".popsection\n\t"
40 91
41#define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \ 92#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
42 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \ 93 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
43 (post_commit_ip - start_ip), abort_ip) 94 (post_commit_ip - start_ip), abort_ip)
44 95
96/*
97 * Exit points of a rseq critical section consist of all instructions outside
98 * of the critical section where a critical section can either branch to or
99 * reach through the normal course of its execution. The abort IP and the
100 * post-commit IP are already part of the __rseq_cs section and should not be
101 * explicitly defined as additional exit points. Knowing all exit points is
102 * useful to assist debuggers stepping over the critical section.
103 */
104#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
105 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
106 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
107 ".popsection\n\t"
108
45#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 109#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
46 RSEQ_INJECT_ASM(1) \ 110 RSEQ_INJECT_ASM(1) \
47 "adr r0, " __rseq_str(cs_label) "\n\t" \ 111 "adr r0, " __rseq_str(cs_label) "\n\t" \
@@ -61,7 +125,8 @@ do { \
61 __rseq_str(table_label) ":\n\t" \ 125 __rseq_str(table_label) ":\n\t" \
62 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 126 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
63 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 127 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
64 ".word " __rseq_str(RSEQ_SIG) "\n\t" \ 128 ".arm\n\t" \
129 ".inst " __rseq_str(RSEQ_SIG_CODE) "\n\t" \
65 __rseq_str(label) ":\n\t" \ 130 __rseq_str(label) ":\n\t" \
66 teardown \ 131 teardown \
67 "b %l[" __rseq_str(abort_label) "]\n\t" 132 "b %l[" __rseq_str(abort_label) "]\n\t"
@@ -86,7 +151,12 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
86 151
87 rseq_workaround_gcc_asm_size_guess(); 152 rseq_workaround_gcc_asm_size_guess();
88 __asm__ __volatile__ goto ( 153 __asm__ __volatile__ goto (
89 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 154 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
155 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
156#ifdef RSEQ_COMPARE_TWICE
157 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
158 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
159#endif
90 /* Start rseq by storing table entry pointer into rseq_cs. */ 160 /* Start rseq by storing table entry pointer into rseq_cs. */
91 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 161 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
92 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 162 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -148,7 +218,12 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
148 218
149 rseq_workaround_gcc_asm_size_guess(); 219 rseq_workaround_gcc_asm_size_guess();
150 __asm__ __volatile__ goto ( 220 __asm__ __volatile__ goto (
151 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 221 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
222 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
223#ifdef RSEQ_COMPARE_TWICE
224 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
225 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
226#endif
152 /* Start rseq by storing table entry pointer into rseq_cs. */ 227 /* Start rseq by storing table entry pointer into rseq_cs. */
153 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 228 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
154 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 229 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -214,7 +289,10 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
214 289
215 rseq_workaround_gcc_asm_size_guess(); 290 rseq_workaround_gcc_asm_size_guess();
216 __asm__ __volatile__ goto ( 291 __asm__ __volatile__ goto (
217 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 292 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
293#ifdef RSEQ_COMPARE_TWICE
294 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
295#endif
218 /* Start rseq by storing table entry pointer into rseq_cs. */ 296 /* Start rseq by storing table entry pointer into rseq_cs. */
219 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 297 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 298 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -266,7 +344,12 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
266 344
267 rseq_workaround_gcc_asm_size_guess(); 345 rseq_workaround_gcc_asm_size_guess();
268 __asm__ __volatile__ goto ( 346 __asm__ __volatile__ goto (
269 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 347 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
348 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
349#ifdef RSEQ_COMPARE_TWICE
350 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
351 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
352#endif
270 /* Start rseq by storing table entry pointer into rseq_cs. */ 353 /* Start rseq by storing table entry pointer into rseq_cs. */
271 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 354 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
272 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 355 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -336,7 +419,12 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
336 419
337 rseq_workaround_gcc_asm_size_guess(); 420 rseq_workaround_gcc_asm_size_guess();
338 __asm__ __volatile__ goto ( 421 __asm__ __volatile__ goto (
339 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 422 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
423 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
424#ifdef RSEQ_COMPARE_TWICE
425 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
426 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
427#endif
340 /* Start rseq by storing table entry pointer into rseq_cs. */ 428 /* Start rseq by storing table entry pointer into rseq_cs. */
341 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 429 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
342 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 430 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -407,7 +495,13 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
407 495
408 rseq_workaround_gcc_asm_size_guess(); 496 rseq_workaround_gcc_asm_size_guess();
409 __asm__ __volatile__ goto ( 497 __asm__ __volatile__ goto (
410 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 498 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
499 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
500#ifdef RSEQ_COMPARE_TWICE
501 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
502 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
503 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
504#endif
411 /* Start rseq by storing table entry pointer into rseq_cs. */ 505 /* Start rseq by storing table entry pointer into rseq_cs. */
412 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 506 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
413 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 507 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -485,7 +579,12 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
485 579
486 rseq_workaround_gcc_asm_size_guess(); 580 rseq_workaround_gcc_asm_size_guess();
487 __asm__ __volatile__ goto ( 581 __asm__ __volatile__ goto (
488 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 582 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
583 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
584#ifdef RSEQ_COMPARE_TWICE
585 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
586 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
587#endif
489 "str %[src], %[rseq_scratch0]\n\t" 588 "str %[src], %[rseq_scratch0]\n\t"
490 "str %[dst], %[rseq_scratch1]\n\t" 589 "str %[dst], %[rseq_scratch1]\n\t"
491 "str %[len], %[rseq_scratch2]\n\t" 590 "str %[len], %[rseq_scratch2]\n\t"
@@ -604,7 +703,12 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
604 703
605 rseq_workaround_gcc_asm_size_guess(); 704 rseq_workaround_gcc_asm_size_guess();
606 __asm__ __volatile__ goto ( 705 __asm__ __volatile__ goto (
607 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 706 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
707 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
708#ifdef RSEQ_COMPARE_TWICE
709 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
710 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
711#endif
608 "str %[src], %[rseq_scratch0]\n\t" 712 "str %[src], %[rseq_scratch0]\n\t"
609 "str %[dst], %[rseq_scratch1]\n\t" 713 "str %[dst], %[rseq_scratch1]\n\t"
610 "str %[len], %[rseq_scratch2]\n\t" 714 "str %[len], %[rseq_scratch2]\n\t"
diff --git a/tools/testing/selftests/rseq/rseq-arm64.h b/tools/testing/selftests/rseq/rseq-arm64.h
index 954f34671ca6..200dae9e4208 100644
--- a/tools/testing/selftests/rseq/rseq-arm64.h
+++ b/tools/testing/selftests/rseq/rseq-arm64.h
@@ -6,7 +6,20 @@
6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> 6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7 */ 7 */
8 8
9#define RSEQ_SIG 0xd428bc00 /* BRK #0x45E0 */ 9/*
10 * aarch64 -mbig-endian generates mixed endianness code vs data:
11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
12 * matches code endianness.
13 */
14#define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
15
16#ifdef __AARCH64EB__
17#define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
18#else
19#define RSEQ_SIG_DATA RSEQ_SIG_CODE
20#endif
21
22#define RSEQ_SIG RSEQ_SIG_DATA
10 23
11#define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory") 24#define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
12#define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory") 25#define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
@@ -82,19 +95,35 @@ do { \
82 95
83#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \ 96#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
84 post_commit_offset, abort_ip) \ 97 post_commit_offset, abort_ip) \
85 " .pushsection __rseq_table, \"aw\"\n" \ 98 " .pushsection __rseq_cs, \"aw\"\n" \
86 " .balign 32\n" \ 99 " .balign 32\n" \
87 __rseq_str(label) ":\n" \ 100 __rseq_str(label) ":\n" \
88 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \ 101 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
89 " .quad " __rseq_str(start_ip) ", " \ 102 " .quad " __rseq_str(start_ip) ", " \
90 __rseq_str(post_commit_offset) ", " \ 103 __rseq_str(post_commit_offset) ", " \
91 __rseq_str(abort_ip) "\n" \ 104 __rseq_str(abort_ip) "\n" \
105 " .popsection\n\t" \
106 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
107 " .quad " __rseq_str(label) "b\n" \
92 " .popsection\n" 108 " .popsection\n"
93 109
94#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 110#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
95 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 111 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
96 (post_commit_ip - start_ip), abort_ip) 112 (post_commit_ip - start_ip), abort_ip)
97 113
114/*
115 * Exit points of a rseq critical section consist of all instructions outside
116 * of the critical section where a critical section can either branch to or
117 * reach through the normal course of its execution. The abort IP and the
118 * post-commit IP are already part of the __rseq_cs section and should not be
119 * explicitly defined as additional exit points. Knowing all exit points is
120 * useful to assist debuggers stepping over the critical section.
121 */
122#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
123 " .pushsection __rseq_exit_point_array, \"aw\"\n" \
124 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
125 " .popsection\n"
126
98#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 127#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
99 RSEQ_INJECT_ASM(1) \ 128 RSEQ_INJECT_ASM(1) \
100 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \ 129 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
@@ -105,7 +134,7 @@ do { \
105 134
106#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ 135#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
107 " b 222f\n" \ 136 " b 222f\n" \
108 " .inst " __rseq_str(RSEQ_SIG) "\n" \ 137 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
109 __rseq_str(label) ":\n" \ 138 __rseq_str(label) ":\n" \
110 " b %l[" __rseq_str(abort_label) "]\n" \ 139 " b %l[" __rseq_str(abort_label) "]\n" \
111 "222:\n" 140 "222:\n"
@@ -182,6 +211,11 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
182 211
183 __asm__ __volatile__ goto ( 212 __asm__ __volatile__ goto (
184 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 213 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
214 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
215#ifdef RSEQ_COMPARE_TWICE
216 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
217 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
218#endif
185 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 219 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
186 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
187 RSEQ_INJECT_ASM(3) 221 RSEQ_INJECT_ASM(3)
@@ -231,6 +265,11 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
231 265
232 __asm__ __volatile__ goto ( 266 __asm__ __volatile__ goto (
233 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 267 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
268 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
269#ifdef RSEQ_COMPARE_TWICE
270 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
271 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
272#endif
234 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 273 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
235 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 274 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
236 RSEQ_INJECT_ASM(3) 275 RSEQ_INJECT_ASM(3)
@@ -282,6 +321,9 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
282 321
283 __asm__ __volatile__ goto ( 322 __asm__ __volatile__ goto (
284 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 323 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
324#ifdef RSEQ_COMPARE_TWICE
325 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
326#endif
285 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 327 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
286 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 328 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
287 RSEQ_INJECT_ASM(3) 329 RSEQ_INJECT_ASM(3)
@@ -325,6 +367,11 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
325 367
326 __asm__ __volatile__ goto ( 368 __asm__ __volatile__ goto (
327 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 369 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
370 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
371#ifdef RSEQ_COMPARE_TWICE
372 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
373 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
374#endif
328 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 375 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
329 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 376 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
330 RSEQ_INJECT_ASM(3) 377 RSEQ_INJECT_ASM(3)
@@ -379,6 +426,11 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
379 426
380 __asm__ __volatile__ goto ( 427 __asm__ __volatile__ goto (
381 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 428 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
429 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
430#ifdef RSEQ_COMPARE_TWICE
431 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
432 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
433#endif
382 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 434 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
383 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 435 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
384 RSEQ_INJECT_ASM(3) 436 RSEQ_INJECT_ASM(3)
@@ -433,6 +485,12 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
433 485
434 __asm__ __volatile__ goto ( 486 __asm__ __volatile__ goto (
435 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 487 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
488 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
489#ifdef RSEQ_COMPARE_TWICE
490 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
491 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
492 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
493#endif
436 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 494 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
437 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 495 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
438 RSEQ_INJECT_ASM(3) 496 RSEQ_INJECT_ASM(3)
@@ -490,6 +548,11 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
490 548
491 __asm__ __volatile__ goto ( 549 __asm__ __volatile__ goto (
492 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 550 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
551 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
552#ifdef RSEQ_COMPARE_TWICE
553 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
554 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
555#endif
493 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 556 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
494 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 557 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
495 RSEQ_INJECT_ASM(3) 558 RSEQ_INJECT_ASM(3)
@@ -545,6 +608,11 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
545 608
546 __asm__ __volatile__ goto ( 609 __asm__ __volatile__ goto (
547 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 610 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
611 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
612#ifdef RSEQ_COMPARE_TWICE
613 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
614 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
615#endif
548 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 616 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
549 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 617 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
550 RSEQ_INJECT_ASM(3) 618 RSEQ_INJECT_ASM(3)
diff --git a/tools/testing/selftests/rseq/rseq-mips.h b/tools/testing/selftests/rseq/rseq-mips.h
index 7f48ecf46994..e989e7c14b09 100644
--- a/tools/testing/selftests/rseq/rseq-mips.h
+++ b/tools/testing/selftests/rseq/rseq-mips.h
@@ -7,7 +7,39 @@
7 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 7 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
8 */ 8 */
9 9
10#define RSEQ_SIG 0x53053053 10/*
11 * RSEQ_SIG uses the break instruction. The instruction pattern is:
12 *
13 * On MIPS:
14 * 0350000d break 0x350
15 *
16 * On nanoMIPS:
17 * 00100350 break 0x350
18 *
19 * On microMIPS:
20 * 0000d407 break 0x350
21 *
22 * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit
23 * halfwords, so the signature halfwords need to be swapped accordingly for
24 * little-endian.
25 */
26#if defined(__nanomips__)
27# ifdef __MIPSEL__
28# define RSEQ_SIG 0x03500010
29# else
30# define RSEQ_SIG 0x00100350
31# endif
32#elif defined(__mips_micromips)
33# ifdef __MIPSEL__
34# define RSEQ_SIG 0xd4070000
35# else
36# define RSEQ_SIG 0x0000d407
37# endif
38#elif defined(__mips__)
39# define RSEQ_SIG 0x0350000d
40#else
41/* Unknown MIPS architecture. */
42#endif
11 43
12#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory") 44#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory")
13#define rseq_smp_rmb() rseq_smp_mb() 45#define rseq_smp_rmb() rseq_smp_mb()
@@ -54,20 +86,38 @@ do { \
54# error unsupported _MIPS_SZLONG 86# error unsupported _MIPS_SZLONG
55#endif 87#endif
56 88
57#define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \ 89#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
58 post_commit_offset, abort_ip) \ 90 post_commit_offset, abort_ip) \
59 ".pushsection __rseq_table, \"aw\"\n\t" \ 91 ".pushsection __rseq_cs, \"aw\"\n\t" \
60 ".balign 32\n\t" \ 92 ".balign 32\n\t" \
93 __rseq_str(label) ":\n\t" \
61 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 94 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
62 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \ 95 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
63 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \ 96 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
64 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \ 97 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
98 ".popsection\n\t" \
99 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
100 LONG " " U32_U64_PAD(__rseq_str(label) "b") "\n\t" \
65 ".popsection\n\t" 101 ".popsection\n\t"
66 102
67#define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \ 103#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
68 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \ 104 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
69 (post_commit_ip - start_ip), abort_ip) 105 (post_commit_ip - start_ip), abort_ip)
70 106
107/*
108 * Exit points of a rseq critical section consist of all instructions outside
109 * of the critical section where a critical section can either branch to or
110 * reach through the normal course of its execution. The abort IP and the
111 * post-commit IP are already part of the __rseq_cs section and should not be
112 * explicitly defined as additional exit points. Knowing all exit points is
113 * useful to assist debuggers stepping over the critical section.
114 */
115#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
116 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
117 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
118 LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \
119 ".popsection\n\t"
120
71#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 121#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
72 RSEQ_INJECT_ASM(1) \ 122 RSEQ_INJECT_ASM(1) \
73 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \ 123 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
@@ -113,7 +163,12 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
113 163
114 rseq_workaround_gcc_asm_size_guess(); 164 rseq_workaround_gcc_asm_size_guess();
115 __asm__ __volatile__ goto ( 165 __asm__ __volatile__ goto (
116 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 166 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
167 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
168#ifdef RSEQ_COMPARE_TWICE
169 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
170 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
171#endif
117 /* Start rseq by storing table entry pointer into rseq_cs. */ 172 /* Start rseq by storing table entry pointer into rseq_cs. */
118 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 173 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
119 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 174 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -173,7 +228,12 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
173 228
174 rseq_workaround_gcc_asm_size_guess(); 229 rseq_workaround_gcc_asm_size_guess();
175 __asm__ __volatile__ goto ( 230 __asm__ __volatile__ goto (
176 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 231 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
232 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
233#ifdef RSEQ_COMPARE_TWICE
234 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
235 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
236#endif
177 /* Start rseq by storing table entry pointer into rseq_cs. */ 237 /* Start rseq by storing table entry pointer into rseq_cs. */
178 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 238 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
179 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 239 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -237,7 +297,10 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
237 297
238 rseq_workaround_gcc_asm_size_guess(); 298 rseq_workaround_gcc_asm_size_guess();
239 __asm__ __volatile__ goto ( 299 __asm__ __volatile__ goto (
240 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 300 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
301#ifdef RSEQ_COMPARE_TWICE
302 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
303#endif
241 /* Start rseq by storing table entry pointer into rseq_cs. */ 304 /* Start rseq by storing table entry pointer into rseq_cs. */
242 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 305 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
243 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 306 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -289,7 +352,12 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
289 352
290 rseq_workaround_gcc_asm_size_guess(); 353 rseq_workaround_gcc_asm_size_guess();
291 __asm__ __volatile__ goto ( 354 __asm__ __volatile__ goto (
292 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 355 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
356 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
357#ifdef RSEQ_COMPARE_TWICE
358 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
359 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
360#endif
293 /* Start rseq by storing table entry pointer into rseq_cs. */ 361 /* Start rseq by storing table entry pointer into rseq_cs. */
294 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 362 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
295 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 363 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -357,7 +425,12 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
357 425
358 rseq_workaround_gcc_asm_size_guess(); 426 rseq_workaround_gcc_asm_size_guess();
359 __asm__ __volatile__ goto ( 427 __asm__ __volatile__ goto (
360 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 428 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
429 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
430#ifdef RSEQ_COMPARE_TWICE
431 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
432 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
433#endif
361 /* Start rseq by storing table entry pointer into rseq_cs. */ 434 /* Start rseq by storing table entry pointer into rseq_cs. */
362 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 435 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
363 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 436 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -426,7 +499,13 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
426 499
427 rseq_workaround_gcc_asm_size_guess(); 500 rseq_workaround_gcc_asm_size_guess();
428 __asm__ __volatile__ goto ( 501 __asm__ __volatile__ goto (
429 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 502 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
503 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
504#ifdef RSEQ_COMPARE_TWICE
505 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
506 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
507 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
508#endif
430 /* Start rseq by storing table entry pointer into rseq_cs. */ 509 /* Start rseq by storing table entry pointer into rseq_cs. */
431 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 510 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
432 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 511 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -500,7 +579,12 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
500 579
501 rseq_workaround_gcc_asm_size_guess(); 580 rseq_workaround_gcc_asm_size_guess();
502 __asm__ __volatile__ goto ( 581 __asm__ __volatile__ goto (
503 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 582 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
583 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
584#ifdef RSEQ_COMPARE_TWICE
585 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
586 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
587#endif
504 LONG_S " %[src], %[rseq_scratch0]\n\t" 588 LONG_S " %[src], %[rseq_scratch0]\n\t"
505 LONG_S " %[dst], %[rseq_scratch1]\n\t" 589 LONG_S " %[dst], %[rseq_scratch1]\n\t"
506 LONG_S " %[len], %[rseq_scratch2]\n\t" 590 LONG_S " %[len], %[rseq_scratch2]\n\t"
@@ -616,7 +700,12 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
616 700
617 rseq_workaround_gcc_asm_size_guess(); 701 rseq_workaround_gcc_asm_size_guess();
618 __asm__ __volatile__ goto ( 702 __asm__ __volatile__ goto (
619 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */ 703 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
704 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
705#ifdef RSEQ_COMPARE_TWICE
706 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
707 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
708#endif
620 LONG_S " %[src], %[rseq_scratch0]\n\t" 709 LONG_S " %[src], %[rseq_scratch0]\n\t"
621 LONG_S " %[dst], %[rseq_scratch1]\n\t" 710 LONG_S " %[dst], %[rseq_scratch1]\n\t"
622 LONG_S " %[len], %[rseq_scratch2]\n\t" 711 LONG_S " %[len], %[rseq_scratch2]\n\t"
diff --git a/tools/testing/selftests/rseq/rseq-ppc.h b/tools/testing/selftests/rseq/rseq-ppc.h
index 52630c9f42be..76be90196fe4 100644
--- a/tools/testing/selftests/rseq/rseq-ppc.h
+++ b/tools/testing/selftests/rseq/rseq-ppc.h
@@ -6,7 +6,15 @@
6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> 6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7 */ 7 */
8 8
9#define RSEQ_SIG 0x53053053 9/*
10 * RSEQ_SIG is used with the following trap instruction:
11 *
12 * powerpc-be: 0f e5 00 0b twui r5,11
13 * powerpc64-le: 0b 00 e5 0f twui r5,11
14 * powerpc64-be: 0f e5 00 0b twui r5,11
15 */
16
17#define RSEQ_SIG 0x0fe5000b
10 18
11#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc") 19#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
12#define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc") 20#define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
@@ -33,8 +41,8 @@ do { \
33#else /* !RSEQ_SKIP_FASTPATH */ 41#else /* !RSEQ_SKIP_FASTPATH */
34 42
35/* 43/*
36 * The __rseq_table section can be used by debuggers to better handle 44 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
37 * single-stepping through the restartable critical sections. 45 * better handle single-stepping through the restartable critical sections.
38 */ 46 */
39 47
40#ifdef __PPC64__ 48#ifdef __PPC64__
@@ -46,11 +54,14 @@ do { \
46 54
47#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 55#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
48 start_ip, post_commit_offset, abort_ip) \ 56 start_ip, post_commit_offset, abort_ip) \
49 ".pushsection __rseq_table, \"aw\"\n\t" \ 57 ".pushsection __rseq_cs, \"aw\"\n\t" \
50 ".balign 32\n\t" \ 58 ".balign 32\n\t" \
51 __rseq_str(label) ":\n\t" \ 59 __rseq_str(label) ":\n\t" \
52 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 60 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
53 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 61 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
62 ".popsection\n\t" \
63 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
64 ".quad " __rseq_str(label) "b\n\t" \
54 ".popsection\n\t" 65 ".popsection\n\t"
55 66
56#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 67#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
@@ -63,6 +74,19 @@ do { \
63 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ 74 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
64 __rseq_str(label) ":\n\t" 75 __rseq_str(label) ":\n\t"
65 76
77/*
78 * Exit points of a rseq critical section consist of all instructions outside
79 * of the critical section where a critical section can either branch to or
80 * reach through the normal course of its execution. The abort IP and the
81 * post-commit IP are already part of the __rseq_cs section and should not be
82 * explicitly defined as additional exit points. Knowing all exit points is
83 * useful to assist debuggers stepping over the critical section.
84 */
85#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
86 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
87 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
88 ".popsection\n\t"
89
66#else /* #ifdef __PPC64__ */ 90#else /* #ifdef __PPC64__ */
67 91
68#define STORE_WORD "stw " 92#define STORE_WORD "stw "
@@ -72,12 +96,29 @@ do { \
72 96
73#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 97#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
74 start_ip, post_commit_offset, abort_ip) \ 98 start_ip, post_commit_offset, abort_ip) \
75 ".pushsection __rseq_table, \"aw\"\n\t" \ 99 ".pushsection __rseq_cs, \"aw\"\n\t" \
76 ".balign 32\n\t" \ 100 ".balign 32\n\t" \
77 __rseq_str(label) ":\n\t" \ 101 __rseq_str(label) ":\n\t" \
78 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 102 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
79 /* 32-bit only supported on BE */ \ 103 /* 32-bit only supported on BE */ \
80 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ 104 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
105 ".popsection\n\t" \
106 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
107 ".long 0x0, " __rseq_str(label) "b\n\t" \
108 ".popsection\n\t"
109
110/*
111 * Exit points of a rseq critical section consist of all instructions outside
112 * of the critical section where a critical section can either branch to or
113 * reach through the normal course of its execution. The abort IP and the
114 * post-commit IP are already part of the __rseq_cs section and should not be
115 * explicitly defined as additional exit points. Knowing all exit points is
116 * useful to assist debuggers stepping over the critical section.
117 */
118#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
119 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
120 /* 32-bit only supported on BE */ \
121 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
81 ".popsection\n\t" 122 ".popsection\n\t"
82 123
83#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 124#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
@@ -169,6 +210,11 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
169 210
170 __asm__ __volatile__ goto ( 211 __asm__ __volatile__ goto (
171 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 212 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
213 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
214#ifdef RSEQ_COMPARE_TWICE
215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
216 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
217#endif
172 /* Start rseq by storing table entry pointer into rseq_cs. */ 218 /* Start rseq by storing table entry pointer into rseq_cs. */
173 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 219 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
174 /* cmp cpuid */ 220 /* cmp cpuid */
@@ -224,6 +270,11 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
224 270
225 __asm__ __volatile__ goto ( 271 __asm__ __volatile__ goto (
226 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 272 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
273 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
274#ifdef RSEQ_COMPARE_TWICE
275 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
276 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
277#endif
227 /* Start rseq by storing table entry pointer into rseq_cs. */ 278 /* Start rseq by storing table entry pointer into rseq_cs. */
228 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 279 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
229 /* cmp cpuid */ 280 /* cmp cpuid */
@@ -286,6 +337,9 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
286 337
287 __asm__ __volatile__ goto ( 338 __asm__ __volatile__ goto (
288 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 339 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
340#ifdef RSEQ_COMPARE_TWICE
341 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
342#endif
289 /* Start rseq by storing table entry pointer into rseq_cs. */ 343 /* Start rseq by storing table entry pointer into rseq_cs. */
290 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 344 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
291 /* cmp cpuid */ 345 /* cmp cpuid */
@@ -337,6 +391,11 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
337 391
338 __asm__ __volatile__ goto ( 392 __asm__ __volatile__ goto (
339 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 393 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
394 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
395#ifdef RSEQ_COMPARE_TWICE
396 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
397 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
398#endif
340 /* Start rseq by storing table entry pointer into rseq_cs. */ 399 /* Start rseq by storing table entry pointer into rseq_cs. */
341 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 400 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
342 /* cmp cpuid */ 401 /* cmp cpuid */
@@ -400,6 +459,11 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
400 459
401 __asm__ __volatile__ goto ( 460 __asm__ __volatile__ goto (
402 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 461 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
462 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
463#ifdef RSEQ_COMPARE_TWICE
464 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
465 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
466#endif
403 /* Start rseq by storing table entry pointer into rseq_cs. */ 467 /* Start rseq by storing table entry pointer into rseq_cs. */
404 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 468 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
405 /* cmp cpuid */ 469 /* cmp cpuid */
@@ -465,6 +529,12 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
465 529
466 __asm__ __volatile__ goto ( 530 __asm__ __volatile__ goto (
467 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 531 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
532 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
533#ifdef RSEQ_COMPARE_TWICE
534 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
535 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
536 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
537#endif
468 /* Start rseq by storing table entry pointer into rseq_cs. */ 538 /* Start rseq by storing table entry pointer into rseq_cs. */
469 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 539 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
470 /* cmp cpuid */ 540 /* cmp cpuid */
@@ -532,6 +602,11 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
532 602
533 __asm__ __volatile__ goto ( 603 __asm__ __volatile__ goto (
534 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 604 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
605 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
606#ifdef RSEQ_COMPARE_TWICE
607 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
608 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
609#endif
535 /* setup for mempcy */ 610 /* setup for mempcy */
536 "mr %%r19, %[len]\n\t" 611 "mr %%r19, %[len]\n\t"
537 "mr %%r20, %[src]\n\t" 612 "mr %%r20, %[src]\n\t"
@@ -601,6 +676,11 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
601 676
602 __asm__ __volatile__ goto ( 677 __asm__ __volatile__ goto (
603 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 678 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
679 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
680#ifdef RSEQ_COMPARE_TWICE
681 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
682 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
683#endif
604 /* setup for mempcy */ 684 /* setup for mempcy */
605 "mr %%r19, %[len]\n\t" 685 "mr %%r19, %[len]\n\t"
606 "mr %%r20, %[src]\n\t" 686 "mr %%r20, %[src]\n\t"
diff --git a/tools/testing/selftests/rseq/rseq-s390.h b/tools/testing/selftests/rseq/rseq-s390.h
index 0afdf7957974..8ef94ad1cbb4 100644
--- a/tools/testing/selftests/rseq/rseq-s390.h
+++ b/tools/testing/selftests/rseq/rseq-s390.h
@@ -44,22 +44,54 @@ do { \
44 44
45#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 45#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
46 start_ip, post_commit_offset, abort_ip) \ 46 start_ip, post_commit_offset, abort_ip) \
47 ".pushsection __rseq_table, \"aw\"\n\t" \ 47 ".pushsection __rseq_cs, \"aw\"\n\t" \
48 ".balign 32\n\t" \ 48 ".balign 32\n\t" \
49 __rseq_str(label) ":\n\t" \ 49 __rseq_str(label) ":\n\t" \
50 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 50 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
51 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 51 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
52 ".popsection\n\t" \
53 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
54 ".quad " __rseq_str(label) "b\n\t" \
55 ".popsection\n\t"
56
57/*
58 * Exit points of a rseq critical section consist of all instructions outside
59 * of the critical section where a critical section can either branch to or
60 * reach through the normal course of its execution. The abort IP and the
61 * post-commit IP are already part of the __rseq_cs section and should not be
62 * explicitly defined as additional exit points. Knowing all exit points is
63 * useful to assist debuggers stepping over the critical section.
64 */
65#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
66 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
67 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
52 ".popsection\n\t" 68 ".popsection\n\t"
53 69
54#elif __s390__ 70#elif __s390__
55 71
56#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 72#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
57 start_ip, post_commit_offset, abort_ip) \ 73 start_ip, post_commit_offset, abort_ip) \
58 ".pushsection __rseq_table, \"aw\"\n\t" \ 74 ".pushsection __rseq_cs, \"aw\"\n\t" \
59 ".balign 32\n\t" \ 75 ".balign 32\n\t" \
60 __rseq_str(label) ":\n\t" \ 76 __rseq_str(label) ":\n\t" \
61 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 77 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
62 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ 78 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
79 ".popsection\n\t" \
80 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
81 ".long 0x0, " __rseq_str(label) "b\n\t" \
82 ".popsection\n\t"
83
84/*
85 * Exit points of a rseq critical section consist of all instructions outside
86 * of the critical section where a critical section can either branch to or
87 * reach through the normal course of its execution. The abort IP and the
88 * post-commit IP are already part of the __rseq_cs section and should not be
89 * explicitly defined as additional exit points. Knowing all exit points is
90 * useful to assist debuggers stepping over the critical section.
91 */
92#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
93 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
94 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
63 ".popsection\n\t" 95 ".popsection\n\t"
64 96
65#define LONG_L "l" 97#define LONG_L "l"
@@ -92,14 +124,14 @@ do { \
92 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 124 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
93 __rseq_str(label) ":\n\t" \ 125 __rseq_str(label) ":\n\t" \
94 teardown \ 126 teardown \
95 "j %l[" __rseq_str(abort_label) "]\n\t" \ 127 "jg %l[" __rseq_str(abort_label) "]\n\t" \
96 ".popsection\n\t" 128 ".popsection\n\t"
97 129
98#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \ 130#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
99 ".pushsection __rseq_failure, \"ax\"\n\t" \ 131 ".pushsection __rseq_failure, \"ax\"\n\t" \
100 __rseq_str(label) ":\n\t" \ 132 __rseq_str(label) ":\n\t" \
101 teardown \ 133 teardown \
102 "j %l[" __rseq_str(cmpfail_label) "]\n\t" \ 134 "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \
103 ".popsection\n\t" 135 ".popsection\n\t"
104 136
105static inline __attribute__((always_inline)) 137static inline __attribute__((always_inline))
@@ -109,6 +141,11 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
109 141
110 __asm__ __volatile__ goto ( 142 __asm__ __volatile__ goto (
111 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 143 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
144 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
145#ifdef RSEQ_COMPARE_TWICE
146 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
147 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
148#endif
112 /* Start rseq by storing table entry pointer into rseq_cs. */ 149 /* Start rseq by storing table entry pointer into rseq_cs. */
113 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 150 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
114 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 151 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -167,6 +204,11 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
167 204
168 __asm__ __volatile__ goto ( 205 __asm__ __volatile__ goto (
169 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 206 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
207 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
208#ifdef RSEQ_COMPARE_TWICE
209 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
210 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
211#endif
170 /* Start rseq by storing table entry pointer into rseq_cs. */ 212 /* Start rseq by storing table entry pointer into rseq_cs. */
171 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 213 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
172 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 214 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -227,6 +269,9 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
227 269
228 __asm__ __volatile__ goto ( 270 __asm__ __volatile__ goto (
229 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 271 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
272#ifdef RSEQ_COMPARE_TWICE
273 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
274#endif
230 /* Start rseq by storing table entry pointer into rseq_cs. */ 275 /* Start rseq by storing table entry pointer into rseq_cs. */
231 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 276 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
232 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 277 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -275,6 +320,11 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
275 320
276 __asm__ __volatile__ goto ( 321 __asm__ __volatile__ goto (
277 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 322 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
323 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
324#ifdef RSEQ_COMPARE_TWICE
325 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
326 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
327#endif
278 /* Start rseq by storing table entry pointer into rseq_cs. */ 328 /* Start rseq by storing table entry pointer into rseq_cs. */
279 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 329 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
280 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 330 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -346,6 +396,12 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
346 396
347 __asm__ __volatile__ goto ( 397 __asm__ __volatile__ goto (
348 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 398 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
399 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
400#ifdef RSEQ_COMPARE_TWICE
401 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
402 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
403 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
404#endif
349 /* Start rseq by storing table entry pointer into rseq_cs. */ 405 /* Start rseq by storing table entry pointer into rseq_cs. */
350 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 406 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
351 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 407 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
@@ -414,6 +470,11 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
414 470
415 __asm__ __volatile__ goto ( 471 __asm__ __volatile__ goto (
416 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 472 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
473 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
474#ifdef RSEQ_COMPARE_TWICE
475 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
476 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
477#endif
417 LONG_S " %[src], %[rseq_scratch0]\n\t" 478 LONG_S " %[src], %[rseq_scratch0]\n\t"
418 LONG_S " %[dst], %[rseq_scratch1]\n\t" 479 LONG_S " %[dst], %[rseq_scratch1]\n\t"
419 LONG_S " %[len], %[rseq_scratch2]\n\t" 480 LONG_S " %[len], %[rseq_scratch2]\n\t"
diff --git a/tools/testing/selftests/rseq/rseq-x86.h b/tools/testing/selftests/rseq/rseq-x86.h
index 089410a314e9..b2da6004fe30 100644
--- a/tools/testing/selftests/rseq/rseq-x86.h
+++ b/tools/testing/selftests/rseq/rseq-x86.h
@@ -7,8 +7,25 @@
7 7
8#include <stdint.h> 8#include <stdint.h>
9 9
10/*
11 * RSEQ_SIG is used with the following reserved undefined instructions, which
12 * trap in user-space:
13 *
14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
16 */
10#define RSEQ_SIG 0x53053053 17#define RSEQ_SIG 0x53053053
11 18
19/*
20 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
21 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
22 * address through a "r" input operand.
23 */
24
25/* Offset of cpu_id and rseq_cs fields in struct rseq. */
26#define RSEQ_CPU_ID_OFFSET 4
27#define RSEQ_CS_OFFSET 8
28
12#ifdef __x86_64__ 29#ifdef __x86_64__
13 30
14#define rseq_smp_mb() \ 31#define rseq_smp_mb() \
@@ -37,32 +54,49 @@ do { \
37 54
38#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 55#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
39 start_ip, post_commit_offset, abort_ip) \ 56 start_ip, post_commit_offset, abort_ip) \
40 ".pushsection __rseq_table, \"aw\"\n\t" \ 57 ".pushsection __rseq_cs, \"aw\"\n\t" \
41 ".balign 32\n\t" \ 58 ".balign 32\n\t" \
42 __rseq_str(label) ":\n\t" \ 59 __rseq_str(label) ":\n\t" \
43 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 60 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
44 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 61 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
62 ".popsection\n\t" \
63 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
64 ".quad " __rseq_str(label) "b\n\t" \
45 ".popsection\n\t" 65 ".popsection\n\t"
46 66
67
47#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 68#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
48 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 69 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
49 (post_commit_ip - start_ip), abort_ip) 70 (post_commit_ip - start_ip), abort_ip)
50 71
72/*
73 * Exit points of a rseq critical section consist of all instructions outside
74 * of the critical section where a critical section can either branch to or
75 * reach through the normal course of its execution. The abort IP and the
76 * post-commit IP are already part of the __rseq_cs section and should not be
77 * explicitly defined as additional exit points. Knowing all exit points is
78 * useful to assist debuggers stepping over the critical section.
79 */
80#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
81 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
82 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
83 ".popsection\n\t"
84
51#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 85#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
52 RSEQ_INJECT_ASM(1) \ 86 RSEQ_INJECT_ASM(1) \
53 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ 87 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
54 "movq %%rax, %[" __rseq_str(rseq_cs) "]\n\t" \ 88 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
55 __rseq_str(label) ":\n\t" 89 __rseq_str(label) ":\n\t"
56 90
57#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 91#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
58 RSEQ_INJECT_ASM(2) \ 92 RSEQ_INJECT_ASM(2) \
59 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ 93 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
60 "jnz " __rseq_str(label) "\n\t" 94 "jnz " __rseq_str(label) "\n\t"
61 95
62#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 96#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
63 ".pushsection __rseq_failure, \"ax\"\n\t" \ 97 ".pushsection __rseq_failure, \"ax\"\n\t" \
64 /* Disassembler-friendly signature: nopl <sig>(%rip). */\ 98 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
65 ".byte 0x0f, 0x1f, 0x05\n\t" \ 99 ".byte 0x0f, 0xb9, 0x3d\n\t" \
66 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 100 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
67 __rseq_str(label) ":\n\t" \ 101 __rseq_str(label) ":\n\t" \
68 teardown \ 102 teardown \
@@ -83,15 +117,20 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
83 117
84 __asm__ __volatile__ goto ( 118 __asm__ __volatile__ goto (
85 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 119 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
120 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
121#ifdef RSEQ_COMPARE_TWICE
122 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
123 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
124#endif
86 /* Start rseq by storing table entry pointer into rseq_cs. */ 125 /* Start rseq by storing table entry pointer into rseq_cs. */
87 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 126 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
88 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 127 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
89 RSEQ_INJECT_ASM(3) 128 RSEQ_INJECT_ASM(3)
90 "cmpq %[v], %[expect]\n\t" 129 "cmpq %[v], %[expect]\n\t"
91 "jnz %l[cmpfail]\n\t" 130 "jnz %l[cmpfail]\n\t"
92 RSEQ_INJECT_ASM(4) 131 RSEQ_INJECT_ASM(4)
93#ifdef RSEQ_COMPARE_TWICE 132#ifdef RSEQ_COMPARE_TWICE
94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 133 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
95 "cmpq %[v], %[expect]\n\t" 134 "cmpq %[v], %[expect]\n\t"
96 "jnz %l[error2]\n\t" 135 "jnz %l[error2]\n\t"
97#endif 136#endif
@@ -102,8 +141,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
102 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 141 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
103 : /* gcc asm goto does not allow outputs */ 142 : /* gcc asm goto does not allow outputs */
104 : [cpu_id] "r" (cpu), 143 : [cpu_id] "r" (cpu),
105 [current_cpu_id] "m" (__rseq_abi.cpu_id), 144 [rseq_abi] "r" (&__rseq_abi),
106 [rseq_cs] "m" (__rseq_abi.rseq_cs),
107 [v] "m" (*v), 145 [v] "m" (*v),
108 [expect] "r" (expect), 146 [expect] "r" (expect),
109 [newv] "r" (newv) 147 [newv] "r" (newv)
@@ -140,16 +178,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
140 178
141 __asm__ __volatile__ goto ( 179 __asm__ __volatile__ goto (
142 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 180 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
181 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
182#ifdef RSEQ_COMPARE_TWICE
183 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
184 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
185#endif
143 /* Start rseq by storing table entry pointer into rseq_cs. */ 186 /* Start rseq by storing table entry pointer into rseq_cs. */
144 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 187 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
145 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 188 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
146 RSEQ_INJECT_ASM(3) 189 RSEQ_INJECT_ASM(3)
147 "movq %[v], %%rbx\n\t" 190 "movq %[v], %%rbx\n\t"
148 "cmpq %%rbx, %[expectnot]\n\t" 191 "cmpq %%rbx, %[expectnot]\n\t"
149 "je %l[cmpfail]\n\t" 192 "je %l[cmpfail]\n\t"
150 RSEQ_INJECT_ASM(4) 193 RSEQ_INJECT_ASM(4)
151#ifdef RSEQ_COMPARE_TWICE 194#ifdef RSEQ_COMPARE_TWICE
152 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 195 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
153 "movq %[v], %%rbx\n\t" 196 "movq %[v], %%rbx\n\t"
154 "cmpq %%rbx, %[expectnot]\n\t" 197 "cmpq %%rbx, %[expectnot]\n\t"
155 "je %l[error2]\n\t" 198 "je %l[error2]\n\t"
@@ -164,8 +207,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
164 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 207 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
165 : /* gcc asm goto does not allow outputs */ 208 : /* gcc asm goto does not allow outputs */
166 : [cpu_id] "r" (cpu), 209 : [cpu_id] "r" (cpu),
167 [current_cpu_id] "m" (__rseq_abi.cpu_id), 210 [rseq_abi] "r" (&__rseq_abi),
168 [rseq_cs] "m" (__rseq_abi.rseq_cs),
169 /* final store input */ 211 /* final store input */
170 [v] "m" (*v), 212 [v] "m" (*v),
171 [expectnot] "r" (expectnot), 213 [expectnot] "r" (expectnot),
@@ -199,12 +241,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
199 241
200 __asm__ __volatile__ goto ( 242 __asm__ __volatile__ goto (
201 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 243 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
244#ifdef RSEQ_COMPARE_TWICE
245 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
246#endif
202 /* Start rseq by storing table entry pointer into rseq_cs. */ 247 /* Start rseq by storing table entry pointer into rseq_cs. */
203 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 248 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
204 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 249 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
205 RSEQ_INJECT_ASM(3) 250 RSEQ_INJECT_ASM(3)
206#ifdef RSEQ_COMPARE_TWICE 251#ifdef RSEQ_COMPARE_TWICE
207 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 252 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
208#endif 253#endif
209 /* final store */ 254 /* final store */
210 "addq %[count], %[v]\n\t" 255 "addq %[count], %[v]\n\t"
@@ -213,8 +258,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
213 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 258 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
214 : /* gcc asm goto does not allow outputs */ 259 : /* gcc asm goto does not allow outputs */
215 : [cpu_id] "r" (cpu), 260 : [cpu_id] "r" (cpu),
216 [current_cpu_id] "m" (__rseq_abi.cpu_id), 261 [rseq_abi] "r" (&__rseq_abi),
217 [rseq_cs] "m" (__rseq_abi.rseq_cs),
218 /* final store input */ 262 /* final store input */
219 [v] "m" (*v), 263 [v] "m" (*v),
220 [count] "er" (count) 264 [count] "er" (count)
@@ -244,15 +288,20 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
244 288
245 __asm__ __volatile__ goto ( 289 __asm__ __volatile__ goto (
246 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 290 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
291 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
292#ifdef RSEQ_COMPARE_TWICE
293 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
294 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
295#endif
247 /* Start rseq by storing table entry pointer into rseq_cs. */ 296 /* Start rseq by storing table entry pointer into rseq_cs. */
248 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 297 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
249 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 298 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
250 RSEQ_INJECT_ASM(3) 299 RSEQ_INJECT_ASM(3)
251 "cmpq %[v], %[expect]\n\t" 300 "cmpq %[v], %[expect]\n\t"
252 "jnz %l[cmpfail]\n\t" 301 "jnz %l[cmpfail]\n\t"
253 RSEQ_INJECT_ASM(4) 302 RSEQ_INJECT_ASM(4)
254#ifdef RSEQ_COMPARE_TWICE 303#ifdef RSEQ_COMPARE_TWICE
255 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 304 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
256 "cmpq %[v], %[expect]\n\t" 305 "cmpq %[v], %[expect]\n\t"
257 "jnz %l[error2]\n\t" 306 "jnz %l[error2]\n\t"
258#endif 307#endif
@@ -266,8 +315,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
266 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 315 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
267 : /* gcc asm goto does not allow outputs */ 316 : /* gcc asm goto does not allow outputs */
268 : [cpu_id] "r" (cpu), 317 : [cpu_id] "r" (cpu),
269 [current_cpu_id] "m" (__rseq_abi.cpu_id), 318 [rseq_abi] "r" (&__rseq_abi),
270 [rseq_cs] "m" (__rseq_abi.rseq_cs),
271 /* try store input */ 319 /* try store input */
272 [v2] "m" (*v2), 320 [v2] "m" (*v2),
273 [newv2] "r" (newv2), 321 [newv2] "r" (newv2),
@@ -314,9 +362,15 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
314 362
315 __asm__ __volatile__ goto ( 363 __asm__ __volatile__ goto (
316 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 364 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
365 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
366#ifdef RSEQ_COMPARE_TWICE
367 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
368 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
369 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
370#endif
317 /* Start rseq by storing table entry pointer into rseq_cs. */ 371 /* Start rseq by storing table entry pointer into rseq_cs. */
318 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 372 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
319 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 373 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
320 RSEQ_INJECT_ASM(3) 374 RSEQ_INJECT_ASM(3)
321 "cmpq %[v], %[expect]\n\t" 375 "cmpq %[v], %[expect]\n\t"
322 "jnz %l[cmpfail]\n\t" 376 "jnz %l[cmpfail]\n\t"
@@ -325,7 +379,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
325 "jnz %l[cmpfail]\n\t" 379 "jnz %l[cmpfail]\n\t"
326 RSEQ_INJECT_ASM(5) 380 RSEQ_INJECT_ASM(5)
327#ifdef RSEQ_COMPARE_TWICE 381#ifdef RSEQ_COMPARE_TWICE
328 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 382 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
329 "cmpq %[v], %[expect]\n\t" 383 "cmpq %[v], %[expect]\n\t"
330 "jnz %l[error2]\n\t" 384 "jnz %l[error2]\n\t"
331 "cmpq %[v2], %[expect2]\n\t" 385 "cmpq %[v2], %[expect2]\n\t"
@@ -338,8 +392,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
338 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 392 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
339 : /* gcc asm goto does not allow outputs */ 393 : /* gcc asm goto does not allow outputs */
340 : [cpu_id] "r" (cpu), 394 : [cpu_id] "r" (cpu),
341 [current_cpu_id] "m" (__rseq_abi.cpu_id), 395 [rseq_abi] "r" (&__rseq_abi),
342 [rseq_cs] "m" (__rseq_abi.rseq_cs),
343 /* cmp2 input */ 396 /* cmp2 input */
344 [v2] "m" (*v2), 397 [v2] "m" (*v2),
345 [expect2] "r" (expect2), 398 [expect2] "r" (expect2),
@@ -381,18 +434,23 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
381 434
382 __asm__ __volatile__ goto ( 435 __asm__ __volatile__ goto (
383 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 436 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
437 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
438#ifdef RSEQ_COMPARE_TWICE
439 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
440 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
441#endif
384 "movq %[src], %[rseq_scratch0]\n\t" 442 "movq %[src], %[rseq_scratch0]\n\t"
385 "movq %[dst], %[rseq_scratch1]\n\t" 443 "movq %[dst], %[rseq_scratch1]\n\t"
386 "movq %[len], %[rseq_scratch2]\n\t" 444 "movq %[len], %[rseq_scratch2]\n\t"
387 /* Start rseq by storing table entry pointer into rseq_cs. */ 445 /* Start rseq by storing table entry pointer into rseq_cs. */
388 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 446 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
389 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 447 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
390 RSEQ_INJECT_ASM(3) 448 RSEQ_INJECT_ASM(3)
391 "cmpq %[v], %[expect]\n\t" 449 "cmpq %[v], %[expect]\n\t"
392 "jnz 5f\n\t" 450 "jnz 5f\n\t"
393 RSEQ_INJECT_ASM(4) 451 RSEQ_INJECT_ASM(4)
394#ifdef RSEQ_COMPARE_TWICE 452#ifdef RSEQ_COMPARE_TWICE
395 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 453 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
396 "cmpq %[v], %[expect]\n\t" 454 "cmpq %[v], %[expect]\n\t"
397 "jnz 7f\n\t" 455 "jnz 7f\n\t"
398#endif 456#endif
@@ -440,8 +498,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
440#endif 498#endif
441 : /* gcc asm goto does not allow outputs */ 499 : /* gcc asm goto does not allow outputs */
442 : [cpu_id] "r" (cpu), 500 : [cpu_id] "r" (cpu),
443 [current_cpu_id] "m" (__rseq_abi.cpu_id), 501 [rseq_abi] "r" (&__rseq_abi),
444 [rseq_cs] "m" (__rseq_abi.rseq_cs),
445 /* final store input */ 502 /* final store input */
446 [v] "m" (*v), 503 [v] "m" (*v),
447 [expect] "r" (expect), 504 [expect] "r" (expect),
@@ -520,31 +577,47 @@ do { \
520 */ 577 */
521#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 578#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
522 start_ip, post_commit_offset, abort_ip) \ 579 start_ip, post_commit_offset, abort_ip) \
523 ".pushsection __rseq_table, \"aw\"\n\t" \ 580 ".pushsection __rseq_cs, \"aw\"\n\t" \
524 ".balign 32\n\t" \ 581 ".balign 32\n\t" \
525 __rseq_str(label) ":\n\t" \ 582 __rseq_str(label) ":\n\t" \
526 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 583 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
527 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \ 584 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
585 ".popsection\n\t" \
586 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
587 ".long " __rseq_str(label) "b, 0x0\n\t" \
528 ".popsection\n\t" 588 ".popsection\n\t"
529 589
530#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 590#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
531 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 591 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
532 (post_commit_ip - start_ip), abort_ip) 592 (post_commit_ip - start_ip), abort_ip)
533 593
594/*
595 * Exit points of a rseq critical section consist of all instructions outside
596 * of the critical section where a critical section can either branch to or
597 * reach through the normal course of its execution. The abort IP and the
598 * post-commit IP are already part of the __rseq_cs section and should not be
599 * explicitly defined as additional exit points. Knowing all exit points is
600 * useful to assist debuggers stepping over the critical section.
601 */
602#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
603 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
604 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
605 ".popsection\n\t"
606
534#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 607#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
535 RSEQ_INJECT_ASM(1) \ 608 RSEQ_INJECT_ASM(1) \
536 "movl $" __rseq_str(cs_label) ", %[rseq_cs]\n\t" \ 609 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
537 __rseq_str(label) ":\n\t" 610 __rseq_str(label) ":\n\t"
538 611
539#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 612#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
540 RSEQ_INJECT_ASM(2) \ 613 RSEQ_INJECT_ASM(2) \
541 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ 614 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
542 "jnz " __rseq_str(label) "\n\t" 615 "jnz " __rseq_str(label) "\n\t"
543 616
544#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ 617#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
545 ".pushsection __rseq_failure, \"ax\"\n\t" \ 618 ".pushsection __rseq_failure, \"ax\"\n\t" \
546 /* Disassembler-friendly signature: nopl <sig>. */ \ 619 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
547 ".byte 0x0f, 0x1f, 0x05\n\t" \ 620 ".byte 0x0f, 0xb9, 0x3d\n\t" \
548 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 621 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
549 __rseq_str(label) ":\n\t" \ 622 __rseq_str(label) ":\n\t" \
550 teardown \ 623 teardown \
@@ -565,15 +638,20 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
565 638
566 __asm__ __volatile__ goto ( 639 __asm__ __volatile__ goto (
567 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 640 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
641 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
642#ifdef RSEQ_COMPARE_TWICE
643 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
644 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
645#endif
568 /* Start rseq by storing table entry pointer into rseq_cs. */ 646 /* Start rseq by storing table entry pointer into rseq_cs. */
569 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 647 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
570 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 648 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
571 RSEQ_INJECT_ASM(3) 649 RSEQ_INJECT_ASM(3)
572 "cmpl %[v], %[expect]\n\t" 650 "cmpl %[v], %[expect]\n\t"
573 "jnz %l[cmpfail]\n\t" 651 "jnz %l[cmpfail]\n\t"
574 RSEQ_INJECT_ASM(4) 652 RSEQ_INJECT_ASM(4)
575#ifdef RSEQ_COMPARE_TWICE 653#ifdef RSEQ_COMPARE_TWICE
576 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 654 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
577 "cmpl %[v], %[expect]\n\t" 655 "cmpl %[v], %[expect]\n\t"
578 "jnz %l[error2]\n\t" 656 "jnz %l[error2]\n\t"
579#endif 657#endif
@@ -584,8 +662,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
584 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 662 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
585 : /* gcc asm goto does not allow outputs */ 663 : /* gcc asm goto does not allow outputs */
586 : [cpu_id] "r" (cpu), 664 : [cpu_id] "r" (cpu),
587 [current_cpu_id] "m" (__rseq_abi.cpu_id), 665 [rseq_abi] "r" (&__rseq_abi),
588 [rseq_cs] "m" (__rseq_abi.rseq_cs),
589 [v] "m" (*v), 666 [v] "m" (*v),
590 [expect] "r" (expect), 667 [expect] "r" (expect),
591 [newv] "r" (newv) 668 [newv] "r" (newv)
@@ -622,16 +699,21 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
622 699
623 __asm__ __volatile__ goto ( 700 __asm__ __volatile__ goto (
624 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 701 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
702 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
703#ifdef RSEQ_COMPARE_TWICE
704 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
705 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
706#endif
625 /* Start rseq by storing table entry pointer into rseq_cs. */ 707 /* Start rseq by storing table entry pointer into rseq_cs. */
626 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 708 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
627 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 709 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
628 RSEQ_INJECT_ASM(3) 710 RSEQ_INJECT_ASM(3)
629 "movl %[v], %%ebx\n\t" 711 "movl %[v], %%ebx\n\t"
630 "cmpl %%ebx, %[expectnot]\n\t" 712 "cmpl %%ebx, %[expectnot]\n\t"
631 "je %l[cmpfail]\n\t" 713 "je %l[cmpfail]\n\t"
632 RSEQ_INJECT_ASM(4) 714 RSEQ_INJECT_ASM(4)
633#ifdef RSEQ_COMPARE_TWICE 715#ifdef RSEQ_COMPARE_TWICE
634 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 716 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
635 "movl %[v], %%ebx\n\t" 717 "movl %[v], %%ebx\n\t"
636 "cmpl %%ebx, %[expectnot]\n\t" 718 "cmpl %%ebx, %[expectnot]\n\t"
637 "je %l[error2]\n\t" 719 "je %l[error2]\n\t"
@@ -646,8 +728,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
646 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 728 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
647 : /* gcc asm goto does not allow outputs */ 729 : /* gcc asm goto does not allow outputs */
648 : [cpu_id] "r" (cpu), 730 : [cpu_id] "r" (cpu),
649 [current_cpu_id] "m" (__rseq_abi.cpu_id), 731 [rseq_abi] "r" (&__rseq_abi),
650 [rseq_cs] "m" (__rseq_abi.rseq_cs),
651 /* final store input */ 732 /* final store input */
652 [v] "m" (*v), 733 [v] "m" (*v),
653 [expectnot] "r" (expectnot), 734 [expectnot] "r" (expectnot),
@@ -681,12 +762,15 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
681 762
682 __asm__ __volatile__ goto ( 763 __asm__ __volatile__ goto (
683 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 764 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
765#ifdef RSEQ_COMPARE_TWICE
766 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
767#endif
684 /* Start rseq by storing table entry pointer into rseq_cs. */ 768 /* Start rseq by storing table entry pointer into rseq_cs. */
685 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 769 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
686 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 770 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
687 RSEQ_INJECT_ASM(3) 771 RSEQ_INJECT_ASM(3)
688#ifdef RSEQ_COMPARE_TWICE 772#ifdef RSEQ_COMPARE_TWICE
689 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 773 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
690#endif 774#endif
691 /* final store */ 775 /* final store */
692 "addl %[count], %[v]\n\t" 776 "addl %[count], %[v]\n\t"
@@ -695,8 +779,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
695 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 779 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
696 : /* gcc asm goto does not allow outputs */ 780 : /* gcc asm goto does not allow outputs */
697 : [cpu_id] "r" (cpu), 781 : [cpu_id] "r" (cpu),
698 [current_cpu_id] "m" (__rseq_abi.cpu_id), 782 [rseq_abi] "r" (&__rseq_abi),
699 [rseq_cs] "m" (__rseq_abi.rseq_cs),
700 /* final store input */ 783 /* final store input */
701 [v] "m" (*v), 784 [v] "m" (*v),
702 [count] "ir" (count) 785 [count] "ir" (count)
@@ -726,15 +809,20 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
726 809
727 __asm__ __volatile__ goto ( 810 __asm__ __volatile__ goto (
728 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 811 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
812 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
813#ifdef RSEQ_COMPARE_TWICE
814 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
815 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
816#endif
729 /* Start rseq by storing table entry pointer into rseq_cs. */ 817 /* Start rseq by storing table entry pointer into rseq_cs. */
730 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 818 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
731 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 819 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
732 RSEQ_INJECT_ASM(3) 820 RSEQ_INJECT_ASM(3)
733 "cmpl %[v], %[expect]\n\t" 821 "cmpl %[v], %[expect]\n\t"
734 "jnz %l[cmpfail]\n\t" 822 "jnz %l[cmpfail]\n\t"
735 RSEQ_INJECT_ASM(4) 823 RSEQ_INJECT_ASM(4)
736#ifdef RSEQ_COMPARE_TWICE 824#ifdef RSEQ_COMPARE_TWICE
737 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 825 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
738 "cmpl %[v], %[expect]\n\t" 826 "cmpl %[v], %[expect]\n\t"
739 "jnz %l[error2]\n\t" 827 "jnz %l[error2]\n\t"
740#endif 828#endif
@@ -749,8 +837,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
749 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 837 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
750 : /* gcc asm goto does not allow outputs */ 838 : /* gcc asm goto does not allow outputs */
751 : [cpu_id] "r" (cpu), 839 : [cpu_id] "r" (cpu),
752 [current_cpu_id] "m" (__rseq_abi.cpu_id), 840 [rseq_abi] "r" (&__rseq_abi),
753 [rseq_cs] "m" (__rseq_abi.rseq_cs),
754 /* try store input */ 841 /* try store input */
755 [v2] "m" (*v2), 842 [v2] "m" (*v2),
756 [newv2] "m" (newv2), 843 [newv2] "m" (newv2),
@@ -788,16 +875,21 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
788 875
789 __asm__ __volatile__ goto ( 876 __asm__ __volatile__ goto (
790 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 877 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
878 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
879#ifdef RSEQ_COMPARE_TWICE
880 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
881 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
882#endif
791 /* Start rseq by storing table entry pointer into rseq_cs. */ 883 /* Start rseq by storing table entry pointer into rseq_cs. */
792 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 884 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
793 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 885 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
794 RSEQ_INJECT_ASM(3) 886 RSEQ_INJECT_ASM(3)
795 "movl %[expect], %%eax\n\t" 887 "movl %[expect], %%eax\n\t"
796 "cmpl %[v], %%eax\n\t" 888 "cmpl %[v], %%eax\n\t"
797 "jnz %l[cmpfail]\n\t" 889 "jnz %l[cmpfail]\n\t"
798 RSEQ_INJECT_ASM(4) 890 RSEQ_INJECT_ASM(4)
799#ifdef RSEQ_COMPARE_TWICE 891#ifdef RSEQ_COMPARE_TWICE
800 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 892 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
801 "movl %[expect], %%eax\n\t" 893 "movl %[expect], %%eax\n\t"
802 "cmpl %[v], %%eax\n\t" 894 "cmpl %[v], %%eax\n\t"
803 "jnz %l[error2]\n\t" 895 "jnz %l[error2]\n\t"
@@ -813,8 +905,7 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
813 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 905 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
814 : /* gcc asm goto does not allow outputs */ 906 : /* gcc asm goto does not allow outputs */
815 : [cpu_id] "r" (cpu), 907 : [cpu_id] "r" (cpu),
816 [current_cpu_id] "m" (__rseq_abi.cpu_id), 908 [rseq_abi] "r" (&__rseq_abi),
817 [rseq_cs] "m" (__rseq_abi.rseq_cs),
818 /* try store input */ 909 /* try store input */
819 [v2] "m" (*v2), 910 [v2] "m" (*v2),
820 [newv2] "r" (newv2), 911 [newv2] "r" (newv2),
@@ -853,9 +944,15 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
853 944
854 __asm__ __volatile__ goto ( 945 __asm__ __volatile__ goto (
855 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 946 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
947 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
948#ifdef RSEQ_COMPARE_TWICE
949 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
950 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
951 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
952#endif
856 /* Start rseq by storing table entry pointer into rseq_cs. */ 953 /* Start rseq by storing table entry pointer into rseq_cs. */
857 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 954 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
858 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 955 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
859 RSEQ_INJECT_ASM(3) 956 RSEQ_INJECT_ASM(3)
860 "cmpl %[v], %[expect]\n\t" 957 "cmpl %[v], %[expect]\n\t"
861 "jnz %l[cmpfail]\n\t" 958 "jnz %l[cmpfail]\n\t"
@@ -864,7 +961,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
864 "jnz %l[cmpfail]\n\t" 961 "jnz %l[cmpfail]\n\t"
865 RSEQ_INJECT_ASM(5) 962 RSEQ_INJECT_ASM(5)
866#ifdef RSEQ_COMPARE_TWICE 963#ifdef RSEQ_COMPARE_TWICE
867 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 964 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
868 "cmpl %[v], %[expect]\n\t" 965 "cmpl %[v], %[expect]\n\t"
869 "jnz %l[error2]\n\t" 966 "jnz %l[error2]\n\t"
870 "cmpl %[expect2], %[v2]\n\t" 967 "cmpl %[expect2], %[v2]\n\t"
@@ -878,8 +975,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
878 RSEQ_ASM_DEFINE_ABORT(4, "", abort) 975 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
879 : /* gcc asm goto does not allow outputs */ 976 : /* gcc asm goto does not allow outputs */
880 : [cpu_id] "r" (cpu), 977 : [cpu_id] "r" (cpu),
881 [current_cpu_id] "m" (__rseq_abi.cpu_id), 978 [rseq_abi] "r" (&__rseq_abi),
882 [rseq_cs] "m" (__rseq_abi.rseq_cs),
883 /* cmp2 input */ 979 /* cmp2 input */
884 [v2] "m" (*v2), 980 [v2] "m" (*v2),
885 [expect2] "r" (expect2), 981 [expect2] "r" (expect2),
@@ -922,19 +1018,24 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
922 1018
923 __asm__ __volatile__ goto ( 1019 __asm__ __volatile__ goto (
924 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1020 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1021 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1022#ifdef RSEQ_COMPARE_TWICE
1023 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1024 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1025#endif
925 "movl %[src], %[rseq_scratch0]\n\t" 1026 "movl %[src], %[rseq_scratch0]\n\t"
926 "movl %[dst], %[rseq_scratch1]\n\t" 1027 "movl %[dst], %[rseq_scratch1]\n\t"
927 "movl %[len], %[rseq_scratch2]\n\t" 1028 "movl %[len], %[rseq_scratch2]\n\t"
928 /* Start rseq by storing table entry pointer into rseq_cs. */ 1029 /* Start rseq by storing table entry pointer into rseq_cs. */
929 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 1030 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
930 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 1031 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
931 RSEQ_INJECT_ASM(3) 1032 RSEQ_INJECT_ASM(3)
932 "movl %[expect], %%eax\n\t" 1033 "movl %[expect], %%eax\n\t"
933 "cmpl %%eax, %[v]\n\t" 1034 "cmpl %%eax, %[v]\n\t"
934 "jnz 5f\n\t" 1035 "jnz 5f\n\t"
935 RSEQ_INJECT_ASM(4) 1036 RSEQ_INJECT_ASM(4)
936#ifdef RSEQ_COMPARE_TWICE 1037#ifdef RSEQ_COMPARE_TWICE
937 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 1038 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
938 "movl %[expect], %%eax\n\t" 1039 "movl %[expect], %%eax\n\t"
939 "cmpl %%eax, %[v]\n\t" 1040 "cmpl %%eax, %[v]\n\t"
940 "jnz 7f\n\t" 1041 "jnz 7f\n\t"
@@ -984,8 +1085,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
984#endif 1085#endif
985 : /* gcc asm goto does not allow outputs */ 1086 : /* gcc asm goto does not allow outputs */
986 : [cpu_id] "r" (cpu), 1087 : [cpu_id] "r" (cpu),
987 [current_cpu_id] "m" (__rseq_abi.cpu_id), 1088 [rseq_abi] "r" (&__rseq_abi),
988 [rseq_cs] "m" (__rseq_abi.rseq_cs),
989 /* final store input */ 1089 /* final store input */
990 [v] "m" (*v), 1090 [v] "m" (*v),
991 [expect] "m" (expect), 1091 [expect] "m" (expect),
@@ -1030,19 +1130,24 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1030 1130
1031 __asm__ __volatile__ goto ( 1131 __asm__ __volatile__ goto (
1032 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 1132 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1133 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1134#ifdef RSEQ_COMPARE_TWICE
1135 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1136 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1137#endif
1033 "movl %[src], %[rseq_scratch0]\n\t" 1138 "movl %[src], %[rseq_scratch0]\n\t"
1034 "movl %[dst], %[rseq_scratch1]\n\t" 1139 "movl %[dst], %[rseq_scratch1]\n\t"
1035 "movl %[len], %[rseq_scratch2]\n\t" 1140 "movl %[len], %[rseq_scratch2]\n\t"
1036 /* Start rseq by storing table entry pointer into rseq_cs. */ 1141 /* Start rseq by storing table entry pointer into rseq_cs. */
1037 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 1142 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1038 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 1143 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1039 RSEQ_INJECT_ASM(3) 1144 RSEQ_INJECT_ASM(3)
1040 "movl %[expect], %%eax\n\t" 1145 "movl %[expect], %%eax\n\t"
1041 "cmpl %%eax, %[v]\n\t" 1146 "cmpl %%eax, %[v]\n\t"
1042 "jnz 5f\n\t" 1147 "jnz 5f\n\t"
1043 RSEQ_INJECT_ASM(4) 1148 RSEQ_INJECT_ASM(4)
1044#ifdef RSEQ_COMPARE_TWICE 1149#ifdef RSEQ_COMPARE_TWICE
1045 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 1150 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1046 "movl %[expect], %%eax\n\t" 1151 "movl %[expect], %%eax\n\t"
1047 "cmpl %%eax, %[v]\n\t" 1152 "cmpl %%eax, %[v]\n\t"
1048 "jnz 7f\n\t" 1153 "jnz 7f\n\t"
@@ -1093,8 +1198,7 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1093#endif 1198#endif
1094 : /* gcc asm goto does not allow outputs */ 1199 : /* gcc asm goto does not allow outputs */
1095 : [cpu_id] "r" (cpu), 1200 : [cpu_id] "r" (cpu),
1096 [current_cpu_id] "m" (__rseq_abi.cpu_id), 1201 [rseq_abi] "r" (&__rseq_abi),
1097 [rseq_cs] "m" (__rseq_abi.rseq_cs),
1098 /* final store input */ 1202 /* final store input */
1099 [v] "m" (*v), 1203 [v] "m" (*v),
1100 [expect] "m" (expect), 1204 [expect] "m" (expect),
diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c
index 4847e97ed049..7159eb777fd3 100644
--- a/tools/testing/selftests/rseq/rseq.c
+++ b/tools/testing/selftests/rseq/rseq.c
@@ -25,18 +25,27 @@
25#include <syscall.h> 25#include <syscall.h>
26#include <assert.h> 26#include <assert.h>
27#include <signal.h> 27#include <signal.h>
28#include <limits.h>
28 29
29#include "rseq.h" 30#include "rseq.h"
30 31
31#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) 32#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
32 33
33__attribute__((tls_model("initial-exec"))) __thread 34__thread volatile struct rseq __rseq_abi = {
34volatile struct rseq __rseq_abi = {
35 .cpu_id = RSEQ_CPU_ID_UNINITIALIZED, 35 .cpu_id = RSEQ_CPU_ID_UNINITIALIZED,
36}; 36};
37 37
38static __attribute__((tls_model("initial-exec"))) __thread 38/*
39volatile int refcount; 39 * Shared with other libraries. This library may take rseq ownership if it is
40 * still 0 when executing the library constructor. Set to 1 by library
41 * constructor when handling rseq. Set to 0 in destructor if handling rseq.
42 */
43int __rseq_handled;
44
45/* Whether this library have ownership of rseq registration. */
46static int rseq_ownership;
47
48static __thread volatile uint32_t __rseq_refcount;
40 49
41static void signal_off_save(sigset_t *oldset) 50static void signal_off_save(sigset_t *oldset)
42{ 51{
@@ -69,8 +78,14 @@ int rseq_register_current_thread(void)
69 int rc, ret = 0; 78 int rc, ret = 0;
70 sigset_t oldset; 79 sigset_t oldset;
71 80
81 if (!rseq_ownership)
82 return 0;
72 signal_off_save(&oldset); 83 signal_off_save(&oldset);
73 if (refcount++) 84 if (__rseq_refcount == UINT_MAX) {
85 ret = -1;
86 goto end;
87 }
88 if (__rseq_refcount++)
74 goto end; 89 goto end;
75 rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), 0, RSEQ_SIG); 90 rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), 0, RSEQ_SIG);
76 if (!rc) { 91 if (!rc) {
@@ -78,9 +93,9 @@ int rseq_register_current_thread(void)
78 goto end; 93 goto end;
79 } 94 }
80 if (errno != EBUSY) 95 if (errno != EBUSY)
81 __rseq_abi.cpu_id = -2; 96 __rseq_abi.cpu_id = RSEQ_CPU_ID_REGISTRATION_FAILED;
82 ret = -1; 97 ret = -1;
83 refcount--; 98 __rseq_refcount--;
84end: 99end:
85 signal_restore(oldset); 100 signal_restore(oldset);
86 return ret; 101 return ret;
@@ -91,13 +106,20 @@ int rseq_unregister_current_thread(void)
91 int rc, ret = 0; 106 int rc, ret = 0;
92 sigset_t oldset; 107 sigset_t oldset;
93 108
109 if (!rseq_ownership)
110 return 0;
94 signal_off_save(&oldset); 111 signal_off_save(&oldset);
95 if (--refcount) 112 if (!__rseq_refcount) {
113 ret = -1;
114 goto end;
115 }
116 if (--__rseq_refcount)
96 goto end; 117 goto end;
97 rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), 118 rc = sys_rseq(&__rseq_abi, sizeof(struct rseq),
98 RSEQ_FLAG_UNREGISTER, RSEQ_SIG); 119 RSEQ_FLAG_UNREGISTER, RSEQ_SIG);
99 if (!rc) 120 if (!rc)
100 goto end; 121 goto end;
122 __rseq_refcount = 1;
101 ret = -1; 123 ret = -1;
102end: 124end:
103 signal_restore(oldset); 125 signal_restore(oldset);
@@ -115,3 +137,20 @@ int32_t rseq_fallback_current_cpu(void)
115 } 137 }
116 return cpu; 138 return cpu;
117} 139}
140
141void __attribute__((constructor)) rseq_init(void)
142{
143 /* Check whether rseq is handled by another library. */
144 if (__rseq_handled)
145 return;
146 __rseq_handled = 1;
147 rseq_ownership = 1;
148}
149
150void __attribute__((destructor)) rseq_fini(void)
151{
152 if (!rseq_ownership)
153 return;
154 __rseq_handled = 0;
155 rseq_ownership = 0;
156}
diff --git a/tools/testing/selftests/rseq/rseq.h b/tools/testing/selftests/rseq/rseq.h
index 6c1126e7f685..d40d60e7499e 100644
--- a/tools/testing/selftests/rseq/rseq.h
+++ b/tools/testing/selftests/rseq/rseq.h
@@ -44,6 +44,7 @@
44#endif 44#endif
45 45
46extern __thread volatile struct rseq __rseq_abi; 46extern __thread volatile struct rseq __rseq_abi;
47extern int __rseq_handled;
47 48
48#define rseq_likely(x) __builtin_expect(!!(x), 1) 49#define rseq_likely(x) __builtin_expect(!!(x), 1)
49#define rseq_unlikely(x) __builtin_expect(!!(x), 0) 50#define rseq_unlikely(x) __builtin_expect(!!(x), 0)
diff --git a/tools/testing/selftests/sigaltstack/sas.c b/tools/testing/selftests/sigaltstack/sas.c
index 228c2ae47687..ad0f8df2ca0a 100644
--- a/tools/testing/selftests/sigaltstack/sas.c
+++ b/tools/testing/selftests/sigaltstack/sas.c
@@ -109,6 +109,7 @@ int main(void)
109 int err; 109 int err;
110 110
111 ksft_print_header(); 111 ksft_print_header();
112 ksft_set_plan(3);
112 113
113 sigemptyset(&act.sa_mask); 114 sigemptyset(&act.sa_mask);
114 act.sa_flags = SA_ONSTACK | SA_SIGINFO; 115 act.sa_flags = SA_ONSTACK | SA_SIGINFO;
diff --git a/tools/testing/selftests/sync/sync_test.c b/tools/testing/selftests/sync/sync_test.c
index 7f7938263c5c..3824b66f41a0 100644
--- a/tools/testing/selftests/sync/sync_test.c
+++ b/tools/testing/selftests/sync/sync_test.c
@@ -86,6 +86,7 @@ int main(void)
86 int err; 86 int err;
87 87
88 ksft_print_header(); 88 ksft_print_header();
89 ksft_set_plan(3 + 7);
89 90
90 sync_api_supported(); 91 sync_api_supported();
91 92
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-25 14:35:51 -0400