rseq/selftests: Provide rseq library

This rseq helper library provides a user-space API to the rseq()
system call.

The rseq fast-path exposes the instruction pointer addresses where the
rseq assembly blocks begin and end, as well as the associated abort
instruction pointer, in the __rseq_table section. This section allows
debuggers may know where to place breakpoints when single-stepping
through assembly blocks which may be aborted at any point by the kernel.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Joel Fernandes <joelaf@google.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Dave Watson <davejwatson@fb.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Shuah Khan <shuahkh@osg.samsung.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: linux-kselftest@vger.kernel.org
Cc: "H . Peter Anvin" <hpa@zytor.com>
Cc: Chris Lameter <cl@linux.com>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Andrew Hunter <ahh@google.com>
Cc: Michael Kerrisk <mtk.manpages@gmail.com>
Cc: "Paul E . McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Paul Turner <pjt@google.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Josh Triplett <josh@joshtriplett.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Ben Maurer <bmaurer@fb.com>
Cc: linux-api@vger.kernel.org
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lkml.kernel.org/r/20180602124408.8430-13-mathieu.desnoyers@efficios.com

6 files changed, 2847 insertions, 0 deletions

diff --git a/tools/testing/selftests/rseq/rseq-arm.h b/tools/testing/selftests/rseq/rseq-arm.h
new file mode 100644
index 000000000000..3b055f9aeaab
--- /dev/null
+++ b/tools/testing/selftests/rseq/rseq-arm.h
@@ -0,0 +1,715 @@
+/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
+/*
+ * rseq-arm.h
+ *
+ * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ */
+
+#define RSEQ_SIG        0x53053053
+
+#define rseq_smp_mb()   __asm__ __volatile__ ("dmb" ::: "memory", "cc")
+#define rseq_smp_rmb()  __asm__ __volatile__ ("dmb" ::: "memory", "cc")
+#define rseq_smp_wmb()  __asm__ __volatile__ ("dmb" ::: "memory", "cc")
+
+#define rseq_smp_load_acquire(p)                                        \
+__extension__ ({                                                        \
+        __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
+        rseq_smp_mb();                                                  \
+        ____p1;                                                         \
+})
+
+#define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
+
+#define rseq_smp_store_release(p, v)                                    \
+do {                                                                    \
+        rseq_smp_mb();                                                  \
+        RSEQ_WRITE_ONCE(*p, v);                                         \
+} while (0)
+
+#ifdef RSEQ_SKIP_FASTPATH
+#include "rseq-skip.h"
+#else /* !RSEQ_SKIP_FASTPATH */
+
+#define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip,               \
+                                post_commit_offset, abort_ip)           \
+                ".pushsection __rseq_table, \"aw\"\n\t"                 \
+                ".balign 32\n\t"                                        \
+                ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
+                ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
+                ".popsection\n\t"
+
+#define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip)       \
+        __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip,                     \
+                                (post_commit_ip - start_ip), abort_ip)
+
+#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
+                RSEQ_INJECT_ASM(1)                                      \
+                "adr r0, " __rseq_str(cs_label) "\n\t"                  \
+                "str r0, %[" __rseq_str(rseq_cs) "]\n\t"                \
+                __rseq_str(label) ":\n\t"
+
+#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
+                RSEQ_INJECT_ASM(2)                                      \
+                "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
+                "cmp %[" __rseq_str(cpu_id) "], r0\n\t"         \
+                "bne " __rseq_str(label) "\n\t"
+
+#define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,           \
+                                abort_label, version, flags,            \
+                                start_ip, post_commit_offset, abort_ip) \
+                __rseq_str(table_label) ":\n\t"                         \
+                ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
+                ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
+                ".word " __rseq_str(RSEQ_SIG) "\n\t"                    \
+                __rseq_str(label) ":\n\t"                               \
+                teardown                                                \
+                "b %l[" __rseq_str(abort_label) "]\n\t"
+
+#define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
+                              start_ip, post_commit_ip, abort_ip)       \
+        __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown,           \
+                                abort_label, 0x0, 0x0, start_ip,        \
+                                (post_commit_ip - start_ip), abort_ip)
+
+#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
+                __rseq_str(label) ":\n\t"                               \
+                teardown                                                \
+                "b %l[" __rseq_str(cmpfail_label) "]\n\t"
+
+#define rseq_workaround_gcc_asm_size_guess()    __asm__ __volatile__("")
+
+static inline __attribute__((always_inline))
+int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
+{
+        RSEQ_INJECT_C(9)
+
+        rseq_workaround_gcc_asm_size_guess();
+        __asm__ __volatile__ goto (
+                RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
+                /* Start rseq by storing table entry pointer into rseq_cs. */
+                RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
+                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
+                RSEQ_INJECT_ASM(3)
+                "ldr r0, %[v]\n\t"
+                "cmp %[expect], r0\n\t"
+                "bne %l[cmpfail]\n\t"
+                RSEQ_INJECT_ASM(4)
+#ifdef RSEQ_COMPARE_TWICE
+                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
+                "ldr r0, %[v]\n\t"
+                "cmp %[expect], r0\n\t"
+                "bne %l[error2]\n\t"
+#endif
+                /* final store */
+                "str %[newv], %[v]\n\t"
+                "2:\n\t"
+                RSEQ_INJECT_ASM(5)
+                "b 5f\n\t"
+                RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
+                "5:\n\t"
+                : /* gcc asm goto does not allow outputs */
+                : [cpu_id]              "r" (cpu),
+                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
+                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
+                  [v]                   "m" (*v),
+                  [expect]              "r" (expect),
+                  [newv]                "r" (newv)
+                  RSEQ_INJECT_INPUT
+                : "r0", "memory", "cc"
+                  RSEQ_INJECT_CLOBBER
+                : abort, cmpfail
+#ifdef RSEQ_COMPARE_TWICE
+                  , error1, error2
+#endif
+        );
+        rseq_workaround_gcc_asm_size_guess();
+        return 0;
+abort:
+        rseq_workaround_gcc_asm_size_guess();
+        RSEQ_INJECT_FAILED
+        return -1;
+cmpfail:
+        rseq_workaround_gcc_asm_size_guess();
+        return 1;
+#ifdef RSEQ_COMPARE_TWICE
+error1:
+        rseq_bug("cpu_id comparison failed");
+error2:
+        rseq_bug("expected value comparison failed");
+#endif
+}
+
+static inline __attribute__((always_inline))
+int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
+                               off_t voffp, intptr_t *load, int cpu)
+{
+        RSEQ_INJECT_C(9)
+
+        rseq_workaround_gcc_asm_size_guess();
+        __asm__ __volatile__ goto (
+                RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
+                /* Start rseq by storing table entry pointer into rseq_cs. */
+                RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
+                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
+                RSEQ_INJECT_ASM(3)
+                "ldr r0, %[v]\n\t"
+                "cmp %[expectnot], r0\n\t"
+                "beq %l[cmpfail]\n\t"
+                RSEQ_INJECT_ASM(4)
+#ifdef RSEQ_COMPARE_TWICE
+                RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
+                "ldr r0, %[v]\n\t"
+                "cmp %[expectnot], r0\n\t"
+                "beq %l[error2]\n\t"
+#endif
+                "str r0, %[load]\n\t"
+                "add r0, %[voffp]\n\t"
+                "ldr r0, [r0]\n\t"
+                /* final store */
+                "str r0, %[v]\n\t"
+                "2:\n\t"
+                RSEQ_INJECT_ASM(5)
+                "b 5f\n\t"
+                RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
+                "5:\n\t"
+                : /* gcc asm goto does not allow outputs */
+                : [cpu_id]              "r" (cpu),
+                  [current_cpu_id]      "m" (__rseq_abi.cpu_id),
+                  [rseq_cs]             "m" (__rseq_abi.rseq_cs),
+                  /* final store input */
+                  [v]                   "m" (*v),
+                  [expectnot]           "r" (expectnot),
+                  [voffp]               "Ir" (voffp),
+                  [loa