bpf, arm64: implement jiting of BPF_XADD

This work adds BPF_XADD for BPF_W/BPF_DW to the arm64 JIT and therefore
completes JITing of all BPF instructions, meaning we can thus also remove
the 'notyet' label and do not need to fall back to the interpreter when
BPF_XADD is used in a program!

This now also brings arm64 JIT in line with x86_64, s390x, ppc64, sparc64,
where all current eBPF features are supported.

BPF_W example from test_bpf:

  .u.insns_int = {
    BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
    BPF_ST_MEM(BPF_W, R10, -40, 0x10),
    BPF_STX_XADD(BPF_W, R10, R0, -40),
    BPF_LDX_MEM(BPF_W, R0, R10, -40),
    BPF_EXIT_INSN(),
  },

  [...]
  00000020:  52800247  mov w7, #0x12 // #18
  00000024:  928004eb  mov x11, #0xffffffffffffffd8 // #-40
  00000028:  d280020a  mov x10, #0x10 // #16
  0000002c:  b82b6b2a  str w10, [x25,x11]
  // start of xadd mapping:
  00000030:  928004ea  mov x10, #0xffffffffffffffd8 // #-40
  00000034:  8b19014a  add x10, x10, x25
  00000038:  f9800151  prfm pstl1strm, [x10]
  0000003c:  885f7d4b  ldxr w11, [x10]
  00000040:  0b07016b  add w11, w11, w7
  00000044:  880b7d4b  stxr w11, w11, [x10]
  00000048:  35ffffab  cbnz w11, 0x0000003c
  // end of xadd mapping:
  [...]

BPF_DW example from test_bpf:

  .u.insns_int = {
    BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
    BPF_ST_MEM(BPF_DW, R10, -40, 0x10),
    BPF_STX_XADD(BPF_DW, R10, R0, -40),
    BPF_LDX_MEM(BPF_DW, R0, R10, -40),
    BPF_EXIT_INSN(),
  },

  [...]
  00000020:  52800247  mov w7,  #0x12 // #18
  00000024:  928004eb  mov x11, #0xffffffffffffffd8 // #-40
  00000028:  d280020a  mov x10, #0x10 // #16
  0000002c:  f82b6b2a  str x10, [x25,x11]
  // start of xadd mapping:
  00000030:  928004ea  mov x10, #0xffffffffffffffd8 // #-40
  00000034:  8b19014a  add x10, x10, x25
  00000038:  f9800151  prfm pstl1strm, [x10]
  0000003c:  c85f7d4b  ldxr x11, [x10]
  00000040:  8b07016b  add x11, x11, x7
  00000044:  c80b7d4b  stxr w11, x11, [x10]
  00000048:  35ffffab  cbnz w11, 0x0000003c
  // end of xadd mapping:
  [...]

Tested on Cavium ThunderX ARMv8, test suite results after the patch:

  No JIT:   [ 3751.855362] test_bpf: Summary: 311 PASSED, 0 FAILED, [0/303 JIT'ed]
  With JIT: [ 3573.759527] test_bpf: Summary: 311 PASSED, 0 FAILED, [303/303 JIT'ed]

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 105 insertions, 0 deletions

diff --git a/lib/test_bpf.c b/lib/test_bpf.c
index 0362da0b66c3..3a7730ca81be 100644
--- a/lib/test_bpf.c
+++ b/lib/test_bpf.c
@@ -434,6 +434,41 @@ loop:
         return 0;
 }
 
+static int __bpf_fill_stxdw(struct bpf_test *self, int size)
+{
+        unsigned int len = BPF_MAXINSNS;
+        struct bpf_insn *insn;
+        int i;
+
+        insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL);
+        if (!insn)
+                return -ENOMEM;
+
+        insn[0] = BPF_ALU32_IMM(BPF_MOV, R0, 1);
+        insn[1] = BPF_ST_MEM(size, R10, -40, 42);
+
+        for (i = 2; i < len - 2; i++)
+                insn[i] = BPF_STX_XADD(size, R10, R0, -40);
+
+        insn[len - 2] = BPF_LDX_MEM(size, R0, R10, -40);
+        insn[len - 1] = BPF_EXIT_INSN();
+
+        self->u.ptr.insns = insn;
+        self->u.ptr.len = len;
+
+        return 0;
+}
+
+static int bpf_fill_stxw(struct bpf_test *self)
+{
+        return __bpf_fill_stxdw(self, BPF_W);
+}
+
+static int bpf_fill_stxdw(struct bpf_test *self)
+{
+        return __bpf_fill_stxdw(self, BPF_DW);
+}
+
 static struct bpf_test tests[] = {
         {
                 "TAX",
@@ -4303,6 +4338,41 @@ static struct bpf_test tests[] = {
                 { { 0, 0x22 } },
         },
         {
+                "STX_XADD_W: Test side-effects, r10: 0x12 + 0x10 = 0x22",
+                .u.insns_int = {
+                        BPF_ALU64_REG(BPF_MOV, R1, R10),
+                        BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
+                        BPF_ST_MEM(BPF_W, R10, -40, 0x10),
+                        BPF_STX_XADD(BPF_W, R10, R0, -40),
+                        BPF_ALU64_REG(BPF_MOV, R0, R10),
+                        BPF_ALU64_REG(BPF_SUB, R0, R1),
+                        BPF_EXIT_INSN(),
+                },
+                INTERNAL,
+                { },
+                { { 0, 0 } },
+        },
+        {
+                "STX_XADD_W: Test side-effects, r0: 0x12 + 0x10 = 0x22",
+                .u.insns_int = {
+                        BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
+                        BPF_ST_MEM(BPF_W, R10, -40, 0x10),
+                        BPF_STX_XADD(BPF_W, R10, R0, -40),
+                        BPF_EXIT_INSN(),
+                },
+                INTERNAL,
+                { },
+                { { 0, 0x12 } },
+        },
+        {
+                "STX_XADD_W: X + 1 + 1 + 1 + ...",
+                { },
+                INTERNAL,
+                { },
+                { { 0, 4134 } },
+                .fill_helper = bpf_fill_stxw,
+        },
+        {
                 "STX_XADD_DW: Test: 0x12 + 0x10 = 0x22",
                 .u.insns_int = {
                         BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
@@ -4315,6 +4385,41 @@ static struct bpf_test tests[] = {
                 { },
                 { { 0, 0x22 } },
         },
+        {
+                "STX_XADD_DW: Test side-effects, r10: 0x12 + 0x10 = 0x22",
+                .u.insns_int = {
+                        BPF_ALU64_REG(BPF_MOV, R1, R10),
+                        BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
+                        BPF_ST_MEM(BPF_DW, R10, -40, 0x10),
+                        BPF_STX_XADD(BPF_DW, R10, R0, -40),
+                        BPF_ALU64_REG(BPF_MOV, R0, R10),
+                        BPF_ALU64_REG(BPF_SUB, R0, R1),
+                        BPF_EXIT_INSN(),
+                },
+                INTERNAL,
+                { },
+                { { 0, 0 } },
+        },
+        {
+                "STX_XADD_DW: Test side-effects, r0: 0x12 + 0x10 = 0x22",
+                .u.insns_int = {
+                        BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
+                        BPF_ST_MEM(BPF_DW, R10, -40, 0x10),
+                        BPF_STX_XADD(BPF_DW, R10, R0, -40),
+                        BPF_EXIT_INSN(),
+                },
+                INTERNAL,
+                { },
+                { { 0, 0x12 } },
+        },
+        {
+                "STX_XADD_DW: X + 1 + 1 + 1 + ...",
+                { },
+                INTERNAL,
+                { },
+                { { 0, 4134 } },
+                .fill_helper = bpf_fill_stxdw,
+        },
         /* BPF_JMP | BPF_EXIT */
         {
                 "JMP_EXIT",