s390: use expoline thunks in the BPF JIT

The BPF JIT need safe guarding against spectre v2 in the sk_load_xxx
assembler stubs and the indirect branches generated by the JIT itself
need to be converted to expolines.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

2 files changed, 71 insertions, 8 deletions

diff --git a/arch/s390/net/bpf_jit.S b/arch/s390/net/bpf_jit.S
index 25bb4643c4f4..9f794869c1b0 100644
--- a/arch/s390/net/bpf_jit.S
+++ b/arch/s390/net/bpf_jit.S
@@ -9,6 +9,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/nospec-insn.h>
 #include "bpf_jit.h"
 
 /*
@@ -54,7 +55,7 @@ ENTRY(sk_load_##NAME##_pos);						\
         clg     %r3,STK_OFF_HLEN(%r15); /* Offset + SIZE > hlen? */     \
         jh      sk_load_##NAME##_slow;                                  \
         LOAD    %r14,-SIZE(%r3,%r12);   /* Get data from skb */         \
-        b       OFF_OK(%r6);            /* Return */                    \
+        B_EX    OFF_OK,%r6;             /* Return */                    \
                                                                         \
 sk_load_##NAME##_slow:;                                                 \
         lgr     %r2,%r7;                /* Arg1 = skb pointer */        \
@@ -64,11 +65,14 @@ sk_load_##NAME##_slow:;							\
         brasl   %r14,skb_copy_bits;     /* Get data from skb */         \
         LOAD    %r14,STK_OFF_TMP(%r15); /* Load from temp bufffer */    \
         ltgr    %r2,%r2;                /* Set cc to (%r2 != 0) */      \
-        br      %r6;                    /* Return */
+        BR_EX   %r6;                    /* Return */
 
 sk_load_common(word, 4, llgf)   /* r14 = *(u32 *) (skb->data+offset) */
 sk_load_common(half, 2, llgh)   /* r14 = *(u16 *) (skb->data+offset) */
 
+        GEN_BR_THUNK %r6
+        GEN_B_THUNK OFF_OK,%r6
+
 /*
  * Load 1 byte from SKB (optimized version)
  */
@@ -80,7 +84,7 @@ ENTRY(sk_load_byte_pos)
         clg     %r3,STK_OFF_HLEN(%r15)  # Offset >= hlen?
         jnl     sk_load_byte_slow
         llgc    %r14,0(%r3,%r12)        # Get byte from skb
-        b       OFF_OK(%r6)             # Return OK
+        B_EX    OFF_OK,%r6              # Return OK
 
 sk_load_byte_slow:
         lgr     %r2,%r7                 # Arg1 = skb pointer
@@ -90,7 +94,7 @@ sk_load_byte_slow:
         brasl   %r14,skb_copy_bits      # Get data from skb
         llgc    %r14,STK_OFF_TMP(%r15)  # Load result from temp buffer
         ltgr    %r2,%r2                 # Set cc to (%r2 != 0)
-        br      %r6                     # Return cc
+        BR_EX   %r6                     # Return cc
 
 #define sk_negative_common(NAME, SIZE, LOAD)                            \
 sk_load_##NAME##_slow_neg:;                                             \
@@ -104,7 +108,7 @@ sk_load_##NAME##_slow_neg:;						\
         jz      bpf_error;                                              \
         LOAD    %r14,0(%r2);            /* Get data from pointer */     \
         xr      %r3,%r3;                /* Set cc to zero */            \
-        br      %r6;                    /* Return cc */
+        BR_EX   %r6;                    /* Return cc */
 
 sk_negative_common(word, 4, llgf)
 sk_negative_common(half, 2, llgh)
@@ -113,4 +117,4 @@ sk_negative_common(byte, 1, llgc)
 bpf_error:
 # force a return 0 from jit handler
         ltgr    %r15,%r15       # Set condition code
-        br      %r6
+        BR_EX   %r6
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 78a19c93b380..dd2bcf0e7d00 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -25,6 +25,8 @@
 #include <linux/bpf.h>
 #include <asm/cacheflush.h>
 #include <asm/dis.h>
+#include <asm/facility.h>
+#include <asm/nospec-branch.h>
 #include <asm/set_memory.h>
 #include "bpf_jit.h"
 
@@ -41,6 +43,8 @@ struct bpf_jit {
         int base_ip;            /* Base address for literal pool */
         int ret0_ip;            /* Address of return 0 */
         int exit_ip;            /* Address of exit */
+        int r1_thunk_ip;        /* Address of expoline thunk for 'br %r1' */
+        int r14_thunk_ip;       /* Address of expoline thunk for 'br %r14' */
         int tail_call_start;    /* Tail call start offset */
         int labels[1];          /* Labels for local jumps */
 };
@@ -250,6 +254,19 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
         REG_SET_SEEN(b2);                                       \
 })
 
+#define EMIT6_PCREL_RILB(op, b, target)                         \
+({                                                              \
+        int rel = (target - jit->prg) / 2;                      \
+        _EMIT6(op | reg_high(b) << 16 | rel >> 16, rel & 0xffff);       \
+        REG_SET_SEEN(b);                                        \
+})
+
+#define EMIT6_PCREL_RIL(op, target)                             \
+({                                                              \
+        int rel = (target - jit->prg) / 2;                      \
+        _EMIT6(op | rel >> 16, rel & 0xffff);                   \
+})
+
 #define _EMIT6_IMM(op, imm)                                     \
 ({                                                              \
         unsigned int __imm = (imm);                             \
@@ -469,8 +486,45 @@ static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)
         EMIT4(0xb9040000, REG_2, BPF_REG_0);
         /* Restore registers */
         save_restore_regs(jit, REGS_RESTORE, stack_depth);
+        if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) {
+                jit->r14_thunk_ip = jit->prg;
+                /* Generate __s390_indirect_jump_r14 thunk */
+                if (test_facility(35)) {
+                        /* exrl %r0,.+10 */
+                        EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
+                } else {
+                        /* larl %r1,.+14 */
+                        EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14);
+                        /* ex 0,0(%r1) */
+                        EMIT4_DISP(0x44000000, REG_0, REG_1, 0);
+                }
+                /* j . */
+                EMIT4_PCREL(0xa7f40000, 0);
+        }
         /* br %r14 */
         _EMIT2(0x07fe);
+
+        if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable &&
+            (jit->seen & SEEN_FUNC)) {
+                jit->r1_thunk_ip = jit->prg;
+                /* Generate __s390_indirect_jump_r1 thunk */
+                if (test_facility(35)) {
+                        /* exrl %r0,.+10 */
+                        EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
+                        /* j . */
+                        EMIT4_PCREL(0xa7f40000, 0);
+                        /* br %r1 */
+                        _EMIT2(0x07f1);
+                } else {
+                        /* larl %r1,.+14 */
+                        EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14);
+                        /* ex 0,S390_lowcore.br_r1_tampoline */
+                        EMIT4_DISP(0x44000000, REG_0, REG_0,
+                                   offsetof(struct lowcore, br_r1_trampoline));
+                        /* j . */
+                        EMIT4_PCREL(0xa7f40000, 0);
+                }
+        }
 }
 
 /*
@@ -966,8 +1020,13 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
                 /* lg %w1,<d(imm)>(%l) */
                 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_W1, REG_0, REG_L,
                               EMIT_CONST_U64(func));
-                /* basr %r14,%w1 */
-                EMIT2(0x0d00, REG_14, REG_W1);
+                if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) {
+                        /* brasl %r14,__s390_indirect_jump_r1 */
+                        EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);
+                } else {
+                        /* basr %r14,%w1 */
+                        EMIT2(0x0d00, REG_14, REG_W1);
+                }
                 /* lgr %b0,%r2: load return value into %b0 */
                 EMIT4(0xb9040000, BPF_REG_0, REG_2);
                 if ((jit->seen & SEEN_SKB) &&