1 files changed, 5 insertions, 4 deletions

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 4923d92f918d..45e4eb5bcbb2 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -13,6 +13,7 @@
 #include <linux/if_vlan.h>
 #include <asm/cacheflush.h>
 #include <asm/set_memory.h>
+#include <asm/nospec-branch.h>
 #include <linux/bpf.h>
 
 /*
@@ -290,7 +291,7 @@ static void emit_bpf_tail_call(u8 **pprog)
         EMIT2(0x89, 0xD2);                        /* mov edx, edx */
         EMIT3(0x39, 0x56,                         /* cmp dword ptr [rsi + 16], edx */
               offsetof(struct bpf_array, map.max_entries));
-#define OFFSET1 43 /* number of bytes to jump */
+#define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* number of bytes to jump */
         EMIT2(X86_JBE, OFFSET1);                  /* jbe out */
         label1 = cnt;
 
@@ -299,7 +300,7 @@ static void emit_bpf_tail_call(u8 **pprog)
          */
         EMIT2_off32(0x8B, 0x85, 36);              /* mov eax, dword ptr [rbp + 36] */
         EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT);     /* cmp eax, MAX_TAIL_CALL_CNT */
-#define OFFSET2 32
+#define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE)
         EMIT2(X86_JA, OFFSET2);                   /* ja out */
         label2 = cnt;
         EMIT3(0x83, 0xC0, 0x01);                  /* add eax, 1 */
@@ -313,7 +314,7 @@ static void emit_bpf_tail_call(u8 **pprog)
          *   goto out;
          */
         EMIT3(0x48, 0x85, 0xC0);                  /* test rax,rax */
-#define OFFSET3 10
+#define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE)
         EMIT2(X86_JE, OFFSET3);                   /* je out */
         label3 = cnt;
 
@@ -326,7 +327,7 @@ static void emit_bpf_tail_call(u8 **pprog)
          * rdi == ctx (1st arg)
          * rax == prog->bpf_func + prologue_size
          */
-        EMIT2(0xFF, 0xE0);                        /* jmp rax */
+        RETPOLINE_RAX_BPF_JIT();
 
         /* out: */
         BUILD_BUG_ON(cnt - label1 != OFFSET1);