1 files changed, 91 insertions, 31 deletions

diff --git a/arch/x86/net/bpf_jit.S b/arch/x86/net/bpf_jit.S
index 66870223f8c..877b9a1b215 100644
--- a/arch/x86/net/bpf_jit.S
+++ b/arch/x86/net/bpf_jit.S
@@ -18,17 +18,17 @@
  * r9d : hlen = skb->len - skb->data_len
  */
 #define SKBDATA %r8
-
-sk_load_word_ind:
-        .globl  sk_load_word_ind
-
-        add     %ebx,%esi       /* offset += X */
-#       test    %esi,%esi       /* if (offset < 0) goto bpf_error; */
-        js      bpf_error
+#define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
 
 sk_load_word:
         .globl  sk_load_word
 
+        test    %esi,%esi
+        js      bpf_slow_path_word_neg
+
+sk_load_word_positive_offset:
+        .globl  sk_load_word_positive_offset
+
         mov     %r9d,%eax               # hlen
         sub     %esi,%eax               # hlen - offset
         cmp     $3,%eax
@@ -37,16 +37,15 @@ sk_load_word:
         bswap   %eax                    /* ntohl() */
         ret
 
-
-sk_load_half_ind:
-        .globl sk_load_half_ind
-
-        add     %ebx,%esi       /* offset += X */
-        js      bpf_error
-
 sk_load_half:
         .globl  sk_load_half
 
+        test    %esi,%esi
+        js      bpf_slow_path_half_neg
+
+sk_load_half_positive_offset:
+        .globl  sk_load_half_positive_offset
+
         mov     %r9d,%eax
         sub     %esi,%eax               #       hlen - offset
         cmp     $1,%eax
@@ -55,14 +54,15 @@ sk_load_half:
         rol     $8,%ax                  # ntohs()
         ret
 
-sk_load_byte_ind:
-        .globl sk_load_byte_ind
-        add     %ebx,%esi       /* offset += X */
-        js      bpf_error
-
 sk_load_byte:
         .globl  sk_load_byte
 
+        test    %esi,%esi
+        js      bpf_slow_path_byte_neg
+
+sk_load_byte_positive_offset:
+        .globl  sk_load_byte_positive_offset
+
         cmp     %esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
         jle     bpf_slow_path_byte
         movzbl  (SKBDATA,%rsi),%eax
@@ -73,25 +73,21 @@ sk_load_byte:
  *
  * Implements BPF_S_LDX_B_MSH : ldxb  4*([offset]&0xf)
  * Must preserve A accumulator (%eax)
- * Inputs : %esi is the offset value, already known positive
+ * Inputs : %esi is the offset value
  */
-ENTRY(sk_load_byte_msh)
-        CFI_STARTPROC
+sk_load_byte_msh:
+        .globl  sk_load_byte_msh
+        test    %esi,%esi
+        js      bpf_slow_path_byte_msh_neg
+
+sk_load_byte_msh_positive_offset:
+        .globl  sk_load_byte_msh_positive_offset
         cmp     %esi,%r9d      /* if (offset >= hlen) goto bpf_slow_path_byte_msh */
         jle     bpf_slow_path_byte_msh
         movzbl  (SKBDATA,%rsi),%ebx
         and     $15,%bl
         shl     $2,%bl
         ret
-        CFI_ENDPROC
-ENDPROC(sk_load_byte_msh)
-
-bpf_error:
-# force a return 0 from jit handler
-        xor             %eax,%eax
-        mov             -8(%rbp),%rbx
-        leaveq
-        ret
 
 /* rsi contains offset and can be scratched */
 #define bpf_slow_path_common(LEN)               \
@@ -138,3 +134,67 @@ bpf_slow_path_byte_msh:
         shl     $2,%al
         xchg    %eax,%ebx
         ret
+
+#define sk_negative_common(SIZE)                                \
+        push    %rdi;   /* save skb */                          \
+        push    %r9;                                            \
+        push    SKBDATA;                                        \
+/* rsi already has offset */                                    \
+        mov     $SIZE,%ecx;     /* size */                      \
+        call    bpf_internal_load_pointer_neg_helper;           \
+        test    %rax,%rax;                                      \
+        pop     SKBDATA;                                        \
+        pop     %r9;                                            \
+        pop     %rdi;                                           \
+        jz      bpf_error
+
+
+bpf_slow_path_word_neg:
+        cmp     SKF_MAX_NEG_OFF, %esi   /* test range */
+        jl      bpf_error       /* offset lower -> error  */
+sk_load_word_negative_offset:
+        .globl  sk_load_word_negative_offset
+        sk_negative_common(4)
+        mov     (%rax), %eax
+        bswap   %eax
+        ret
+
+bpf_slow_path_half_neg:
+        cmp     SKF_MAX_NEG_OFF, %esi
+        jl      bpf_error
+sk_load_half_negative_offset:
+        .globl  sk_load_half_negative_offset
+        sk_negative_common(2)
+        mov     (%rax),%ax
+        rol     $8,%ax
+        movzwl  %ax,%eax
+        ret
+
+bpf_slow_path_byte_neg:
+        cmp     SKF_MAX_NEG_OFF, %esi
+        jl      bpf_error
+sk_load_byte_negative_offset:
+        .globl  sk_load_byte_negative_offset
+        sk_negative_common(1)
+        movzbl  (%rax), %eax
+        ret
+
+bpf_slow_path_byte_msh_neg:
+        cmp     SKF_MAX_NEG_OFF, %esi
+        jl      bpf_error
+sk_load_byte_msh_negative_offset:
+        .globl  sk_load_byte_msh_negative_offset
+        xchg    %eax,%ebx /* dont lose A , X is about to be scratched */
+        sk_negative_common(1)
+        movzbl  (%rax),%eax
+        and     $15,%al
+        shl     $2,%al
+        xchg    %eax,%ebx
+        ret
+
+bpf_error:
+# force a return 0 from jit handler
+        xor             %eax,%eax
+        mov             -8(%rbp),%rbx
+        leaveq
+        ret