1 files changed, 77 insertions, 52 deletions

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 5c8cb8043c5a..d56cd1f515bd 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -8,12 +8,10 @@
  * as published by the Free Software Foundation; version 2
  * of the License.
  */
-#include <linux/moduleloader.h>
-#include <asm/cacheflush.h>
 #include <linux/netdevice.h>
 #include <linux/filter.h>
 #include <linux/if_vlan.h>
-#include <linux/random.h>
+#include <asm/cacheflush.h>
 
 int bpf_jit_enable __read_mostly;
 
@@ -109,39 +107,6 @@ static inline void bpf_flush_icache(void *start, void *end)
 #define CHOOSE_LOAD_FUNC(K, func) \
         ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
 
-struct bpf_binary_header {
-        unsigned int    pages;
-        /* Note : for security reasons, bpf code will follow a randomly
-         * sized amount of int3 instructions
-         */
-        u8              image[];
-};
-
-static struct bpf_binary_header *bpf_alloc_binary(unsigned int proglen,
-                                                  u8 **image_ptr)
-{
-        unsigned int sz, hole;
-        struct bpf_binary_header *header;
-
-        /* Most of BPF filters are really small,
-         * but if some of them fill a page, allow at least
-         * 128 extra bytes to insert a random section of int3
-         */
-        sz = round_up(proglen + sizeof(*header) + 128, PAGE_SIZE);
-        header = module_alloc(sz);
-        if (!header)
-                return NULL;
-
-        memset(header, 0xcc, sz); /* fill whole space with int3 instructions */
-
-        header->pages = sz / PAGE_SIZE;
-        hole = min(sz - (proglen + sizeof(*header)), PAGE_SIZE - sizeof(*header));
-
-        /* insert a random number of int3 instructions before BPF code */
-        *image_ptr = &header->image[prandom_u32() % hole];
-        return header;
-}
-
 /* pick a register outside of BPF range for JIT internal work */
 #define AUX_REG (MAX_BPF_REG + 1)
 
@@ -206,6 +171,12 @@ static inline u8 add_2reg(u8 byte, u32 dst_reg, u32 src_reg)
         return byte + reg2hex[dst_reg] + (reg2hex[src_reg] << 3);
 }
 
+static void jit_fill_hole(void *area, unsigned int size)
+{
+        /* fill whole space with int3 instructions */
+        memset(area, 0xcc, size);
+}
+
 struct jit_context {
         unsigned int cleanup_addr; /* epilogue code offset */
         bool seen_ld_abs;
@@ -393,6 +364,23 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
                         EMIT1_off32(add_1reg(0xB8, dst_reg), imm32);
                         break;
 
+                case BPF_LD | BPF_IMM | BPF_DW:
+                        if (insn[1].code != 0 || insn[1].src_reg != 0 ||
+                            insn[1].dst_reg != 0 || insn[1].off != 0) {
+                                /* verifier must catch invalid insns */
+                                pr_err("invalid BPF_LD_IMM64 insn\n");
+                                return -EINVAL;
+                        }
+
+                        /* movabsq %rax, imm64 */
+                        EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg));
+                        EMIT(insn[0].imm, 4);
+                        EMIT(insn[1].imm, 4);
+
+                        insn++;
+                        i++;
+                        break;
+
                         /* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */
                 case BPF_ALU | BPF_MOD | BPF_X:
                 case BPF_ALU | BPF_DIV | BPF_X:
@@ -515,6 +503,48 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
                         EMIT3(0xC1, add_1reg(b3, dst_reg), imm32);
                         break;
 
+                case BPF_ALU | BPF_LSH | BPF_X:
+                case BPF_ALU | BPF_RSH | BPF_X:
+                case BPF_ALU | BPF_ARSH | BPF_X:
+                case BPF_ALU64 | BPF_LSH | BPF_X:
+                case BPF_ALU64 | BPF_RSH | BPF_X:
+                case BPF_ALU64 | BPF_ARSH | BPF_X:
+
+                        /* check for bad case when dst_reg == rcx */
+                        if (dst_reg == BPF_REG_4) {
+                                /* mov r11, dst_reg */
+                                EMIT_mov(AUX_REG, dst_reg);
+                                dst_reg = AUX_REG;
+                        }
+
+                        if (src_reg != BPF_REG_4) { /* common case */
+                                EMIT1(0x51); /* push rcx */
+
+                                /* mov rcx, src_reg */
+                                EMIT_mov(BPF_REG_4, src_reg);
+                        }
+
+                        /* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
+                        if (BPF_CLASS(insn->code) == BPF_ALU64)
+                                EMIT1(add_1mod(0x48, dst_reg));
+                        else if (is_ereg(dst_reg))
+                                EMIT1(add_1mod(0x40, dst_reg));
+
+                        switch (BPF_OP(insn->code)) {
+                        case BPF_LSH: b3 = 0xE0; break;
+                        case BPF_RSH: b3 = 0xE8; break;
+                        case BPF_ARSH: b3 = 0xF8; break;
+                        }
+                        EMIT2(0xD3, add_1reg(b3, dst_reg));
+
+                        if (src_reg != BPF_REG_4)
+                                EMIT1(0x59); /* pop rcx */
+
+                        if (insn->dst_reg == BPF_REG_4)
+                                /* mov dst_reg, r11 */
+                                EMIT_mov(insn->dst_reg, AUX_REG);
+                        break;
+
                 case BPF_ALU | BPF_END | BPF_FROM_BE:
                         switch (imm32) {
                         case 16:
@@ -900,7 +930,7 @@ void bpf_int_jit_compile(struct bpf_prog *prog)
                 if (proglen <= 0) {
                         image = NULL;
                         if (header)
-                                module_free(NULL, header);
+                                bpf_jit_binary_free(header);
                         goto out;
                 }
                 if (image) {
@@ -910,7 +940,8 @@ void bpf_int_jit_compile(struct bpf_prog *prog)
                         break;
                 }
                 if (proglen == oldproglen) {
-                        header = bpf_alloc_binary(proglen, &image);
+                        header = bpf_jit_binary_alloc(proglen, &image,
+                                                      1, jit_fill_hole);
                         if (!header)
                                 goto out;
                 }
@@ -924,29 +955,23 @@ void bpf_int_jit_compile(struct bpf_prog *prog)
                 bpf_flush_icache(header, image + proglen);
                 set_memory_ro((unsigned long)header, header->pages);
                 prog->bpf_func = (void *)image;
-                prog->jited = 1;
+                prog->jited = true;
         }
 out:
         kfree(addrs);
 }
 
-static void bpf_jit_free_deferred(struct work_struct *work)
+void bpf_jit_free(struct bpf_prog *fp)
 {
-        struct bpf_prog *fp = container_of(work, struct bpf_prog, work);
         unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
         struct bpf_binary_header *header = (void *)addr;
 
+        if (!fp->jited)
+                goto free_filter;
+
         set_memory_rw(addr, header->pages);
-        module_free(NULL, header);
-        kfree(fp);
-}
+        bpf_jit_binary_free(header);
 
-void bpf_jit_free(struct bpf_prog *fp)
-{
-        if (fp->jited) {
-                INIT_WORK(&fp->work, bpf_jit_free_deferred);
-                schedule_work(&fp->work);
-        } else {
-                kfree(fp);
-        }
+free_filter:
+        bpf_prog_unlock_free(fp);
 }