8 files changed, 210 insertions, 116 deletions

diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index a5ae60f0b0a2..e6983be12bd3 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -1,5 +1,2 @@
 obj-y := core.o
 obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o hashtab.o arraymap.o helpers.o
-ifdef CONFIG_TEST_BPF
-obj-$(CONFIG_BPF_SYSCALL) += test_stub.o
-endif
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 9eb4d8a7cd87..8a6616583f38 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -134,7 +134,7 @@ static void array_map_free(struct bpf_map *map)
         kvfree(array);
 }
 
-static struct bpf_map_ops array_ops = {
+static const struct bpf_map_ops array_ops = {
         .map_alloc = array_map_alloc,
         .map_free = array_map_free,
         .map_get_next_key = array_map_get_next_key,
@@ -143,14 +143,14 @@ static struct bpf_map_ops array_ops = {
         .map_delete_elem = array_map_delete_elem,
 };
 
-static struct bpf_map_type_list tl = {
+static struct bpf_map_type_list array_type __read_mostly = {
         .ops = &array_ops,
         .type = BPF_MAP_TYPE_ARRAY,
 };
 
 static int __init register_array_map(void)
 {
-        bpf_register_map_type(&tl);
+        bpf_register_map_type(&array_type);
         return 0;
 }
 late_initcall(register_array_map);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index a64e7a207d2b..4139a0f8b558 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -656,6 +656,14 @@ void bpf_prog_free(struct bpf_prog *fp)
 }
 EXPORT_SYMBOL_GPL(bpf_prog_free);
 
+/* Weak definitions of helper functions in case we don't have bpf syscall. */
+const struct bpf_func_proto bpf_map_lookup_elem_proto __weak;
+const struct bpf_func_proto bpf_map_update_elem_proto __weak;
+const struct bpf_func_proto bpf_map_delete_elem_proto __weak;
+
+const struct bpf_func_proto bpf_get_prandom_u32_proto __weak;
+const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak;
+
 /* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
  * skb_copy_bits(), so provide a weak definition of it for NET-less config.
  */
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index b3ba43674310..83c209d9b17a 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -345,7 +345,7 @@ static void htab_map_free(struct bpf_map *map)
         kfree(htab);
 }
 
-static struct bpf_map_ops htab_ops = {
+static const struct bpf_map_ops htab_ops = {
         .map_alloc = htab_map_alloc,
         .map_free = htab_map_free,
         .map_get_next_key = htab_map_get_next_key,
@@ -354,14 +354,14 @@ static struct bpf_map_ops htab_ops = {
         .map_delete_elem = htab_map_delete_elem,
 };
 
-static struct bpf_map_type_list tl = {
+static struct bpf_map_type_list htab_type __read_mostly = {
         .ops = &htab_ops,
         .type = BPF_MAP_TYPE_HASH,
 };
 
 static int __init register_htab_map(void)
 {
-        bpf_register_map_type(&tl);
+        bpf_register_map_type(&htab_type);
         return 0;
 }
 late_initcall(register_htab_map);
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 9e3414d85459..bd7f5988ed9c 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -11,6 +11,8 @@
  */
 #include <linux/bpf.h>
 #include <linux/rcupdate.h>
+#include <linux/random.h>
+#include <linux/smp.h>
 
 /* If kernel subsystem is allowing eBPF programs to call this function,
  * inside its own verifier_ops->get_func_proto() callback it should return
@@ -41,7 +43,7 @@ static u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
         return (unsigned long) value;
 }
 
-struct bpf_func_proto bpf_map_lookup_elem_proto = {
+const struct bpf_func_proto bpf_map_lookup_elem_proto = {
         .func = bpf_map_lookup_elem,
         .gpl_only = false,
         .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
@@ -60,7 +62,7 @@ static u64 bpf_map_update_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
         return map->ops->map_update_elem(map, key, value, r4);
 }
 
-struct bpf_func_proto bpf_map_update_elem_proto = {
+const struct bpf_func_proto bpf_map_update_elem_proto = {
         .func = bpf_map_update_elem,
         .gpl_only = false,
         .ret_type = RET_INTEGER,
@@ -80,10 +82,32 @@ static u64 bpf_map_delete_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
         return map->ops->map_delete_elem(map, key);
 }
 
-struct bpf_func_proto bpf_map_delete_elem_proto = {
+const struct bpf_func_proto bpf_map_delete_elem_proto = {
         .func = bpf_map_delete_elem,
         .gpl_only = false,
         .ret_type = RET_INTEGER,
         .arg1_type = ARG_CONST_MAP_PTR,
         .arg2_type = ARG_PTR_TO_MAP_KEY,
 };
+
+static u64 bpf_get_prandom_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
+{
+        return prandom_u32();
+}
+
+const struct bpf_func_proto bpf_get_prandom_u32_proto = {
+        .func           = bpf_get_prandom_u32,
+        .gpl_only       = false,
+        .ret_type       = RET_INTEGER,
+};
+
+static u64 bpf_get_smp_processor_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
+{
+        return raw_smp_processor_id();
+}
+
+const struct bpf_func_proto bpf_get_smp_processor_id_proto = {
+        .func           = bpf_get_smp_processor_id,
+        .gpl_only       = false,
+        .ret_type       = RET_INTEGER,
+};
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 536edc2be307..3bae6c591914 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -16,6 +16,7 @@
 #include <linux/file.h>
 #include <linux/license.h>
 #include <linux/filter.h>
+#include <linux/version.h>
 
 static LIST_HEAD(bpf_map_types);
 
@@ -354,10 +355,11 @@ static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
         list_for_each_entry(tl, &bpf_prog_types, list_node) {
                 if (tl->type == type) {
                         prog->aux->ops = tl->ops;
-                        prog->aux->prog_type = type;
+                        prog->type = type;
                         return 0;
                 }
         }
+
         return -EINVAL;
 }
 
@@ -418,6 +420,7 @@ void bpf_prog_put(struct bpf_prog *prog)
                 bpf_prog_free(prog);
         }
 }
+EXPORT_SYMBOL_GPL(bpf_prog_put);
 
 static int bpf_prog_release(struct inode *inode, struct file *filp)
 {
@@ -465,9 +468,10 @@ struct bpf_prog *bpf_prog_get(u32 ufd)
         fdput(f);
         return prog;
 }
+EXPORT_SYMBOL_GPL(bpf_prog_get);
 
 /* last field in 'union bpf_attr' used by this command */
-#define BPF_PROG_LOAD_LAST_FIELD log_buf
+#define BPF_PROG_LOAD_LAST_FIELD kern_version
 
 static int bpf_prog_load(union bpf_attr *attr)
 {
@@ -492,6 +496,10 @@ static int bpf_prog_load(union bpf_attr *attr)
         if (attr->insn_cnt >= BPF_MAXINSNS)
                 return -EINVAL;
 
+        if (type == BPF_PROG_TYPE_KPROBE &&
+            attr->kern_version != LINUX_VERSION_CODE)
+                return -EINVAL;
+
         /* plain bpf_prog allocation */
         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
         if (!prog)
@@ -508,7 +516,7 @@ static int bpf_prog_load(union bpf_attr *attr)
         prog->jited = false;
 
         atomic_set(&prog->aux->refcnt, 1);
-        prog->aux->is_gpl_compatible = is_gpl;
+        prog->gpl_compatible = is_gpl;
 
         /* find program type: socket_filter vs tracing_filter */
         err = find_prog_type(type, prog);
@@ -516,8 +524,7 @@ static int bpf_prog_load(union bpf_attr *attr)
                 goto free_prog;
 
         /* run eBPF verifier */
-        err = bpf_check(prog, attr);
-
+        err = bpf_check(&prog, attr);
         if (err < 0)
                 goto free_used_maps;
 
@@ -528,7 +535,6 @@ static int bpf_prog_load(union bpf_attr *attr)
         bpf_prog_select_runtime(prog);
 
         err = anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog, O_RDWR | O_CLOEXEC);
-
         if (err < 0)
                 /* failed to allocate fd */
                 goto free_used_maps;
diff --git a/kernel/bpf/test_stub.c b/kernel/bpf/test_stub.c
deleted file mode 100644
index 0ceae1e6e8b5..000000000000
--- a/kernel/bpf/test_stub.c
+++ /dev/null
@@ -1,78 +0,0 @@
-/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of version 2 of the GNU General Public
- * License as published by the Free Software Foundation.
- */
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/slab.h>
-#include <linux/err.h>
-#include <linux/bpf.h>
-
-/* test stubs for BPF_MAP_TYPE_UNSPEC and for BPF_PROG_TYPE_UNSPEC
- * to be used by user space verifier testsuite
- */
-struct bpf_context {
-        u64 arg1;
-        u64 arg2;
-};
-
-static const struct bpf_func_proto *test_func_proto(enum bpf_func_id func_id)
-{
-        switch (func_id) {
-        case BPF_FUNC_map_lookup_elem:
-                return &bpf_map_lookup_elem_proto;
-        case BPF_FUNC_map_update_elem:
-                return &bpf_map_update_elem_proto;
-        case BPF_FUNC_map_delete_elem:
-                return &bpf_map_delete_elem_proto;
-        default:
-                return NULL;
-        }
-}
-
-static const struct bpf_context_access {
-        int size;
-        enum bpf_access_type type;
-} test_ctx_access[] = {
-        [offsetof(struct bpf_context, arg1)] = {
-                FIELD_SIZEOF(struct bpf_context, arg1),
-                BPF_READ
-        },
-        [offsetof(struct bpf_context, arg2)] = {
-                FIELD_SIZEOF(struct bpf_context, arg2),
-                BPF_READ
-        },
-};
-
-static bool test_is_valid_access(int off, int size, enum bpf_access_type type)
-{
-        const struct bpf_context_access *access;
-
-        if (off < 0 || off >= ARRAY_SIZE(test_ctx_access))
-                return false;
-
-        access = &test_ctx_access[off];
-        if (access->size == size && (access->type & type))
-                return true;
-
-        return false;
-}
-
-static struct bpf_verifier_ops test_ops = {
-        .get_func_proto = test_func_proto,
-        .is_valid_access = test_is_valid_access,
-};
-
-static struct bpf_prog_type_list tl_prog = {
-        .ops = &test_ops,
-        .type = BPF_PROG_TYPE_UNSPEC,
-};
-
-static int __init register_test_ops(void)
-{
-        bpf_register_prog_type(&tl_prog);
-        return 0;
-}
-late_initcall(register_test_ops);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index a28e09c7825d..630a7bac1e51 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -755,7 +755,7 @@ static int check_func_arg(struct verifier_env *env, u32 regno,
         enum bpf_reg_type expected_type;
         int err = 0;
 
-        if (arg_type == ARG_ANYTHING)
+        if (arg_type == ARG_DONTCARE)
                 return 0;
 
         if (reg->type == NOT_INIT) {
@@ -763,6 +763,9 @@ static int check_func_arg(struct verifier_env *env, u32 regno,
                 return -EACCES;
         }
 
+        if (arg_type == ARG_ANYTHING)
+                return 0;
+
         if (arg_type == ARG_PTR_TO_STACK || arg_type == ARG_PTR_TO_MAP_KEY ||
             arg_type == ARG_PTR_TO_MAP_VALUE) {
                 expected_type = PTR_TO_STACK;
@@ -770,6 +773,8 @@ static int check_func_arg(struct verifier_env *env, u32 regno,
                 expected_type = CONST_IMM;
         } else if (arg_type == ARG_CONST_MAP_PTR) {
                 expected_type = CONST_PTR_TO_MAP;
+        } else if (arg_type == ARG_PTR_TO_CTX) {
+                expected_type = PTR_TO_CTX;
         } else {
                 verbose("unsupported arg_type %d\n", arg_type);
                 return -EFAULT;
@@ -852,7 +857,7 @@ static int check_call(struct verifier_env *env, int func_id)
         }
 
         /* eBPF programs must be GPL compatible to use GPL-ed functions */
-        if (!env->prog->aux->is_gpl_compatible && fn->gpl_only) {
+        if (!env->prog->gpl_compatible && fn->gpl_only) {
                 verbose("cannot call GPL only function from proprietary program\n");
                 return -EINVAL;
         }
@@ -1172,6 +1177,18 @@ static int check_ld_imm(struct verifier_env *env, struct bpf_insn *insn)
         return 0;
 }
 
+static bool may_access_skb(enum bpf_prog_type type)
+{
+        switch (type) {
+        case BPF_PROG_TYPE_SOCKET_FILTER:
+        case BPF_PROG_TYPE_SCHED_CLS:
+        case BPF_PROG_TYPE_SCHED_ACT:
+                return true;
+        default:
+                return false;
+        }
+}
+
 /* verify safety of LD_ABS|LD_IND instructions:
  * - they can only appear in the programs where ctx == skb
  * - since they are wrappers of function calls, they scratch R1-R5 registers,
@@ -1194,8 +1211,8 @@ static int check_ld_abs(struct verifier_env *env, struct bpf_insn *insn)
         struct reg_state *reg;
         int i, err;
 
-        if (env->prog->aux->prog_type != BPF_PROG_TYPE_SOCKET_FILTER) {
-                verbose("BPF_LD_ABS|IND instructions are only allowed in socket filters\n");
+        if (!may_access_skb(env->prog->type)) {
+                verbose("BPF_LD_ABS|IND instructions not allowed for this program type\n");
                 return -EINVAL;
         }
 
@@ -1606,11 +1623,10 @@ static int do_check(struct verifier_env *env)
                                 return err;
 
                 } else if (class == BPF_LDX) {
-                        if (BPF_MODE(insn->code) != BPF_MEM ||
-                            insn->imm != 0) {
-                                verbose("BPF_LDX uses reserved fields\n");
-                                return -EINVAL;
-                        }
+                        enum bpf_reg_type src_reg_type;
+
+                        /* check for reserved fields is already done */
+
                         /* check src operand */
                         err = check_reg_arg(regs, insn->src_reg, SRC_OP);
                         if (err)
@@ -1629,6 +1645,29 @@ static int do_check(struct verifier_env *env)
                         if (err)
                                 return err;
 
+                        src_reg_type = regs[insn->src_reg].type;
+
+                        if (insn->imm == 0 && BPF_SIZE(insn->code) == BPF_W) {
+                                /* saw a valid insn
+                                 * dst_reg = *(u32 *)(src_reg + off)
+                                 * use reserved 'imm' field to mark this insn
+                                 */
+                                insn->imm = src_reg_type;
+
+                        } else if (src_reg_type != insn->imm &&
+                                   (src_reg_type == PTR_TO_CTX ||
+                                    insn->imm == PTR_TO_CTX)) {
+                                /* ABuser program is trying to use the same insn
+                                 * dst_reg = *(u32*) (src_reg + off)
+                                 * with different pointer types:
+                                 * src_reg == ctx in one branch and
+                                 * src_reg == stack|map in some other branch.
+                                 * Reject it.
+                                 */
+                                verbose("same insn cannot be used with different pointers\n");
+                                return -EINVAL;
+                        }
+
                 } else if (class == BPF_STX) {
                         if (BPF_MODE(insn->code) == BPF_XADD) {
                                 err = check_xadd(env, insn);
@@ -1776,6 +1815,13 @@ static int replace_map_fd_with_map_ptr(struct verifier_env *env)
         int i, j;
 
         for (i = 0; i < insn_cnt; i++, insn++) {
+                if (BPF_CLASS(insn->code) == BPF_LDX &&
+                    (BPF_MODE(insn->code) != BPF_MEM ||
+                     insn->imm != 0)) {
+                        verbose("BPF_LDX uses reserved fields\n");
+                        return -EINVAL;
+                }
+
                 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
                         struct bpf_map *map;
                         struct fd f;
@@ -1867,6 +1913,92 @@ static void convert_pseudo_ld_imm64(struct verifier_env *env)
                         insn->src_reg = 0;
 }
 
+static void adjust_branches(struct bpf_prog *prog, int pos, int delta)
+{
+        struct bpf_insn *insn = prog->insnsi;
+        int insn_cnt = prog->len;
+        int i;
+
+        for (i = 0; i < insn_cnt; i++, insn++) {
+                if (BPF_CLASS(insn->code) != BPF_JMP ||
+                    BPF_OP(insn->code) == BPF_CALL ||
+                    BPF_OP(insn->code) == BPF_EXIT)
+                        continue;
+
+                /* adjust offset of jmps if necessary */
+                if (i < pos && i + insn->off + 1 > pos)
+                        insn->off += delta;
+                else if (i > pos && i + insn->off + 1 < pos)
+                        insn->off -= delta;
+        }
+}
+
+/* convert load instructions that access fields of 'struct __sk_buff'
+ * into sequence of instructions that access fields of 'struct sk_buff'
+ */
+static int convert_ctx_accesses(struct verifier_env *env)
+{
+        struct bpf_insn *insn = env->prog->insnsi;
+        int insn_cnt = env->prog->len;
+        struct bpf_insn insn_buf[16];
+        struct bpf_prog *new_prog;
+        u32 cnt;
+        int i;
+
+        if (!env->prog->aux->ops->convert_ctx_access)
+                return 0;
+
+        for (i = 0; i < insn_cnt; i++, insn++) {
+                if (insn->code != (BPF_LDX | BPF_MEM | BPF_W))
+                        continue;
+
+                if (insn->imm != PTR_TO_CTX) {
+                        /* clear internal mark */
+                        insn->imm = 0;
+                        continue;
+                }
+
+                cnt = env->prog->aux->ops->
+                        convert_ctx_access(insn->dst_reg, insn->src_reg,
+                                           insn->off, insn_buf);
+                if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
+                        verbose("bpf verifier is misconfigured\n");
+                        return -EINVAL;
+                }
+
+                if (cnt == 1) {
+                        memcpy(insn, insn_buf, sizeof(*insn));
+                        continue;
+                }
+
+                /* several new insns need to be inserted. Make room for them */
+                insn_cnt += cnt - 1;
+                new_prog = bpf_prog_realloc(env->prog,
+                                            bpf_prog_size(insn_cnt),
+                                            GFP_USER);
+                if (!new_prog)
+                        return -ENOMEM;
+
+                new_prog->len = insn_cnt;
+
+                memmove(new_prog->insnsi + i + cnt, new_prog->insns + i + 1,
+                        sizeof(*insn) * (insn_cnt - i - cnt));
+
+                /* copy substitute insns in place of load instruction */
+                memcpy(new_prog->insnsi + i, insn_buf, sizeof(*insn) * cnt);
+
+                /* adjust branches in the whole program */
+                adjust_branches(new_prog, i, cnt - 1);
+
+                /* keep walking new program and skip insns we just inserted */
+                env->prog = new_prog;
+                insn = new_prog->insnsi + i + cnt - 1;
+                i += cnt - 1;
+        }
+
+        return 0;
+}
+
 static void free_states(struct verifier_env *env)
 {
         struct verifier_state_list *sl, *sln;
@@ -1889,13 +2021,13 @@ static void free_states(struct verifier_env *env)
         kfree(env->explored_states);
 }
 
-int bpf_check(struct bpf_prog *prog, union bpf_attr *attr)
+int bpf_check(struct bpf_prog **prog, union bpf_attr *attr)
 {
         char __user *log_ubuf = NULL;
         struct verifier_env *env;
         int ret = -EINVAL;
 
-        if (prog->len <= 0 || prog->len > BPF_MAXINSNS)
+        if ((*prog)->len <= 0 || (*prog)->len > BPF_MAXINSNS)
                 return -E2BIG;
 
         /* 'struct verifier_env' can be global, but since it's not small,
@@ -1905,7 +2037,7 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr *attr)
         if (!env)
                 return -ENOMEM;
 
-        env->prog = prog;
+        env->prog = *prog;
 
         /* grab the mutex to protect few globals used by verifier */
         mutex_lock(&bpf_verifier_lock);
@@ -1937,7 +2069,7 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr *attr)
         if (ret < 0)
                 goto skip_full_check;
 
-        env->explored_states = kcalloc(prog->len,
+        env->explored_states = kcalloc(env->prog->len,
                                        sizeof(struct verifier_state_list *),
                                        GFP_USER);
         ret = -ENOMEM;
@@ -1954,6 +2086,10 @@ skip_full_check:
         while (pop_stack(env, NULL) >= 0);
         free_states(env);
 
+        if (ret == 0)
+                /* program is valid, convert *(u32*)(ctx + off) accesses */
+                ret = convert_ctx_accesses(env);
+
         if (log_level && log_len >= log_size - 1) {
                 BUG_ON(log_len >= log_size);
                 /* verifier log exceeded user supplied buffer */
@@ -1969,18 +2105,18 @@ skip_full_check:
 
         if (ret == 0 && env->used_map_cnt) {
                 /* if program passed verifier, update used_maps in bpf_prog_info */
-                prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
-                                                     sizeof(env->used_maps[0]),
-                                                     GFP_KERNEL);
+                env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
+                                                          sizeof(env->used_maps[0]),
+                                                          GFP_KERNEL);
 
-                if (!prog->aux->used_maps) {
+                if (!env->prog->aux->used_maps) {
                         ret = -ENOMEM;
                         goto free_log_buf;
                 }
 
-                memcpy(prog->aux->used_maps, env->used_maps,
+                memcpy(env->prog->aux->used_maps, env->used_maps,
                        sizeof(env->used_maps[0]) * env->used_map_cnt);
-                prog->aux->used_map_cnt = env->used_map_cnt;
+                env->prog->aux->used_map_cnt = env->used_map_cnt;
 
                 /* program is valid. Convert pseudo bpf_ld_imm64 into generic
                  * bpf_ld_imm64 instructions
@@ -1992,11 +2128,12 @@ free_log_buf:
         if (log_level)
                 vfree(log_buf);
 free_env:
-        if (!prog->aux->used_maps)
+        if (!env->prog->aux->used_maps)
                 /* if we didn't copy map pointers into bpf_prog_info, release
                  * them now. Otherwise free_bpf_prog_info() will release them.
                  */
                 release_maps(env);
+        *prog = env->prog;
         kfree(env);
         mutex_unlock(&bpf_verifier_lock);
         return ret;