5 files changed, 93 insertions, 32 deletions

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 8796ff03f472..f74ae68086dc 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -216,7 +216,7 @@ u64 bpf_tail_call(u64 ctx, u64 r2, u64 index, u64 r4, u64 r5);
 u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
 
 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
-void bpf_prog_calc_digest(struct bpf_prog *fp);
+int bpf_prog_calc_digest(struct bpf_prog *fp);
 
 const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
 
@@ -238,6 +238,8 @@ struct bpf_prog * __must_check bpf_prog_add(struct bpf_prog *prog, int i);
 void bpf_prog_sub(struct bpf_prog *prog, int i);
 struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog);
 void bpf_prog_put(struct bpf_prog *prog);
+int __bpf_prog_charge(struct user_struct *user, u32 pages);
+void __bpf_prog_uncharge(struct user_struct *user, u32 pages);
 
 struct bpf_map *bpf_map_get_with_uref(u32 ufd);
 struct bpf_map *__bpf_map_get(struct fd f);
@@ -318,6 +320,15 @@ static inline struct bpf_prog * __must_check bpf_prog_inc(struct bpf_prog *prog)
 {
         return ERR_PTR(-EOPNOTSUPP);
 }
+
+static inline int __bpf_prog_charge(struct user_struct *user, u32 pages)
+{
+        return 0;
+}
+
+static inline void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
+{
+}
 #endif /* CONFIG_BPF_SYSCALL */
 
 /* verifier prototypes for helper functions called from eBPF programs */
diff --git a/include/linux/filter.h b/include/linux/filter.h
index af8a1804cac6..702314253797 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -57,9 +57,6 @@ struct bpf_prog_aux;
 /* BPF program can access up to 512 bytes of stack space. */
 #define MAX_BPF_STACK   512
 
-/* Maximum BPF program size in bytes. */
-#define MAX_BPF_SIZE    (BPF_MAXINSNS * sizeof(struct bpf_insn))
-
 /* Helper macros for filter block array initializers. */
 
 /* ALU ops on registers, bpf_add|sub|...: dst_reg += src_reg */
@@ -517,6 +514,17 @@ static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
         return BPF_PROG_RUN(prog, xdp);
 }
 
+static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog)
+{
+        return prog->len * sizeof(struct bpf_insn);
+}
+
+static inline u32 bpf_prog_digest_scratch_size(const struct bpf_prog *prog)
+{
+        return round_up(bpf_prog_insn_size(prog) +
+                        sizeof(__be64) + 1, SHA_MESSAGE_BYTES);
+}
+
 static inline unsigned int bpf_prog_size(unsigned int proglen)
 {
         return max(sizeof(struct bpf_prog),
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 83e0d153b0b4..1eb4f1303756 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -105,19 +105,29 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
         gfp_t gfp_flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO |
                           gfp_extra_flags;
         struct bpf_prog *fp;
+        u32 pages, delta;
+        int ret;
 
         BUG_ON(fp_old == NULL);
 
         size = round_up(size, PAGE_SIZE);
-        if (size <= fp_old->pages * PAGE_SIZE)
+        pages = size / PAGE_SIZE;
+        if (pages <= fp_old->pages)
                 return fp_old;
 
+        delta = pages - fp_old->pages;
+        ret = __bpf_prog_charge(fp_old->aux->user, delta);
+        if (ret)
+                return NULL;
+
         fp = __vmalloc(size, gfp_flags, PAGE_KERNEL);
-        if (fp != NULL) {
+        if (fp == NULL) {
+                __bpf_prog_uncharge(fp_old->aux->user, delta);
+        } else {
                 kmemcheck_annotate_bitfield(fp, meta);
 
                 memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE);
-                fp->pages = size / PAGE_SIZE;
+                fp->pages = pages;
                 fp->aux->prog = fp;
 
                 /* We keep fp->aux from fp_old around in the new
@@ -136,28 +146,29 @@ void __bpf_prog_free(struct bpf_prog *fp)
         vfree(fp);
 }
 
-#define SHA_BPF_RAW_SIZE                                                \
-        round_up(MAX_BPF_SIZE + sizeof(__be64) + 1, SHA_MESSAGE_BYTES)
-
-/* Called under verifier mutex. */
-void bpf_prog_calc_digest(struct bpf_prog *fp)
+int bpf_prog_calc_digest(struct bpf_prog *fp)
 {
         const u32 bits_offset = SHA_MESSAGE_BYTES - sizeof(__be64);
-        static u32 ws[SHA_WORKSPACE_WORDS];
-        static u8 raw[SHA_BPF_RAW_SIZE];
-        struct bpf_insn *dst = (void *)raw;
+        u32 raw_size = bpf_prog_digest_scratch_size(fp);
+        u32 ws[SHA_WORKSPACE_WORDS];
         u32 i, bsize, psize, blocks;
+        struct bpf_insn *dst;
         bool was_ld_map;
-        u8 *todo = raw;
+        u8 *raw, *todo;
         __be32 *result;
         __be64 *bits;
 
+        raw = vmalloc(raw_size);
+        if (!raw)
+                return -ENOMEM;
+
         sha_init(fp->digest);
         memset(ws, 0, sizeof(ws));
 
         /* We need to take out the map fd for the digest calculation
          * since they are unstable from user space side.
          */
+        dst = (void *)raw;
         for (i = 0, was_ld_map = false; i < fp->len; i++) {
                 dst[i] = fp->insnsi[i];
                 if (!was_ld_map &&
@@ -177,12 +188,13 @@ void bpf_prog_calc_digest(struct bpf_prog *fp)
                 }
         }
 
-        psize = fp->len * sizeof(struct bpf_insn);
-        memset(&raw[psize], 0, sizeof(raw) - psize);
+        psize = bpf_prog_insn_size(fp);
+        memset(&raw[psize], 0, raw_size - psize);
         raw[psize++] = 0x80;
 
         bsize  = round_up(psize, SHA_MESSAGE_BYTES);
         blocks = bsize / SHA_MESSAGE_BYTES;
+        todo   = raw;
         if (bsize - psize >= sizeof(__be64)) {
                 bits = (__be64 *)(todo + bsize - sizeof(__be64));
         } else {
@@ -199,6 +211,9 @@ void bpf_prog_calc_digest(struct bpf_prog *fp)
         result = (__force __be32 *)fp->digest;
         for (i = 0; i < SHA_DIGEST_WORDS; i++)
                 result[i] = cpu_to_be32(fp->digest[i]);
+
+        vfree(raw);
+        return 0;
 }
 
 static bool bpf_is_jmp_and_has_target(const struct bpf_insn *insn)
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 4819ec9d95f6..e89acea22ecf 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -615,19 +615,39 @@ static void free_used_maps(struct bpf_prog_aux *aux)
         kfree(aux->used_maps);
 }
 
+int __bpf_prog_charge(struct user_struct *user, u32 pages)
+{
+        unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+        unsigned long user_bufs;
+
+        if (user) {
+                user_bufs = atomic_long_add_return(pages, &user->locked_vm);
+                if (user_bufs > memlock_limit) {
+                        atomic_long_sub(pages, &user->locked_vm);
+                        return -EPERM;
+                }
+        }
+
+        return 0;
+}
+
+void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
+{
+        if (user)
+                atomic_long_sub(pages, &user->locked_vm);
+}
+
 static int bpf_prog_charge_memlock(struct bpf_prog *prog)
 {
         struct user_struct *user = get_current_user();
-        unsigned long memlock_limit;
-
-        memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+        int ret;
 
-        atomic_long_add(prog->pages, &user->locked_vm);
-        if (atomic_long_read(&user->locked_vm) > memlock_limit) {
-                atomic_long_sub(prog->pages, &user->locked_vm);
+        ret = __bpf_prog_charge(user, prog->pages);
+        if (ret) {
                 free_uid(user);
-                return -EPERM;
+                return ret;
         }
+
         prog->aux->user = user;
         return 0;
 }
@@ -636,7 +656,7 @@ static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
 {
         struct user_struct *user = prog->aux->user;
 
-        atomic_long_sub(prog->pages, &user->locked_vm);
+        __bpf_prog_uncharge(user, prog->pages);
         free_uid(user);
 }
 
@@ -811,7 +831,7 @@ static int bpf_prog_load(union bpf_attr *attr)
 
         err = -EFAULT;
         if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
-                           prog->len * sizeof(struct bpf_insn)) != 0)
+                           bpf_prog_insn_size(prog)) != 0)
                 goto free_prog;
 
         prog->orig_prog = NULL;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 81e267bc4640..83ed2f8f6f22 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -462,14 +462,19 @@ static void init_reg_state(struct bpf_reg_state *regs)
         regs[BPF_REG_1].type = PTR_TO_CTX;
 }
 
-static void mark_reg_unknown_value(struct bpf_reg_state *regs, u32 regno)
+static void __mark_reg_unknown_value(struct bpf_reg_state *regs, u32 regno)
 {
-        BUG_ON(regno >= MAX_BPF_REG);
         regs[regno].type = UNKNOWN_VALUE;
         regs[regno].id = 0;
         regs[regno].imm = 0;
 }
 
+static void mark_reg_unknown_value(struct bpf_reg_state *regs, u32 regno)
+{
+        BUG_ON(regno >= MAX_BPF_REG);
+        __mark_reg_unknown_value(regs, regno);
+}
+
 static void reset_reg_range_values(struct bpf_reg_state *regs, u32 regno)
 {
         regs[regno].min_value = BPF_REGISTER_MIN_RANGE;
@@ -1976,7 +1981,7 @@ static void mark_map_reg(struct bpf_reg_state *regs, u32 regno, u32 id,
                  */
                 reg->id = 0;
                 if (type == UNKNOWN_VALUE)
-                        mark_reg_unknown_value(regs, regno);
+                        __mark_reg_unknown_value(regs, regno);
         }
 }
 
@@ -2931,6 +2936,10 @@ static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env)
         int insn_cnt = env->prog->len;
         int i, j, err;
 
+        err = bpf_prog_calc_digest(env->prog);
+        if (err)
+                return err;
+
         for (i = 0; i < insn_cnt; i++, insn++) {
                 if (BPF_CLASS(insn->code) == BPF_LDX &&
                     (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) {
@@ -3178,8 +3187,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr)
                 log_level = 0;
         }
 
-        bpf_prog_calc_digest(env->prog);
-
         ret = replace_map_fd_with_map_ptr(env);
         if (ret < 0)
                 goto skip_full_check;