bpf: add bpf_patch_insn_single helper

Move the functionality to patch instructions out of the verifier
code and into the core as the new bpf_patch_insn_single() helper
will be needed later on for blinding as well. No changes in
functionality.

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>

2 files changed, 80 insertions, 44 deletions

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 5313d09d4b62..49b5538a5301 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -136,6 +136,77 @@ void __bpf_prog_free(struct bpf_prog *fp)
         vfree(fp);
 }
 
+static bool bpf_is_jmp_and_has_target(const struct bpf_insn *insn)
+{
+        return BPF_CLASS(insn->code) == BPF_JMP  &&
+               /* Call and Exit are both special jumps with no
+                * target inside the BPF instruction image.
+                */
+               BPF_OP(insn->code) != BPF_CALL &&
+               BPF_OP(insn->code) != BPF_EXIT;
+}
+
+static void bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta)
+{
+        struct bpf_insn *insn = prog->insnsi;
+        u32 i, insn_cnt = prog->len;
+
+        for (i = 0; i < insn_cnt; i++, insn++) {
+                if (!bpf_is_jmp_and_has_target(insn))
+                        continue;
+
+                /* Adjust offset of jmps if we cross boundaries. */
+                if (i < pos && i + insn->off + 1 > pos)
+                        insn->off += delta;
+                else if (i > pos + delta && i + insn->off + 1 <= pos + delta)
+                        insn->off -= delta;
+        }
+}
+
+struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
+                                       const struct bpf_insn *patch, u32 len)
+{
+        u32 insn_adj_cnt, insn_rest, insn_delta = len - 1;
+        struct bpf_prog *prog_adj;
+
+        /* Since our patchlet doesn't expand the image, we're done. */
+        if (insn_delta == 0) {
+                memcpy(prog->insnsi + off, patch, sizeof(*patch));
+                return prog;
+        }
+
+        insn_adj_cnt = prog->len + insn_delta;
+
+        /* Several new instructions need to be inserted. Make room
+         * for them. Likely, there's no need for a new allocation as
+         * last page could have large enough tailroom.
+         */
+        prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt),
+                                    GFP_USER);
+        if (!prog_adj)
+                return NULL;
+
+        prog_adj->len = insn_adj_cnt;
+
+        /* Patching happens in 3 steps:
+         *
+         * 1) Move over tail of insnsi from next instruction onwards,
+         *    so we can patch the single target insn with one or more
+         *    new ones (patching is always from 1 to n insns, n > 0).
+         * 2) Inject new instructions at the target location.
+         * 3) Adjust branch offsets if necessary.
+         */
+        insn_rest = insn_adj_cnt - off - len;
+
+        memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1,
+                sizeof(*patch) * insn_rest);
+        memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len);
+
+        bpf_adj_branches(prog_adj, off, insn_delta);
+
+        return prog_adj;
+}
+
 #ifdef CONFIG_BPF_JIT
 struct bpf_binary_header *
 bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 84bff68cf80e..a08d66215245 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -2587,26 +2587,6 @@ static void convert_pseudo_ld_imm64(struct verifier_env *env)
                         insn->src_reg = 0;
 }
 
-static void adjust_branches(struct bpf_prog *prog, int pos, int delta)
-{
-        struct bpf_insn *insn = prog->insnsi;
-        int insn_cnt = prog->len;
-        int i;
-
-        for (i = 0; i < insn_cnt; i++, insn++) {
-                if (BPF_CLASS(insn->code) != BPF_JMP ||
-                    BPF_OP(insn->code) == BPF_CALL ||
-                    BPF_OP(insn->code) == BPF_EXIT)
-                        continue;
-
-                /* adjust offset of jmps if necessary */
-                if (i < pos && i + insn->off + 1 > pos)
-                        insn->off += delta;
-                else if (i > pos + delta && i + insn->off + 1 <= pos + delta)
-                        insn->off -= delta;
-        }
-}
-
 /* convert load instructions that access fields of 'struct __sk_buff'
  * into sequence of instructions that access fields of 'struct sk_buff'
  */
@@ -2616,14 +2596,15 @@ static int convert_ctx_accesses(struct verifier_env *env)
         int insn_cnt = env->prog->len;
         struct bpf_insn insn_buf[16];
         struct bpf_prog *new_prog;
-        u32 cnt;
-        int i;
         enum bpf_access_type type;
+        int i;
 
         if (!env->prog->aux->ops->convert_ctx_access)
                 return 0;
 
         for (i = 0; i < insn_cnt; i++, insn++) {
+                u32 insn_delta, cnt;
+
                 if (insn->code == (BPF_LDX | BPF_MEM | BPF_W))
                         type = BPF_READ;
                 else if (insn->code == (BPF_STX | BPF_MEM | BPF_W))
@@ -2645,34 +2626,18 @@ static int convert_ctx_accesses(struct verifier_env *env)
                         return -EINVAL;
                 }
 
-                if (cnt == 1) {
-                        memcpy(insn, insn_buf, sizeof(*insn));
-                        continue;
-                }
-
-                /* several new insns need to be inserted. Make room for them */
-                insn_cnt += cnt - 1;
-                new_prog = bpf_prog_realloc(env->prog,
-                                            bpf_prog_size(insn_cnt),
-                                            GFP_USER);
+                new_prog = bpf_patch_insn_single(env->prog, i, insn_buf, cnt);
                 if (!new_prog)
                         return -ENOMEM;
 
-                new_prog->len = insn_cnt;
-
-                memmove(new_prog->insnsi + i + cnt, new_prog->insns + i + 1,
-                        sizeof(*insn) * (insn_cnt - i - cnt));
-
-                /* copy substitute insns in place of load instruction */
-                memcpy(new_prog->insnsi + i, insn_buf, sizeof(*insn) * cnt);
-
-                /* adjust branches in the whole program */
-                adjust_branches(new_prog, i, cnt - 1);
+                insn_delta = cnt - 1;
 
                 /* keep walking new program and skip insns we just inserted */
                 env->prog = new_prog;
-                insn = new_prog->insnsi + i + cnt - 1;
-                i += cnt - 1;
+                insn      = new_prog->insnsi + i + insn_delta;
+
+                insn_cnt += insn_delta;
+                i        += insn_delta;
         }
 
         return 0;