Merge branch 'bpf-prog-stats'

Alexei Starovoitov says:

====================
Introduce per program stats to monitor the usage BPF.

v2->v3:
- rename to run_time_ns/run_cnt everywhere

v1->v2:
- fixed u64 stats on 32-bit archs. Thanks Eric
- use more verbose run_time_ns in json output as suggested by Andrii
- refactored prog_alloc and clarified behavior of stats in subprogs
====================

Acked-by: Andrii Nakryiko <andriin@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

4 files changed, 106 insertions, 5 deletions

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index ef88b167959d..1c14c347f3cf 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -78,7 +78,7 @@ void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, uns
         return NULL;
 }
 
-struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)
+struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags)
 {
         gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
         struct bpf_prog_aux *aux;
@@ -104,6 +104,26 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)
 
         return fp;
 }
+
+struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)
+{
+        gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags;
+        struct bpf_prog *prog;
+
+        prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags);
+        if (!prog)
+                return NULL;
+
+        prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
+        if (!prog->aux->stats) {
+                kfree(prog->aux);
+                vfree(prog);
+                return NULL;
+        }
+
+        u64_stats_init(&prog->aux->stats->syncp);
+        return prog;
+}
 EXPORT_SYMBOL_GPL(bpf_prog_alloc);
 
 int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)
@@ -231,7 +251,10 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
 
 void __bpf_prog_free(struct bpf_prog *fp)
 {
-        kfree(fp->aux);
+        if (fp->aux) {
+                free_percpu(fp->aux->stats);
+                kfree(fp->aux);
+        }
         vfree(fp);
 }
 
@@ -2069,6 +2092,10 @@ int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to,
         return -EFAULT;
 }
 
+DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key);
+EXPORT_SYMBOL(bpf_stats_enabled_key);
+int sysctl_bpf_stats_enabled __read_mostly;
+
 /* All definitions of tracepoints related to BPF. */
 #define CREATE_TRACE_POINTS
 #include <linux/bpf_trace.h>
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index ec7c552af76b..174581dfe225 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1283,24 +1283,54 @@ static int bpf_prog_release(struct inode *inode, struct file *filp)
         return 0;
 }
 
+static void bpf_prog_get_stats(const struct bpf_prog *prog,
+                               struct bpf_prog_stats *stats)
+{
+        u64 nsecs = 0, cnt = 0;
+        int cpu;
+
+        for_each_possible_cpu(cpu) {
+                const struct bpf_prog_stats *st;
+                unsigned int start;
+                u64 tnsecs, tcnt;
+
+                st = per_cpu_ptr(prog->aux->stats, cpu);
+                do {
+                        start = u64_stats_fetch_begin_irq(&st->syncp);
+                        tnsecs = st->nsecs;
+                        tcnt = st->cnt;
+                } while (u64_stats_fetch_retry_irq(&st->syncp, start));
+                nsecs += tnsecs;
+                cnt += tcnt;
+        }
+        stats->nsecs = nsecs;
+        stats->cnt = cnt;
+}
+
 #ifdef CONFIG_PROC_FS
 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
 {
         const struct bpf_prog *prog = filp->private_data;
         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
+        struct bpf_prog_stats stats;
 
+        bpf_prog_get_stats(prog, &stats);
         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
         seq_printf(m,
                    "prog_type:\t%u\n"
                    "prog_jited:\t%u\n"
                    "prog_tag:\t%s\n"
                    "memlock:\t%llu\n"
-                   "prog_id:\t%u\n",
+                   "prog_id:\t%u\n"
+                   "run_time_ns:\t%llu\n"
+                   "run_cnt:\t%llu\n",
                    prog->type,
                    prog->jited,
                    prog_tag,
                    prog->pages * 1ULL << PAGE_SHIFT,
-                   prog->aux->id);
+                   prog->aux->id,
+                   stats.nsecs,
+                   stats.cnt);
 }
 #endif
 
@@ -2122,6 +2152,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
         struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
         struct bpf_prog_info info = {};
         u32 info_len = attr->info.info_len;
+        struct bpf_prog_stats stats;
         char __user *uinsns;
         u32 ulen;
         int err;
@@ -2161,6 +2192,10 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
         if (err)
                 return err;
 
+        bpf_prog_get_stats(prog, &stats);
+        info.run_time_ns = stats.nsecs;
+        info.run_cnt = stats.cnt;
+
         if (!capable(CAP_SYS_ADMIN)) {
                 info.jited_prog_len = 0;
                 info.xlated_prog_len = 0;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 1b9496c41383..0e4edd7e3c5f 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -7320,7 +7320,12 @@ static int jit_subprogs(struct bpf_verifier_env *env)
                 subprog_end = env->subprog_info[i + 1].start;
 
                 len = subprog_end - subprog_start;
-                func[i] = bpf_prog_alloc(bpf_prog_size(len), GFP_USER);
+                /* BPF_PROG_RUN doesn't call subprogs directly,
+                 * hence main prog stats include the runtime of subprogs.
+                 * subprogs don't have IDs and not reachable via prog_get_next_id
+                 * func[i]->aux->stats will never be accessed and stays NULL
+                 */
+                func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
                 if (!func[i])
                         goto out_free;
                 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index ba4d9e85feb8..86e0771352f2 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -224,6 +224,9 @@ static int proc_dostring_coredump(struct ctl_table *table, int write,
 #endif
 static int proc_dopipe_max_size(struct ctl_table *table, int write,
                 void __user *buffer, size_t *lenp, loff_t *ppos);
+static int proc_dointvec_minmax_bpf_stats(struct ctl_table *table, int write,
+                                          void __user *buffer, size_t *lenp,
+                                          loff_t *ppos);
 
 #ifdef CONFIG_MAGIC_SYSRQ
 /* Note: sysrq code uses its own private copy */
@@ -1230,6 +1233,15 @@ static struct ctl_table kern_table[] = {
                 .extra2         = &one,
         },
 #endif
+        {
+                .procname       = "bpf_stats_enabled",
+                .data           = &sysctl_bpf_stats_enabled,
+                .maxlen         = sizeof(sysctl_bpf_stats_enabled),
+                .mode           = 0644,
+                .proc_handler   = proc_dointvec_minmax_bpf_stats,
+                .extra1         = &zero,
+                .extra2         = &one,
+        },
 #if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU)
         {
                 .procname       = "panic_on_rcu_stall",
@@ -3260,6 +3272,28 @@ int proc_doulongvec_ms_jiffies_minmax(struct ctl_table *table, int write,
 
 #endif /* CONFIG_PROC_SYSCTL */
 
+static int proc_dointvec_minmax_bpf_stats(struct ctl_table *table, int write,
+                                          void __user *buffer, size_t *lenp,
+                                          loff_t *ppos)
+{
+        int ret, bpf_stats = *(int *)table->data;
+        struct ctl_table tmp = *table;
+
+        if (write && !capable(CAP_SYS_ADMIN))
+                return -EPERM;
+
+        tmp.data = &bpf_stats;
+        ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
+        if (write && !ret) {
+                *(int *)table->data = bpf_stats;
+                if (bpf_stats)
+                        static_branch_enable(&bpf_stats_enabled_key);
+                else
+                        static_branch_disable(&bpf_stats_enabled_key);
+        }
+        return ret;
+}
+
 /*
  * No sense putting this after each symbol definition, twice,
  * exception granted :-)