8 files changed, 219 insertions, 1 deletions
diff --git a/include/linux/ftrace_event.h b/include/linux/ftrace_event.h
index 77325e1a1816..0aa535bc9f05 100644
--- a/include/linux/ftrace_event.h
+++ b/include/linux/ftrace_event.h
@@ -13,6 +13,7 @@ struct trace_array;
 struct trace_buffer;
 struct tracer;
 struct dentry;
+struct bpf_prog;
 struct trace_print_flags {
        unsigned long           mask;
@@ -306,6 +307,7 @@ struct ftrace_event_call {
 #ifdef CONFIG_PERF_EVENTS
        int                             perf_refcount;
        struct hlist_head __percpu      *perf_events;
+        struct bpf_prog                 *prog;
        int     (*perf_perm)(struct ftrace_event_call *,
                             struct perf_event *);
@@ -551,6 +553,15 @@ event_trigger_unlock_commit_regs(struct ftrace_event_file *file,
                event_triggers_post_call(file, tt);
 }
+#ifdef CONFIG_BPF_SYSCALL
+unsigned int trace_call_bpf(struct bpf_prog *prog, void *ctx);
+#else
+static inline unsigned int trace_call_bpf(struct bpf_prog *prog, void *ctx)
+{
+        return 1;
+}
+#endif
 enum {
        FILTER_OTHER = 0,
        FILTER_STATIC_STRING,
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 45da7ec7d274..b2948feeb70b 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -118,6 +118,7 @@ enum bpf_map_type {
 enum bpf_prog_type {
        BPF_PROG_TYPE_UNSPEC,
        BPF_PROG_TYPE_SOCKET_FILTER,
+        BPF_PROG_TYPE_KPROBE,
 };
 /* flags for BPF_MAP_UPDATE_ELEM command */
@@ -151,6 +152,7 @@ union bpf_attr {
                __u32           log_level;      /* verbosity level of verifier */
                __u32           log_size;       /* size of user buffer */
                __aligned_u64   log_buf;        /* user supplied buffer */
+                __u32           kern_version;   /* checked when prog_type=kprobe */
        };
 } __attribute__((aligned(8)));
@@ -162,6 +164,7 @@ enum bpf_func_id {
        BPF_FUNC_map_lookup_elem, /* void *map_lookup_elem(&map, &key) */
        BPF_FUNC_map_update_elem, /* int map_update_elem(&map, &key, &value, flags) */
        BPF_FUNC_map_delete_elem, /* int map_delete_elem(&map, &key) */
+        BPF_FUNC_probe_read,      /* int bpf_probe_read(void *dst, int size, void *src) */
        __BPF_FUNC_MAX_ID,
 };
diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
index 3bb40ddadbe5..91803e54ee73 100644
--- a/include/uapi/linux/perf_event.h
+++ b/include/uapi/linux/perf_event.h
@@ -381,6 +381,7 @@ struct perf_event_attr {
 #define PERF_EVENT_IOC_SET_OUTPUT       _IO ('$', 5)
 #define PERF_EVENT_IOC_SET_FILTER       _IOW('$', 6, char *)
 #define PERF_EVENT_IOC_ID               _IOR('$', 7, __u64 *)
+#define PERF_EVENT_IOC_SET_BPF          _IOW('$', 8, __u32)
 enum perf_event_ioc_flags {
        PERF_IOC_FLAG_GROUP             = 1U << 0,
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 536edc2be307..504c10b990ef 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -16,6 +16,7 @@
 #include <linux/file.h>
 #include <linux/license.h>
 #include <linux/filter.h>
+#include <linux/version.h>
 static LIST_HEAD(bpf_map_types);
@@ -467,7 +468,7 @@ struct bpf_prog *bpf_prog_get(u32 ufd)
 }
 /* last field in 'union bpf_attr' used by this command */
-#define BPF_PROG_LOAD_LAST_FIELD log_buf
+#define BPF_PROG_LOAD_LAST_FIELD kern_version
 static int bpf_prog_load(union bpf_attr *attr)
 {
@@ -492,6 +493,10 @@ static int bpf_prog_load(union bpf_attr *attr)
        if (attr->insn_cnt >= BPF_MAXINSNS)
                return -EINVAL;
+        if (type == BPF_PROG_TYPE_KPROBE &&
+            attr->kern_version != LINUX_VERSION_CODE)
+                return -EINVAL;
        /* plain bpf_prog allocation */
        prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
        if (!prog)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index c40c2cac2d8e..5c13862d3e85 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -42,6 +42,8 @@
 #include <linux/module.h>
 #include <linux/mman.h>
 #include <linux/compat.h>
+#include <linux/bpf.h>
+#include <linux/filter.h>
 #include "internal.h"
@@ -3407,6 +3409,7 @@ errout:
 }
 static void perf_event_free_filter(struct perf_event *event);
+static void perf_event_free_bpf_prog(struct perf_event *event);
 static void free_event_rcu(struct rcu_head *head)
 {
@@ -3416,6 +3419,7 @@ static void free_event_rcu(struct rcu_head *head)
        if (event->ns)
                put_pid_ns(event->ns);
        perf_event_free_filter(event);
+        perf_event_free_bpf_prog(event);
        kfree(event);
 }
@@ -3928,6 +3932,7 @@ static inline int perf_fget_light(int fd, struct fd *p)
 static int perf_event_set_output(struct perf_event *event,
                                 struct perf_event *output_event);
 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
+static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd);
 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
 {
@@ -3981,6 +3986,9 @@ static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned lon
        case PERF_EVENT_IOC_SET_FILTER:
                return perf_event_set_filter(event, (void __user *)arg);
+        case PERF_EVENT_IOC_SET_BPF:
+                return perf_event_set_bpf_prog(event, arg);
        default:
                return -ENOTTY;
        }
@@ -6455,6 +6463,49 @@ static void perf_event_free_filter(struct perf_event *event)
        ftrace_profile_free_filter(event);
 }
+static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
+{
+        struct bpf_prog *prog;
+        if (event->attr.type != PERF_TYPE_TRACEPOINT)
+                return -EINVAL;
+        if (event->tp_event->prog)
+                return -EEXIST;
+        if (!(event->tp_event->flags & TRACE_EVENT_FL_KPROBE))
+                /* bpf programs can only be attached to kprobes */
+                return -EINVAL;
+        prog = bpf_prog_get(prog_fd);
+        if (IS_ERR(prog))
+                return PTR_ERR(prog);
+        if (prog->aux->prog_type != BPF_PROG_TYPE_KPROBE) {
+                /* valid fd, but invalid bpf program type */
+                bpf_prog_put(prog);
+                return -EINVAL;
+        }
+        event->tp_event->prog = prog;
+        return 0;
+}
+static void perf_event_free_bpf_prog(struct perf_event *event)
+{
+        struct bpf_prog *prog;
+        if (!event->tp_event)
+                return;
+        prog = event->tp_event->prog;
+        if (prog) {
+                event->tp_event->prog = NULL;
+                bpf_prog_put(prog);
+        }
+}
 #else
 static inline void perf_tp_register(void)
@@ -6470,6 +6521,14 @@ static void perf_event_free_filter(struct perf_event *event)
 {
 }
+static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
+{
+        return -ENOENT;
+}
+static void perf_event_free_bpf_prog(struct perf_event *event)
+{
+}
 #endif /* CONFIG_EVENT_TRACING */
 #ifdef CONFIG_HAVE_HW_BREAKPOINT
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index 98f26588255e..c575a300103b 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -53,6 +53,7 @@ obj-$(CONFIG_EVENT_TRACING) += trace_event_perf.o
 endif
 obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o
 obj-$(CONFIG_EVENT_TRACING) += trace_events_trigger.o
+obj-$(CONFIG_BPF_SYSCALL) += bpf_trace.o
 obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o
 obj-$(CONFIG_TRACEPOINTS) += power-traces.o
 ifeq ($(CONFIG_PM),y)
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
new file mode 100644
index 000000000000..f1e87da91da3
--- /dev/null
+++ b/kernel/trace/bpf_trace.c
@@ -0,0 +1,130 @@
+/* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU General Public
+ * License as published by the Free Software Foundation.
+ */
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/bpf.h>
+#include <linux/filter.h>
+#include <linux/uaccess.h>
+#include "trace.h"
+static DEFINE_PER_CPU(int, bpf_prog_active);
+/**
+ * trace_call_bpf - invoke BPF program
+ * @prog: BPF program
+ * @ctx: opaque context pointer
+ *
+ * kprobe handlers execute BPF programs via this helper.
+ * Can be used from static tracepoints in the future.
+ *
+ * Return: BPF programs always return an integer which is interpreted by
+ * kprobe handler as:
+ * 0 - return from kprobe (event is filtered out)
+ * 1 - store kprobe event into ring buffer
+ * Other values are reserved and currently alias to 1
+ */
+unsigned int trace_call_bpf(struct bpf_prog *prog, void *ctx)
+{
+        unsigned int ret;
+        if (in_nmi()) /* not supported yet */
+                return 1;
+        preempt_disable();
+        if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
+                /*
+                 * since some bpf program is already running on this cpu,
+                 * don't call into another bpf program (same or different)
+                 * and don't send kprobe event into ring-buffer,
+                 * so return zero here
+                 */
+                ret = 0;
+                goto out;
+        }
+        rcu_read_lock();
+        ret = BPF_PROG_RUN(prog, ctx);
+        rcu_read_unlock();
+ out:
+        __this_cpu_dec(bpf_prog_active);
+        preempt_enable();
+        return ret;
+}
+EXPORT_SYMBOL_GPL(trace_call_bpf);
+static u64 bpf_probe_read(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
+{
+        void *dst = (void *) (long) r1;
+        int size = (int) r2;
+        void *unsafe_ptr = (void *) (long) r3;
+        return probe_kernel_read(dst, unsafe_ptr, size);
+}
+static const struct bpf_func_proto bpf_probe_read_proto = {
+        .func           = bpf_probe_read,
+        .gpl_only       = true,
+        .ret_type       = RET_INTEGER,
+        .arg1_type      = ARG_PTR_TO_STACK,
+        .arg2_type      = ARG_CONST_STACK_SIZE,
+        .arg3_type      = ARG_ANYTHING,
+};
+static const struct bpf_func_proto *kprobe_prog_func_proto(enum bpf_func_id func_id)
+{
+        switch (func_id) {
+        case BPF_FUNC_map_lookup_elem:
+                return &bpf_map_lookup_elem_proto;
+        case BPF_FUNC_map_update_elem:
+                return &bpf_map_update_elem_proto;
+        case BPF_FUNC_map_delete_elem:
+                return &bpf_map_delete_elem_proto;
+        case BPF_FUNC_probe_read:
+                return &bpf_probe_read_proto;
+        default:
+                return NULL;
+        }
+}
+/* bpf+kprobe programs can access fields of 'struct pt_regs' */
+static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type)
+{
+        /* check bounds */
+        if (off < 0 || off >= sizeof(struct pt_regs))
+                return false;
+        /* only read is allowed */
+        if (type != BPF_READ)
+                return false;
+        /* disallow misaligned access */
+        if (off % size != 0)
+                return false;
+        return true;
+}
+static struct bpf_verifier_ops kprobe_prog_ops = {
+        .get_func_proto  = kprobe_prog_func_proto,
+        .is_valid_access = kprobe_prog_is_valid_access,
+};
+static struct bpf_prog_type_list kprobe_tl = {
+        .ops    = &kprobe_prog_ops,
+        .type   = BPF_PROG_TYPE_KPROBE,
+};
+static int __init register_kprobe_prog_ops(void)
+{
+        bpf_register_prog_type(&kprobe_tl);
+        return 0;
+}
+late_initcall(register_kprobe_prog_ops);
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index 8fa549f6f528..dc3462507d7c 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -1134,11 +1134,15 @@ static void
 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
 {
        struct ftrace_event_call *call = &tk->tp.call;
+        struct bpf_prog *prog = call->prog;
        struct kprobe_trace_entry_head *entry;
        struct hlist_head *head;
        int size, __size, dsize;
        int rctx;
+        if (prog && !trace_call_bpf(prog, regs))
+                return;
        head = this_cpu_ptr(call->perf_events);
        if (hlist_empty(head))
                return;
@@ -1165,11 +1169,15 @@ kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
                    struct pt_regs *regs)
 {
        struct ftrace_event_call *call = &tk->tp.call;
+        struct bpf_prog *prog = call->prog;
        struct kretprobe_trace_entry_head *entry;
        struct hlist_head *head;
        int size, __size, dsize;
        int rctx;
+        if (prog && !trace_call_bpf(prog, regs))
+                return;
        head = this_cpu_ptr(call->perf_events);
        if (hlist_empty(head))
                return;

diff --git a/include/linux/ftrace_event.h b/include/linux/ftrace_event.h index 77325e1a1816..0aa535bc9f05 100644 --- a/include/linux/ftrace_event.h +++ b/include/linux/ftrace_event.h
@@ -13,6 +13,7 @@ struct trace_array;
13	struct trace_buffer;	13	struct trace_buffer;
14	struct tracer;	14	struct tracer;
15	struct dentry;	15	struct dentry;
		16	struct bpf_prog;
16		17
17	struct trace_print_flags {	18	struct trace_print_flags {
18	unsigned long mask;	19	unsigned long mask;
@@ -306,6 +307,7 @@ struct ftrace_event_call {
306	#ifdef CONFIG_PERF_EVENTS	307	#ifdef CONFIG_PERF_EVENTS
307	int perf_refcount;	308	int perf_refcount;
308	struct hlist_head __percpu *perf_events;	309	struct hlist_head __percpu *perf_events;
		310	struct bpf_prog *prog;
309		311
310	int (perf_perm)(struct ftrace_event_call ,	312	int (perf_perm)(struct ftrace_event_call ,
311	struct perf_event *);	313	struct perf_event *);
@@ -551,6 +553,15 @@ event_trigger_unlock_commit_regs(struct ftrace_event_file *file,
551	event_triggers_post_call(file, tt);	553	event_triggers_post_call(file, tt);
552	}	554	}
553		555
		556	#ifdef CONFIG_BPF_SYSCALL
		557	unsigned int trace_call_bpf(struct bpf_prog prog, void ctx);
		558	#else
		559	static inline unsigned int trace_call_bpf(struct bpf_prog prog, void ctx)
		560	{
		561	return 1;
		562	}
		563	#endif
		564
554	enum {	565	enum {
555	FILTER_OTHER = 0,	566	FILTER_OTHER = 0,
556	FILTER_STATIC_STRING,	567	FILTER_STATIC_STRING,


diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 45da7ec7d274..b2948feeb70b 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h
@@ -118,6 +118,7 @@ enum bpf_map_type {
118	enum bpf_prog_type {	118	enum bpf_prog_type {
119	BPF_PROG_TYPE_UNSPEC,	119	BPF_PROG_TYPE_UNSPEC,
120	BPF_PROG_TYPE_SOCKET_FILTER,	120	BPF_PROG_TYPE_SOCKET_FILTER,
		121	BPF_PROG_TYPE_KPROBE,
121	};	122	};
122		123
123	/* flags for BPF_MAP_UPDATE_ELEM command */	124	/* flags for BPF_MAP_UPDATE_ELEM command */
@@ -151,6 +152,7 @@ union bpf_attr {
151	__u32 log_level; /* verbosity level of verifier */	152	__u32 log_level; /* verbosity level of verifier */
152	__u32 log_size; /* size of user buffer */	153	__u32 log_size; /* size of user buffer */
153	__aligned_u64 log_buf; /* user supplied buffer */	154	__aligned_u64 log_buf; /* user supplied buffer */
		155	__u32 kern_version; /* checked when prog_type=kprobe */
154	};	156	};
155	} __attribute__((aligned(8)));	157	} __attribute__((aligned(8)));
156		158
@@ -162,6 +164,7 @@ enum bpf_func_id {
162	BPF_FUNC_map_lookup_elem, /* void map_lookup_elem(&map, &key) /	164	BPF_FUNC_map_lookup_elem, /* void map_lookup_elem(&map, &key) /
163	BPF_FUNC_map_update_elem, /* int map_update_elem(&map, &key, &value, flags) */	165	BPF_FUNC_map_update_elem, /* int map_update_elem(&map, &key, &value, flags) */
164	BPF_FUNC_map_delete_elem, /* int map_delete_elem(&map, &key) */	166	BPF_FUNC_map_delete_elem, /* int map_delete_elem(&map, &key) */
		167	BPF_FUNC_probe_read, /* int bpf_probe_read(void dst, int size, void src) */
165	__BPF_FUNC_MAX_ID,	168	__BPF_FUNC_MAX_ID,
166	};	169	};
167		170


diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h index 3bb40ddadbe5..91803e54ee73 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h
@@ -381,6 +381,7 @@ struct perf_event_attr {
381	#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5)	381	#define PERF_EVENT_IOC_SET_OUTPUT _IO ('$', 5)
382	#define PERF_EVENT_IOC_SET_FILTER _IOW('$', 6, char *)	382	#define PERF_EVENT_IOC_SET_FILTER _IOW('$', 6, char *)
383	#define PERF_EVENT_IOC_ID _IOR('$', 7, __u64 *)	383	#define PERF_EVENT_IOC_ID _IOR('$', 7, __u64 *)
		384	#define PERF_EVENT_IOC_SET_BPF _IOW('$', 8, __u32)
384		385
385	enum perf_event_ioc_flags {	386	enum perf_event_ioc_flags {
386	PERF_IOC_FLAG_GROUP = 1U << 0,	387	PERF_IOC_FLAG_GROUP = 1U << 0,


diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 536edc2be307..504c10b990ef 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c
@@ -16,6 +16,7 @@
16	#include <linux/file.h>	16	#include <linux/file.h>
17	#include <linux/license.h>	17	#include <linux/license.h>
18	#include <linux/filter.h>	18	#include <linux/filter.h>
		19	#include <linux/version.h>
19		20
20	static LIST_HEAD(bpf_map_types);	21	static LIST_HEAD(bpf_map_types);
21		22
@@ -467,7 +468,7 @@ struct bpf_prog *bpf_prog_get(u32 ufd)
467	}	468	}
468		469
469	/* last field in 'union bpf_attr' used by this command */	470	/* last field in 'union bpf_attr' used by this command */
470	#define BPF_PROG_LOAD_LAST_FIELD log_buf	471	#define BPF_PROG_LOAD_LAST_FIELD kern_version
471		472
472	static int bpf_prog_load(union bpf_attr *attr)	473	static int bpf_prog_load(union bpf_attr *attr)
473	{	474	{
@@ -492,6 +493,10 @@ static int bpf_prog_load(union bpf_attr *attr)
492	if (attr->insn_cnt >= BPF_MAXINSNS)	493	if (attr->insn_cnt >= BPF_MAXINSNS)
493	return -EINVAL;	494	return -EINVAL;
494		495
		496	if (type == BPF_PROG_TYPE_KPROBE &&
		497	attr->kern_version != LINUX_VERSION_CODE)
		498	return -EINVAL;
		499
495	/* plain bpf_prog allocation */	500	/* plain bpf_prog allocation */
496	prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);	501	prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
497	if (!prog)	502	if (!prog)


diff --git a/kernel/events/core.c b/kernel/events/core.c index c40c2cac2d8e..5c13862d3e85 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c
@@ -42,6 +42,8 @@
42	#include <linux/module.h>	42	#include <linux/module.h>
43	#include <linux/mman.h>	43	#include <linux/mman.h>
44	#include <linux/compat.h>	44	#include <linux/compat.h>
		45	#include <linux/bpf.h>
		46	#include <linux/filter.h>
45		47
46	#include "internal.h"	48	#include "internal.h"
47		49
@@ -3407,6 +3409,7 @@ errout:
3407	}	3409	}
3408		3410
3409	static void perf_event_free_filter(struct perf_event *event);	3411	static void perf_event_free_filter(struct perf_event *event);
		3412	static void perf_event_free_bpf_prog(struct perf_event *event);
3410		3413
3411	static void free_event_rcu(struct rcu_head *head)	3414	static void free_event_rcu(struct rcu_head *head)
3412	{	3415	{
@@ -3416,6 +3419,7 @@ static void free_event_rcu(struct rcu_head *head)
3416	if (event->ns)	3419	if (event->ns)
3417	put_pid_ns(event->ns);	3420	put_pid_ns(event->ns);
3418	perf_event_free_filter(event);	3421	perf_event_free_filter(event);
		3422	perf_event_free_bpf_prog(event);
3419	kfree(event);	3423	kfree(event);
3420	}	3424	}
3421		3425
@@ -3928,6 +3932,7 @@ static inline int perf_fget_light(int fd, struct fd *p)
3928	static int perf_event_set_output(struct perf_event *event,	3932	static int perf_event_set_output(struct perf_event *event,
3929	struct perf_event *output_event);	3933	struct perf_event *output_event);
3930	static int perf_event_set_filter(struct perf_event event, void __user arg);	3934	static int perf_event_set_filter(struct perf_event event, void __user arg);
		3935	static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd);
3931		3936
3932	static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)	3937	static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)
3933	{	3938	{
@@ -3981,6 +3986,9 @@ static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned lon
3981	case PERF_EVENT_IOC_SET_FILTER:	3986	case PERF_EVENT_IOC_SET_FILTER:
3982	return perf_event_set_filter(event, (void __user *)arg);	3987	return perf_event_set_filter(event, (void __user *)arg);
3983		3988
		3989	case PERF_EVENT_IOC_SET_BPF:
		3990	return perf_event_set_bpf_prog(event, arg);
		3991
3984	default:	3992	default:
3985	return -ENOTTY;	3993	return -ENOTTY;
3986	}	3994	}
@@ -6455,6 +6463,49 @@ static void perf_event_free_filter(struct perf_event *event)
6455	ftrace_profile_free_filter(event);	6463	ftrace_profile_free_filter(event);
6456	}	6464	}
6457		6465
		6466	static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
		6467	{
		6468	struct bpf_prog *prog;
		6469
		6470	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		6471	return -EINVAL;
		6472
		6473	if (event->tp_event->prog)
		6474	return -EEXIST;
		6475
		6476	if (!(event->tp_event->flags & TRACE_EVENT_FL_KPROBE))
		6477	/* bpf programs can only be attached to kprobes */
		6478	return -EINVAL;
		6479
		6480	prog = bpf_prog_get(prog_fd);
		6481	if (IS_ERR(prog))
		6482	return PTR_ERR(prog);
		6483
		6484	if (prog->aux->prog_type != BPF_PROG_TYPE_KPROBE) {
		6485	/* valid fd, but invalid bpf program type */
		6486	bpf_prog_put(prog);
		6487	return -EINVAL;
		6488	}
		6489
		6490	event->tp_event->prog = prog;
		6491
		6492	return 0;
		6493	}
		6494
		6495	static void perf_event_free_bpf_prog(struct perf_event *event)
		6496	{
		6497	struct bpf_prog *prog;
		6498
		6499	if (!event->tp_event)
		6500	return;
		6501
		6502	prog = event->tp_event->prog;
		6503	if (prog) {
		6504	event->tp_event->prog = NULL;
		6505	bpf_prog_put(prog);
		6506	}
		6507	}
		6508
6458	#else	6509	#else
6459		6510
6460	static inline void perf_tp_register(void)	6511	static inline void perf_tp_register(void)
@@ -6470,6 +6521,14 @@ static void perf_event_free_filter(struct perf_event *event)
6470	{	6521	{
6471	}	6522	}
6472		6523
		6524	static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)
		6525	{
		6526	return -ENOENT;
		6527	}
		6528
		6529	static void perf_event_free_bpf_prog(struct perf_event *event)
		6530	{
		6531	}
6473	#endif /* CONFIG_EVENT_TRACING */	6532	#endif /* CONFIG_EVENT_TRACING */
6474		6533
6475	#ifdef CONFIG_HAVE_HW_BREAKPOINT	6534	#ifdef CONFIG_HAVE_HW_BREAKPOINT


diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index 98f26588255e..c575a300103b 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile
@@ -53,6 +53,7 @@ obj-$(CONFIG_EVENT_TRACING) += trace_event_perf.o
53	endif	53	endif
54	obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o	54	obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o
55	obj-$(CONFIG_EVENT_TRACING) += trace_events_trigger.o	55	obj-$(CONFIG_EVENT_TRACING) += trace_events_trigger.o
		56	obj-$(CONFIG_BPF_SYSCALL) += bpf_trace.o
56	obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o	57	obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o
57	obj-$(CONFIG_TRACEPOINTS) += power-traces.o	58	obj-$(CONFIG_TRACEPOINTS) += power-traces.o
58	ifeq ($(CONFIG_PM),y)	59	ifeq ($(CONFIG_PM),y)


diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c new file mode 100644 index 000000000000..f1e87da91da3 --- /dev/null +++ b/kernel/trace/bpf_trace.c
@@ -0,0 +1,130 @@
		1	/* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com
		2	*
		3	* This program is free software; you can redistribute it and/or
		4	* modify it under the terms of version 2 of the GNU General Public
		5	* License as published by the Free Software Foundation.
		6	*/
		7	#include <linux/kernel.h>
		8	#include <linux/types.h>
		9	#include <linux/slab.h>
		10	#include <linux/bpf.h>
		11	#include <linux/filter.h>
		12	#include <linux/uaccess.h>
		13	#include "trace.h"
		14
		15	static DEFINE_PER_CPU(int, bpf_prog_active);
		16
		17	/**
		18	* trace_call_bpf - invoke BPF program
		19	* @prog: BPF program
		20	* @ctx: opaque context pointer
		21	*
		22	* kprobe handlers execute BPF programs via this helper.
		23	* Can be used from static tracepoints in the future.
		24	*
		25	* Return: BPF programs always return an integer which is interpreted by
		26	* kprobe handler as:
		27	* 0 - return from kprobe (event is filtered out)
		28	* 1 - store kprobe event into ring buffer
		29	* Other values are reserved and currently alias to 1
		30	*/
		31	unsigned int trace_call_bpf(struct bpf_prog prog, void ctx)
		32	{
		33	unsigned int ret;
		34
		35	if (in_nmi()) /* not supported yet */
		36	return 1;
		37
		38	preempt_disable();
		39
		40	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
		41	/*
		42	* since some bpf program is already running on this cpu,
		43	* don't call into another bpf program (same or different)
		44	* and don't send kprobe event into ring-buffer,
		45	* so return zero here
		46	*/
		47	ret = 0;
		48	goto out;
		49	}
		50
		51	rcu_read_lock();
		52	ret = BPF_PROG_RUN(prog, ctx);
		53	rcu_read_unlock();
		54
		55	out:
		56	__this_cpu_dec(bpf_prog_active);
		57	preempt_enable();
		58
		59	return ret;
		60	}
		61	EXPORT_SYMBOL_GPL(trace_call_bpf);
		62
		63	static u64 bpf_probe_read(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
		64	{
		65	void dst = (void ) (long) r1;
		66	int size = (int) r2;
		67	void unsafe_ptr = (void ) (long) r3;
		68
		69	return probe_kernel_read(dst, unsafe_ptr, size);
		70	}
		71
		72	static const struct bpf_func_proto bpf_probe_read_proto = {
		73	.func = bpf_probe_read,
		74	.gpl_only = true,
		75	.ret_type = RET_INTEGER,
		76	.arg1_type = ARG_PTR_TO_STACK,
		77	.arg2_type = ARG_CONST_STACK_SIZE,
		78	.arg3_type = ARG_ANYTHING,
		79	};
		80
		81	static const struct bpf_func_proto *kprobe_prog_func_proto(enum bpf_func_id func_id)
		82	{
		83	switch (func_id) {
		84	case BPF_FUNC_map_lookup_elem:
		85	return &bpf_map_lookup_elem_proto;
		86	case BPF_FUNC_map_update_elem:
		87	return &bpf_map_update_elem_proto;
		88	case BPF_FUNC_map_delete_elem:
		89	return &bpf_map_delete_elem_proto;
		90	case BPF_FUNC_probe_read:
		91	return &bpf_probe_read_proto;
		92	default:
		93	return NULL;
		94	}
		95	}
		96
		97	/* bpf+kprobe programs can access fields of 'struct pt_regs' */
		98	static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type)
		99	{
		100	/* check bounds */
		101	if (off < 0 \|\| off >= sizeof(struct pt_regs))
		102	return false;
		103
		104	/* only read is allowed */
		105	if (type != BPF_READ)
		106	return false;
		107
		108	/* disallow misaligned access */
		109	if (off % size != 0)
		110	return false;
		111
		112	return true;
		113	}
		114
		115	static struct bpf_verifier_ops kprobe_prog_ops = {
		116	.get_func_proto = kprobe_prog_func_proto,
		117	.is_valid_access = kprobe_prog_is_valid_access,
		118	};
		119
		120	static struct bpf_prog_type_list kprobe_tl = {
		121	.ops = &kprobe_prog_ops,
		122	.type = BPF_PROG_TYPE_KPROBE,
		123	};
		124
		125	static int __init register_kprobe_prog_ops(void)
		126	{
		127	bpf_register_prog_type(&kprobe_tl);
		128	return 0;
		129	}
		130	late_initcall(register_kprobe_prog_ops);


diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 8fa549f6f528..dc3462507d7c 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c
@@ -1134,11 +1134,15 @@ static void
1134	kprobe_perf_func(struct trace_kprobe tk, struct pt_regs regs)	1134	kprobe_perf_func(struct trace_kprobe tk, struct pt_regs regs)
1135	{	1135	{
1136	struct ftrace_event_call *call = &tk->tp.call;	1136	struct ftrace_event_call *call = &tk->tp.call;
		1137	struct bpf_prog *prog = call->prog;
1137	struct kprobe_trace_entry_head *entry;	1138	struct kprobe_trace_entry_head *entry;
1138	struct hlist_head *head;	1139	struct hlist_head *head;
1139	int size, __size, dsize;	1140	int size, __size, dsize;
1140	int rctx;	1141	int rctx;
1141		1142
		1143	if (prog && !trace_call_bpf(prog, regs))
		1144	return;
		1145
1142	head = this_cpu_ptr(call->perf_events);	1146	head = this_cpu_ptr(call->perf_events);
1143	if (hlist_empty(head))	1147	if (hlist_empty(head))
1144	return;	1148	return;
@@ -1165,11 +1169,15 @@ kretprobe_perf_func(struct trace_kprobe tk, struct kretprobe_instance ri,
1165	struct pt_regs *regs)	1169	struct pt_regs *regs)
1166	{	1170	{
1167	struct ftrace_event_call *call = &tk->tp.call;	1171	struct ftrace_event_call *call = &tk->tp.call;
		1172	struct bpf_prog *prog = call->prog;
1168	struct kretprobe_trace_entry_head *entry;	1173	struct kretprobe_trace_entry_head *entry;
1169	struct hlist_head *head;	1174	struct hlist_head *head;
1170	int size, __size, dsize;	1175	int size, __size, dsize;
1171	int rctx;	1176	int rctx;
1172		1177
		1178	if (prog && !trace_call_bpf(prog, regs))
		1179	return;
		1180
1173	head = this_cpu_ptr(call->perf_events);	1181	head = this_cpu_ptr(call->perf_events);
1174	if (hlist_empty(head))	1182	if (hlist_empty(head))
1175	return;	1183	return;