5 files changed, 148 insertions, 185 deletions

diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 900dbb1efff2..8b488f4dd8e8 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -89,16 +89,16 @@ struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
 /* What to set function_trace_op to */
 static struct ftrace_ops *set_function_trace_op;
 
-/* List for set_ftrace_pid's pids. */
-LIST_HEAD(ftrace_pids);
-struct ftrace_pid {
-        struct list_head list;
-        struct pid *pid;
-};
-
-static bool ftrace_pids_enabled(void)
+static bool ftrace_pids_enabled(struct ftrace_ops *ops)
 {
-        return !list_empty(&ftrace_pids);
+        struct trace_array *tr;
+
+        if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
+                return false;
+
+        tr = ops->private;
+
+        return tr->function_pids != NULL;
 }
 
 static void ftrace_update_trampoline(struct ftrace_ops *ops);
@@ -179,7 +179,9 @@ int ftrace_nr_registered_ops(void)
 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
                             struct ftrace_ops *op, struct pt_regs *regs)
 {
-        if (!test_tsk_trace_trace(current))
+        struct trace_array *tr = op->private;
+
+        if (tr && this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid))
                 return;
 
         op->saved_func(ip, parent_ip, op, regs);
@@ -417,7 +419,7 @@ static int __register_ftrace_function(struct ftrace_ops *ops)
         /* Always save the function, and reset at unregistering */
         ops->saved_func = ops->func;
 
-        if (ops->flags & FTRACE_OPS_FL_PID && ftrace_pids_enabled())
+        if (ftrace_pids_enabled(ops))
                 ops->func = ftrace_pid_func;
 
         ftrace_update_trampoline(ops);
@@ -450,7 +452,6 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
 
 static void ftrace_update_pid_func(void)
 {
-        bool enabled = ftrace_pids_enabled();
         struct ftrace_ops *op;
 
         /* Only do something if we are tracing something */
@@ -459,8 +460,8 @@ static void ftrace_update_pid_func(void)
 
         do_for_each_ftrace_op(op, ftrace_ops_list) {
                 if (op->flags & FTRACE_OPS_FL_PID) {
-                        op->func = enabled ? ftrace_pid_func :
-                                op->saved_func;
+                        op->func = ftrace_pids_enabled(op) ?
+                                ftrace_pid_func : op->saved_func;
                         ftrace_update_trampoline(op);
                 }
         } while_for_each_ftrace_op(op);
@@ -5324,179 +5325,99 @@ ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
         return ops->func;
 }
 
-static void clear_ftrace_swapper(void)
+static void
+ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
+                    struct task_struct *prev, struct task_struct *next)
 {
-        struct task_struct *p;
-        int cpu;
+        struct trace_array *tr = data;
+        struct trace_pid_list *pid_list;
 
-        get_online_cpus();
-        for_each_online_cpu(cpu) {
-                p = idle_task(cpu);
-                clear_tsk_trace_trace(p);
-        }
-        put_online_cpus();
-}
-
-static void set_ftrace_swapper(void)
-{
-        struct task_struct *p;
-        int cpu;
+        pid_list = rcu_dereference_sched(tr->function_pids);
 
-        get_online_cpus();
-        for_each_online_cpu(cpu) {
-                p = idle_task(cpu);
-                set_tsk_trace_trace(p);
-        }
-        put_online_cpus();
+        this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
+                       trace_ignore_this_task(pid_list, next));
 }
 
-static void clear_ftrace_pid(struct pid *pid)
+static void clear_ftrace_pids(struct trace_array *tr)
 {
-        struct task_struct *p;
+        struct trace_pid_list *pid_list;
+        int cpu;
 
-        rcu_read_lock();
-        do_each_pid_task(pid, PIDTYPE_PID, p) {
-                clear_tsk_trace_trace(p);
-        } while_each_pid_task(pid, PIDTYPE_PID, p);
-        rcu_read_unlock();
+        pid_list = rcu_dereference_protected(tr->function_pids,
+                                             lockdep_is_held(&ftrace_lock));
+        if (!pid_list)
+                return;
 
-        put_pid(pid);
-}
+        unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
 
-static void set_ftrace_pid(struct pid *pid)
-{
-        struct task_struct *p;
+        for_each_possible_cpu(cpu)
+                per_cpu_ptr(tr->trace_buffer.data, cpu)->ftrace_ignore_pid = false;
 
-        rcu_read_lock();
-        do_each_pid_task(pid, PIDTYPE_PID, p) {
-                set_tsk_trace_trace(p);
-        } while_each_pid_task(pid, PIDTYPE_PID, p);
-        rcu_read_unlock();
-}
+        rcu_assign_pointer(tr->function_pids, NULL);
 
-static void clear_ftrace_pid_task(struct pid *pid)
-{
-        if (pid == ftrace_swapper_pid)
-                clear_ftrace_swapper();
-        else
-                clear_ftrace_pid(pid);
-}
+        /* Wait till all users are no longer using pid filtering */
+        synchronize_sched();
 
-static void set_ftrace_pid_task(struct pid *pid)
-{
-        if (pid == ftrace_swapper_pid)
-                set_ftrace_swapper();
-        else
-                set_ftrace_pid(pid);
+        trace_free_pid_list(pid_list);
 }
 
-static int ftrace_pid_add(int p)
+static void ftrace_pid_reset(struct trace_array *tr)
 {
-        struct pid *pid;
-        struct ftrace_pid *fpid;
-        int ret = -EINVAL;
-
         mutex_lock(&ftrace_lock);
-
-        if (!p)
-                pid = ftrace_swapper_pid;
-        else
-                pid = find_get_pid(p);
-
-        if (!pid)
-                goto out;
-
-        ret = 0;
-
-        list_for_each_entry(fpid, &ftrace_pids, list)
-                if (fpid->pid == pid)
-                        goto out_put;
-
-        ret = -ENOMEM;
-
-        fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
-        if (!fpid)
-                goto out_put;
-
-        list_add(&fpid->list, &ftrace_pids);
-        fpid->pid = pid;
-
-        set_ftrace_pid_task(pid);
+        clear_ftrace_pids(tr);
 
         ftrace_update_pid_func();
-
         ftrace_startup_all(0);
 
         mutex_unlock(&ftrace_lock);
-        return 0;
-
-out_put:
-        if (pid != ftrace_swapper_pid)
-                put_pid(pid);
-
-out:
-        mutex_unlock(&ftrace_lock);
-        return ret;
 }
 
-static void ftrace_pid_reset(void)
-{
-        struct ftrace_pid *fpid, *safe;
-
-        mutex_lock(&ftrace_lock);
-        list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
-                struct pid *pid = fpid->pid;
-
-                clear_ftrace_pid_task(pid);
-
-                list_del(&fpid->list);
-                kfree(fpid);
-        }
-
-        ftrace_update_pid_func();
-        ftrace_startup_all(0);
-
-        mutex_unlock(&ftrace_lock);
-}
+/* Greater than any max PID */
+#define FTRACE_NO_PIDS          (void *)(PID_MAX_LIMIT + 1)
 
 static void *fpid_start(struct seq_file *m, loff_t *pos)
+        __acquires(RCU)
 {
+        struct trace_pid_list *pid_list;
+        struct trace_array *tr = m->private;
+
         mutex_lock(&ftrace_lock);
+        rcu_read_lock_sched();
 
-        if (!ftrace_pids_enabled() && (!*pos))
-                return (void *) 1;
+        pid_list = rcu_dereference_sched(tr->function_pids);
 
-        return seq_list_start(&ftrace_pids, *pos);
+        if (!pid_list)
+                return !(*pos) ? FTRACE_NO_PIDS : NULL;
+
+        return trace_pid_start(pid_list, pos);
 }
 
 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
 {
-        if (v == (void *)1)
+        struct trace_array *tr = m->private;
+        struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);
+
+        if (v == FTRACE_NO_PIDS)
                 return NULL;
 
-        return seq_list_next(v, &ftrace_pids, pos);
+        return trace_pid_next(pid_list, v, pos);
 }
 
 static void fpid_stop(struct seq_file *m, void *p)
+        __releases(RCU)
 {
+        rcu_read_unlock_sched();
         mutex_unlock(&ftrace_lock);
 }
 
 static int fpid_show(struct seq_file *m, void *v)
 {
-        const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
-
-        if (v == (void *)1) {
+        if (v == FTRACE_NO_PIDS) {
                 seq_puts(m, "no pid\n");
                 return 0;
         }
 
-        if (fpid->pid == ftrace_swapper_pid)
-                seq_puts(m, "swapper tasks\n");
-        else
-                seq_printf(m, "%u\n", pid_vnr(fpid->pid));
-
-        return 0;
+        return trace_pid_show(m, v);
 }
 
 static const struct seq_operations ftrace_pid_sops = {
@@ -5509,58 +5430,103 @@ static const struct seq_operations ftrace_pid_sops = {
 static int
 ftrace_pid_open(struct inode *inode, struct file *file)
 {
+        struct trace_array *tr = inode->i_private;
+        struct seq_file *m;
         int ret = 0;
 
+        if (trace_array_get(tr) < 0)
+                return -ENODEV;
+
         if ((file->f_mode & FMODE_WRITE) &&
             (file->f_flags & O_TRUNC))
-                ftrace_pid_reset();
+                ftrace_pid_reset(tr);
 
-        if (file->f_mode & FMODE_READ)
-                ret = seq_open(file, &ftrace_pid_sops);
+        ret = seq_open(file, &ftrace_pid_sops);
+        if (ret < 0) {
+                trace_array_put(tr);
+        } else {
+                m = file->private_data;
+                /* copy tr over to seq ops */
+                m->private = tr;
+        }
 
         return ret;
 }
 
+static void ignore_task_cpu(void *data)
+{
+        struct trace_array *tr = data;
+        struct trace_pid_list *pid_list;
+
+        /*
+         * This function is called by on_each_cpu() while the
+         * event_mutex is held.
+         */
+        pid_list = rcu_dereference_protected(tr->function_pids,
+                                             mutex_is_locked(&ftrace_lock));
+
+        this_cpu_write(tr->trace_buffer.data->ftrace_ignore_pid,
+                       trace_ignore_this_task(pid_list, current));
+}
+
 static ssize_t
 ftrace_pid_write(struct file *filp, const char __user *ubuf,
                    size_t cnt, loff_t *ppos)
 {
-        char buf[64], *tmp;
-        long val;
-        int ret;
+        struct seq_file *m = filp->private_data;
+        struct trace_array *tr = m->private;
+        struct trace_pid_list *filtered_pids = NULL;
+        struct trace_pid_list *pid_list;
+        ssize_t ret;
 
-        if (cnt >= sizeof(buf))
-                return -EINVAL;
+        if (!cnt)
+                return 0;
+
+        mutex_lock(&ftrace_lock);
+
+        filtered_pids = rcu_dereference_protected(tr->function_pids,
+                                             lockdep_is_held(&ftrace_lock));
+
+        ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
+        if (ret < 0)
+                goto out;
 
-        if (copy_from_user(&buf, ubuf, cnt))
-                return -EFAULT;
+        rcu_assign_pointer(tr->function_pids, pid_list);
 
-        buf[cnt] = 0;
+        if (filtered_pids) {
+                synchronize_sched();
+                trace_free_pid_list(filtered_pids);
+        } else if (pid_list) {
+                /* Register a probe to set whether to ignore the tracing of a task */
+                register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
+        }
 
         /*
-         * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
-         * to clean the filter quietly.
+         * Ignoring of pids is done at task switch. But we have to
+         * check for those tasks that are currently running.
+         * Always do this in case a pid was appended or removed.
          */
-        tmp = strstrip(buf);
-        if (strlen(tmp) == 0)
-                return 1;
+        on_each_cpu(ignore_task_cpu, tr, 1);
 
-        ret = kstrtol(tmp, 10, &val);
-        if (ret < 0)
-                return ret;
+        ftrace_update_pid_func();
+        ftrace_startup_all(0);
+ out:
+        mutex_unlock(&ftrace_lock);
 
-        ret = ftrace_pid_add(val);
+        if (ret > 0)
+                *ppos += ret;
 
-        return ret ? ret : cnt;
+        return ret;
 }
 
 static int
 ftrace_pid_release(struct inode *inode, struct file *file)
 {
-        if (file->f_mode & FMODE_READ)
-                seq_release(inode, file);
+        struct trace_array *tr = inode->i_private;
 
-        return 0;
+        trace_array_put(tr);
+
+        return seq_release(inode, file);
 }
 
 static const struct file_operations ftrace_pid_fops = {
@@ -5571,24 +5537,17 @@ static const struct file_operations ftrace_pid_fops = {
         .release        = ftrace_pid_release,
 };
 
-static __init int ftrace_init_tracefs(void)
+void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer)
 {
-        struct dentry *d_tracer;
-
-        d_tracer = tracing_init_dentry();
-        if (IS_ERR(d_tracer))
-                return 0;
-
-        ftrace_init_dyn_tracefs(d_tracer);
+        /* Only the top level directory has the dyn_tracefs and profile */
+        if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
+                ftrace_init_dyn_tracefs(d_tracer);
+                ftrace_profile_tracefs(d_tracer);
+        }
 
         trace_create_file("set_ftrace_pid", 0644, d_tracer,
-                            NULL, &ftrace_pid_fops);
-
-        ftrace_profile_tracefs(d_tracer);
-
-        return 0;
+                            tr, &ftrace_pid_fops);
 }
-fs_initcall(ftrace_init_tracefs);
 
 /**
  * ftrace_kill - kill ftrace
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index a8bb7485fd1d..aa240551fc5d 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -7233,6 +7233,7 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
         for_each_tracing_cpu(cpu)
                 tracing_init_tracefs_percpu(tr, cpu);
 
+        ftrace_init_tracefs(tr, d_tracer);
 }
 
 static struct vfsmount *trace_automount(void *ingore)
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index a4dce1ef9e03..eaee458755a4 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -156,6 +156,9 @@ struct trace_array_cpu {
         char                    comm[TASK_COMM_LEN];
 
         bool                    ignore_pid;
+#ifdef CONFIG_FUNCTION_TRACER
+        bool                    ftrace_ignore_pid;
+#endif
 };
 
 struct tracer;
@@ -247,6 +250,7 @@ struct trace_array {
         int                     ref;
 #ifdef CONFIG_FUNCTION_TRACER
         struct ftrace_ops       *ops;
+        struct trace_pid_list   __rcu *function_pids;
         /* function tracing enabled */
         int                     function_enabled;
 #endif
@@ -840,12 +844,9 @@ extern struct list_head ftrace_pids;
 
 #ifdef CONFIG_FUNCTION_TRACER
 extern bool ftrace_filter_param __initdata;
-static inline int ftrace_trace_task(struct task_struct *task)
+static inline int ftrace_trace_task(struct trace_array *tr)
 {
-        if (list_empty(&ftrace_pids))
-                return 1;
-
-        return test_tsk_trace_trace(task);
+        return !this_cpu_read(tr->trace_buffer.data->ftrace_ignore_pid);
 }
 extern int ftrace_is_dead(void);
 int ftrace_create_function_files(struct trace_array *tr,
@@ -855,8 +856,9 @@ void ftrace_init_global_array_ops(struct trace_array *tr);
 void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func);
 void ftrace_reset_array_ops(struct trace_array *tr);
 int using_ftrace_ops_list_func(void);
+void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer);
 #else
-static inline int ftrace_trace_task(struct task_struct *task)
+static inline int ftrace_trace_task(struct trace_array *tr)
 {
         return 1;
 }
@@ -871,6 +873,7 @@ static inline void ftrace_destroy_function_files(struct trace_array *tr) { }
 static inline __init void
 ftrace_init_global_array_ops(struct trace_array *tr) { }
 static inline void ftrace_reset_array_ops(struct trace_array *tr) { }
+static inline void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d) { }
 /* ftace_func_t type is not defined, use macro instead of static inline */
 #define ftrace_init_array_ops(tr, func) do { } while (0)
 #endif /* CONFIG_FUNCTION_TRACER */
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index 5a095c2e4b69..0efa00d80623 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -43,7 +43,7 @@ static int allocate_ftrace_ops(struct trace_array *tr)
 
         /* Currently only the non stack verision is supported */
         ops->func = function_trace_call;
-        ops->flags = FTRACE_OPS_FL_RECURSION_SAFE;
+        ops->flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_PID;
 
         tr->ops = ops;
         ops->private = tr;
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 3a0244ff7ea8..67cce7896aeb 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -319,7 +319,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)
         int cpu;
         int pc;
 
-        if (!ftrace_trace_task(current))
+        if (!ftrace_trace_task(tr))
                 return 0;
 
         /* trace it when it is-nested-in or is a function enabled. */