1 files changed, 533 insertions, 166 deletions

diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index cb6c0d2af68..11847bf1e8c 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -13,6 +13,7 @@
 #include <linux/mm.h>
 #include <linux/cpu.h>
 #include <linux/smp.h>
+#include <linux/idr.h>
 #include <linux/file.h>
 #include <linux/poll.h>
 #include <linux/slab.h>
@@ -21,7 +22,9 @@
 #include <linux/dcache.h>
 #include <linux/percpu.h>
 #include <linux/ptrace.h>
+#include <linux/reboot.h>
 #include <linux/vmstat.h>
+#include <linux/device.h>
 #include <linux/vmalloc.h>
 #include <linux/hardirq.h>
 #include <linux/rculist.h>
@@ -31,6 +34,7 @@
 #include <linux/kernel_stat.h>
 #include <linux/perf_event.h>
 #include <linux/ftrace_event.h>
+#include <linux/hw_breakpoint.h>
 
 #include <asm/irq_regs.h>
 
@@ -132,6 +136,28 @@ static void unclone_ctx(struct perf_event_context *ctx)
         }
 }
 
+static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
+{
+        /*
+         * only top level events have the pid namespace they were created in
+         */
+        if (event->parent)
+                event = event->parent;
+
+        return task_tgid_nr_ns(p, event->ns);
+}
+
+static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
+{
+        /*
+         * only top level events have the pid namespace they were created in
+         */
+        if (event->parent)
+                event = event->parent;
+
+        return task_pid_nr_ns(p, event->ns);
+}
+
 /*
  * If we inherit events we want to return the parent event id
  * to userspace.
@@ -311,9 +337,84 @@ list_add_event(struct perf_event *event, struct perf_event_context *ctx)
                 ctx->nr_stat++;
 }
 
+/*
+ * Called at perf_event creation and when events are attached/detached from a
+ * group.
+ */
+static void perf_event__read_size(struct perf_event *event)
+{
+        int entry = sizeof(u64); /* value */
+        int size = 0;
+        int nr = 1;
+
+        if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+                size += sizeof(u64);
+
+        if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+                size += sizeof(u64);
+
+        if (event->attr.read_format & PERF_FORMAT_ID)
+                entry += sizeof(u64);
+
+        if (event->attr.read_format & PERF_FORMAT_GROUP) {
+                nr += event->group_leader->nr_siblings;
+                size += sizeof(u64);
+        }
+
+        size += entry * nr;
+        event->read_size = size;
+}
+
+static void perf_event__header_size(struct perf_event *event)
+{
+        struct perf_sample_data *data;
+        u64 sample_type = event->attr.sample_type;
+        u16 size = 0;
+
+        perf_event__read_size(event);
+
+        if (sample_type & PERF_SAMPLE_IP)
+                size += sizeof(data->ip);
+
+        if (sample_type & PERF_SAMPLE_ADDR)
+                size += sizeof(data->addr);
+
+        if (sample_type & PERF_SAMPLE_PERIOD)
+                size += sizeof(data->period);
+
+        if (sample_type & PERF_SAMPLE_READ)
+                size += event->read_size;
+
+        event->header_size = size;
+}
+
+static void perf_event__id_header_size(struct perf_event *event)
+{
+        struct perf_sample_data *data;
+        u64 sample_type = event->attr.sample_type;
+        u16 size = 0;
+
+        if (sample_type & PERF_SAMPLE_TID)
+                size += sizeof(data->tid_entry);
+
+        if (sample_type & PERF_SAMPLE_TIME)
+                size += sizeof(data->time);
+
+        if (sample_type & PERF_SAMPLE_ID)
+                size += sizeof(data->id);
+
+        if (sample_type & PERF_SAMPLE_STREAM_ID)
+                size += sizeof(data->stream_id);
+
+        if (sample_type & PERF_SAMPLE_CPU)
+                size += sizeof(data->cpu_entry);
+
+        event->id_header_size = size;
+}
+
 static void perf_group_attach(struct perf_event *event)
 {
-        struct perf_event *group_leader = event->group_leader;
+        struct perf_event *group_leader = event->group_leader, *pos;
 
         /*
          * We can have double attach due to group movement in perf_event_open.
@@ -332,6 +433,11 @@ static void perf_group_attach(struct perf_event *event)
 
         list_add_tail(&event->group_entry, &group_leader->sibling_list);
         group_leader->nr_siblings++;
+
+        perf_event__header_size(group_leader);
+
+        list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
+                perf_event__header_size(pos);
 }
 
 /*
@@ -390,7 +496,7 @@ static void perf_group_detach(struct perf_event *event)
         if (event->group_leader != event) {
                 list_del_init(&event->group_entry);
                 event->group_leader->nr_siblings--;
-                return;
+                goto out;
         }
 
         if (!list_empty(&event->group_entry))
@@ -409,6 +515,12 @@ static void perf_group_detach(struct perf_event *event)
                 /* Inherit group flags from the previous leader */
                 sibling->group_flags = event->group_flags;
         }
+
+out:
+        perf_event__header_size(event->group_leader);
+
+        list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
+                perf_event__header_size(tmp);
 }
 
 static inline int
@@ -1072,7 +1184,7 @@ static int perf_event_refresh(struct perf_event *event, int refresh)
         /*
          * not supported on inherited events
          */
-        if (event->attr.inherit)
+        if (event->attr.inherit || !is_sampling_event(event))
                 return -EINVAL;
 
         atomic_add(refresh, &event->event_limit);
@@ -1286,8 +1398,6 @@ void __perf_event_task_sched_out(struct task_struct *task,
 {
         int ctxn;
 
-        perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
-
         for_each_task_context_nr(ctxn)
                 perf_event_context_sched_out(task, ctxn, next);
 }
@@ -1621,8 +1731,12 @@ static void rotate_ctx(struct perf_event_context *ctx)
 {
         raw_spin_lock(&ctx->lock);
 
-        /* Rotate the first entry last of non-pinned groups */
-        list_rotate_left(&ctx->flexible_groups);
+        /*
+         * Rotate the first entry last of non-pinned groups. Rotation might be
+         * disabled by the inheritance code.
+         */
+        if (!ctx->rotate_disable)
+                list_rotate_left(&ctx->flexible_groups);
 
         raw_spin_unlock(&ctx->lock);
 }
@@ -2234,11 +2348,6 @@ int perf_event_release_kernel(struct perf_event *event)
         raw_spin_unlock_irq(&ctx->lock);
         mutex_unlock(&ctx->mutex);
 
-        mutex_lock(&event->owner->perf_event_mutex);
-        list_del_init(&event->owner_entry);
-        mutex_unlock(&event->owner->perf_event_mutex);
-        put_task_struct(event->owner);
-
         free_event(event);
 
         return 0;
@@ -2251,35 +2360,44 @@ EXPORT_SYMBOL_GPL(perf_event_release_kernel);
 static int perf_release(struct inode *inode, struct file *file)
 {
         struct perf_event *event = file->private_data;
+        struct task_struct *owner;
 
         file->private_data = NULL;
 
-        return perf_event_release_kernel(event);
-}
-
-static int perf_event_read_size(struct perf_event *event)
-{
-        int entry = sizeof(u64); /* value */
-        int size = 0;
-        int nr = 1;
-
-        if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
-                size += sizeof(u64);
-
-        if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
-                size += sizeof(u64);
-
-        if (event->attr.read_format & PERF_FORMAT_ID)
-                entry += sizeof(u64);
-
-        if (event->attr.read_format & PERF_FORMAT_GROUP) {
-                nr += event->group_leader->nr_siblings;
-                size += sizeof(u64);
+        rcu_read_lock();
+        owner = ACCESS_ONCE(event->owner);
+        /*
+         * Matches the smp_wmb() in perf_event_exit_task(). If we observe
+         * !owner it means the list deletion is complete and we can indeed
+         * free this event, otherwise we need to serialize on
+         * owner->perf_event_mutex.
+         */
+        smp_read_barrier_depends();
+        if (owner) {
+                /*
+                 * Since delayed_put_task_struct() also drops the last
+                 * task reference we can safely take a new reference
+                 * while holding the rcu_read_lock().
+                 */
+                get_task_struct(owner);
         }
+        rcu_read_unlock();
 
-        size += entry * nr;
+        if (owner) {
+                mutex_lock(&owner->perf_event_mutex);
+                /*
+                 * We have to re-check the event->owner field, if it is cleared
+                 * we raced with perf_event_exit_task(), acquiring the mutex
+                 * ensured they're done, and we can proceed with freeing the
+                 * event.
+                 */
+                if (event->owner)
+                        list_del_init(&event->owner_entry);
+                mutex_unlock(&owner->perf_event_mutex);
+                put_task_struct(owner);
+        }
 
-        return size;
+        return perf_event_release_kernel(event);
 }
 
 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
@@ -2396,7 +2514,7 @@ perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
         if (event->state == PERF_EVENT_STATE_ERROR)
                 return 0;
 
-        if (count < perf_event_read_size(event))
+        if (count < event->read_size)
                 return -ENOSPC;
 
         WARN_ON_ONCE(event->ctx->parent_ctx);
@@ -2482,7 +2600,7 @@ static int perf_event_period(struct perf_event *event, u64 __user *arg)
         int ret = 0;
         u64 value;
 
-        if (!event->attr.sample_period)
+        if (!is_sampling_event(event))
                 return -EINVAL;
 
         if (copy_from_user(&value, arg, sizeof(value)))
@@ -3273,6 +3391,73 @@ __always_inline void perf_output_copy(struct perf_output_handle *handle,
         } while (len);
 }
 
+static void __perf_event_header__init_id(struct perf_event_header *header,
+                                         struct perf_sample_data *data,
+                                         struct perf_event *event)
+{
+        u64 sample_type = event->attr.sample_type;
+
+        data->type = sample_type;
+        header->size += event->id_header_size;
+
+        if (sample_type & PERF_SAMPLE_TID) {
+                /* namespace issues */
+                data->tid_entry.pid = perf_event_pid(event, current);
+                data->tid_entry.tid = perf_event_tid(event, current);
+        }
+
+        if (sample_type & PERF_SAMPLE_TIME)
+                data->time = perf_clock();
+
+        if (sample_type & PERF_SAMPLE_ID)
+                data->id = primary_event_id(event);
+
+        if (sample_type & PERF_SAMPLE_STREAM_ID)
+                data->stream_id = event->id;
+
+        if (sample_type & PERF_SAMPLE_CPU) {
+                data->cpu_entry.cpu      = raw_smp_processor_id();
+                data->cpu_entry.reserved = 0;
+        }
+}
+
+static void perf_event_header__init_id(struct perf_event_header *header,
+                                       struct perf_sample_data *data,
+                                       struct perf_event *event)
+{
+        if (event->attr.sample_id_all)
+                __perf_event_header__init_id(header, data, event);
+}
+
+static void __perf_event__output_id_sample(struct perf_output_handle *handle,
+                                           struct perf_sample_data *data)
+{
+        u64 sample_type = data->type;
+
+        if (sample_type & PERF_SAMPLE_TID)
+                perf_output_put(handle, data->tid_entry);
+
+        if (sample_type & PERF_SAMPLE_TIME)
+                perf_output_put(handle, data->time);
+
+        if (sample_type & PERF_SAMPLE_ID)
+                perf_output_put(handle, data->id);
+
+        if (sample_type & PERF_SAMPLE_STREAM_ID)
+                perf_output_put(handle, data->stream_id);
+
+        if (sample_type & PERF_SAMPLE_CPU)
+                perf_output_put(handle, data->cpu_entry);
+}
+
+static void perf_event__output_id_sample(struct perf_event *event,
+                                         struct perf_output_handle *handle,
+                                         struct perf_sample_data *sample)
+{
+        if (event->attr.sample_id_all)
+                __perf_event__output_id_sample(handle, sample);
+}
+
 int perf_output_begin(struct perf_output_handle *handle,
                       struct perf_event *event, unsigned int size,
                       int nmi, int sample)
@@ -3280,6 +3465,7 @@ int perf_output_begin(struct perf_output_handle *handle,
         struct perf_buffer *buffer;
         unsigned long tail, offset, head;
         int have_lost;
+        struct perf_sample_data sample_data;
         struct {
                 struct perf_event_header header;
                 u64                      id;
@@ -3306,8 +3492,12 @@ int perf_output_begin(struct perf_output_handle *handle,
                 goto out;
 
         have_lost = local_read(&buffer->lost);
-        if (have_lost)
-                size += sizeof(lost_event);
+        if (have_lost) {
+                lost_event.header.size = sizeof(lost_event);
+                perf_event_header__init_id(&lost_event.header, &sample_data,
+                                           event);
+                size += lost_event.header.size;
+        }
 
         perf_output_get_handle(handle);
 
@@ -3338,11 +3528,11 @@ int perf_output_begin(struct perf_output_handle *handle,
         if (have_lost) {
                 lost_event.header.type = PERF_RECORD_LOST;
                 lost_event.header.misc = 0;
-                lost_event.header.size = sizeof(lost_event);
                 lost_event.id          = event->id;
                 lost_event.lost        = local_xchg(&buffer->lost, 0);
 
                 perf_output_put(handle, lost_event);
+                perf_event__output_id_sample(event, handle, &sample_data);
         }
 
         return 0;
@@ -3375,28 +3565,6 @@ void perf_output_end(struct perf_output_handle *handle)
         rcu_read_unlock();
 }
 
-static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
-{
-        /*
-         * only top level events have the pid namespace they were created in
-         */
-        if (event->parent)
-                event = event->parent;
-
-        return task_tgid_nr_ns(p, event->ns);
-}
-
-static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
-{
-        /*
-         * only top level events have the pid namespace they were created in
-         */
-        if (event->parent)
-                event = event->parent;
-
-        return task_pid_nr_ns(p, event->ns);
-}
-
 static void perf_output_read_one(struct perf_output_handle *handle,
                                  struct perf_event *event,
                                  u64 enabled, u64 running)
@@ -3571,61 +3739,16 @@ void perf_prepare_sample(struct perf_event_header *header,
 {
         u64 sample_type = event->attr.sample_type;
 
-        data->type = sample_type;
-
         header->type = PERF_RECORD_SAMPLE;
-        header->size = sizeof(*header);
+        header->size = sizeof(*header) + event->header_size;
 
         header->misc = 0;
         header->misc |= perf_misc_flags(regs);
 
-        if (sample_type & PERF_SAMPLE_IP) {
-                data->ip = perf_instruction_pointer(regs);
-
-                header->size += sizeof(data->ip);
-        }
-
-        if (sample_type & PERF_SAMPLE_TID) {
-                /* namespace issues */
-                data->tid_entry.pid = perf_event_pid(event, current);
-                data->tid_entry.tid = perf_event_tid(event, current);
-
-                header->size += sizeof(data->tid_entry);
-        }
-
-        if (sample_type & PERF_SAMPLE_TIME) {
-                data->time = perf_clock();
-
-                header->size += sizeof(data->time);
-        }
-
-        if (sample_type & PERF_SAMPLE_ADDR)
-                header->size += sizeof(data->addr);
-
-        if (sample_type & PERF_SAMPLE_ID) {
-                data->id = primary_event_id(event);
-
-                header->size += sizeof(data->id);
-        }
-
-        if (sample_type & PERF_SAMPLE_STREAM_ID) {
-                data->stream_id = event->id;
-
-                header->size += sizeof(data->stream_id);
-        }
-
-        if (sample_type & PERF_SAMPLE_CPU) {
-                data->cpu_entry.cpu             = raw_smp_processor_id();
-                data->cpu_entry.reserved        = 0;
-
-                header->size += sizeof(data->cpu_entry);
-        }
-
-        if (sample_type & PERF_SAMPLE_PERIOD)
-                header->size += sizeof(data->period);
+        __perf_event_header__init_id(header, data, event);
 
-        if (sample_type & PERF_SAMPLE_READ)
-                header->size += perf_event_read_size(event);
+        if (sample_type & PERF_SAMPLE_IP)
+                data->ip = perf_instruction_pointer(regs);
 
         if (sample_type & PERF_SAMPLE_CALLCHAIN) {
                 int size = 1;
@@ -3690,23 +3813,26 @@ perf_event_read_event(struct perf_event *event,
                         struct task_struct *task)
 {
         struct perf_output_handle handle;
+        struct perf_sample_data sample;
         struct perf_read_event read_event = {
                 .header = {
                         .type = PERF_RECORD_READ,
                         .misc = 0,
-                        .size = sizeof(read_event) + perf_event_read_size(event),
+                        .size = sizeof(read_event) + event->read_size,
                 },
                 .pid = perf_event_pid(event, task),
                 .tid = perf_event_tid(event, task),
         };
         int ret;
 
+        perf_event_header__init_id(&read_event.header, &sample, event);
         ret = perf_output_begin(&handle, event, read_event.header.size, 0, 0);
         if (ret)
                 return;
 
         perf_output_put(&handle, read_event);
         perf_output_read(&handle, event);
+        perf_event__output_id_sample(event, &handle, &sample);
 
         perf_output_end(&handle);
 }
@@ -3736,14 +3862,16 @@ static void perf_event_task_output(struct perf_event *event,
                                      struct perf_task_event *task_event)
 {
         struct perf_output_handle handle;
+        struct perf_sample_data sample;
         struct task_struct *task = task_event->task;
-        int size, ret;
+        int ret, size = task_event->event_id.header.size;
 
-        size  = task_event->event_id.header.size;
-        ret = perf_output_begin(&handle, event, size, 0, 0);
+        perf_event_header__init_id(&task_event->event_id.header, &sample, event);
 
+        ret = perf_output_begin(&handle, event,
+                                task_event->event_id.header.size, 0, 0);
         if (ret)
-                return;
+                goto out;
 
         task_event->event_id.pid = perf_event_pid(event, task);
         task_event->event_id.ppid = perf_event_pid(event, current);
@@ -3753,7 +3881,11 @@ static void perf_event_task_output(struct perf_event *event,
 
         perf_output_put(&handle, task_event->event_id);
 
+        perf_event__output_id_sample(event, &handle, &sample);
+
         perf_output_end(&handle);
+out:
+        task_event->event_id.header.size = size;
 }
 
 static int perf_event_task_match(struct perf_event *event)
@@ -3792,6 +3924,8 @@ static void perf_event_task_event(struct perf_task_event *task_event)
         rcu_read_lock();
         list_for_each_entry_rcu(pmu, &pmus, entry) {
                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
+                if (cpuctx->active_pmu != pmu)
+                        goto next;
                 perf_event_task_ctx(&cpuctx->ctx, task_event);
 
                 ctx = task_event->task_ctx;
@@ -3866,11 +4000,16 @@ static void perf_event_comm_output(struct perf_event *event,
                                      struct perf_comm_event *comm_event)
 {
         struct perf_output_handle handle;
+        struct perf_sample_data sample;
         int size = comm_event->event_id.header.size;
-        int ret = perf_output_begin(&handle, event, size, 0, 0);
+        int ret;
+
+        perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
+        ret = perf_output_begin(&handle, event,
+                                comm_event->event_id.header.size, 0, 0);
 
         if (ret)
-                return;
+                goto out;
 
         comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
         comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
@@ -3878,7 +4017,12 @@ static void perf_event_comm_output(struct perf_event *event,
         perf_output_put(&handle, comm_event->event_id);
         perf_output_copy(&handle, comm_event->comm,
                                    comm_event->comm_size);
+
+        perf_event__output_id_sample(event, &handle, &sample);
+
         perf_output_end(&handle);
+out:
+        comm_event->event_id.header.size = size;
 }
 
 static int perf_event_comm_match(struct perf_event *event)
@@ -3923,10 +4067,11 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event)
         comm_event->comm_size = size;
 
         comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
-
         rcu_read_lock();
         list_for_each_entry_rcu(pmu, &pmus, entry) {
                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
+                if (cpuctx->active_pmu != pmu)
+                        goto next;
                 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
 
                 ctxn = pmu->task_ctx_nr;
@@ -4002,11 +4147,15 @@ static void perf_event_mmap_output(struct perf_event *event,
                                      struct perf_mmap_event *mmap_event)
 {
         struct perf_output_handle handle;
+        struct perf_sample_data sample;
         int size = mmap_event->event_id.header.size;
-        int ret = perf_output_begin(&handle, event, size, 0, 0);
+        int ret;
 
+        perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
+        ret = perf_output_begin(&handle, event,
+                                mmap_event->event_id.header.size, 0, 0);
         if (ret)
-                return;
+                goto out;
 
         mmap_event->event_id.pid = perf_event_pid(event, current);
         mmap_event->event_id.tid = perf_event_tid(event, current);
@@ -4014,7 +4163,12 @@ static void perf_event_mmap_output(struct perf_event *event,
         perf_output_put(&handle, mmap_event->event_id);
         perf_output_copy(&handle, mmap_event->file_name,
                                    mmap_event->file_size);
+
+        perf_event__output_id_sample(event, &handle, &sample);
+
         perf_output_end(&handle);
+out:
+        mmap_event->event_id.header.size = size;
 }
 
 static int perf_event_mmap_match(struct perf_event *event,
@@ -4112,6 +4266,8 @@ got_name:
         rcu_read_lock();
         list_for_each_entry_rcu(pmu, &pmus, entry) {
                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
+                if (cpuctx->active_pmu != pmu)
+                        goto next;
                 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
                                         vma->vm_flags & VM_EXEC);
 
@@ -4167,6 +4323,7 @@ void perf_event_mmap(struct vm_area_struct *vma)
 static void perf_log_throttle(struct perf_event *event, int enable)
 {
         struct perf_output_handle handle;
+        struct perf_sample_data sample;
         int ret;
 
         struct {
@@ -4188,11 +4345,15 @@ static void perf_log_throttle(struct perf_event *event, int enable)
         if (enable)
                 throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
 
-        ret = perf_output_begin(&handle, event, sizeof(throttle_event), 1, 0);
+        perf_event_header__init_id(&throttle_event.header, &sample, event);
+
+        ret = perf_output_begin(&handle, event,
+                                throttle_event.header.size, 1, 0);
         if (ret)
                 return;
 
         perf_output_put(&handle, throttle_event);
+        perf_event__output_id_sample(event, &handle, &sample);
         perf_output_end(&handle);
 }
 
@@ -4208,6 +4369,13 @@ static int __perf_event_overflow(struct perf_event *event, int nmi,
         struct hw_perf_event *hwc = &event->hw;
         int ret = 0;
 
+        /*
+         * Non-sampling counters might still use the PMI to fold short
+         * hardware counters, ignore those.
+         */
+        if (unlikely(!is_sampling_event(event)))
+                return 0;
+
         if (!throttle) {
                 hwc->interrupts++;
         } else {
@@ -4353,7 +4521,7 @@ static void perf_swevent_event(struct perf_event *event, u64 nr,
         if (!regs)
                 return;
 
-        if (!hwc->sample_period)
+        if (!is_sampling_event(event))
                 return;
 
         if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
@@ -4516,7 +4684,7 @@ static int perf_swevent_add(struct perf_event *event, int flags)
         struct hw_perf_event *hwc = &event->hw;
         struct hlist_head *head;
 
-        if (hwc->sample_period) {
+        if (is_sampling_event(event)) {
                 hwc->last_period = hwc->sample_period;
                 perf_swevent_set_period(event);
         }
@@ -4681,7 +4849,7 @@ static int perf_swevent_init(struct perf_event *event)
                 break;
         }
 
-        if (event_id > PERF_COUNT_SW_MAX)
+        if (event_id >= PERF_COUNT_SW_MAX)
                 return -ENOENT;
 
         if (!event->parent) {
@@ -4773,15 +4941,6 @@ static int perf_tp_event_init(struct perf_event *event)
         if (event->attr.type != PERF_TYPE_TRACEPOINT)
                 return -ENOENT;
 
-        /*
-         * Raw tracepoint data is a severe data leak, only allow root to
-         * have these.
-         */
-        if ((event->attr.sample_type & PERF_SAMPLE_RAW) &&
-                        perf_paranoid_tracepoint_raw() &&
-                        !capable(CAP_SYS_ADMIN))
-                return -EPERM;
-
         err = perf_trace_init(event);
         if (err)
                 return err;
@@ -4804,7 +4963,7 @@ static struct pmu perf_tracepoint = {
 
 static inline void perf_tp_register(void)
 {
-        perf_pmu_register(&perf_tracepoint);
+        perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
 }
 
 static int perf_event_set_filter(struct perf_event *event, void __user *arg)
@@ -4894,31 +5053,33 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
 static void perf_swevent_start_hrtimer(struct perf_event *event)
 {
         struct hw_perf_event *hwc = &event->hw;
+        s64 period;
+
+        if (!is_sampling_event(event))
+                return;
 
         hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
         hwc->hrtimer.function = perf_swevent_hrtimer;
-        if (hwc->sample_period) {
-                s64 period = local64_read(&hwc->period_left);
 
-                if (period) {
-                        if (period < 0)
-                                period = 10000;
+        period = local64_read(&hwc->period_left);
+        if (period) {
+                if (period < 0)
+                        period = 10000;
 
-                        local64_set(&hwc->period_left, 0);
-                } else {
-                        period = max_t(u64, 10000, hwc->sample_period);
-                }
-                __hrtimer_start_range_ns(&hwc->hrtimer,
+                local64_set(&hwc->period_left, 0);
+        } else {
+                period = max_t(u64, 10000, hwc->sample_period);
+        }
+        __hrtimer_start_range_ns(&hwc->hrtimer,
                                 ns_to_ktime(period), 0,
                                 HRTIMER_MODE_REL_PINNED, 0);
-        }
 }
 
 static void perf_swevent_cancel_hrtimer(struct perf_event *event)
 {
         struct hw_perf_event *hwc = &event->hw;
 
-        if (hwc->sample_period) {
+        if (is_sampling_event(event)) {
                 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
                 local64_set(&hwc->period_left, ktime_to_ns(remaining));
 
@@ -5113,25 +5274,94 @@ static void *find_pmu_context(int ctxn)
         return NULL;
 }
 
-static void free_pmu_context(void * __percpu cpu_context)
+static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
 {
-        struct pmu *pmu;
+        int cpu;
+
+        for_each_possible_cpu(cpu) {
+                struct perf_cpu_context *cpuctx;
+
+                cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
+
+                if (cpuctx->active_pmu == old_pmu)
+                        cpuctx->active_pmu = pmu;
+        }
+}
+
+static void free_pmu_context(struct pmu *pmu)
+{
+        struct pmu *i;
 
         mutex_lock(&pmus_lock);
         /*
          * Like a real lame refcount.
          */
-        list_for_each_entry(pmu, &pmus, entry) {
-                if (pmu->pmu_cpu_context == cpu_context)
+        list_for_each_entry(i, &pmus, entry) {
+                if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
+                        update_pmu_context(i, pmu);
                         goto out;
+                }
         }
 
-        free_percpu(cpu_context);
+        free_percpu(pmu->pmu_cpu_context);
 out:
         mutex_unlock(&pmus_lock);
 }
+static struct idr pmu_idr;
+
+static ssize_t
+type_show(struct device *dev, struct device_attribute *attr, char *page)
+{
+        struct pmu *pmu = dev_get_drvdata(dev);
+
+        return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
+}
+
+static struct device_attribute pmu_dev_attrs[] = {
+       __ATTR_RO(type),
+       __ATTR_NULL,
+};
+
+static int pmu_bus_running;
+static struct bus_type pmu_bus = {
+        .name           = "event_source",
+        .dev_attrs      = pmu_dev_attrs,
+};
+
+static void pmu_dev_release(struct device *dev)
+{
+        kfree(dev);
+}
+
+static int pmu_dev_alloc(struct pmu *pmu)
+{
+        int ret = -ENOMEM;
+
+        pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
+        if (!pmu->dev)
+                goto out;
+
+        device_initialize(pmu->dev);
+        ret = dev_set_name(pmu->dev, "%s", pmu->name);
+        if (ret)
+                goto free_dev;
+
+        dev_set_drvdata(pmu->dev, pmu);
+        pmu->dev->bus = &pmu_bus;
+        pmu->dev->release = pmu_dev_release;
+        ret = device_add(pmu->dev);
+        if (ret)
+                goto free_dev;
+
+out:
+        return ret;
+
+free_dev:
+        put_device(pmu->dev);
+        goto out;
+}
 
-int perf_pmu_register(struct pmu *pmu)
+int perf_pmu_register(struct pmu *pmu, char *name, int type)
 {
         int cpu, ret;
 
@@ -5141,13 +5371,38 @@ int perf_pmu_register(struct pmu *pmu)
         if (!pmu->pmu_disable_count)
                 goto unlock;
 
+        pmu->type = -1;
+        if (!name)
+                goto skip_type;
+        pmu->name = name;
+
+        if (type < 0) {
+                int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
+                if (!err)
+                        goto free_pdc;
+
+                err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
+                if (err) {
+                        ret = err;
+                        goto free_pdc;
+                }
+        }
+        pmu->type = type;
+
+        if (pmu_bus_running) {
+                ret = pmu_dev_alloc(pmu);
+                if (ret)
+                        goto free_idr;
+        }
+
+skip_type:
         pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
         if (pmu->pmu_cpu_context)
                 goto got_cpu_context;
 
         pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
         if (!pmu->pmu_cpu_context)
-                goto free_pdc;
+                goto free_dev;
 
         for_each_possible_cpu(cpu) {
                 struct perf_cpu_context *cpuctx;
@@ -5158,6 +5413,7 @@ int perf_pmu_register(struct pmu *pmu)
                 cpuctx->ctx.pmu = pmu;
                 cpuctx->jiffies_interval = 1;
                 INIT_LIST_HEAD(&cpuctx->rotation_list);
+                cpuctx->active_pmu = pmu;
         }
 
 got_cpu_context:
@@ -5190,6 +5446,14 @@ unlock:
 
         return ret;
 
+free_dev:
+        device_del(pmu->dev);
+        put_device(pmu->dev);
+
+free_idr:
+        if (pmu->type >= PERF_TYPE_MAX)
+                idr_remove(&pmu_idr, pmu->type);
+
 free_pdc:
         free_percpu(pmu->pmu_disable_count);
         goto unlock;
@@ -5209,7 +5473,11 @@ void perf_pmu_unregister(struct pmu *pmu)
         synchronize_rcu();
 
         free_percpu(pmu->pmu_disable_count);
-        free_pmu_context(pmu->pmu_cpu_context);
+        if (pmu->type >= PERF_TYPE_MAX)
+                idr_remove(&pmu_idr, pmu->type);
+        device_del(pmu->dev);
+        put_device(pmu->dev);
+        free_pmu_context(pmu);
 }
 
 struct pmu *perf_init_event(struct perf_event *event)
@@ -5218,6 +5486,13 @@ struct pmu *perf_init_event(struct perf_event *event)
         int idx;
 
         idx = srcu_read_lock(&pmus_srcu);
+
+        rcu_read_lock();
+        pmu = idr_find(&pmu_idr, event->attr.type);
+        rcu_read_unlock();
+        if (pmu)
+                goto unlock;
+
         list_for_each_entry_rcu(pmu, &pmus, entry) {
                 int ret = pmu->event_init(event);
                 if (!ret)
@@ -5677,12 +5952,18 @@ SYSCALL_DEFINE5(perf_event_open,
         mutex_unlock(&ctx->mutex);
 
         event->owner = current;
-        get_task_struct(current);
+
         mutex_lock(&current->perf_event_mutex);
         list_add_tail(&event->owner_entry, &current->perf_event_list);
         mutex_unlock(&current->perf_event_mutex);
 
         /*
+         * Precalculate sample_data sizes
+         */
+        perf_event__header_size(event);
+        perf_event__id_header_size(event);
+
+        /*
          * Drop the reference on the group_event after placing the
          * new event on the sibling_list. This ensures destruction
          * of the group leader will find the pointer to itself in
@@ -5745,12 +6026,6 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
         ++ctx->generation;
         mutex_unlock(&ctx->mutex);
 
-        event->owner = current;
-        get_task_struct(current);
-        mutex_lock(&current->perf_event_mutex);
-        list_add_tail(&event->owner_entry, &current->perf_event_list);
-        mutex_unlock(&current->perf_event_mutex);
-
         return event;
 
 err_free:
@@ -5901,8 +6176,24 @@ again:
  */
 void perf_event_exit_task(struct task_struct *child)
 {
+        struct perf_event *event, *tmp;
         int ctxn;
 
+        mutex_lock(&child->perf_event_mutex);
+        list_for_each_entry_safe(event, tmp, &child->perf_event_list,
+                                 owner_entry) {
+                list_del_init(&event->owner_entry);
+
+                /*
+                 * Ensure the list deletion is visible before we clear
+                 * the owner, closes a race against perf_release() where
+                 * we need to serialize on the owner->perf_event_mutex.
+                 */
+                smp_wmb();
+                event->owner = NULL;
+        }
+        mutex_unlock(&child->perf_event_mutex);
+
         for_each_task_context_nr(ctxn)
                 perf_event_exit_task_context(child, ctxn);
 }
@@ -6025,6 +6316,12 @@ inherit_event(struct perf_event *parent_event,
         child_event->overflow_handler = parent_event->overflow_handler;
 
         /*
+         * Precalculate sample_data sizes
+         */
+        perf_event__header_size(child_event);
+        perf_event__id_header_size(child_event);
+
+        /*
          * Link it up in the child's context:
          */
         raw_spin_lock_irqsave(&child_ctx->lock, flags);
@@ -6122,6 +6419,7 @@ int perf_event_init_context(struct task_struct *child, int ctxn)
         struct perf_event *event;
         struct task_struct *parent = current;
         int inherited_all = 1;
+        unsigned long flags;
         int ret = 0;
 
         child->perf_event_ctxp[ctxn] = NULL;
@@ -6162,6 +6460,15 @@ int perf_event_init_context(struct task_struct *child, int ctxn)
                         break;
         }
 
+        /*
+         * We can't hold ctx->lock when iterating the ->flexible_group list due
+         * to allocations, but we need to prevent rotation because
+         * rotate_ctx() will change the list from interrupt context.
+         */
+        raw_spin_lock_irqsave(&parent_ctx->lock, flags);
+        parent_ctx->rotate_disable = 1;
+        raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
+
         list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
                 ret = inherit_task_group(event, parent, parent_ctx,
                                          child, ctxn, &inherited_all);
@@ -6169,6 +6476,10 @@ int perf_event_init_context(struct task_struct *child, int ctxn)
                         break;
         }
 
+        raw_spin_lock_irqsave(&parent_ctx->lock, flags);
+        parent_ctx->rotate_disable = 0;
+        raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
+
         child_ctx = child->perf_event_ctxp[ctxn];
 
         if (child_ctx && inherited_all) {
@@ -6241,7 +6552,7 @@ static void __cpuinit perf_event_init_cpu(int cpu)
         mutex_unlock(&swhash->hlist_mutex);
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
+#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
 static void perf_pmu_rotate_stop(struct pmu *pmu)
 {
         struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
@@ -6295,6 +6606,26 @@ static void perf_event_exit_cpu(int cpu)
 static inline void perf_event_exit_cpu(int cpu) { }
 #endif
 
+static int
+perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
+{
+        int cpu;
+
+        for_each_online_cpu(cpu)
+                perf_event_exit_cpu(cpu);
+
+        return NOTIFY_OK;
+}
+
+/*
+ * Run the perf reboot notifier at the very last possible moment so that
+ * the generic watchdog code runs as long as possible.
+ */
+static struct notifier_block perf_reboot_notifier = {
+        .notifier_call = perf_reboot,
+        .priority = INT_MIN,
+};
+
 static int __cpuinit
 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
 {
@@ -6321,11 +6652,47 @@ perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
 
 void __init perf_event_init(void)
 {
+        int ret;
+
+        idr_init(&pmu_idr);
+
         perf_event_init_all_cpus();
         init_srcu_struct(&pmus_srcu);
-        perf_pmu_register(&perf_swevent);
-        perf_pmu_register(&perf_cpu_clock);
-        perf_pmu_register(&perf_task_clock);
+        perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
+        perf_pmu_register(&perf_cpu_clock, NULL, -1);
+        perf_pmu_register(&perf_task_clock, NULL, -1);
         perf_tp_register();
         perf_cpu_notifier(perf_cpu_notify);
+        register_reboot_notifier(&perf_reboot_notifier);
+
+        ret = init_hw_breakpoint();
+        WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
+}
+
+static int __init perf_event_sysfs_init(void)
+{
+        struct pmu *pmu;
+        int ret;
+
+        mutex_lock(&pmus_lock);
+
+        ret = bus_register(&pmu_bus);
+        if (ret)
+                goto unlock;
+
+        list_for_each_entry(pmu, &pmus, entry) {
+                if (!pmu->name || pmu->type < 0)
+                        continue;
+
+                ret = pmu_dev_alloc(pmu);
+                WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
+        }
+        pmu_bus_running = 1;
+        ret = 0;
+
+unlock:
+        mutex_unlock(&pmus_lock);
+
+        return ret;
 }
+device_initcall(perf_event_sysfs_init);