1 files changed, 261 insertions, 155 deletions

diff --git a/kernel/events/core.c b/kernel/events/core.c
index eba8fb5834ae..2207efc941d1 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -145,6 +145,7 @@ static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
 static atomic_t nr_mmap_events __read_mostly;
 static atomic_t nr_comm_events __read_mostly;
 static atomic_t nr_task_events __read_mostly;
+static atomic_t nr_freq_events __read_mostly;
 
 static LIST_HEAD(pmus);
 static DEFINE_MUTEX(pmus_lock);
@@ -340,8 +341,8 @@ struct perf_cgroup {
 static inline struct perf_cgroup *
 perf_cgroup_from_task(struct task_struct *task)
 {
-        return container_of(task_subsys_state(task, perf_subsys_id),
-                        struct perf_cgroup, css);
+        return container_of(task_css(task, perf_subsys_id),
+                            struct perf_cgroup, css);
 }
 
 static inline bool
@@ -591,7 +592,9 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
         if (!f.file)
                 return -EBADF;
 
-        css = cgroup_css_from_dir(f.file, perf_subsys_id);
+        rcu_read_lock();
+
+        css = css_from_dir(f.file->f_dentry, &perf_subsys);
         if (IS_ERR(css)) {
                 ret = PTR_ERR(css);
                 goto out;
@@ -617,6 +620,7 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
                 ret = -EINVAL;
         }
 out:
+        rcu_read_unlock();
         fdput(f);
         return ret;
 }
@@ -869,12 +873,8 @@ static void perf_pmu_rotate_start(struct pmu *pmu)
 
         WARN_ON(!irqs_disabled());
 
-        if (list_empty(&cpuctx->rotation_list)) {
-                int was_empty = list_empty(head);
+        if (list_empty(&cpuctx->rotation_list))
                 list_add(&cpuctx->rotation_list, head);
-                if (was_empty)
-                        tick_nohz_full_kick();
-        }
 }
 
 static void get_ctx(struct perf_event_context *ctx)
@@ -1216,6 +1216,9 @@ static void perf_event__id_header_size(struct perf_event *event)
         if (sample_type & PERF_SAMPLE_TIME)
                 size += sizeof(data->time);
 
+        if (sample_type & PERF_SAMPLE_IDENTIFIER)
+                size += sizeof(data->id);
+
         if (sample_type & PERF_SAMPLE_ID)
                 size += sizeof(data->id);
 
@@ -2712,7 +2715,7 @@ static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
 
                 hwc = &event->hw;
 
-                if (needs_unthr && hwc->interrupts == MAX_INTERRUPTS) {
+                if (hwc->interrupts == MAX_INTERRUPTS) {
                         hwc->interrupts = 0;
                         perf_log_throttle(event, 1);
                         event->pmu->start(event, 0);
@@ -2811,10 +2814,11 @@ done:
 #ifdef CONFIG_NO_HZ_FULL
 bool perf_event_can_stop_tick(void)
 {
-        if (list_empty(&__get_cpu_var(rotation_list)))
-                return true;
-        else
+        if (atomic_read(&nr_freq_events) ||
+            __this_cpu_read(perf_throttled_count))
                 return false;
+        else
+                return true;
 }
 #endif
 
@@ -3128,36 +3132,63 @@ static void free_event_rcu(struct rcu_head *head)
 static void ring_buffer_put(struct ring_buffer *rb);
 static void ring_buffer_detach(struct perf_event *event, struct ring_buffer *rb);
 
-static void free_event(struct perf_event *event)
+static void unaccount_event_cpu(struct perf_event *event, int cpu)
 {
-        irq_work_sync(&event->pending);
+        if (event->parent)
+                return;
+
+        if (has_branch_stack(event)) {
+                if (!(event->attach_state & PERF_ATTACH_TASK))
+                        atomic_dec(&per_cpu(perf_branch_stack_events, cpu));
+        }
+        if (is_cgroup_event(event))
+                atomic_dec(&per_cpu(perf_cgroup_events, cpu));
+}
+
+static void unaccount_event(struct perf_event *event)
+{
+        if (event->parent)
+                return;
+
+        if (event->attach_state & PERF_ATTACH_TASK)
+                static_key_slow_dec_deferred(&perf_sched_events);
+        if (event->attr.mmap || event->attr.mmap_data)
+                atomic_dec(&nr_mmap_events);
+        if (event->attr.comm)
+                atomic_dec(&nr_comm_events);
+        if (event->attr.task)
+                atomic_dec(&nr_task_events);
+        if (event->attr.freq)
+                atomic_dec(&nr_freq_events);
+        if (is_cgroup_event(event))
+                static_key_slow_dec_deferred(&perf_sched_events);
+        if (has_branch_stack(event))
+                static_key_slow_dec_deferred(&perf_sched_events);
+
+        unaccount_event_cpu(event, event->cpu);
+}
 
+static void __free_event(struct perf_event *event)
+{
         if (!event->parent) {
-                if (event->attach_state & PERF_ATTACH_TASK)
-                        static_key_slow_dec_deferred(&perf_sched_events);
-                if (event->attr.mmap || event->attr.mmap_data)
-                        atomic_dec(&nr_mmap_events);
-                if (event->attr.comm)
-                        atomic_dec(&nr_comm_events);
-                if (event->attr.task)
-                        atomic_dec(&nr_task_events);
                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
                         put_callchain_buffers();
-                if (is_cgroup_event(event)) {
-                        atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
-                        static_key_slow_dec_deferred(&perf_sched_events);
-                }
-
-                if (has_branch_stack(event)) {
-                        static_key_slow_dec_deferred(&perf_sched_events);
-                        /* is system-wide event */
-                        if (!(event->attach_state & PERF_ATTACH_TASK)) {
-                                atomic_dec(&per_cpu(perf_branch_stack_events,
-                                                    event->cpu));
-                        }
-                }
         }
 
+        if (event->destroy)
+                event->destroy(event);
+
+        if (event->ctx)
+                put_ctx(event->ctx);
+
+        call_rcu(&event->rcu_head, free_event_rcu);
+}
+static void free_event(struct perf_event *event)
+{
+        irq_work_sync(&event->pending);
+
+        unaccount_event(event);
+
         if (event->rb) {
                 struct ring_buffer *rb;
 
@@ -3180,13 +3211,8 @@ static void free_event(struct perf_event *event)
         if (is_cgroup_event(event))
                 perf_detach_cgroup(event);
 
-        if (event->destroy)
-                event->destroy(event);
 
-        if (event->ctx)
-                put_ctx(event->ctx);
-
-        call_rcu(&event->rcu_head, free_event_rcu);
+        __free_event(event);
 }
 
 int perf_event_release_kernel(struct perf_event *event)
@@ -3544,6 +3570,15 @@ static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
         case PERF_EVENT_IOC_PERIOD:
                 return perf_event_period(event, (u64 __user *)arg);
 
+        case PERF_EVENT_IOC_ID:
+        {
+                u64 id = primary_event_id(event);
+
+                if (copy_to_user((void __user *)arg, &id, sizeof(id)))
+                        return -EFAULT;
+                return 0;
+        }
+
         case PERF_EVENT_IOC_SET_OUTPUT:
         {
                 int ret;
@@ -3641,6 +3676,10 @@ void perf_event_update_userpage(struct perf_event *event)
         u64 enabled, running, now;
 
         rcu_read_lock();
+        rb = rcu_dereference(event->rb);
+        if (!rb)
+                goto unlock;
+
         /*
          * compute total_time_enabled, total_time_running
          * based on snapshot values taken when the event
@@ -3651,12 +3690,8 @@ void perf_event_update_userpage(struct perf_event *event)
          * NMI context
          */
         calc_timer_values(event, &now, &enabled, &running);
-        rb = rcu_dereference(event->rb);
-        if (!rb)
-                goto unlock;
 
         userpg = rb->user_page;
-
         /*
          * Disable preemption so as to not let the corresponding user-space
          * spin too long if we get preempted.
@@ -4251,7 +4286,7 @@ static void __perf_event_header__init_id(struct perf_event_header *header,
         if (sample_type & PERF_SAMPLE_TIME)
                 data->time = perf_clock();
 
-        if (sample_type & PERF_SAMPLE_ID)
+        if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
                 data->id = primary_event_id(event);
 
         if (sample_type & PERF_SAMPLE_STREAM_ID)
@@ -4290,6 +4325,9 @@ static void __perf_event__output_id_sample(struct perf_output_handle *handle,
 
         if (sample_type & PERF_SAMPLE_CPU)
                 perf_output_put(handle, data->cpu_entry);
+
+        if (sample_type & PERF_SAMPLE_IDENTIFIER)
+                perf_output_put(handle, data->id);
 }
 
 void perf_event__output_id_sample(struct perf_event *event,
@@ -4355,7 +4393,8 @@ static void perf_output_read_group(struct perf_output_handle *handle,
         list_for_each_entry(sub, &leader->sibling_list, group_entry) {
                 n = 0;
 
-                if (sub != event)
+                if ((sub != event) &&
+                    (sub->state == PERF_EVENT_STATE_ACTIVE))
                         sub->pmu->read(sub);
 
                 values[n++] = perf_event_count(sub);
@@ -4402,6 +4441,9 @@ void perf_output_sample(struct perf_output_handle *handle,
 
         perf_output_put(handle, *header);
 
+        if (sample_type & PERF_SAMPLE_IDENTIFIER)
+                perf_output_put(handle, data->id);
+
         if (sample_type & PERF_SAMPLE_IP)
                 perf_output_put(handle, data->ip);
 
@@ -4462,20 +4504,6 @@ void perf_output_sample(struct perf_output_handle *handle,
                 }
         }
 
-        if (!event->attr.watermark) {
-                int wakeup_events = event->attr.wakeup_events;
-
-                if (wakeup_events) {
-                        struct ring_buffer *rb = handle->rb;
-                        int events = local_inc_return(&rb->events);
-
-                        if (events >= wakeup_events) {
-                                local_sub(wakeup_events, &rb->events);
-                                local_inc(&rb->wakeup);
-                        }
-                }
-        }
-
         if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
                 if (data->br_stack) {
                         size_t size;
@@ -4511,16 +4539,31 @@ void perf_output_sample(struct perf_output_handle *handle,
                 }
         }
 
-        if (sample_type & PERF_SAMPLE_STACK_USER)
+        if (sample_type & PERF_SAMPLE_STACK_USER) {
                 perf_output_sample_ustack(handle,
                                           data->stack_user_size,
                                           data->regs_user.regs);
+        }
 
         if (sample_type & PERF_SAMPLE_WEIGHT)
                 perf_output_put(handle, data->weight);
 
         if (sample_type & PERF_SAMPLE_DATA_SRC)
                 perf_output_put(handle, data->data_src.val);
+
+        if (!event->attr.watermark) {
+                int wakeup_events = event->attr.wakeup_events;
+
+                if (wakeup_events) {
+                        struct ring_buffer *rb = handle->rb;
+                        int events = local_inc_return(&rb->events);
+
+                        if (events >= wakeup_events) {
+                                local_sub(wakeup_events, &rb->events);
+                                local_inc(&rb->wakeup);
+                        }
+                }
+        }
 }
 
 void perf_prepare_sample(struct perf_event_header *header,
@@ -4680,12 +4723,10 @@ perf_event_read_event(struct perf_event *event,
         perf_output_end(&handle);
 }
 
-typedef int  (perf_event_aux_match_cb)(struct perf_event *event, void *data);
 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
 
 static void
 perf_event_aux_ctx(struct perf_event_context *ctx,
-                   perf_event_aux_match_cb match,
                    perf_event_aux_output_cb output,
                    void *data)
 {
@@ -4696,15 +4737,12 @@ perf_event_aux_ctx(struct perf_event_context *ctx,
                         continue;
                 if (!event_filter_match(event))
                         continue;
-                if (match(event, data))
-                        output(event, data);
+                output(event, data);
         }
 }
 
 static void
-perf_event_aux(perf_event_aux_match_cb match,
-               perf_event_aux_output_cb output,
-               void *data,
+perf_event_aux(perf_event_aux_output_cb output, void *data,
                struct perf_event_context *task_ctx)
 {
         struct perf_cpu_context *cpuctx;
@@ -4717,7 +4755,7 @@ perf_event_aux(perf_event_aux_match_cb match,
                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
                 if (cpuctx->unique_pmu != pmu)
                         goto next;
-                perf_event_aux_ctx(&cpuctx->ctx, match, output, data);
+                perf_event_aux_ctx(&cpuctx->ctx, output, data);
                 if (task_ctx)
                         goto next;
                 ctxn = pmu->task_ctx_nr;
@@ -4725,14 +4763,14 @@ perf_event_aux(perf_event_aux_match_cb match,
                         goto next;
                 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
                 if (ctx)
-                        perf_event_aux_ctx(ctx, match, output, data);
+                        perf_event_aux_ctx(ctx, output, data);
 next:
                 put_cpu_ptr(pmu->pmu_cpu_context);
         }
 
         if (task_ctx) {
                 preempt_disable();
-                perf_event_aux_ctx(task_ctx, match, output, data);
+                perf_event_aux_ctx(task_ctx, output, data);
                 preempt_enable();
         }
         rcu_read_unlock();
@@ -4741,7 +4779,7 @@ next:
 /*
  * task tracking -- fork/exit
  *
- * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
+ * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
  */
 
 struct perf_task_event {
@@ -4759,6 +4797,13 @@ struct perf_task_event {
         } event_id;
 };
 
+static int perf_event_task_match(struct perf_event *event)
+{
+        return event->attr.comm  || event->attr.mmap ||
+               event->attr.mmap2 || event->attr.mmap_data ||
+               event->attr.task;
+}
+
 static void perf_event_task_output(struct perf_event *event,
                                    void *data)
 {
@@ -4768,6 +4813,9 @@ static void perf_event_task_output(struct perf_event *event,
         struct task_struct *task = task_event->task;
         int ret, size = task_event->event_id.header.size;
 
+        if (!perf_event_task_match(event))
+                return;
+
         perf_event_header__init_id(&task_event->event_id.header, &sample, event);
 
         ret = perf_output_begin(&handle, event,
@@ -4790,13 +4838,6 @@ out:
         task_event->event_id.header.size = size;
 }
 
-static int perf_event_task_match(struct perf_event *event,
-                                 void *data __maybe_unused)
-{
-        return event->attr.comm || event->attr.mmap ||
-               event->attr.mmap_data || event->attr.task;
-}
-
 static void perf_event_task(struct task_struct *task,
                               struct perf_event_context *task_ctx,
                               int new)
@@ -4825,8 +4866,7 @@ static void perf_event_task(struct task_struct *task,
                 },
         };
 
-        perf_event_aux(perf_event_task_match,
-                       perf_event_task_output,
+        perf_event_aux(perf_event_task_output,
                        &task_event,
                        task_ctx);
 }
@@ -4853,6 +4893,11 @@ struct perf_comm_event {
         } event_id;
 };
 
+static int perf_event_comm_match(struct perf_event *event)
+{
+        return event->attr.comm;
+}
+
 static void perf_event_comm_output(struct perf_event *event,
                                    void *data)
 {
@@ -4862,6 +4907,9 @@ static void perf_event_comm_output(struct perf_event *event,
         int size = comm_event->event_id.header.size;
         int ret;
 
+        if (!perf_event_comm_match(event))
+                return;
+
         perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
         ret = perf_output_begin(&handle, event,
                                 comm_event->event_id.header.size);
@@ -4883,12 +4931,6 @@ out:
         comm_event->event_id.header.size = size;
 }
 
-static int perf_event_comm_match(struct perf_event *event,
-                                 void *data __maybe_unused)
-{
-        return event->attr.comm;
-}
-
 static void perf_event_comm_event(struct perf_comm_event *comm_event)
 {
         char comm[TASK_COMM_LEN];
@@ -4903,8 +4945,7 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event)
 
         comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
 
-        perf_event_aux(perf_event_comm_match,
-                       perf_event_comm_output,
+        perf_event_aux(perf_event_comm_output,
                        comm_event,
                        NULL);
 }
@@ -4955,6 +4996,9 @@ struct perf_mmap_event {
 
         const char              *file_name;
         int                     file_size;
+        int                     maj, min;
+        u64                     ino;
+        u64                     ino_generation;
 
         struct {
                 struct perf_event_header        header;
@@ -4967,6 +5011,17 @@ struct perf_mmap_event {
         } event_id;
 };
 
+static int perf_event_mmap_match(struct perf_event *event,
+                                 void *data)
+{
+        struct perf_mmap_event *mmap_event = data;
+        struct vm_area_struct *vma = mmap_event->vma;
+        int executable = vma->vm_flags & VM_EXEC;
+
+        return (!executable && event->attr.mmap_data) ||
+               (executable && (event->attr.mmap || event->attr.mmap2));
+}
+
 static void perf_event_mmap_output(struct perf_event *event,
                                    void *data)
 {
@@ -4976,6 +5031,16 @@ static void perf_event_mmap_output(struct perf_event *event,
         int size = mmap_event->event_id.header.size;
         int ret;
 
+        if (!perf_event_mmap_match(event, data))
+                return;
+
+        if (event->attr.mmap2) {
+                mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
+                mmap_event->event_id.header.size += sizeof(mmap_event->maj);
+                mmap_event->event_id.header.size += sizeof(mmap_event->min);
+                mmap_event->event_id.header.size += sizeof(mmap_event->ino);
+        }
+
         perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
         ret = perf_output_begin(&handle, event,
                                 mmap_event->event_id.header.size);
@@ -4986,6 +5051,14 @@ static void perf_event_mmap_output(struct perf_event *event,
         mmap_event->event_id.tid = perf_event_tid(event, current);
 
         perf_output_put(&handle, mmap_event->event_id);
+
+        if (event->attr.mmap2) {
+                perf_output_put(&handle, mmap_event->maj);
+                perf_output_put(&handle, mmap_event->min);
+                perf_output_put(&handle, mmap_event->ino);
+                perf_output_put(&handle, mmap_event->ino_generation);
+        }
+
         __output_copy(&handle, mmap_event->file_name,
                                    mmap_event->file_size);
 
@@ -4996,21 +5069,12 @@ out:
         mmap_event->event_id.header.size = size;
 }
 
-static int perf_event_mmap_match(struct perf_event *event,
-                                 void *data)
-{
-        struct perf_mmap_event *mmap_event = data;
-        struct vm_area_struct *vma = mmap_event->vma;
-        int executable = vma->vm_flags & VM_EXEC;
-
-        return (!executable && event->attr.mmap_data) ||
-               (executable && event->attr.mmap);
-}
-
 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
 {
         struct vm_area_struct *vma = mmap_event->vma;
         struct file *file = vma->vm_file;
+        int maj = 0, min = 0;
+        u64 ino = 0, gen = 0;
         unsigned int size;
         char tmp[16];
         char *buf = NULL;
@@ -5019,6 +5083,8 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
         memset(tmp, 0, sizeof(tmp));
 
         if (file) {
+                struct inode *inode;
+                dev_t dev;
                 /*
                  * d_path works from the end of the rb backwards, so we
                  * need to add enough zero bytes after the string to handle
@@ -5034,6 +5100,13 @@ static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
                         name = strncpy(tmp, "//toolong", sizeof(tmp));
                         goto got_name;
                 }
+                inode = file_inode(vma->vm_file);
+                dev = inode->i_sb->s_dev;
+                ino = inode->i_ino;
+                gen = inode->i_generation;
+                maj = MAJOR(dev);
+                min = MINOR(dev);
+
         } else {
                 if (arch_vma_name(mmap_event->vma)) {
                         name = strncpy(tmp, arch_vma_name(mmap_event->vma),
@@ -5064,14 +5137,17 @@ got_name:
 
         mmap_event->file_name = name;
         mmap_event->file_size = size;
+        mmap_event->maj = maj;
+        mmap_event->min = min;
+        mmap_event->ino = ino;
+        mmap_event->ino_generation = gen;
 
         if (!(vma->vm_flags & VM_EXEC))
                 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
 
         mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
 
-        perf_event_aux(perf_event_mmap_match,
-                       perf_event_mmap_output,
+        perf_event_aux(perf_event_mmap_output,
                        mmap_event,
                        NULL);
 
@@ -5101,6 +5177,10 @@ void perf_event_mmap(struct vm_area_struct *vma)
                         .len    = vma->vm_end - vma->vm_start,
                         .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
                 },
+                /* .maj (attr_mmap2 only) */
+                /* .min (attr_mmap2 only) */
+                /* .ino (attr_mmap2 only) */
+                /* .ino_generation (attr_mmap2 only) */
         };
 
         perf_event_mmap_event(&mmap_event);
@@ -5178,6 +5258,7 @@ static int __perf_event_overflow(struct perf_event *event,
                         __this_cpu_inc(perf_throttled_count);
                         hwc->interrupts = MAX_INTERRUPTS;
                         perf_log_throttle(event, 0);
+                        tick_nohz_full_kick();
                         ret = 1;
                 }
         }
@@ -6234,8 +6315,6 @@ perf_event_mux_interval_ms_store(struct device *dev,
         return count;
 }
 
-#define __ATTR_RW(attr) __ATTR(attr, 0644, attr##_show, attr##_store)
-
 static struct device_attribute pmu_dev_attrs[] = {
         __ATTR_RO(type),
         __ATTR_RW(perf_event_mux_interval_ms),
@@ -6445,6 +6524,44 @@ unlock:
         return pmu;
 }
 
+static void account_event_cpu(struct perf_event *event, int cpu)
+{
+        if (event->parent)
+                return;
+
+        if (has_branch_stack(event)) {
+                if (!(event->attach_state & PERF_ATTACH_TASK))
+                        atomic_inc(&per_cpu(perf_branch_stack_events, cpu));
+        }
+        if (is_cgroup_event(event))
+                atomic_inc(&per_cpu(perf_cgroup_events, cpu));
+}
+
+static void account_event(struct perf_event *event)
+{
+        if (event->parent)
+                return;
+
+        if (event->attach_state & PERF_ATTACH_TASK)
+                static_key_slow_inc(&perf_sched_events.key);
+        if (event->attr.mmap || event->attr.mmap_data)
+                atomic_inc(&nr_mmap_events);
+        if (event->attr.comm)
+                atomic_inc(&nr_comm_events);
+        if (event->attr.task)
+                atomic_inc(&nr_task_events);
+        if (event->attr.freq) {
+                if (atomic_inc_return(&nr_freq_events) == 1)
+                        tick_nohz_full_kick_all();
+        }
+        if (has_branch_stack(event))
+                static_key_slow_inc(&perf_sched_events.key);
+        if (is_cgroup_event(event))
+                static_key_slow_inc(&perf_sched_events.key);
+
+        account_event_cpu(event, event->cpu);
+}
+
 /*
  * Allocate and initialize a event structure
  */
@@ -6459,7 +6576,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
         struct pmu *pmu;
         struct perf_event *event;
         struct hw_perf_event *hwc;
-        long err;
+        long err = -EINVAL;
 
         if ((unsigned)cpu >= nr_cpu_ids) {
                 if (!task || cpu != -1)
@@ -6542,49 +6659,35 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
          * we currently do not support PERF_FORMAT_GROUP on inherited events
          */
         if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
-                goto done;
+                goto err_ns;
 
         pmu = perf_init_event(event);
-
-done:
-        err = 0;
         if (!pmu)
-                err = -EINVAL;
-        else if (IS_ERR(pmu))
+                goto err_ns;
+        else if (IS_ERR(pmu)) {
                 err = PTR_ERR(pmu);
-
-        if (err) {
-                if (event->ns)
-                        put_pid_ns(event->ns);
-                kfree(event);
-                return ERR_PTR(err);
+                goto err_ns;
         }
 
         if (!event->parent) {
-                if (event->attach_state & PERF_ATTACH_TASK)
-                        static_key_slow_inc(&perf_sched_events.key);
-                if (event->attr.mmap || event->attr.mmap_data)
-                        atomic_inc(&nr_mmap_events);
-                if (event->attr.comm)
-                        atomic_inc(&nr_comm_events);
-                if (event->attr.task)
-                        atomic_inc(&nr_task_events);
                 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
                         err = get_callchain_buffers();
-                        if (err) {
-                                free_event(event);
-                                return ERR_PTR(err);
-                        }
-                }
-                if (has_branch_stack(event)) {
-                        static_key_slow_inc(&perf_sched_events.key);
-                        if (!(event->attach_state & PERF_ATTACH_TASK))
-                                atomic_inc(&per_cpu(perf_branch_stack_events,
-                                                    event->cpu));
+                        if (err)
+                                goto err_pmu;
                 }
         }
 
         return event;
+
+err_pmu:
+        if (event->destroy)
+                event->destroy(event);
+err_ns:
+        if (event->ns)
+                put_pid_ns(event->ns);
+        kfree(event);
+
+        return ERR_PTR(err);
 }
 
 static int perf_copy_attr(struct perf_event_attr __user *uattr,
@@ -6866,17 +6969,14 @@ SYSCALL_DEFINE5(perf_event_open,
 
         if (flags & PERF_FLAG_PID_CGROUP) {
                 err = perf_cgroup_connect(pid, event, &attr, group_leader);
-                if (err)
-                        goto err_alloc;
-                /*
-                 * one more event:
-                 * - that has cgroup constraint on event->cpu
-                 * - that may need work on context switch
-                 */
-                atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
-                static_key_slow_inc(&perf_sched_events.key);
+                if (err) {
+                        __free_event(event);
+                        goto err_task;
+                }
         }
 
+        account_event(event);
+
         /*
          * Special case software events and allow them to be part of
          * any hardware group.
@@ -7072,6 +7172,8 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
                 goto err;
         }
 
+        account_event(event);
+
         ctx = find_get_context(event->pmu, task, cpu);
         if (IS_ERR(ctx)) {
                 err = PTR_ERR(ctx);
@@ -7108,6 +7210,7 @@ void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
         list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
                                  event_entry) {
                 perf_remove_from_context(event);
+                unaccount_event_cpu(event, src_cpu);
                 put_ctx(src_ctx);
                 list_add(&event->event_entry, &events);
         }
@@ -7120,6 +7223,7 @@ void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
                 list_del(&event->event_entry);
                 if (event->state >= PERF_EVENT_STATE_OFF)
                         event->state = PERF_EVENT_STATE_INACTIVE;
+                account_event_cpu(event, dst_cpu);
                 perf_install_in_context(dst_ctx, event, dst_cpu);
                 get_ctx(dst_ctx);
         }
@@ -7630,7 +7734,7 @@ static void __init perf_event_init_all_cpus(void)
         }
 }
 
-static void __cpuinit perf_event_init_cpu(int cpu)
+static void perf_event_init_cpu(int cpu)
 {
         struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
 
@@ -7719,7 +7823,7 @@ static struct notifier_block perf_reboot_notifier = {
         .priority = INT_MIN,
 };
 
-static int __cpuinit
+static int
 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
 {
         unsigned int cpu = (long)hcpu;
@@ -7800,7 +7904,8 @@ unlock:
 device_initcall(perf_event_sysfs_init);
 
 #ifdef CONFIG_CGROUP_PERF
-static struct cgroup_subsys_state *perf_cgroup_css_alloc(struct cgroup *cont)
+static struct cgroup_subsys_state *
+perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 {
         struct perf_cgroup *jc;
 
@@ -7817,11 +7922,10 @@ static struct cgroup_subsys_state *perf_cgroup_css_alloc(struct cgroup *cont)
         return &jc->css;
 }
 
-static void perf_cgroup_css_free(struct cgroup *cont)
+static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
 {
-        struct perf_cgroup *jc;
-        jc = container_of(cgroup_subsys_state(cont, perf_subsys_id),
-                          struct perf_cgroup, css);
+        struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
+
         free_percpu(jc->info);
         kfree(jc);
 }
@@ -7833,15 +7937,17 @@ static int __perf_cgroup_move(void *info)
         return 0;
 }
 
-static void perf_cgroup_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
+static void perf_cgroup_attach(struct cgroup_subsys_state *css,
+                               struct cgroup_taskset *tset)
 {
         struct task_struct *task;
 
-        cgroup_taskset_for_each(task, cgrp, tset)
+        cgroup_taskset_for_each(task, css, tset)
                 task_function_call(task, __perf_cgroup_move, task);
 }
 
-static void perf_cgroup_exit(struct cgroup *cgrp, struct cgroup *old_cgrp,
+static void perf_cgroup_exit(struct cgroup_subsys_state *css,
+                             struct cgroup_subsys_state *old_css,
                              struct task_struct *task)
 {
         /*