3 files changed, 89 insertions, 72 deletions

diff --git a/kernel/events/core.c b/kernel/events/core.c
index 2ba890450d15..dbccf83c134d 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -372,6 +372,8 @@ void perf_cgroup_switch(struct task_struct *task, int mode)
 
         list_for_each_entry_rcu(pmu, &pmus, entry) {
                 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
+                if (cpuctx->unique_pmu != pmu)
+                        continue; /* ensure we process each cpuctx once */
 
                 /*
                  * perf_cgroup_events says at least one
@@ -395,9 +397,10 @@ void perf_cgroup_switch(struct task_struct *task, int mode)
 
                         if (mode & PERF_CGROUP_SWIN) {
                                 WARN_ON_ONCE(cpuctx->cgrp);
-                                /* set cgrp before ctxsw in to
-                                 * allow event_filter_match() to not
-                                 * have to pass task around
+                                /*
+                                 * set cgrp before ctxsw in to allow
+                                 * event_filter_match() to not have to pass
+                                 * task around
                                  */
                                 cpuctx->cgrp = perf_cgroup_from_task(task);
                                 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
@@ -468,14 +471,13 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
 {
         struct perf_cgroup *cgrp;
         struct cgroup_subsys_state *css;
-        struct file *file;
-        int ret = 0, fput_needed;
+        struct fd f = fdget(fd);
+        int ret = 0;
 
-        file = fget_light(fd, &fput_needed);
-        if (!file)
+        if (!f.file)
                 return -EBADF;
 
-        css = cgroup_css_from_dir(file, perf_subsys_id);
+        css = cgroup_css_from_dir(f.file, perf_subsys_id);
         if (IS_ERR(css)) {
                 ret = PTR_ERR(css);
                 goto out;
@@ -501,7 +503,7 @@ static inline int perf_cgroup_connect(int fd, struct perf_event *event,
                 ret = -EINVAL;
         }
 out:
-        fput_light(file, fput_needed);
+        fdput(f);
         return ret;
 }
 
@@ -1254,7 +1256,7 @@ retry:
 /*
  * Cross CPU call to disable a performance event
  */
-static int __perf_event_disable(void *info)
+int __perf_event_disable(void *info)
 {
         struct perf_event *event = info;
         struct perf_event_context *ctx = event->ctx;
@@ -2936,12 +2938,12 @@ EXPORT_SYMBOL_GPL(perf_event_release_kernel);
 /*
  * Called when the last reference to the file is gone.
  */
-static int perf_release(struct inode *inode, struct file *file)
+static void put_event(struct perf_event *event)
 {
-        struct perf_event *event = file->private_data;
         struct task_struct *owner;
 
-        file->private_data = NULL;
+        if (!atomic_long_dec_and_test(&event->refcount))
+                return;
 
         rcu_read_lock();
         owner = ACCESS_ONCE(event->owner);
@@ -2976,7 +2978,13 @@ static int perf_release(struct inode *inode, struct file *file)
                 put_task_struct(owner);
         }
 
-        return perf_event_release_kernel(event);
+        perf_event_release_kernel(event);
+}
+
+static int perf_release(struct inode *inode, struct file *file)
+{
+        put_event(file->private_data);
+        return 0;
 }
 
 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
@@ -3228,21 +3236,18 @@ unlock:
 
 static const struct file_operations perf_fops;
 
-static struct perf_event *perf_fget_light(int fd, int *fput_needed)
+static inline int perf_fget_light(int fd, struct fd *p)
 {
-        struct file *file;
-
-        file = fget_light(fd, fput_needed);
-        if (!file)
-                return ERR_PTR(-EBADF);
+        struct fd f = fdget(fd);
+        if (!f.file)
+                return -EBADF;
 
-        if (file->f_op != &perf_fops) {
-                fput_light(file, *fput_needed);
-                *fput_needed = 0;
-                return ERR_PTR(-EBADF);
+        if (f.file->f_op != &perf_fops) {
+                fdput(f);
+                return -EBADF;
         }
-
-        return file->private_data;
+        *p = f;
+        return 0;
 }
 
 static int perf_event_set_output(struct perf_event *event,
@@ -3274,20 +3279,19 @@ static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 
         case PERF_EVENT_IOC_SET_OUTPUT:
         {
-                struct perf_event *output_event = NULL;
-                int fput_needed = 0;
                 int ret;
-
                 if (arg != -1) {
-                        output_event = perf_fget_light(arg, &fput_needed);
-                        if (IS_ERR(output_event))
-                                return PTR_ERR(output_event);
+                        struct perf_event *output_event;
+                        struct fd output;
+                        ret = perf_fget_light(arg, &output);
+                        if (ret)
+                                return ret;
+                        output_event = output.file->private_data;
+                        ret = perf_event_set_output(event, output_event);
+                        fdput(output);
+                } else {
+                        ret = perf_event_set_output(event, NULL);
                 }
-
-                ret = perf_event_set_output(event, output_event);
-                if (output_event)
-                        fput_light(output_event->filp, fput_needed);
-
                 return ret;
         }
 
@@ -3670,7 +3674,7 @@ unlock:
                 atomic_inc(&event->mmap_count);
         mutex_unlock(&event->mmap_mutex);
 
-        vma->vm_flags |= VM_RESERVED;
+        vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
         vma->vm_ops = &perf_mmap_vmops;
 
         return ret;
@@ -4411,7 +4415,7 @@ static void perf_event_task_event(struct perf_task_event *task_event)
         rcu_read_lock();
         list_for_each_entry_rcu(pmu, &pmus, entry) {
                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
-                if (cpuctx->active_pmu != pmu)
+                if (cpuctx->unique_pmu != pmu)
                         goto next;
                 perf_event_task_ctx(&cpuctx->ctx, task_event);
 
@@ -4557,7 +4561,7 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event)
         rcu_read_lock();
         list_for_each_entry_rcu(pmu, &pmus, entry) {
                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
-                if (cpuctx->active_pmu != pmu)
+                if (cpuctx->unique_pmu != pmu)
                         goto next;
                 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
 
@@ -4753,7 +4757,7 @@ got_name:
         rcu_read_lock();
         list_for_each_entry_rcu(pmu, &pmus, entry) {
                 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
-                if (cpuctx->active_pmu != pmu)
+                if (cpuctx->unique_pmu != pmu)
                         goto next;
                 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
                                         vma->vm_flags & VM_EXEC);
@@ -5854,8 +5858,8 @@ static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
 
                 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
 
-                if (cpuctx->active_pmu == old_pmu)
-                        cpuctx->active_pmu = pmu;
+                if (cpuctx->unique_pmu == old_pmu)
+                        cpuctx->unique_pmu = pmu;
         }
 }
 
@@ -5990,7 +5994,7 @@ skip_type:
                 cpuctx->ctx.pmu = pmu;
                 cpuctx->jiffies_interval = 1;
                 INIT_LIST_HEAD(&cpuctx->rotation_list);
-                cpuctx->active_pmu = pmu;
+                cpuctx->unique_pmu = pmu;
         }
 
 got_cpu_context:
@@ -6142,6 +6146,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
 
         mutex_init(&event->mmap_mutex);
 
+        atomic_long_set(&event->refcount, 1);
         event->cpu              = cpu;
         event->attr             = *attr;
         event->group_leader     = group_leader;
@@ -6434,12 +6439,11 @@ SYSCALL_DEFINE5(perf_event_open,
         struct perf_event_attr attr;
         struct perf_event_context *ctx;
         struct file *event_file = NULL;
-        struct file *group_file = NULL;
+        struct fd group = {NULL, 0};
         struct task_struct *task = NULL;
         struct pmu *pmu;
         int event_fd;
         int move_group = 0;
-        int fput_needed = 0;
         int err;
 
         /* for future expandability... */
@@ -6469,17 +6473,15 @@ SYSCALL_DEFINE5(perf_event_open,
         if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
                 return -EINVAL;
 
-        event_fd = get_unused_fd_flags(O_RDWR);
+        event_fd = get_unused_fd();
         if (event_fd < 0)
                 return event_fd;
 
         if (group_fd != -1) {
-                group_leader = perf_fget_light(group_fd, &fput_needed);
-                if (IS_ERR(group_leader)) {
-                        err = PTR_ERR(group_leader);
+                err = perf_fget_light(group_fd, &group);
+                if (err)
                         goto err_fd;
-                }
-                group_file = group_leader->filp;
+                group_leader = group.file->private_data;
                 if (flags & PERF_FLAG_FD_OUTPUT)
                         output_event = group_leader;
                 if (flags & PERF_FLAG_FD_NO_GROUP)
@@ -6616,7 +6618,6 @@ SYSCALL_DEFINE5(perf_event_open,
                 put_ctx(gctx);
         }
 
-        event->filp = event_file;
         WARN_ON_ONCE(ctx->parent_ctx);
         mutex_lock(&ctx->mutex);
 
@@ -6656,7 +6657,7 @@ SYSCALL_DEFINE5(perf_event_open,
          * of the group leader will find the pointer to itself in
          * perf_group_detach().
          */
-        fput_light(group_file, fput_needed);
+        fdput(group);
         fd_install(event_fd, event_file);
         return event_fd;
 
@@ -6670,7 +6671,7 @@ err_task:
         if (task)
                 put_task_struct(task);
 err_group_fd:
-        fput_light(group_file, fput_needed);
+        fdput(group);
 err_fd:
         put_unused_fd(event_fd);
         return err;
@@ -6710,7 +6711,6 @@ perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
                 goto err_free;
         }
 
-        event->filp = NULL;
         WARN_ON_ONCE(ctx->parent_ctx);
         mutex_lock(&ctx->mutex);
         perf_install_in_context(ctx, event, cpu);
@@ -6792,7 +6792,7 @@ static void sync_child_event(struct perf_event *child_event,
          * Release the parent event, if this was the last
          * reference to it.
          */
-        fput(parent_event->filp);
+        put_event(parent_event);
 }
 
 static void
@@ -6868,9 +6868,8 @@ static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
          *
          *   __perf_event_exit_task()
          *     sync_child_event()
-         *       fput(parent_event->filp)
-         *         perf_release()
-         *           mutex_lock(&ctx->mutex)
+         *       put_event()
+         *         mutex_lock(&ctx->mutex)
          *
          * But since its the parent context it won't be the same instance.
          */
@@ -6938,7 +6937,7 @@ static void perf_free_event(struct perf_event *event,
         list_del_init(&event->child_list);
         mutex_unlock(&parent->child_mutex);
 
-        fput(parent->filp);
+        put_event(parent);
 
         perf_group_detach(event);
         list_del_event(event, ctx);
@@ -7018,6 +7017,12 @@ inherit_event(struct perf_event *parent_event,
                                            NULL, NULL);
         if (IS_ERR(child_event))
                 return child_event;
+
+        if (!atomic_long_inc_not_zero(&parent_event->refcount)) {
+                free_event(child_event);
+                return NULL;
+        }
+
         get_ctx(child_ctx);
 
         /*
@@ -7059,14 +7064,6 @@ inherit_event(struct perf_event *parent_event,
         raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
 
         /*
-         * Get a reference to the parent filp - we will fput it
-         * when the child event exits. This is safe to do because
-         * we are in the parent and we know that the filp still
-         * exists and has a nonzero count:
-         */
-        atomic_long_inc(&parent_event->filp->f_count);
-
-        /*
          * Link this into the parent event's child list
          */
         WARN_ON_ONCE(parent_event->ctx->parent_ctx);
@@ -7499,5 +7496,12 @@ struct cgroup_subsys perf_subsys = {
         .destroy        = perf_cgroup_destroy,
         .exit           = perf_cgroup_exit,
         .attach         = perf_cgroup_attach,
+
+        /*
+         * perf_event cgroup doesn't handle nesting correctly.
+         * ctx->nr_cgroups adjustments should be propagated through the
+         * cgroup hierarchy.  Fix it and remove the following.
+         */
+        .broken_hierarchy = true,
 };
 #endif /* CONFIG_CGROUP_PERF */
diff --git a/kernel/events/hw_breakpoint.c b/kernel/events/hw_breakpoint.c
index bb38c4d3ee12..9a7b487c6fe2 100644
--- a/kernel/events/hw_breakpoint.c
+++ b/kernel/events/hw_breakpoint.c
@@ -453,7 +453,16 @@ int modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *att
         int old_type = bp->attr.bp_type;
         int err = 0;
 
-        perf_event_disable(bp);
+        /*
+         * modify_user_hw_breakpoint can be invoked with IRQs disabled and hence it
+         * will not be possible to raise IPIs that invoke __perf_event_disable.
+         * So call the function directly after making sure we are targeting the
+         * current task.
+         */
+        if (irqs_disabled() && bp->ctx && bp->ctx->task == current)
+                __perf_event_disable(bp);
+        else
+                perf_event_disable(bp);
 
         bp->attr.bp_addr = attr->bp_addr;
         bp->attr.bp_type = attr->bp_type;
diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index c92651d619ca..5cc4e7e42e68 100644
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -144,10 +144,14 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
         spinlock_t *ptl;
         pte_t *ptep;
         int err;
+        /* For mmu_notifiers */
+        const unsigned long mmun_start = addr;
+        const unsigned long mmun_end   = addr + PAGE_SIZE;
 
         /* For try_to_free_swap() and munlock_vma_page() below */
         lock_page(page);
 
+        mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
         err = -EAGAIN;
         ptep = page_check_address(page, mm, addr, &ptl, 0);
         if (!ptep)
@@ -176,6 +180,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
 
         err = 0;
  unlock:
+        mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
         unlock_page(page);
         return err;
 }
@@ -690,7 +695,6 @@ static struct map_info *
 build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
 {
         unsigned long pgoff = offset >> PAGE_SHIFT;
-        struct prio_tree_iter iter;
         struct vm_area_struct *vma;
         struct map_info *curr = NULL;
         struct map_info *prev = NULL;
@@ -699,7 +703,7 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register)
 
  again:
         mutex_lock(&mapping->i_mmap_mutex);
-        vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
+        vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
                 if (!valid_vma(vma, is_register))
                         continue;