1 files changed, 83 insertions, 108 deletions

diff --git a/kernel/events/hw_breakpoint.c b/kernel/events/hw_breakpoint.c
index 20185ea64aa6..1559fb0b9296 100644
--- a/kernel/events/hw_breakpoint.c
+++ b/kernel/events/hw_breakpoint.c
@@ -46,23 +46,26 @@
 #include <linux/smp.h>
 
 #include <linux/hw_breakpoint.h>
-
-
 /*
  * Constraints data
  */
+struct bp_cpuinfo {
+        /* Number of pinned cpu breakpoints in a cpu */
+        unsigned int    cpu_pinned;
+        /* tsk_pinned[n] is the number of tasks having n+1 breakpoints */
+        unsigned int    *tsk_pinned;
+        /* Number of non-pinned cpu/task breakpoints in a cpu */
+        unsigned int    flexible; /* XXX: placeholder, see fetch_this_slot() */
+};
 
-/* Number of pinned cpu breakpoints in a cpu */
-static DEFINE_PER_CPU(unsigned int, nr_cpu_bp_pinned[TYPE_MAX]);
-
-/* Number of pinned task breakpoints in a cpu */
-static DEFINE_PER_CPU(unsigned int *, nr_task_bp_pinned[TYPE_MAX]);
-
-/* Number of non-pinned cpu/task breakpoints in a cpu */
-static DEFINE_PER_CPU(unsigned int, nr_bp_flexible[TYPE_MAX]);
-
+static DEFINE_PER_CPU(struct bp_cpuinfo, bp_cpuinfo[TYPE_MAX]);
 static int nr_slots[TYPE_MAX];
 
+static struct bp_cpuinfo *get_bp_info(int cpu, enum bp_type_idx type)
+{
+        return per_cpu_ptr(bp_cpuinfo + type, cpu);
+}
+
 /* Keep track of the breakpoints attached to tasks */
 static LIST_HEAD(bp_task_head);
 
@@ -96,8 +99,8 @@ static inline enum bp_type_idx find_slot_idx(struct perf_event *bp)
  */
 static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type)
 {
+        unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned;
         int i;
-        unsigned int *tsk_pinned = per_cpu(nr_task_bp_pinned[type], cpu);
 
         for (i = nr_slots[type] - 1; i >= 0; i--) {
                 if (tsk_pinned[i] > 0)
@@ -127,6 +130,13 @@ static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type)
         return count;
 }
 
+static const struct cpumask *cpumask_of_bp(struct perf_event *bp)
+{
+        if (bp->cpu >= 0)
+                return cpumask_of(bp->cpu);
+        return cpu_possible_mask;
+}
+
 /*
  * Report the number of pinned/un-pinned breakpoints we have in
  * a given cpu (cpu > -1) or in all of them (cpu = -1).
@@ -135,25 +145,15 @@ static void
 fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
                     enum bp_type_idx type)
 {
-        int cpu = bp->cpu;
-        struct task_struct *tsk = bp->hw.bp_target;
-
-        if (cpu >= 0) {
-                slots->pinned = per_cpu(nr_cpu_bp_pinned[type], cpu);
-                if (!tsk)
-                        slots->pinned += max_task_bp_pinned(cpu, type);
-                else
-                        slots->pinned += task_bp_pinned(cpu, bp, type);
-                slots->flexible = per_cpu(nr_bp_flexible[type], cpu);
-
-                return;
-        }
+        const struct cpumask *cpumask = cpumask_of_bp(bp);
+        int cpu;
 
-        for_each_possible_cpu(cpu) {
-                unsigned int nr;
+        for_each_cpu(cpu, cpumask) {
+                struct bp_cpuinfo *info = get_bp_info(cpu, type);
+                int nr;
 
-                nr = per_cpu(nr_cpu_bp_pinned[type], cpu);
-                if (!tsk)
+                nr = info->cpu_pinned;
+                if (!bp->hw.bp_target)
                         nr += max_task_bp_pinned(cpu, type);
                 else
                         nr += task_bp_pinned(cpu, bp, type);
@@ -161,8 +161,7 @@ fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp,
                 if (nr > slots->pinned)
                         slots->pinned = nr;
 
-                nr = per_cpu(nr_bp_flexible[type], cpu);
-
+                nr = info->flexible;
                 if (nr > slots->flexible)
                         slots->flexible = nr;
         }
@@ -182,29 +181,19 @@ fetch_this_slot(struct bp_busy_slots *slots, int weight)
 /*
  * Add a pinned breakpoint for the given task in our constraint table
  */
-static void toggle_bp_task_slot(struct perf_event *bp, int cpu, bool enable,
+static void toggle_bp_task_slot(struct perf_event *bp, int cpu,
                                 enum bp_type_idx type, int weight)
 {
-        unsigned int *tsk_pinned;
-        int old_count = 0;
-        int old_idx = 0;
-        int idx = 0;
-
-        old_count = task_bp_pinned(cpu, bp, type);
-        old_idx = old_count - 1;
-        idx = old_idx + weight;
-
-        /* tsk_pinned[n] is the number of tasks having n breakpoints */
-        tsk_pinned = per_cpu(nr_task_bp_pinned[type], cpu);
-        if (enable) {
-                tsk_pinned[idx]++;
-                if (old_count > 0)
-                        tsk_pinned[old_idx]--;
-        } else {
-                tsk_pinned[idx]--;
-                if (old_count > 0)
-                        tsk_pinned[old_idx]++;
-        }
+        unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned;
+        int old_idx, new_idx;
+
+        old_idx = task_bp_pinned(cpu, bp, type) - 1;
+        new_idx = old_idx + weight;
+
+        if (old_idx >= 0)
+                tsk_pinned[old_idx]--;
+        if (new_idx >= 0)
+                tsk_pinned[new_idx]++;
 }
 
 /*
@@ -214,33 +203,26 @@ static void
 toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type,
                int weight)
 {
-        int cpu = bp->cpu;
-        struct task_struct *tsk = bp->hw.bp_target;
+        const struct cpumask *cpumask = cpumask_of_bp(bp);
+        int cpu;
 
-        /* Pinned counter cpu profiling */
-        if (!tsk) {
+        if (!enable)
+                weight = -weight;
 
-                if (enable)
-                        per_cpu(nr_cpu_bp_pinned[type], bp->cpu) += weight;
-                else
-                        per_cpu(nr_cpu_bp_pinned[type], bp->cpu) -= weight;
+        /* Pinned counter cpu profiling */
+        if (!bp->hw.bp_target) {
+                get_bp_info(bp->cpu, type)->cpu_pinned += weight;
                 return;
         }
 
         /* Pinned counter task profiling */
-
-        if (!enable)
-                list_del(&bp->hw.bp_list);
-
-        if (cpu >= 0) {
-                toggle_bp_task_slot(bp, cpu, enable, type, weight);
-        } else {
-                for_each_possible_cpu(cpu)
-                        toggle_bp_task_slot(bp, cpu, enable, type, weight);
-        }
+        for_each_cpu(cpu, cpumask)
+                toggle_bp_task_slot(bp, cpu, type, weight);
 
         if (enable)
                 list_add_tail(&bp->hw.bp_list, &bp_task_head);
+        else
+                list_del(&bp->hw.bp_list);
 }
 
 /*
@@ -261,8 +243,8 @@ __weak void arch_unregister_hw_breakpoint(struct perf_event *bp)
  *
  *   - If attached to a single cpu, check:
  *
- *       (per_cpu(nr_bp_flexible, cpu) || (per_cpu(nr_cpu_bp_pinned, cpu)
- *           + max(per_cpu(nr_task_bp_pinned, cpu)))) < HBP_NUM
+ *       (per_cpu(info->flexible, cpu) || (per_cpu(info->cpu_pinned, cpu)
+ *           + max(per_cpu(info->tsk_pinned, cpu)))) < HBP_NUM
  *
  *       -> If there are already non-pinned counters in this cpu, it means
  *          there is already a free slot for them.
@@ -272,8 +254,8 @@ __weak void arch_unregister_hw_breakpoint(struct perf_event *bp)
  *
  *   - If attached to every cpus, check:
  *
- *       (per_cpu(nr_bp_flexible, *) || (max(per_cpu(nr_cpu_bp_pinned, *))
- *           + max(per_cpu(nr_task_bp_pinned, *)))) < HBP_NUM
+ *       (per_cpu(info->flexible, *) || (max(per_cpu(info->cpu_pinned, *))
+ *           + max(per_cpu(info->tsk_pinned, *)))) < HBP_NUM
  *
  *       -> This is roughly the same, except we check the number of per cpu
  *          bp for every cpu and we keep the max one. Same for the per tasks
@@ -284,16 +266,16 @@ __weak void arch_unregister_hw_breakpoint(struct perf_event *bp)
  *
  *   - If attached to a single cpu, check:
  *
- *       ((per_cpu(nr_bp_flexible, cpu) > 1) + per_cpu(nr_cpu_bp_pinned, cpu)
- *            + max(per_cpu(nr_task_bp_pinned, cpu))) < HBP_NUM
+ *       ((per_cpu(info->flexible, cpu) > 1) + per_cpu(info->cpu_pinned, cpu)
+ *            + max(per_cpu(info->tsk_pinned, cpu))) < HBP_NUM
  *
- *       -> Same checks as before. But now the nr_bp_flexible, if any, must keep
+ *       -> Same checks as before. But now the info->flexible, if any, must keep
  *          one register at least (or they will never be fed).
  *
  *   - If attached to every cpus, check:
  *
- *       ((per_cpu(nr_bp_flexible, *) > 1) + max(per_cpu(nr_cpu_bp_pinned, *))
- *            + max(per_cpu(nr_task_bp_pinned, *))) < HBP_NUM
+ *       ((per_cpu(info->flexible, *) > 1) + max(per_cpu(info->cpu_pinned, *))
+ *            + max(per_cpu(info->tsk_pinned, *))) < HBP_NUM
  */
 static int __reserve_bp_slot(struct perf_event *bp)
 {
@@ -518,8 +500,8 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
                             perf_overflow_handler_t triggered,
                             void *context)
 {
-        struct perf_event * __percpu *cpu_events, **pevent, *bp;
-        long err;
+        struct perf_event * __percpu *cpu_events, *bp;
+        long err = 0;
         int cpu;
 
         cpu_events = alloc_percpu(typeof(*cpu_events));
@@ -528,31 +510,21 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr,
 
         get_online_cpus();
         for_each_online_cpu(cpu) {
-                pevent = per_cpu_ptr(cpu_events, cpu);
                 bp = perf_event_create_kernel_counter(attr, cpu, NULL,
                                                       triggered, context);
-
-                *pevent = bp;
-
                 if (IS_ERR(bp)) {
                         err = PTR_ERR(bp);
-                        goto fail;
+                        break;
                 }
-        }
-        put_online_cpus();
 
-        return cpu_events;
-
-fail:
-        for_each_online_cpu(cpu) {
-                pevent = per_cpu_ptr(cpu_events, cpu);
-                if (IS_ERR(*pevent))
-                        break;
-                unregister_hw_breakpoint(*pevent);
+                per_cpu(*cpu_events, cpu) = bp;
         }
         put_online_cpus();
 
-        free_percpu(cpu_events);
+        if (likely(!err))
+                return cpu_events;
+
+        unregister_wide_hw_breakpoint(cpu_events);
         return (void __percpu __force *)ERR_PTR(err);
 }
 EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
@@ -564,12 +536,10 @@ EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint);
 void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events)
 {
         int cpu;
-        struct perf_event **pevent;
 
-        for_each_possible_cpu(cpu) {
-                pevent = per_cpu_ptr(cpu_events, cpu);
-                unregister_hw_breakpoint(*pevent);
-        }
+        for_each_possible_cpu(cpu)
+                unregister_hw_breakpoint(per_cpu(*cpu_events, cpu));
+
         free_percpu(cpu_events);
 }
 EXPORT_SYMBOL_GPL(unregister_wide_hw_breakpoint);
@@ -612,6 +582,11 @@ static int hw_breakpoint_add(struct perf_event *bp, int flags)
         if (!(flags & PERF_EF_START))
                 bp->hw.state = PERF_HES_STOPPED;
 
+        if (is_sampling_event(bp)) {
+                bp->hw.last_period = bp->hw.sample_period;
+                perf_swevent_set_period(bp);
+        }
+
         return arch_install_hw_breakpoint(bp);
 }
 
@@ -650,7 +625,6 @@ static struct pmu perf_breakpoint = {
 
 int __init init_hw_breakpoint(void)
 {
-        unsigned int **task_bp_pinned;
         int cpu, err_cpu;
         int i;
 
@@ -659,10 +633,11 @@ int __init init_hw_breakpoint(void)
 
         for_each_possible_cpu(cpu) {
                 for (i = 0; i < TYPE_MAX; i++) {
-                        task_bp_pinned = &per_cpu(nr_task_bp_pinned[i], cpu);
-                        *task_bp_pinned = kzalloc(sizeof(int) * nr_slots[i],
-                                                  GFP_KERNEL);
-                        if (!*task_bp_pinned)
+                        struct bp_cpuinfo *info = get_bp_info(cpu, i);
+
+                        info->tsk_pinned = kcalloc(nr_slots[i], sizeof(int),
+                                                        GFP_KERNEL);
+                        if (!info->tsk_pinned)
                                 goto err_alloc;
                 }
         }
@@ -676,7 +651,7 @@ int __init init_hw_breakpoint(void)
  err_alloc:
         for_each_possible_cpu(err_cpu) {
                 for (i = 0; i < TYPE_MAX; i++)
-                        kfree(per_cpu(nr_task_bp_pinned[i], err_cpu));
+                        kfree(get_bp_info(err_cpu, i)->tsk_pinned);
                 if (err_cpu == cpu)
                         break;
         }