1 files changed, 123 insertions, 157 deletions

diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 03a5b0385ad6..fe73c1844a9a 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -531,7 +531,7 @@ static int x86_pmu_hw_config(struct perf_event *event)
 /*
  * Setup the hardware configuration for a given attr_type
  */
-static int __hw_perf_event_init(struct perf_event *event)
+static int __x86_pmu_event_init(struct perf_event *event)
 {
         int err;
 
@@ -584,7 +584,7 @@ static void x86_pmu_disable_all(void)
         }
 }
 
-void hw_perf_disable(void)
+static void x86_pmu_disable(struct pmu *pmu)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 
@@ -619,7 +619,7 @@ static void x86_pmu_enable_all(int added)
         }
 }
 
-static const struct pmu pmu;
+static struct pmu pmu;
 
 static inline int is_x86_event(struct perf_event *event)
 {
@@ -801,10 +801,10 @@ static inline int match_prev_assignment(struct hw_perf_event *hwc,
                 hwc->last_tag == cpuc->tags[i];
 }
 
-static int x86_pmu_start(struct perf_event *event);
-static void x86_pmu_stop(struct perf_event *event);
+static void x86_pmu_start(struct perf_event *event, int flags);
+static void x86_pmu_stop(struct perf_event *event, int flags);
 
-void hw_perf_enable(void)
+static void x86_pmu_enable(struct pmu *pmu)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         struct perf_event *event;
@@ -840,7 +840,14 @@ void hw_perf_enable(void)
                             match_prev_assignment(hwc, cpuc, i))
                                 continue;
 
-                        x86_pmu_stop(event);
+                        /*
+                         * Ensure we don't accidentally enable a stopped
+                         * counter simply because we rescheduled.
+                         */
+                        if (hwc->state & PERF_HES_STOPPED)
+                                hwc->state |= PERF_HES_ARCH;
+
+                        x86_pmu_stop(event, PERF_EF_UPDATE);
                 }
 
                 for (i = 0; i < cpuc->n_events; i++) {
@@ -852,7 +859,10 @@ void hw_perf_enable(void)
                         else if (i < n_running)
                                 continue;
 
-                        x86_pmu_start(event);
+                        if (hwc->state & PERF_HES_ARCH)
+                                continue;
+
+                        x86_pmu_start(event, PERF_EF_RELOAD);
                 }
                 cpuc->n_added = 0;
                 perf_events_lapic_init();
@@ -953,15 +963,12 @@ static void x86_pmu_enable_event(struct perf_event *event)
 }
 
 /*
- * activate a single event
+ * Add a single event to the PMU.
  *
  * The event is added to the group of enabled events
  * but only if it can be scehduled with existing events.
- *
- * Called with PMU disabled. If successful and return value 1,
- * then guaranteed to call perf_enable() and hw_perf_enable()
  */
-static int x86_pmu_enable(struct perf_event *event)
+static int x86_pmu_add(struct perf_event *event, int flags)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         struct hw_perf_event *hwc;
@@ -970,58 +977,67 @@ static int x86_pmu_enable(struct perf_event *event)
 
         hwc = &event->hw;
 
+        perf_pmu_disable(event->pmu);
         n0 = cpuc->n_events;
-        n = collect_events(cpuc, event, false);
-        if (n < 0)
-                return n;
+        ret = n = collect_events(cpuc, event, false);
+        if (ret < 0)
+                goto out;
+
+        hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
+        if (!(flags & PERF_EF_START))
+                hwc->state |= PERF_HES_ARCH;
 
         /*
          * If group events scheduling transaction was started,
          * skip the schedulability test here, it will be peformed
-         * at commit time(->commit_txn) as a whole
+         * at commit time (->commit_txn) as a whole
          */
         if (cpuc->group_flag & PERF_EVENT_TXN)
-                goto out;
+                goto done_collect;
 
         ret = x86_pmu.schedule_events(cpuc, n, assign);
         if (ret)
-                return ret;
+                goto out;
         /*
          * copy new assignment, now we know it is possible
          * will be used by hw_perf_enable()
          */
         memcpy(cpuc->assign, assign, n*sizeof(int));
 
-out:
+done_collect:
         cpuc->n_events = n;
         cpuc->n_added += n - n0;
         cpuc->n_txn += n - n0;
 
-        return 0;
+        ret = 0;
+out:
+        perf_pmu_enable(event->pmu);
+        return ret;
 }
 
-static int x86_pmu_start(struct perf_event *event)
+static void x86_pmu_start(struct perf_event *event, int flags)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         int idx = event->hw.idx;
 
-        if (idx == -1)
-                return -EAGAIN;
+        if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
+                return;
+
+        if (WARN_ON_ONCE(idx == -1))
+                return;
+
+        if (flags & PERF_EF_RELOAD) {
+                WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
+                x86_perf_event_set_period(event);
+        }
+
+        event->hw.state = 0;
 
-        x86_perf_event_set_period(event);
         cpuc->events[idx] = event;
         __set_bit(idx, cpuc->active_mask);
         __set_bit(idx, cpuc->running);
         x86_pmu.enable(event);
         perf_event_update_userpage(event);
-
-        return 0;
-}
-
-static void x86_pmu_unthrottle(struct perf_event *event)
-{
-        int ret = x86_pmu_start(event);
-        WARN_ON_ONCE(ret);
 }
 
 void perf_event_print_debug(void)
@@ -1078,27 +1094,29 @@ void perf_event_print_debug(void)
         local_irq_restore(flags);
 }
 
-static void x86_pmu_stop(struct perf_event *event)
+static void x86_pmu_stop(struct perf_event *event, int flags)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         struct hw_perf_event *hwc = &event->hw;
-        int idx = hwc->idx;
 
-        if (!__test_and_clear_bit(idx, cpuc->active_mask))
-                return;
-
-        x86_pmu.disable(event);
-
-        /*
-         * Drain the remaining delta count out of a event
-         * that we are disabling:
-         */
-        x86_perf_event_update(event);
+        if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
+                x86_pmu.disable(event);
+                cpuc->events[hwc->idx] = NULL;
+                WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
+                hwc->state |= PERF_HES_STOPPED;
+        }
 
-        cpuc->events[idx] = NULL;
+        if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
+                /*
+                 * Drain the remaining delta count out of a event
+                 * that we are disabling:
+                 */
+                x86_perf_event_update(event);
+                hwc->state |= PERF_HES_UPTODATE;
+        }
 }
 
-static void x86_pmu_disable(struct perf_event *event)
+static void x86_pmu_del(struct perf_event *event, int flags)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         int i;
@@ -1111,7 +1129,7 @@ static void x86_pmu_disable(struct perf_event *event)
         if (cpuc->group_flag & PERF_EVENT_TXN)
                 return;
 
-        x86_pmu_stop(event);
+        x86_pmu_stop(event, PERF_EF_UPDATE);
 
         for (i = 0; i < cpuc->n_events; i++) {
                 if (event == cpuc->event_list[i]) {
@@ -1134,7 +1152,6 @@ static int x86_pmu_handle_irq(struct pt_regs *regs)
         struct perf_sample_data data;
         struct cpu_hw_events *cpuc;
         struct perf_event *event;
-        struct hw_perf_event *hwc;
         int idx, handled = 0;
         u64 val;
 
@@ -1155,7 +1172,6 @@ static int x86_pmu_handle_irq(struct pt_regs *regs)
                 }
 
                 event = cpuc->events[idx];
-                hwc = &event->hw;
 
                 val = x86_perf_event_update(event);
                 if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
@@ -1171,7 +1187,7 @@ static int x86_pmu_handle_irq(struct pt_regs *regs)
                         continue;
 
                 if (perf_event_overflow(event, 1, &data, regs))
-                        x86_pmu_stop(event);
+                        x86_pmu_stop(event, 0);
         }
 
         if (handled)
@@ -1180,25 +1196,6 @@ static int x86_pmu_handle_irq(struct pt_regs *regs)
         return handled;
 }
 
-void smp_perf_pending_interrupt(struct pt_regs *regs)
-{
-        irq_enter();
-        ack_APIC_irq();
-        inc_irq_stat(apic_pending_irqs);
-        perf_event_do_pending();
-        irq_exit();
-}
-
-void set_perf_event_pending(void)
-{
-#ifdef CONFIG_X86_LOCAL_APIC
-        if (!x86_pmu.apic || !x86_pmu_initialized())
-                return;
-
-        apic->send_IPI_self(LOCAL_PENDING_VECTOR);
-#endif
-}
-
 void perf_events_lapic_init(void)
 {
         if (!x86_pmu.apic || !x86_pmu_initialized())
@@ -1388,7 +1385,6 @@ void __init init_hw_perf_events(void)
                 x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
         }
         x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
-        perf_max_events = x86_pmu.num_counters;
 
         if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
                 WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
@@ -1424,6 +1420,7 @@ void __init init_hw_perf_events(void)
         pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_counters_fixed);
         pr_info("... event mask:             %016Lx\n", x86_pmu.intel_ctrl);
 
+        perf_pmu_register(&pmu);
         perf_cpu_notifier(x86_pmu_notifier);
 }
 
@@ -1437,10 +1434,11 @@ static inline void x86_pmu_read(struct perf_event *event)
  * Set the flag to make pmu::enable() not perform the
  * schedulability test, it will be performed at commit time
  */
-static void x86_pmu_start_txn(const struct pmu *pmu)
+static void x86_pmu_start_txn(struct pmu *pmu)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 
+        perf_pmu_disable(pmu);
         cpuc->group_flag |= PERF_EVENT_TXN;
         cpuc->n_txn = 0;
 }
@@ -1450,7 +1448,7 @@ static void x86_pmu_start_txn(const struct pmu *pmu)
  * Clear the flag and pmu::enable() will perform the
  * schedulability test.
  */
-static void x86_pmu_cancel_txn(const struct pmu *pmu)
+static void x86_pmu_cancel_txn(struct pmu *pmu)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 
@@ -1460,6 +1458,7 @@ static void x86_pmu_cancel_txn(const struct pmu *pmu)
          */
         cpuc->n_added -= cpuc->n_txn;
         cpuc->n_events -= cpuc->n_txn;
+        perf_pmu_enable(pmu);
 }
 
 /*
@@ -1467,7 +1466,7 @@ static void x86_pmu_cancel_txn(const struct pmu *pmu)
  * Perform the group schedulability test as a whole
  * Return 0 if success
  */
-static int x86_pmu_commit_txn(const struct pmu *pmu)
+static int x86_pmu_commit_txn(struct pmu *pmu)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         int assign[X86_PMC_IDX_MAX];
@@ -1489,22 +1488,10 @@ static int x86_pmu_commit_txn(const struct pmu *pmu)
         memcpy(cpuc->assign, assign, n*sizeof(int));
 
         cpuc->group_flag &= ~PERF_EVENT_TXN;
-
+        perf_pmu_enable(pmu);
         return 0;
 }
 
-static const struct pmu pmu = {
-        .enable         = x86_pmu_enable,
-        .disable        = x86_pmu_disable,
-        .start          = x86_pmu_start,
-        .stop           = x86_pmu_stop,
-        .read           = x86_pmu_read,
-        .unthrottle     = x86_pmu_unthrottle,
-        .start_txn      = x86_pmu_start_txn,
-        .cancel_txn     = x86_pmu_cancel_txn,
-        .commit_txn     = x86_pmu_commit_txn,
-};
-
 /*
  * validate that we can schedule this event
  */
@@ -1579,12 +1566,22 @@ out:
         return ret;
 }
 
-const struct pmu *hw_perf_event_init(struct perf_event *event)
+int x86_pmu_event_init(struct perf_event *event)
 {
-        const struct pmu *tmp;
+        struct pmu *tmp;
         int err;
 
-        err = __hw_perf_event_init(event);
+        switch (event->attr.type) {
+        case PERF_TYPE_RAW:
+        case PERF_TYPE_HARDWARE:
+        case PERF_TYPE_HW_CACHE:
+                break;
+
+        default:
+                return -ENOENT;
+        }
+
+        err = __x86_pmu_event_init(event);
         if (!err) {
                 /*
                  * we temporarily connect event to its pmu
@@ -1604,26 +1601,31 @@ const struct pmu *hw_perf_event_init(struct perf_event *event)
         if (err) {
                 if (event->destroy)
                         event->destroy(event);
-                return ERR_PTR(err);
         }
 
-        return &pmu;
+        return err;
 }
 
-/*
- * callchain support
- */
+static struct pmu pmu = {
+        .pmu_enable     = x86_pmu_enable,
+        .pmu_disable    = x86_pmu_disable,
 
-static inline
-void callchain_store(struct perf_callchain_entry *entry, u64 ip)
-{
-        if (entry->nr < PERF_MAX_STACK_DEPTH)
-                entry->ip[entry->nr++] = ip;
-}
+        .event_init     = x86_pmu_event_init,
 
-static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
-static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
+        .add            = x86_pmu_add,
+        .del            = x86_pmu_del,
+        .start          = x86_pmu_start,
+        .stop           = x86_pmu_stop,
+        .read           = x86_pmu_read,
 
+        .start_txn      = x86_pmu_start_txn,
+        .cancel_txn     = x86_pmu_cancel_txn,
+        .commit_txn     = x86_pmu_commit_txn,
+};
+
+/*
+ * callchain support
+ */
 
 static void
 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
@@ -1645,7 +1647,7 @@ static void backtrace_address(void *data, unsigned long addr, int reliable)
 {
         struct perf_callchain_entry *entry = data;
 
-        callchain_store(entry, addr);
+        perf_callchain_store(entry, addr);
 }
 
 static const struct stacktrace_ops backtrace_ops = {
@@ -1656,11 +1658,15 @@ static const struct stacktrace_ops backtrace_ops = {
         .walk_stack             = print_context_stack_bp,
 };
 
-static void
-perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
+void
+perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
 {
-        callchain_store(entry, PERF_CONTEXT_KERNEL);
-        callchain_store(entry, regs->ip);
+        if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
+                /* TODO: We don't support guest os callchain now */
+                return;
+        }
+
+        perf_callchain_store(entry, regs->ip);
 
         dump_trace(NULL, regs, NULL, regs->bp, &backtrace_ops, entry);
 }
@@ -1689,7 +1695,7 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
                 if (fp < compat_ptr(regs->sp))
                         break;
 
-                callchain_store(entry, frame.return_address);
+                perf_callchain_store(entry, frame.return_address);
                 fp = compat_ptr(frame.next_frame);
         }
         return 1;
@@ -1702,19 +1708,20 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry *entry)
 }
 #endif
 
-static void
-perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
+void
+perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
 {
         struct stack_frame frame;
         const void __user *fp;
 
-        if (!user_mode(regs))
-                regs = task_pt_regs(current);
+        if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
+                /* TODO: We don't support guest os callchain now */
+                return;
+        }
 
         fp = (void __user *)regs->bp;
 
-        callchain_store(entry, PERF_CONTEXT_USER);
-        callchain_store(entry, regs->ip);
+        perf_callchain_store(entry, regs->ip);
 
         if (perf_callchain_user32(regs, entry))
                 return;
@@ -1731,52 +1738,11 @@ perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
                 if ((unsigned long)fp < regs->sp)
                         break;
 
-                callchain_store(entry, frame.return_address);
+                perf_callchain_store(entry, frame.return_address);
                 fp = frame.next_frame;
         }
 }
 
-static void
-perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
-{
-        int is_user;
-
-        if (!regs)
-                return;
-
-        is_user = user_mode(regs);
-
-        if (is_user && current->state != TASK_RUNNING)
-                return;
-
-        if (!is_user)
-                perf_callchain_kernel(regs, entry);
-
-        if (current->mm)
-                perf_callchain_user(regs, entry);
-}
-
-struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
-{
-        struct perf_callchain_entry *entry;
-
-        if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
-                /* TODO: We don't support guest os callchain now */
-                return NULL;
-        }
-
-        if (in_nmi())
-                entry = &__get_cpu_var(pmc_nmi_entry);
-        else
-                entry = &__get_cpu_var(pmc_irq_entry);
-
-        entry->nr = 0;
-
-        perf_do_callchain(regs, entry);
-
-        return entry;
-}
-
 unsigned long perf_instruction_pointer(struct pt_regs *regs)
 {
         unsigned long ip;