4 files changed, 122 insertions, 1 deletions

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 92eb1f42240d..6795dedcbd1e 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -856,6 +856,11 @@
                         causing system reset or hang due to sending
                         INIT from AP to BSP.
 
+        disable_counter_freezing [HW]
+                        Disable Intel PMU counter freezing feature.
+                        The feature only exists starting from
+                        Arch Perfmon v4 (Skylake and newer).
+
         disable_ddw     [PPC/PSERIES]
                         Disable Dynamic DMA Window support. Use this if
                         to workaround buggy firmware.
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index 9b320a51f82f..bd3b8f3600b2 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -1995,6 +1995,18 @@ static void intel_pmu_nhm_enable_all(int added)
         intel_pmu_enable_all(added);
 }
 
+static void enable_counter_freeze(void)
+{
+        update_debugctlmsr(get_debugctlmsr() |
+                        DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI);
+}
+
+static void disable_counter_freeze(void)
+{
+        update_debugctlmsr(get_debugctlmsr() &
+                        ~DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI);
+}
+
 static inline u64 intel_pmu_get_status(void)
 {
         u64 status;
@@ -2290,6 +2302,91 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
         return handled;
 }
 
+static bool disable_counter_freezing;
+static int __init intel_perf_counter_freezing_setup(char *s)
+{
+        disable_counter_freezing = true;
+        pr_info("Intel PMU Counter freezing feature disabled\n");
+        return 1;
+}
+__setup("disable_counter_freezing", intel_perf_counter_freezing_setup);
+
+/*
+ * Simplified handler for Arch Perfmon v4:
+ * - We rely on counter freezing/unfreezing to enable/disable the PMU.
+ * This is done automatically on PMU ack.
+ * - Ack the PMU only after the APIC.
+ */
+
+static int intel_pmu_handle_irq_v4(struct pt_regs *regs)
+{
+        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+        int handled = 0;
+        bool bts = false;
+        u64 status;
+        int pmu_enabled = cpuc->enabled;
+        int loops = 0;
+
+        /* PMU has been disabled because of counter freezing */
+        cpuc->enabled = 0;
+        if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
+                bts = true;
+                intel_bts_disable_local();
+                handled = intel_pmu_drain_bts_buffer();
+                handled += intel_bts_interrupt();
+        }
+        status = intel_pmu_get_status();
+        if (!status)
+                goto done;
+again:
+        intel_pmu_lbr_read();
+        if (++loops > 100) {
+                static bool warned;
+
+                if (!warned) {
+                        WARN(1, "perfevents: irq loop stuck!\n");
+                        perf_event_print_debug();
+                        warned = true;
+                }
+                intel_pmu_reset();
+                goto done;
+        }
+
+
+        handled += handle_pmi_common(regs, status);
+done:
+        /* Ack the PMI in the APIC */
+        apic_write(APIC_LVTPC, APIC_DM_NMI);
+
+        /*
+         * The counters start counting immediately while ack the status.
+         * Make it as close as possible to IRET. This avoids bogus
+         * freezing on Skylake CPUs.
+         */
+        if (status) {
+                intel_pmu_ack_status(status);
+        } else {
+                /*
+                 * CPU may issues two PMIs very close to each other.
+                 * When the PMI handler services the first one, the
+                 * GLOBAL_STATUS is already updated to reflect both.
+                 * When it IRETs, the second PMI is immediately
+                 * handled and it sees clear status. At the meantime,
+                 * there may be a third PMI, because the freezing bit
+                 * isn't set since the ack in first PMI handlers.
+                 * Double check if there is more work to be done.
+                 */
+                status = intel_pmu_get_status();
+                if (status)
+                        goto again;
+        }
+
+        if (bts)
+                intel_bts_enable_local();
+        cpuc->enabled = pmu_enabled;
+        return handled;
+}
+
 /*
  * This handler is triggered by the local APIC, so the APIC IRQ handling
  * rules apply:
@@ -3361,6 +3458,9 @@ static void intel_pmu_cpu_starting(int cpu)
         if (x86_pmu.version > 1)
                 flip_smm_bit(&x86_pmu.attr_freeze_on_smi);
 
+        if (x86_pmu.counter_freezing)
+                enable_counter_freeze();
+
         if (!cpuc->shared_regs)
                 return;
 
@@ -3432,6 +3532,9 @@ static void intel_pmu_cpu_dying(int cpu)
         free_excl_cntrs(cpu);
 
         fini_debug_store_on_cpu(cpu);
+
+        if (x86_pmu.counter_freezing)
+                disable_counter_freeze();
 }
 
 static void intel_pmu_sched_task(struct perf_event_context *ctx,
@@ -3946,6 +4049,9 @@ __init int intel_pmu_init(void)
                         max((int)edx.split.num_counters_fixed, assume);
         }
 
+        if (version >= 4)
+                x86_pmu.counter_freezing = !disable_counter_freezing;
+
         if (boot_cpu_has(X86_FEATURE_PDCM)) {
                 u64 capabilities;
 
@@ -4442,6 +4548,13 @@ __init int intel_pmu_init(void)
                 pr_cont("full-width counters, ");
         }
 
+        /*
+         * For arch perfmon 4 use counter freezing to avoid
+         * several MSR accesses in the PMI.
+         */
+        if (x86_pmu.counter_freezing)
+                x86_pmu.handle_irq = intel_pmu_handle_irq_v4;
+
         kfree(to_free);
         return 0;
 }
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index 156286335351..adae087cecdd 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -560,9 +560,11 @@ struct x86_pmu {
         struct event_constraint *event_constraints;
         struct x86_pmu_quirk *quirks;
         int             perfctr_second_write;
-        bool            late_ack;
         u64             (*limit_period)(struct perf_event *event, u64 l);
 
+        /* PMI handler bits */
+        unsigned int    late_ack                :1,
+                        counter_freezing        :1;
         /*
          * sysfs attrs
          */
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 4731f0cf97c5..80f4a4f38c79 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -164,6 +164,7 @@
 #define DEBUGCTLMSR_BTS_OFF_OS          (1UL <<  9)
 #define DEBUGCTLMSR_BTS_OFF_USR         (1UL << 10)
 #define DEBUGCTLMSR_FREEZE_LBRS_ON_PMI  (1UL << 11)
+#define DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI       (1UL << 12)
 #define DEBUGCTLMSR_FREEZE_IN_SMM_BIT   14
 #define DEBUGCTLMSR_FREEZE_IN_SMM       (1UL << DEBUGCTLMSR_FREEZE_IN_SMM_BIT)