Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 perf updates from Ingo Molnar:
 "This series tightens up RDPMC permissions: currently even highly
  sandboxed x86 execution environments (such as seccomp) have permission
  to execute RDPMC, which may leak various perf events / PMU state such
  as timing information and other CPU execution details.

  This 'all is allowed' RDPMC mode is still preserved as the
  (non-default) /sys/devices/cpu/rdpmc=2 setting.  The new default is
  that RDPMC access is only allowed if a perf event is mmap-ed (which is
  needed to correctly interpret RDPMC counter values in any case).

  As a side effect of these changes CR4 handling is cleaned up in the
  x86 code and a shadow copy of the CR4 value is added.

  The extra CR4 manipulation adds ~ <50ns to the context switch cost
  between rdpmc-capable and rdpmc-non-capable mms"

* 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  perf/x86: Add /sys/devices/cpu/rdpmc=2 to allow rdpmc for all tasks
  perf/x86: Only allow rdpmc if a perf_event is mapped
  perf: Pass the event to arch_perf_update_userpage()
  perf: Add pmu callbacks to track event mapping and unmapping
  x86: Add a comment clarifying LDT context switching
  x86: Store a per-cpu shadow copy of CR4
  x86: Clean up cr4 manipulation

17 files changed, 98 insertions, 40 deletions

diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index 31368207837c..d1daead5fcdd 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -78,7 +78,7 @@ int x86_acpi_suspend_lowlevel(void)
 
         header->pmode_cr0 = read_cr0();
         if (__this_cpu_read(cpu_info.cpuid_level) >= 0) {
-                header->pmode_cr4 = read_cr4();
+                header->pmode_cr4 = __read_cr4();
                 header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_CR4);
         }
         if (!rdmsr_safe(MSR_IA32_MISC_ENABLE,
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index b15bffcaba6d..b5c8ff5e9dfc 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -19,6 +19,7 @@
 #include <asm/archrandom.h>
 #include <asm/hypervisor.h>
 #include <asm/processor.h>
+#include <asm/tlbflush.h>
 #include <asm/debugreg.h>
 #include <asm/sections.h>
 #include <asm/vsyscall.h>
@@ -278,7 +279,7 @@ __setup("nosmep", setup_disable_smep);
 static __always_inline void setup_smep(struct cpuinfo_x86 *c)
 {
         if (cpu_has(c, X86_FEATURE_SMEP))
-                set_in_cr4(X86_CR4_SMEP);
+                cr4_set_bits(X86_CR4_SMEP);
 }
 
 static __init int setup_disable_smap(char *arg)
@@ -298,9 +299,9 @@ static __always_inline void setup_smap(struct cpuinfo_x86 *c)
 
         if (cpu_has(c, X86_FEATURE_SMAP)) {
 #ifdef CONFIG_X86_SMAP
-                set_in_cr4(X86_CR4_SMAP);
+                cr4_set_bits(X86_CR4_SMAP);
 #else
-                clear_in_cr4(X86_CR4_SMAP);
+                cr4_clear_bits(X86_CR4_SMAP);
 #endif
         }
 }
@@ -1295,6 +1296,12 @@ void cpu_init(void)
         wait_for_master_cpu(cpu);
 
         /*
+         * Initialize the CR4 shadow before doing anything that could
+         * try to read it.
+         */
+        cr4_init_shadow();
+
+        /*
          * Load microcode on this cpu if a valid microcode is available.
          * This is early microcode loading procedure.
          */
@@ -1313,7 +1320,7 @@ void cpu_init(void)
 
         pr_debug("Initializing CPU#%d\n", cpu);
 
-        clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
+        cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
 
         /*
          * Initialize the per-CPU GDT with the boot GDT,
@@ -1394,7 +1401,7 @@ void cpu_init(void)
         printk(KERN_INFO "Initializing CPU#%d\n", cpu);
 
         if (cpu_feature_enabled(X86_FEATURE_VME) || cpu_has_tsc || cpu_has_de)
-                clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
+                cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
 
         load_current_idt();
         switch_to_new_gdt(cpu);
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index cdfed7953963..3be9fa69f875 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -44,6 +44,7 @@
 
 #include <asm/processor.h>
 #include <asm/traps.h>
+#include <asm/tlbflush.h>
 #include <asm/mce.h>
 #include <asm/msr.h>
 
@@ -1452,7 +1453,7 @@ static void __mcheck_cpu_init_generic(void)
         bitmap_fill(all_banks, MAX_NR_BANKS);
         machine_check_poll(MCP_UC | m_fl, &all_banks);
 
-        set_in_cr4(X86_CR4_MCE);
+        cr4_set_bits(X86_CR4_MCE);
 
         rdmsrl(MSR_IA32_MCG_CAP, cap);
         if (cap & MCG_CTL_P)
diff --git a/arch/x86/kernel/cpu/mcheck/p5.c b/arch/x86/kernel/cpu/mcheck/p5.c
index ec2663a708e4..737b0ad4e61a 100644
--- a/arch/x86/kernel/cpu/mcheck/p5.c
+++ b/arch/x86/kernel/cpu/mcheck/p5.c
@@ -9,6 +9,7 @@
 
 #include <asm/processor.h>
 #include <asm/traps.h>
+#include <asm/tlbflush.h>
 #include <asm/mce.h>
 #include <asm/msr.h>
 
@@ -65,7 +66,7 @@ void intel_p5_mcheck_init(struct cpuinfo_x86 *c)
                "Intel old style machine check architecture supported.\n");
 
         /* Enable MCE: */
-        set_in_cr4(X86_CR4_MCE);
+        cr4_set_bits(X86_CR4_MCE);
         printk(KERN_INFO
                "Intel old style machine check reporting enabled on CPU#%d.\n",
                smp_processor_id());
diff --git a/arch/x86/kernel/cpu/mcheck/winchip.c b/arch/x86/kernel/cpu/mcheck/winchip.c
index bd5d46a32210..44f138296fbe 100644
--- a/arch/x86/kernel/cpu/mcheck/winchip.c
+++ b/arch/x86/kernel/cpu/mcheck/winchip.c
@@ -8,6 +8,7 @@
 
 #include <asm/processor.h>
 #include <asm/traps.h>
+#include <asm/tlbflush.h>
 #include <asm/mce.h>
 #include <asm/msr.h>
 
@@ -36,7 +37,7 @@ void winchip_mcheck_init(struct cpuinfo_x86 *c)
         lo &= ~(1<<4);  /* Enable MCE */
         wrmsr(MSR_IDT_FCR1, lo, hi);
 
-        set_in_cr4(X86_CR4_MCE);
+        cr4_set_bits(X86_CR4_MCE);
 
         printk(KERN_INFO
                "Winchip machine check reporting enabled on CPU#0.\n");
diff --git a/arch/x86/kernel/cpu/mtrr/cyrix.c b/arch/x86/kernel/cpu/mtrr/cyrix.c
index 9e451b0876b5..f8c81ba0b465 100644
--- a/arch/x86/kernel/cpu/mtrr/cyrix.c
+++ b/arch/x86/kernel/cpu/mtrr/cyrix.c
@@ -138,8 +138,8 @@ static void prepare_set(void)
 
         /*  Save value of CR4 and clear Page Global Enable (bit 7)  */
         if (cpu_has_pge) {
-                cr4 = read_cr4();
-                write_cr4(cr4 & ~X86_CR4_PGE);
+                cr4 = __read_cr4();
+                __write_cr4(cr4 & ~X86_CR4_PGE);
         }
 
         /*
@@ -171,7 +171,7 @@ static void post_set(void)
 
         /* Restore value of CR4 */
         if (cpu_has_pge)
-                write_cr4(cr4);
+                __write_cr4(cr4);
 }
 
 static void cyrix_set_arr(unsigned int reg, unsigned long base,
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index 0e25a1bc5ab5..7d74f7b3c6ba 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -678,8 +678,8 @@ static void prepare_set(void) __acquires(set_atomicity_lock)
 
         /* Save value of CR4 and clear Page Global Enable (bit 7) */
         if (cpu_has_pge) {
-                cr4 = read_cr4();
-                write_cr4(cr4 & ~X86_CR4_PGE);
+                cr4 = __read_cr4();
+                __write_cr4(cr4 & ~X86_CR4_PGE);
         }
 
         /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
@@ -708,7 +708,7 @@ static void post_set(void) __releases(set_atomicity_lock)
 
         /* Restore value of CR4 */
         if (cpu_has_pge)
-                write_cr4(cr4);
+                __write_cr4(cr4);
         raw_spin_unlock(&set_atomicity_lock);
 }
 
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 143e5f5dc855..b71a7f86d68a 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -31,6 +31,8 @@
 #include <asm/nmi.h>
 #include <asm/smp.h>
 #include <asm/alternative.h>
+#include <asm/mmu_context.h>
+#include <asm/tlbflush.h>
 #include <asm/timer.h>
 #include <asm/desc.h>
 #include <asm/ldt.h>
@@ -43,6 +45,8 @@ DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
         .enabled = 1,
 };
 
+struct static_key rdpmc_always_available = STATIC_KEY_INIT_FALSE;
+
 u64 __read_mostly hw_cache_event_ids
                                 [PERF_COUNT_HW_CACHE_MAX]
                                 [PERF_COUNT_HW_CACHE_OP_MAX]
@@ -1327,8 +1331,6 @@ x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
                 break;
 
         case CPU_STARTING:
-                if (x86_pmu.attr_rdpmc)
-                        set_in_cr4(X86_CR4_PCE);
                 if (x86_pmu.cpu_starting)
                         x86_pmu.cpu_starting(cpu);
                 break;
@@ -1804,14 +1806,44 @@ static int x86_pmu_event_init(struct perf_event *event)
                         event->destroy(event);
         }
 
+        if (ACCESS_ONCE(x86_pmu.attr_rdpmc))
+                event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED;
+
         return err;
 }
 
+static void refresh_pce(void *ignored)
+{
+        if (current->mm)
+                load_mm_cr4(current->mm);
+}
+
+static void x86_pmu_event_mapped(struct perf_event *event)
+{
+        if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
+                return;
+
+        if (atomic_inc_return(&current->mm->context.perf_rdpmc_allowed) == 1)
+                on_each_cpu_mask(mm_cpumask(current->mm), refresh_pce, NULL, 1);
+}
+
+static void x86_pmu_event_unmapped(struct perf_event *event)
+{
+        if (!current->mm)
+                return;
+
+        if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
+                return;
+
+        if (atomic_dec_and_test(&current->mm->context.perf_rdpmc_allowed))
+                on_each_cpu_mask(mm_cpumask(current->mm), refresh_pce, NULL, 1);
+}
+
 static int x86_pmu_event_idx(struct perf_event *event)
 {
         int idx = event->hw.idx;
 
-        if (!x86_pmu.attr_rdpmc)
+        if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
                 return 0;
 
         if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
@@ -1829,16 +1861,6 @@ static ssize_t get_attr_rdpmc(struct device *cdev,
         return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
 }
 
-static void change_rdpmc(void *info)
-{
-        bool enable = !!(unsigned long)info;
-
-        if (enable)
-                set_in_cr4(X86_CR4_PCE);
-        else
-                clear_in_cr4(X86_CR4_PCE);
-}
-
 static ssize_t set_attr_rdpmc(struct device *cdev,
                               struct device_attribute *attr,
                               const char *buf, size_t count)
@@ -1850,14 +1872,27 @@ static ssize_t set_attr_rdpmc(struct device *cdev,
         if (ret)
                 return ret;
 
+        if (val > 2)
+                return -EINVAL;
+
         if (x86_pmu.attr_rdpmc_broken)
                 return -ENOTSUPP;
 
-        if (!!val != !!x86_pmu.attr_rdpmc) {
-                x86_pmu.attr_rdpmc = !!val;
-                on_each_cpu(change_rdpmc, (void *)val, 1);
+        if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) {
+                /*
+                 * Changing into or out of always available, aka
+                 * perf-event-bypassing mode.  This path is extremely slow,
+                 * but only root can trigger it, so it's okay.
+                 */
+                if (val == 2)
+                        static_key_slow_inc(&rdpmc_always_available);
+                else
+                        static_key_slow_dec(&rdpmc_always_available);
+                on_each_cpu(refresh_pce, NULL, 1);
         }
 
+        x86_pmu.attr_rdpmc = val;
+
         return count;
 }
 
@@ -1900,6 +1935,9 @@ static struct pmu pmu = {
 
         .event_init             = x86_pmu_event_init,
 
+        .event_mapped           = x86_pmu_event_mapped,
+        .event_unmapped         = x86_pmu_event_unmapped,
+
         .add                    = x86_pmu_add,
         .del                    = x86_pmu_del,
         .start                  = x86_pmu_start,
@@ -1914,13 +1952,15 @@ static struct pmu pmu = {
         .flush_branch_stack     = x86_pmu_flush_branch_stack,
 };
 
-void arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
+void arch_perf_update_userpage(struct perf_event *event,
+                               struct perf_event_mmap_page *userpg, u64 now)
 {
         struct cyc2ns_data *data;
 
         userpg->cap_user_time = 0;
         userpg->cap_user_time_zero = 0;
-        userpg->cap_user_rdpmc = x86_pmu.attr_rdpmc;
+        userpg->cap_user_rdpmc =
+                !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED);
         userpg->pmc_width = x86_pmu.cntval_bits;
 
         if (!sched_clock_stable())
diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index 4e6cdb0ddc70..df525d2be1e8 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -71,6 +71,8 @@ struct event_constraint {
 #define PERF_X86_EVENT_COMMITTED        0x8 /* event passed commit_txn */
 #define PERF_X86_EVENT_PEBS_LD_HSW      0x10 /* haswell style datala, load */
 #define PERF_X86_EVENT_PEBS_NA_HSW      0x20 /* haswell style datala, unknown */
+#define PERF_X86_EVENT_RDPMC_ALLOWED    0x40 /* grant rdpmc permission */
+
 
 struct amd_nb {
         int nb_id;  /* NorthBridge id */
diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c
index d6c1b9836995..2911ef3a9f1c 100644
--- a/arch/x86/kernel/head32.c
+++ b/arch/x86/kernel/head32.c
@@ -31,6 +31,7 @@ static void __init i386_default_early_setup(void)
 
 asmlinkage __visible void __init i386_start_kernel(void)
 {
+        cr4_init_shadow();
         sanitize_boot_params(&boot_params);
 
         /* Call the subarch specific early setup function */
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index efcddfaf05f9..c4f8d4659070 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -156,6 +156,8 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
                                 (__START_KERNEL & PGDIR_MASK)));
         BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
 
+        cr4_init_shadow();
+
         /* Kill off the identity-map trampoline */
         reset_early_page_tables();
 
diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index 81049ffab2d6..d5651fce0b71 100644
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -13,6 +13,7 @@
 #include <asm/sigcontext.h>
 #include <asm/processor.h>
 #include <asm/math_emu.h>
+#include <asm/tlbflush.h>
 #include <asm/uaccess.h>
 #include <asm/ptrace.h>
 #include <asm/i387.h>
@@ -193,7 +194,7 @@ void fpu_init(void)
         if (cpu_has_xmm)
                 cr4_mask |= X86_CR4_OSXMMEXCPT;
         if (cr4_mask)
-                set_in_cr4(cr4_mask);
+                cr4_set_bits(cr4_mask);
 
         cr0 = read_cr0();
         cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index e127ddaa2d5a..046e2d620bbe 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -28,6 +28,7 @@
 #include <asm/fpu-internal.h>
 #include <asm/debugreg.h>
 #include <asm/nmi.h>
+#include <asm/tlbflush.h>
 
 /*
  * per-CPU TSS segments. Threads are completely 'soft' on Linux,
@@ -141,7 +142,7 @@ void flush_thread(void)
 
 static void hard_disable_TSC(void)
 {
-        write_cr4(read_cr4() | X86_CR4_TSD);
+        cr4_set_bits(X86_CR4_TSD);
 }
 
 void disable_TSC(void)
@@ -158,7 +159,7 @@ void disable_TSC(void)
 
 static void hard_enable_TSC(void)
 {
-        write_cr4(read_cr4() & ~X86_CR4_TSD);
+        cr4_clear_bits(X86_CR4_TSD);
 }
 
 static void enable_TSC(void)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 8f3ebfe710d0..603c4f99cb5a 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -101,7 +101,7 @@ void __show_regs(struct pt_regs *regs, int all)
         cr0 = read_cr0();
         cr2 = read_cr2();
         cr3 = read_cr3();
-        cr4 = read_cr4_safe();
+        cr4 = __read_cr4_safe();
         printk(KERN_DEFAULT "CR0: %08lx CR2: %08lx CR3: %08lx CR4: %08lx\n",
                         cr0, cr2, cr3, cr4);
 
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 5a2c02913af3..67fcc43577d2 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -93,7 +93,7 @@ void __show_regs(struct pt_regs *regs, int all)
         cr0 = read_cr0();
         cr2 = read_cr2();
         cr3 = read_cr3();
-        cr4 = read_cr4();
+        cr4 = __read_cr4();
 
         printk(KERN_DEFAULT "FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
                fs, fsindex, gs, gsindex, shadowgs);
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 27d200929864..0a2421cca01f 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -1179,7 +1179,7 @@ void __init setup_arch(char **cmdline_p)
 
         if (boot_cpu_data.cpuid_level >= 0) {
                 /* A CPU has %cr4 if and only if it has CPUID */
-                mmu_cr4_features = read_cr4();
+                mmu_cr4_features = __read_cr4();
                 if (trampoline_cr4_features)
                         *trampoline_cr4_features = mmu_cr4_features;
         }
diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c
index 0de1fae2bdf0..34f66e58a896 100644
--- a/arch/x86/kernel/xsave.c
+++ b/arch/x86/kernel/xsave.c
@@ -12,6 +12,7 @@
 #include <asm/i387.h>
 #include <asm/fpu-internal.h>
 #include <asm/sigframe.h>
+#include <asm/tlbflush.h>
 #include <asm/xcr.h>
 
 /*
@@ -453,7 +454,7 @@ static void prepare_fx_sw_frame(void)
  */
 static inline void xstate_enable(void)
 {
-        set_in_cr4(X86_CR4_OSXSAVE);
+        cr4_set_bits(X86_CR4_OSXSAVE);
         xsetbv(XCR_XFEATURE_ENABLED_MASK, pcntxt_mask);
 }