Merge tag 'v4.3-rc1' into perf/core, to refresh the tree

Signed-off-by: Ingo Molnar <mingo@kernel.org>

1 files changed, 51 insertions, 9 deletions

diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c
index 39d6a3db0b96..e912b2f6d36e 100644
--- a/arch/x86/lib/delay.c
+++ b/arch/x86/lib/delay.c
@@ -20,6 +20,7 @@
 #include <asm/processor.h>
 #include <asm/delay.h>
 #include <asm/timer.h>
+#include <asm/mwait.h>
 
 #ifdef CONFIG_SMP
 # include <asm/smp.h>
@@ -49,16 +50,14 @@ static void delay_loop(unsigned long loops)
 /* TSC based delay: */
 static void delay_tsc(unsigned long __loops)
 {
-        u32 bclock, now, loops = __loops;
+        u64 bclock, now, loops = __loops;
         int cpu;
 
         preempt_disable();
         cpu = smp_processor_id();
-        rdtsc_barrier();
-        rdtscl(bclock);
+        bclock = rdtsc_ordered();
         for (;;) {
-                rdtsc_barrier();
-                rdtscl(now);
+                now = rdtsc_ordered();
                 if ((now - bclock) >= loops)
                         break;
 
@@ -79,14 +78,51 @@ static void delay_tsc(unsigned long __loops)
                 if (unlikely(cpu != smp_processor_id())) {
                         loops -= (now - bclock);
                         cpu = smp_processor_id();
-                        rdtsc_barrier();
-                        rdtscl(bclock);
+                        bclock = rdtsc_ordered();
                 }
         }
         preempt_enable();
 }
 
 /*
+ * On some AMD platforms, MWAITX has a configurable 32-bit timer, that
+ * counts with TSC frequency. The input value is the loop of the
+ * counter, it will exit when the timer expires.
+ */
+static void delay_mwaitx(unsigned long __loops)
+{
+        u64 start, end, delay, loops = __loops;
+
+        start = rdtsc_ordered();
+
+        for (;;) {
+                delay = min_t(u64, MWAITX_MAX_LOOPS, loops);
+
+                /*
+                 * Use cpu_tss as a cacheline-aligned, seldomly
+                 * accessed per-cpu variable as the monitor target.
+                 */
+                __monitorx(this_cpu_ptr(&cpu_tss), 0, 0);
+
+                /*
+                 * AMD, like Intel, supports the EAX hint and EAX=0xf
+                 * means, do not enter any deep C-state and we use it
+                 * here in delay() to minimize wakeup latency.
+                 */
+                __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE);
+
+                end = rdtsc_ordered();
+
+                if (loops <= end - start)
+                        break;
+
+                loops -= end - start;
+
+                start = end;
+        }
+}
+
+/*
  * Since we calibrate only once at boot, this
  * function should be set once at boot and not changed
  */
@@ -94,13 +130,19 @@ static void (*delay_fn)(unsigned long) = delay_loop;
 
 void use_tsc_delay(void)
 {
-        delay_fn = delay_tsc;
+        if (delay_fn == delay_loop)
+                delay_fn = delay_tsc;
+}
+
+void use_mwaitx_delay(void)
+{
+        delay_fn = delay_mwaitx;
 }
 
 int read_current_timer(unsigned long *timer_val)
 {
         if (delay_fn == delay_tsc) {
-                rdtscll(*timer_val);
+                *timer_val = rdtsc();
                 return 0;
         }
         return -1;