Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

2 files changed, 134 insertions, 41 deletions

diff --git a/drivers/idle/i7300_idle.c b/drivers/idle/i7300_idle.c
index 15341fc1c68b..c976285d313e 100644
--- a/drivers/idle/i7300_idle.c
+++ b/drivers/idle/i7300_idle.c
@@ -536,6 +536,7 @@ static ssize_t stats_read_ul(struct file *fp, char __user *ubuf, size_t count,
 static const struct file_operations idle_fops = {
         .open   = stats_open_generic,
         .read   = stats_read_ul,
+        .llseek = default_llseek,
 };
 
 struct debugfs_file_info {
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
index c37ef64d1465..a46dddf61078 100644
--- a/drivers/idle/intel_idle.c
+++ b/drivers/idle/intel_idle.c
@@ -59,18 +59,14 @@
 #include <linux/hrtimer.h>      /* ktime_get_real() */
 #include <trace/events/power.h>
 #include <linux/sched.h>
+#include <linux/notifier.h>
+#include <linux/cpu.h>
+#include <asm/mwait.h>
+#include <asm/msr.h>
 
 #define INTEL_IDLE_VERSION "0.4"
 #define PREFIX "intel_idle: "
 
-#define MWAIT_SUBSTATE_MASK     (0xf)
-#define MWAIT_CSTATE_MASK       (0xf)
-#define MWAIT_SUBSTATE_SIZE     (4)
-#define MWAIT_MAX_NUM_CSTATES   8
-#define CPUID_MWAIT_LEAF (5)
-#define CPUID5_ECX_EXTENSIONS_SUPPORTED (0x1)
-#define CPUID5_ECX_INTERRUPT_BREAK      (0x2)
-
 static struct cpuidle_driver intel_idle_driver = {
         .name = "intel_idle",
         .owner = THIS_MODULE,
@@ -80,8 +76,9 @@ static int max_cstate = MWAIT_MAX_NUM_CSTATES - 1;
 
 static unsigned int mwait_substates;
 
+#define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
 /* Reliable LAPIC Timer States, bit 1 for C1 etc.  */
-static unsigned int lapic_timer_reliable_states;
+static unsigned int lapic_timer_reliable_states = (1 << 1);      /* Default to only C1 */
 
 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
 static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state);
@@ -89,6 +86,20 @@ static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state);
 static struct cpuidle_state *cpuidle_state_table;
 
 /*
+ * Hardware C-state auto-demotion may not always be optimal.
+ * Indicate which enable bits to clear here.
+ */
+static unsigned long long auto_demotion_disable_flags;
+
+/*
+ * Set this flag for states where the HW flushes the TLB for us
+ * and so we don't need cross-calls to keep it consistent.
+ * If this flag is set, SW flushes the TLB, so even if the
+ * HW doesn't do the flushing, this flag is safe to use.
+ */
+#define CPUIDLE_FLAG_TLB_FLUSHED        0x10000
+
+/*
  * States are indexed by the cstate number,
  * which is also the index into the MWAIT hint array.
  * Thus C0 is a dummy.
@@ -96,72 +107,101 @@ static struct cpuidle_state *cpuidle_state_table;
 static struct cpuidle_state nehalem_cstates[MWAIT_MAX_NUM_CSTATES] = {
         { /* MWAIT C0 */ },
         { /* MWAIT C1 */
-                .name = "NHM-C1",
+                .name = "C1-NHM",
                 .desc = "MWAIT 0x00",
                 .driver_data = (void *) 0x00,
                 .flags = CPUIDLE_FLAG_TIME_VALID,
                 .exit_latency = 3,
-                .power_usage = 1000,
                 .target_residency = 6,
                 .enter = &intel_idle },
         { /* MWAIT C2 */
-                .name = "NHM-C3",
+                .name = "C3-NHM",
                 .desc = "MWAIT 0x10",
                 .driver_data = (void *) 0x10,
                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
                 .exit_latency = 20,
-                .power_usage = 500,
                 .target_residency = 80,
                 .enter = &intel_idle },
         { /* MWAIT C3 */
-                .name = "NHM-C6",
+                .name = "C6-NHM",
                 .desc = "MWAIT 0x20",
                 .driver_data = (void *) 0x20,
                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
                 .exit_latency = 200,
-                .power_usage = 350,
                 .target_residency = 800,
                 .enter = &intel_idle },
 };
 
+static struct cpuidle_state snb_cstates[MWAIT_MAX_NUM_CSTATES] = {
+        { /* MWAIT C0 */ },
+        { /* MWAIT C1 */
+                .name = "C1-SNB",
+                .desc = "MWAIT 0x00",
+                .driver_data = (void *) 0x00,
+                .flags = CPUIDLE_FLAG_TIME_VALID,
+                .exit_latency = 1,
+                .target_residency = 1,
+                .enter = &intel_idle },
+        { /* MWAIT C2 */
+                .name = "C3-SNB",
+                .desc = "MWAIT 0x10",
+                .driver_data = (void *) 0x10,
+                .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
+                .exit_latency = 80,
+                .target_residency = 211,
+                .enter = &intel_idle },
+        { /* MWAIT C3 */
+                .name = "C6-SNB",
+                .desc = "MWAIT 0x20",
+                .driver_data = (void *) 0x20,
+                .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
+                .exit_latency = 104,
+                .target_residency = 345,
+                .enter = &intel_idle },
+        { /* MWAIT C4 */
+                .name = "C7-SNB",
+                .desc = "MWAIT 0x30",
+                .driver_data = (void *) 0x30,
+                .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
+                .exit_latency = 109,
+                .target_residency = 345,
+                .enter = &intel_idle },
+};
+
 static struct cpuidle_state atom_cstates[MWAIT_MAX_NUM_CSTATES] = {
         { /* MWAIT C0 */ },
         { /* MWAIT C1 */
-                .name = "ATM-C1",
+                .name = "C1-ATM",
                 .desc = "MWAIT 0x00",
                 .driver_data = (void *) 0x00,
                 .flags = CPUIDLE_FLAG_TIME_VALID,
                 .exit_latency = 1,
-                .power_usage = 1000,
                 .target_residency = 4,
                 .enter = &intel_idle },
         { /* MWAIT C2 */
-                .name = "ATM-C2",
+                .name = "C2-ATM",
                 .desc = "MWAIT 0x10",
                 .driver_data = (void *) 0x10,
                 .flags = CPUIDLE_FLAG_TIME_VALID,
                 .exit_latency = 20,
-                .power_usage = 500,
                 .target_residency = 80,
                 .enter = &intel_idle },
         { /* MWAIT C3 */ },
         { /* MWAIT C4 */
-                .name = "ATM-C4",
+                .name = "C4-ATM",
                 .desc = "MWAIT 0x30",
                 .driver_data = (void *) 0x30,
                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
                 .exit_latency = 100,
-                .power_usage = 250,
                 .target_residency = 400,
                 .enter = &intel_idle },
         { /* MWAIT C5 */ },
         { /* MWAIT C6 */
-                .name = "ATM-C6",
+                .name = "C6-ATM",
                 .desc = "MWAIT 0x52",
                 .driver_data = (void *) 0x52,
                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
                 .exit_latency = 140,
-                .power_usage = 150,
                 .target_residency = 560,
                 .enter = &intel_idle },
 };
@@ -186,13 +226,10 @@ static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state)
         local_irq_disable();
 
         /*
-         * If the state flag indicates that the TLB will be flushed or if this
-         * is the deepest c-state supported, do a voluntary leave mm to avoid
-         * costly and mostly unnecessary wakeups for flushing the user TLB's
-         * associated with the active mm.
+         * leave_mm() to avoid costly and often unnecessary wakeups
+         * for flushing the user TLB's associated with the active mm.
          */
-        if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED ||
-            (&dev->states[dev->state_count - 1] == state))
+        if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
                 leave_mm(cpu);
 
         if (!(lapic_timer_reliable_states & (1 << (cstate))))
@@ -201,9 +238,6 @@ static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state)
         kt_before = ktime_get_real();
 
         stop_critical_timings();
-#ifndef MODULE
-        trace_power_start(POWER_CSTATE, (eax >> 4) + 1, cpu);
-#endif
         if (!need_resched()) {
 
                 __monitor((void *)&current_thread_info()->flags, 0, 0);
@@ -225,6 +259,44 @@ static int intel_idle(struct cpuidle_device *dev, struct cpuidle_state *state)
         return usec_delta;
 }
 
+static void __setup_broadcast_timer(void *arg)
+{
+        unsigned long reason = (unsigned long)arg;
+        int cpu = smp_processor_id();
+
+        reason = reason ?
+                CLOCK_EVT_NOTIFY_BROADCAST_ON : CLOCK_EVT_NOTIFY_BROADCAST_OFF;
+
+        clockevents_notify(reason, &cpu);
+}
+
+static int setup_broadcast_cpuhp_notify(struct notifier_block *n,
+                unsigned long action, void *hcpu)
+{
+        int hotcpu = (unsigned long)hcpu;
+
+        switch (action & 0xf) {
+        case CPU_ONLINE:
+                smp_call_function_single(hotcpu, __setup_broadcast_timer,
+                        (void *)true, 1);
+                break;
+        }
+        return NOTIFY_OK;
+}
+
+static struct notifier_block setup_broadcast_notifier = {
+        .notifier_call = setup_broadcast_cpuhp_notify,
+};
+
+static void auto_demotion_disable(void *dummy)
+{
+        unsigned long long msr_bits;
+
+        rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
+        msr_bits &= ~auto_demotion_disable_flags;
+        wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
+}
+
 /*
  * intel_idle_probe()
  */
@@ -254,8 +326,6 @@ static int intel_idle_probe(void)
 
         pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates);
 
-        if (boot_cpu_has(X86_FEATURE_ARAT))     /* Always Reliable APIC Timer */
-                lapic_timer_reliable_states = 0xFFFFFFFF;
 
         if (boot_cpu_data.x86 != 6)     /* family 6 */
                 return -ENODEV;
@@ -267,22 +337,26 @@ static int intel_idle_probe(void)
         case 0x1F:      /* Core i7 and i5 Processor - Nehalem */
         case 0x2E:      /* Nehalem-EX Xeon */
         case 0x2F:      /* Westmere-EX Xeon */
-                lapic_timer_reliable_states = (1 << 1);  /* C1 */
-
         case 0x25:      /* Westmere */
         case 0x2C:      /* Westmere */
                 cpuidle_state_table = nehalem_cstates;
+                auto_demotion_disable_flags =
+                        (NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE);
                 break;
 
         case 0x1C:      /* 28 - Atom Processor */
+                cpuidle_state_table = atom_cstates;
+                break;
+
         case 0x26:      /* 38 - Lincroft Atom Processor */
-                lapic_timer_reliable_states = (1 << 2) | (1 << 1); /* C2, C1 */
                 cpuidle_state_table = atom_cstates;
+                auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE;
+                break;
+
+        case 0x2A:      /* SNB */
+        case 0x2D:      /* SNB Xeon */
+                cpuidle_state_table = snb_cstates;
                 break;
-#ifdef FUTURE_USE
-        case 0x17:      /* 23 - Core 2 Duo */
-                lapic_timer_reliable_states = (1 << 2) | (1 << 1); /* C2, C1 */
-#endif
 
         default:
                 pr_debug(PREFIX "does not run on family %d model %d\n",
@@ -290,6 +364,13 @@ static int intel_idle_probe(void)
                 return -ENODEV;
         }
 
+        if (boot_cpu_has(X86_FEATURE_ARAT))     /* Always Reliable APIC Timer */
+                lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE;
+        else {
+                smp_call_function(__setup_broadcast_timer, (void *)true, 1);
+                register_cpu_notifier(&setup_broadcast_notifier);
+        }
+
         pr_debug(PREFIX "v" INTEL_IDLE_VERSION
                 " model 0x%X\n", boot_cpu_data.x86_model);
 
@@ -377,6 +458,8 @@ static int intel_idle_cpuidle_devices_init(void)
                         return -EIO;
                 }
         }
+        if (auto_demotion_disable_flags)
+                smp_call_function(auto_demotion_disable, NULL, 1);
 
         return 0;
 }
@@ -386,6 +469,10 @@ static int __init intel_idle_init(void)
 {
         int retval;
 
+        /* Do not load intel_idle at all for now if idle= is passed */
+        if (boot_option_idle_override != IDLE_NO_OVERRIDE)
+                return -ENODEV;
+
         retval = intel_idle_probe();
         if (retval)
                 return retval;
@@ -411,6 +498,11 @@ static void __exit intel_idle_exit(void)
         intel_idle_cpuidle_devices_uninit();
         cpuidle_unregister_driver(&intel_idle_driver);
 
+        if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) {
+                smp_call_function(__setup_broadcast_timer, (void *)false, 1);
+                unregister_cpu_notifier(&setup_broadcast_notifier);
+        }
+
         return;
 }