10 files changed, 37 insertions, 32 deletions

diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 35a5d5dca2fa..c36140d788fe 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -556,10 +556,6 @@ static void early_init_amd(struct cpuinfo_x86 *c)
         if (c->x86_power & (1 << 8)) {
                 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
                 set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
-                if (check_tsc_unstable())
-                        clear_sched_clock_stable();
-        } else {
-                clear_sched_clock_stable();
         }
 
         /* Bit 12 of 8000_0007 edx is accumulated power mechanism. */
diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c
index adc0ebd8bed0..43955ee6715b 100644
--- a/arch/x86/kernel/cpu/centaur.c
+++ b/arch/x86/kernel/cpu/centaur.c
@@ -105,8 +105,6 @@ static void early_init_centaur(struct cpuinfo_x86 *c)
 #ifdef CONFIG_X86_64
         set_cpu_cap(c, X86_FEATURE_SYSENTER32);
 #endif
-
-        clear_sched_clock_stable();
 }
 
 static void init_centaur(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index b11b38c3b0bd..58094a1f9e9d 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -88,7 +88,6 @@ static void default_init(struct cpuinfo_x86 *c)
                         strcpy(c->x86_model_id, "386");
         }
 #endif
-        clear_sched_clock_stable();
 }
 
 static const struct cpu_dev default_cpu = {
@@ -1077,8 +1076,6 @@ static void identify_cpu(struct cpuinfo_x86 *c)
          */
         if (this_cpu->c_init)
                 this_cpu->c_init(c);
-        else
-                clear_sched_clock_stable();
 
         /* Disable the PN if appropriate */
         squash_the_stupid_serial_number(c);
diff --git a/arch/x86/kernel/cpu/cyrix.c b/arch/x86/kernel/cpu/cyrix.c
index 0a3bc19de017..a70fd61095f8 100644
--- a/arch/x86/kernel/cpu/cyrix.c
+++ b/arch/x86/kernel/cpu/cyrix.c
@@ -185,7 +185,6 @@ static void early_init_cyrix(struct cpuinfo_x86 *c)
                 set_cpu_cap(c, X86_FEATURE_CYRIX_ARR);
                 break;
         }
-        clear_sched_clock_stable();
 }
 
 static void init_cyrix(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index fe0a615a051b..063197771b8d 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -162,10 +162,6 @@ static void early_init_intel(struct cpuinfo_x86 *c)
         if (c->x86_power & (1 << 8)) {
                 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
                 set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
-                if (check_tsc_unstable())
-                        clear_sched_clock_stable();
-        } else {
-                clear_sched_clock_stable();
         }
 
         /* Penwell and Cloverview have the TSC which doesn't sleep on S3 */
diff --git a/arch/x86/kernel/cpu/transmeta.c b/arch/x86/kernel/cpu/transmeta.c
index 8457b4978668..d77d07ab310b 100644
--- a/arch/x86/kernel/cpu/transmeta.c
+++ b/arch/x86/kernel/cpu/transmeta.c
@@ -16,8 +16,6 @@ static void early_init_transmeta(struct cpuinfo_x86 *c)
                 if (xlvl >= 0x80860001)
                         c->x86_capability[CPUID_8086_0001_EDX] = cpuid_edx(0x80860001);
         }
-
-        clear_sched_clock_stable();
 }
 
 static void init_transmeta(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 46bcda4cb1c2..4f7a9833d8e5 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -327,9 +327,16 @@ unsigned long long sched_clock(void)
 {
         return paravirt_sched_clock();
 }
+
+static inline bool using_native_sched_clock(void)
+{
+        return pv_time_ops.sched_clock == native_sched_clock;
+}
 #else
 unsigned long long
 sched_clock(void) __attribute__((alias("native_sched_clock")));
+
+static inline bool using_native_sched_clock(void) { return true; }
 #endif
 
 int check_tsc_unstable(void)
@@ -1112,8 +1119,10 @@ static void tsc_cs_mark_unstable(struct clocksource *cs)
 {
         if (tsc_unstable)
                 return;
+
         tsc_unstable = 1;
-        clear_sched_clock_stable();
+        if (using_native_sched_clock())
+                clear_sched_clock_stable();
         disable_sched_clock_irqtime();
         pr_info("Marking TSC unstable due to clocksource watchdog\n");
 }
@@ -1135,18 +1144,20 @@ static struct clocksource clocksource_tsc = {
 
 void mark_tsc_unstable(char *reason)
 {
-        if (!tsc_unstable) {
-                tsc_unstable = 1;
+        if (tsc_unstable)
+                return;
+
+        tsc_unstable = 1;
+        if (using_native_sched_clock())
                 clear_sched_clock_stable();
-                disable_sched_clock_irqtime();
-                pr_info("Marking TSC unstable due to %s\n", reason);
-                /* Change only the rating, when not registered */
-                if (clocksource_tsc.mult)
-                        clocksource_mark_unstable(&clocksource_tsc);
-                else {
-                        clocksource_tsc.flags |= CLOCK_SOURCE_UNSTABLE;
-                        clocksource_tsc.rating = 0;
-                }
+        disable_sched_clock_irqtime();
+        pr_info("Marking TSC unstable due to %s\n", reason);
+        /* Change only the rating, when not registered */
+        if (clocksource_tsc.mult) {
+                clocksource_mark_unstable(&clocksource_tsc);
+        } else {
+                clocksource_tsc.flags |= CLOCK_SOURCE_UNSTABLE;
+                clocksource_tsc.rating = 0;
         }
 }
 
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 956383844116..3b31fc05a0f1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3287,10 +3287,15 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
         struct task_struct *p;
 
         /*
-         * Optimization: we know that if all tasks are in
-         * the fair class we can call that function directly:
+         * Optimization: we know that if all tasks are in the fair class we can
+         * call that function directly, but only if the @prev task wasn't of a
+         * higher scheduling class, because otherwise those loose the
+         * opportunity to pull in more work from other CPUs.
          */
-        if (likely(rq->nr_running == rq->cfs.h_nr_running)) {
+        if (likely((prev->sched_class == &idle_sched_class ||
+                    prev->sched_class == &fair_sched_class) &&
+                   rq->nr_running == rq->cfs.h_nr_running)) {
+
                 p = fair_sched_class.pick_next_task(rq, prev, rf);
                 if (unlikely(p == RETRY_TASK))
                         goto again;
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 3e88b35ac157..dea138964b91 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5799,7 +5799,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t
          * Due to large variance we need a large fuzz factor; hackbench in
          * particularly is sensitive here.
          */
-        if ((avg_idle / 512) < avg_cost)
+        if (sched_feat(SIS_AVG_CPU) && (avg_idle / 512) < avg_cost)
                 return -1;
 
         time = local_clock();
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 69631fa46c2f..1b3c8189b286 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -51,6 +51,11 @@ SCHED_FEAT(NONTASK_CAPACITY, true)
  */
 SCHED_FEAT(TTWU_QUEUE, true)
 
+/*
+ * When doing wakeups, attempt to limit superfluous scans of the LLC domain.
+ */
+SCHED_FEAT(SIS_AVG_CPU, false)
+
 #ifdef HAVE_RT_PUSH_IPI
 /*
  * In order to avoid a thundering herd attack of CPUs that are