Merge branch 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fixes from Ingo Molnar:
 "A fix for KVM's scheduler clock which (erroneously) was always marked
  unstable, a fix for RT/DL load balancing, plus latency fixes"

* 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  sched/clock, x86/tsc: Rework the x86 'unstable' sched_clock() interface
  sched/core: Fix pick_next_task() for RT,DL
  sched/fair: Make select_idle_cpu() more aggressive

3 files changed, 14 insertions, 4 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 956383844116..3b31fc05a0f1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3287,10 +3287,15 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
         struct task_struct *p;
 
         /*
-         * Optimization: we know that if all tasks are in
-         * the fair class we can call that function directly:
+         * Optimization: we know that if all tasks are in the fair class we can
+         * call that function directly, but only if the @prev task wasn't of a
+         * higher scheduling class, because otherwise those loose the
+         * opportunity to pull in more work from other CPUs.
          */
-        if (likely(rq->nr_running == rq->cfs.h_nr_running)) {
+        if (likely((prev->sched_class == &idle_sched_class ||
+                    prev->sched_class == &fair_sched_class) &&
+                   rq->nr_running == rq->cfs.h_nr_running)) {
+
                 p = fair_sched_class.pick_next_task(rq, prev, rf);
                 if (unlikely(p == RETRY_TASK))
                         goto again;
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 3e88b35ac157..dea138964b91 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5799,7 +5799,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t
          * Due to large variance we need a large fuzz factor; hackbench in
          * particularly is sensitive here.
          */
-        if ((avg_idle / 512) < avg_cost)
+        if (sched_feat(SIS_AVG_CPU) && (avg_idle / 512) < avg_cost)
                 return -1;
 
         time = local_clock();
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 69631fa46c2f..1b3c8189b286 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -51,6 +51,11 @@ SCHED_FEAT(NONTASK_CAPACITY, true)
  */
 SCHED_FEAT(TTWU_QUEUE, true)
 
+/*
+ * When doing wakeups, attempt to limit superfluous scans of the LLC domain.
+ */
+SCHED_FEAT(SIS_AVG_CPU, false)
+
 #ifdef HAVE_RT_PUSH_IPI
 /*
  * In order to avoid a thundering herd attack of CPUs that are