5 files changed, 62 insertions, 14 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 956383844116..3b31fc05a0f1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3287,10 +3287,15 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
         struct task_struct *p;
 
         /*
-         * Optimization: we know that if all tasks are in
-         * the fair class we can call that function directly:
+         * Optimization: we know that if all tasks are in the fair class we can
+         * call that function directly, but only if the @prev task wasn't of a
+         * higher scheduling class, because otherwise those loose the
+         * opportunity to pull in more work from other CPUs.
          */
-        if (likely(rq->nr_running == rq->cfs.h_nr_running)) {
+        if (likely((prev->sched_class == &idle_sched_class ||
+                    prev->sched_class == &fair_sched_class) &&
+                   rq->nr_running == rq->cfs.h_nr_running)) {
+
                 p = fair_sched_class.pick_next_task(rq, prev, rf);
                 if (unlikely(p == RETRY_TASK))
                         goto again;
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index 8f8de3d4d6b7..cd7cd489f739 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -36,6 +36,7 @@ struct sugov_policy {
         u64 last_freq_update_time;
         s64 freq_update_delay_ns;
         unsigned int next_freq;
+        unsigned int cached_raw_freq;
 
         /* The next fields are only needed if fast switch cannot be used. */
         struct irq_work irq_work;
@@ -52,7 +53,6 @@ struct sugov_cpu {
         struct update_util_data update_util;
         struct sugov_policy *sg_policy;
 
-        unsigned int cached_raw_freq;
         unsigned long iowait_boost;
         unsigned long iowait_boost_max;
         u64 last_update;
@@ -116,7 +116,7 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time,
 
 /**
  * get_next_freq - Compute a new frequency for a given cpufreq policy.
- * @sg_cpu: schedutil cpu object to compute the new frequency for.
+ * @sg_policy: schedutil policy object to compute the new frequency for.
  * @util: Current CPU utilization.
  * @max: CPU capacity.
  *
@@ -136,19 +136,18 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time,
  * next_freq (as calculated above) is returned, subject to policy min/max and
  * cpufreq driver limitations.
  */
-static unsigned int get_next_freq(struct sugov_cpu *sg_cpu, unsigned long util,
-                                  unsigned long max)
+static unsigned int get_next_freq(struct sugov_policy *sg_policy,
+                                  unsigned long util, unsigned long max)
 {
-        struct sugov_policy *sg_policy = sg_cpu->sg_policy;
         struct cpufreq_policy *policy = sg_policy->policy;
         unsigned int freq = arch_scale_freq_invariant() ?
                                 policy->cpuinfo.max_freq : policy->cur;
 
         freq = (freq + (freq >> 2)) * util / max;
 
-        if (freq == sg_cpu->cached_raw_freq && sg_policy->next_freq != UINT_MAX)
+        if (freq == sg_policy->cached_raw_freq && sg_policy->next_freq != UINT_MAX)
                 return sg_policy->next_freq;
-        sg_cpu->cached_raw_freq = freq;
+        sg_policy->cached_raw_freq = freq;
         return cpufreq_driver_resolve_freq(policy, freq);
 }
 
@@ -213,7 +212,7 @@ static void sugov_update_single(struct update_util_data *hook, u64 time,
         } else {
                 sugov_get_util(&util, &max);
                 sugov_iowait_boost(sg_cpu, &util, &max);
-                next_f = get_next_freq(sg_cpu, util, max);
+                next_f = get_next_freq(sg_policy, util, max);
         }
         sugov_update_commit(sg_policy, time, next_f);
 }
@@ -267,7 +266,7 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu,
                 sugov_iowait_boost(j_sg_cpu, &util, &max);
         }
 
-        return get_next_freq(sg_cpu, util, max);
+        return get_next_freq(sg_policy, util, max);
 }
 
 static void sugov_update_shared(struct update_util_data *hook, u64 time,
@@ -580,6 +579,7 @@ static int sugov_start(struct cpufreq_policy *policy)
         sg_policy->next_freq = UINT_MAX;
         sg_policy->work_in_progress = false;
         sg_policy->need_freq_update = false;
+        sg_policy->cached_raw_freq = 0;
 
         for_each_cpu(cpu, policy->cpus) {
                 struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
@@ -590,7 +590,6 @@ static int sugov_start(struct cpufreq_policy *policy)
                         sg_cpu->max = 0;
                         sg_cpu->flags = SCHED_CPUFREQ_RT;
                         sg_cpu->last_update = 0;
-                        sg_cpu->cached_raw_freq = 0;
                         sg_cpu->iowait_boost = 0;
                         sg_cpu->iowait_boost_max = policy->cpuinfo.max_freq;
                         cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util,
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 3e88b35ac157..dea138964b91 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5799,7 +5799,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t
          * Due to large variance we need a large fuzz factor; hackbench in
          * particularly is sensitive here.
          */
-        if ((avg_idle / 512) < avg_cost)
+        if (sched_feat(SIS_AVG_CPU) && (avg_idle / 512) < avg_cost)
                 return -1;
 
         time = local_clock();
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 69631fa46c2f..1b3c8189b286 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -51,6 +51,11 @@ SCHED_FEAT(NONTASK_CAPACITY, true)
  */
 SCHED_FEAT(TTWU_QUEUE, true)
 
+/*
+ * When doing wakeups, attempt to limit superfluous scans of the LLC domain.
+ */
+SCHED_FEAT(SIS_AVG_CPU, false)
+
 #ifdef HAVE_RT_PUSH_IPI
 /*
  * In order to avoid a thundering herd attack of CPUs that are
diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c
index 4d2ea6f25568..b8c84c6dee64 100644
--- a/kernel/sched/wait.c
+++ b/kernel/sched/wait.c
@@ -242,6 +242,45 @@ long prepare_to_wait_event(wait_queue_head_t *q, wait_queue_t *wait, int state)
 }
 EXPORT_SYMBOL(prepare_to_wait_event);
 
+/*
+ * Note! These two wait functions are entered with the
+ * wait-queue lock held (and interrupts off in the _irq
+ * case), so there is no race with testing the wakeup
+ * condition in the caller before they add the wait
+ * entry to the wake queue.
+ */
+int do_wait_intr(wait_queue_head_t *wq, wait_queue_t *wait)
+{
+        if (likely(list_empty(&wait->task_list)))
+                __add_wait_queue_tail(wq, wait);
+
+        set_current_state(TASK_INTERRUPTIBLE);
+        if (signal_pending(current))
+                return -ERESTARTSYS;
+
+        spin_unlock(&wq->lock);
+        schedule();
+        spin_lock(&wq->lock);
+        return 0;
+}
+EXPORT_SYMBOL(do_wait_intr);
+
+int do_wait_intr_irq(wait_queue_head_t *wq, wait_queue_t *wait)
+{
+        if (likely(list_empty(&wait->task_list)))
+                __add_wait_queue_tail(wq, wait);
+
+        set_current_state(TASK_INTERRUPTIBLE);
+        if (signal_pending(current))
+                return -ERESTARTSYS;
+
+        spin_unlock_irq(&wq->lock);
+        schedule();
+        spin_lock_irq(&wq->lock);
+        return 0;
+}
+EXPORT_SYMBOL(do_wait_intr_irq);
+
 /**
  * finish_wait - clean up after waiting in a queue
  * @q: waitqueue waited on