5 files changed, 112 insertions, 61 deletions

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f12225f26b70..6fedf3a98581 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5738,15 +5738,10 @@ int sched_cpu_activate(unsigned int cpu)
 
 #ifdef CONFIG_SCHED_SMT
         /*
-         * The sched_smt_present static key needs to be evaluated on every
-         * hotplug event because at boot time SMT might be disabled when
-         * the number of booted CPUs is limited.
-         *
-         * If then later a sibling gets hotplugged, then the key would stay
-         * off and SMT scheduling would never be functional.
+         * When going up, increment the number of cores with SMT present.
          */
-        if (cpumask_weight(cpu_smt_mask(cpu)) > 1)
-                static_branch_enable_cpuslocked(&sched_smt_present);
+        if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
+                static_branch_inc_cpuslocked(&sched_smt_present);
 #endif
         set_cpu_active(cpu, true);
 
@@ -5790,6 +5785,14 @@ int sched_cpu_deactivate(unsigned int cpu)
          */
         synchronize_rcu_mult(call_rcu, call_rcu_sched);
 
+#ifdef CONFIG_SCHED_SMT
+        /*
+         * When going down, decrement the number of cores with SMT present.
+         */
+        if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
+                static_branch_dec_cpuslocked(&sched_smt_present);
+#endif
+
         if (!sched_smp_initialized)
                 return 0;
 
@@ -5851,11 +5854,14 @@ void __init sched_init_smp(void)
         /*
          * There's no userspace yet to cause hotplug operations; hence all the
          * CPU masks are stable and all blatant races in the below code cannot
-         * happen.
+         * happen. The hotplug lock is nevertheless taken to satisfy lockdep,
+         * but there won't be any contention on it.
          */
+        cpus_read_lock();
         mutex_lock(&sched_domains_mutex);
         sched_init_domains(cpu_active_mask);
         mutex_unlock(&sched_domains_mutex);
+        cpus_read_unlock();
 
         /* Move init over to a non-isolated CPU */
         if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_FLAG_DOMAIN)) < 0)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index ee271bb661cc..ac855b2f4774 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2400,8 +2400,8 @@ void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
                 local = 1;
 
         /*
-         * Retry task to preferred node migration periodically, in case it
-         * case it previously failed, or the scheduler moved us.
+         * Retry to migrate task to preferred node periodically, in case it
+         * previously failed, or the scheduler moved us.
          */
         if (time_after(jiffies, p->numa_migrate_retry)) {
                 task_numa_placement(p);
@@ -5674,11 +5674,11 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p,
         return target;
 }
 
-static unsigned long cpu_util_wake(int cpu, struct task_struct *p);
+static unsigned long cpu_util_without(int cpu, struct task_struct *p);
 
-static unsigned long capacity_spare_wake(int cpu, struct task_struct *p)
+static unsigned long capacity_spare_without(int cpu, struct task_struct *p)
 {
-        return max_t(long, capacity_of(cpu) - cpu_util_wake(cpu, p), 0);
+        return max_t(long, capacity_of(cpu) - cpu_util_without(cpu, p), 0);
 }
 
 /*
@@ -5738,7 +5738,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p,
 
                         avg_load += cfs_rq_load_avg(&cpu_rq(i)->cfs);
 
-                        spare_cap = capacity_spare_wake(i, p);
+                        spare_cap = capacity_spare_without(i, p);
 
                         if (spare_cap > max_spare_cap)
                                 max_spare_cap = spare_cap;
@@ -5889,8 +5889,8 @@ static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p
                 return prev_cpu;
 
         /*
-         * We need task's util for capacity_spare_wake, sync it up to prev_cpu's
-         * last_update_time.
+         * We need task's util for capacity_spare_without, sync it up to
+         * prev_cpu's last_update_time.
          */
         if (!(sd_flag & SD_BALANCE_FORK))
                 sync_entity_load_avg(&p->se);
@@ -6216,10 +6216,19 @@ static inline unsigned long cpu_util(int cpu)
 }
 
 /*
- * cpu_util_wake: Compute CPU utilization with any contributions from
- * the waking task p removed.
+ * cpu_util_without: compute cpu utilization without any contributions from *p
+ * @cpu: the CPU which utilization is requested
+ * @p: the task which utilization should be discounted
+ *
+ * The utilization of a CPU is defined by the utilization of tasks currently
+ * enqueued on that CPU as well as tasks which are currently sleeping after an
+ * execution on that CPU.
+ *
+ * This method returns the utilization of the specified CPU by discounting the
+ * utilization of the specified task, whenever the task is currently
+ * contributing to the CPU utilization.
  */
-static unsigned long cpu_util_wake(int cpu, struct task_struct *p)
+static unsigned long cpu_util_without(int cpu, struct task_struct *p)
 {
         struct cfs_rq *cfs_rq;
         unsigned int util;
@@ -6231,7 +6240,7 @@ static unsigned long cpu_util_wake(int cpu, struct task_struct *p)
         cfs_rq = &cpu_rq(cpu)->cfs;
         util = READ_ONCE(cfs_rq->avg.util_avg);
 
-        /* Discount task's blocked util from CPU's util */
+        /* Discount task's util from CPU's util */
         util -= min_t(unsigned int, util, task_util(p));
 
         /*
@@ -6240,14 +6249,14 @@ static unsigned long cpu_util_wake(int cpu, struct task_struct *p)
          * a) if *p is the only task sleeping on this CPU, then:
          *      cpu_util (== task_util) > util_est (== 0)
          *    and thus we return:
-         *      cpu_util_wake = (cpu_util - task_util) = 0
+         *      cpu_util_without = (cpu_util - task_util) = 0
          *
          * b) if other tasks are SLEEPING on this CPU, which is now exiting
          *    IDLE, then:
          *      cpu_util >= task_util
          *      cpu_util > util_est (== 0)
          *    and thus we discount *p's blocked utilization to return:
-         *      cpu_util_wake = (cpu_util - task_util) >= 0
+         *      cpu_util_without = (cpu_util - task_util) >= 0
          *
          * c) if other tasks are RUNNABLE on that CPU and
          *      util_est > cpu_util
@@ -6260,8 +6269,33 @@ static unsigned long cpu_util_wake(int cpu, struct task_struct *p)
          * covered by the following code when estimated utilization is
          * enabled.
          */
-        if (sched_feat(UTIL_EST))
-                util = max(util, READ_ONCE(cfs_rq->avg.util_est.enqueued));
+        if (sched_feat(UTIL_EST)) {
+                unsigned int estimated =
+                        READ_ONCE(cfs_rq->avg.util_est.enqueued);
+
+                /*
+                 * Despite the following checks we still have a small window
+                 * for a possible race, when an execl's select_task_rq_fair()
+                 * races with LB's detach_task():
+                 *
+                 *   detach_task()
+                 *     p->on_rq = TASK_ON_RQ_MIGRATING;
+                 *     ---------------------------------- A
+                 *     deactivate_task()                   \
+                 *       dequeue_task()                     + RaceTime
+                 *         util_est_dequeue()              /
+                 *     ---------------------------------- B
+                 *
+                 * The additional check on "current == p" it's required to
+                 * properly fix the execl regression and it helps in further
+                 * reducing the chances for the above race.
+                 */
+                if (unlikely(task_on_rq_queued(p) || current == p)) {
+                        estimated -= min_t(unsigned int, estimated,
+                                           (_task_util_est(p) | UTIL_AVG_UNCHANGED));
+                }
+                util = max(util, estimated);
+        }
 
         /*
          * Utilization (estimated) can exceed the CPU capacity, thus let's
diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c
index 7cdecfc010af..fe24de3fbc93 100644
--- a/kernel/sched/psi.c
+++ b/kernel/sched/psi.c
@@ -136,8 +136,18 @@
 
 static int psi_bug __read_mostly;
 
-bool psi_disabled __read_mostly;
-core_param(psi_disabled, psi_disabled, bool, 0644);
+DEFINE_STATIC_KEY_FALSE(psi_disabled);
+
+#ifdef CONFIG_PSI_DEFAULT_DISABLED
+bool psi_enable;
+#else
+bool psi_enable = true;
+#endif
+static int __init setup_psi(char *str)
+{
+        return kstrtobool(str, &psi_enable) == 0;
+}
+__setup("psi=", setup_psi);
 
 /* Running averages - we need to be higher-res than loadavg */
 #define PSI_FREQ        (2*HZ+1)        /* 2 sec intervals */
@@ -169,8 +179,10 @@ static void group_init(struct psi_group *group)
 
 void __init psi_init(void)
 {
-        if (psi_disabled)
+        if (!psi_enable) {
+                static_branch_enable(&psi_disabled);
                 return;
+        }
 
         psi_period = jiffies_to_nsecs(PSI_FREQ);
         group_init(&psi_system);
@@ -549,7 +561,7 @@ void psi_memstall_enter(unsigned long *flags)
         struct rq_flags rf;
         struct rq *rq;
 
-        if (psi_disabled)
+        if (static_branch_likely(&psi_disabled))
                 return;
 
         *flags = current->flags & PF_MEMSTALL;
@@ -579,7 +591,7 @@ void psi_memstall_leave(unsigned long *flags)
         struct rq_flags rf;
         struct rq *rq;
 
-        if (psi_disabled)
+        if (static_branch_likely(&psi_disabled))
                 return;
 
         if (*flags)
@@ -600,7 +612,7 @@ void psi_memstall_leave(unsigned long *flags)
 #ifdef CONFIG_CGROUPS
 int psi_cgroup_alloc(struct cgroup *cgroup)
 {
-        if (psi_disabled)
+        if (static_branch_likely(&psi_disabled))
                 return 0;
 
         cgroup->psi.pcpu = alloc_percpu(struct psi_group_cpu);
@@ -612,7 +624,7 @@ int psi_cgroup_alloc(struct cgroup *cgroup)
 
 void psi_cgroup_free(struct cgroup *cgroup)
 {
-        if (psi_disabled)
+        if (static_branch_likely(&psi_disabled))
                 return;
 
         cancel_delayed_work_sync(&cgroup->psi.clock_work);
@@ -633,38 +645,39 @@ void psi_cgroup_free(struct cgroup *cgroup)
  */
 void cgroup_move_task(struct task_struct *task, struct css_set *to)
 {
-        bool move_psi = !psi_disabled;
         unsigned int task_flags = 0;
         struct rq_flags rf;
         struct rq *rq;
 
-        if (move_psi) {
-                rq = task_rq_lock(task, &rf);
+        if (static_branch_likely(&psi_disabled)) {
+                /*
+                 * Lame to do this here, but the scheduler cannot be locked
+                 * from the outside, so we move cgroups from inside sched/.
+                 */
+                rcu_assign_pointer(task->cgroups, to);
+                return;
+        }
 
-                if (task_on_rq_queued(task))
-                        task_flags = TSK_RUNNING;
-                else if (task->in_iowait)
-                        task_flags = TSK_IOWAIT;
+        rq = task_rq_lock(task, &rf);
 
-                if (task->flags & PF_MEMSTALL)
-                        task_flags |= TSK_MEMSTALL;
+        if (task_on_rq_queued(task))
+                task_flags = TSK_RUNNING;
+        else if (task->in_iowait)
+                task_flags = TSK_IOWAIT;
 
-                if (task_flags)
-                        psi_task_change(task, task_flags, 0);
-        }
+        if (task->flags & PF_MEMSTALL)
+                task_flags |= TSK_MEMSTALL;
 
-        /*
-         * Lame to do this here, but the scheduler cannot be locked
-         * from the outside, so we move cgroups from inside sched/.
-         */
+        if (task_flags)
+                psi_task_change(task, task_flags, 0);
+
+        /* See comment above */
         rcu_assign_pointer(task->cgroups, to);
 
-        if (move_psi) {
-                if (task_flags)
-                        psi_task_change(task, 0, task_flags);
+        if (task_flags)
+                psi_task_change(task, 0, task_flags);
 
-                task_rq_unlock(rq, task, &rf);
-        }
+        task_rq_unlock(rq, task, &rf);
 }
 #endif /* CONFIG_CGROUPS */
 
@@ -672,7 +685,7 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res)
 {
         int full;
 
-        if (psi_disabled)
+        if (static_branch_likely(&psi_disabled))
                 return -EOPNOTSUPP;
 
         update_stats(group);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 618577fc9aa8..4e524ab589c9 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -23,6 +23,7 @@
 #include <linux/sched/prio.h>
 #include <linux/sched/rt.h>
 #include <linux/sched/signal.h>
+#include <linux/sched/smt.h>
 #include <linux/sched/stat.h>
 #include <linux/sched/sysctl.h>
 #include <linux/sched/task.h>
@@ -936,9 +937,6 @@ static inline int cpu_of(struct rq *rq)
 
 
 #ifdef CONFIG_SCHED_SMT
-
-extern struct static_key_false sched_smt_present;
-
 extern void __update_idle_core(struct rq *rq);
 
 static inline void update_idle_core(struct rq *rq)
diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h
index 4904c4677000..aa0de240fb41 100644
--- a/kernel/sched/stats.h
+++ b/kernel/sched/stats.h
@@ -66,7 +66,7 @@ static inline void psi_enqueue(struct task_struct *p, bool wakeup)
 {
         int clear = 0, set = TSK_RUNNING;
 
-        if (psi_disabled)
+        if (static_branch_likely(&psi_disabled))
                 return;
 
         if (!wakeup || p->sched_psi_wake_requeue) {
@@ -86,7 +86,7 @@ static inline void psi_dequeue(struct task_struct *p, bool sleep)
 {
         int clear = TSK_RUNNING, set = 0;
 
-        if (psi_disabled)
+        if (static_branch_likely(&psi_disabled))
                 return;
 
         if (!sleep) {
@@ -102,7 +102,7 @@ static inline void psi_dequeue(struct task_struct *p, bool sleep)
 
 static inline void psi_ttwu_dequeue(struct task_struct *p)
 {
-        if (psi_disabled)
+        if (static_branch_likely(&psi_disabled))
                 return;
         /*
          * Is the task being migrated during a wakeup? Make sure to
@@ -128,7 +128,7 @@ static inline void psi_ttwu_dequeue(struct task_struct *p)
 
 static inline void psi_task_tick(struct rq *rq)
 {
-        if (psi_disabled)
+        if (static_branch_likely(&psi_disabled))
                 return;
 
         if (unlikely(rq->curr->flags & PF_MEMSTALL))