sched/core: Provide a pointer to the valid CPU mask

In commit:

  4b53a3412d66 ("sched/core: Remove the tsk_nr_cpus_allowed() wrapper")

the tsk_nr_cpus_allowed() wrapper was removed. There was not
much difference in !RT but in RT we used this to implement
migrate_disable(). Within a migrate_disable() section the CPU mask is
restricted to single CPU while the "normal" CPU mask remains untouched.

As an alternative implementation Ingo suggested to use:

	struct task_struct {
		const cpumask_t		*cpus_ptr;
		cpumask_t		cpus_mask;
        };
with
	t->cpus_ptr = &t->cpus_mask;

In -RT we then can switch the cpus_ptr to:

	t->cpus_ptr = &cpumask_of(task_cpu(p));

in a migration disabled region. The rules are simple:

 - Code that 'uses' ->cpus_allowed would use the pointer.
 - Code that 'modifies' ->cpus_allowed would use the direct mask.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20190423142636.14347-1-bigeasy@linutronix.de
Signed-off-by: Ingo Molnar <mingo@kernel.org>

9 files changed, 50 insertions, 48 deletions

diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 6a1942ed781c..fe90fa1899e6 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -2829,7 +2829,7 @@ static void cpuset_fork(struct task_struct *task)
         if (task_css_is_root(task, cpuset_cgrp_id))
                 return;
 
-        set_cpus_allowed_ptr(task, &current->cpus_allowed);
+        set_cpus_allowed_ptr(task, current->cpus_ptr);
         task->mems_allowed = current->mems_allowed;
 }
 
diff --git a/kernel/fork.c b/kernel/fork.c
index 75675b9bf6df..6be686283e55 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -894,6 +894,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 #ifdef CONFIG_STACKPROTECTOR
         tsk->stack_canary = get_random_canary();
 #endif
+        if (orig->cpus_ptr == &orig->cpus_mask)
+                tsk->cpus_ptr = &tsk->cpus_mask;
 
         /*
          * One for us, one for whoever does the "release_task()" (usually
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 874c427742a9..93ab85f0d076 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -930,7 +930,7 @@ static inline bool is_per_cpu_kthread(struct task_struct *p)
  */
 static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
 {
-        if (!cpumask_test_cpu(cpu, &p->cpus_allowed))
+        if (!cpumask_test_cpu(cpu, p->cpus_ptr))
                 return false;
 
         if (is_per_cpu_kthread(p))
@@ -1025,7 +1025,7 @@ static int migration_cpu_stop(void *data)
         local_irq_disable();
         /*
          * We need to explicitly wake pending tasks before running
-         * __migrate_task() such that we will not miss enforcing cpus_allowed
+         * __migrate_task() such that we will not miss enforcing cpus_ptr
          * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
          */
         sched_ttwu_pending();
@@ -1056,7 +1056,7 @@ static int migration_cpu_stop(void *data)
  */
 void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask)
 {
-        cpumask_copy(&p->cpus_allowed, new_mask);
+        cpumask_copy(&p->cpus_mask, new_mask);
         p->nr_cpus_allowed = cpumask_weight(new_mask);
 }
 
@@ -1126,7 +1126,7 @@ static int __set_cpus_allowed_ptr(struct task_struct *p,
                 goto out;
         }
 
-        if (cpumask_equal(&p->cpus_allowed, new_mask))
+        if (cpumask_equal(p->cpus_ptr, new_mask))
                 goto out;
 
         if (!cpumask_intersects(new_mask, cpu_valid_mask)) {
@@ -1286,10 +1286,10 @@ static int migrate_swap_stop(void *data)
         if (task_cpu(arg->src_task) != arg->src_cpu)
                 goto unlock;
 
-        if (!cpumask_test_cpu(arg->dst_cpu, &arg->src_task->cpus_allowed))
+        if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr))
                 goto unlock;
 
-        if (!cpumask_test_cpu(arg->src_cpu, &arg->dst_task->cpus_allowed))
+        if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr))
                 goto unlock;
 
         __migrate_swap_task(arg->src_task, arg->dst_cpu);
@@ -1331,10 +1331,10 @@ int migrate_swap(struct task_struct *cur, struct task_struct *p,
         if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
                 goto out;
 
-        if (!cpumask_test_cpu(arg.dst_cpu, &arg.src_task->cpus_allowed))
+        if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr))
                 goto out;
 
-        if (!cpumask_test_cpu(arg.src_cpu, &arg.dst_task->cpus_allowed))
+        if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr))
                 goto out;
 
         trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
@@ -1479,7 +1479,7 @@ void kick_process(struct task_struct *p)
 EXPORT_SYMBOL_GPL(kick_process);
 
 /*
- * ->cpus_allowed is protected by both rq->lock and p->pi_lock
+ * ->cpus_ptr is protected by both rq->lock and p->pi_lock
  *
  * A few notes on cpu_active vs cpu_online:
  *
@@ -1519,14 +1519,14 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
                 for_each_cpu(dest_cpu, nodemask) {
                         if (!cpu_active(dest_cpu))
                                 continue;
-                        if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
+                        if (cpumask_test_cpu(dest_cpu, p->cpus_ptr))
                                 return dest_cpu;
                 }
         }
 
         for (;;) {
                 /* Any allowed, online CPU? */
-                for_each_cpu(dest_cpu, &p->cpus_allowed) {
+                for_each_cpu(dest_cpu, p->cpus_ptr) {
                         if (!is_cpu_allowed(p, dest_cpu))
                                 continue;
 
@@ -1570,7 +1570,7 @@ out:
 }
 
 /*
- * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
+ * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable.
  */
 static inline
 int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
@@ -1580,11 +1580,11 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
         if (p->nr_cpus_allowed > 1)
                 cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
         else
-                cpu = cpumask_any(&p->cpus_allowed);
+                cpu = cpumask_any(p->cpus_ptr);
 
         /*
          * In order not to call set_task_cpu() on a blocking task we need
-         * to rely on ttwu() to place the task on a valid ->cpus_allowed
+         * to rely on ttwu() to place the task on a valid ->cpus_ptr
          * CPU.
          *
          * Since this is common to all placement strategies, this lives here.
@@ -2395,7 +2395,7 @@ void wake_up_new_task(struct task_struct *p)
 #ifdef CONFIG_SMP
         /*
          * Fork balancing, do it here and not earlier because:
-         *  - cpus_allowed can change in the fork path
+         *  - cpus_ptr can change in the fork path
          *  - any previously selected CPU might disappear through hotplug
          *
          * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq,
@@ -4267,7 +4267,7 @@ change:
                          * the entire root_domain to become SCHED_DEADLINE. We
                          * will also fail if there's no bandwidth available.
                          */
-                        if (!cpumask_subset(span, &p->cpus_allowed) ||
+                        if (!cpumask_subset(span, p->cpus_ptr) ||
                             rq->rd->dl_bw.bw == 0) {
                                 task_rq_unlock(rq, p, &rf);
                                 return -EPERM;
@@ -4866,7 +4866,7 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask)
                 goto out_unlock;
 
         raw_spin_lock_irqsave(&p->pi_lock, flags);
-        cpumask_and(mask, &p->cpus_allowed, cpu_active_mask);
+        cpumask_and(mask, &p->cpus_mask, cpu_active_mask);
         raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 
 out_unlock:
@@ -5443,7 +5443,7 @@ int task_can_attach(struct task_struct *p,
          * allowed nodes is unnecessary.  Thus, cpusets are not
          * applicable for such threads.  This prevents checking for
          * success of set_cpus_allowed_ptr() on all attached tasks
-         * before cpus_allowed may be changed.
+         * before cpus_mask may be changed.
          */
         if (p->flags & PF_NO_SETAFFINITY) {
                 ret = -EINVAL;
@@ -5470,7 +5470,7 @@ int migrate_task_to(struct task_struct *p, int target_cpu)
         if (curr_cpu == target_cpu)
                 return 0;
 
-        if (!cpumask_test_cpu(target_cpu, &p->cpus_allowed))
+        if (!cpumask_test_cpu(target_cpu, p->cpus_ptr))
                 return -EINVAL;
 
         /* TODO: This is not properly updating schedstats */
@@ -5608,7 +5608,7 @@ static void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf)
                 put_prev_task(rq, next);
 
                 /*
-                 * Rules for changing task_struct::cpus_allowed are holding
+                 * Rules for changing task_struct::cpus_mask are holding
                  * both pi_lock and rq->lock, such that holding either
                  * stabilizes the mask.
                  *
diff --git a/kernel/sched/cpudeadline.c b/kernel/sched/cpudeadline.c
index 50316455ea66..d57fb2f8ae67 100644
--- a/kernel/sched/cpudeadline.c
+++ b/kernel/sched/cpudeadline.c
@@ -124,14 +124,14 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p,
         const struct sched_dl_entity *dl_se = &p->dl;
 
         if (later_mask &&
-            cpumask_and(later_mask, cp->free_cpus, &p->cpus_allowed)) {
+            cpumask_and(later_mask, cp->free_cpus, p->cpus_ptr)) {
                 return 1;
         } else {
                 int best_cpu = cpudl_maximum(cp);
 
                 WARN_ON(best_cpu != -1 && !cpu_present(best_cpu));
 
-                if (cpumask_test_cpu(best_cpu, &p->cpus_allowed) &&
+                if (cpumask_test_cpu(best_cpu, p->cpus_ptr) &&
                     dl_time_before(dl_se->deadline, cp->elements[0].dl)) {
                         if (later_mask)
                                 cpumask_set_cpu(best_cpu, later_mask);
diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c
index daaadf939ccb..f7d2c10b4c92 100644
--- a/kernel/sched/cpupri.c
+++ b/kernel/sched/cpupri.c
@@ -98,11 +98,11 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p,
                 if (skip)
                         continue;
 
-                if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids)
+                if (cpumask_any_and(p->cpus_ptr, vec->mask) >= nr_cpu_ids)
                         continue;
 
                 if (lowest_mask) {
-                        cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask);
+                        cpumask_and(lowest_mask, p->cpus_ptr, vec->mask);
 
                         /*
                          * We have to ensure that we have at least one bit
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 43901fa3f269..c1ef30861068 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -538,7 +538,7 @@ static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p
                  * If we cannot preempt any rq, fall back to pick any
                  * online CPU:
                  */
-                cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
+                cpu = cpumask_any_and(cpu_active_mask, p->cpus_ptr);
                 if (cpu >= nr_cpu_ids) {
                         /*
                          * Failed to find any suitable CPU.
@@ -1824,7 +1824,7 @@ static void set_curr_task_dl(struct rq *rq)
 static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu)
 {
         if (!task_running(rq, p) &&
-            cpumask_test_cpu(cpu, &p->cpus_allowed))
+            cpumask_test_cpu(cpu, p->cpus_ptr))
                 return 1;
         return 0;
 }
@@ -1974,7 +1974,7 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq)
                 /* Retry if something changed. */
                 if (double_lock_balance(rq, later_rq)) {
                         if (unlikely(task_rq(task) != rq ||
-                                     !cpumask_test_cpu(later_rq->cpu, &task->cpus_allowed) ||
+                                     !cpumask_test_cpu(later_rq->cpu, task->cpus_ptr) ||
                                      task_running(rq, task) ||
                                      !dl_task(task) ||
                                      !task_on_rq_queued(task))) {
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index f35930f5e528..8691a8fffe40 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1621,7 +1621,7 @@ static void task_numa_compare(struct task_numa_env *env,
          * be incurred if the tasks were swapped.
          */
         /* Skip this swap candidate if cannot move to the source cpu */
-        if (!cpumask_test_cpu(env->src_cpu, &cur->cpus_allowed))
+        if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr))
                 goto unlock;
 
         /*
@@ -1718,7 +1718,7 @@ static void task_numa_find_cpu(struct task_numa_env *env,
 
         for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {
                 /* Skip this CPU if the source task cannot migrate */
-                if (!cpumask_test_cpu(cpu, &env->p->cpus_allowed))
+                if (!cpumask_test_cpu(cpu, env->p->cpus_ptr))
                         continue;
 
                 env->dst_cpu = cpu;
@@ -5831,7 +5831,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p,
 
                 /* Skip over this group if it has no CPUs allowed */
                 if (!cpumask_intersects(sched_group_span(group),
-                                        &p->cpus_allowed))
+                                        p->cpus_ptr))
                         continue;
 
                 local_group = cpumask_test_cpu(this_cpu,
@@ -5963,7 +5963,7 @@ find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this
                 return cpumask_first(sched_group_span(group));
 
         /* Traverse only the allowed CPUs */
-        for_each_cpu_and(i, sched_group_span(group), &p->cpus_allowed) {
+        for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) {
                 if (available_idle_cpu(i)) {
                         struct rq *rq = cpu_rq(i);
                         struct cpuidle_state *idle = idle_get_state(rq);
@@ -6003,7 +6003,7 @@ static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p
 {
         int new_cpu = cpu;
 
-        if (!cpumask_intersects(sched_domain_span(sd), &p->cpus_allowed))
+        if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr))
                 return prev_cpu;
 
         /*
@@ -6120,7 +6120,7 @@ static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int
         if (!test_idle_cores(target, false))
                 return -1;
 
-        cpumask_and(cpus, sched_domain_span(sd), &p->cpus_allowed);
+        cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
 
         for_each_cpu_wrap(core, cpus, target) {
                 bool idle = true;
@@ -6154,7 +6154,7 @@ static int select_idle_smt(struct task_struct *p, int target)
                 return -1;
 
         for_each_cpu(cpu, cpu_smt_mask(target)) {
-                if (!cpumask_test_cpu(cpu, &p->cpus_allowed))
+                if (!cpumask_test_cpu(cpu, p->cpus_ptr))
                         continue;
                 if (available_idle_cpu(cpu))
                         return cpu;
@@ -6217,7 +6217,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t
         for_each_cpu_wrap(cpu, sched_domain_span(sd), target) {
                 if (!--nr)
                         return -1;
-                if (!cpumask_test_cpu(cpu, &p->cpus_allowed))
+                if (!cpumask_test_cpu(cpu, p->cpus_ptr))
                         continue;
                 if (available_idle_cpu(cpu))
                         break;
@@ -6254,7 +6254,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
             recent_used_cpu != target &&
             cpus_share_cache(recent_used_cpu, target) &&
             available_idle_cpu(recent_used_cpu) &&
-            cpumask_test_cpu(p->recent_used_cpu, &p->cpus_allowed)) {
+            cpumask_test_cpu(p->recent_used_cpu, p->cpus_ptr)) {
                 /*
                  * Replace recent_used_cpu with prev as it is a potential
                  * candidate for the next wake:
@@ -6600,7 +6600,7 @@ static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
                 int max_spare_cap_cpu = -1;
 
                 for_each_cpu_and(cpu, perf_domain_span(pd), sched_domain_span(sd)) {
-                        if (!cpumask_test_cpu(cpu, &p->cpus_allowed))
+                        if (!cpumask_test_cpu(cpu, p->cpus_ptr))
                                 continue;
 
                         /* Skip CPUs that will be overutilized. */
@@ -6689,7 +6689,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f
                 }
 
                 want_affine = !wake_wide(p) && !wake_cap(p, cpu, prev_cpu) &&
-                              cpumask_test_cpu(cpu, &p->cpus_allowed);
+                              cpumask_test_cpu(cpu, p->cpus_ptr);
         }
 
         rcu_read_lock();
@@ -7445,14 +7445,14 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
         /*
          * We do not migrate tasks that are:
          * 1) throttled_lb_pair, or
-         * 2) cannot be migrated to this CPU due to cpus_allowed, or
+         * 2) cannot be migrated to this CPU due to cpus_ptr, or
          * 3) running (obviously), or
          * 4) are cache-hot on their current CPU.
          */
         if (throttled_lb_pair(task_group(p), env->src_cpu, env->dst_cpu))
                 return 0;
 
-        if (!cpumask_test_cpu(env->dst_cpu, &p->cpus_allowed)) {
+        if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) {
                 int cpu;
 
                 schedstat_inc(p->se.statistics.nr_failed_migrations_affine);
@@ -7472,7 +7472,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
 
                 /* Prevent to re-select dst_cpu via env's CPUs: */
                 for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) {
-                        if (cpumask_test_cpu(cpu, &p->cpus_allowed)) {
+                        if (cpumask_test_cpu(cpu, p->cpus_ptr)) {
                                 env->flags |= LBF_DST_PINNED;
                                 env->new_dst_cpu = cpu;
                                 break;
@@ -8099,7 +8099,7 @@ static inline int check_misfit_status(struct rq *rq, struct sched_domain *sd)
 
 /*
  * Group imbalance indicates (and tries to solve) the problem where balancing
- * groups is inadequate due to ->cpus_allowed constraints.
+ * groups is inadequate due to ->cpus_ptr constraints.
  *
  * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a
  * cpumask covering 1 CPU of the first group and 3 CPUs of the second group.
@@ -8768,7 +8768,7 @@ static struct sched_group *find_busiest_group(struct lb_env *env)
         /*
          * If the busiest group is imbalanced the below checks don't
          * work because they assume all things are equal, which typically
-         * isn't true due to cpus_allowed constraints and the like.
+         * isn't true due to cpus_ptr constraints and the like.
          */
         if (busiest->group_type == group_imbalanced)
                 goto force_balance;
@@ -9210,7 +9210,7 @@ more_balance:
                          * if the curr task on busiest CPU can't be
                          * moved to this_cpu:
                          */
-                        if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) {
+                        if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) {
                                 raw_spin_unlock_irqrestore(&busiest->lock,
                                                             flags);
                                 env.flags |= LBF_ALL_PINNED;
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 1e6b909dca36..63ad7c90822c 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1614,7 +1614,7 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
 static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
 {
         if (!task_running(rq, p) &&
-            cpumask_test_cpu(cpu, &p->cpus_allowed))
+            cpumask_test_cpu(cpu, p->cpus_ptr))
                 return 1;
 
         return 0;
@@ -1751,7 +1751,7 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
                          * Also make sure that it wasn't scheduled on its rq.
                          */
                         if (unlikely(task_rq(task) != rq ||
-                                     !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_allowed) ||
+                                     !cpumask_test_cpu(lowest_rq->cpu, task->cpus_ptr) ||
                                      task_running(rq, task) ||
                                      !rt_task(task) ||
                                      !task_on_rq_queued(task))) {
diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c
index 1e6db9cbe4dc..fa95139445b2 100644
--- a/kernel/trace/trace_hwlat.c
+++ b/kernel/trace/trace_hwlat.c
@@ -277,7 +277,7 @@ static void move_to_next_cpu(void)
          * of this thread, than stop migrating for the duration
          * of the current test.
          */
-        if (!cpumask_equal(current_mask, &current->cpus_allowed))
+        if (!cpumask_equal(current_mask, current->cpus_ptr))
                 goto disable;
 
         get_online_cpus();