cgroup: Implement css_task_iter_skip()

When a task is moved out of a cset, task iterators pointing to the
task are advanced using the normal css_task_iter_advance() call.  This
is fine but we'll be tracking dying tasks on csets and thus moving
tasks from cset->tasks to (to be added) cset->dying_tasks.  When we
remove a task from cset->tasks, if we advance the iterators, they may
move over to the next cset before we had the chance to add the task
back on the dying list, which can allow the task to escape iteration.

This patch separates out skipping from advancing.  Skipping only moves
the affected iterators to the next pointer rather than fully advancing
it and the following advancing will recognize that the cursor has
already been moved forward and do the rest of advancing.  This ensures
that when a task moves from one list to another in its cset, as long
as it moves in the right direction, it's always visible to iteration.

This doesn't cause any visible behavior changes.

Signed-off-by: Tejun Heo <tj@kernel.org>
Cc: Oleg Nesterov <oleg@redhat.com>

2 files changed, 39 insertions, 24 deletions

diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index a7e4611e20c8..05ed2a209e74 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -43,6 +43,9 @@
 /* walk all threaded css_sets in the domain */
 #define CSS_TASK_ITER_THREADED          (1U << 1)
 
+/* internal flags */
+#define CSS_TASK_ITER_SKIPPED           (1U << 16)
+
 /* a css_task_iter should be treated as an opaque object */
 struct css_task_iter {
         struct cgroup_subsys            *ss;
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 217cec4e22c6..035aee466bbf 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -215,7 +215,8 @@ static struct cftype cgroup_base_files[];
 
 static int cgroup_apply_control(struct cgroup *cgrp);
 static void cgroup_finalize_control(struct cgroup *cgrp, int ret);
-static void css_task_iter_advance(struct css_task_iter *it);
+static void css_task_iter_skip(struct css_task_iter *it,
+                               struct task_struct *task);
 static int cgroup_destroy_locked(struct cgroup *cgrp);
 static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
                                               struct cgroup_subsys *ss);
@@ -843,6 +844,21 @@ static void css_set_update_populated(struct css_set *cset, bool populated)
                 cgroup_update_populated(link->cgrp, populated);
 }
 
+/*
+ * @task is leaving, advance task iterators which are pointing to it so
+ * that they can resume at the next position.  Advancing an iterator might
+ * remove it from the list, use safe walk.  See css_task_iter_skip() for
+ * details.
+ */
+static void css_set_skip_task_iters(struct css_set *cset,
+                                    struct task_struct *task)
+{
+        struct css_task_iter *it, *pos;
+
+        list_for_each_entry_safe(it, pos, &cset->task_iters, iters_node)
+                css_task_iter_skip(it, task);
+}
+
 /**
  * css_set_move_task - move a task from one css_set to another
  * @task: task being moved
@@ -868,22 +884,9 @@ static void css_set_move_task(struct task_struct *task,
                 css_set_update_populated(to_cset, true);
 
         if (from_cset) {
-                struct css_task_iter *it, *pos;
-
                 WARN_ON_ONCE(list_empty(&task->cg_list));
 
-                /*
-                 * @task is leaving, advance task iterators which are
-                 * pointing to it so that they can resume at the next
-                 * position.  Advancing an iterator might remove it from
-                 * the list, use safe walk.  See css_task_iter_advance*()
-                 * for details.
-                 */
-                list_for_each_entry_safe(it, pos, &from_cset->task_iters,
-                                         iters_node)
-                        if (it->task_pos == &task->cg_list)
-                                css_task_iter_advance(it);
-
+                css_set_skip_task_iters(from_cset, task);
                 list_del_init(&task->cg_list);
                 if (!css_set_populated(from_cset))
                         css_set_update_populated(from_cset, false);
@@ -4430,10 +4433,19 @@ static void css_task_iter_advance_css_set(struct css_task_iter *it)
         list_add(&it->iters_node, &cset->task_iters);
 }
 
-static void css_task_iter_advance(struct css_task_iter *it)
+static void css_task_iter_skip(struct css_task_iter *it,
+                               struct task_struct *task)
 {
-        struct list_head *next;
+        lockdep_assert_held(&css_set_lock);
+
+        if (it->task_pos == &task->cg_list) {
+                it->task_pos = it->task_pos->next;
+                it->flags |= CSS_TASK_ITER_SKIPPED;
+        }
+}
 
+static void css_task_iter_advance(struct css_task_iter *it)
+{
         lockdep_assert_held(&css_set_lock);
 repeat:
         if (it->task_pos) {
@@ -4442,15 +4454,15 @@ repeat:
                  * consumed first and then ->mg_tasks.  After ->mg_tasks,
                  * we move onto the next cset.
                  */
-                next = it->task_pos->next;
-
-                if (next == it->tasks_head)
-                        next = it->mg_tasks_head->next;
+                if (it->flags & CSS_TASK_ITER_SKIPPED)
+                        it->flags &= ~CSS_TASK_ITER_SKIPPED;
+                else
+                        it->task_pos = it->task_pos->next;
 
-                if (next == it->mg_tasks_head)
+                if (it->task_pos == it->tasks_head)
+                        it->task_pos = it->mg_tasks_head->next;
+                if (it->task_pos == it->mg_tasks_head)
                         css_task_iter_advance_css_set(it);
-                else
-                        it->task_pos = next;
         } else {
                 /* called from start, proceed to the first cset */
                 css_task_iter_advance_css_set(it);