sched: maintain only task entities in cfs_rq->tasks list

cfs_rq->tasks list is used by the load balancer to iterate
over all the tasks. Currently it holds all the entities
(both task and group entities) because of which there is
a need to check for group entities explicitly during load
balancing. This patch changes the cfs_rq->tasks list to
hold only task entities.

Signed-off-by: Bharata B Rao <bharata@linux.vnet.ibm.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

1 files changed, 9 insertions, 17 deletions

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index e3f3c10f7033..95c1295ad26d 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -528,11 +528,12 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
         update_load_add(&cfs_rq->load, se->load.weight);
         if (!parent_entity(se))
                 inc_cpu_load(rq_of(cfs_rq), se->load.weight);
-        if (entity_is_task(se))
+        if (entity_is_task(se)) {
                 add_cfs_task_weight(cfs_rq, se->load.weight);
+                list_add(&se->group_node, &cfs_rq->tasks);
+        }
         cfs_rq->nr_running++;
         se->on_rq = 1;
-        list_add(&se->group_node, &cfs_rq->tasks);
 }
 
 static void
@@ -541,11 +542,12 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
         update_load_sub(&cfs_rq->load, se->load.weight);
         if (!parent_entity(se))
                 dec_cpu_load(rq_of(cfs_rq), se->load.weight);
-        if (entity_is_task(se))
+        if (entity_is_task(se)) {
                 add_cfs_task_weight(cfs_rq, -se->load.weight);
+                list_del_init(&se->group_node);
+        }
         cfs_rq->nr_running--;
         se->on_rq = 0;
-        list_del_init(&se->group_node);
 }
 
 static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
@@ -1335,19 +1337,9 @@ __load_balance_iterator(struct cfs_rq *cfs_rq, struct list_head *next)
         if (next == &cfs_rq->tasks)
                 return NULL;
 
-        /* Skip over entities that are not tasks */
-        do {
-                se = list_entry(next, struct sched_entity, group_node);
-                next = next->next;
-        } while (next != &cfs_rq->tasks && !entity_is_task(se));
-
-        if (next == &cfs_rq->tasks && !entity_is_task(se))
-                return NULL;
-
-        cfs_rq->balance_iterator = next;
-
-        if (entity_is_task(se))
-                p = task_of(se);
+        se = list_entry(next, struct sched_entity, group_node);
+        p = task_of(se);
+        cfs_rq->balance_iterator = next->next;
 
         return p;
 }