1 files changed, 96 insertions, 116 deletions

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 6f579ff5a9bc..e91db32cadfd 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -222,21 +222,25 @@ niced_granularity(struct sched_entity *curr, unsigned long granularity)
 {
         u64 tmp;
 
+        if (likely(curr->load.weight == NICE_0_LOAD))
+                return granularity;
         /*
-         * Negative nice levels get the same granularity as nice-0:
+         * Positive nice levels get the same granularity as nice-0:
          */
-        if (likely(curr->load.weight >= NICE_0_LOAD))
-                return granularity;
+        if (likely(curr->load.weight < NICE_0_LOAD)) {
+                tmp = curr->load.weight * (u64)granularity;
+                return (long) (tmp >> NICE_0_SHIFT);
+        }
         /*
-         * Positive nice level tasks get linearly finer
+         * Negative nice level tasks get linearly finer
          * granularity:
          */
-        tmp = curr->load.weight * (u64)granularity;
+        tmp = curr->load.inv_weight * (u64)granularity;
 
         /*
          * It will always fit into 'long':
          */
-        return (long) (tmp >> NICE_0_SHIFT);
+        return (long) (tmp >> WMULT_SHIFT);
 }
 
 static inline void
@@ -281,26 +285,25 @@ add_wait_runtime(struct cfs_rq *cfs_rq, struct sched_entity *se, long delta)
  * are not in our scheduling class.
  */
 static inline void
-__update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr, u64 now)
+__update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr)
 {
-        unsigned long delta, delta_exec, delta_fair;
-        long delta_mine;
+        unsigned long delta, delta_exec, delta_fair, delta_mine;
         struct load_weight *lw = &cfs_rq->load;
         unsigned long load = lw->weight;
 
-        if (unlikely(!load))
-                return;
-
         delta_exec = curr->delta_exec;
         schedstat_set(curr->exec_max, max((u64)delta_exec, curr->exec_max));
 
         curr->sum_exec_runtime += delta_exec;
         cfs_rq->exec_clock += delta_exec;
 
+        if (unlikely(!load))
+                return;
+
         delta_fair = calc_delta_fair(delta_exec, lw);
         delta_mine = calc_delta_mine(delta_exec, curr->load.weight, lw);
 
-        if (cfs_rq->sleeper_bonus > sysctl_sched_stat_granularity) {
+        if (cfs_rq->sleeper_bonus > sysctl_sched_granularity) {
                 delta = calc_delta_mine(cfs_rq->sleeper_bonus,
                                         curr->load.weight, lw);
                 if (unlikely(delta > cfs_rq->sleeper_bonus))
@@ -321,7 +324,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr, u64 now)
         add_wait_runtime(cfs_rq, curr, delta_mine - delta_exec);
 }
 
-static void update_curr(struct cfs_rq *cfs_rq, u64 now)
+static void update_curr(struct cfs_rq *cfs_rq)
 {
         struct sched_entity *curr = cfs_rq_curr(cfs_rq);
         unsigned long delta_exec;
@@ -334,22 +337,22 @@ static void update_curr(struct cfs_rq *cfs_rq, u64 now)
          * since the last time we changed load (this cannot
          * overflow on 32 bits):
          */
-        delta_exec = (unsigned long)(now - curr->exec_start);
+        delta_exec = (unsigned long)(rq_of(cfs_rq)->clock - curr->exec_start);
 
         curr->delta_exec += delta_exec;
 
         if (unlikely(curr->delta_exec > sysctl_sched_stat_granularity)) {
-                __update_curr(cfs_rq, curr, now);
+                __update_curr(cfs_rq, curr);
                 curr->delta_exec = 0;
         }
-        curr->exec_start = now;
+        curr->exec_start = rq_of(cfs_rq)->clock;
 }
 
 static inline void
-update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
+update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         se->wait_start_fair = cfs_rq->fair_clock;
-        schedstat_set(se->wait_start, now);
+        schedstat_set(se->wait_start, rq_of(cfs_rq)->clock);
 }
 
 /*
@@ -377,8 +380,7 @@ calc_weighted(unsigned long delta, unsigned long weight, int shift)
 /*
  * Task is being enqueued - update stats:
  */
-static void
-update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
+static void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         s64 key;
 
@@ -387,7 +389,7 @@ update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
          * a dequeue/enqueue event is a NOP)
          */
         if (se != cfs_rq_curr(cfs_rq))
-                update_stats_wait_start(cfs_rq, se, now);
+                update_stats_wait_start(cfs_rq, se);
         /*
          * Update the key:
          */
@@ -407,7 +409,8 @@ update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
                                         (WMULT_SHIFT - NICE_0_SHIFT);
                 } else {
                         tmp = se->wait_runtime;
-                        key -= (tmp * se->load.weight) >> NICE_0_SHIFT;
+                        key -= (tmp * se->load.inv_weight) >>
+                                        (WMULT_SHIFT - NICE_0_SHIFT);
                 }
         }
 
@@ -418,11 +421,12 @@ update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
  * Note: must be called with a freshly updated rq->fair_clock.
  */
 static inline void
-__update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
+__update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         unsigned long delta_fair = se->delta_fair_run;
 
-        schedstat_set(se->wait_max, max(se->wait_max, now - se->wait_start));
+        schedstat_set(se->wait_max, max(se->wait_max,
+                        rq_of(cfs_rq)->clock - se->wait_start));
 
         if (unlikely(se->load.weight != NICE_0_LOAD))
                 delta_fair = calc_weighted(delta_fair, se->load.weight,
@@ -432,7 +436,7 @@ __update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
 }
 
 static void
-update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
+update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         unsigned long delta_fair;
 
@@ -442,7 +446,7 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
         se->delta_fair_run += delta_fair;
         if (unlikely(abs(se->delta_fair_run) >=
                                 sysctl_sched_stat_granularity)) {
-                __update_stats_wait_end(cfs_rq, se, now);
+                __update_stats_wait_end(cfs_rq, se);
                 se->delta_fair_run = 0;
         }
 
@@ -451,34 +455,34 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
 }
 
 static inline void
-update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
+update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-        update_curr(cfs_rq, now);
+        update_curr(cfs_rq);
         /*
          * Mark the end of the wait period if dequeueing a
          * waiting task:
          */
         if (se != cfs_rq_curr(cfs_rq))
-                update_stats_wait_end(cfs_rq, se, now);
+                update_stats_wait_end(cfs_rq, se);
 }
 
 /*
  * We are picking a new current task - update its stats:
  */
 static inline void
-update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
+update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         /*
          * We are starting a new run period:
          */
-        se->exec_start = now;
+        se->exec_start = rq_of(cfs_rq)->clock;
 }
 
 /*
  * We are descheduling a task - update its stats:
  */
 static inline void
-update_stats_curr_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
+update_stats_curr_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         se->exec_start = 0;
 }
@@ -487,8 +491,7 @@ update_stats_curr_end(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
  * Scheduling class queueing methods:
  */
 
-static void
-__enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
+static void __enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         unsigned long load = cfs_rq->load.weight, delta_fair;
         long prev_runtime;
@@ -522,8 +525,7 @@ __enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
         schedstat_add(cfs_rq, wait_runtime, se->wait_runtime);
 }
 
-static void
-enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
+static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         struct task_struct *tsk = task_of(se);
         unsigned long delta_fair;
@@ -538,7 +540,7 @@ enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
         se->delta_fair_sleep += delta_fair;
         if (unlikely(abs(se->delta_fair_sleep) >=
                                 sysctl_sched_stat_granularity)) {
-                __enqueue_sleeper(cfs_rq, se, now);
+                __enqueue_sleeper(cfs_rq, se);
                 se->delta_fair_sleep = 0;
         }
 
@@ -546,7 +548,7 @@ enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
 
 #ifdef CONFIG_SCHEDSTATS
         if (se->sleep_start) {
-                u64 delta = now - se->sleep_start;
+                u64 delta = rq_of(cfs_rq)->clock - se->sleep_start;
 
                 if ((s64)delta < 0)
                         delta = 0;
@@ -558,7 +560,7 @@ enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
                 se->sum_sleep_runtime += delta;
         }
         if (se->block_start) {
-                u64 delta = now - se->block_start;
+                u64 delta = rq_of(cfs_rq)->clock - se->block_start;
 
                 if ((s64)delta < 0)
                         delta = 0;
@@ -573,26 +575,24 @@ enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
 }
 
 static void
-enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
-               int wakeup, u64 now)
+enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup)
 {
         /*
          * Update the fair clock.
          */
-        update_curr(cfs_rq, now);
+        update_curr(cfs_rq);
 
         if (wakeup)
-                enqueue_sleeper(cfs_rq, se, now);
+                enqueue_sleeper(cfs_rq, se);
 
-        update_stats_enqueue(cfs_rq, se, now);
+        update_stats_enqueue(cfs_rq, se);
         __enqueue_entity(cfs_rq, se);
 }
 
 static void
-dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
-               int sleep, u64 now)
+dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
 {
-        update_stats_dequeue(cfs_rq, se, now);
+        update_stats_dequeue(cfs_rq, se);
         if (sleep) {
                 se->sleep_start_fair = cfs_rq->fair_clock;
 #ifdef CONFIG_SCHEDSTATS
@@ -600,9 +600,9 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
                         struct task_struct *tsk = task_of(se);
 
                         if (tsk->state & TASK_INTERRUPTIBLE)
-                                se->sleep_start = now;
+                                se->sleep_start = rq_of(cfs_rq)->clock;
                         if (tsk->state & TASK_UNINTERRUPTIBLE)
-                                se->block_start = now;
+                                se->block_start = rq_of(cfs_rq)->clock;
                 }
                 cfs_rq->wait_runtime -= se->wait_runtime;
 #endif
@@ -629,7 +629,7 @@ __check_preempt_curr_fair(struct cfs_rq *cfs_rq, struct sched_entity *se,
 }
 
 static inline void
-set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
+set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         /*
          * Any task has to be enqueued before it get to execute on
@@ -638,49 +638,46 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 now)
          * done a put_prev_task_fair() shortly before this, which
          * updated rq->fair_clock - used by update_stats_wait_end())
          */
-        update_stats_wait_end(cfs_rq, se, now);
-        update_stats_curr_start(cfs_rq, se, now);
+        update_stats_wait_end(cfs_rq, se);
+        update_stats_curr_start(cfs_rq, se);
         set_cfs_rq_curr(cfs_rq, se);
 }
 
-static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq, u64 now)
+static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq)
 {
         struct sched_entity *se = __pick_next_entity(cfs_rq);
 
-        set_next_entity(cfs_rq, se, now);
+        set_next_entity(cfs_rq, se);
 
         return se;
 }
 
-static void
-put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev, u64 now)
+static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
 {
         /*
          * If still on the runqueue then deactivate_task()
          * was not called and update_curr() has to be done:
          */
         if (prev->on_rq)
-                update_curr(cfs_rq, now);
+                update_curr(cfs_rq);
 
-        update_stats_curr_end(cfs_rq, prev, now);
+        update_stats_curr_end(cfs_rq, prev);
 
         if (prev->on_rq)
-                update_stats_wait_start(cfs_rq, prev, now);
+                update_stats_wait_start(cfs_rq, prev);
         set_cfs_rq_curr(cfs_rq, NULL);
 }
 
 static void entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
 {
-        struct rq *rq = rq_of(cfs_rq);
         struct sched_entity *next;
-        u64 now = __rq_clock(rq);
 
         /*
          * Dequeue and enqueue the task to update its
          * position within the tree:
          */
-        dequeue_entity(cfs_rq, curr, 0, now);
-        enqueue_entity(cfs_rq, curr, 0, now);
+        dequeue_entity(cfs_rq, curr, 0);
+        enqueue_entity(cfs_rq, curr, 0);
 
         /*
          * Reschedule if another task tops the current one.
@@ -785,8 +782,7 @@ static inline int is_same_group(struct task_struct *curr, struct task_struct *p)
  * increased. Here we update the fair scheduling stats and
  * then put the task into the rbtree:
  */
-static void
-enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
+static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup)
 {
         struct cfs_rq *cfs_rq;
         struct sched_entity *se = &p->se;
@@ -795,7 +791,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
                 if (se->on_rq)
                         break;
                 cfs_rq = cfs_rq_of(se);
-                enqueue_entity(cfs_rq, se, wakeup, now);
+                enqueue_entity(cfs_rq, se, wakeup);
         }
 }
 
@@ -804,15 +800,14 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup, u64 now)
  * decreased. We remove the task from the rbtree and
  * update the fair scheduling stats:
  */
-static void
-dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep, u64 now)
+static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep)
 {
         struct cfs_rq *cfs_rq;
         struct sched_entity *se = &p->se;
 
         for_each_sched_entity(se) {
                 cfs_rq = cfs_rq_of(se);
-                dequeue_entity(cfs_rq, se, sleep, now);
+                dequeue_entity(cfs_rq, se, sleep);
                 /* Don't dequeue parent if it has other entities besides us */
                 if (cfs_rq->load.weight)
                         break;
@@ -825,14 +820,14 @@ dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep, u64 now)
 static void yield_task_fair(struct rq *rq, struct task_struct *p)
 {
         struct cfs_rq *cfs_rq = task_cfs_rq(p);
-        u64 now = __rq_clock(rq);
 
+        __update_rq_clock(rq);
         /*
          * Dequeue and enqueue the task to update its
          * position within the tree:
          */
-        dequeue_entity(cfs_rq, &p->se, 0, now);
-        enqueue_entity(cfs_rq, &p->se, 0, now);
+        dequeue_entity(cfs_rq, &p->se, 0);
+        enqueue_entity(cfs_rq, &p->se, 0);
 }
 
 /*
@@ -845,7 +840,8 @@ static void check_preempt_curr_fair(struct rq *rq, struct task_struct *p)
         unsigned long gran;
 
         if (unlikely(rt_prio(p->prio))) {
-                update_curr(cfs_rq, rq_clock(rq));
+                update_rq_clock(rq);
+                update_curr(cfs_rq);
                 resched_task(curr);
                 return;
         }
@@ -861,7 +857,7 @@ static void check_preempt_curr_fair(struct rq *rq, struct task_struct *p)
                 __check_preempt_curr_fair(cfs_rq, &p->se, &curr->se, gran);
 }
 
-static struct task_struct *pick_next_task_fair(struct rq *rq, u64 now)
+static struct task_struct *pick_next_task_fair(struct rq *rq)
 {
         struct cfs_rq *cfs_rq = &rq->cfs;
         struct sched_entity *se;
@@ -870,7 +866,7 @@ static struct task_struct *pick_next_task_fair(struct rq *rq, u64 now)
                 return NULL;
 
         do {
-                se = pick_next_entity(cfs_rq, now);
+                se = pick_next_entity(cfs_rq);
                 cfs_rq = group_cfs_rq(se);
         } while (cfs_rq);
 
@@ -880,14 +876,14 @@ static struct task_struct *pick_next_task_fair(struct rq *rq, u64 now)
 /*
  * Account for a descheduled task:
  */
-static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, u64 now)
+static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
 {
         struct sched_entity *se = &prev->se;
         struct cfs_rq *cfs_rq;
 
         for_each_sched_entity(se) {
                 cfs_rq = cfs_rq_of(se);
-                put_prev_entity(cfs_rq, se, now);
+                put_prev_entity(cfs_rq, se);
         }
 }
 
@@ -930,6 +926,7 @@ static struct task_struct *load_balance_next_fair(void *arg)
         return __load_balance_iterator(cfs_rq, cfs_rq->rb_load_balance_curr);
 }
 
+#ifdef CONFIG_FAIR_GROUP_SCHED
 static int cfs_rq_best_prio(struct cfs_rq *cfs_rq)
 {
         struct sched_entity *curr;
@@ -943,12 +940,13 @@ static int cfs_rq_best_prio(struct cfs_rq *cfs_rq)
 
         return p->prio;
 }
+#endif
 
-static int
+static unsigned long
 load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
-                        unsigned long max_nr_move, unsigned long max_load_move,
-                        struct sched_domain *sd, enum cpu_idle_type idle,
-                        int *all_pinned, unsigned long *total_load_moved)
+                  unsigned long max_nr_move, unsigned long max_load_move,
+                  struct sched_domain *sd, enum cpu_idle_type idle,
+                  int *all_pinned, int *this_best_prio)
 {
         struct cfs_rq *busy_cfs_rq;
         unsigned long load_moved, total_nr_moved = 0, nr_moved;
@@ -959,10 +957,10 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
         cfs_rq_iterator.next = load_balance_next_fair;
 
         for_each_leaf_cfs_rq(busiest, busy_cfs_rq) {
+#ifdef CONFIG_FAIR_GROUP_SCHED
                 struct cfs_rq *this_cfs_rq;
-                long imbalance;
+                long imbalances;
                 unsigned long maxload;
-                int this_best_prio, best_prio, best_prio_seen = 0;
 
                 this_cfs_rq = cpu_cfs_rq(busy_cfs_rq, this_cpu);
 
@@ -976,27 +974,17 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
                 imbalance /= 2;
                 maxload = min(rem_load_move, imbalance);
 
-                this_best_prio = cfs_rq_best_prio(this_cfs_rq);
-                best_prio = cfs_rq_best_prio(busy_cfs_rq);
-
-                /*
-                 * Enable handling of the case where there is more than one task
-                 * with the best priority. If the current running task is one
-                 * of those with prio==best_prio we know it won't be moved
-                 * and therefore it's safe to override the skip (based on load)
-                 * of any task we find with that prio.
-                 */
-                if (cfs_rq_curr(busy_cfs_rq) == &busiest->curr->se)
-                        best_prio_seen = 1;
-
+                *this_best_prio = cfs_rq_best_prio(this_cfs_rq);
+#else
+#define maxload rem_load_move
+#endif
                 /* pass busy_cfs_rq argument into
                  * load_balance_[start|next]_fair iterators
                  */
                 cfs_rq_iterator.arg = busy_cfs_rq;
                 nr_moved = balance_tasks(this_rq, this_cpu, busiest,
                                 max_nr_move, maxload, sd, idle, all_pinned,
-                                &load_moved, this_best_prio, best_prio,
-                                best_prio_seen, &cfs_rq_iterator);
+                                &load_moved, this_best_prio, &cfs_rq_iterator);
 
                 total_nr_moved += nr_moved;
                 max_nr_move -= nr_moved;
@@ -1006,9 +994,7 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
                         break;
         }
 
-        *total_load_moved = max_load_move - rem_load_move;
-
-        return total_nr_moved;
+        return max_load_move - rem_load_move;
 }
 
 /*
@@ -1032,14 +1018,14 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr)
  * monopolize the CPU. Note: the parent runqueue is locked,
  * the child is not running yet.
  */
-static void task_new_fair(struct rq *rq, struct task_struct *p, u64 now)
+static void task_new_fair(struct rq *rq, struct task_struct *p)
 {
         struct cfs_rq *cfs_rq = task_cfs_rq(p);
         struct sched_entity *se = &p->se;
 
         sched_info_queued(p);
 
-        update_stats_enqueue(cfs_rq, se, now);
+        update_stats_enqueue(cfs_rq, se);
         /*
          * Child runs first: we let it run before the parent
          * until it reschedules once. We set up the key so that
@@ -1072,15 +1058,10 @@ static void task_new_fair(struct rq *rq, struct task_struct *p, u64 now)
  */
 static void set_curr_task_fair(struct rq *rq)
 {
-        struct task_struct *curr = rq->curr;
-        struct sched_entity *se = &curr->se;
-        u64 now = rq_clock(rq);
-        struct cfs_rq *cfs_rq;
+        struct sched_entity *se = &rq->curr.se;
 
-        for_each_sched_entity(se) {
-                cfs_rq = cfs_rq_of(se);
-                set_next_entity(cfs_rq, se, now);
-        }
+        for_each_sched_entity(se)
+                set_next_entity(cfs_rq_of(se), se);
 }
 #else
 static void set_curr_task_fair(struct rq *rq)
@@ -1109,12 +1090,11 @@ struct sched_class fair_sched_class __read_mostly = {
 };
 
 #ifdef CONFIG_SCHED_DEBUG
-void print_cfs_stats(struct seq_file *m, int cpu, u64 now)
+static void print_cfs_stats(struct seq_file *m, int cpu)
 {
-        struct rq *rq = cpu_rq(cpu);
         struct cfs_rq *cfs_rq;
 
-        for_each_leaf_cfs_rq(rq, cfs_rq)
-                print_cfs_rq(m, cpu, cfs_rq, now);
+        for_each_leaf_cfs_rq(cpu_rq(cpu), cfs_rq)
+                print_cfs_rq(m, cpu, cfs_rq);
 }
 #endif