1 files changed, 78 insertions, 35 deletions

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 56b7d4b83947..51a45502d8a6 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -20,8 +20,8 @@
  *  Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
  */
 
-#include <linux/latencytop.h>
 #include <linux/sched.h>
+#include <linux/latencytop.h>
 #include <linux/cpumask.h>
 #include <linux/cpuidle.h>
 #include <linux/slab.h>
@@ -755,7 +755,9 @@ static void
 update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         struct task_struct *p;
-        u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start;
+        u64 delta;
+
+        delta = rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start;
 
         if (entity_is_task(se)) {
                 p = task_of(se);
@@ -776,22 +778,12 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
         se->statistics.wait_sum += delta;
         se->statistics.wait_start = 0;
 }
-#else
-static inline void
-update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
-{
-}
-
-static inline void
-update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
-{
-}
-#endif
 
 /*
  * Task is being enqueued - update stats:
  */
-static void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
+static inline void
+update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
         /*
          * Are we enqueueing a waiting task? (for current tasks
@@ -802,7 +794,7 @@ static void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 }
 
 static inline void
-update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
+update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 {
         /*
          * Mark the end of the wait period if dequeueing a
@@ -810,7 +802,40 @@ update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
          */
         if (se != cfs_rq->curr)
                 update_stats_wait_end(cfs_rq, se);
+
+        if (flags & DEQUEUE_SLEEP) {
+                if (entity_is_task(se)) {
+                        struct task_struct *tsk = task_of(se);
+
+                        if (tsk->state & TASK_INTERRUPTIBLE)
+                                se->statistics.sleep_start = rq_clock(rq_of(cfs_rq));
+                        if (tsk->state & TASK_UNINTERRUPTIBLE)
+                                se->statistics.block_start = rq_clock(rq_of(cfs_rq));
+                }
+        }
+
+}
+#else
+static inline void
+update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
+{
+}
+
+static inline void
+update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
+{
+}
+
+static inline void
+update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
+{
+}
+
+static inline void
+update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
+{
 }
+#endif
 
 /*
  * We are picking a new current task - update its stats:
@@ -3102,6 +3127,26 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
 
 static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
 
+static inline void check_schedstat_required(void)
+{
+#ifdef CONFIG_SCHEDSTATS
+        if (schedstat_enabled())
+                return;
+
+        /* Force schedstat enabled if a dependent tracepoint is active */
+        if (trace_sched_stat_wait_enabled()    ||
+                        trace_sched_stat_sleep_enabled()   ||
+                        trace_sched_stat_iowait_enabled()  ||
+                        trace_sched_stat_blocked_enabled() ||
+                        trace_sched_stat_runtime_enabled())  {
+                pr_warn_once("Scheduler tracepoints stat_sleep, stat_iowait, "
+                             "stat_blocked and stat_runtime require the "
+                             "kernel parameter schedstats=enabled or "
+                             "kernel.sched_schedstats=1\n");
+        }
+#endif
+}
+
 static void
 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 {
@@ -3122,11 +3167,15 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 
         if (flags & ENQUEUE_WAKEUP) {
                 place_entity(cfs_rq, se, 0);
-                enqueue_sleeper(cfs_rq, se);
+                if (schedstat_enabled())
+                        enqueue_sleeper(cfs_rq, se);
         }
 
-        update_stats_enqueue(cfs_rq, se);
-        check_spread(cfs_rq, se);
+        check_schedstat_required();
+        if (schedstat_enabled()) {
+                update_stats_enqueue(cfs_rq, se);
+                check_spread(cfs_rq, se);
+        }
         if (se != cfs_rq->curr)
                 __enqueue_entity(cfs_rq, se);
         se->on_rq = 1;
@@ -3193,19 +3242,8 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
         update_curr(cfs_rq);
         dequeue_entity_load_avg(cfs_rq, se);
 
-        update_stats_dequeue(cfs_rq, se);
-        if (flags & DEQUEUE_SLEEP) {
-#ifdef CONFIG_SCHEDSTATS
-                if (entity_is_task(se)) {
-                        struct task_struct *tsk = task_of(se);
-
-                        if (tsk->state & TASK_INTERRUPTIBLE)
-                                se->statistics.sleep_start = rq_clock(rq_of(cfs_rq));
-                        if (tsk->state & TASK_UNINTERRUPTIBLE)
-                                se->statistics.block_start = rq_clock(rq_of(cfs_rq));
-                }
-#endif
-        }
+        if (schedstat_enabled())
+                update_stats_dequeue(cfs_rq, se, flags);
 
         clear_buddies(cfs_rq, se);
 
@@ -3279,7 +3317,8 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
                  * a CPU. So account for the time it spent waiting on the
                  * runqueue.
                  */
-                update_stats_wait_end(cfs_rq, se);
+                if (schedstat_enabled())
+                        update_stats_wait_end(cfs_rq, se);
                 __dequeue_entity(cfs_rq, se);
                 update_load_avg(se, 1);
         }
@@ -3292,7 +3331,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
          * least twice that of our own weight (i.e. dont track it
          * when there are only lesser-weight tasks around):
          */
-        if (rq_of(cfs_rq)->load.weight >= 2*se->load.weight) {
+        if (schedstat_enabled() && rq_of(cfs_rq)->load.weight >= 2*se->load.weight) {
                 se->statistics.slice_max = max(se->statistics.slice_max,
                         se->sum_exec_runtime - se->prev_sum_exec_runtime);
         }
@@ -3375,9 +3414,13 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
         /* throttle cfs_rqs exceeding runtime */
         check_cfs_rq_runtime(cfs_rq);
 
-        check_spread(cfs_rq, prev);
+        if (schedstat_enabled()) {
+                check_spread(cfs_rq, prev);
+                if (prev->on_rq)
+                        update_stats_wait_start(cfs_rq, prev);
+        }
+
         if (prev->on_rq) {
-                update_stats_wait_start(cfs_rq, prev);
                 /* Put 'current' back into the tree. */
                 __enqueue_entity(cfs_rq, prev);
                 /* in !on_rq case, update occurred at dequeue */