sched/core: Rework rq->clock update skips

The original purpose of rq::skip_clock_update was to avoid 'costly' clock
updates for back to back wakeup-preempt pairs. The big problem with it
has always been that the rq variable is unaware of the context and
causes indiscrimiate clock skips.

Rework the entire thing and create a sense of context by only allowing
schedule() to skip clock updates. (XXX can we measure the cost of the
added store?)

By ensuring only schedule can ever skip an update, we guarantee we're
never more than 1 tick behind on the update.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: umgwanakikbuti@gmail.com
Link: http://lkml.kernel.org/r/20150105103554.432381549@infradead.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

1 files changed, 13 insertions, 2 deletions

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index bd2373273a9e..0870db23d79c 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -558,8 +558,6 @@ struct rq {
 #ifdef CONFIG_NO_HZ_FULL
         unsigned long last_sched_tick;
 #endif
-        int skip_clock_update;
-
         /* capture load from *all* tasks on this cpu: */
         struct load_weight load;
         unsigned long nr_load_updates;
@@ -588,6 +586,7 @@ struct rq {
         unsigned long next_balance;
         struct mm_struct *prev_mm;
 
+        unsigned int clock_skip_update;
         u64 clock;
         u64 clock_task;
 
@@ -704,6 +703,18 @@ static inline u64 rq_clock_task(struct rq *rq)
         return rq->clock_task;
 }
 
+#define RQCF_REQ_SKIP   0x01
+#define RQCF_ACT_SKIP   0x02
+
+static inline void rq_clock_skip_update(struct rq *rq, bool skip)
+{
+        lockdep_assert_held(&rq->lock);
+        if (skip)
+                rq->clock_skip_update |= RQCF_REQ_SKIP;
+        else
+                rq->clock_skip_update &= ~RQCF_REQ_SKIP;
+}
+
 #ifdef CONFIG_NUMA
 enum numa_topology_type {
         NUMA_DIRECT,