1 files changed, 227 insertions, 90 deletions

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 4e777b47eeda..8fe7ee81c552 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -21,6 +21,7 @@
  */
 
 #include <linux/latencytop.h>
+#include <linux/sched.h>
 
 /*
  * Targeted preemption latency for CPU-bound tasks:
@@ -35,12 +36,26 @@
  *  run vmstat and monitor the context-switches (cs) field)
  */
 unsigned int sysctl_sched_latency = 5000000ULL;
+unsigned int normalized_sysctl_sched_latency = 5000000ULL;
+
+/*
+ * The initial- and re-scaling of tunables is configurable
+ * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
+ *
+ * Options are:
+ * SCHED_TUNABLESCALING_NONE - unscaled, always *1
+ * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
+ * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
+ */
+enum sched_tunable_scaling sysctl_sched_tunable_scaling
+        = SCHED_TUNABLESCALING_LOG;
 
 /*
  * Minimal preemption granularity for CPU-bound tasks:
  * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
  */
 unsigned int sysctl_sched_min_granularity = 1000000ULL;
+unsigned int normalized_sysctl_sched_min_granularity = 1000000ULL;
 
 /*
  * is kept at sysctl_sched_latency / sysctl_sched_min_granularity
@@ -70,6 +85,7 @@ unsigned int __read_mostly sysctl_sched_compat_yield;
  * have immediate wakeup/sleep latencies.
  */
 unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
+unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL;
 
 const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
 
@@ -383,11 +399,12 @@ static struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
  */
 
 #ifdef CONFIG_SCHED_DEBUG
-int sched_nr_latency_handler(struct ctl_table *table, int write,
+int sched_proc_update_handler(struct ctl_table *table, int write,
                 void __user *buffer, size_t *lenp,
                 loff_t *ppos)
 {
         int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+        int factor = get_update_sysctl_factor();
 
         if (ret || !write)
                 return ret;
@@ -395,6 +412,14 @@ int sched_nr_latency_handler(struct ctl_table *table, int write,
         sched_nr_latency = DIV_ROUND_UP(sysctl_sched_latency,
                                         sysctl_sched_min_granularity);
 
+#define WRT_SYSCTL(name) \
+        (normalized_sysctl_##name = sysctl_##name / (factor))
+        WRT_SYSCTL(sched_min_granularity);
+        WRT_SYSCTL(sched_latency);
+        WRT_SYSCTL(sched_wakeup_granularity);
+        WRT_SYSCTL(sched_shares_ratelimit);
+#undef WRT_SYSCTL
+
         return 0;
 }
 #endif
@@ -485,6 +510,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
         curr->sum_exec_runtime += delta_exec;
         schedstat_add(cfs_rq, exec_clock, delta_exec);
         delta_exec_weighted = calc_delta_fair(delta_exec, curr);
+
         curr->vruntime += delta_exec_weighted;
         update_min_vruntime(cfs_rq);
 }
@@ -740,16 +766,26 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
         se->vruntime = vruntime;
 }
 
+#define ENQUEUE_WAKEUP  1
+#define ENQUEUE_MIGRATE 2
+
 static void
-enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup)
+enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 {
         /*
+         * Update the normalized vruntime before updating min_vruntime
+         * through callig update_curr().
+         */
+        if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATE))
+                se->vruntime += cfs_rq->min_vruntime;
+
+        /*
          * Update run-time statistics of the 'current'.
          */
         update_curr(cfs_rq);
         account_entity_enqueue(cfs_rq, se);
 
-        if (wakeup) {
+        if (flags & ENQUEUE_WAKEUP) {
                 place_entity(cfs_rq, se, 0);
                 enqueue_sleeper(cfs_rq, se);
         }
@@ -803,6 +839,14 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
                 __dequeue_entity(cfs_rq, se);
         account_entity_dequeue(cfs_rq, se);
         update_min_vruntime(cfs_rq);
+
+        /*
+         * Normalize the entity after updating the min_vruntime because the
+         * update can refer to the ->curr item and we need to reflect this
+         * movement in our normalized position.
+         */
+        if (!sleep)
+                se->vruntime -= cfs_rq->min_vruntime;
 }
 
 /*
@@ -822,6 +866,26 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr)
                  * re-elected due to buddy favours.
                  */
                 clear_buddies(cfs_rq, curr);
+                return;
+        }
+
+        /*
+         * Ensure that a task that missed wakeup preemption by a
+         * narrow margin doesn't have to wait for a full slice.
+         * This also mitigates buddy induced latencies under load.
+         */
+        if (!sched_feat(WAKEUP_PREEMPT))
+                return;
+
+        if (delta_exec < sysctl_sched_min_granularity)
+                return;
+
+        if (cfs_rq->nr_running > 1) {
+                struct sched_entity *se = __pick_next_entity(cfs_rq);
+                s64 delta = curr->vruntime - se->vruntime;
+
+                if (delta > ideal_runtime)
+                        resched_task(rq_of(cfs_rq)->curr);
         }
 }
 
@@ -861,12 +925,18 @@ wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
 static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq)
 {
         struct sched_entity *se = __pick_next_entity(cfs_rq);
+        struct sched_entity *left = se;
 
-        if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, se) < 1)
-                return cfs_rq->next;
+        if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1)
+                se = cfs_rq->next;
 
-        if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, se) < 1)
-                return cfs_rq->last;
+        /*
+         * Prefer last buddy, try to return the CPU to a preempted task.
+         */
+        if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1)
+                se = cfs_rq->last;
+
+        clear_buddies(cfs_rq, se);
 
         return se;
 }
@@ -987,13 +1057,19 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup)
 {
         struct cfs_rq *cfs_rq;
         struct sched_entity *se = &p->se;
+        int flags = 0;
+
+        if (wakeup)
+                flags |= ENQUEUE_WAKEUP;
+        if (p->state == TASK_WAKING)
+                flags |= ENQUEUE_MIGRATE;
 
         for_each_sched_entity(se) {
                 if (se->on_rq)
                         break;
                 cfs_rq = cfs_rq_of(se);
-                enqueue_entity(cfs_rq, se, wakeup);
-                wakeup = 1;
+                enqueue_entity(cfs_rq, se, flags);
+                flags = ENQUEUE_WAKEUP;
         }
 
         hrtick_update(rq);
@@ -1069,6 +1145,14 @@ static void yield_task_fair(struct rq *rq)
 
 #ifdef CONFIG_SMP
 
+static void task_waking_fair(struct rq *rq, struct task_struct *p)
+{
+        struct sched_entity *se = &p->se;
+        struct cfs_rq *cfs_rq = cfs_rq_of(se);
+
+        se->vruntime -= cfs_rq->min_vruntime;
+}
+
 #ifdef CONFIG_FAIR_GROUP_SCHED
 /*
  * effective_load() calculates the load change as seen from the root_task_group
@@ -1319,6 +1403,37 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
 }
 
 /*
+ * Try and locate an idle CPU in the sched_domain.
+ */
+static int
+select_idle_sibling(struct task_struct *p, struct sched_domain *sd, int target)
+{
+        int cpu = smp_processor_id();
+        int prev_cpu = task_cpu(p);
+        int i;
+
+        /*
+         * If this domain spans both cpu and prev_cpu (see the SD_WAKE_AFFINE
+         * test in select_task_rq_fair) and the prev_cpu is idle then that's
+         * always a better target than the current cpu.
+         */
+        if (target == cpu && !cpu_rq(prev_cpu)->cfs.nr_running)
+                return prev_cpu;
+
+        /*
+         * Otherwise, iterate the domain and find an elegible idle cpu.
+         */
+        for_each_cpu_and(i, sched_domain_span(sd), &p->cpus_allowed) {
+                if (!cpu_rq(i)->cfs.nr_running) {
+                        target = i;
+                        break;
+                }
+        }
+
+        return target;
+}
+
+/*
  * sched_balance_self: balance the current task (running on cpu) in domains
  * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
  * SD_BALANCE_EXEC.
@@ -1346,8 +1461,10 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag
                 new_cpu = prev_cpu;
         }
 
-        rcu_read_lock();
         for_each_domain(cpu, tmp) {
+                if (!(tmp->flags & SD_LOAD_BALANCE))
+                        continue;
+
                 /*
                  * If power savings logic is enabled for a domain, see if we
                  * are not overloaded, if so, don't balance wider.
@@ -1372,11 +1489,35 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag
                                 want_sd = 0;
                 }
 
-                if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
-                    cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
+                /*
+                 * While iterating the domains looking for a spanning
+                 * WAKE_AFFINE domain, adjust the affine target to any idle cpu
+                 * in cache sharing domains along the way.
+                 */
+                if (want_affine) {
+                        int target = -1;
 
-                        affine_sd = tmp;
-                        want_affine = 0;
+                        /*
+                         * If both cpu and prev_cpu are part of this domain,
+                         * cpu is a valid SD_WAKE_AFFINE target.
+                         */
+                        if (cpumask_test_cpu(prev_cpu, sched_domain_span(tmp)))
+                                target = cpu;
+
+                        /*
+                         * If there's an idle sibling in this domain, make that
+                         * the wake_affine target instead of the current cpu.
+                         */
+                        if (tmp->flags & SD_SHARE_PKG_RESOURCES)
+                                target = select_idle_sibling(p, tmp, target);
+
+                        if (target >= 0) {
+                                if (tmp->flags & SD_WAKE_AFFINE) {
+                                        affine_sd = tmp;
+                                        want_affine = 0;
+                                }
+                                cpu = target;
+                        }
                 }
 
                 if (!want_sd && !want_affine)
@@ -1403,10 +1544,8 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag
                         update_shares(tmp);
         }
 
-        if (affine_sd && wake_affine(affine_sd, p, sync)) {
-                new_cpu = cpu;
-                goto out;
-        }
+        if (affine_sd && wake_affine(affine_sd, p, sync))
+                return cpu;
 
         while (sd) {
                 int load_idx = sd->forkexec_idx;
@@ -1447,8 +1586,6 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag
                 /* while loop will break here if sd == NULL */
         }
 
-out:
-        rcu_read_unlock();
         return new_cpu;
 }
 #endif /* CONFIG_SMP */
@@ -1568,13 +1705,10 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
         struct sched_entity *se = &curr->se, *pse = &p->se;
         struct cfs_rq *cfs_rq = task_cfs_rq(curr);
         int sync = wake_flags & WF_SYNC;
+        int scale = cfs_rq->nr_running >= sched_nr_latency;
 
-        update_curr(cfs_rq);
-
-        if (unlikely(rt_prio(p->prio))) {
-                resched_task(curr);
-                return;
-        }
+        if (unlikely(rt_prio(p->prio)))
+                goto preempt;
 
         if (unlikely(p->sched_class != &fair_sched_class))
                 return;
@@ -1582,18 +1716,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
         if (unlikely(se == pse))
                 return;
 
-        /*
-         * Only set the backward buddy when the current task is still on the
-         * rq. This can happen when a wakeup gets interleaved with schedule on
-         * the ->pre_schedule() or idle_balance() point, either of which can
-         * drop the rq lock.
-         *
-         * Also, during early boot the idle thread is in the fair class, for
-         * obvious reasons its a bad idea to schedule back to the idle thread.
-         */
-        if (sched_feat(LAST_BUDDY) && likely(se->on_rq && curr != rq->idle))
-                set_last_buddy(se);
-        if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK))
+        if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK))
                 set_next_buddy(pse);
 
         /*
@@ -1611,36 +1734,44 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
                 return;
 
         /* Idle tasks are by definition preempted by everybody. */
-        if (unlikely(curr->policy == SCHED_IDLE)) {
-                resched_task(curr);
-                return;
-        }
+        if (unlikely(curr->policy == SCHED_IDLE))
+                goto preempt;
 
-        if ((sched_feat(WAKEUP_SYNC) && sync) ||
-            (sched_feat(WAKEUP_OVERLAP) &&
-             (se->avg_overlap < sysctl_sched_migration_cost &&
-              pse->avg_overlap < sysctl_sched_migration_cost))) {
-                resched_task(curr);
-                return;
-        }
+        if (sched_feat(WAKEUP_SYNC) && sync)
+                goto preempt;
 
-        if (sched_feat(WAKEUP_RUNNING)) {
-                if (pse->avg_running < se->avg_running) {
-                        set_next_buddy(pse);
-                        resched_task(curr);
-                        return;
-                }
-        }
+        if (sched_feat(WAKEUP_OVERLAP) &&
+                        se->avg_overlap < sysctl_sched_migration_cost &&
+                        pse->avg_overlap < sysctl_sched_migration_cost)
+                goto preempt;
 
         if (!sched_feat(WAKEUP_PREEMPT))
                 return;
 
+        update_curr(cfs_rq);
         find_matching_se(&se, &pse);
-
         BUG_ON(!pse);
-
         if (wakeup_preempt_entity(se, pse) == 1)
-                resched_task(curr);
+                goto preempt;
+
+        return;
+
+preempt:
+        resched_task(curr);
+        /*
+         * Only set the backward buddy when the current task is still
+         * on the rq. This can happen when a wakeup gets interleaved
+         * with schedule on the ->pre_schedule() or idle_balance()
+         * point, either of which can * drop the rq lock.
+         *
+         * Also, during early boot the idle thread is in the fair class,
+         * for obvious reasons its a bad idea to schedule back to it.
+         */
+        if (unlikely(!se->on_rq || curr == rq->idle))
+                return;
+
+        if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se))
+                set_last_buddy(se);
 }
 
 static struct task_struct *pick_next_task_fair(struct rq *rq)
@@ -1649,21 +1780,11 @@ static struct task_struct *pick_next_task_fair(struct rq *rq)
         struct cfs_rq *cfs_rq = &rq->cfs;
         struct sched_entity *se;
 
-        if (unlikely(!cfs_rq->nr_running))
+        if (!cfs_rq->nr_running)
                 return NULL;
 
         do {
                 se = pick_next_entity(cfs_rq);
-                /*
-                 * If se was a buddy, clear it so that it will have to earn
-                 * the favour again.
-                 *
-                 * If se was not a buddy, clear the buddies because neither
-                 * was elegible to run, let them earn it again.
-                 *
-                 * IOW. unconditionally clear buddies.
-                 */
-                __clear_buddies(cfs_rq, NULL);
                 set_next_entity(cfs_rq, se);
                 cfs_rq = group_cfs_rq(se);
         } while (cfs_rq);
@@ -1830,6 +1951,17 @@ move_one_task_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
 
         return 0;
 }
+
+static void rq_online_fair(struct rq *rq)
+{
+        update_sysctl();
+}
+
+static void rq_offline_fair(struct rq *rq)
+{
+        update_sysctl();
+}
+
 #endif /* CONFIG_SMP */
 
 /*
@@ -1847,28 +1979,30 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
 }
 
 /*
- * Share the fairness runtime between parent and child, thus the
- * total amount of pressure for CPU stays equal - new tasks
- * get a chance to run but frequent forkers are not allowed to
- * monopolize the CPU. Note: the parent runqueue is locked,
- * the child is not running yet.
+ * called on fork with the child task as argument from the parent's context
+ *  - child not yet on the tasklist
+ *  - preemption disabled
  */
-static void task_new_fair(struct rq *rq, struct task_struct *p)
+static void task_fork_fair(struct task_struct *p)
 {
-        struct cfs_rq *cfs_rq = task_cfs_rq(p);
+        struct cfs_rq *cfs_rq = task_cfs_rq(current);
         struct sched_entity *se = &p->se, *curr = cfs_rq->curr;
         int this_cpu = smp_processor_id();
+        struct rq *rq = this_rq();
+        unsigned long flags;
+
+        raw_spin_lock_irqsave(&rq->lock, flags);
 
-        sched_info_queued(p);
+        if (unlikely(task_cpu(p) != this_cpu))
+                __set_task_cpu(p, this_cpu);
 
         update_curr(cfs_rq);
+
         if (curr)
                 se->vruntime = curr->vruntime;
         place_entity(cfs_rq, se, 1);
 
-        /* 'curr' will be NULL if the child belongs to a different group */
-        if (sysctl_sched_child_runs_first && this_cpu == task_cpu(p) &&
-                        curr && entity_before(curr, se)) {
+        if (sysctl_sched_child_runs_first && curr && entity_before(curr, se)) {
                 /*
                  * Upon rescheduling, sched_class::put_prev_task() will place
                  * 'current' within the tree based on its new key value.
@@ -1877,7 +2011,9 @@ static void task_new_fair(struct rq *rq, struct task_struct *p)
                 resched_task(rq->curr);
         }
 
-        enqueue_task_fair(rq, p, 0);
+        se->vruntime -= cfs_rq->min_vruntime;
+
+        raw_spin_unlock_irqrestore(&rq->lock, flags);
 }
 
 /*
@@ -1930,30 +2066,27 @@ static void set_curr_task_fair(struct rq *rq)
 }
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-static void moved_group_fair(struct task_struct *p)
+static void moved_group_fair(struct task_struct *p, int on_rq)
 {
         struct cfs_rq *cfs_rq = task_cfs_rq(p);
 
         update_curr(cfs_rq);
-        place_entity(cfs_rq, &p->se, 1);
+        if (!on_rq)
+                place_entity(cfs_rq, &p->se, 1);
 }
 #endif
 
-unsigned int get_rr_interval_fair(struct task_struct *task)
+unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task)
 {
         struct sched_entity *se = &task->se;
-        unsigned long flags;
-        struct rq *rq;
         unsigned int rr_interval = 0;
 
         /*
          * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise
          * idle runqueue:
          */
-        rq = task_rq_lock(task, &flags);
         if (rq->cfs.load.weight)
                 rr_interval = NS_TO_JIFFIES(sched_slice(&rq->cfs, se));
-        task_rq_unlock(rq, &flags);
 
         return rr_interval;
 }
@@ -1977,11 +2110,15 @@ static const struct sched_class fair_sched_class = {
 
         .load_balance           = load_balance_fair,
         .move_one_task          = move_one_task_fair,
+        .rq_online              = rq_online_fair,
+        .rq_offline             = rq_offline_fair,
+
+        .task_waking            = task_waking_fair,
 #endif
 
         .set_curr_task          = set_curr_task_fair,
         .task_tick              = task_tick_fair,
-        .task_new               = task_new_fair,
+        .task_fork              = task_fork_fair,
 
         .prio_changed           = prio_changed_fair,
         .switched_to            = switched_to_fair,