/* * kernel/sched.c * * Kernel scheduler and related syscalls * * Copyright (C) 1991-2002 Linus Torvalds * * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and * make semaphores SMP safe * 1998-11-19 Implemented schedule_timeout() and related stuff * by Andrea Arcangeli * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar: * hybrid priority-list and round-robin design with * an array-switch method of distributing timeslices * and per-CPU runqueues. Cleanups and useful suggestions * by Davide Libenzi, preemptible kernel bits by Robert Love. * 2003-09-03 Interactivity tuning by Con Kolivas. * 2004-04-02 Scheduler domains code by Nick Piggin * 2007-04-15 Work begun on replacing all interactivity tuning with a * fair scheduling design by Con Kolivas. * 2007-05-05 Load balancing (smp-nice) and other improvements * by Peter Williams * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Scheduler clock - returns current time in nanosec units. * This is default implementation. * Architectures and sub-architectures can override this. */ unsigned long long __attribute__((weak)) sched_clock(void) { return (unsigned long long)jiffies * (1000000000 / HZ); } /* * Convert user-nice values [ -20 ... 0 ... 19 ] * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], * and back. */ #define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20) #define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20) #define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio) /* * 'User priority' is the nice value converted to something we * can work with better when scaling various scheduler parameters, * it's a [ 0 ... 39 ] range. */ #define USER_PRIO(p) ((p)-MAX_RT_PRIO) #define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio) #define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) /* * Some helpers for converting nanosecond timing to jiffy resolution */ #define NS_TO_JIFFIES(TIME) ((TIME) / (1000000000 / HZ)) #define JIFFIES_TO_NS(TIME) ((TIME) * (1000000000 / HZ)) #define NICE_0_LOAD SCHED_LOAD_SCALE #define NICE_0_SHIFT SCHED_LOAD_SHIFT /* * These are the 'tuning knobs' of the scheduler: * * Minimum timeslice is 5 msecs (or 1 jiffy, whichever is larger), * default timeslice is 100 msecs, maximum timeslice is 800 msecs. * Timeslices get refilled after they expire. */ #define MIN_TIMESLICE max(5 * HZ / 1000, 1) #define DEF_TIMESLICE (100 * HZ / 1000) #ifdef CONFIG_SMP /* * Divide a load by a sched group cpu_power : (load / sg->__cpu_power) * Since cpu_power is a 'constant', we can use a reciprocal divide. */ static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load) { return reciprocal_divide(load, sg->reciprocal_cpu_power); } /* * Each time a sched group cpu_power is changed, * we must compute its reciprocal value */ static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val) { sg->__cpu_power += val; sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power); } #endif #define SCALE_PRIO(x, prio) \ max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO / 2), MIN_TIMESLICE) /* * static_prio_timeslice() scales user-nice values [ -20 ... 0 ... 19 ] * to time slice values: [800ms ... 100ms ... 5ms] */ static unsigned int static_prio_timeslice(int static_prio) { if (static_prio == NICE_TO_PRIO(19)) return 1; if (static_prio < NICE_TO_PRIO(0)) return SCALE_PRIO(DEF_TIMESLICE * 4, static_prio); else return SCALE_PRIO(DEF_TIMESLICE, static_prio); } static inline int rt_policy(int policy) { if (unlikely(policy == SCHED_FIFO) || unlikely(policy == SCHED_RR)) return 1; return 0; } static inline int task_has_rt_policy(struct task_struct *p) { return rt_policy(p->policy); } /* * This is the priority-queue data structure of the RT scheduling class: */ struct rt_prio_array { DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */ struct list_head queue[MAX_RT_PRIO]; }; struct load_stat { struct load_weight load; u64 load_update_start, load_update_last; unsigned long delta_fair, delta_exec, delta_stat; }; /* CFS-related fields in a runqueue */ struct cfs_rq { struct load_weight load; unsigned long nr_running; s64 fair_clock; u64 exec_clock; s64 wait_runtime; u64 sleeper_bonus; unsigned long wait_runtime_overruns, wait_runtime_underruns; struct rb_root tasks_timeline; struct rb_node *rb_leftmost; struct rb_node *rb_load_balance_curr; #ifdef CONFIG_FAIR_GROUP_SCHED /* 'curr' points to currently running entity on this cfs_rq. * It is set to NULL otherwise (i.e when none are currently running). */ struct sched_entity *curr; struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */ /* leaf cfs_rqs are those that hold tasks (lowest schedulable entity in * a hierarchy). Non-leaf lrqs hold other higher schedulable entities * (like users, containers etc.) * * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This * list is used during load balance. */ struct list_head leaf_cfs_rq_list; /* Better name : task_cfs_rq_list? */ #endif }; /* Real-Time classes' related field in a runqueue: */ struct rt_rq { struct rt_prio_array active; int rt_load_balance_idx; struct list_head *rt_load_balance_head, *rt_load_balance_curr; }; /* * This is the main, per-CPU runqueue data structure. * * Locking rule: those places that want to lock multiple runqueues * (such as the load balancing or the thread migration code), lock * acquire operations must be ordered by ascending &runqueue. */ struct rq { spinlock_t lock; /* runqueue lock */ /* * nr_running and cpu_load should be in the same cacheline because * remote CPUs use both these fields when doing load calculation. */ unsigned long nr_running; #define CPU_LOAD_IDX_MAX 5 unsigned long cpu_load[CPU_LOAD_IDX_MAX]; unsigned char idle_at_tick; #ifdef CONFIG_NO_HZ unsigned char in_nohz_recently; #endif struct load_stat ls; /* capture load from *all* tasks on this cpu */ unsigned long nr_load_updates; u64 nr_switches; struct cfs_rq cfs; #ifdef CONFIG_FAIR_GROUP_SCHED struct list_head leaf_cfs_rq_list; /* list of leaf cfs_rq on this cpu */ #endif struct rt_rq rt; /* * This is part of a global counter where only the total sum * over all CPUs matters. A task can increase this counter on * one CPU and if it got migrated afterwards it may decrease * it on another CPU. Always updated under the runqueue lock: */ unsigned long nr_uninterruptible; struct task_struct *curr, *idle; unsigned long next_balance; struct mm_struct *prev_mm; u64 clock, prev_clock_raw; s64 clock_max_delta; unsigned int clock_warps, clock_overflows; unsigned int clock_unstable_events; struct sched_class *load_balance_class; atomic_t nr_iowait; #ifdef CONFIG_SMP struct sched_domain *sd; /* For active balancing */ int active_balance; int push_cpu; int cpu; /* cpu of this runqueue */ struct task_struct *migration_thread; struct list_head migration_queue; #endif #ifdef CONFIG_SCHEDSTATS /* latency stats */ struct sched_info rq_sched_info; /* sys_sched_yield() stats */ unsigned long yld_exp_empty; unsigned long yld_act_empty; unsigned long yld_both_empty; unsigned long yld_cnt; /* schedule() stats */ unsigned long sched_switch; unsigned long sched_cnt; unsigned long sched_goidle; /* try_to_wake_up() stats */ unsigned long ttwu_cnt; unsigned long ttwu_local; #endif struct lock_class_key rq_lock_key; }; static DEFINE_PER_CPU(struct rq, runqueues) ____cacheline_aligned_in_smp; static DEFINE_MUTEX(sched_hotcpu_mutex); static inline void check_preempt_curr(struct rq *rq, struct task_struct *p) { rq->curr->sched_class->check_preempt_curr(rq, p); } static inline int cpu_of(struct rq *rq) { #ifdef CONFIG_SMP return rq->cpu; #else return 0; #endif } /* * Per-runqueue clock, as finegrained as the platform can give us: */ static unsigned long long __rq_clock(struct rq *rq) { u64 prev_raw = rq->prev_clock_raw; u64 now = sched_clock(); s64 delta = now - prev_raw; u64 clock = rq->clock; /* * Protect against sched_clock() occasionally going backwards: */ if (unlikely(delta < 0)) { clock++; rq->clock_warps++; } else { /* * Catch too large forward jumps too: */ if (unlikely(delta > 2*TICK_NSEC)) { clock++; rq->clock_overflows++; } else { if (unlikely(delta > rq->clock_max_delta)) rq->clock_max_delta = delta; clock += delta; } } rq->prev_clock_raw = now; rq->clock = clock; return clock; } static inline unsigned long long rq_clock(struct rq *rq) { int this_cpu = smp_processor_id(); if (this_cpu == cpu_of(rq)) return __rq_clock(rq); return rq->clock; } /* * The domain tree (rq->sd) is protected by RCU's quiescent state transition. * See detach_destroy_domains: synchronize_sched for details. * * The domain tree of any CPU may only be accessed from within * preempt-disabled sections. */ #define for_each_domain(cpu, __sd) \ for (__sd = rcu_dereference(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent) #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) #define this_rq() (&__get_cpu_var(runqueues)) #define task_rq(p) cpu_rq(task_cpu(p)) #define cpu_curr(cpu) (cpu_rq(cpu)->curr) #ifdef CONFIG_FAIR_GROUP_SCHED /* Change a task's ->cfs_rq if it moves across CPUs */ static inline void set_task_cfs_rq(struct task_struct *p) { p->se.cfs_rq = &task_rq(p)->cfs; } #else static inline void set_task_cfs_rq(struct task_struct *p) { } #endif #ifndef prepare_arch_switch # define prepare_arch_switch(next) do { } while (0) #endif #ifndef finish_arch_switch # define finish_arch_switch(prev) do { } while (0) #endif #ifndef __ARCH_WANT_UNLOCKED_CTXSW static inline int task_running(struct rq *rq, struct task_struct *p) { return rq->curr == p; } static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) { } static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) { #ifdef CONFIG_DEBUG_SPINLOCK /* this is a valid case when another task releases the spinlock */ rq->lock.owner = current; #endif /* * If we are tracking spinlock dependencies then we have to * fix up the runqueue lock - which gets 'carried over' from * prev into current: */ spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_); spin_unlock_irq(&rq->lock); } #else /* __ARCH_WANT_UNLOCKED_CTXSW */ static inline int task_running(struct rq *rq, struct task_struct *p) { #ifdef CONFIG_SMP return p->oncpu; #else return rq->curr == p; #endif } static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) { #ifdef CONFIG_SMP /* * We can optimise this out completely for !SMP, because the * SMP rebalancing from interrupt is the only thing that cares * here. */ next->oncpu = 1; #endif #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW spin_unlock_irq(&rq->lock); #else spin_unlock(&rq->lock); #endif } static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) { #ifdef CONFIG_SMP /* * After ->oncpu is cleared, the task can be moved to a different CPU. * We must ensure this doesn't happen until the switch is completely * finished. */ smp_wmb(); prev->oncpu = 0; #endif #ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW local_irq_enable(); #endif } #endif /* __ARCH_WANT_UNLOCKED_CTXSW */ /* * __task_rq_lock - lock the runqueue a given task resides on. * Must be called interrupts disabled. */ static inline struct rq *__task_rq_lock(struct task_struct *p) __acquires(rq->lock) { struct rq *rq; repeat_lock_task: rq = task_rq(p); spin_lock(&rq->lock); if (unlikely(rq != task_rq(p))) { spin_unlock(&rq->lock); goto repeat_lock_task; } return rq; } /* * task_rq_lock - lock the runqueue a given task resides on and disable * interrupts. Note the ordering: we can safely lookup the task_rq without * explicitly disabling preemption. */ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags) __acquires(rq->lock) { struct rq *rq; repeat_lock_task: local_irq_save(*flags); rq = task_rq(p); spin_lock(&rq->lock); if (unlikely(rq != task_rq(p))) { spin_unlock_irqrestore(&rq->lock, *flags); goto repeat_lock_task; } return rq; } static inline void __task_rq_unlock(struct rq *rq) __releases(rq->lock) { spin_unlock(&rq->lock); } static inline void task_rq_unlock(struct rq *rq, unsigned long *flags) __releases(rq->lock) { spin_unlock_irqrestore(&rq->lock, *flags); } /* * this_rq_lock - lock this runqueue and disable interrupts. */ static inline struct rq *this_rq_lock(void) __acquires(rq->lock) { struct rq *rq; local_irq_disable(); rq = this_rq(); spin_lock(&rq->lock); return rq; } /* * CPU frequency is/was unstable - start new by setting prev_clock_raw: */ void sched_clock_unstable_event(void) { unsigned long flags; struct rq *rq; rq = task_rq_lock(current, &flags); rq->prev_clock_raw = sched_clock(); rq->clock_unstable_events++; task_rq_unlock(rq, &flags); } /* * resched_task - mark a task 'to be rescheduled now'. * * On UP this means the setting of the need_resched flag, on SMP it * might also involve a cross-CPU call to trigger the scheduler on * the target CPU. */ #ifdef CONFIG_SMP #ifndef tsk_is_polling #define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG) #endif static void resched_task(struct task_struct *p) { int cpu; assert_spin_locked(&task_rq(p)->lock); if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED))) return; set_tsk_thread_flag(p, TIF_NEED_RESCHED); cpu = task_cpu(p); if (cpu == smp_processor_id()) return; /* NEED_RESCHED must be visible before we test polling */ smp_mb(); if (!tsk_is_polling(p)) smp_send_reschedule(cpu); } static void resched_cpu(int cpu) { struct rq *rq = cpu_rq(cpu); unsigned long flags; if (!spin_trylock_irqsave(&rq->lock, flags)) return; resched_task(cpu_curr(cpu)); spin_unlock_irqrestore(&rq->lock, flags); } #else static inline void resched_task(struct task_struct *p) { assert_spin_locked(&task_rq(p)->lock); set_tsk_need_resched(p); } #endif static u64 div64_likely32(u64 divident, unsigned long divisor) { #if BITS_PER_LONG == 32 if (likely(divident <= 0xffffffffULL)) return (u32)divident / divisor; do_div(divident, divisor); return divident; #else return divident / divisor; #endif } #if BITS_PER_LONG == 32 # define WMULT_CONST (~0UL) #else # define WMULT_CONST (1UL << 32) #endif #define WMULT_SHIFT 32 static inline unsigned long calc_delta_mine(unsigned long delta_exec, unsigned long weight, struct load_weight *lw) { u64 tmp; if (unlikely(!lw->inv_weight)) lw->inv_weight = WMULT_CONST / lw->weight; tmp = (u64)delta_exec * weight; /* * Check whether we'd overflow the 64-bit multiplication: */ if (unlikely(tmp > WMULT_CONST)) { tmp = ((tmp >> WMULT_SHIFT/2) * lw->inv_weight) >> (WMULT_SHIFT/2); } else { tmp = (tmp * lw->inv_weight) >> WMULT_SHIFT; } return (unsigned long)min(tmp, (u64)sysctl_sched_runtime_limit); } static inline unsigned long calc_delta_fair(unsigned long delta_exec, struct load_weight *lw) { return calc_delta_mine(delta_exec, NICE_0_LOAD, lw); } static void update_load_add(struct load_weight *lw, unsigned long inc) { lw->weight += inc; lw->inv_weight = 0; } static void update_load_sub(struct load_weight *lw, unsigned long dec) { lw->weight -= dec; lw->inv_weight = 0; } static void __update_curr_load(struct rq *rq, struct load_stat *ls) { if (rq->curr != rq->idle && ls->load.weight) { ls->delta_exec += ls->delta_stat; ls->delta_fair += calc_delta_fair(ls->delta_stat, &ls->load); ls->delta_stat = 0; } } /* * Update delta_exec, delta_fair fields for rq. * * delta_fair clock advances at a rate inversely proportional to * total load (rq->ls.load.weight) on the runqueue, while * delta_exec advances at the same rate as wall-clock (provided * cpu is not idle). * * delta_exec / delta_fair is a measure of the (smoothened) load on this * runqueue over any given interval. This (smoothened) load is used * during load balance. * * This function is called /before/ updating rq->ls.load * and when switching tasks. */ static void update_curr_load(struct rq *rq, u64 now) { struct load_stat *ls = &rq->ls; u64 start; start = ls->load_update_start; ls->load_update_start = now; ls->delta_stat += now - start; /* * Stagger updates to ls->delta_fair. Very frequent updates * can be expensive. */ if (ls->delta_stat >= sysctl_sched_stat_granularity) __update_curr_load(rq, ls); } /* * To aid in avoiding the subversion of "niceness" due to uneven distribution * of tasks with abnormal "nice" values across CPUs the contribution that * each task makes to its run queue's load is weighted according to its * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a * scaled version of the new time slice allocation that they receive on time * slice expiry etc. */ /* * Assume: static_prio_timeslice(NICE_TO_PRIO(0)) == DEF_TIMESLICE * If static_prio_timeslice() is ever changed to break this assumption then * this code will need modification */ #define TIME_SLICE_NICE_ZERO DEF_TIMESLICE #define load_weight(lp) \ (((lp) * SCHED_LOAD_SCALE) / TIME_SLICE_NICE_ZERO) #define PRIO_TO_LOAD_WEIGHT(prio) \ load_weight(static_prio_timeslice(prio)) #define RTPRIO_TO_LOAD_WEIGHT(rp) \ (PRIO_TO_LOAD_WEIGHT(MAX_RT_PRIO) + load_weight(rp)) #define WEIGHT_IDLEPRIO 2 #define WMULT_IDLEPRIO (1 << 31) /* * Nice levels are multiplicative, with a gentle 10% change for every * nice level changed. I.e. when a CPU-bound task goes from nice 0 to * nice 1, it will get ~10% less CPU time than another CPU-bound task * that remained on nice 0. * * The "10% effect" is relative and cumulative: from _any_ nice level, * if you go up 1 level, it's -10% CPU usage, if you go down 1 level * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. * If a task goes up by ~10% and another task goes down by ~10% then * the relative distance between them is ~25%.) */ static const int prio_to_weight[40] = { /* -20 */ 88818, 71054, 56843, 45475, 36380, 29104, 23283, 18626, 14901, 11921, /* -10 */ 9537, 7629, 6103, 4883, 3906, 3125, 2500, 2000, 1600, 1280, /* 0 */ NICE_0_LOAD /* 1024 */, /* 1 */ 819, 655, 524, 419, 336, 268, 215, 172, 137, /* 10 */ 110, 87, 70, 56, 45, 36, 29, 23, 18, 15, }; /* * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated. * * In cases where the weight does not change often, we can use the * precalculated inverse to speed up arithmetics by turning divisions * into multiplications: */ static const u32 prio_to_wmult[40] = { /* -20 */ 48356, 60446, 75558, 94446, 118058, /* -15 */ 147573, 184467, 230589, 288233, 360285, /* -10 */ 450347, 562979, 703746, 879575, 1099582, /* -5 */ 1374389, 1717986, 2147483, 2684354, 3355443, /* 0 */ 4194304, 5244160, 6557201, 8196502, 10250518, /* 5 */ 12782640, 16025997, 19976592, 24970740, 31350126, /* 10 */ 39045157, 49367440, 61356675, 76695844, 95443717, /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, }; static inline void inc_load(struct rq *rq, const struct task_struct *p, u64 now) { update_curr_load(rq, now); update_load_add(&rq->ls.load, p->se.load.weight); } static inline void dec_load(struct rq *rq, const struct task_struct *p, u64 now) { update_curr_load(rq, now); update_load_sub(&rq->ls.load, p->se.load.weight); } static inline void inc_nr_running(struct task_struct *p, struct rq *rq, u64 now) { rq->nr_running++; inc_load(rq, p, now); } static inline void dec_nr_running(struct task_struct *p, struct rq *rq, u64 now) { rq->nr_running--; dec_load(rq, p, now); } static void activate_task(struct rq *rq, struct task_struct *p, int wakeup); /* * runqueue iterator, to support SMP load-balancing between different * scheduling classes, without having to expose their internal data * structures to the load-balancing proper: */ struct rq_iterator { void *arg; struct task_struct *(*start)(void *); struct task_struct *(*next)(void *); }; static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, unsigned long max_nr_move, unsigned long max_load_move, struct sched_domain *sd, enum cpu_idle_type idle, int *all_pinned, unsigned long *load_moved, int this_best_prio, int best_prio, int best_prio_seen, struct rq_iterator *iterator); #include "sched_stats.h" #include "sched_rt.c" #include "sched_fair.c" #include "sched_idletask.c" #ifdef CONFIG_SCHED_DEBUG # include "sched_debug.c" #endif #define sched_class_highest (&rt_sched_class) static void set_load_weight(struct task_struct *p) { task_rq(p)->cfs.wait_runtime -= p->se.wait_runtime; p->se.wait_runtime = 0; if (task_has_rt_policy(p)) { p->se.load.weight = prio_to_weight[0] * 2; p->se.load.inv_weight = prio_to_wmult[0] >> 1; return; } /* * SCHED_IDLE tasks get minimal weight: */ if (p->policy == SCHED_IDLE) { p->se.load.weight = WEIGHT_IDLEPRIO; p->se.load.inv_weight = WMULT_IDLEPRIO; return; } p->se.load.weight = prio_to_weight[p->static_prio - MAX_RT_PRIO]; p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO]; } static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup, u64 now) { sched_info_queued(p); p->sched_class->enqueue_task(rq, p, wakeup, now); p->se.on_rq = 1; } static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep, u64 now) { p->sched_class->dequeue_task(rq, p, sleep, now); p->se.on_rq = 0; } /* * __normal_prio - return the priority that is based on the static prio */ static inline int __normal_prio(struct task_struct *p) { return p->static_prio; } /* * Calculate the expected normal priority: i.e. priority * without taking RT-inheritance into account. Might be * boosted by interactivity modifiers. Changes upon fork, * setprio syscalls, and whenever the interactivity * estimator recalculates. */ static inline int normal_prio(struct task_struct *p) { int prio; if (task_has_rt_policy(p)) prio = MAX_RT_PRIO-1 - p->rt_priority; else prio = __normal_prio(p); return prio; } /* * Calculate the current priority, i.e. the priority * taken into account by the scheduler. This value might * be boosted by RT tasks, or might be boosted by * interactivity modifiers. Will be RT if the task got * RT-boosted. If not then it returns p->normal_prio. */ static int effective_prio(struct task_struct *p) { p->normal_prio = normal_prio(p); /* * If we are RT tasks or we were boosted to RT priority, * keep the priority unchanged. Otherwise, update priority * to the normal priority: */ if (!rt_prio(p->prio)) return p->normal_prio; return p->prio; } /* * activate_task - move a task to the runqueue. */ static void activate_task(struct rq *rq, struct task_struct *p, int wakeup) { u64 now = rq_clock(rq); if (p->state == TASK_UNINTERRUPTIBLE) rq->nr_uninterruptible--; enqueue_task(rq, p, wakeup, now); inc_nr_running(p, rq, now); } /* * activate_idle_task - move idle task to the _front_ of runqueue. */ static inline void activate_idle_task(struct task_struct *p, struct rq *rq) { u64 now = rq_clock(rq); if (p->state == TASK_UNINTERRUPTIBLE) rq->nr_uninterruptible--; enqueue_task(rq, p, 0, now); inc_nr_running(p, rq, now); } /* * deactivate_task - remove a task from the runqueue. */ static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep) { u64 now = rq_clock(rq); if (p->state == TASK_UNINTERRUPTIBLE) rq->nr_uninterruptible++; dequeue_task(rq, p, sleep, now); dec_nr_running(p, rq, now); } /** * task_curr - is this task currently executing on a CPU? * @p: the task in question. */ inline int task_curr(const struct task_struct *p) { return cpu_curr(task_cpu(p)) == p; } /* Used instead of source_load when we know the type == 0 */ unsigned long weighted_cpuload(const int cpu) { return cpu_rq(cpu)->ls.load.weight; } static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) { #ifdef CONFIG_SMP task_thread_info(p)->cpu = cpu; set_task_cfs_rq(p); #endif } #ifdef CONFIG_SMP void set_task_cpu(struct task_struct *p, unsigned int new_cpu) { int old_cpu = task_cpu(p); struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu); u64 clock_offset, fair_clock_offset; clock_offset = old_rq->clock - new_rq->clock; fair_clock_offset = old_rq->cfs.fair_clock - new_rq->cfs.fair_clock; if (p->se.wait_start) p->se.wait_start -= clock_offset; if (p->se.wait_start_fair) p->se.wait_start_fair -= fair_clock_offset; if (p->se.sleep_start) p->se.sleep_start -= clock_offset; if (p->se.block_start) p->se.block_start -= clock_offset; if (p->se.sleep_start_fair) p->se.sleep_start_fair -= fair_clock_offset; __set_task_cpu(p, new_cpu); } struct migration_req { struct list_head list; struct task_struct *task; int dest_cpu; struct completion done; }; /* * The task's runqueue lock must be held. * Returns true if you have to wait for migration thread. */ static int migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req) { struct rq *rq = task_rq(p); /* * If the task is not on a runqueue (and not running), then * it is sufficient to simply update the task's cpu field. */ if (!p->se.on_rq && !task_running(rq, p)) { set_task_cpu(p, dest_cpu); return 0; } init_completion(&req->done); req->task = p; req->dest_cpu = dest_cpu; list_add(&req->list, &rq->migration_queue); return 1; } /* * wait_task_inactive - wait for a thread to unschedule. * * The caller must ensure that the task *will* unschedule sometime soon, * else this function might spin for a *long* time. This function can't * be called with interrupts off, or it may introduce deadlock with * smp_call_function() if an IPI is sent by the same process we are * waiting to become inactive. */ void wait_task_inactive(struct task_struct *p) { unsigned long flags; int running, on_rq; struct rq *rq; repeat: /* * We do the initial early heuristics without holding * any task-queue locks at all. We'll only try to get * the runqueue lock when things look like they will * work out! */ rq = task_rq(p); /* * If the task is actively running on another CPU * still, just relax and busy-wait without holding * any locks. * * NOTE! Since we don't hold any locks, it's not * even sure that "rq" stays as the right runqueue! * But we don't care, since "task_running()" will * return false if the runqueue has changed and p * is actually now running somewhere else! */ while (task_running(rq, p)) cpu_relax(); /* * Ok, time to look more closely! We need the rq * lock now, to be *sure*. If we're wrong, we'll * just go back and repeat. */ rq = task_rq_lock(p, &flags); running = task_running(rq, p); on_rq = p->se.on_rq; task_rq_unlock(rq, &flags); /* * Was it really running after all now that we * checked with the proper locks actually held? * * Oops. Go back and try again.. */ if (unlikely(running)) { cpu_relax(); goto repeat; } /* * It's not enough that it's not actively running, * it must be off the runqueue _entirely_, and not * preempted! * * So if it wa still runnable (but just not actively * running right now), it's preempted, and we should * yield - it could be a while. */ if (unlikely(on_rq)) { yield(); goto repeat; } /* * Ahh, all good. It wasn't running, and it wasn't * runnable, which means that it will never become * running in the future either. We're all done! */ } /*** * kick_process - kick a running thread to enter/exit the kernel * @p: the to-be-kicked thread * * Cause a process which is running on another CPU to enter * kernel-mode, without any delay. (to get signals handled.) * * NOTE: this function doesnt have to take the runqueue lock, * because all it wants to ensure is that the remote task enters * the kernel. If the IPI races and the task has been migrated * to another CPU then no harm is done and the purpose has been * achieved as well. */ void kick_process(struct task_struct *p) { int cpu; preempt_disable(); cpu = task_cpu(p); if ((cpu != smp_processor_id()) && task_curr(p)) smp_send_reschedule(cpu); preempt_enable(); } /* * Return a low guess at the load of a migration-source cpu weighted * according to the scheduling class and "nice" value. * * We want to under-estimate the load of migration sources, to * balance conservatively. */ static inline unsigned long source_load(int cpu, int type) { struct rq *rq = cpu_rq(cpu); unsigned long total = weighted_cpuload(cpu); if (type == 0) return total; return min(rq->cpu_load[type-1], total); } /* * Return a high guess at the load of a migration-target cpu weighted * according to the scheduling class and "nice" value. */ static inline unsigned long target_load(int cpu, int type) { struct rq *rq = cpu_rq(cpu); unsigned long total = weighted_cpuload(cpu); if (type == 0) return total; return max(rq->cpu_load[type-1], total); } /* * Return the average load per task on the cpu's run queue */ static inline unsigned long cpu_avg_load_per_task(int cpu) { struct rq *rq = cpu_rq(cpu); unsigned long total = weighted_cpuload(cpu); unsigned long n = rq->nr_running; return n ? total / n : SCHED_LOAD_SCALE; } /* * find_idlest_group finds and returns the least busy CPU group within the * domain. */ static struct sched_group * find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) { struct sched_group *idlest = NULL, *this = NULL, *group = sd->groups; unsigned long min_load = ULONG_MAX, this_load = 0; int load_idx = sd->forkexec_idx; int imbalance = 100 + (sd->imbalance_pct-100)/2; do { unsigned long load, avg_load; int local_group; int i; /* Skip over this group if it has no CPUs allowed */ if (!cpus_intersects(group->cpumask, p->cpus_allowed)) goto nextgroup; local_group = cpu_isset(this_cpu, group->cpumask); /* Tally up the load of all CPUs in the group */ avg_load = 0; for_each_cpu_mask(i, group->cpumask) { /* Bias balancing toward cpus of our domain */ if (local_group) load = source_load(i, load_idx); else load = target_load(i, load_idx); avg_load += load; } /* Adjust by relative CPU power of the group */ avg_load = sg_div_cpu_power(group, avg_load * SCHED_LOAD_SCALE); if (local_group) { this_load = avg_load; this = group; } else if (avg_load < min_load) { min_load = avg_load; idlest = group; } nextgroup: group = group->next; } while (group != sd->groups); if (!idlest || 100*this_load < imbalance*min_load) return NULL; return idlest; } /* * find_idlest_cpu - find the idlest cpu among the cpus in group. */ static int find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) { cpumask_t tmp; unsigned long load, min_load = ULONG_MAX; int idlest = -1; int i; /* Traverse only the allowed CPUs */ cpus_and(tmp, group->cpumask, p->cpus_allowed); for_each_cpu_mask(i, tmp) { load = weighted_cpuload(i); if (load < min_load || (load == min_load && i == this_cpu)) { min_load = load; idlest = i; } } return idlest; } /* * sched_balance_self: balance the current task (running on cpu) in domains * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and * SD_BALANCE_EXEC. * * Balance, ie. select the least loaded group. * * Returns the target CPU number, or the same CPU if no balancing is needed. * * preempt must be disabled. */ static int sched_balance_self(int cpu, int flag) { struct task_struct *t = current; struct sched_domain *tmp, *sd = NULL; for_each_domain(cpu, tmp) { /* * If power savings logic is enabled for a domain, stop there. */ if (tmp->flags & SD_POWERSAVINGS_BALANCE) break; if (tmp->flags & flag) sd = tmp; } while (sd) { cpumask_t span; struct sched_group *group; int new_cpu, weight; if (!(sd->flags & flag)) { sd = sd->child; continue; } span = sd->span; group = find_idlest_group(sd, t, cpu); if (!group) { sd = sd->child; continue; } new_cpu = find_idlest_cpu(group, t, cpu); if (new_cpu == -1 || new_cpu == cpu) { /* Now try balancing at a lower domain level of cpu */ sd = sd->child; continue; } /* Now try balancing at a lower domain level of new_cpu */ cpu = new_cpu; sd = NULL; weight = cpus_weight(span); for_each_domain(cpu, tmp) { if (weight <= cpus_weight(tmp->span)) break; if (tmp->flags & flag) sd = tmp; } /* while loop will break here if sd == NULL */ } return cpu; } #endif /* CONFIG_SMP */ /* * wake_idle() will wake a task on an idle cpu if task->cpu is * not idle and an idle cpu is available. The span of cpus to * search starts with cpus closest then further out as needed, * so we always favor a closer, idle cpu. * * Returns the CPU we should wake onto. */ #if defined(ARCH_HAS_SCHED_WAKE_IDLE) static int wake_idle(int cpu, struct task_struct *p) { cpumask_t tmp; struct sched_domain *sd; int i; /* * If it is idle, then it is the best cpu to run this task. * * This cpu is also the best, if it has more than one task already. * Siblings must be also busy(in most cases) as they didn't already * pickup the extra load from this cpu and hence we need not check * sibling runqueue info. This will avoid the checks and cache miss * penalities associated with that. */ if (idle_cpu(cpu) || cpu_rq(cpu)->nr_running > 1) return cpu; for_each_domain(cpu, sd) { if (sd->flags & SD_WAKE_IDLE) { cpus_and(tmp, sd->span, p->cpus_allowed); for_each_cpu_mask(i, tmp) { if (idle_cpu(i)) return i; } } else { break; } } return cpu; } #else static inline int wake_idle(int cpu, struct task_struct *p) { return cpu; } #endif /*** * try_to_wake_up - wake up a thread * @p: the to-be-woken-up thread * @state: the mask of task states that can be woken * @sync: do a synchronous wakeup? * * Put it on the run-queue if it's not already there. The "current" * thread is always on the run-queue (except when the actual * re-schedule is in progress), and as such you're allowed to do * the simpler "current->state = TASK_RUNNING" to mark yourself * runnable without the overhead of this. * * returns failure only if the task is already active. */ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync) { int cpu, this_cpu, success = 0; unsigned long flags; long old_state; struct rq *rq; #ifdef CONFIG_SMP struct sched_domain *sd, *this_sd = NULL; unsigned long load, this_load; int new_cpu; #endif rq = task_rq_lock(p, &flags); old_state = p->state; if (!(old_state & state)) goto out; if (p->se.on_rq) goto out_running; cpu = task_cpu(p); this_cpu = smp_processor_id(); #ifdef CONFIG_SMP if (unlikely(task_running(rq, p))) goto out_activate; new_cpu = cpu; schedstat_inc(rq, ttwu_cnt); if (cpu == this_cpu) { schedstat_inc(rq, ttwu_local); goto out_set_cpu; } for_each_domain(this_cpu, sd) { if (cpu_isset(cpu, sd->span)) { schedstat_inc(sd, ttwu_wake_remote); this_sd = sd; break; } } if (unlikely(!cpu_isset(this_cpu, p->cpus_allowed))) goto out_set_cpu; /* * Check for affine wakeup and passive balancing possibilities. */ if (this_sd) { int idx = this_sd->wake_idx; unsigned int imbalance; imbalance = 100 + (this_sd->imbalance_pct - 100) / 2; load = source_load(cpu, idx); this_load = target_load(this_cpu, idx); new_cpu = this_cpu; /* Wake to this CPU if we can */ if (this_sd->flags & SD_WAKE_AFFINE) { unsigned long tl = this_load; unsigned long tl_per_task; tl_per_task = cpu_avg_load_per_task(this_cpu); /* * If sync wakeup then subtract the (maximum possible) * effect of the currently running task from the load * of the current CPU: */ if (sync) tl -= current->se.load.weight; if ((tl <= load && tl + target_load(cpu, idx) <= tl_per_task) || 100*(tl + p->se.load.weight) <= imbalance*load) { /* * This domain has SD_WAKE_AFFINE and * p is cache cold in this domain, and * there is no bad imbalance. */ schedstat_inc(this_sd, ttwu_move_affine); goto out_set_cpu; } } /* * Start passive balancing when half the imbalance_pct * limit is reached. */ if (this_sd->flags & SD_WAKE_BALANCE) { if (imbalance*this_load <= 100*load) { schedstat_inc(this_sd, ttwu_move_balance); goto out_set_cpu; } } } new_cpu = cpu; /* Could not wake to this_cpu. Wake to cpu instead */ out_set_cpu: new_cpu = wake_idle(new_cpu, p); if (new_cpu != cpu) { set_task_cpu(p, new_cpu); task_rq_unlock(rq, &flags); /* might preempt at this point */ rq = task_rq_lock(p, &flags); old_state = p->state; if (!(old_state & state)) goto out; if (p->se.on_rq) goto out_running; this_cpu = smp_processor_id(); cpu = task_cpu(p); } out_activate: #endif /* CONFIG_SMP */ activate_task(rq, p, 1); /* * Sync wakeups (i.e. those types of wakeups where the waker * has indicated that it will leave the CPU in short order) * don't trigger a preemption, if the woken up task will run on * this cpu. (in this case the 'I will reschedule' promise of * the waker guarantees that the freshly woken up task is going * to be considered on this CPU.) */ if (!sync || cpu != this_cpu) check_preempt_curr(rq, p); success = 1; out_running: p->state = TASK_RUNNING; out: task_rq_unlock(rq, &flags); return success; } int fastcall wake_up_process(struct task_struct *p) { return try_to_wake_up(p, TASK_STOPPED | TASK_TRACED | TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0); } EXPORT_SYMBOL(wake_up_process); int fastcall wake_up_state(struct task_struct *p, unsigned int state) { return try_to_wake_up(p, state, 0); } /* * Perform scheduler related setup for a newly forked process p. * p is forked by current. * * __sched_fork() is basic setup used by init_idle() too: */ static void __sched_fork(struct task_struct *p) { p->se.wait_start_fair = 0; p->se.wait_start = 0; p->se.exec_start = 0; p->se.sum_exec_runtime = 0; p->se.delta_exec = 0; p->se.delta_fair_run = 0; p->se.delta_fair_sleep = 0; p->se.wait_runtime = 0; p->se.sum_wait_runtime = 0; p->se.sum_sleep_runtime = 0; p->se.sleep_start = 0; p->se.sleep_start_fair = 0; p->se.block_start = 0; p->se.sleep_max = 0; p->se.block_max = 0; p->se.exec_max = 0; p->se.wait_max = 0; p->se.wait_runtime_overruns = 0; p->se.wait_runtime_underruns = 0; INIT_LIST_HEAD(&p->run_list); p->se.on_rq = 0; /* * We mark the process as running here, but have not actually * inserted it onto the runqueue yet. This guarantees that * nobody will actually run it, and a signal or other external * event cannot wake it up and insert it on the runqueue either. */ p->state = TASK_RUNNING; } /* * fork()/clone()-time setup: */ void sched_fork(struct task_struct *p, int clone_flags) { int cpu = get_cpu(); __sched_fork(p); #ifdef CONFIG_SMP cpu = sched_balance_self(cpu, SD_BALANCE_FORK); #endif __set_task_cpu(p, cpu); /* * Make sure we do not leak PI boosting priority to the child: */ p->prio = current->normal_prio; #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) if (likely(sched_info_on())) memset(&p->sched_info, 0, sizeof(p->sched_info)); #endif #if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW) p->oncpu = 0; #endif #ifdef CONFIG_PREEMPT /* Want to start with kernel preemption disabled. */ task_thread_info(p)->preempt_count = 1; #endif put_cpu(); } /* * After fork, child runs first. (default) If set to 0 then * parent will (try to) run first. */ unsigned int __read_mostly sysctl_sched_child_runs_first = 1; /* * wake_up_new_task - wake up a newly created task for the first time. * * This function will do some initial scheduler statistics housekeeping * that must be done for every newly created context, then puts the task * on the runqueue and wakes it. */ void fastcall wake_up_new_task(struct task_struct *p, unsigned long clone_flags) { unsigned long flags; struct rq *rq; int this_cpu; rq = task_rq_lock(p, &flags); BUG_ON(p->state != TASK_RUNNING); this_cpu = smp_processor_id(); /* parent's CPU */ p->prio = effective_prio(p); if (!sysctl_sched_child_runs_first || (clone_flags & CLONE_VM) || task_cpu(p) != this_cpu || !current->se.on_rq) { activate_task(rq, p, 0); } else { /* * Let the scheduling class do new task startup * management (if any): */ p->sched_class->task_new(rq, p); } check_preempt_curr(rq, p); task_rq_unlock(rq, &flags); } /** * prepare_task_switch - prepare to switch tasks * @rq: the runqueue preparing to switch * @next: the task we are going to switch to. * * This is called with the rq lock held and interrupts off. It must * be paired with a subsequent finish_task_switch after the context * switch. * * prepare_task_switch sets up locking and calls architecture specific * hooks. */ static inline void prepare_task_switch(struct rq *rq, struct task_struct *next) { prepare_lock_switch(rq, next); prepare_arch_switch(next); } /** * finish_task_switch - clean up after a task-switch * @rq: runqueue associated with task-switch * @prev: the thread we just switched away from. * * finish_task_switch must be called after the context switch, paired * with a prepare_task_switch call before the context switch. * finish_task_switch will reconcile locking set up by prepare_task_switch, * and do any other architecture-specific cleanup actions. * * Note that we may have delayed dropping an mm in context_switch(). If * so, we finish that here outside of the runqueue lock. (Doing it * with the lock held can cause deadlocks; see schedule() for * details.) */ static inline void finish_task_switch(struct rq *rq, struct task_struct *prev) __releases(rq->lock) { struct mm_struct *mm = rq->prev_mm; long prev_state; rq->prev_mm = NULL; /* * A task struct has one reference for the use as "current". * If a task dies, then it sets TASK_DEAD in tsk->state and calls * schedule one last time. The schedule call will never return, and * the scheduled task must drop that reference. * The test for TASK_DEAD must occur while the runqueue locks are * still held, otherwise prev could be scheduled on another cpu, die * there before we look at prev->state, and then the reference would * be dropped twice. * Manfred Spraul */ prev_state = prev->state; finish_arch_switch(prev); finish_lock_switch(rq, prev); if (mm) mmdrop(mm); if (unlikely(prev_state == TASK_DEAD)) { /* * Remove function-return probe instances associated with this * task and put them back on the free list. */ kprobe_flush_task(prev); put_task_struct(prev); } } /** * schedule_tail - first thing a freshly forked thread must call. * @prev: the thread we just switched away from. */ asmlinkage void schedule_tail(struct task_struct *prev) __releases(rq->lock) { struct rq *rq = this_rq(); finish_task_switch(rq, prev); #ifdef __ARCH_WANT_UNLOCKED_CTXSW /* In this case, finish_task_switch does not reenable preemption */ preempt_enable(); #endif if (current->set_child_tid) put_user(current->pid, current->set_child_tid); } /* * context_switch - switch to the new MM and the new * thread's register state. */ static inline void context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) { struct mm_struct *mm, *oldmm; prepare_task_switch(rq, next); mm = next->mm; oldmm = prev->active_mm; /* * For paravirt, this is coupled with an exit in switch_to to * combine the page table reload and the switch backend into * one hypercall. */ arch_enter_lazy_cpu_mode(); if (unlikely(!mm)) { next->active_mm = oldmm; atomic_inc(&oldmm->mm_count); enter_lazy_tlb(oldmm, next); } else switch_mm(oldmm, mm, next); if (unlikely(!prev->mm)) { prev->active_mm = NULL; rq->prev_mm = oldmm; } /* * Since the runqueue lock will be released by the next * task (which is an invalid locking op but in the case * of the scheduler it's an obvious special-case), so we * do an early lockdep release here: */ #ifndef __ARCH_WANT_UNLOCKED_CTXSW spin_release(&rq->lock.dep_map, 1, _THIS_IP_); #endif /* Here we just switch the register state and the stack. */ switch_to(prev, next, prev); barrier(); /* * this_rq must be evaluated again because prev may have moved * CPUs since it called schedule(), thus the 'rq' on its stack * frame will be invalid. */ finish_task_switch(this_rq(), prev); } /* * nr_running, nr_uninterruptible and nr_context_switches: * * externally visible scheduler statistics: current number of runnable * threads, current number of uninterruptible-sleeping threads, total * number of context switches performed since bootup. */ unsigned long nr_running(void) { unsigned long i, sum = 0; for_each_online_cpu(i) sum += cpu_rq(i)->nr_running; return sum; } unsigned long nr_uninterruptible(void) { unsigned long i, sum = 0; for_each_possible_cpu(i) sum += cpu_rq(i)->nr_uninterruptible; /* * Since we read the counters lockless, it might be slightly * inaccurate. Do not allow it to go below zero though: */ if (unlikely((long)sum < 0)) sum = 0; return sum; } unsigned long long nr_context_switches(void) { int i; unsigned long long sum = 0; for_each_possible_cpu(i) sum += cpu_rq(i)->nr_switches; return sum; } unsigned long nr_iowait(void) { unsigned long i, sum = 0; for_each_possible_cpu(i) sum += atomic_read(&cpu_rq(i)->nr_iowait); return sum; } unsigned long nr_active(void) { unsigned long i, running = 0, uninterruptible = 0; for_each_online_cpu(i) { running += cpu_rq(i)->nr_running; uninterruptible += cpu_rq(i)->nr_uninterruptible; } if (unlikely((long)uninterruptible < 0)) uninterruptible = 0; return running + uninterruptible; } /* * Update rq->cpu_load[] statistics. This function is usually called every * scheduler tick (TICK_NSEC). */ static void update_cpu_load(struct rq *this_rq) { u64 fair_delta64, exec_delta64, idle_delta64, sample_interval64, tmp64; unsigned long total_load = this_rq->ls.load.weight; unsigned long this_load = total_load; struct load_stat *ls = &this_rq->ls; u64 now = __rq_clock(this_rq); int i, scale; this_rq->nr_load_updates++; if (unlikely(!(sysctl_sched_features & SCHED_FEAT_PRECISE_CPU_LOAD))) goto do_avg; /* Update delta_fair/delta_exec fields first */ update_curr_load(this_rq, now); fair_delta64 = ls->delta_fair + 1; ls->delta_fair = 0; exec_delta64 = ls->delta_exec + 1; ls->delta_exec = 0; sample_interval64 = now - ls->load_update_last; ls->load_update_last = now; if ((s64)sample_interval64 < (s64)TICK_NSEC) sample_interval64 = TICK_NSEC; if (exec_delta64 > sample_interval64) exec_delta64 = sample_interval64; idle_delta64 = sample_interval64 - exec_delta64; tmp64 = div64_64(SCHED_LOAD_SCALE * exec_delta64, fair_delta64); tmp64 = div64_64(tmp64 * exec_delta64, sample_interval64); this_load = (unsigned long)tmp64; do_avg: /* Update our load: */ for (i = 0, scale = 1; i < CPU_LOAD_IDX_MAX; i++, scale += scale) { unsigned long old_load, new_load; /* scale is effectively 1 << i now, and >> i divides by scale */ old_load = this_rq->cpu_load[i]; new_load = this_load; this_rq->cpu_load[i] = (old_load*(scale-1) + new_load) >> i; } } #ifdef CONFIG_SMP /* * double_rq_lock - safely lock two runqueues * * Note this does not disable interrupts like task_rq_lock, * you need to do so manually before calling. */ static void double_rq_lock(struct rq *rq1, struct rq *rq2) __acquires(rq1->lock) __acquires(rq2->lock) { BUG_ON(!irqs_disabled()); if (rq1 == rq2) { spin_lock(&rq1->lock); __acquire(rq2->lock); /* Fake it out ;) */ } else { if (rq1 < rq2) { spin_lock(&rq1->lock); spin_lock(&rq2->lock); } else { spin_lock(&rq2->lock); spin_lock(&rq1->lock); } } } /* * double_rq_unlock - safely unlock two runqueues * * Note this does not restore interrupts like task_rq_unlock, * you need to do so manually after calling. */ static void double_rq_unlock(struct rq *rq1, struct rq *rq2) __releases(rq1->lock) __releases(rq2->lock) { spin_unlock(&rq1->lock); if (rq1 != rq2) spin_unlock(&rq2->lock); else __release(rq2->lock); } /* * double_lock_balance - lock the busiest runqueue, this_rq is locked already. */ static void double_lock_balance(struct rq *this_rq, struct rq *busiest) __releases(this_rq->lock) __acquires(busiest->lock) __acquires(this_rq->lock) { if (unlikely(!irqs_disabled())) { /* printk() doesn't work good under rq->lock */ spin_unlock(&this_rq->lock); BUG_ON(1); } if (unlikely(!spin_trylock(&busiest->lock))) { if (busiest < this_rq) { spin_unlock(&this_rq->lock); spin_lock(&busiest->lock); spin_lock(&this_rq->lock); } else spin_lock(&busiest->lock); } } /* * If dest_cpu is allowed for this process, migrate the task to it. * This is accomplished by forcing the cpu_allowed mask to only * allow dest_cpu, which will force the cpu onto dest_cpu. Then * the cpu_allowed mask is restored. */ static void sched_migrate_task(struct task_struct *p, int dest_cpu) { struct migration_req req; unsigned long flags; struct rq *rq; rq = task_rq_lock(p, &flags); if (!cpu_isset(dest_cpu, p->cpus_allowed) || unlikely(cpu_is_offline(dest_cpu))) goto out; /* force the process onto the specified CPU */ if (migrate_task(p, dest_cpu, &req)) { /* Need to wait for migration thread (might exit: take ref). */ struct task_struct *mt = rq->migration_thread; get_task_struct(mt); task_rq_unlock(rq, &flags); wake_up_process(mt); put_task_struct(mt); wait_for_completion(&req.done); return; } out: task_rq_unlock(rq, &flags); } /* * sched_exec - execve() is a valuable balancing opportunity, because at * this point the task has the smallest effective memory and cache footprint. */ void sched_exec(void) { int new_cpu, this_cpu = get_cpu(); new_cpu = sched_balance_self(this_cpu, SD_BALANCE_EXEC); put_cpu(); if (new_cpu != this_cpu) sched_migrate_task(current, new_cpu); } /* * pull_task - move a task from a remote runqueue to the local runqueue. * Both runqueues must be locked. */ static void pull_task(struct rq *src_rq, struct task_struct *p, struct rq *this_rq, int this_cpu) { deactivate_task(src_rq, p, 0); set_task_cpu(p, this_cpu); activate_task(this_rq, p, 0); /* * Note that idle threads have a prio of MAX_PRIO, for this test * to be always true for them. */ check_preempt_curr(this_rq, p); } /* * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? */ static int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu, struct sched_domain *sd, enum cpu_idle_type idle, int *all_pinned) { /* * We do not migrate tasks that are: * 1) running (obviously), or * 2) cannot be migrated to this CPU due to cpus_allowed, or * 3) are cache-hot on their current CPU. */ if (!cpu_isset(this_cpu, p->cpus_allowed)) return 0; *all_pinned = 0; if (task_running(rq, p)) return 0; /* * Aggressive migration if too many balance attempts have failed: */ if (sd->nr_balance_failed > sd->cache_nice_tries) return 1; return 1; } static int balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, unsigned long max_nr_move, unsigned long max_load_move, struct sched_domain *sd, enum cpu_idle_type idle, int *all_pinned, unsigned long *load_moved, int this_best_prio, int best_prio, int best_prio_seen, struct rq_iterator *iterator) { int pulled = 0, pinned = 0, skip_for_load; struct task_struct *p; long rem_load_move = max_load_move; if (max_nr_move == 0 || max_load_move == 0) goto out; pinned = 1; /* * Start the load-balancing iterator: */ p = iterator->start(iterator->arg); next: if (!p) goto out; /* * To help distribute high priority tasks accross CPUs we don't * skip a task if it will be the highest priority task (i.e. smallest * prio value) on its new queue regardless of its load weight */ skip_for_load = (p->se.load.weight >> 1) > rem_load_move + SCHED_LOAD_SCALE_FUZZ; if (skip_for_load && p->prio < this_best_prio) skip_for_load = !best_prio_seen && p->prio == best_prio; if (skip_for_load || !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) { best_prio_seen |= p->prio == best_prio; p = iterator->next(iterator->arg); goto next; } pull_task(busiest, p, this_rq, this_cpu); pulled++; rem_load_move -= p->se.load.weight; /* * We only want to steal up to the prescribed number of tasks * and the prescribed amount of weighted load. */ if (pulled < max_nr_move && rem_load_move > 0) { if (p->prio < this_best_prio) this_best_prio = p->prio; p = iterator->next(iterator->arg); goto next; } out: /* * Right now, this is the only place pull_task() is called, * so we can safely collect pull_task() stats here rather than * inside pull_task(). */ schedstat_add(sd, lb_gained[idle], pulled); if (all_pinned) *all_pinned = pinned; *load_moved = max_load_move - rem_load_move; return pulled; } /* * move_tasks tries to move up to max_nr_move tasks and max_load_move weighted * load from busiest to this_rq, as part of a balancing operation within * "domain". Returns the number of tasks moved. * * Called with both runqueues locked. */ static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest, unsigned long max_nr_move, unsigned long max_load_move, struct sched_domain *sd, enum cpu_idle_type idle, int *all_pinned) { struct sched_class *class = sched_class_highest; unsigned long load_moved, total_nr_moved = 0, nr_moved; long rem_load_move = max_load_move; do { nr_moved = class->load_balance(this_rq, this_cpu, busiest, max_nr_move, (unsigned long)rem_load_move, sd, idle, all_pinned, &load_moved); total_nr_moved += nr_moved; max_nr_move -= nr_moved; rem_load_move -= load_moved; class = class->next; } while (class && max_nr_move && rem_load_move > 0); return total_nr_moved; } /* * find_busiest_group finds and returns the busiest CPU group within the * domain. It calculates and returns the amount of weighted load which * should be moved to restore balance via the imbalance parameter. */ static struct sched_group * find_busiest_group(struct sched_domain *sd, int this_cpu, unsigned long *imbalance, enum cpu_idle_type idle, int *sd_idle, cpumask_t *cpus, int *balance) { struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups; unsigned long max_load, avg_load, total_load, this_load, total_pwr; unsigned long max_pull; unsigned long busiest_load_per_task, busiest_nr_running; unsigned long this_load_per_task, this_nr_running; int load_idx; #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT) int power_savings_balance = 1; unsigned long leader_nr_running = 0, min_load_per_task = 0; unsigned long min_nr_running = ULONG_MAX; struct sched_group *group_min = NULL, *group_leader = NULL; #endif max_load = this_load = total_load = total_pwr = 0; busiest_load_per_task = busiest_nr_running = 0; this_load_per_task = this_nr_running = 0; if (idle == CPU_NOT_IDLE) load_idx = sd->busy_idx; else if (idle == CPU_NEWLY_IDLE) load_idx = sd->newidle_idx; else load_idx = sd->idle_idx; do { unsigned long load, group_capacity; int local_group; int i; unsigned int balance_cpu = -1, first_idle_cpu = 0; unsigned long sum_nr_running, sum_weighted_load; local_group = cpu_isset(this_cpu, group->cpumask); if (local_group) balance_cpu = first_cpu(group->cpumask); /* Tally up the load of all CPUs in the group */ sum_weighted_load = sum_nr_running = avg_load = 0; for_each_cpu_mask(i, group->cpumask) { struct rq *rq; if (!cpu_isset(i, *cpus)) continue; rq = cpu_rq(i); if (*sd_idle && !idle_cpu(i)) *sd_idle = 0; /* Bias balancing toward cpus of our domain */ if (local_group) { if (idle_cpu(i) && !first_idle_cpu) { first_idle_cpu = 1; balance_cpu = i; } load = target_load(i, load_idx); } else load = source_load(i, load_idx); avg_load += load; sum_nr_running += rq->nr_running; sum_weighted_load += weighted_cpuload(i); } /* * First idle cpu or the first cpu(busiest) in this sched group * is eligible for doing load balancing at this and above * domains. */ if (local_group && balance_cpu != this_cpu && balance) { *balance = 0; goto ret; } total_load += avg_load; total_pwr += group->__cpu_power; /* Adjust by relative CPU power of the group */ avg_load = sg_div_cpu_power(group, avg_load * SCHED_LOAD_SCALE); group_capacity = group->__cpu_power / SCHED_LOAD_SCALE; if (local_group) { this_load = avg_load; this = group; this_nr_running = sum_nr_running; this_load_per_task = sum_weighted_load; } else if (avg_load > max_load && sum_nr_running > group_capacity) { max_load = avg_load; busiest = group; busiest_nr_running = sum_nr_running; busiest_load_per_task = sum_weighted_load; } #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT) /* * Busy processors will not participate in power savings * balance. */ if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE)) goto group_next; /* * If the local group is idle or completely loaded * no need to do power savings balance at this domain */ if (local_group && (this_nr_running >= group_capacity || !this_nr_running)) power_savings_balance = 0; /* * If a group is already running at full capacity or idle, * don't include that group in power savings calculations */ if (!power_savings_balance || sum_nr_running >= group_capacity || !sum_nr_running) goto group_next; /* * Calculate the group which has the least non-idle load. * This is the group from where we need to pick up the load * for saving power */ if ((sum_nr_running < min_nr_running) || (sum_nr_running == min_nr_running && first_cpu(group->cpumask) < first_cpu(group_min->cpumask))) { group_min = group; min_nr_running = sum_nr_running; min_load_per_task = sum_weighted_load / sum_nr_running; } /* * Calculate the group which is almost near its * capacity but still has some space to pick up some load * from other group and save more power */ if (sum_nr_running <= group_capacity - 1) { if (sum_nr_running > leader_nr_running || (sum_nr_running == leader_nr_running && first_cpu(group->cpumask) > first_cpu(group_leader->cpumask))) { group_leader = group; leader_nr_running = sum_nr_running; } } group_next: #endif group = group->next; } while (group != sd->groups); if (!busiest || this_load >= max_load || busiest_nr_running == 0) goto out_balanced; avg_load = (SCHED_LOAD_SCALE * total_load) / total_pwr; if (this_load >= avg_load || 100*max_load <= sd->imbalance_pct*this_load) goto out_balanced; busiest_load_per_task /= busiest_nr_running; /* * We're trying to get all the cpus to the average_load, so we don't * want to push ourselves above the average load, nor do we wish to * reduce the max loaded cpu below the average load, as either of these * actions would just result in more rebalancing later, and ping-pong * tasks around. Thus we look for the minimum possible imbalance. * Negative imbalances (*we* are more loaded than anyone else) will * be counted as no imbalance for these purposes -- we can't fix that * by pulling tasks to us. Be careful of negative numbers as they'll * appear as very large values with unsigned longs. */ if (max_load <= busiest_load_per_task) goto out_balanced; /* * In the presence of smp nice balancing, certain scenarios can have * max load less than avg load(as we skip the groups at or below * its cpu_power, while calculating max_load..) */ if (max_load < avg_load) { *imbalance = 0; goto small_imbalance; } /* Don't want to pull so many tasks that a group would go idle */ max_pull = min(max_load - avg_load, max_load - busiest_load_per_task); /* How much load to actually move to equalise the imbalance */ *imbalance = min(max_pull * busiest->__cpu_power, (avg_load - this_load) * this->__cpu_power) / SCHED_LOAD_SCALE; /* * if *imbalance is less than the average load per runnable task * there is no gaurantee that any tasks will be moved so we'll have * a think about bumping its value to force at least one task to be * moved */ if (*imbalance + SCHED_LOAD_SCALE_FUZZ < busiest_load_per_task/2) { unsigned long tmp, pwr_now, pwr_move; unsigned int imbn; small_imbalance: pwr_move = pwr_now = 0; imbn = 2; if (this_nr_running) { this_load_per_task /= this_nr_running; if (busiest_load_per_task > this_load_per_task) imbn = 1; } else this_load_per_task = SCHED_LOAD_SCALE; if (max_load - this_load + SCHED_LOAD_SCALE_FUZZ >= busiest_load_per_task * imbn) { *imbalance = busiest_load_per_task; return busiest; } /* * OK, we don't have enough imbalance to justify moving tasks, * however we may be able to increase total CPU power used by * moving them. */ pwr_now += busiest->__cpu_power * min(busiest_load_per_task, max_load); pwr_now += this->__cpu_power * min(this_load_per_task, this_load); pwr_now /= SCHED_LOAD_SCALE; /* Amount of load we'd subtract */ tmp = sg_div_cpu_power(busiest, busiest_load_per_task * SCHED_LOAD_SCALE); if (max_load > tmp) pwr_move += busiest->__cpu_power * min(busiest_load_per_task, max_load - tmp); /* Amount of load we'd add */ if (max_load * busiest->__cpu_power < busiest_load_per_task * SCHED_LOAD_SCALE) tmp = sg_div_cpu_power(this, max_load * busiest->__cpu_power); else tmp = sg_div_cpu_power(this, busiest_load_per_task * SCHED_LOAD_SCALE); pwr_move += this->__cpu_power * min(this_load_per_task, this_load + tmp); pwr_move /= SCHED_LOAD_SCALE; /* Move if we gain throughput */ if (pwr_move <= pwr_now) goto out_balanced; *imbalance = busiest_load_per_task; } return busiest; out_balanced: #if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT) if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE)) goto ret; if (this == group_leader && group_leader != group_min) { *imbalance = min_load_per_task; return group_min; } #endif ret: *imbalance = 0; return NULL; } /* * find_busiest_queue - find the busiest runqueue among the cpus in group. */ static struct rq * find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle, unsigned long imbalance, cpumask_t *cpus) { struct rq *busiest = NULL, *rq; unsigned long max_load = 0; int i; for_each_cpu_mask(i, group->cpumask) { unsigned long wl; if (!cpu_isset(i, *cpus)) continue; rq = cpu_rq(i); wl = weighted_cpuload(i); if (rq->nr_running == 1 && wl > imbalance) continue; if (wl > max_load) { max_load = wl; busiest = rq; } } return busiest; } /* * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but * so long as it is large enough. */ #define MAX_PINNED_INTERVAL 512 static inline unsigned long minus_1_or_zero(unsigned long n) { return n > 0 ? n - 1 : 0; } /* * Check this_cpu to ensure it is balanced within domain. Attempt