aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched.c')
-rw-r--r--kernel/sched.c697
1 files changed, 394 insertions, 303 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index 3c11ae0a948d..18cceeecce35 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -141,7 +141,7 @@ struct rt_prio_array {
141 141
142struct rt_bandwidth { 142struct rt_bandwidth {
143 /* nests inside the rq lock: */ 143 /* nests inside the rq lock: */
144 spinlock_t rt_runtime_lock; 144 raw_spinlock_t rt_runtime_lock;
145 ktime_t rt_period; 145 ktime_t rt_period;
146 u64 rt_runtime; 146 u64 rt_runtime;
147 struct hrtimer rt_period_timer; 147 struct hrtimer rt_period_timer;
@@ -178,7 +178,7 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
178 rt_b->rt_period = ns_to_ktime(period); 178 rt_b->rt_period = ns_to_ktime(period);
179 rt_b->rt_runtime = runtime; 179 rt_b->rt_runtime = runtime;
180 180
181 spin_lock_init(&rt_b->rt_runtime_lock); 181 raw_spin_lock_init(&rt_b->rt_runtime_lock);
182 182
183 hrtimer_init(&rt_b->rt_period_timer, 183 hrtimer_init(&rt_b->rt_period_timer,
184 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 184 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
@@ -200,7 +200,7 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
200 if (hrtimer_active(&rt_b->rt_period_timer)) 200 if (hrtimer_active(&rt_b->rt_period_timer))
201 return; 201 return;
202 202
203 spin_lock(&rt_b->rt_runtime_lock); 203 raw_spin_lock(&rt_b->rt_runtime_lock);
204 for (;;) { 204 for (;;) {
205 unsigned long delta; 205 unsigned long delta;
206 ktime_t soft, hard; 206 ktime_t soft, hard;
@@ -217,7 +217,7 @@ static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
217 __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta, 217 __hrtimer_start_range_ns(&rt_b->rt_period_timer, soft, delta,
218 HRTIMER_MODE_ABS_PINNED, 0); 218 HRTIMER_MODE_ABS_PINNED, 0);
219 } 219 }
220 spin_unlock(&rt_b->rt_runtime_lock); 220 raw_spin_unlock(&rt_b->rt_runtime_lock);
221} 221}
222 222
223#ifdef CONFIG_RT_GROUP_SCHED 223#ifdef CONFIG_RT_GROUP_SCHED
@@ -298,7 +298,7 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct cfs_rq, init_tg_cfs_rq);
298 298
299#ifdef CONFIG_RT_GROUP_SCHED 299#ifdef CONFIG_RT_GROUP_SCHED
300static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); 300static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
301static DEFINE_PER_CPU_SHARED_ALIGNED(struct rt_rq, init_rt_rq); 301static DEFINE_PER_CPU_SHARED_ALIGNED(struct rt_rq, init_rt_rq_var);
302#endif /* CONFIG_RT_GROUP_SCHED */ 302#endif /* CONFIG_RT_GROUP_SCHED */
303#else /* !CONFIG_USER_SCHED */ 303#else /* !CONFIG_USER_SCHED */
304#define root_task_group init_task_group 304#define root_task_group init_task_group
@@ -470,7 +470,7 @@ struct rt_rq {
470 u64 rt_time; 470 u64 rt_time;
471 u64 rt_runtime; 471 u64 rt_runtime;
472 /* Nests inside the rq lock: */ 472 /* Nests inside the rq lock: */
473 spinlock_t rt_runtime_lock; 473 raw_spinlock_t rt_runtime_lock;
474 474
475#ifdef CONFIG_RT_GROUP_SCHED 475#ifdef CONFIG_RT_GROUP_SCHED
476 unsigned long rt_nr_boosted; 476 unsigned long rt_nr_boosted;
@@ -525,7 +525,7 @@ static struct root_domain def_root_domain;
525 */ 525 */
526struct rq { 526struct rq {
527 /* runqueue lock: */ 527 /* runqueue lock: */
528 spinlock_t lock; 528 raw_spinlock_t lock;
529 529
530 /* 530 /*
531 * nr_running and cpu_load should be in the same cacheline because 531 * nr_running and cpu_load should be in the same cacheline because
@@ -535,14 +535,12 @@ struct rq {
535 #define CPU_LOAD_IDX_MAX 5 535 #define CPU_LOAD_IDX_MAX 5
536 unsigned long cpu_load[CPU_LOAD_IDX_MAX]; 536 unsigned long cpu_load[CPU_LOAD_IDX_MAX];
537#ifdef CONFIG_NO_HZ 537#ifdef CONFIG_NO_HZ
538 unsigned long last_tick_seen;
539 unsigned char in_nohz_recently; 538 unsigned char in_nohz_recently;
540#endif 539#endif
541 /* capture load from *all* tasks on this cpu: */ 540 /* capture load from *all* tasks on this cpu: */
542 struct load_weight load; 541 struct load_weight load;
543 unsigned long nr_load_updates; 542 unsigned long nr_load_updates;
544 u64 nr_switches; 543 u64 nr_switches;
545 u64 nr_migrations_in;
546 544
547 struct cfs_rq cfs; 545 struct cfs_rq cfs;
548 struct rt_rq rt; 546 struct rt_rq rt;
@@ -591,6 +589,8 @@ struct rq {
591 589
592 u64 rt_avg; 590 u64 rt_avg;
593 u64 age_stamp; 591 u64 age_stamp;
592 u64 idle_stamp;
593 u64 avg_idle;
594#endif 594#endif
595 595
596 /* calc_load related fields */ 596 /* calc_load related fields */
@@ -685,7 +685,7 @@ inline void update_rq_clock(struct rq *rq)
685 */ 685 */
686int runqueue_is_locked(int cpu) 686int runqueue_is_locked(int cpu)
687{ 687{
688 return spin_is_locked(&cpu_rq(cpu)->lock); 688 return raw_spin_is_locked(&cpu_rq(cpu)->lock);
689} 689}
690 690
691/* 691/*
@@ -772,7 +772,7 @@ sched_feat_write(struct file *filp, const char __user *ubuf,
772 if (!sched_feat_names[i]) 772 if (!sched_feat_names[i])
773 return -EINVAL; 773 return -EINVAL;
774 774
775 filp->f_pos += cnt; 775 *ppos += cnt;
776 776
777 return cnt; 777 return cnt;
778} 778}
@@ -814,6 +814,7 @@ const_debug unsigned int sysctl_sched_nr_migrate = 32;
814 * default: 0.25ms 814 * default: 0.25ms
815 */ 815 */
816unsigned int sysctl_sched_shares_ratelimit = 250000; 816unsigned int sysctl_sched_shares_ratelimit = 250000;
817unsigned int normalized_sysctl_sched_shares_ratelimit = 250000;
817 818
818/* 819/*
819 * Inject some fuzzyness into changing the per-cpu group shares 820 * Inject some fuzzyness into changing the per-cpu group shares
@@ -892,7 +893,7 @@ static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
892 */ 893 */
893 spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_); 894 spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
894 895
895 spin_unlock_irq(&rq->lock); 896 raw_spin_unlock_irq(&rq->lock);
896} 897}
897 898
898#else /* __ARCH_WANT_UNLOCKED_CTXSW */ 899#else /* __ARCH_WANT_UNLOCKED_CTXSW */
@@ -916,9 +917,9 @@ static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next)
916 next->oncpu = 1; 917 next->oncpu = 1;
917#endif 918#endif
918#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW 919#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
919 spin_unlock_irq(&rq->lock); 920 raw_spin_unlock_irq(&rq->lock);
920#else 921#else
921 spin_unlock(&rq->lock); 922 raw_spin_unlock(&rq->lock);
922#endif 923#endif
923} 924}
924 925
@@ -948,10 +949,10 @@ static inline struct rq *__task_rq_lock(struct task_struct *p)
948{ 949{
949 for (;;) { 950 for (;;) {
950 struct rq *rq = task_rq(p); 951 struct rq *rq = task_rq(p);
951 spin_lock(&rq->lock); 952 raw_spin_lock(&rq->lock);
952 if (likely(rq == task_rq(p))) 953 if (likely(rq == task_rq(p)))
953 return rq; 954 return rq;
954 spin_unlock(&rq->lock); 955 raw_spin_unlock(&rq->lock);
955 } 956 }
956} 957}
957 958
@@ -968,10 +969,10 @@ static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags)
968 for (;;) { 969 for (;;) {
969 local_irq_save(*flags); 970 local_irq_save(*flags);
970 rq = task_rq(p); 971 rq = task_rq(p);
971 spin_lock(&rq->lock); 972 raw_spin_lock(&rq->lock);
972 if (likely(rq == task_rq(p))) 973 if (likely(rq == task_rq(p)))
973 return rq; 974 return rq;
974 spin_unlock_irqrestore(&rq->lock, *flags); 975 raw_spin_unlock_irqrestore(&rq->lock, *flags);
975 } 976 }
976} 977}
977 978
@@ -980,19 +981,19 @@ void task_rq_unlock_wait(struct task_struct *p)
980 struct rq *rq = task_rq(p); 981 struct rq *rq = task_rq(p);
981 982
982 smp_mb(); /* spin-unlock-wait is not a full memory barrier */ 983 smp_mb(); /* spin-unlock-wait is not a full memory barrier */
983 spin_unlock_wait(&rq->lock); 984 raw_spin_unlock_wait(&rq->lock);
984} 985}
985 986
986static void __task_rq_unlock(struct rq *rq) 987static void __task_rq_unlock(struct rq *rq)
987 __releases(rq->lock) 988 __releases(rq->lock)
988{ 989{
989 spin_unlock(&rq->lock); 990 raw_spin_unlock(&rq->lock);
990} 991}
991 992
992static inline void task_rq_unlock(struct rq *rq, unsigned long *flags) 993static inline void task_rq_unlock(struct rq *rq, unsigned long *flags)
993 __releases(rq->lock) 994 __releases(rq->lock)
994{ 995{
995 spin_unlock_irqrestore(&rq->lock, *flags); 996 raw_spin_unlock_irqrestore(&rq->lock, *flags);
996} 997}
997 998
998/* 999/*
@@ -1005,7 +1006,7 @@ static struct rq *this_rq_lock(void)
1005 1006
1006 local_irq_disable(); 1007 local_irq_disable();
1007 rq = this_rq(); 1008 rq = this_rq();
1008 spin_lock(&rq->lock); 1009 raw_spin_lock(&rq->lock);
1009 1010
1010 return rq; 1011 return rq;
1011} 1012}
@@ -1052,10 +1053,10 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer)
1052 1053
1053 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 1054 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
1054 1055
1055 spin_lock(&rq->lock); 1056 raw_spin_lock(&rq->lock);
1056 update_rq_clock(rq); 1057 update_rq_clock(rq);
1057 rq->curr->sched_class->task_tick(rq, rq->curr, 1); 1058 rq->curr->sched_class->task_tick(rq, rq->curr, 1);
1058 spin_unlock(&rq->lock); 1059 raw_spin_unlock(&rq->lock);
1059 1060
1060 return HRTIMER_NORESTART; 1061 return HRTIMER_NORESTART;
1061} 1062}
@@ -1068,10 +1069,10 @@ static void __hrtick_start(void *arg)
1068{ 1069{
1069 struct rq *rq = arg; 1070 struct rq *rq = arg;
1070 1071
1071 spin_lock(&rq->lock); 1072 raw_spin_lock(&rq->lock);
1072 hrtimer_restart(&rq->hrtick_timer); 1073 hrtimer_restart(&rq->hrtick_timer);
1073 rq->hrtick_csd_pending = 0; 1074 rq->hrtick_csd_pending = 0;
1074 spin_unlock(&rq->lock); 1075 raw_spin_unlock(&rq->lock);
1075} 1076}
1076 1077
1077/* 1078/*
@@ -1178,7 +1179,7 @@ static void resched_task(struct task_struct *p)
1178{ 1179{
1179 int cpu; 1180 int cpu;
1180 1181
1181 assert_spin_locked(&task_rq(p)->lock); 1182 assert_raw_spin_locked(&task_rq(p)->lock);
1182 1183
1183 if (test_tsk_need_resched(p)) 1184 if (test_tsk_need_resched(p))
1184 return; 1185 return;
@@ -1200,10 +1201,10 @@ static void resched_cpu(int cpu)
1200 struct rq *rq = cpu_rq(cpu); 1201 struct rq *rq = cpu_rq(cpu);
1201 unsigned long flags; 1202 unsigned long flags;
1202 1203
1203 if (!spin_trylock_irqsave(&rq->lock, flags)) 1204 if (!raw_spin_trylock_irqsave(&rq->lock, flags))
1204 return; 1205 return;
1205 resched_task(cpu_curr(cpu)); 1206 resched_task(cpu_curr(cpu));
1206 spin_unlock_irqrestore(&rq->lock, flags); 1207 raw_spin_unlock_irqrestore(&rq->lock, flags);
1207} 1208}
1208 1209
1209#ifdef CONFIG_NO_HZ 1210#ifdef CONFIG_NO_HZ
@@ -1272,7 +1273,7 @@ static void sched_rt_avg_update(struct rq *rq, u64 rt_delta)
1272#else /* !CONFIG_SMP */ 1273#else /* !CONFIG_SMP */
1273static void resched_task(struct task_struct *p) 1274static void resched_task(struct task_struct *p)
1274{ 1275{
1275 assert_spin_locked(&task_rq(p)->lock); 1276 assert_raw_spin_locked(&task_rq(p)->lock);
1276 set_tsk_need_resched(p); 1277 set_tsk_need_resched(p);
1277} 1278}
1278 1279
@@ -1599,11 +1600,11 @@ static void update_group_shares_cpu(struct task_group *tg, int cpu,
1599 struct rq *rq = cpu_rq(cpu); 1600 struct rq *rq = cpu_rq(cpu);
1600 unsigned long flags; 1601 unsigned long flags;
1601 1602
1602 spin_lock_irqsave(&rq->lock, flags); 1603 raw_spin_lock_irqsave(&rq->lock, flags);
1603 tg->cfs_rq[cpu]->rq_weight = boost ? 0 : rq_weight; 1604 tg->cfs_rq[cpu]->rq_weight = boost ? 0 : rq_weight;
1604 tg->cfs_rq[cpu]->shares = boost ? 0 : shares; 1605 tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
1605 __set_se_shares(tg->se[cpu], shares); 1606 __set_se_shares(tg->se[cpu], shares);
1606 spin_unlock_irqrestore(&rq->lock, flags); 1607 raw_spin_unlock_irqrestore(&rq->lock, flags);
1607 } 1608 }
1608} 1609}
1609 1610
@@ -1614,7 +1615,7 @@ static void update_group_shares_cpu(struct task_group *tg, int cpu,
1614 */ 1615 */
1615static int tg_shares_up(struct task_group *tg, void *data) 1616static int tg_shares_up(struct task_group *tg, void *data)
1616{ 1617{
1617 unsigned long weight, rq_weight = 0, shares = 0; 1618 unsigned long weight, rq_weight = 0, sum_weight = 0, shares = 0;
1618 unsigned long *usd_rq_weight; 1619 unsigned long *usd_rq_weight;
1619 struct sched_domain *sd = data; 1620 struct sched_domain *sd = data;
1620 unsigned long flags; 1621 unsigned long flags;
@@ -1630,6 +1631,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
1630 weight = tg->cfs_rq[i]->load.weight; 1631 weight = tg->cfs_rq[i]->load.weight;
1631 usd_rq_weight[i] = weight; 1632 usd_rq_weight[i] = weight;
1632 1633
1634 rq_weight += weight;
1633 /* 1635 /*
1634 * If there are currently no tasks on the cpu pretend there 1636 * If there are currently no tasks on the cpu pretend there
1635 * is one of average load so that when a new task gets to 1637 * is one of average load so that when a new task gets to
@@ -1638,10 +1640,13 @@ static int tg_shares_up(struct task_group *tg, void *data)
1638 if (!weight) 1640 if (!weight)
1639 weight = NICE_0_LOAD; 1641 weight = NICE_0_LOAD;
1640 1642
1641 rq_weight += weight; 1643 sum_weight += weight;
1642 shares += tg->cfs_rq[i]->shares; 1644 shares += tg->cfs_rq[i]->shares;
1643 } 1645 }
1644 1646
1647 if (!rq_weight)
1648 rq_weight = sum_weight;
1649
1645 if ((!shares && rq_weight) || shares > tg->shares) 1650 if ((!shares && rq_weight) || shares > tg->shares)
1646 shares = tg->shares; 1651 shares = tg->shares;
1647 1652
@@ -1701,9 +1706,9 @@ static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
1701 if (root_task_group_empty()) 1706 if (root_task_group_empty())
1702 return; 1707 return;
1703 1708
1704 spin_unlock(&rq->lock); 1709 raw_spin_unlock(&rq->lock);
1705 update_shares(sd); 1710 update_shares(sd);
1706 spin_lock(&rq->lock); 1711 raw_spin_lock(&rq->lock);
1707} 1712}
1708 1713
1709static void update_h_load(long cpu) 1714static void update_h_load(long cpu)
@@ -1743,7 +1748,7 @@ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
1743 __acquires(busiest->lock) 1748 __acquires(busiest->lock)
1744 __acquires(this_rq->lock) 1749 __acquires(this_rq->lock)
1745{ 1750{
1746 spin_unlock(&this_rq->lock); 1751 raw_spin_unlock(&this_rq->lock);
1747 double_rq_lock(this_rq, busiest); 1752 double_rq_lock(this_rq, busiest);
1748 1753
1749 return 1; 1754 return 1;
@@ -1764,14 +1769,16 @@ static int _double_lock_balance(struct rq *this_rq, struct rq *busiest)
1764{ 1769{
1765 int ret = 0; 1770 int ret = 0;
1766 1771
1767 if (unlikely(!spin_trylock(&busiest->lock))) { 1772 if (unlikely(!raw_spin_trylock(&busiest->lock))) {
1768 if (busiest < this_rq) { 1773 if (busiest < this_rq) {
1769 spin_unlock(&this_rq->lock); 1774 raw_spin_unlock(&this_rq->lock);
1770 spin_lock(&busiest->lock); 1775 raw_spin_lock(&busiest->lock);
1771 spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING); 1776 raw_spin_lock_nested(&this_rq->lock,
1777 SINGLE_DEPTH_NESTING);
1772 ret = 1; 1778 ret = 1;
1773 } else 1779 } else
1774 spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING); 1780 raw_spin_lock_nested(&busiest->lock,
1781 SINGLE_DEPTH_NESTING);
1775 } 1782 }
1776 return ret; 1783 return ret;
1777} 1784}
@@ -1785,7 +1792,7 @@ static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
1785{ 1792{
1786 if (unlikely(!irqs_disabled())) { 1793 if (unlikely(!irqs_disabled())) {
1787 /* printk() doesn't work good under rq->lock */ 1794 /* printk() doesn't work good under rq->lock */
1788 spin_unlock(&this_rq->lock); 1795 raw_spin_unlock(&this_rq->lock);
1789 BUG_ON(1); 1796 BUG_ON(1);
1790 } 1797 }
1791 1798
@@ -1795,7 +1802,7 @@ static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
1795static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest) 1802static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
1796 __releases(busiest->lock) 1803 __releases(busiest->lock)
1797{ 1804{
1798 spin_unlock(&busiest->lock); 1805 raw_spin_unlock(&busiest->lock);
1799 lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_); 1806 lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
1800} 1807}
1801#endif 1808#endif
@@ -1810,6 +1817,22 @@ static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
1810#endif 1817#endif
1811 1818
1812static void calc_load_account_active(struct rq *this_rq); 1819static void calc_load_account_active(struct rq *this_rq);
1820static void update_sysctl(void);
1821static int get_update_sysctl_factor(void);
1822
1823static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
1824{
1825 set_task_rq(p, cpu);
1826#ifdef CONFIG_SMP
1827 /*
1828 * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
1829 * successfuly executed on another CPU. We must ensure that updates of
1830 * per-task data have been completed by this moment.
1831 */
1832 smp_wmb();
1833 task_thread_info(p)->cpu = cpu;
1834#endif
1835}
1813 1836
1814#include "sched_stats.h" 1837#include "sched_stats.h"
1815#include "sched_idletask.c" 1838#include "sched_idletask.c"
@@ -1967,20 +1990,6 @@ inline int task_curr(const struct task_struct *p)
1967 return cpu_curr(task_cpu(p)) == p; 1990 return cpu_curr(task_cpu(p)) == p;
1968} 1991}
1969 1992
1970static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
1971{
1972 set_task_rq(p, cpu);
1973#ifdef CONFIG_SMP
1974 /*
1975 * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
1976 * successfuly executed on another CPU. We must ensure that updates of
1977 * per-task data have been completed by this moment.
1978 */
1979 smp_wmb();
1980 task_thread_info(p)->cpu = cpu;
1981#endif
1982}
1983
1984static inline void check_class_changed(struct rq *rq, struct task_struct *p, 1993static inline void check_class_changed(struct rq *rq, struct task_struct *p,
1985 const struct sched_class *prev_class, 1994 const struct sched_class *prev_class,
1986 int oldprio, int running) 1995 int oldprio, int running)
@@ -2016,12 +2025,13 @@ void kthread_bind(struct task_struct *p, unsigned int cpu)
2016 return; 2025 return;
2017 } 2026 }
2018 2027
2019 spin_lock_irqsave(&rq->lock, flags); 2028 raw_spin_lock_irqsave(&rq->lock, flags);
2029 update_rq_clock(rq);
2020 set_task_cpu(p, cpu); 2030 set_task_cpu(p, cpu);
2021 p->cpus_allowed = cpumask_of_cpu(cpu); 2031 p->cpus_allowed = cpumask_of_cpu(cpu);
2022 p->rt.nr_cpus_allowed = 1; 2032 p->rt.nr_cpus_allowed = 1;
2023 p->flags |= PF_THREAD_BOUND; 2033 p->flags |= PF_THREAD_BOUND;
2024 spin_unlock_irqrestore(&rq->lock, flags); 2034 raw_spin_unlock_irqrestore(&rq->lock, flags);
2025} 2035}
2026EXPORT_SYMBOL(kthread_bind); 2036EXPORT_SYMBOL(kthread_bind);
2027 2037
@@ -2059,30 +2069,13 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
2059void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 2069void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
2060{ 2070{
2061 int old_cpu = task_cpu(p); 2071 int old_cpu = task_cpu(p);
2062 struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu);
2063 struct cfs_rq *old_cfsrq = task_cfs_rq(p), 2072 struct cfs_rq *old_cfsrq = task_cfs_rq(p),
2064 *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu); 2073 *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
2065 u64 clock_offset;
2066
2067 clock_offset = old_rq->clock - new_rq->clock;
2068 2074
2069 trace_sched_migrate_task(p, new_cpu); 2075 trace_sched_migrate_task(p, new_cpu);
2070 2076
2071#ifdef CONFIG_SCHEDSTATS
2072 if (p->se.wait_start)
2073 p->se.wait_start -= clock_offset;
2074 if (p->se.sleep_start)
2075 p->se.sleep_start -= clock_offset;
2076 if (p->se.block_start)
2077 p->se.block_start -= clock_offset;
2078#endif
2079 if (old_cpu != new_cpu) { 2077 if (old_cpu != new_cpu) {
2080 p->se.nr_migrations++; 2078 p->se.nr_migrations++;
2081 new_rq->nr_migrations_in++;
2082#ifdef CONFIG_SCHEDSTATS
2083 if (task_hot(p, old_rq->clock, NULL))
2084 schedstat_inc(p, se.nr_forced2_migrations);
2085#endif
2086 perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 2079 perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS,
2087 1, 1, NULL, 0); 2080 1, 1, NULL, 0);
2088 } 2081 }
@@ -2115,6 +2108,7 @@ migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
2115 * it is sufficient to simply update the task's cpu field. 2108 * it is sufficient to simply update the task's cpu field.
2116 */ 2109 */
2117 if (!p->se.on_rq && !task_running(rq, p)) { 2110 if (!p->se.on_rq && !task_running(rq, p)) {
2111 update_rq_clock(rq);
2118 set_task_cpu(p, dest_cpu); 2112 set_task_cpu(p, dest_cpu);
2119 return 0; 2113 return 0;
2120 } 2114 }
@@ -2322,6 +2316,14 @@ void task_oncpu_function_call(struct task_struct *p,
2322 preempt_enable(); 2316 preempt_enable();
2323} 2317}
2324 2318
2319#ifdef CONFIG_SMP
2320static inline
2321int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
2322{
2323 return p->sched_class->select_task_rq(p, sd_flags, wake_flags);
2324}
2325#endif
2326
2325/*** 2327/***
2326 * try_to_wake_up - wake up a thread 2328 * try_to_wake_up - wake up a thread
2327 * @p: the to-be-woken-up thread 2329 * @p: the to-be-woken-up thread
@@ -2373,16 +2375,14 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state,
2373 if (task_contributes_to_load(p)) 2375 if (task_contributes_to_load(p))
2374 rq->nr_uninterruptible--; 2376 rq->nr_uninterruptible--;
2375 p->state = TASK_WAKING; 2377 p->state = TASK_WAKING;
2376 task_rq_unlock(rq, &flags); 2378 __task_rq_unlock(rq);
2377 2379
2378 cpu = p->sched_class->select_task_rq(p, SD_BALANCE_WAKE, wake_flags); 2380 cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
2379 if (cpu != orig_cpu) 2381 if (cpu != orig_cpu)
2380 set_task_cpu(p, cpu); 2382 set_task_cpu(p, cpu);
2381 2383
2382 rq = task_rq_lock(p, &flags); 2384 rq = __task_rq_lock(p);
2383 2385 update_rq_clock(rq);
2384 if (rq != orig_rq)
2385 update_rq_clock(rq);
2386 2386
2387 WARN_ON(p->state != TASK_WAKING); 2387 WARN_ON(p->state != TASK_WAKING);
2388 cpu = task_cpu(p); 2388 cpu = task_cpu(p);
@@ -2440,6 +2440,17 @@ out_running:
2440#ifdef CONFIG_SMP 2440#ifdef CONFIG_SMP
2441 if (p->sched_class->task_wake_up) 2441 if (p->sched_class->task_wake_up)
2442 p->sched_class->task_wake_up(rq, p); 2442 p->sched_class->task_wake_up(rq, p);
2443
2444 if (unlikely(rq->idle_stamp)) {
2445 u64 delta = rq->clock - rq->idle_stamp;
2446 u64 max = 2*sysctl_sched_migration_cost;
2447
2448 if (delta > max)
2449 rq->avg_idle = max;
2450 else
2451 update_avg(&rq->avg_idle, delta);
2452 rq->idle_stamp = 0;
2453 }
2443#endif 2454#endif
2444out: 2455out:
2445 task_rq_unlock(rq, &flags); 2456 task_rq_unlock(rq, &flags);
@@ -2486,7 +2497,6 @@ static void __sched_fork(struct task_struct *p)
2486 p->se.avg_overlap = 0; 2497 p->se.avg_overlap = 0;
2487 p->se.start_runtime = 0; 2498 p->se.start_runtime = 0;
2488 p->se.avg_wakeup = sysctl_sched_wakeup_granularity; 2499 p->se.avg_wakeup = sysctl_sched_wakeup_granularity;
2489 p->se.avg_running = 0;
2490 2500
2491#ifdef CONFIG_SCHEDSTATS 2501#ifdef CONFIG_SCHEDSTATS
2492 p->se.wait_start = 0; 2502 p->se.wait_start = 0;
@@ -2508,7 +2518,6 @@ static void __sched_fork(struct task_struct *p)
2508 p->se.nr_failed_migrations_running = 0; 2518 p->se.nr_failed_migrations_running = 0;
2509 p->se.nr_failed_migrations_hot = 0; 2519 p->se.nr_failed_migrations_hot = 0;
2510 p->se.nr_forced_migrations = 0; 2520 p->se.nr_forced_migrations = 0;
2511 p->se.nr_forced2_migrations = 0;
2512 2521
2513 p->se.nr_wakeups = 0; 2522 p->se.nr_wakeups = 0;
2514 p->se.nr_wakeups_sync = 0; 2523 p->se.nr_wakeups_sync = 0;
@@ -2578,8 +2587,11 @@ void sched_fork(struct task_struct *p, int clone_flags)
2578 if (!rt_prio(p->prio)) 2587 if (!rt_prio(p->prio))
2579 p->sched_class = &fair_sched_class; 2588 p->sched_class = &fair_sched_class;
2580 2589
2590 if (p->sched_class->task_fork)
2591 p->sched_class->task_fork(p);
2592
2581#ifdef CONFIG_SMP 2593#ifdef CONFIG_SMP
2582 cpu = p->sched_class->select_task_rq(p, SD_BALANCE_FORK, 0); 2594 cpu = select_task_rq(p, SD_BALANCE_FORK, 0);
2583#endif 2595#endif
2584 set_task_cpu(p, cpu); 2596 set_task_cpu(p, cpu);
2585 2597
@@ -2614,17 +2626,7 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
2614 rq = task_rq_lock(p, &flags); 2626 rq = task_rq_lock(p, &flags);
2615 BUG_ON(p->state != TASK_RUNNING); 2627 BUG_ON(p->state != TASK_RUNNING);
2616 update_rq_clock(rq); 2628 update_rq_clock(rq);
2617 2629 activate_task(rq, p, 0);
2618 if (!p->sched_class->task_new || !current->se.on_rq) {
2619 activate_task(rq, p, 0);
2620 } else {
2621 /*
2622 * Let the scheduling class do new task startup
2623 * management (if any):
2624 */
2625 p->sched_class->task_new(rq, p);
2626 inc_nr_running(rq);
2627 }
2628 trace_sched_wakeup_new(rq, p, 1); 2630 trace_sched_wakeup_new(rq, p, 1);
2629 check_preempt_curr(rq, p, WF_FORK); 2631 check_preempt_curr(rq, p, WF_FORK);
2630#ifdef CONFIG_SMP 2632#ifdef CONFIG_SMP
@@ -2781,10 +2783,10 @@ static inline void post_schedule(struct rq *rq)
2781 if (rq->post_schedule) { 2783 if (rq->post_schedule) {
2782 unsigned long flags; 2784 unsigned long flags;
2783 2785
2784 spin_lock_irqsave(&rq->lock, flags); 2786 raw_spin_lock_irqsave(&rq->lock, flags);
2785 if (rq->curr->sched_class->post_schedule) 2787 if (rq->curr->sched_class->post_schedule)
2786 rq->curr->sched_class->post_schedule(rq); 2788 rq->curr->sched_class->post_schedule(rq);
2787 spin_unlock_irqrestore(&rq->lock, flags); 2789 raw_spin_unlock_irqrestore(&rq->lock, flags);
2788 2790
2789 rq->post_schedule = 0; 2791 rq->post_schedule = 0;
2790 } 2792 }
@@ -2848,14 +2850,14 @@ context_switch(struct rq *rq, struct task_struct *prev,
2848 */ 2850 */
2849 arch_start_context_switch(prev); 2851 arch_start_context_switch(prev);
2850 2852
2851 if (unlikely(!mm)) { 2853 if (likely(!mm)) {
2852 next->active_mm = oldmm; 2854 next->active_mm = oldmm;
2853 atomic_inc(&oldmm->mm_count); 2855 atomic_inc(&oldmm->mm_count);
2854 enter_lazy_tlb(oldmm, next); 2856 enter_lazy_tlb(oldmm, next);
2855 } else 2857 } else
2856 switch_mm(oldmm, mm, next); 2858 switch_mm(oldmm, mm, next);
2857 2859
2858 if (unlikely(!prev->mm)) { 2860 if (likely(!prev->mm)) {
2859 prev->active_mm = NULL; 2861 prev->active_mm = NULL;
2860 rq->prev_mm = oldmm; 2862 rq->prev_mm = oldmm;
2861 } 2863 }
@@ -3018,15 +3020,6 @@ static void calc_load_account_active(struct rq *this_rq)
3018} 3020}
3019 3021
3020/* 3022/*
3021 * Externally visible per-cpu scheduler statistics:
3022 * cpu_nr_migrations(cpu) - number of migrations into that cpu
3023 */
3024u64 cpu_nr_migrations(int cpu)
3025{
3026 return cpu_rq(cpu)->nr_migrations_in;
3027}
3028
3029/*
3030 * Update rq->cpu_load[] statistics. This function is usually called every 3023 * Update rq->cpu_load[] statistics. This function is usually called every
3031 * scheduler tick (TICK_NSEC). 3024 * scheduler tick (TICK_NSEC).
3032 */ 3025 */
@@ -3075,15 +3068,15 @@ static void double_rq_lock(struct rq *rq1, struct rq *rq2)
3075{ 3068{
3076 BUG_ON(!irqs_disabled()); 3069 BUG_ON(!irqs_disabled());
3077 if (rq1 == rq2) { 3070 if (rq1 == rq2) {
3078 spin_lock(&rq1->lock); 3071 raw_spin_lock(&rq1->lock);
3079 __acquire(rq2->lock); /* Fake it out ;) */ 3072 __acquire(rq2->lock); /* Fake it out ;) */
3080 } else { 3073 } else {
3081 if (rq1 < rq2) { 3074 if (rq1 < rq2) {
3082 spin_lock(&rq1->lock); 3075 raw_spin_lock(&rq1->lock);
3083 spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING); 3076 raw_spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING);
3084 } else { 3077 } else {
3085 spin_lock(&rq2->lock); 3078 raw_spin_lock(&rq2->lock);
3086 spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING); 3079 raw_spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING);
3087 } 3080 }
3088 } 3081 }
3089 update_rq_clock(rq1); 3082 update_rq_clock(rq1);
@@ -3100,9 +3093,9 @@ static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
3100 __releases(rq1->lock) 3093 __releases(rq1->lock)
3101 __releases(rq2->lock) 3094 __releases(rq2->lock)
3102{ 3095{
3103 spin_unlock(&rq1->lock); 3096 raw_spin_unlock(&rq1->lock);
3104 if (rq1 != rq2) 3097 if (rq1 != rq2)
3105 spin_unlock(&rq2->lock); 3098 raw_spin_unlock(&rq2->lock);
3106 else 3099 else
3107 __release(rq2->lock); 3100 __release(rq2->lock);
3108} 3101}
@@ -3148,7 +3141,7 @@ out:
3148void sched_exec(void) 3141void sched_exec(void)
3149{ 3142{
3150 int new_cpu, this_cpu = get_cpu(); 3143 int new_cpu, this_cpu = get_cpu();
3151 new_cpu = current->sched_class->select_task_rq(current, SD_BALANCE_EXEC, 0); 3144 new_cpu = select_task_rq(current, SD_BALANCE_EXEC, 0);
3152 put_cpu(); 3145 put_cpu();
3153 if (new_cpu != this_cpu) 3146 if (new_cpu != this_cpu)
3154 sched_migrate_task(current, new_cpu); 3147 sched_migrate_task(current, new_cpu);
@@ -3164,10 +3157,6 @@ static void pull_task(struct rq *src_rq, struct task_struct *p,
3164 deactivate_task(src_rq, p, 0); 3157 deactivate_task(src_rq, p, 0);
3165 set_task_cpu(p, this_cpu); 3158 set_task_cpu(p, this_cpu);
3166 activate_task(this_rq, p, 0); 3159 activate_task(this_rq, p, 0);
3167 /*
3168 * Note that idle threads have a prio of MAX_PRIO, for this test
3169 * to be always true for them.
3170 */
3171 check_preempt_curr(this_rq, p, 0); 3160 check_preempt_curr(this_rq, p, 0);
3172} 3161}
3173 3162
@@ -4126,7 +4115,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
4126 unsigned long flags; 4115 unsigned long flags;
4127 struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask); 4116 struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
4128 4117
4129 cpumask_setall(cpus); 4118 cpumask_copy(cpus, cpu_active_mask);
4130 4119
4131 /* 4120 /*
4132 * When power savings policy is enabled for the parent domain, idle 4121 * When power savings policy is enabled for the parent domain, idle
@@ -4199,14 +4188,15 @@ redo:
4199 4188
4200 if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) { 4189 if (unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2)) {
4201 4190
4202 spin_lock_irqsave(&busiest->lock, flags); 4191 raw_spin_lock_irqsave(&busiest->lock, flags);
4203 4192
4204 /* don't kick the migration_thread, if the curr 4193 /* don't kick the migration_thread, if the curr
4205 * task on busiest cpu can't be moved to this_cpu 4194 * task on busiest cpu can't be moved to this_cpu
4206 */ 4195 */
4207 if (!cpumask_test_cpu(this_cpu, 4196 if (!cpumask_test_cpu(this_cpu,
4208 &busiest->curr->cpus_allowed)) { 4197 &busiest->curr->cpus_allowed)) {
4209 spin_unlock_irqrestore(&busiest->lock, flags); 4198 raw_spin_unlock_irqrestore(&busiest->lock,
4199 flags);
4210 all_pinned = 1; 4200 all_pinned = 1;
4211 goto out_one_pinned; 4201 goto out_one_pinned;
4212 } 4202 }
@@ -4216,7 +4206,7 @@ redo:
4216 busiest->push_cpu = this_cpu; 4206 busiest->push_cpu = this_cpu;
4217 active_balance = 1; 4207 active_balance = 1;
4218 } 4208 }
4219 spin_unlock_irqrestore(&busiest->lock, flags); 4209 raw_spin_unlock_irqrestore(&busiest->lock, flags);
4220 if (active_balance) 4210 if (active_balance)
4221 wake_up_process(busiest->migration_thread); 4211 wake_up_process(busiest->migration_thread);
4222 4212
@@ -4289,7 +4279,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd)
4289 int all_pinned = 0; 4279 int all_pinned = 0;
4290 struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask); 4280 struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
4291 4281
4292 cpumask_setall(cpus); 4282 cpumask_copy(cpus, cpu_active_mask);
4293 4283
4294 /* 4284 /*
4295 * When power savings policy is enabled for the parent domain, idle 4285 * When power savings policy is enabled for the parent domain, idle
@@ -4398,10 +4388,10 @@ redo:
4398 /* 4388 /*
4399 * Should not call ttwu while holding a rq->lock 4389 * Should not call ttwu while holding a rq->lock
4400 */ 4390 */
4401 spin_unlock(&this_rq->lock); 4391 raw_spin_unlock(&this_rq->lock);
4402 if (active_balance) 4392 if (active_balance)
4403 wake_up_process(busiest->migration_thread); 4393 wake_up_process(busiest->migration_thread);
4404 spin_lock(&this_rq->lock); 4394 raw_spin_lock(&this_rq->lock);
4405 4395
4406 } else 4396 } else
4407 sd->nr_balance_failed = 0; 4397 sd->nr_balance_failed = 0;
@@ -4429,6 +4419,11 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
4429 int pulled_task = 0; 4419 int pulled_task = 0;
4430 unsigned long next_balance = jiffies + HZ; 4420 unsigned long next_balance = jiffies + HZ;
4431 4421
4422 this_rq->idle_stamp = this_rq->clock;
4423
4424 if (this_rq->avg_idle < sysctl_sched_migration_cost)
4425 return;
4426
4432 for_each_domain(this_cpu, sd) { 4427 for_each_domain(this_cpu, sd) {
4433 unsigned long interval; 4428 unsigned long interval;
4434 4429
@@ -4443,8 +4438,10 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
4443 interval = msecs_to_jiffies(sd->balance_interval); 4438 interval = msecs_to_jiffies(sd->balance_interval);
4444 if (time_after(next_balance, sd->last_balance + interval)) 4439 if (time_after(next_balance, sd->last_balance + interval))
4445 next_balance = sd->last_balance + interval; 4440 next_balance = sd->last_balance + interval;
4446 if (pulled_task) 4441 if (pulled_task) {
4442 this_rq->idle_stamp = 0;
4447 break; 4443 break;
4444 }
4448 } 4445 }
4449 if (pulled_task || time_after(jiffies, this_rq->next_balance)) { 4446 if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
4450 /* 4447 /*
@@ -4679,7 +4676,7 @@ int select_nohz_load_balancer(int stop_tick)
4679 cpumask_set_cpu(cpu, nohz.cpu_mask); 4676 cpumask_set_cpu(cpu, nohz.cpu_mask);
4680 4677
4681 /* time for ilb owner also to sleep */ 4678 /* time for ilb owner also to sleep */
4682 if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) { 4679 if (cpumask_weight(nohz.cpu_mask) == num_active_cpus()) {
4683 if (atomic_read(&nohz.load_balancer) == cpu) 4680 if (atomic_read(&nohz.load_balancer) == cpu)
4684 atomic_set(&nohz.load_balancer, -1); 4681 atomic_set(&nohz.load_balancer, -1);
4685 return 0; 4682 return 0;
@@ -5046,8 +5043,13 @@ static void account_guest_time(struct task_struct *p, cputime_t cputime,
5046 p->gtime = cputime_add(p->gtime, cputime); 5043 p->gtime = cputime_add(p->gtime, cputime);
5047 5044
5048 /* Add guest time to cpustat. */ 5045 /* Add guest time to cpustat. */
5049 cpustat->user = cputime64_add(cpustat->user, tmp); 5046 if (TASK_NICE(p) > 0) {
5050 cpustat->guest = cputime64_add(cpustat->guest, tmp); 5047 cpustat->nice = cputime64_add(cpustat->nice, tmp);
5048 cpustat->guest_nice = cputime64_add(cpustat->guest_nice, tmp);
5049 } else {
5050 cpustat->user = cputime64_add(cpustat->user, tmp);
5051 cpustat->guest = cputime64_add(cpustat->guest, tmp);
5052 }
5051} 5053}
5052 5054
5053/* 5055/*
@@ -5162,60 +5164,86 @@ void account_idle_ticks(unsigned long ticks)
5162 * Use precise platform statistics if available: 5164 * Use precise platform statistics if available:
5163 */ 5165 */
5164#ifdef CONFIG_VIRT_CPU_ACCOUNTING 5166#ifdef CONFIG_VIRT_CPU_ACCOUNTING
5165cputime_t task_utime(struct task_struct *p) 5167void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
5166{ 5168{
5167 return p->utime; 5169 *ut = p->utime;
5170 *st = p->stime;
5168} 5171}
5169 5172
5170cputime_t task_stime(struct task_struct *p) 5173void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
5171{ 5174{
5172 return p->stime; 5175 struct task_cputime cputime;
5176
5177 thread_group_cputime(p, &cputime);
5178
5179 *ut = cputime.utime;
5180 *st = cputime.stime;
5173} 5181}
5174#else 5182#else
5175cputime_t task_utime(struct task_struct *p) 5183
5184#ifndef nsecs_to_cputime
5185# define nsecs_to_cputime(__nsecs) nsecs_to_jiffies(__nsecs)
5186#endif
5187
5188void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
5176{ 5189{
5177 clock_t utime = cputime_to_clock_t(p->utime), 5190 cputime_t rtime, utime = p->utime, total = cputime_add(utime, p->stime);
5178 total = utime + cputime_to_clock_t(p->stime);
5179 u64 temp;
5180 5191
5181 /* 5192 /*
5182 * Use CFS's precise accounting: 5193 * Use CFS's precise accounting:
5183 */ 5194 */
5184 temp = (u64)nsec_to_clock_t(p->se.sum_exec_runtime); 5195 rtime = nsecs_to_cputime(p->se.sum_exec_runtime);
5185 5196
5186 if (total) { 5197 if (total) {
5187 temp *= utime; 5198 u64 temp;
5199
5200 temp = (u64)(rtime * utime);
5188 do_div(temp, total); 5201 do_div(temp, total);
5189 } 5202 utime = (cputime_t)temp;
5190 utime = (clock_t)temp; 5203 } else
5204 utime = rtime;
5191 5205
5192 p->prev_utime = max(p->prev_utime, clock_t_to_cputime(utime)); 5206 /*
5193 return p->prev_utime; 5207 * Compare with previous values, to keep monotonicity:
5208 */
5209 p->prev_utime = max(p->prev_utime, utime);
5210 p->prev_stime = max(p->prev_stime, cputime_sub(rtime, p->prev_utime));
5211
5212 *ut = p->prev_utime;
5213 *st = p->prev_stime;
5194} 5214}
5195 5215
5196cputime_t task_stime(struct task_struct *p) 5216/*
5217 * Must be called with siglock held.
5218 */
5219void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st)
5197{ 5220{
5198 clock_t stime; 5221 struct signal_struct *sig = p->signal;
5222 struct task_cputime cputime;
5223 cputime_t rtime, utime, total;
5199 5224
5200 /* 5225 thread_group_cputime(p, &cputime);
5201 * Use CFS's precise accounting. (we subtract utime from
5202 * the total, to make sure the total observed by userspace
5203 * grows monotonically - apps rely on that):
5204 */
5205 stime = nsec_to_clock_t(p->se.sum_exec_runtime) -
5206 cputime_to_clock_t(task_utime(p));
5207 5226
5208 if (stime >= 0) 5227 total = cputime_add(cputime.utime, cputime.stime);
5209 p->prev_stime = max(p->prev_stime, clock_t_to_cputime(stime)); 5228 rtime = nsecs_to_cputime(cputime.sum_exec_runtime);
5210 5229
5211 return p->prev_stime; 5230 if (total) {
5212} 5231 u64 temp;
5213#endif
5214 5232
5215inline cputime_t task_gtime(struct task_struct *p) 5233 temp = (u64)(rtime * cputime.utime);
5216{ 5234 do_div(temp, total);
5217 return p->gtime; 5235 utime = (cputime_t)temp;
5236 } else
5237 utime = rtime;
5238
5239 sig->prev_utime = max(sig->prev_utime, utime);
5240 sig->prev_stime = max(sig->prev_stime,
5241 cputime_sub(rtime, sig->prev_utime));
5242
5243 *ut = sig->prev_utime;
5244 *st = sig->prev_stime;
5218} 5245}
5246#endif
5219 5247
5220/* 5248/*
5221 * This function gets called by the timer code, with HZ frequency. 5249 * This function gets called by the timer code, with HZ frequency.
@@ -5232,11 +5260,11 @@ void scheduler_tick(void)
5232 5260
5233 sched_clock_tick(); 5261 sched_clock_tick();
5234 5262
5235 spin_lock(&rq->lock); 5263 raw_spin_lock(&rq->lock);
5236 update_rq_clock(rq); 5264 update_rq_clock(rq);
5237 update_cpu_load(rq); 5265 update_cpu_load(rq);
5238 curr->sched_class->task_tick(rq, curr, 0); 5266 curr->sched_class->task_tick(rq, curr, 0);
5239 spin_unlock(&rq->lock); 5267 raw_spin_unlock(&rq->lock);
5240 5268
5241 perf_event_task_tick(curr, cpu); 5269 perf_event_task_tick(curr, cpu);
5242 5270
@@ -5350,13 +5378,14 @@ static inline void schedule_debug(struct task_struct *prev)
5350#endif 5378#endif
5351} 5379}
5352 5380
5353static void put_prev_task(struct rq *rq, struct task_struct *p) 5381static void put_prev_task(struct rq *rq, struct task_struct *prev)
5354{ 5382{
5355 u64 runtime = p->se.sum_exec_runtime - p->se.prev_sum_exec_runtime; 5383 if (prev->state == TASK_RUNNING) {
5384 u64 runtime = prev->se.sum_exec_runtime;
5356 5385
5357 update_avg(&p->se.avg_running, runtime); 5386 runtime -= prev->se.prev_sum_exec_runtime;
5387 runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost);
5358 5388
5359 if (p->state == TASK_RUNNING) {
5360 /* 5389 /*
5361 * In order to avoid avg_overlap growing stale when we are 5390 * In order to avoid avg_overlap growing stale when we are
5362 * indeed overlapping and hence not getting put to sleep, grow 5391 * indeed overlapping and hence not getting put to sleep, grow
@@ -5366,12 +5395,9 @@ static void put_prev_task(struct rq *rq, struct task_struct *p)
5366 * correlates to the amount of cache footprint a task can 5395 * correlates to the amount of cache footprint a task can
5367 * build up. 5396 * build up.
5368 */ 5397 */
5369 runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost); 5398 update_avg(&prev->se.avg_overlap, runtime);
5370 update_avg(&p->se.avg_overlap, runtime);
5371 } else {
5372 update_avg(&p->se.avg_running, 0);
5373 } 5399 }
5374 p->sched_class->put_prev_task(rq, p); 5400 prev->sched_class->put_prev_task(rq, prev);
5375} 5401}
5376 5402
5377/* 5403/*
@@ -5432,7 +5458,7 @@ need_resched_nonpreemptible:
5432 if (sched_feat(HRTICK)) 5458 if (sched_feat(HRTICK))
5433 hrtick_clear(rq); 5459 hrtick_clear(rq);
5434 5460
5435 spin_lock_irq(&rq->lock); 5461 raw_spin_lock_irq(&rq->lock);
5436 update_rq_clock(rq); 5462 update_rq_clock(rq);
5437 clear_tsk_need_resched(prev); 5463 clear_tsk_need_resched(prev);
5438 5464
@@ -5468,7 +5494,7 @@ need_resched_nonpreemptible:
5468 cpu = smp_processor_id(); 5494 cpu = smp_processor_id();
5469 rq = cpu_rq(cpu); 5495 rq = cpu_rq(cpu);
5470 } else 5496 } else
5471 spin_unlock_irq(&rq->lock); 5497 raw_spin_unlock_irq(&rq->lock);
5472 5498
5473 post_schedule(rq); 5499 post_schedule(rq);
5474 5500
@@ -5481,7 +5507,7 @@ need_resched_nonpreemptible:
5481} 5507}
5482EXPORT_SYMBOL(schedule); 5508EXPORT_SYMBOL(schedule);
5483 5509
5484#ifdef CONFIG_SMP 5510#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
5485/* 5511/*
5486 * Look out! "owner" is an entirely speculative pointer 5512 * Look out! "owner" is an entirely speculative pointer
5487 * access and not reliable. 5513 * access and not reliable.
@@ -6175,22 +6201,14 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
6175 BUG_ON(p->se.on_rq); 6201 BUG_ON(p->se.on_rq);
6176 6202
6177 p->policy = policy; 6203 p->policy = policy;
6178 switch (p->policy) {
6179 case SCHED_NORMAL:
6180 case SCHED_BATCH:
6181 case SCHED_IDLE:
6182 p->sched_class = &fair_sched_class;
6183 break;
6184 case SCHED_FIFO:
6185 case SCHED_RR:
6186 p->sched_class = &rt_sched_class;
6187 break;
6188 }
6189
6190 p->rt_priority = prio; 6204 p->rt_priority = prio;
6191 p->normal_prio = normal_prio(p); 6205 p->normal_prio = normal_prio(p);
6192 /* we are holding p->pi_lock already */ 6206 /* we are holding p->pi_lock already */
6193 p->prio = rt_mutex_getprio(p); 6207 p->prio = rt_mutex_getprio(p);
6208 if (rt_prio(p->prio))
6209 p->sched_class = &rt_sched_class;
6210 else
6211 p->sched_class = &fair_sched_class;
6194 set_load_weight(p); 6212 set_load_weight(p);
6195} 6213}
6196 6214
@@ -6305,7 +6323,7 @@ recheck:
6305 * make sure no PI-waiters arrive (or leave) while we are 6323 * make sure no PI-waiters arrive (or leave) while we are
6306 * changing the priority of the task: 6324 * changing the priority of the task:
6307 */ 6325 */
6308 spin_lock_irqsave(&p->pi_lock, flags); 6326 raw_spin_lock_irqsave(&p->pi_lock, flags);
6309 /* 6327 /*
6310 * To be able to change p->policy safely, the apropriate 6328 * To be able to change p->policy safely, the apropriate
6311 * runqueue lock must be held. 6329 * runqueue lock must be held.
@@ -6315,7 +6333,7 @@ recheck:
6315 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { 6333 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
6316 policy = oldpolicy = -1; 6334 policy = oldpolicy = -1;
6317 __task_rq_unlock(rq); 6335 __task_rq_unlock(rq);
6318 spin_unlock_irqrestore(&p->pi_lock, flags); 6336 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
6319 goto recheck; 6337 goto recheck;
6320 } 6338 }
6321 update_rq_clock(rq); 6339 update_rq_clock(rq);
@@ -6339,7 +6357,7 @@ recheck:
6339 check_class_changed(rq, p, prev_class, oldprio, running); 6357 check_class_changed(rq, p, prev_class, oldprio, running);
6340 } 6358 }
6341 __task_rq_unlock(rq); 6359 __task_rq_unlock(rq);
6342 spin_unlock_irqrestore(&p->pi_lock, flags); 6360 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
6343 6361
6344 rt_mutex_adjust_pi(p); 6362 rt_mutex_adjust_pi(p);
6345 6363
@@ -6593,6 +6611,8 @@ SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
6593long sched_getaffinity(pid_t pid, struct cpumask *mask) 6611long sched_getaffinity(pid_t pid, struct cpumask *mask)
6594{ 6612{
6595 struct task_struct *p; 6613 struct task_struct *p;
6614 unsigned long flags;
6615 struct rq *rq;
6596 int retval; 6616 int retval;
6597 6617
6598 get_online_cpus(); 6618 get_online_cpus();
@@ -6607,7 +6627,9 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask)
6607 if (retval) 6627 if (retval)
6608 goto out_unlock; 6628 goto out_unlock;
6609 6629
6630 rq = task_rq_lock(p, &flags);
6610 cpumask_and(mask, &p->cpus_allowed, cpu_online_mask); 6631 cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
6632 task_rq_unlock(rq, &flags);
6611 6633
6612out_unlock: 6634out_unlock:
6613 read_unlock(&tasklist_lock); 6635 read_unlock(&tasklist_lock);
@@ -6665,7 +6687,7 @@ SYSCALL_DEFINE0(sched_yield)
6665 */ 6687 */
6666 __release(rq->lock); 6688 __release(rq->lock);
6667 spin_release(&rq->lock.dep_map, 1, _THIS_IP_); 6689 spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
6668 _raw_spin_unlock(&rq->lock); 6690 do_raw_spin_unlock(&rq->lock);
6669 preempt_enable_no_resched(); 6691 preempt_enable_no_resched();
6670 6692
6671 schedule(); 6693 schedule();
@@ -6845,6 +6867,8 @@ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
6845{ 6867{
6846 struct task_struct *p; 6868 struct task_struct *p;
6847 unsigned int time_slice; 6869 unsigned int time_slice;
6870 unsigned long flags;
6871 struct rq *rq;
6848 int retval; 6872 int retval;
6849 struct timespec t; 6873 struct timespec t;
6850 6874
@@ -6861,7 +6885,9 @@ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
6861 if (retval) 6885 if (retval)
6862 goto out_unlock; 6886 goto out_unlock;
6863 6887
6864 time_slice = p->sched_class->get_rr_interval(p); 6888 rq = task_rq_lock(p, &flags);
6889 time_slice = p->sched_class->get_rr_interval(rq, p);
6890 task_rq_unlock(rq, &flags);
6865 6891
6866 read_unlock(&tasklist_lock); 6892 read_unlock(&tasklist_lock);
6867 jiffies_to_timespec(time_slice, &t); 6893 jiffies_to_timespec(time_slice, &t);
@@ -6935,7 +6961,7 @@ void show_state_filter(unsigned long state_filter)
6935 /* 6961 /*
6936 * Only show locks if all tasks are dumped: 6962 * Only show locks if all tasks are dumped:
6937 */ 6963 */
6938 if (state_filter == -1) 6964 if (!state_filter)
6939 debug_show_all_locks(); 6965 debug_show_all_locks();
6940} 6966}
6941 6967
@@ -6957,12 +6983,11 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
6957 struct rq *rq = cpu_rq(cpu); 6983 struct rq *rq = cpu_rq(cpu);
6958 unsigned long flags; 6984 unsigned long flags;
6959 6985
6960 spin_lock_irqsave(&rq->lock, flags); 6986 raw_spin_lock_irqsave(&rq->lock, flags);
6961 6987
6962 __sched_fork(idle); 6988 __sched_fork(idle);
6963 idle->se.exec_start = sched_clock(); 6989 idle->se.exec_start = sched_clock();
6964 6990
6965 idle->prio = idle->normal_prio = MAX_PRIO;
6966 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu)); 6991 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
6967 __set_task_cpu(idle, cpu); 6992 __set_task_cpu(idle, cpu);
6968 6993
@@ -6970,7 +6995,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
6970#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW) 6995#if defined(CONFIG_SMP) && defined(__ARCH_WANT_UNLOCKED_CTXSW)
6971 idle->oncpu = 1; 6996 idle->oncpu = 1;
6972#endif 6997#endif
6973 spin_unlock_irqrestore(&rq->lock, flags); 6998 raw_spin_unlock_irqrestore(&rq->lock, flags);
6974 6999
6975 /* Set the preempt count _outside_ the spinlocks! */ 7000 /* Set the preempt count _outside_ the spinlocks! */
6976#if defined(CONFIG_PREEMPT) 7001#if defined(CONFIG_PREEMPT)
@@ -7003,22 +7028,43 @@ cpumask_var_t nohz_cpu_mask;
7003 * 7028 *
7004 * This idea comes from the SD scheduler of Con Kolivas: 7029 * This idea comes from the SD scheduler of Con Kolivas:
7005 */ 7030 */
7006static inline void sched_init_granularity(void) 7031static int get_update_sysctl_factor(void)
7007{ 7032{
7008 unsigned int factor = 1 + ilog2(num_online_cpus()); 7033 unsigned int cpus = min_t(int, num_online_cpus(), 8);
7009 const unsigned long limit = 200000000; 7034 unsigned int factor;
7010 7035
7011 sysctl_sched_min_granularity *= factor; 7036 switch (sysctl_sched_tunable_scaling) {
7012 if (sysctl_sched_min_granularity > limit) 7037 case SCHED_TUNABLESCALING_NONE:
7013 sysctl_sched_min_granularity = limit; 7038 factor = 1;
7039 break;
7040 case SCHED_TUNABLESCALING_LINEAR:
7041 factor = cpus;
7042 break;
7043 case SCHED_TUNABLESCALING_LOG:
7044 default:
7045 factor = 1 + ilog2(cpus);
7046 break;
7047 }
7014 7048
7015 sysctl_sched_latency *= factor; 7049 return factor;
7016 if (sysctl_sched_latency > limit) 7050}
7017 sysctl_sched_latency = limit;
7018 7051
7019 sysctl_sched_wakeup_granularity *= factor; 7052static void update_sysctl(void)
7053{
7054 unsigned int factor = get_update_sysctl_factor();
7055
7056#define SET_SYSCTL(name) \
7057 (sysctl_##name = (factor) * normalized_sysctl_##name)
7058 SET_SYSCTL(sched_min_granularity);
7059 SET_SYSCTL(sched_latency);
7060 SET_SYSCTL(sched_wakeup_granularity);
7061 SET_SYSCTL(sched_shares_ratelimit);
7062#undef SET_SYSCTL
7063}
7020 7064
7021 sysctl_sched_shares_ratelimit *= factor; 7065static inline void sched_init_granularity(void)
7066{
7067 update_sysctl();
7022} 7068}
7023 7069
7024#ifdef CONFIG_SMP 7070#ifdef CONFIG_SMP
@@ -7055,7 +7101,7 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
7055 int ret = 0; 7101 int ret = 0;
7056 7102
7057 rq = task_rq_lock(p, &flags); 7103 rq = task_rq_lock(p, &flags);
7058 if (!cpumask_intersects(new_mask, cpu_online_mask)) { 7104 if (!cpumask_intersects(new_mask, cpu_active_mask)) {
7059 ret = -EINVAL; 7105 ret = -EINVAL;
7060 goto out; 7106 goto out;
7061 } 7107 }
@@ -7077,7 +7123,7 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
7077 if (cpumask_test_cpu(task_cpu(p), new_mask)) 7123 if (cpumask_test_cpu(task_cpu(p), new_mask))
7078 goto out; 7124 goto out;
7079 7125
7080 if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) { 7126 if (migrate_task(p, cpumask_any_and(cpu_active_mask, new_mask), &req)) {
7081 /* Need help from migration thread: drop lock and wait. */ 7127 /* Need help from migration thread: drop lock and wait. */
7082 struct task_struct *mt = rq->migration_thread; 7128 struct task_struct *mt = rq->migration_thread;
7083 7129
@@ -7166,10 +7212,10 @@ static int migration_thread(void *data)
7166 struct migration_req *req; 7212 struct migration_req *req;
7167 struct list_head *head; 7213 struct list_head *head;
7168 7214
7169 spin_lock_irq(&rq->lock); 7215 raw_spin_lock_irq(&rq->lock);
7170 7216
7171 if (cpu_is_offline(cpu)) { 7217 if (cpu_is_offline(cpu)) {
7172 spin_unlock_irq(&rq->lock); 7218 raw_spin_unlock_irq(&rq->lock);
7173 break; 7219 break;
7174 } 7220 }
7175 7221
@@ -7181,7 +7227,7 @@ static int migration_thread(void *data)
7181 head = &rq->migration_queue; 7227 head = &rq->migration_queue;
7182 7228
7183 if (list_empty(head)) { 7229 if (list_empty(head)) {
7184 spin_unlock_irq(&rq->lock); 7230 raw_spin_unlock_irq(&rq->lock);
7185 schedule(); 7231 schedule();
7186 set_current_state(TASK_INTERRUPTIBLE); 7232 set_current_state(TASK_INTERRUPTIBLE);
7187 continue; 7233 continue;
@@ -7190,14 +7236,14 @@ static int migration_thread(void *data)
7190 list_del_init(head->next); 7236 list_del_init(head->next);
7191 7237
7192 if (req->task != NULL) { 7238 if (req->task != NULL) {
7193 spin_unlock(&rq->lock); 7239 raw_spin_unlock(&rq->lock);
7194 __migrate_task(req->task, cpu, req->dest_cpu); 7240 __migrate_task(req->task, cpu, req->dest_cpu);
7195 } else if (likely(cpu == (badcpu = smp_processor_id()))) { 7241 } else if (likely(cpu == (badcpu = smp_processor_id()))) {
7196 req->dest_cpu = RCU_MIGRATION_GOT_QS; 7242 req->dest_cpu = RCU_MIGRATION_GOT_QS;
7197 spin_unlock(&rq->lock); 7243 raw_spin_unlock(&rq->lock);
7198 } else { 7244 } else {
7199 req->dest_cpu = RCU_MIGRATION_MUST_SYNC; 7245 req->dest_cpu = RCU_MIGRATION_MUST_SYNC;
7200 spin_unlock(&rq->lock); 7246 raw_spin_unlock(&rq->lock);
7201 WARN_ONCE(1, "migration_thread() on CPU %d, expected %d\n", badcpu, cpu); 7247 WARN_ONCE(1, "migration_thread() on CPU %d, expected %d\n", badcpu, cpu);
7202 } 7248 }
7203 local_irq_enable(); 7249 local_irq_enable();
@@ -7231,19 +7277,19 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
7231 7277
7232again: 7278again:
7233 /* Look for allowed, online CPU in same node. */ 7279 /* Look for allowed, online CPU in same node. */
7234 for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask) 7280 for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
7235 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed)) 7281 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
7236 goto move; 7282 goto move;
7237 7283
7238 /* Any allowed, online CPU? */ 7284 /* Any allowed, online CPU? */
7239 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask); 7285 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
7240 if (dest_cpu < nr_cpu_ids) 7286 if (dest_cpu < nr_cpu_ids)
7241 goto move; 7287 goto move;
7242 7288
7243 /* No more Mr. Nice Guy. */ 7289 /* No more Mr. Nice Guy. */
7244 if (dest_cpu >= nr_cpu_ids) { 7290 if (dest_cpu >= nr_cpu_ids) {
7245 cpuset_cpus_allowed_locked(p, &p->cpus_allowed); 7291 cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
7246 dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed); 7292 dest_cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
7247 7293
7248 /* 7294 /*
7249 * Don't tell them about moving exiting tasks or 7295 * Don't tell them about moving exiting tasks or
@@ -7272,7 +7318,7 @@ move:
7272 */ 7318 */
7273static void migrate_nr_uninterruptible(struct rq *rq_src) 7319static void migrate_nr_uninterruptible(struct rq *rq_src)
7274{ 7320{
7275 struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask)); 7321 struct rq *rq_dest = cpu_rq(cpumask_any(cpu_active_mask));
7276 unsigned long flags; 7322 unsigned long flags;
7277 7323
7278 local_irq_save(flags); 7324 local_irq_save(flags);
@@ -7320,14 +7366,14 @@ void sched_idle_next(void)
7320 * Strictly not necessary since rest of the CPUs are stopped by now 7366 * Strictly not necessary since rest of the CPUs are stopped by now
7321 * and interrupts disabled on the current cpu. 7367 * and interrupts disabled on the current cpu.
7322 */ 7368 */
7323 spin_lock_irqsave(&rq->lock, flags); 7369 raw_spin_lock_irqsave(&rq->lock, flags);
7324 7370
7325 __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1); 7371 __setscheduler(rq, p, SCHED_FIFO, MAX_RT_PRIO-1);
7326 7372
7327 update_rq_clock(rq); 7373 update_rq_clock(rq);
7328 activate_task(rq, p, 0); 7374 activate_task(rq, p, 0);
7329 7375
7330 spin_unlock_irqrestore(&rq->lock, flags); 7376 raw_spin_unlock_irqrestore(&rq->lock, flags);
7331} 7377}
7332 7378
7333/* 7379/*
@@ -7363,9 +7409,9 @@ static void migrate_dead(unsigned int dead_cpu, struct task_struct *p)
7363 * that's OK. No task can be added to this CPU, so iteration is 7409 * that's OK. No task can be added to this CPU, so iteration is
7364 * fine. 7410 * fine.
7365 */ 7411 */
7366 spin_unlock_irq(&rq->lock); 7412 raw_spin_unlock_irq(&rq->lock);
7367 move_task_off_dead_cpu(dead_cpu, p); 7413 move_task_off_dead_cpu(dead_cpu, p);
7368 spin_lock_irq(&rq->lock); 7414 raw_spin_lock_irq(&rq->lock);
7369 7415
7370 put_task_struct(p); 7416 put_task_struct(p);
7371} 7417}
@@ -7406,17 +7452,16 @@ static struct ctl_table sd_ctl_dir[] = {
7406 .procname = "sched_domain", 7452 .procname = "sched_domain",
7407 .mode = 0555, 7453 .mode = 0555,
7408 }, 7454 },
7409 {0, }, 7455 {}
7410}; 7456};
7411 7457
7412static struct ctl_table sd_ctl_root[] = { 7458static struct ctl_table sd_ctl_root[] = {
7413 { 7459 {
7414 .ctl_name = CTL_KERN,
7415 .procname = "kernel", 7460 .procname = "kernel",
7416 .mode = 0555, 7461 .mode = 0555,
7417 .child = sd_ctl_dir, 7462 .child = sd_ctl_dir,
7418 }, 7463 },
7419 {0, }, 7464 {}
7420}; 7465};
7421 7466
7422static struct ctl_table *sd_alloc_ctl_entry(int n) 7467static struct ctl_table *sd_alloc_ctl_entry(int n)
@@ -7526,7 +7571,7 @@ static ctl_table *sd_alloc_ctl_cpu_table(int cpu)
7526static struct ctl_table_header *sd_sysctl_header; 7571static struct ctl_table_header *sd_sysctl_header;
7527static void register_sched_domain_sysctl(void) 7572static void register_sched_domain_sysctl(void)
7528{ 7573{
7529 int i, cpu_num = num_online_cpus(); 7574 int i, cpu_num = num_possible_cpus();
7530 struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1); 7575 struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
7531 char buf[32]; 7576 char buf[32];
7532 7577
@@ -7536,7 +7581,7 @@ static void register_sched_domain_sysctl(void)
7536 if (entry == NULL) 7581 if (entry == NULL)
7537 return; 7582 return;
7538 7583
7539 for_each_online_cpu(i) { 7584 for_each_possible_cpu(i) {
7540 snprintf(buf, 32, "cpu%d", i); 7585 snprintf(buf, 32, "cpu%d", i);
7541 entry->procname = kstrdup(buf, GFP_KERNEL); 7586 entry->procname = kstrdup(buf, GFP_KERNEL);
7542 entry->mode = 0555; 7587 entry->mode = 0555;
@@ -7632,13 +7677,13 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
7632 7677
7633 /* Update our root-domain */ 7678 /* Update our root-domain */
7634 rq = cpu_rq(cpu); 7679 rq = cpu_rq(cpu);
7635 spin_lock_irqsave(&rq->lock, flags); 7680 raw_spin_lock_irqsave(&rq->lock, flags);
7636 if (rq->rd) { 7681 if (rq->rd) {
7637 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 7682 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7638 7683
7639 set_rq_online(rq); 7684 set_rq_online(rq);
7640 } 7685 }
7641 spin_unlock_irqrestore(&rq->lock, flags); 7686 raw_spin_unlock_irqrestore(&rq->lock, flags);
7642 break; 7687 break;
7643 7688
7644#ifdef CONFIG_HOTPLUG_CPU 7689#ifdef CONFIG_HOTPLUG_CPU
@@ -7663,14 +7708,13 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
7663 put_task_struct(rq->migration_thread); 7708 put_task_struct(rq->migration_thread);
7664 rq->migration_thread = NULL; 7709 rq->migration_thread = NULL;
7665 /* Idle task back to normal (off runqueue, low prio) */ 7710 /* Idle task back to normal (off runqueue, low prio) */
7666 spin_lock_irq(&rq->lock); 7711 raw_spin_lock_irq(&rq->lock);
7667 update_rq_clock(rq); 7712 update_rq_clock(rq);
7668 deactivate_task(rq, rq->idle, 0); 7713 deactivate_task(rq, rq->idle, 0);
7669 rq->idle->static_prio = MAX_PRIO;
7670 __setscheduler(rq, rq->idle, SCHED_NORMAL, 0); 7714 __setscheduler(rq, rq->idle, SCHED_NORMAL, 0);
7671 rq->idle->sched_class = &idle_sched_class; 7715 rq->idle->sched_class = &idle_sched_class;
7672 migrate_dead_tasks(cpu); 7716 migrate_dead_tasks(cpu);
7673 spin_unlock_irq(&rq->lock); 7717 raw_spin_unlock_irq(&rq->lock);
7674 cpuset_unlock(); 7718 cpuset_unlock();
7675 migrate_nr_uninterruptible(rq); 7719 migrate_nr_uninterruptible(rq);
7676 BUG_ON(rq->nr_running != 0); 7720 BUG_ON(rq->nr_running != 0);
@@ -7680,30 +7724,30 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
7680 * they didn't take sched_hotcpu_mutex. Just wake up 7724 * they didn't take sched_hotcpu_mutex. Just wake up
7681 * the requestors. 7725 * the requestors.
7682 */ 7726 */
7683 spin_lock_irq(&rq->lock); 7727 raw_spin_lock_irq(&rq->lock);
7684 while (!list_empty(&rq->migration_queue)) { 7728 while (!list_empty(&rq->migration_queue)) {
7685 struct migration_req *req; 7729 struct migration_req *req;
7686 7730
7687 req = list_entry(rq->migration_queue.next, 7731 req = list_entry(rq->migration_queue.next,
7688 struct migration_req, list); 7732 struct migration_req, list);
7689 list_del_init(&req->list); 7733 list_del_init(&req->list);
7690 spin_unlock_irq(&rq->lock); 7734 raw_spin_unlock_irq(&rq->lock);
7691 complete(&req->done); 7735 complete(&req->done);
7692 spin_lock_irq(&rq->lock); 7736 raw_spin_lock_irq(&rq->lock);
7693 } 7737 }
7694 spin_unlock_irq(&rq->lock); 7738 raw_spin_unlock_irq(&rq->lock);
7695 break; 7739 break;
7696 7740
7697 case CPU_DYING: 7741 case CPU_DYING:
7698 case CPU_DYING_FROZEN: 7742 case CPU_DYING_FROZEN:
7699 /* Update our root-domain */ 7743 /* Update our root-domain */
7700 rq = cpu_rq(cpu); 7744 rq = cpu_rq(cpu);
7701 spin_lock_irqsave(&rq->lock, flags); 7745 raw_spin_lock_irqsave(&rq->lock, flags);
7702 if (rq->rd) { 7746 if (rq->rd) {
7703 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 7747 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7704 set_rq_offline(rq); 7748 set_rq_offline(rq);
7705 } 7749 }
7706 spin_unlock_irqrestore(&rq->lock, flags); 7750 raw_spin_unlock_irqrestore(&rq->lock, flags);
7707 break; 7751 break;
7708#endif 7752#endif
7709 } 7753 }
@@ -7740,6 +7784,16 @@ early_initcall(migration_init);
7740 7784
7741#ifdef CONFIG_SCHED_DEBUG 7785#ifdef CONFIG_SCHED_DEBUG
7742 7786
7787static __read_mostly int sched_domain_debug_enabled;
7788
7789static int __init sched_domain_debug_setup(char *str)
7790{
7791 sched_domain_debug_enabled = 1;
7792
7793 return 0;
7794}
7795early_param("sched_debug", sched_domain_debug_setup);
7796
7743static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 7797static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
7744 struct cpumask *groupmask) 7798 struct cpumask *groupmask)
7745{ 7799{
@@ -7826,6 +7880,9 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
7826 cpumask_var_t groupmask; 7880 cpumask_var_t groupmask;
7827 int level = 0; 7881 int level = 0;
7828 7882
7883 if (!sched_domain_debug_enabled)
7884 return;
7885
7829 if (!sd) { 7886 if (!sd) {
7830 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); 7887 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
7831 return; 7888 return;
@@ -7905,6 +7962,8 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
7905 7962
7906static void free_rootdomain(struct root_domain *rd) 7963static void free_rootdomain(struct root_domain *rd)
7907{ 7964{
7965 synchronize_sched();
7966
7908 cpupri_cleanup(&rd->cpupri); 7967 cpupri_cleanup(&rd->cpupri);
7909 7968
7910 free_cpumask_var(rd->rto_mask); 7969 free_cpumask_var(rd->rto_mask);
@@ -7918,7 +7977,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
7918 struct root_domain *old_rd = NULL; 7977 struct root_domain *old_rd = NULL;
7919 unsigned long flags; 7978 unsigned long flags;
7920 7979
7921 spin_lock_irqsave(&rq->lock, flags); 7980 raw_spin_lock_irqsave(&rq->lock, flags);
7922 7981
7923 if (rq->rd) { 7982 if (rq->rd) {
7924 old_rd = rq->rd; 7983 old_rd = rq->rd;
@@ -7944,7 +8003,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
7944 if (cpumask_test_cpu(rq->cpu, cpu_active_mask)) 8003 if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
7945 set_rq_online(rq); 8004 set_rq_online(rq);
7946 8005
7947 spin_unlock_irqrestore(&rq->lock, flags); 8006 raw_spin_unlock_irqrestore(&rq->lock, flags);
7948 8007
7949 if (old_rd) 8008 if (old_rd)
7950 free_rootdomain(old_rd); 8009 free_rootdomain(old_rd);
@@ -8045,6 +8104,7 @@ static cpumask_var_t cpu_isolated_map;
8045/* Setup the mask of cpus configured for isolated domains */ 8104/* Setup the mask of cpus configured for isolated domains */
8046static int __init isolated_cpu_setup(char *str) 8105static int __init isolated_cpu_setup(char *str)
8047{ 8106{
8107 alloc_bootmem_cpumask_var(&cpu_isolated_map);
8048 cpulist_parse(str, cpu_isolated_map); 8108 cpulist_parse(str, cpu_isolated_map);
8049 return 1; 8109 return 1;
8050} 8110}
@@ -8229,14 +8289,14 @@ enum s_alloc {
8229 */ 8289 */
8230#ifdef CONFIG_SCHED_SMT 8290#ifdef CONFIG_SCHED_SMT
8231static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains); 8291static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
8232static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus); 8292static DEFINE_PER_CPU(struct static_sched_group, sched_groups);
8233 8293
8234static int 8294static int
8235cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map, 8295cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
8236 struct sched_group **sg, struct cpumask *unused) 8296 struct sched_group **sg, struct cpumask *unused)
8237{ 8297{
8238 if (sg) 8298 if (sg)
8239 *sg = &per_cpu(sched_group_cpus, cpu).sg; 8299 *sg = &per_cpu(sched_groups, cpu).sg;
8240 return cpu; 8300 return cpu;
8241} 8301}
8242#endif /* CONFIG_SCHED_SMT */ 8302#endif /* CONFIG_SCHED_SMT */
@@ -8881,7 +8941,7 @@ static int build_sched_domains(const struct cpumask *cpu_map)
8881 return __build_sched_domains(cpu_map, NULL); 8941 return __build_sched_domains(cpu_map, NULL);
8882} 8942}
8883 8943
8884static struct cpumask *doms_cur; /* current sched domains */ 8944static cpumask_var_t *doms_cur; /* current sched domains */
8885static int ndoms_cur; /* number of sched domains in 'doms_cur' */ 8945static int ndoms_cur; /* number of sched domains in 'doms_cur' */
8886static struct sched_domain_attr *dattr_cur; 8946static struct sched_domain_attr *dattr_cur;
8887 /* attribues of custom domains in 'doms_cur' */ 8947 /* attribues of custom domains in 'doms_cur' */
@@ -8903,6 +8963,31 @@ int __attribute__((weak)) arch_update_cpu_topology(void)
8903 return 0; 8963 return 0;
8904} 8964}
8905 8965
8966cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
8967{
8968 int i;
8969 cpumask_var_t *doms;
8970
8971 doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
8972 if (!doms)
8973 return NULL;
8974 for (i = 0; i < ndoms; i++) {
8975 if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
8976 free_sched_domains(doms, i);
8977 return NULL;
8978 }
8979 }
8980 return doms;
8981}
8982
8983void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
8984{
8985 unsigned int i;
8986 for (i = 0; i < ndoms; i++)
8987 free_cpumask_var(doms[i]);
8988 kfree(doms);
8989}
8990
8906/* 8991/*
8907 * Set up scheduler domains and groups. Callers must hold the hotplug lock. 8992 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
8908 * For now this just excludes isolated cpus, but could be used to 8993 * For now this just excludes isolated cpus, but could be used to
@@ -8914,12 +8999,12 @@ static int arch_init_sched_domains(const struct cpumask *cpu_map)
8914 8999
8915 arch_update_cpu_topology(); 9000 arch_update_cpu_topology();
8916 ndoms_cur = 1; 9001 ndoms_cur = 1;
8917 doms_cur = kmalloc(cpumask_size(), GFP_KERNEL); 9002 doms_cur = alloc_sched_domains(ndoms_cur);
8918 if (!doms_cur) 9003 if (!doms_cur)
8919 doms_cur = fallback_doms; 9004 doms_cur = &fallback_doms;
8920 cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map); 9005 cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
8921 dattr_cur = NULL; 9006 dattr_cur = NULL;
8922 err = build_sched_domains(doms_cur); 9007 err = build_sched_domains(doms_cur[0]);
8923 register_sched_domain_sysctl(); 9008 register_sched_domain_sysctl();
8924 9009
8925 return err; 9010 return err;
@@ -8969,19 +9054,19 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
8969 * doms_new[] to the current sched domain partitioning, doms_cur[]. 9054 * doms_new[] to the current sched domain partitioning, doms_cur[].
8970 * It destroys each deleted domain and builds each new domain. 9055 * It destroys each deleted domain and builds each new domain.
8971 * 9056 *
8972 * 'doms_new' is an array of cpumask's of length 'ndoms_new'. 9057 * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
8973 * The masks don't intersect (don't overlap.) We should setup one 9058 * The masks don't intersect (don't overlap.) We should setup one
8974 * sched domain for each mask. CPUs not in any of the cpumasks will 9059 * sched domain for each mask. CPUs not in any of the cpumasks will
8975 * not be load balanced. If the same cpumask appears both in the 9060 * not be load balanced. If the same cpumask appears both in the
8976 * current 'doms_cur' domains and in the new 'doms_new', we can leave 9061 * current 'doms_cur' domains and in the new 'doms_new', we can leave
8977 * it as it is. 9062 * it as it is.
8978 * 9063 *
8979 * The passed in 'doms_new' should be kmalloc'd. This routine takes 9064 * The passed in 'doms_new' should be allocated using
8980 * ownership of it and will kfree it when done with it. If the caller 9065 * alloc_sched_domains. This routine takes ownership of it and will
8981 * failed the kmalloc call, then it can pass in doms_new == NULL && 9066 * free_sched_domains it when done with it. If the caller failed the
8982 * ndoms_new == 1, and partition_sched_domains() will fallback to 9067 * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
8983 * the single partition 'fallback_doms', it also forces the domains 9068 * and partition_sched_domains() will fallback to the single partition
8984 * to be rebuilt. 9069 * 'fallback_doms', it also forces the domains to be rebuilt.
8985 * 9070 *
8986 * If doms_new == NULL it will be replaced with cpu_online_mask. 9071 * If doms_new == NULL it will be replaced with cpu_online_mask.
8987 * ndoms_new == 0 is a special case for destroying existing domains, 9072 * ndoms_new == 0 is a special case for destroying existing domains,
@@ -8989,8 +9074,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
8989 * 9074 *
8990 * Call with hotplug lock held 9075 * Call with hotplug lock held
8991 */ 9076 */
8992/* FIXME: Change to struct cpumask *doms_new[] */ 9077void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
8993void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
8994 struct sched_domain_attr *dattr_new) 9078 struct sched_domain_attr *dattr_new)
8995{ 9079{
8996 int i, j, n; 9080 int i, j, n;
@@ -9009,40 +9093,40 @@ void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
9009 /* Destroy deleted domains */ 9093 /* Destroy deleted domains */
9010 for (i = 0; i < ndoms_cur; i++) { 9094 for (i = 0; i < ndoms_cur; i++) {
9011 for (j = 0; j < n && !new_topology; j++) { 9095 for (j = 0; j < n && !new_topology; j++) {
9012 if (cpumask_equal(&doms_cur[i], &doms_new[j]) 9096 if (cpumask_equal(doms_cur[i], doms_new[j])
9013 && dattrs_equal(dattr_cur, i, dattr_new, j)) 9097 && dattrs_equal(dattr_cur, i, dattr_new, j))
9014 goto match1; 9098 goto match1;
9015 } 9099 }
9016 /* no match - a current sched domain not in new doms_new[] */ 9100 /* no match - a current sched domain not in new doms_new[] */
9017 detach_destroy_domains(doms_cur + i); 9101 detach_destroy_domains(doms_cur[i]);
9018match1: 9102match1:
9019 ; 9103 ;
9020 } 9104 }
9021 9105
9022 if (doms_new == NULL) { 9106 if (doms_new == NULL) {
9023 ndoms_cur = 0; 9107 ndoms_cur = 0;
9024 doms_new = fallback_doms; 9108 doms_new = &fallback_doms;
9025 cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map); 9109 cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
9026 WARN_ON_ONCE(dattr_new); 9110 WARN_ON_ONCE(dattr_new);
9027 } 9111 }
9028 9112
9029 /* Build new domains */ 9113 /* Build new domains */
9030 for (i = 0; i < ndoms_new; i++) { 9114 for (i = 0; i < ndoms_new; i++) {
9031 for (j = 0; j < ndoms_cur && !new_topology; j++) { 9115 for (j = 0; j < ndoms_cur && !new_topology; j++) {
9032 if (cpumask_equal(&doms_new[i], &doms_cur[j]) 9116 if (cpumask_equal(doms_new[i], doms_cur[j])
9033 && dattrs_equal(dattr_new, i, dattr_cur, j)) 9117 && dattrs_equal(dattr_new, i, dattr_cur, j))
9034 goto match2; 9118 goto match2;
9035 } 9119 }
9036 /* no match - add a new doms_new */ 9120 /* no match - add a new doms_new */
9037 __build_sched_domains(doms_new + i, 9121 __build_sched_domains(doms_new[i],
9038 dattr_new ? dattr_new + i : NULL); 9122 dattr_new ? dattr_new + i : NULL);
9039match2: 9123match2:
9040 ; 9124 ;
9041 } 9125 }
9042 9126
9043 /* Remember the new sched domains */ 9127 /* Remember the new sched domains */
9044 if (doms_cur != fallback_doms) 9128 if (doms_cur != &fallback_doms)
9045 kfree(doms_cur); 9129 free_sched_domains(doms_cur, ndoms_cur);
9046 kfree(dattr_cur); /* kfree(NULL) is safe */ 9130 kfree(dattr_cur); /* kfree(NULL) is safe */
9047 doms_cur = doms_new; 9131 doms_cur = doms_new;
9048 dattr_cur = dattr_new; 9132 dattr_cur = dattr_new;
@@ -9153,8 +9237,10 @@ static int update_sched_domains(struct notifier_block *nfb,
9153 switch (action) { 9237 switch (action) {
9154 case CPU_ONLINE: 9238 case CPU_ONLINE:
9155 case CPU_ONLINE_FROZEN: 9239 case CPU_ONLINE_FROZEN:
9156 case CPU_DEAD: 9240 case CPU_DOWN_PREPARE:
9157 case CPU_DEAD_FROZEN: 9241 case CPU_DOWN_PREPARE_FROZEN:
9242 case CPU_DOWN_FAILED:
9243 case CPU_DOWN_FAILED_FROZEN:
9158 partition_sched_domains(1, NULL, NULL); 9244 partition_sched_domains(1, NULL, NULL);
9159 return NOTIFY_OK; 9245 return NOTIFY_OK;
9160 9246
@@ -9201,7 +9287,7 @@ void __init sched_init_smp(void)
9201#endif 9287#endif
9202 get_online_cpus(); 9288 get_online_cpus();
9203 mutex_lock(&sched_domains_mutex); 9289 mutex_lock(&sched_domains_mutex);
9204 arch_init_sched_domains(cpu_online_mask); 9290 arch_init_sched_domains(cpu_active_mask);
9205 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map); 9291 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
9206 if (cpumask_empty(non_isolated_cpus)) 9292 if (cpumask_empty(non_isolated_cpus))
9207 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus); 9293 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
@@ -9274,13 +9360,13 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
9274#ifdef CONFIG_SMP 9360#ifdef CONFIG_SMP
9275 rt_rq->rt_nr_migratory = 0; 9361 rt_rq->rt_nr_migratory = 0;
9276 rt_rq->overloaded = 0; 9362 rt_rq->overloaded = 0;
9277 plist_head_init(&rt_rq->pushable_tasks, &rq->lock); 9363 plist_head_init_raw(&rt_rq->pushable_tasks, &rq->lock);
9278#endif 9364#endif
9279 9365
9280 rt_rq->rt_time = 0; 9366 rt_rq->rt_time = 0;
9281 rt_rq->rt_throttled = 0; 9367 rt_rq->rt_throttled = 0;
9282 rt_rq->rt_runtime = 0; 9368 rt_rq->rt_runtime = 0;
9283 spin_lock_init(&rt_rq->rt_runtime_lock); 9369 raw_spin_lock_init(&rt_rq->rt_runtime_lock);
9284 9370
9285#ifdef CONFIG_RT_GROUP_SCHED 9371#ifdef CONFIG_RT_GROUP_SCHED
9286 rt_rq->rt_nr_boosted = 0; 9372 rt_rq->rt_nr_boosted = 0;
@@ -9364,10 +9450,6 @@ void __init sched_init(void)
9364#ifdef CONFIG_CPUMASK_OFFSTACK 9450#ifdef CONFIG_CPUMASK_OFFSTACK
9365 alloc_size += num_possible_cpus() * cpumask_size(); 9451 alloc_size += num_possible_cpus() * cpumask_size();
9366#endif 9452#endif
9367 /*
9368 * As sched_init() is called before page_alloc is setup,
9369 * we use alloc_bootmem().
9370 */
9371 if (alloc_size) { 9453 if (alloc_size) {
9372 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT); 9454 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
9373 9455
@@ -9444,7 +9526,7 @@ void __init sched_init(void)
9444 struct rq *rq; 9526 struct rq *rq;
9445 9527
9446 rq = cpu_rq(i); 9528 rq = cpu_rq(i);
9447 spin_lock_init(&rq->lock); 9529 raw_spin_lock_init(&rq->lock);
9448 rq->nr_running = 0; 9530 rq->nr_running = 0;
9449 rq->calc_load_active = 0; 9531 rq->calc_load_active = 0;
9450 rq->calc_load_update = jiffies + LOAD_FREQ; 9532 rq->calc_load_update = jiffies + LOAD_FREQ;
@@ -9504,7 +9586,7 @@ void __init sched_init(void)
9504#elif defined CONFIG_USER_SCHED 9586#elif defined CONFIG_USER_SCHED
9505 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, 0, NULL); 9587 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, 0, NULL);
9506 init_tg_rt_entry(&init_task_group, 9588 init_tg_rt_entry(&init_task_group,
9507 &per_cpu(init_rt_rq, i), 9589 &per_cpu(init_rt_rq_var, i),
9508 &per_cpu(init_sched_rt_entity, i), i, 1, 9590 &per_cpu(init_sched_rt_entity, i), i, 1,
9509 root_task_group.rt_se[i]); 9591 root_task_group.rt_se[i]);
9510#endif 9592#endif
@@ -9522,6 +9604,8 @@ void __init sched_init(void)
9522 rq->cpu = i; 9604 rq->cpu = i;
9523 rq->online = 0; 9605 rq->online = 0;
9524 rq->migration_thread = NULL; 9606 rq->migration_thread = NULL;
9607 rq->idle_stamp = 0;
9608 rq->avg_idle = 2*sysctl_sched_migration_cost;
9525 INIT_LIST_HEAD(&rq->migration_queue); 9609 INIT_LIST_HEAD(&rq->migration_queue);
9526 rq_attach_root(rq, &def_root_domain); 9610 rq_attach_root(rq, &def_root_domain);
9527#endif 9611#endif
@@ -9540,7 +9624,7 @@ void __init sched_init(void)
9540#endif 9624#endif
9541 9625
9542#ifdef CONFIG_RT_MUTEXES 9626#ifdef CONFIG_RT_MUTEXES
9543 plist_head_init(&init_task.pi_waiters, &init_task.pi_lock); 9627 plist_head_init_raw(&init_task.pi_waiters, &init_task.pi_lock);
9544#endif 9628#endif
9545 9629
9546 /* 9630 /*
@@ -9571,7 +9655,9 @@ void __init sched_init(void)
9571 zalloc_cpumask_var(&nohz.cpu_mask, GFP_NOWAIT); 9655 zalloc_cpumask_var(&nohz.cpu_mask, GFP_NOWAIT);
9572 alloc_cpumask_var(&nohz.ilb_grp_nohz_mask, GFP_NOWAIT); 9656 alloc_cpumask_var(&nohz.ilb_grp_nohz_mask, GFP_NOWAIT);
9573#endif 9657#endif
9574 zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT); 9658 /* May be allocated at isolcpus cmdline parse time */
9659 if (cpu_isolated_map == NULL)
9660 zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
9575#endif /* SMP */ 9661#endif /* SMP */
9576 9662
9577 perf_event_init(); 9663 perf_event_init();
@@ -9663,13 +9749,13 @@ void normalize_rt_tasks(void)
9663 continue; 9749 continue;
9664 } 9750 }
9665 9751
9666 spin_lock(&p->pi_lock); 9752 raw_spin_lock(&p->pi_lock);
9667 rq = __task_rq_lock(p); 9753 rq = __task_rq_lock(p);
9668 9754
9669 normalize_task(rq, p); 9755 normalize_task(rq, p);
9670 9756
9671 __task_rq_unlock(rq); 9757 __task_rq_unlock(rq);
9672 spin_unlock(&p->pi_lock); 9758 raw_spin_unlock(&p->pi_lock);
9673 } while_each_thread(g, p); 9759 } while_each_thread(g, p);
9674 9760
9675 read_unlock_irqrestore(&tasklist_lock, flags); 9761 read_unlock_irqrestore(&tasklist_lock, flags);
@@ -9765,13 +9851,15 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
9765 se = kzalloc_node(sizeof(struct sched_entity), 9851 se = kzalloc_node(sizeof(struct sched_entity),
9766 GFP_KERNEL, cpu_to_node(i)); 9852 GFP_KERNEL, cpu_to_node(i));
9767 if (!se) 9853 if (!se)
9768 goto err; 9854 goto err_free_rq;
9769 9855
9770 init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]); 9856 init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
9771 } 9857 }
9772 9858
9773 return 1; 9859 return 1;
9774 9860
9861 err_free_rq:
9862 kfree(cfs_rq);
9775 err: 9863 err:
9776 return 0; 9864 return 0;
9777} 9865}
@@ -9853,13 +9941,15 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
9853 rt_se = kzalloc_node(sizeof(struct sched_rt_entity), 9941 rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
9854 GFP_KERNEL, cpu_to_node(i)); 9942 GFP_KERNEL, cpu_to_node(i));
9855 if (!rt_se) 9943 if (!rt_se)
9856 goto err; 9944 goto err_free_rq;
9857 9945
9858 init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]); 9946 init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
9859 } 9947 }
9860 9948
9861 return 1; 9949 return 1;
9862 9950
9951 err_free_rq:
9952 kfree(rt_rq);
9863 err: 9953 err:
9864 return 0; 9954 return 0;
9865} 9955}
@@ -10028,9 +10118,9 @@ static void set_se_shares(struct sched_entity *se, unsigned long shares)
10028 struct rq *rq = cfs_rq->rq; 10118 struct rq *rq = cfs_rq->rq;
10029 unsigned long flags; 10119 unsigned long flags;
10030 10120
10031 spin_lock_irqsave(&rq->lock, flags); 10121 raw_spin_lock_irqsave(&rq->lock, flags);
10032 __set_se_shares(se, shares); 10122 __set_se_shares(se, shares);
10033 spin_unlock_irqrestore(&rq->lock, flags); 10123 raw_spin_unlock_irqrestore(&rq->lock, flags);
10034} 10124}
10035 10125
10036static DEFINE_MUTEX(shares_mutex); 10126static DEFINE_MUTEX(shares_mutex);
@@ -10215,18 +10305,18 @@ static int tg_set_bandwidth(struct task_group *tg,
10215 if (err) 10305 if (err)
10216 goto unlock; 10306 goto unlock;
10217 10307
10218 spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); 10308 raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
10219 tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); 10309 tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
10220 tg->rt_bandwidth.rt_runtime = rt_runtime; 10310 tg->rt_bandwidth.rt_runtime = rt_runtime;
10221 10311
10222 for_each_possible_cpu(i) { 10312 for_each_possible_cpu(i) {
10223 struct rt_rq *rt_rq = tg->rt_rq[i]; 10313 struct rt_rq *rt_rq = tg->rt_rq[i];
10224 10314
10225 spin_lock(&rt_rq->rt_runtime_lock); 10315 raw_spin_lock(&rt_rq->rt_runtime_lock);
10226 rt_rq->rt_runtime = rt_runtime; 10316 rt_rq->rt_runtime = rt_runtime;
10227 spin_unlock(&rt_rq->rt_runtime_lock); 10317 raw_spin_unlock(&rt_rq->rt_runtime_lock);
10228 } 10318 }
10229 spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); 10319 raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
10230 unlock: 10320 unlock:
10231 read_unlock(&tasklist_lock); 10321 read_unlock(&tasklist_lock);
10232 mutex_unlock(&rt_constraints_mutex); 10322 mutex_unlock(&rt_constraints_mutex);
@@ -10331,15 +10421,15 @@ static int sched_rt_global_constraints(void)
10331 if (sysctl_sched_rt_runtime == 0) 10421 if (sysctl_sched_rt_runtime == 0)
10332 return -EBUSY; 10422 return -EBUSY;
10333 10423
10334 spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); 10424 raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
10335 for_each_possible_cpu(i) { 10425 for_each_possible_cpu(i) {
10336 struct rt_rq *rt_rq = &cpu_rq(i)->rt; 10426 struct rt_rq *rt_rq = &cpu_rq(i)->rt;
10337 10427
10338 spin_lock(&rt_rq->rt_runtime_lock); 10428 raw_spin_lock(&rt_rq->rt_runtime_lock);
10339 rt_rq->rt_runtime = global_rt_runtime(); 10429 rt_rq->rt_runtime = global_rt_runtime();
10340 spin_unlock(&rt_rq->rt_runtime_lock); 10430 raw_spin_unlock(&rt_rq->rt_runtime_lock);
10341 } 10431 }
10342 spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); 10432 raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
10343 10433
10344 return 0; 10434 return 0;
10345} 10435}
@@ -10630,9 +10720,9 @@ static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
10630 /* 10720 /*
10631 * Take rq->lock to make 64-bit read safe on 32-bit platforms. 10721 * Take rq->lock to make 64-bit read safe on 32-bit platforms.
10632 */ 10722 */
10633 spin_lock_irq(&cpu_rq(cpu)->lock); 10723 raw_spin_lock_irq(&cpu_rq(cpu)->lock);
10634 data = *cpuusage; 10724 data = *cpuusage;
10635 spin_unlock_irq(&cpu_rq(cpu)->lock); 10725 raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
10636#else 10726#else
10637 data = *cpuusage; 10727 data = *cpuusage;
10638#endif 10728#endif
@@ -10648,9 +10738,9 @@ static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
10648 /* 10738 /*
10649 * Take rq->lock to make 64-bit write safe on 32-bit platforms. 10739 * Take rq->lock to make 64-bit write safe on 32-bit platforms.
10650 */ 10740 */
10651 spin_lock_irq(&cpu_rq(cpu)->lock); 10741 raw_spin_lock_irq(&cpu_rq(cpu)->lock);
10652 *cpuusage = val; 10742 *cpuusage = val;
10653 spin_unlock_irq(&cpu_rq(cpu)->lock); 10743 raw_spin_unlock_irq(&cpu_rq(cpu)->lock);
10654#else 10744#else
10655 *cpuusage = val; 10745 *cpuusage = val;
10656#endif 10746#endif
@@ -10884,9 +10974,9 @@ void synchronize_sched_expedited(void)
10884 init_completion(&req->done); 10974 init_completion(&req->done);
10885 req->task = NULL; 10975 req->task = NULL;
10886 req->dest_cpu = RCU_MIGRATION_NEED_QS; 10976 req->dest_cpu = RCU_MIGRATION_NEED_QS;
10887 spin_lock_irqsave(&rq->lock, flags); 10977 raw_spin_lock_irqsave(&rq->lock, flags);
10888 list_add(&req->list, &rq->migration_queue); 10978 list_add(&req->list, &rq->migration_queue);
10889 spin_unlock_irqrestore(&rq->lock, flags); 10979 raw_spin_unlock_irqrestore(&rq->lock, flags);
10890 wake_up_process(rq->migration_thread); 10980 wake_up_process(rq->migration_thread);
10891 } 10981 }
10892 for_each_online_cpu(cpu) { 10982 for_each_online_cpu(cpu) {
@@ -10894,13 +10984,14 @@ void synchronize_sched_expedited(void)
10894 req = &per_cpu(rcu_migration_req, cpu); 10984 req = &per_cpu(rcu_migration_req, cpu);
10895 rq = cpu_rq(cpu); 10985 rq = cpu_rq(cpu);
10896 wait_for_completion(&req->done); 10986 wait_for_completion(&req->done);
10897 spin_lock_irqsave(&rq->lock, flags); 10987 raw_spin_lock_irqsave(&rq->lock, flags);
10898 if (unlikely(req->dest_cpu == RCU_MIGRATION_MUST_SYNC)) 10988 if (unlikely(req->dest_cpu == RCU_MIGRATION_MUST_SYNC))
10899 need_full_sync = 1; 10989 need_full_sync = 1;
10900 req->dest_cpu = RCU_MIGRATION_IDLE; 10990 req->dest_cpu = RCU_MIGRATION_IDLE;
10901 spin_unlock_irqrestore(&rq->lock, flags); 10991 raw_spin_unlock_irqrestore(&rq->lock, flags);
10902 } 10992 }
10903 rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE; 10993 rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE;
10994 synchronize_sched_expedited_count++;
10904 mutex_unlock(&rcu_sched_expedited_mutex); 10995 mutex_unlock(&rcu_sched_expedited_mutex);
10905 put_online_cpus(); 10996 put_online_cpus();
10906 if (need_full_sync) 10997 if (need_full_sync)
generated by cgit v1.2.2 (git 2.25.0) at 2025-05-21 01:21:36 -0400