aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched.c')
-rw-r--r--kernel/sched.c29
1 files changed, 16 insertions, 13 deletions
diff --git a/kernel/sched.c b/kernel/sched.c
index 93309c3034de..2f8ea99df16a 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -3758,10 +3758,9 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3758#ifdef CONFIG_NO_HZ 3758#ifdef CONFIG_NO_HZ
3759static struct { 3759static struct {
3760 atomic_t load_balancer; 3760 atomic_t load_balancer;
3761 cpumask_t cpu_mask; 3761 cpumask_var_t cpu_mask;
3762} nohz ____cacheline_aligned = { 3762} nohz ____cacheline_aligned = {
3763 .load_balancer = ATOMIC_INIT(-1), 3763 .load_balancer = ATOMIC_INIT(-1),
3764 .cpu_mask = CPU_MASK_NONE,
3765}; 3764};
3766 3765
3767/* 3766/*
@@ -3789,7 +3788,7 @@ int select_nohz_load_balancer(int stop_tick)
3789 int cpu = smp_processor_id(); 3788 int cpu = smp_processor_id();
3790 3789
3791 if (stop_tick) { 3790 if (stop_tick) {
3792 cpu_set(cpu, nohz.cpu_mask); 3791 cpumask_set_cpu(cpu, nohz.cpu_mask);
3793 cpu_rq(cpu)->in_nohz_recently = 1; 3792 cpu_rq(cpu)->in_nohz_recently = 1;
3794 3793
3795 /* 3794 /*
@@ -3803,7 +3802,7 @@ int select_nohz_load_balancer(int stop_tick)
3803 } 3802 }
3804 3803
3805 /* time for ilb owner also to sleep */ 3804 /* time for ilb owner also to sleep */
3806 if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 3805 if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
3807 if (atomic_read(&nohz.load_balancer) == cpu) 3806 if (atomic_read(&nohz.load_balancer) == cpu)
3808 atomic_set(&nohz.load_balancer, -1); 3807 atomic_set(&nohz.load_balancer, -1);
3809 return 0; 3808 return 0;
@@ -3816,10 +3815,10 @@ int select_nohz_load_balancer(int stop_tick)
3816 } else if (atomic_read(&nohz.load_balancer) == cpu) 3815 } else if (atomic_read(&nohz.load_balancer) == cpu)
3817 return 1; 3816 return 1;
3818 } else { 3817 } else {
3819 if (!cpu_isset(cpu, nohz.cpu_mask)) 3818 if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
3820 return 0; 3819 return 0;
3821 3820
3822 cpu_clear(cpu, nohz.cpu_mask); 3821 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3823 3822
3824 if (atomic_read(&nohz.load_balancer) == cpu) 3823 if (atomic_read(&nohz.load_balancer) == cpu)
3825 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu) 3824 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
@@ -3930,12 +3929,13 @@ static void run_rebalance_domains(struct softirq_action *h)
3930 */ 3929 */
3931 if (this_rq->idle_at_tick && 3930 if (this_rq->idle_at_tick &&
3932 atomic_read(&nohz.load_balancer) == this_cpu) { 3931 atomic_read(&nohz.load_balancer) == this_cpu) {
3933 cpumask_t cpus = nohz.cpu_mask;
3934 struct rq *rq; 3932 struct rq *rq;
3935 int balance_cpu; 3933 int balance_cpu;
3936 3934
3937 cpu_clear(this_cpu, cpus); 3935 for_each_cpu(balance_cpu, nohz.cpu_mask) {
3938 for_each_cpu(balance_cpu, &cpus) { 3936 if (balance_cpu == this_cpu)
3937 continue;
3938
3939 /* 3939 /*
3940 * If this cpu gets work to do, stop the load balancing 3940 * If this cpu gets work to do, stop the load balancing
3941 * work being done for other cpus. Next load 3941 * work being done for other cpus. Next load
@@ -3973,7 +3973,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
3973 rq->in_nohz_recently = 0; 3973 rq->in_nohz_recently = 0;
3974 3974
3975 if (atomic_read(&nohz.load_balancer) == cpu) { 3975 if (atomic_read(&nohz.load_balancer) == cpu) {
3976 cpu_clear(cpu, nohz.cpu_mask); 3976 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3977 atomic_set(&nohz.load_balancer, -1); 3977 atomic_set(&nohz.load_balancer, -1);
3978 } 3978 }
3979 3979
@@ -3986,7 +3986,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
3986 * TBD: Traverse the sched domains and nominate 3986 * TBD: Traverse the sched domains and nominate
3987 * the nearest cpu in the nohz.cpu_mask. 3987 * the nearest cpu in the nohz.cpu_mask.
3988 */ 3988 */
3989 int ilb = first_cpu(nohz.cpu_mask); 3989 int ilb = cpumask_first(nohz.cpu_mask);
3990 3990
3991 if (ilb < nr_cpu_ids) 3991 if (ilb < nr_cpu_ids)
3992 resched_cpu(ilb); 3992 resched_cpu(ilb);
@@ -3998,7 +3998,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
3998 * cpus with ticks stopped, is it time for that to stop? 3998 * cpus with ticks stopped, is it time for that to stop?
3999 */ 3999 */
4000 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu && 4000 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4001 cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 4001 cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
4002 resched_cpu(cpu); 4002 resched_cpu(cpu);
4003 return; 4003 return;
4004 } 4004 }
@@ -4008,7 +4008,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4008 * someone else, then no need raise the SCHED_SOFTIRQ 4008 * someone else, then no need raise the SCHED_SOFTIRQ
4009 */ 4009 */
4010 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu && 4010 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4011 cpu_isset(cpu, nohz.cpu_mask)) 4011 cpumask_test_cpu(cpu, nohz.cpu_mask))
4012 return; 4012 return;
4013#endif 4013#endif
4014 if (time_after_eq(jiffies, rq->next_balance)) 4014 if (time_after_eq(jiffies, rq->next_balance))
@@ -8309,6 +8309,9 @@ void __init sched_init(void)
8309 8309
8310 /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */ 8310 /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
8311 alloc_bootmem_cpumask_var(&nohz_cpu_mask); 8311 alloc_bootmem_cpumask_var(&nohz_cpu_mask);
8312#ifdef CONFIG_NO_HZ
8313 alloc_bootmem_cpumask_var(&nohz.cpu_mask);
8314#endif
8312 8315
8313 scheduler_running = 1; 8316 scheduler_running = 1;
8314} 8317}