diff options
author | Peter Zijlstra <a.p.zijlstra@chello.nl> | 2012-05-02 08:20:37 -0400 |
---|---|---|
committer | Ingo Molnar <mingo@kernel.org> | 2012-05-09 09:00:54 -0400 |
commit | bd939f45da24e25e08a8f5c993c50b1afada0fef (patch) | |
tree | a103544f605e5e0e3f1c20a1fc9fffcaeba1a33c /kernel/sched | |
parent | 0ce90475dcdbe90affc218e9688c8401e468e84d (diff) |
sched/fair: Propagate 'struct lb_env' usage into find_busiest_group
More function argument passing reduction.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/n/tip-v66ivjfqdiqdso01lqgqx6qf@git.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Diffstat (limited to 'kernel/sched')
-rw-r--r-- | kernel/sched/fair.c | 175 |
1 files changed, 82 insertions, 93 deletions
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index cf86f74bcac2..9bd3366dbb1c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c | |||
@@ -3082,7 +3082,7 @@ struct lb_env { | |||
3082 | struct rq *dst_rq; | 3082 | struct rq *dst_rq; |
3083 | 3083 | ||
3084 | enum cpu_idle_type idle; | 3084 | enum cpu_idle_type idle; |
3085 | long load_move; | 3085 | long imbalance; |
3086 | unsigned int flags; | 3086 | unsigned int flags; |
3087 | 3087 | ||
3088 | unsigned int loop; | 3088 | unsigned int loop; |
@@ -3218,7 +3218,7 @@ static unsigned long task_h_load(struct task_struct *p); | |||
3218 | static const unsigned int sched_nr_migrate_break = 32; | 3218 | static const unsigned int sched_nr_migrate_break = 32; |
3219 | 3219 | ||
3220 | /* | 3220 | /* |
3221 | * move_tasks tries to move up to load_move weighted load from busiest to | 3221 | * move_tasks tries to move up to imbalance weighted load from busiest to |
3222 | * this_rq, as part of a balancing operation within domain "sd". | 3222 | * this_rq, as part of a balancing operation within domain "sd". |
3223 | * Returns 1 if successful and 0 otherwise. | 3223 | * Returns 1 if successful and 0 otherwise. |
3224 | * | 3224 | * |
@@ -3231,7 +3231,7 @@ static int move_tasks(struct lb_env *env) | |||
3231 | unsigned long load; | 3231 | unsigned long load; |
3232 | int pulled = 0; | 3232 | int pulled = 0; |
3233 | 3233 | ||
3234 | if (env->load_move <= 0) | 3234 | if (env->imbalance <= 0) |
3235 | return 0; | 3235 | return 0; |
3236 | 3236 | ||
3237 | while (!list_empty(tasks)) { | 3237 | while (!list_empty(tasks)) { |
@@ -3257,7 +3257,7 @@ static int move_tasks(struct lb_env *env) | |||
3257 | if (sched_feat(LB_MIN) && load < 16 && !env->sd->nr_balance_failed) | 3257 | if (sched_feat(LB_MIN) && load < 16 && !env->sd->nr_balance_failed) |
3258 | goto next; | 3258 | goto next; |
3259 | 3259 | ||
3260 | if ((load / 2) > env->load_move) | 3260 | if ((load / 2) > env->imbalance) |
3261 | goto next; | 3261 | goto next; |
3262 | 3262 | ||
3263 | if (!can_migrate_task(p, env)) | 3263 | if (!can_migrate_task(p, env)) |
@@ -3265,7 +3265,7 @@ static int move_tasks(struct lb_env *env) | |||
3265 | 3265 | ||
3266 | move_task(p, env); | 3266 | move_task(p, env); |
3267 | pulled++; | 3267 | pulled++; |
3268 | env->load_move -= load; | 3268 | env->imbalance -= load; |
3269 | 3269 | ||
3270 | #ifdef CONFIG_PREEMPT | 3270 | #ifdef CONFIG_PREEMPT |
3271 | /* | 3271 | /* |
@@ -3281,7 +3281,7 @@ static int move_tasks(struct lb_env *env) | |||
3281 | * We only want to steal up to the prescribed amount of | 3281 | * We only want to steal up to the prescribed amount of |
3282 | * weighted load. | 3282 | * weighted load. |
3283 | */ | 3283 | */ |
3284 | if (env->load_move <= 0) | 3284 | if (env->imbalance <= 0) |
3285 | break; | 3285 | break; |
3286 | 3286 | ||
3287 | continue; | 3287 | continue; |
@@ -3578,10 +3578,9 @@ static inline void update_sd_power_savings_stats(struct sched_group *group, | |||
3578 | 3578 | ||
3579 | /** | 3579 | /** |
3580 | * check_power_save_busiest_group - see if there is potential for some power-savings balance | 3580 | * check_power_save_busiest_group - see if there is potential for some power-savings balance |
3581 | * @env: load balance environment | ||
3581 | * @sds: Variable containing the statistics of the sched_domain | 3582 | * @sds: Variable containing the statistics of the sched_domain |
3582 | * under consideration. | 3583 | * under consideration. |
3583 | * @this_cpu: Cpu at which we're currently performing load-balancing. | ||
3584 | * @imbalance: Variable to store the imbalance. | ||
3585 | * | 3584 | * |
3586 | * Description: | 3585 | * Description: |
3587 | * Check if we have potential to perform some power-savings balance. | 3586 | * Check if we have potential to perform some power-savings balance. |
@@ -3591,8 +3590,8 @@ static inline void update_sd_power_savings_stats(struct sched_group *group, | |||
3591 | * Returns 1 if there is potential to perform power-savings balance. | 3590 | * Returns 1 if there is potential to perform power-savings balance. |
3592 | * Else returns 0. | 3591 | * Else returns 0. |
3593 | */ | 3592 | */ |
3594 | static inline int check_power_save_busiest_group(struct sd_lb_stats *sds, | 3593 | static inline |
3595 | int this_cpu, unsigned long *imbalance) | 3594 | int check_power_save_busiest_group(struct lb_env *env, struct sd_lb_stats *sds) |
3596 | { | 3595 | { |
3597 | if (!sds->power_savings_balance) | 3596 | if (!sds->power_savings_balance) |
3598 | return 0; | 3597 | return 0; |
@@ -3601,7 +3600,7 @@ static inline int check_power_save_busiest_group(struct sd_lb_stats *sds, | |||
3601 | sds->group_leader == sds->group_min) | 3600 | sds->group_leader == sds->group_min) |
3602 | return 0; | 3601 | return 0; |
3603 | 3602 | ||
3604 | *imbalance = sds->min_load_per_task; | 3603 | env->imbalance = sds->min_load_per_task; |
3605 | sds->busiest = sds->group_min; | 3604 | sds->busiest = sds->group_min; |
3606 | 3605 | ||
3607 | return 1; | 3606 | return 1; |
@@ -3620,8 +3619,8 @@ static inline void update_sd_power_savings_stats(struct sched_group *group, | |||
3620 | return; | 3619 | return; |
3621 | } | 3620 | } |
3622 | 3621 | ||
3623 | static inline int check_power_save_busiest_group(struct sd_lb_stats *sds, | 3622 | static inline |
3624 | int this_cpu, unsigned long *imbalance) | 3623 | int check_power_save_busiest_group(struct lb_env *env, struct sd_lb_stats *sds) |
3625 | { | 3624 | { |
3626 | return 0; | 3625 | return 0; |
3627 | } | 3626 | } |
@@ -3765,25 +3764,22 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group) | |||
3765 | * update_sg_lb_stats - Update sched_group's statistics for load balancing. | 3764 | * update_sg_lb_stats - Update sched_group's statistics for load balancing. |
3766 | * @sd: The sched_domain whose statistics are to be updated. | 3765 | * @sd: The sched_domain whose statistics are to be updated. |
3767 | * @group: sched_group whose statistics are to be updated. | 3766 | * @group: sched_group whose statistics are to be updated. |
3768 | * @this_cpu: Cpu for which load balance is currently performed. | ||
3769 | * @idle: Idle status of this_cpu | ||
3770 | * @load_idx: Load index of sched_domain of this_cpu for load calc. | 3767 | * @load_idx: Load index of sched_domain of this_cpu for load calc. |
3771 | * @local_group: Does group contain this_cpu. | 3768 | * @local_group: Does group contain this_cpu. |
3772 | * @cpus: Set of cpus considered for load balancing. | 3769 | * @cpus: Set of cpus considered for load balancing. |
3773 | * @balance: Should we balance. | 3770 | * @balance: Should we balance. |
3774 | * @sgs: variable to hold the statistics for this group. | 3771 | * @sgs: variable to hold the statistics for this group. |
3775 | */ | 3772 | */ |
3776 | static inline void update_sg_lb_stats(struct sched_domain *sd, | 3773 | static inline void update_sg_lb_stats(struct lb_env *env, |
3777 | struct sched_group *group, int this_cpu, | 3774 | struct sched_group *group, int load_idx, |
3778 | enum cpu_idle_type idle, int load_idx, | ||
3779 | int local_group, const struct cpumask *cpus, | 3775 | int local_group, const struct cpumask *cpus, |
3780 | int *balance, struct sg_lb_stats *sgs) | 3776 | int *balance, struct sg_lb_stats *sgs) |
3781 | { | 3777 | { |
3782 | unsigned long load, max_cpu_load, min_cpu_load, max_nr_running; | 3778 | unsigned long load, max_cpu_load, min_cpu_load, max_nr_running; |
3783 | int i; | ||
3784 | unsigned int balance_cpu = -1; | 3779 | unsigned int balance_cpu = -1; |
3785 | unsigned long balance_load = ~0UL; | 3780 | unsigned long balance_load = ~0UL; |
3786 | unsigned long avg_load_per_task = 0; | 3781 | unsigned long avg_load_per_task = 0; |
3782 | int i; | ||
3787 | 3783 | ||
3788 | if (local_group) | 3784 | if (local_group) |
3789 | balance_cpu = group_first_cpu(group); | 3785 | balance_cpu = group_first_cpu(group); |
@@ -3827,15 +3823,15 @@ static inline void update_sg_lb_stats(struct sched_domain *sd, | |||
3827 | * to do the newly idle load balance. | 3823 | * to do the newly idle load balance. |
3828 | */ | 3824 | */ |
3829 | if (local_group) { | 3825 | if (local_group) { |
3830 | if (idle != CPU_NEWLY_IDLE) { | 3826 | if (env->idle != CPU_NEWLY_IDLE) { |
3831 | if (balance_cpu != this_cpu || | 3827 | if (balance_cpu != env->dst_cpu || |
3832 | cmpxchg(&group->balance_cpu, -1, balance_cpu) != -1) { | 3828 | cmpxchg(&group->balance_cpu, -1, balance_cpu) != -1) { |
3833 | *balance = 0; | 3829 | *balance = 0; |
3834 | return; | 3830 | return; |
3835 | } | 3831 | } |
3836 | update_group_power(sd, this_cpu); | 3832 | update_group_power(env->sd, env->dst_cpu); |
3837 | } else if (time_after_eq(jiffies, group->sgp->next_update)) | 3833 | } else if (time_after_eq(jiffies, group->sgp->next_update)) |
3838 | update_group_power(sd, this_cpu); | 3834 | update_group_power(env->sd, env->dst_cpu); |
3839 | } | 3835 | } |
3840 | 3836 | ||
3841 | /* Adjust by relative CPU power of the group */ | 3837 | /* Adjust by relative CPU power of the group */ |
@@ -3859,7 +3855,7 @@ static inline void update_sg_lb_stats(struct sched_domain *sd, | |||
3859 | sgs->group_capacity = DIV_ROUND_CLOSEST(group->sgp->power, | 3855 | sgs->group_capacity = DIV_ROUND_CLOSEST(group->sgp->power, |
3860 | SCHED_POWER_SCALE); | 3856 | SCHED_POWER_SCALE); |
3861 | if (!sgs->group_capacity) | 3857 | if (!sgs->group_capacity) |
3862 | sgs->group_capacity = fix_small_capacity(sd, group); | 3858 | sgs->group_capacity = fix_small_capacity(env->sd, group); |
3863 | sgs->group_weight = group->group_weight; | 3859 | sgs->group_weight = group->group_weight; |
3864 | 3860 | ||
3865 | if (sgs->group_capacity > sgs->sum_nr_running) | 3861 | if (sgs->group_capacity > sgs->sum_nr_running) |
@@ -3877,11 +3873,10 @@ static inline void update_sg_lb_stats(struct sched_domain *sd, | |||
3877 | * Determine if @sg is a busier group than the previously selected | 3873 | * Determine if @sg is a busier group than the previously selected |
3878 | * busiest group. | 3874 | * busiest group. |
3879 | */ | 3875 | */ |
3880 | static bool update_sd_pick_busiest(struct sched_domain *sd, | 3876 | static bool update_sd_pick_busiest(struct lb_env *env, |
3881 | struct sd_lb_stats *sds, | 3877 | struct sd_lb_stats *sds, |
3882 | struct sched_group *sg, | 3878 | struct sched_group *sg, |
3883 | struct sg_lb_stats *sgs, | 3879 | struct sg_lb_stats *sgs) |
3884 | int this_cpu) | ||
3885 | { | 3880 | { |
3886 | if (sgs->avg_load <= sds->max_load) | 3881 | if (sgs->avg_load <= sds->max_load) |
3887 | return false; | 3882 | return false; |
@@ -3897,8 +3892,8 @@ static bool update_sd_pick_busiest(struct sched_domain *sd, | |||
3897 | * numbered CPUs in the group, therefore mark all groups | 3892 | * numbered CPUs in the group, therefore mark all groups |
3898 | * higher than ourself as busy. | 3893 | * higher than ourself as busy. |
3899 | */ | 3894 | */ |
3900 | if ((sd->flags & SD_ASYM_PACKING) && sgs->sum_nr_running && | 3895 | if ((env->sd->flags & SD_ASYM_PACKING) && sgs->sum_nr_running && |
3901 | this_cpu < group_first_cpu(sg)) { | 3896 | env->dst_cpu < group_first_cpu(sg)) { |
3902 | if (!sds->busiest) | 3897 | if (!sds->busiest) |
3903 | return true; | 3898 | return true; |
3904 | 3899 | ||
@@ -3918,28 +3913,28 @@ static bool update_sd_pick_busiest(struct sched_domain *sd, | |||
3918 | * @balance: Should we balance. | 3913 | * @balance: Should we balance. |
3919 | * @sds: variable to hold the statistics for this sched_domain. | 3914 | * @sds: variable to hold the statistics for this sched_domain. |
3920 | */ | 3915 | */ |
3921 | static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu, | 3916 | static inline void update_sd_lb_stats(struct lb_env *env, |
3922 | enum cpu_idle_type idle, const struct cpumask *cpus, | 3917 | const struct cpumask *cpus, |
3923 | int *balance, struct sd_lb_stats *sds) | 3918 | int *balance, struct sd_lb_stats *sds) |
3924 | { | 3919 | { |
3925 | struct sched_domain *child = sd->child; | 3920 | struct sched_domain *child = env->sd->child; |
3926 | struct sched_group *sg = sd->groups; | 3921 | struct sched_group *sg = env->sd->groups; |
3927 | struct sg_lb_stats sgs; | 3922 | struct sg_lb_stats sgs; |
3928 | int load_idx, prefer_sibling = 0; | 3923 | int load_idx, prefer_sibling = 0; |
3929 | 3924 | ||
3930 | if (child && child->flags & SD_PREFER_SIBLING) | 3925 | if (child && child->flags & SD_PREFER_SIBLING) |
3931 | prefer_sibling = 1; | 3926 | prefer_sibling = 1; |
3932 | 3927 | ||
3933 | init_sd_power_savings_stats(sd, sds, idle); | 3928 | init_sd_power_savings_stats(env->sd, sds, env->idle); |
3934 | load_idx = get_sd_load_idx(sd, idle); | 3929 | load_idx = get_sd_load_idx(env->sd, env->idle); |
3935 | 3930 | ||
3936 | do { | 3931 | do { |
3937 | int local_group; | 3932 | int local_group; |
3938 | 3933 | ||
3939 | local_group = cpumask_test_cpu(this_cpu, sched_group_cpus(sg)); | 3934 | local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg)); |
3940 | memset(&sgs, 0, sizeof(sgs)); | 3935 | memset(&sgs, 0, sizeof(sgs)); |
3941 | update_sg_lb_stats(sd, sg, this_cpu, idle, load_idx, | 3936 | update_sg_lb_stats(env, sg, load_idx, local_group, |
3942 | local_group, cpus, balance, &sgs); | 3937 | cpus, balance, &sgs); |
3943 | 3938 | ||
3944 | if (local_group && !(*balance)) | 3939 | if (local_group && !(*balance)) |
3945 | return; | 3940 | return; |
@@ -3967,7 +3962,7 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu, | |||
3967 | sds->this_load_per_task = sgs.sum_weighted_load; | 3962 | sds->this_load_per_task = sgs.sum_weighted_load; |
3968 | sds->this_has_capacity = sgs.group_has_capacity; | 3963 | sds->this_has_capacity = sgs.group_has_capacity; |
3969 | sds->this_idle_cpus = sgs.idle_cpus; | 3964 | sds->this_idle_cpus = sgs.idle_cpus; |
3970 | } else if (update_sd_pick_busiest(sd, sds, sg, &sgs, this_cpu)) { | 3965 | } else if (update_sd_pick_busiest(env, sds, sg, &sgs)) { |
3971 | sds->max_load = sgs.avg_load; | 3966 | sds->max_load = sgs.avg_load; |
3972 | sds->busiest = sg; | 3967 | sds->busiest = sg; |
3973 | sds->busiest_nr_running = sgs.sum_nr_running; | 3968 | sds->busiest_nr_running = sgs.sum_nr_running; |
@@ -3981,7 +3976,7 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu, | |||
3981 | 3976 | ||
3982 | update_sd_power_savings_stats(sg, sds, local_group, &sgs); | 3977 | update_sd_power_savings_stats(sg, sds, local_group, &sgs); |
3983 | sg = sg->next; | 3978 | sg = sg->next; |
3984 | } while (sg != sd->groups); | 3979 | } while (sg != env->sd->groups); |
3985 | } | 3980 | } |
3986 | 3981 | ||
3987 | /** | 3982 | /** |
@@ -4009,24 +4004,23 @@ static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu, | |||
4009 | * @this_cpu: The cpu at whose sched_domain we're performing load-balance. | 4004 | * @this_cpu: The cpu at whose sched_domain we're performing load-balance. |
4010 | * @imbalance: returns amount of imbalanced due to packing. | 4005 | * @imbalance: returns amount of imbalanced due to packing. |
4011 | */ | 4006 | */ |
4012 | static int check_asym_packing(struct sched_domain *sd, | 4007 | static int check_asym_packing(struct lb_env *env, struct sd_lb_stats *sds) |
4013 | struct sd_lb_stats *sds, | ||
4014 | int this_cpu, unsigned long *imbalance) | ||
4015 | { | 4008 | { |
4016 | int busiest_cpu; | 4009 | int busiest_cpu; |
4017 | 4010 | ||
4018 | if (!(sd->flags & SD_ASYM_PACKING)) | 4011 | if (!(env->sd->flags & SD_ASYM_PACKING)) |
4019 | return 0; | 4012 | return 0; |
4020 | 4013 | ||
4021 | if (!sds->busiest) | 4014 | if (!sds->busiest) |
4022 | return 0; | 4015 | return 0; |
4023 | 4016 | ||
4024 | busiest_cpu = group_first_cpu(sds->busiest); | 4017 | busiest_cpu = group_first_cpu(sds->busiest); |
4025 | if (this_cpu > busiest_cpu) | 4018 | if (env->dst_cpu > busiest_cpu) |
4026 | return 0; | 4019 | return 0; |
4027 | 4020 | ||
4028 | *imbalance = DIV_ROUND_CLOSEST(sds->max_load * sds->busiest->sgp->power, | 4021 | env->imbalance = DIV_ROUND_CLOSEST( |
4029 | SCHED_POWER_SCALE); | 4022 | sds->max_load * sds->busiest->sgp->power, SCHED_POWER_SCALE); |
4023 | |||
4030 | return 1; | 4024 | return 1; |
4031 | } | 4025 | } |
4032 | 4026 | ||
@@ -4038,8 +4032,8 @@ static int check_asym_packing(struct sched_domain *sd, | |||
4038 | * @this_cpu: The cpu at whose sched_domain we're performing load-balance. | 4032 | * @this_cpu: The cpu at whose sched_domain we're performing load-balance. |
4039 | * @imbalance: Variable to store the imbalance. | 4033 | * @imbalance: Variable to store the imbalance. |
4040 | */ | 4034 | */ |
4041 | static inline void fix_small_imbalance(struct sd_lb_stats *sds, | 4035 | static inline |
4042 | int this_cpu, unsigned long *imbalance) | 4036 | void fix_small_imbalance(struct lb_env *env, struct sd_lb_stats *sds) |
4043 | { | 4037 | { |
4044 | unsigned long tmp, pwr_now = 0, pwr_move = 0; | 4038 | unsigned long tmp, pwr_now = 0, pwr_move = 0; |
4045 | unsigned int imbn = 2; | 4039 | unsigned int imbn = 2; |
@@ -4050,9 +4044,10 @@ static inline void fix_small_imbalance(struct sd_lb_stats *sds, | |||
4050 | if (sds->busiest_load_per_task > | 4044 | if (sds->busiest_load_per_task > |
4051 | sds->this_load_per_task) | 4045 | sds->this_load_per_task) |
4052 | imbn = 1; | 4046 | imbn = 1; |
4053 | } else | 4047 | } else { |
4054 | sds->this_load_per_task = | 4048 | sds->this_load_per_task = |
4055 | cpu_avg_load_per_task(this_cpu); | 4049 | cpu_avg_load_per_task(env->dst_cpu); |
4050 | } | ||
4056 | 4051 | ||
4057 | scaled_busy_load_per_task = sds->busiest_load_per_task | 4052 | scaled_busy_load_per_task = sds->busiest_load_per_task |
4058 | * SCHED_POWER_SCALE; | 4053 | * SCHED_POWER_SCALE; |
@@ -4060,7 +4055,7 @@ static inline void fix_small_imbalance(struct sd_lb_stats *sds, | |||
4060 | 4055 | ||
4061 | if (sds->max_load - sds->this_load + scaled_busy_load_per_task >= | 4056 | if (sds->max_load - sds->this_load + scaled_busy_load_per_task >= |
4062 | (scaled_busy_load_per_task * imbn)) { | 4057 | (scaled_busy_load_per_task * imbn)) { |
4063 | *imbalance = sds->busiest_load_per_task; | 4058 | env->imbalance = sds->busiest_load_per_task; |
4064 | return; | 4059 | return; |
4065 | } | 4060 | } |
4066 | 4061 | ||
@@ -4097,18 +4092,16 @@ static inline void fix_small_imbalance(struct sd_lb_stats *sds, | |||
4097 | 4092 | ||
4098 | /* Move if we gain throughput */ | 4093 | /* Move if we gain throughput */ |
4099 | if (pwr_move > pwr_now) | 4094 | if (pwr_move > pwr_now) |
4100 | *imbalance = sds->busiest_load_per_task; | 4095 | env->imbalance = sds->busiest_load_per_task; |
4101 | } | 4096 | } |
4102 | 4097 | ||
4103 | /** | 4098 | /** |
4104 | * calculate_imbalance - Calculate the amount of imbalance present within the | 4099 | * calculate_imbalance - Calculate the amount of imbalance present within the |
4105 | * groups of a given sched_domain during load balance. | 4100 | * groups of a given sched_domain during load balance. |
4101 | * @env: load balance environment | ||
4106 | * @sds: statistics of the sched_domain whose imbalance is to be calculated. | 4102 | * @sds: statistics of the sched_domain whose imbalance is to be calculated. |
4107 | * @this_cpu: Cpu for which currently load balance is being performed. | ||
4108 | * @imbalance: The variable to store the imbalance. | ||
4109 | */ | 4103 | */ |
4110 | static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu, | 4104 | static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) |
4111 | unsigned long *imbalance) | ||
4112 | { | 4105 | { |
4113 | unsigned long max_pull, load_above_capacity = ~0UL; | 4106 | unsigned long max_pull, load_above_capacity = ~0UL; |
4114 | 4107 | ||
@@ -4124,8 +4117,8 @@ static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu, | |||
4124 | * its cpu_power, while calculating max_load..) | 4117 | * its cpu_power, while calculating max_load..) |
4125 | */ | 4118 | */ |
4126 | if (sds->max_load < sds->avg_load) { | 4119 | if (sds->max_load < sds->avg_load) { |
4127 | *imbalance = 0; | 4120 | env->imbalance = 0; |
4128 | return fix_small_imbalance(sds, this_cpu, imbalance); | 4121 | return fix_small_imbalance(env, sds); |
4129 | } | 4122 | } |
4130 | 4123 | ||
4131 | if (!sds->group_imb) { | 4124 | if (!sds->group_imb) { |
@@ -4153,7 +4146,7 @@ static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu, | |||
4153 | max_pull = min(sds->max_load - sds->avg_load, load_above_capacity); | 4146 | max_pull = min(sds->max_load - sds->avg_load, load_above_capacity); |
4154 | 4147 | ||
4155 | /* How much load to actually move to equalise the imbalance */ | 4148 | /* How much load to actually move to equalise the imbalance */ |
4156 | *imbalance = min(max_pull * sds->busiest->sgp->power, | 4149 | env->imbalance = min(max_pull * sds->busiest->sgp->power, |
4157 | (sds->avg_load - sds->this_load) * sds->this->sgp->power) | 4150 | (sds->avg_load - sds->this_load) * sds->this->sgp->power) |
4158 | / SCHED_POWER_SCALE; | 4151 | / SCHED_POWER_SCALE; |
4159 | 4152 | ||
@@ -4163,8 +4156,8 @@ static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu, | |||
4163 | * a think about bumping its value to force at least one task to be | 4156 | * a think about bumping its value to force at least one task to be |
4164 | * moved | 4157 | * moved |
4165 | */ | 4158 | */ |
4166 | if (*imbalance < sds->busiest_load_per_task) | 4159 | if (env->imbalance < sds->busiest_load_per_task) |
4167 | return fix_small_imbalance(sds, this_cpu, imbalance); | 4160 | return fix_small_imbalance(env, sds); |
4168 | 4161 | ||
4169 | } | 4162 | } |
4170 | 4163 | ||
@@ -4195,9 +4188,7 @@ static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu, | |||
4195 | * put to idle by rebalancing its tasks onto our group. | 4188 | * put to idle by rebalancing its tasks onto our group. |
4196 | */ | 4189 | */ |
4197 | static struct sched_group * | 4190 | static struct sched_group * |
4198 | find_busiest_group(struct sched_domain *sd, int this_cpu, | 4191 | find_busiest_group(struct lb_env *env, const struct cpumask *cpus, int *balance) |
4199 | unsigned long *imbalance, enum cpu_idle_type idle, | ||
4200 | const struct cpumask *cpus, int *balance) | ||
4201 | { | 4192 | { |
4202 | struct sd_lb_stats sds; | 4193 | struct sd_lb_stats sds; |
4203 | 4194 | ||
@@ -4207,7 +4198,7 @@ find_busiest_group(struct sched_domain *sd, int this_cpu, | |||
4207 | * Compute the various statistics relavent for load balancing at | 4198 | * Compute the various statistics relavent for load balancing at |
4208 | * this level. | 4199 | * this level. |
4209 | */ | 4200 | */ |
4210 | update_sd_lb_stats(sd, this_cpu, idle, cpus, balance, &sds); | 4201 | update_sd_lb_stats(env, cpus, balance, &sds); |
4211 | 4202 | ||
4212 | /* | 4203 | /* |
4213 | * this_cpu is not the appropriate cpu to perform load balancing at | 4204 | * this_cpu is not the appropriate cpu to perform load balancing at |
@@ -4216,8 +4207,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu, | |||
4216 | if (!(*balance)) | 4207 | if (!(*balance)) |
4217 | goto ret; | 4208 | goto ret; |
4218 | 4209 | ||
4219 | if ((idle == CPU_IDLE || idle == CPU_NEWLY_IDLE) && | 4210 | if ((env->idle == CPU_IDLE || env->idle == CPU_NEWLY_IDLE) && |
4220 | check_asym_packing(sd, &sds, this_cpu, imbalance)) | 4211 | check_asym_packing(env, &sds)) |
4221 | return sds.busiest; | 4212 | return sds.busiest; |
4222 | 4213 | ||
4223 | /* There is no busy sibling group to pull tasks from */ | 4214 | /* There is no busy sibling group to pull tasks from */ |
@@ -4235,7 +4226,7 @@ find_busiest_group(struct sched_domain *sd, int this_cpu, | |||
4235 | goto force_balance; | 4226 | goto force_balance; |
4236 | 4227 | ||
4237 | /* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */ | 4228 | /* SD_BALANCE_NEWIDLE trumps SMP nice when underutilized */ |
4238 | if (idle == CPU_NEWLY_IDLE && sds.this_has_capacity && | 4229 | if (env->idle == CPU_NEWLY_IDLE && sds.this_has_capacity && |
4239 | !sds.busiest_has_capacity) | 4230 | !sds.busiest_has_capacity) |
4240 | goto force_balance; | 4231 | goto force_balance; |
4241 | 4232 | ||
@@ -4253,7 +4244,7 @@ find_busiest_group(struct sched_domain *sd, int this_cpu, | |||
4253 | if (sds.this_load >= sds.avg_load) | 4244 | if (sds.this_load >= sds.avg_load) |
4254 | goto out_balanced; | 4245 | goto out_balanced; |
4255 | 4246 | ||
4256 | if (idle == CPU_IDLE) { | 4247 | if (env->idle == CPU_IDLE) { |
4257 | /* | 4248 | /* |
4258 | * This cpu is idle. If the busiest group load doesn't | 4249 | * This cpu is idle. If the busiest group load doesn't |
4259 | * have more tasks than the number of available cpu's and | 4250 | * have more tasks than the number of available cpu's and |
@@ -4268,13 +4259,13 @@ find_busiest_group(struct sched_domain *sd, int this_cpu, | |||
4268 | * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use | 4259 | * In the CPU_NEWLY_IDLE, CPU_NOT_IDLE cases, use |
4269 | * imbalance_pct to be conservative. | 4260 | * imbalance_pct to be conservative. |
4270 | */ | 4261 | */ |
4271 | if (100 * sds.max_load <= sd->imbalance_pct * sds.this_load) | 4262 | if (100 * sds.max_load <= env->sd->imbalance_pct * sds.this_load) |
4272 | goto out_balanced; | 4263 | goto out_balanced; |
4273 | } | 4264 | } |
4274 | 4265 | ||
4275 | force_balance: | 4266 | force_balance: |
4276 | /* Looks like there is an imbalance. Compute it */ | 4267 | /* Looks like there is an imbalance. Compute it */ |
4277 | calculate_imbalance(&sds, this_cpu, imbalance); | 4268 | calculate_imbalance(env, &sds); |
4278 | return sds.busiest; | 4269 | return sds.busiest; |
4279 | 4270 | ||
4280 | out_balanced: | 4271 | out_balanced: |
@@ -4282,20 +4273,19 @@ out_balanced: | |||
4282 | * There is no obvious imbalance. But check if we can do some balancing | 4273 | * There is no obvious imbalance. But check if we can do some balancing |
4283 | * to save power. | 4274 | * to save power. |
4284 | */ | 4275 | */ |
4285 | if (check_power_save_busiest_group(&sds, this_cpu, imbalance)) | 4276 | if (check_power_save_busiest_group(env, &sds)) |
4286 | return sds.busiest; | 4277 | return sds.busiest; |
4287 | ret: | 4278 | ret: |
4288 | *imbalance = 0; | 4279 | env->imbalance = 0; |
4289 | return NULL; | 4280 | return NULL; |
4290 | } | 4281 | } |
4291 | 4282 | ||
4292 | /* | 4283 | /* |
4293 | * find_busiest_queue - find the busiest runqueue among the cpus in group. | 4284 | * find_busiest_queue - find the busiest runqueue among the cpus in group. |
4294 | */ | 4285 | */ |
4295 | static struct rq * | 4286 | static struct rq *find_busiest_queue(struct lb_env *env, |
4296 | find_busiest_queue(struct sched_domain *sd, struct sched_group *group, | 4287 | struct sched_group *group, |
4297 | enum cpu_idle_type idle, unsigned long imbalance, | 4288 | const struct cpumask *cpus) |
4298 | const struct cpumask *cpus) | ||
4299 | { | 4289 | { |
4300 | struct rq *busiest = NULL, *rq; | 4290 | struct rq *busiest = NULL, *rq; |
4301 | unsigned long max_load = 0; | 4291 | unsigned long max_load = 0; |
@@ -4308,7 +4298,7 @@ find_busiest_queue(struct sched_domain *sd, struct sched_group *group, | |||
4308 | unsigned long wl; | 4298 | unsigned long wl; |
4309 | 4299 | ||
4310 | if (!capacity) | 4300 | if (!capacity) |
4311 | capacity = fix_small_capacity(sd, group); | 4301 | capacity = fix_small_capacity(env->sd, group); |
4312 | 4302 | ||
4313 | if (!cpumask_test_cpu(i, cpus)) | 4303 | if (!cpumask_test_cpu(i, cpus)) |
4314 | continue; | 4304 | continue; |
@@ -4320,7 +4310,7 @@ find_busiest_queue(struct sched_domain *sd, struct sched_group *group, | |||
4320 | * When comparing with imbalance, use weighted_cpuload() | 4310 | * When comparing with imbalance, use weighted_cpuload() |
4321 | * which is not scaled with the cpu power. | 4311 | * which is not scaled with the cpu power. |
4322 | */ | 4312 | */ |
4323 | if (capacity && rq->nr_running == 1 && wl > imbalance) | 4313 | if (capacity && rq->nr_running == 1 && wl > env->imbalance) |
4324 | continue; | 4314 | continue; |
4325 | 4315 | ||
4326 | /* | 4316 | /* |
@@ -4349,17 +4339,18 @@ find_busiest_queue(struct sched_domain *sd, struct sched_group *group, | |||
4349 | /* Working cpumask for load_balance and load_balance_newidle. */ | 4339 | /* Working cpumask for load_balance and load_balance_newidle. */ |
4350 | DEFINE_PER_CPU(cpumask_var_t, load_balance_tmpmask); | 4340 | DEFINE_PER_CPU(cpumask_var_t, load_balance_tmpmask); |
4351 | 4341 | ||
4352 | static int need_active_balance(struct sched_domain *sd, int idle, | 4342 | static int need_active_balance(struct lb_env *env) |
4353 | int busiest_cpu, int this_cpu) | ||
4354 | { | 4343 | { |
4355 | if (idle == CPU_NEWLY_IDLE) { | 4344 | struct sched_domain *sd = env->sd; |
4345 | |||
4346 | if (env->idle == CPU_NEWLY_IDLE) { | ||
4356 | 4347 | ||
4357 | /* | 4348 | /* |
4358 | * ASYM_PACKING needs to force migrate tasks from busy but | 4349 | * ASYM_PACKING needs to force migrate tasks from busy but |
4359 | * higher numbered CPUs in order to pack all tasks in the | 4350 | * higher numbered CPUs in order to pack all tasks in the |
4360 | * lowest numbered CPUs. | 4351 | * lowest numbered CPUs. |
4361 | */ | 4352 | */ |
4362 | if ((sd->flags & SD_ASYM_PACKING) && busiest_cpu > this_cpu) | 4353 | if ((sd->flags & SD_ASYM_PACKING) && env->src_cpu > env->dst_cpu) |
4363 | return 1; | 4354 | return 1; |
4364 | 4355 | ||
4365 | /* | 4356 | /* |
@@ -4400,7 +4391,6 @@ static int load_balance(int this_cpu, struct rq *this_rq, | |||
4400 | { | 4391 | { |
4401 | int ld_moved, active_balance = 0; | 4392 | int ld_moved, active_balance = 0; |
4402 | struct sched_group *group; | 4393 | struct sched_group *group; |
4403 | unsigned long imbalance; | ||
4404 | struct rq *busiest; | 4394 | struct rq *busiest; |
4405 | unsigned long flags; | 4395 | unsigned long flags; |
4406 | struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask); | 4396 | struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask); |
@@ -4418,8 +4408,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, | |||
4418 | schedstat_inc(sd, lb_count[idle]); | 4408 | schedstat_inc(sd, lb_count[idle]); |
4419 | 4409 | ||
4420 | redo: | 4410 | redo: |
4421 | group = find_busiest_group(sd, this_cpu, &imbalance, idle, | 4411 | group = find_busiest_group(&env, cpus, balance); |
4422 | cpus, balance); | ||
4423 | 4412 | ||
4424 | if (*balance == 0) | 4413 | if (*balance == 0) |
4425 | goto out_balanced; | 4414 | goto out_balanced; |
@@ -4429,7 +4418,7 @@ redo: | |||
4429 | goto out_balanced; | 4418 | goto out_balanced; |
4430 | } | 4419 | } |
4431 | 4420 | ||
4432 | busiest = find_busiest_queue(sd, group, idle, imbalance, cpus); | 4421 | busiest = find_busiest_queue(&env, group, cpus); |
4433 | if (!busiest) { | 4422 | if (!busiest) { |
4434 | schedstat_inc(sd, lb_nobusyq[idle]); | 4423 | schedstat_inc(sd, lb_nobusyq[idle]); |
4435 | goto out_balanced; | 4424 | goto out_balanced; |
@@ -4437,7 +4426,7 @@ redo: | |||
4437 | 4426 | ||
4438 | BUG_ON(busiest == this_rq); | 4427 | BUG_ON(busiest == this_rq); |
4439 | 4428 | ||
4440 | schedstat_add(sd, lb_imbalance[idle], imbalance); | 4429 | schedstat_add(sd, lb_imbalance[idle], env.imbalance); |
4441 | 4430 | ||
4442 | ld_moved = 0; | 4431 | ld_moved = 0; |
4443 | if (busiest->nr_running > 1) { | 4432 | if (busiest->nr_running > 1) { |
@@ -4448,7 +4437,6 @@ redo: | |||
4448 | * correctly treated as an imbalance. | 4437 | * correctly treated as an imbalance. |
4449 | */ | 4438 | */ |
4450 | env.flags |= LBF_ALL_PINNED; | 4439 | env.flags |= LBF_ALL_PINNED; |
4451 | env.load_move = imbalance; | ||
4452 | env.src_cpu = busiest->cpu; | 4440 | env.src_cpu = busiest->cpu; |
4453 | env.src_rq = busiest; | 4441 | env.src_rq = busiest; |
4454 | env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); | 4442 | env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); |
@@ -4493,7 +4481,7 @@ more_balance: | |||
4493 | if (idle != CPU_NEWLY_IDLE) | 4481 | if (idle != CPU_NEWLY_IDLE) |
4494 | sd->nr_balance_failed++; | 4482 | sd->nr_balance_failed++; |
4495 | 4483 | ||
4496 | if (need_active_balance(sd, idle, cpu_of(busiest), this_cpu)) { | 4484 | if (need_active_balance(&env)) { |
4497 | raw_spin_lock_irqsave(&busiest->lock, flags); | 4485 | raw_spin_lock_irqsave(&busiest->lock, flags); |
4498 | 4486 | ||
4499 | /* don't kick the active_load_balance_cpu_stop, | 4487 | /* don't kick the active_load_balance_cpu_stop, |
@@ -4520,10 +4508,11 @@ more_balance: | |||
4520 | } | 4508 | } |
4521 | raw_spin_unlock_irqrestore(&busiest->lock, flags); | 4509 | raw_spin_unlock_irqrestore(&busiest->lock, flags); |
4522 | 4510 | ||
4523 | if (active_balance) | 4511 | if (active_balance) { |
4524 | stop_one_cpu_nowait(cpu_of(busiest), | 4512 | stop_one_cpu_nowait(cpu_of(busiest), |
4525 | active_load_balance_cpu_stop, busiest, | 4513 | active_load_balance_cpu_stop, busiest, |
4526 | &busiest->active_balance_work); | 4514 | &busiest->active_balance_work); |
4515 | } | ||
4527 | 4516 | ||
4528 | /* | 4517 | /* |
4529 | * We've kicked active balancing, reset the failure | 4518 | * We've kicked active balancing, reset the failure |