aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched_fair.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched_fair.c')
-rw-r--r--kernel/sched_fair.c2003
1 files changed, 1769 insertions, 234 deletions
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 8fe7ee81c552..217e4a9393e4 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -35,8 +35,8 @@
35 * (to see the precise effective timeslice length of your workload, 35 * (to see the precise effective timeslice length of your workload,
36 * run vmstat and monitor the context-switches (cs) field) 36 * run vmstat and monitor the context-switches (cs) field)
37 */ 37 */
38unsigned int sysctl_sched_latency = 5000000ULL; 38unsigned int sysctl_sched_latency = 6000000ULL;
39unsigned int normalized_sysctl_sched_latency = 5000000ULL; 39unsigned int normalized_sysctl_sched_latency = 6000000ULL;
40 40
41/* 41/*
42 * The initial- and re-scaling of tunables is configurable 42 * The initial- and re-scaling of tunables is configurable
@@ -52,15 +52,15 @@ enum sched_tunable_scaling sysctl_sched_tunable_scaling
52 52
53/* 53/*
54 * Minimal preemption granularity for CPU-bound tasks: 54 * Minimal preemption granularity for CPU-bound tasks:
55 * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds) 55 * (default: 2 msec * (1 + ilog(ncpus)), units: nanoseconds)
56 */ 56 */
57unsigned int sysctl_sched_min_granularity = 1000000ULL; 57unsigned int sysctl_sched_min_granularity = 2000000ULL;
58unsigned int normalized_sysctl_sched_min_granularity = 1000000ULL; 58unsigned int normalized_sysctl_sched_min_granularity = 2000000ULL;
59 59
60/* 60/*
61 * is kept at sysctl_sched_latency / sysctl_sched_min_granularity 61 * is kept at sysctl_sched_latency / sysctl_sched_min_granularity
62 */ 62 */
63static unsigned int sched_nr_latency = 5; 63static unsigned int sched_nr_latency = 3;
64 64
65/* 65/*
66 * After fork, child runs first. If set to 0 (default) then 66 * After fork, child runs first. If set to 0 (default) then
@@ -505,7 +505,8 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
505{ 505{
506 unsigned long delta_exec_weighted; 506 unsigned long delta_exec_weighted;
507 507
508 schedstat_set(curr->exec_max, max((u64)delta_exec, curr->exec_max)); 508 schedstat_set(curr->statistics.exec_max,
509 max((u64)delta_exec, curr->statistics.exec_max));
509 510
510 curr->sum_exec_runtime += delta_exec; 511 curr->sum_exec_runtime += delta_exec;
511 schedstat_add(cfs_rq, exec_clock, delta_exec); 512 schedstat_add(cfs_rq, exec_clock, delta_exec);
@@ -548,7 +549,7 @@ static void update_curr(struct cfs_rq *cfs_rq)
548static inline void 549static inline void
549update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 550update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
550{ 551{
551 schedstat_set(se->wait_start, rq_of(cfs_rq)->clock); 552 schedstat_set(se->statistics.wait_start, rq_of(cfs_rq)->clock);
552} 553}
553 554
554/* 555/*
@@ -567,18 +568,18 @@ static void update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
567static void 568static void
568update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se) 569update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
569{ 570{
570 schedstat_set(se->wait_max, max(se->wait_max, 571 schedstat_set(se->statistics.wait_max, max(se->statistics.wait_max,
571 rq_of(cfs_rq)->clock - se->wait_start)); 572 rq_of(cfs_rq)->clock - se->statistics.wait_start));
572 schedstat_set(se->wait_count, se->wait_count + 1); 573 schedstat_set(se->statistics.wait_count, se->statistics.wait_count + 1);
573 schedstat_set(se->wait_sum, se->wait_sum + 574 schedstat_set(se->statistics.wait_sum, se->statistics.wait_sum +
574 rq_of(cfs_rq)->clock - se->wait_start); 575 rq_of(cfs_rq)->clock - se->statistics.wait_start);
575#ifdef CONFIG_SCHEDSTATS 576#ifdef CONFIG_SCHEDSTATS
576 if (entity_is_task(se)) { 577 if (entity_is_task(se)) {
577 trace_sched_stat_wait(task_of(se), 578 trace_sched_stat_wait(task_of(se),
578 rq_of(cfs_rq)->clock - se->wait_start); 579 rq_of(cfs_rq)->clock - se->statistics.wait_start);
579 } 580 }
580#endif 581#endif
581 schedstat_set(se->wait_start, 0); 582 schedstat_set(se->statistics.wait_start, 0);
582} 583}
583 584
584static inline void 585static inline void
@@ -657,39 +658,39 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
657 if (entity_is_task(se)) 658 if (entity_is_task(se))
658 tsk = task_of(se); 659 tsk = task_of(se);
659 660
660 if (se->sleep_start) { 661 if (se->statistics.sleep_start) {
661 u64 delta = rq_of(cfs_rq)->clock - se->sleep_start; 662 u64 delta = rq_of(cfs_rq)->clock - se->statistics.sleep_start;
662 663
663 if ((s64)delta < 0) 664 if ((s64)delta < 0)
664 delta = 0; 665 delta = 0;
665 666
666 if (unlikely(delta > se->sleep_max)) 667 if (unlikely(delta > se->statistics.sleep_max))
667 se->sleep_max = delta; 668 se->statistics.sleep_max = delta;
668 669
669 se->sleep_start = 0; 670 se->statistics.sleep_start = 0;
670 se->sum_sleep_runtime += delta; 671 se->statistics.sum_sleep_runtime += delta;
671 672
672 if (tsk) { 673 if (tsk) {
673 account_scheduler_latency(tsk, delta >> 10, 1); 674 account_scheduler_latency(tsk, delta >> 10, 1);
674 trace_sched_stat_sleep(tsk, delta); 675 trace_sched_stat_sleep(tsk, delta);
675 } 676 }
676 } 677 }
677 if (se->block_start) { 678 if (se->statistics.block_start) {
678 u64 delta = rq_of(cfs_rq)->clock - se->block_start; 679 u64 delta = rq_of(cfs_rq)->clock - se->statistics.block_start;
679 680
680 if ((s64)delta < 0) 681 if ((s64)delta < 0)
681 delta = 0; 682 delta = 0;
682 683
683 if (unlikely(delta > se->block_max)) 684 if (unlikely(delta > se->statistics.block_max))
684 se->block_max = delta; 685 se->statistics.block_max = delta;
685 686
686 se->block_start = 0; 687 se->statistics.block_start = 0;
687 se->sum_sleep_runtime += delta; 688 se->statistics.sum_sleep_runtime += delta;
688 689
689 if (tsk) { 690 if (tsk) {
690 if (tsk->in_iowait) { 691 if (tsk->in_iowait) {
691 se->iowait_sum += delta; 692 se->statistics.iowait_sum += delta;
692 se->iowait_count++; 693 se->statistics.iowait_count++;
693 trace_sched_stat_iowait(tsk, delta); 694 trace_sched_stat_iowait(tsk, delta);
694 } 695 }
695 696
@@ -737,20 +738,10 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
737 vruntime += sched_vslice(cfs_rq, se); 738 vruntime += sched_vslice(cfs_rq, se);
738 739
739 /* sleeps up to a single latency don't count. */ 740 /* sleeps up to a single latency don't count. */
740 if (!initial && sched_feat(FAIR_SLEEPERS)) { 741 if (!initial) {
741 unsigned long thresh = sysctl_sched_latency; 742 unsigned long thresh = sysctl_sched_latency;
742 743
743 /* 744 /*
744 * Convert the sleeper threshold into virtual time.
745 * SCHED_IDLE is a special sub-class. We care about
746 * fairness only relative to other SCHED_IDLE tasks,
747 * all of which have the same weight.
748 */
749 if (sched_feat(NORMALIZED_SLEEPER) && (!entity_is_task(se) ||
750 task_of(se)->policy != SCHED_IDLE))
751 thresh = calc_delta_fair(thresh, se);
752
753 /*
754 * Halve their sleep time's effect, to allow 745 * Halve their sleep time's effect, to allow
755 * for a gentler effect of sleepers: 746 * for a gentler effect of sleepers:
756 */ 747 */
@@ -766,9 +757,6 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
766 se->vruntime = vruntime; 757 se->vruntime = vruntime;
767} 758}
768 759
769#define ENQUEUE_WAKEUP 1
770#define ENQUEUE_MIGRATE 2
771
772static void 760static void
773enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 761enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
774{ 762{
@@ -776,7 +764,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
776 * Update the normalized vruntime before updating min_vruntime 764 * Update the normalized vruntime before updating min_vruntime
777 * through callig update_curr(). 765 * through callig update_curr().
778 */ 766 */
779 if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATE)) 767 if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_WAKING))
780 se->vruntime += cfs_rq->min_vruntime; 768 se->vruntime += cfs_rq->min_vruntime;
781 769
782 /* 770 /*
@@ -812,7 +800,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
812} 800}
813 801
814static void 802static void
815dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) 803dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
816{ 804{
817 /* 805 /*
818 * Update run-time statistics of the 'current'. 806 * Update run-time statistics of the 'current'.
@@ -820,15 +808,15 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
820 update_curr(cfs_rq); 808 update_curr(cfs_rq);
821 809
822 update_stats_dequeue(cfs_rq, se); 810 update_stats_dequeue(cfs_rq, se);
823 if (sleep) { 811 if (flags & DEQUEUE_SLEEP) {
824#ifdef CONFIG_SCHEDSTATS 812#ifdef CONFIG_SCHEDSTATS
825 if (entity_is_task(se)) { 813 if (entity_is_task(se)) {
826 struct task_struct *tsk = task_of(se); 814 struct task_struct *tsk = task_of(se);
827 815
828 if (tsk->state & TASK_INTERRUPTIBLE) 816 if (tsk->state & TASK_INTERRUPTIBLE)
829 se->sleep_start = rq_of(cfs_rq)->clock; 817 se->statistics.sleep_start = rq_of(cfs_rq)->clock;
830 if (tsk->state & TASK_UNINTERRUPTIBLE) 818 if (tsk->state & TASK_UNINTERRUPTIBLE)
831 se->block_start = rq_of(cfs_rq)->clock; 819 se->statistics.block_start = rq_of(cfs_rq)->clock;
832 } 820 }
833#endif 821#endif
834 } 822 }
@@ -845,7 +833,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
845 * update can refer to the ->curr item and we need to reflect this 833 * update can refer to the ->curr item and we need to reflect this
846 * movement in our normalized position. 834 * movement in our normalized position.
847 */ 835 */
848 if (!sleep) 836 if (!(flags & DEQUEUE_SLEEP))
849 se->vruntime -= cfs_rq->min_vruntime; 837 se->vruntime -= cfs_rq->min_vruntime;
850} 838}
851 839
@@ -912,7 +900,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
912 * when there are only lesser-weight tasks around): 900 * when there are only lesser-weight tasks around):
913 */ 901 */
914 if (rq_of(cfs_rq)->load.weight >= 2*se->load.weight) { 902 if (rq_of(cfs_rq)->load.weight >= 2*se->load.weight) {
915 se->slice_max = max(se->slice_max, 903 se->statistics.slice_max = max(se->statistics.slice_max,
916 se->sum_exec_runtime - se->prev_sum_exec_runtime); 904 se->sum_exec_runtime - se->prev_sum_exec_runtime);
917 } 905 }
918#endif 906#endif
@@ -1053,16 +1041,11 @@ static inline void hrtick_update(struct rq *rq)
1053 * increased. Here we update the fair scheduling stats and 1041 * increased. Here we update the fair scheduling stats and
1054 * then put the task into the rbtree: 1042 * then put the task into the rbtree:
1055 */ 1043 */
1056static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup) 1044static void
1045enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
1057{ 1046{
1058 struct cfs_rq *cfs_rq; 1047 struct cfs_rq *cfs_rq;
1059 struct sched_entity *se = &p->se; 1048 struct sched_entity *se = &p->se;
1060 int flags = 0;
1061
1062 if (wakeup)
1063 flags |= ENQUEUE_WAKEUP;
1064 if (p->state == TASK_WAKING)
1065 flags |= ENQUEUE_MIGRATE;
1066 1049
1067 for_each_sched_entity(se) { 1050 for_each_sched_entity(se) {
1068 if (se->on_rq) 1051 if (se->on_rq)
@@ -1080,18 +1063,18 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup)
1080 * decreased. We remove the task from the rbtree and 1063 * decreased. We remove the task from the rbtree and
1081 * update the fair scheduling stats: 1064 * update the fair scheduling stats:
1082 */ 1065 */
1083static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int sleep) 1066static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
1084{ 1067{
1085 struct cfs_rq *cfs_rq; 1068 struct cfs_rq *cfs_rq;
1086 struct sched_entity *se = &p->se; 1069 struct sched_entity *se = &p->se;
1087 1070
1088 for_each_sched_entity(se) { 1071 for_each_sched_entity(se) {
1089 cfs_rq = cfs_rq_of(se); 1072 cfs_rq = cfs_rq_of(se);
1090 dequeue_entity(cfs_rq, se, sleep); 1073 dequeue_entity(cfs_rq, se, flags);
1091 /* Don't dequeue parent if it has other entities besides us */ 1074 /* Don't dequeue parent if it has other entities besides us */
1092 if (cfs_rq->load.weight) 1075 if (cfs_rq->load.weight)
1093 break; 1076 break;
1094 sleep = 1; 1077 flags |= DEQUEUE_SLEEP;
1095 } 1078 }
1096 1079
1097 hrtick_update(rq); 1080 hrtick_update(rq);
@@ -1239,7 +1222,6 @@ static inline unsigned long effective_load(struct task_group *tg, int cpu,
1239 1222
1240static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync) 1223static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
1241{ 1224{
1242 struct task_struct *curr = current;
1243 unsigned long this_load, load; 1225 unsigned long this_load, load;
1244 int idx, this_cpu, prev_cpu; 1226 int idx, this_cpu, prev_cpu;
1245 unsigned long tl_per_task; 1227 unsigned long tl_per_task;
@@ -1254,18 +1236,6 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
1254 load = source_load(prev_cpu, idx); 1236 load = source_load(prev_cpu, idx);
1255 this_load = target_load(this_cpu, idx); 1237 this_load = target_load(this_cpu, idx);
1256 1238
1257 if (sync) {
1258 if (sched_feat(SYNC_LESS) &&
1259 (curr->se.avg_overlap > sysctl_sched_migration_cost ||
1260 p->se.avg_overlap > sysctl_sched_migration_cost))
1261 sync = 0;
1262 } else {
1263 if (sched_feat(SYNC_MORE) &&
1264 (curr->se.avg_overlap < sysctl_sched_migration_cost &&
1265 p->se.avg_overlap < sysctl_sched_migration_cost))
1266 sync = 1;
1267 }
1268
1269 /* 1239 /*
1270 * If sync wakeup then subtract the (maximum possible) 1240 * If sync wakeup then subtract the (maximum possible)
1271 * effect of the currently running task from the load 1241 * effect of the currently running task from the load
@@ -1305,7 +1275,7 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
1305 if (sync && balanced) 1275 if (sync && balanced)
1306 return 1; 1276 return 1;
1307 1277
1308 schedstat_inc(p, se.nr_wakeups_affine_attempts); 1278 schedstat_inc(p, se.statistics.nr_wakeups_affine_attempts);
1309 tl_per_task = cpu_avg_load_per_task(this_cpu); 1279 tl_per_task = cpu_avg_load_per_task(this_cpu);
1310 1280
1311 if (balanced || 1281 if (balanced ||
@@ -1317,7 +1287,7 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
1317 * there is no bad imbalance. 1287 * there is no bad imbalance.
1318 */ 1288 */
1319 schedstat_inc(sd, ttwu_move_affine); 1289 schedstat_inc(sd, ttwu_move_affine);
1320 schedstat_inc(p, se.nr_wakeups_affine); 1290 schedstat_inc(p, se.statistics.nr_wakeups_affine);
1321 1291
1322 return 1; 1292 return 1;
1323 } 1293 }
@@ -1405,29 +1375,48 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
1405/* 1375/*
1406 * Try and locate an idle CPU in the sched_domain. 1376 * Try and locate an idle CPU in the sched_domain.
1407 */ 1377 */
1408static int 1378static int select_idle_sibling(struct task_struct *p, int target)
1409select_idle_sibling(struct task_struct *p, struct sched_domain *sd, int target)
1410{ 1379{
1411 int cpu = smp_processor_id(); 1380 int cpu = smp_processor_id();
1412 int prev_cpu = task_cpu(p); 1381 int prev_cpu = task_cpu(p);
1382 struct sched_domain *sd;
1413 int i; 1383 int i;
1414 1384
1415 /* 1385 /*
1416 * If this domain spans both cpu and prev_cpu (see the SD_WAKE_AFFINE 1386 * If the task is going to be woken-up on this cpu and if it is
1417 * test in select_task_rq_fair) and the prev_cpu is idle then that's 1387 * already idle, then it is the right target.
1418 * always a better target than the current cpu. 1388 */
1389 if (target == cpu && idle_cpu(cpu))
1390 return cpu;
1391
1392 /*
1393 * If the task is going to be woken-up on the cpu where it previously
1394 * ran and if it is currently idle, then it the right target.
1419 */ 1395 */
1420 if (target == cpu && !cpu_rq(prev_cpu)->cfs.nr_running) 1396 if (target == prev_cpu && idle_cpu(prev_cpu))
1421 return prev_cpu; 1397 return prev_cpu;
1422 1398
1423 /* 1399 /*
1424 * Otherwise, iterate the domain and find an elegible idle cpu. 1400 * Otherwise, iterate the domains and find an elegible idle cpu.
1425 */ 1401 */
1426 for_each_cpu_and(i, sched_domain_span(sd), &p->cpus_allowed) { 1402 for_each_domain(target, sd) {
1427 if (!cpu_rq(i)->cfs.nr_running) { 1403 if (!(sd->flags & SD_SHARE_PKG_RESOURCES))
1428 target = i;
1429 break; 1404 break;
1405
1406 for_each_cpu_and(i, sched_domain_span(sd), &p->cpus_allowed) {
1407 if (idle_cpu(i)) {
1408 target = i;
1409 break;
1410 }
1430 } 1411 }
1412
1413 /*
1414 * Lets stop looking for an idle sibling when we reached
1415 * the domain that spans the current cpu and prev_cpu.
1416 */
1417 if (cpumask_test_cpu(cpu, sched_domain_span(sd)) &&
1418 cpumask_test_cpu(prev_cpu, sched_domain_span(sd)))
1419 break;
1431 } 1420 }
1432 1421
1433 return target; 1422 return target;
@@ -1444,7 +1433,8 @@ select_idle_sibling(struct task_struct *p, struct sched_domain *sd, int target)
1444 * 1433 *
1445 * preempt must be disabled. 1434 * preempt must be disabled.
1446 */ 1435 */
1447static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags) 1436static int
1437select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_flags)
1448{ 1438{
1449 struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL; 1439 struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
1450 int cpu = smp_processor_id(); 1440 int cpu = smp_processor_id();
@@ -1455,8 +1445,7 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag
1455 int sync = wake_flags & WF_SYNC; 1445 int sync = wake_flags & WF_SYNC;
1456 1446
1457 if (sd_flag & SD_BALANCE_WAKE) { 1447 if (sd_flag & SD_BALANCE_WAKE) {
1458 if (sched_feat(AFFINE_WAKEUPS) && 1448 if (cpumask_test_cpu(cpu, &p->cpus_allowed))
1459 cpumask_test_cpu(cpu, &p->cpus_allowed))
1460 want_affine = 1; 1449 want_affine = 1;
1461 new_cpu = prev_cpu; 1450 new_cpu = prev_cpu;
1462 } 1451 }
@@ -1490,34 +1479,13 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag
1490 } 1479 }
1491 1480
1492 /* 1481 /*
1493 * While iterating the domains looking for a spanning 1482 * If both cpu and prev_cpu are part of this domain,
1494 * WAKE_AFFINE domain, adjust the affine target to any idle cpu 1483 * cpu is a valid SD_WAKE_AFFINE target.
1495 * in cache sharing domains along the way.
1496 */ 1484 */
1497 if (want_affine) { 1485 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
1498 int target = -1; 1486 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
1499 1487 affine_sd = tmp;
1500 /* 1488 want_affine = 0;
1501 * If both cpu and prev_cpu are part of this domain,
1502 * cpu is a valid SD_WAKE_AFFINE target.
1503 */
1504 if (cpumask_test_cpu(prev_cpu, sched_domain_span(tmp)))
1505 target = cpu;
1506
1507 /*
1508 * If there's an idle sibling in this domain, make that
1509 * the wake_affine target instead of the current cpu.
1510 */
1511 if (tmp->flags & SD_SHARE_PKG_RESOURCES)
1512 target = select_idle_sibling(p, tmp, target);
1513
1514 if (target >= 0) {
1515 if (tmp->flags & SD_WAKE_AFFINE) {
1516 affine_sd = tmp;
1517 want_affine = 0;
1518 }
1519 cpu = target;
1520 }
1521 } 1489 }
1522 1490
1523 if (!want_sd && !want_affine) 1491 if (!want_sd && !want_affine)
@@ -1530,22 +1498,29 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag
1530 sd = tmp; 1498 sd = tmp;
1531 } 1499 }
1532 1500
1501#ifdef CONFIG_FAIR_GROUP_SCHED
1533 if (sched_feat(LB_SHARES_UPDATE)) { 1502 if (sched_feat(LB_SHARES_UPDATE)) {
1534 /* 1503 /*
1535 * Pick the largest domain to update shares over 1504 * Pick the largest domain to update shares over
1536 */ 1505 */
1537 tmp = sd; 1506 tmp = sd;
1538 if (affine_sd && (!tmp || 1507 if (affine_sd && (!tmp || affine_sd->span_weight > sd->span_weight))
1539 cpumask_weight(sched_domain_span(affine_sd)) >
1540 cpumask_weight(sched_domain_span(sd))))
1541 tmp = affine_sd; 1508 tmp = affine_sd;
1542 1509
1543 if (tmp) 1510 if (tmp) {
1511 raw_spin_unlock(&rq->lock);
1544 update_shares(tmp); 1512 update_shares(tmp);
1513 raw_spin_lock(&rq->lock);
1514 }
1545 } 1515 }
1516#endif
1546 1517
1547 if (affine_sd && wake_affine(affine_sd, p, sync)) 1518 if (affine_sd) {
1548 return cpu; 1519 if (cpu == prev_cpu || wake_affine(affine_sd, p, sync))
1520 return select_idle_sibling(p, cpu);
1521 else
1522 return select_idle_sibling(p, prev_cpu);
1523 }
1549 1524
1550 while (sd) { 1525 while (sd) {
1551 int load_idx = sd->forkexec_idx; 1526 int load_idx = sd->forkexec_idx;
@@ -1575,10 +1550,10 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag
1575 1550
1576 /* Now try balancing at a lower domain level of new_cpu */ 1551 /* Now try balancing at a lower domain level of new_cpu */
1577 cpu = new_cpu; 1552 cpu = new_cpu;
1578 weight = cpumask_weight(sched_domain_span(sd)); 1553 weight = sd->span_weight;
1579 sd = NULL; 1554 sd = NULL;
1580 for_each_domain(cpu, tmp) { 1555 for_each_domain(cpu, tmp) {
1581 if (weight <= cpumask_weight(sched_domain_span(tmp))) 1556 if (weight <= tmp->span_weight)
1582 break; 1557 break;
1583 if (tmp->flags & sd_flag) 1558 if (tmp->flags & sd_flag)
1584 sd = tmp; 1559 sd = tmp;
@@ -1590,63 +1565,26 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag
1590} 1565}
1591#endif /* CONFIG_SMP */ 1566#endif /* CONFIG_SMP */
1592 1567
1593/*
1594 * Adaptive granularity
1595 *
1596 * se->avg_wakeup gives the average time a task runs until it does a wakeup,
1597 * with the limit of wakeup_gran -- when it never does a wakeup.
1598 *
1599 * So the smaller avg_wakeup is the faster we want this task to preempt,
1600 * but we don't want to treat the preemptee unfairly and therefore allow it
1601 * to run for at least the amount of time we'd like to run.
1602 *
1603 * NOTE: we use 2*avg_wakeup to increase the probability of actually doing one
1604 *
1605 * NOTE: we use *nr_running to scale with load, this nicely matches the
1606 * degrading latency on load.
1607 */
1608static unsigned long
1609adaptive_gran(struct sched_entity *curr, struct sched_entity *se)
1610{
1611 u64 this_run = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
1612 u64 expected_wakeup = 2*se->avg_wakeup * cfs_rq_of(se)->nr_running;
1613 u64 gran = 0;
1614
1615 if (this_run < expected_wakeup)
1616 gran = expected_wakeup - this_run;
1617
1618 return min_t(s64, gran, sysctl_sched_wakeup_granularity);
1619}
1620
1621static unsigned long 1568static unsigned long
1622wakeup_gran(struct sched_entity *curr, struct sched_entity *se) 1569wakeup_gran(struct sched_entity *curr, struct sched_entity *se)
1623{ 1570{
1624 unsigned long gran = sysctl_sched_wakeup_granularity; 1571 unsigned long gran = sysctl_sched_wakeup_granularity;
1625 1572
1626 if (cfs_rq_of(curr)->curr && sched_feat(ADAPTIVE_GRAN))
1627 gran = adaptive_gran(curr, se);
1628
1629 /* 1573 /*
1630 * Since its curr running now, convert the gran from real-time 1574 * Since its curr running now, convert the gran from real-time
1631 * to virtual-time in his units. 1575 * to virtual-time in his units.
1576 *
1577 * By using 'se' instead of 'curr' we penalize light tasks, so
1578 * they get preempted easier. That is, if 'se' < 'curr' then
1579 * the resulting gran will be larger, therefore penalizing the
1580 * lighter, if otoh 'se' > 'curr' then the resulting gran will
1581 * be smaller, again penalizing the lighter task.
1582 *
1583 * This is especially important for buddies when the leftmost
1584 * task is higher priority than the buddy.
1632 */ 1585 */
1633 if (sched_feat(ASYM_GRAN)) { 1586 if (unlikely(se->load.weight != NICE_0_LOAD))
1634 /* 1587 gran = calc_delta_fair(gran, se);
1635 * By using 'se' instead of 'curr' we penalize light tasks, so
1636 * they get preempted easier. That is, if 'se' < 'curr' then
1637 * the resulting gran will be larger, therefore penalizing the
1638 * lighter, if otoh 'se' > 'curr' then the resulting gran will
1639 * be smaller, again penalizing the lighter task.
1640 *
1641 * This is especially important for buddies when the leftmost
1642 * task is higher priority than the buddy.
1643 */
1644 if (unlikely(se->load.weight != NICE_0_LOAD))
1645 gran = calc_delta_fair(gran, se);
1646 } else {
1647 if (unlikely(curr->load.weight != NICE_0_LOAD))
1648 gran = calc_delta_fair(gran, curr);
1649 }
1650 1588
1651 return gran; 1589 return gran;
1652} 1590}
@@ -1704,7 +1642,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
1704 struct task_struct *curr = rq->curr; 1642 struct task_struct *curr = rq->curr;
1705 struct sched_entity *se = &curr->se, *pse = &p->se; 1643 struct sched_entity *se = &curr->se, *pse = &p->se;
1706 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 1644 struct cfs_rq *cfs_rq = task_cfs_rq(curr);
1707 int sync = wake_flags & WF_SYNC;
1708 int scale = cfs_rq->nr_running >= sched_nr_latency; 1645 int scale = cfs_rq->nr_running >= sched_nr_latency;
1709 1646
1710 if (unlikely(rt_prio(p->prio))) 1647 if (unlikely(rt_prio(p->prio)))
@@ -1737,14 +1674,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_
1737 if (unlikely(curr->policy == SCHED_IDLE)) 1674 if (unlikely(curr->policy == SCHED_IDLE))
1738 goto preempt; 1675 goto preempt;
1739 1676
1740 if (sched_feat(WAKEUP_SYNC) && sync)
1741 goto preempt;
1742
1743 if (sched_feat(WAKEUP_OVERLAP) &&
1744 se->avg_overlap < sysctl_sched_migration_cost &&
1745 pse->avg_overlap < sysctl_sched_migration_cost)
1746 goto preempt;
1747
1748 if (!sched_feat(WAKEUP_PREEMPT)) 1677 if (!sched_feat(WAKEUP_PREEMPT))
1749 return; 1678 return;
1750 1679
@@ -1815,57 +1744,164 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
1815 */ 1744 */
1816 1745
1817/* 1746/*
1818 * Load-balancing iterator. Note: while the runqueue stays locked 1747 * pull_task - move a task from a remote runqueue to the local runqueue.
1819 * during the whole iteration, the current task might be 1748 * Both runqueues must be locked.
1820 * dequeued so the iterator has to be dequeue-safe. Here we
1821 * achieve that by always pre-iterating before returning
1822 * the current task:
1823 */ 1749 */
1824static struct task_struct * 1750static void pull_task(struct rq *src_rq, struct task_struct *p,
1825__load_balance_iterator(struct cfs_rq *cfs_rq, struct list_head *next) 1751 struct rq *this_rq, int this_cpu)
1826{ 1752{
1827 struct task_struct *p = NULL; 1753 deactivate_task(src_rq, p, 0);
1828 struct sched_entity *se; 1754 set_task_cpu(p, this_cpu);
1755 activate_task(this_rq, p, 0);
1756 check_preempt_curr(this_rq, p, 0);
1757}
1829 1758
1830 if (next == &cfs_rq->tasks) 1759/*
1831 return NULL; 1760 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
1761 */
1762static
1763int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
1764 struct sched_domain *sd, enum cpu_idle_type idle,
1765 int *all_pinned)
1766{
1767 int tsk_cache_hot = 0;
1768 /*
1769 * We do not migrate tasks that are:
1770 * 1) running (obviously), or
1771 * 2) cannot be migrated to this CPU due to cpus_allowed, or
1772 * 3) are cache-hot on their current CPU.
1773 */
1774 if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
1775 schedstat_inc(p, se.statistics.nr_failed_migrations_affine);
1776 return 0;
1777 }
1778 *all_pinned = 0;
1832 1779
1833 se = list_entry(next, struct sched_entity, group_node); 1780 if (task_running(rq, p)) {
1834 p = task_of(se); 1781 schedstat_inc(p, se.statistics.nr_failed_migrations_running);
1835 cfs_rq->balance_iterator = next->next; 1782 return 0;
1783 }
1836 1784
1837 return p; 1785 /*
1838} 1786 * Aggressive migration if:
1787 * 1) task is cache cold, or
1788 * 2) too many balance attempts have failed.
1789 */
1839 1790
1840static struct task_struct *load_balance_start_fair(void *arg) 1791 tsk_cache_hot = task_hot(p, rq->clock, sd);
1841{ 1792 if (!tsk_cache_hot ||
1842 struct cfs_rq *cfs_rq = arg; 1793 sd->nr_balance_failed > sd->cache_nice_tries) {
1794#ifdef CONFIG_SCHEDSTATS
1795 if (tsk_cache_hot) {
1796 schedstat_inc(sd, lb_hot_gained[idle]);
1797 schedstat_inc(p, se.statistics.nr_forced_migrations);
1798 }
1799#endif
1800 return 1;
1801 }
1843 1802
1844 return __load_balance_iterator(cfs_rq, cfs_rq->tasks.next); 1803 if (tsk_cache_hot) {
1804 schedstat_inc(p, se.statistics.nr_failed_migrations_hot);
1805 return 0;
1806 }
1807 return 1;
1845} 1808}
1846 1809
1847static struct task_struct *load_balance_next_fair(void *arg) 1810/*
1811 * move_one_task tries to move exactly one task from busiest to this_rq, as
1812 * part of active balancing operations within "domain".
1813 * Returns 1 if successful and 0 otherwise.
1814 *
1815 * Called with both runqueues locked.
1816 */
1817static int
1818move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
1819 struct sched_domain *sd, enum cpu_idle_type idle)
1848{ 1820{
1849 struct cfs_rq *cfs_rq = arg; 1821 struct task_struct *p, *n;
1822 struct cfs_rq *cfs_rq;
1823 int pinned = 0;
1824
1825 for_each_leaf_cfs_rq(busiest, cfs_rq) {
1826 list_for_each_entry_safe(p, n, &cfs_rq->tasks, se.group_node) {
1827
1828 if (!can_migrate_task(p, busiest, this_cpu,
1829 sd, idle, &pinned))
1830 continue;
1850 1831
1851 return __load_balance_iterator(cfs_rq, cfs_rq->balance_iterator); 1832 pull_task(busiest, p, this_rq, this_cpu);
1833 /*
1834 * Right now, this is only the second place pull_task()
1835 * is called, so we can safely collect pull_task()
1836 * stats here rather than inside pull_task().
1837 */
1838 schedstat_inc(sd, lb_gained[idle]);
1839 return 1;
1840 }
1841 }
1842
1843 return 0;
1852} 1844}
1853 1845
1854static unsigned long 1846static unsigned long
1855__load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, 1847balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
1856 unsigned long max_load_move, struct sched_domain *sd, 1848 unsigned long max_load_move, struct sched_domain *sd,
1857 enum cpu_idle_type idle, int *all_pinned, int *this_best_prio, 1849 enum cpu_idle_type idle, int *all_pinned,
1858 struct cfs_rq *cfs_rq) 1850 int *this_best_prio, struct cfs_rq *busiest_cfs_rq)
1859{ 1851{
1860 struct rq_iterator cfs_rq_iterator; 1852 int loops = 0, pulled = 0, pinned = 0;
1853 long rem_load_move = max_load_move;
1854 struct task_struct *p, *n;
1861 1855
1862 cfs_rq_iterator.start = load_balance_start_fair; 1856 if (max_load_move == 0)
1863 cfs_rq_iterator.next = load_balance_next_fair; 1857 goto out;
1864 cfs_rq_iterator.arg = cfs_rq;
1865 1858
1866 return balance_tasks(this_rq, this_cpu, busiest, 1859 pinned = 1;
1867 max_load_move, sd, idle, all_pinned, 1860
1868 this_best_prio, &cfs_rq_iterator); 1861 list_for_each_entry_safe(p, n, &busiest_cfs_rq->tasks, se.group_node) {
1862 if (loops++ > sysctl_sched_nr_migrate)
1863 break;
1864
1865 if ((p->se.load.weight >> 1) > rem_load_move ||
1866 !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned))
1867 continue;
1868
1869 pull_task(busiest, p, this_rq, this_cpu);
1870 pulled++;
1871 rem_load_move -= p->se.load.weight;
1872
1873#ifdef CONFIG_PREEMPT
1874 /*
1875 * NEWIDLE balancing is a source of latency, so preemptible
1876 * kernels will stop after the first task is pulled to minimize
1877 * the critical section.
1878 */
1879 if (idle == CPU_NEWLY_IDLE)
1880 break;
1881#endif
1882
1883 /*
1884 * We only want to steal up to the prescribed amount of
1885 * weighted load.
1886 */
1887 if (rem_load_move <= 0)
1888 break;
1889
1890 if (p->prio < *this_best_prio)
1891 *this_best_prio = p->prio;
1892 }
1893out:
1894 /*
1895 * Right now, this is one of only two places pull_task() is called,
1896 * so we can safely collect pull_task() stats here rather than
1897 * inside pull_task().
1898 */
1899 schedstat_add(sd, lb_gained[idle], pulled);
1900
1901 if (all_pinned)
1902 *all_pinned = pinned;
1903
1904 return max_load_move - rem_load_move;
1869} 1905}
1870 1906
1871#ifdef CONFIG_FAIR_GROUP_SCHED 1907#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -1897,9 +1933,9 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
1897 rem_load = (u64)rem_load_move * busiest_weight; 1933 rem_load = (u64)rem_load_move * busiest_weight;
1898 rem_load = div_u64(rem_load, busiest_h_load + 1); 1934 rem_load = div_u64(rem_load, busiest_h_load + 1);
1899 1935
1900 moved_load = __load_balance_fair(this_rq, this_cpu, busiest, 1936 moved_load = balance_tasks(this_rq, this_cpu, busiest,
1901 rem_load, sd, idle, all_pinned, this_best_prio, 1937 rem_load, sd, idle, all_pinned, this_best_prio,
1902 tg->cfs_rq[busiest_cpu]); 1938 busiest_cfs_rq);
1903 1939
1904 if (!moved_load) 1940 if (!moved_load)
1905 continue; 1941 continue;
@@ -1922,35 +1958,1527 @@ load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
1922 struct sched_domain *sd, enum cpu_idle_type idle, 1958 struct sched_domain *sd, enum cpu_idle_type idle,
1923 int *all_pinned, int *this_best_prio) 1959 int *all_pinned, int *this_best_prio)
1924{ 1960{
1925 return __load_balance_fair(this_rq, this_cpu, busiest, 1961 return balance_tasks(this_rq, this_cpu, busiest,
1926 max_load_move, sd, idle, all_pinned, 1962 max_load_move, sd, idle, all_pinned,
1927 this_best_prio, &busiest->cfs); 1963 this_best_prio, &busiest->cfs);
1928} 1964}
1929#endif 1965#endif
1930 1966
1931static int 1967/*
1932move_one_task_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, 1968 * move_tasks tries to move up to max_load_move weighted load from busiest to
1933 struct sched_domain *sd, enum cpu_idle_type idle) 1969 * this_rq, as part of a balancing operation within domain "sd".
1970 * Returns 1 if successful and 0 otherwise.
1971 *
1972 * Called with both runqueues locked.
1973 */
1974static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
1975 unsigned long max_load_move,
1976 struct sched_domain *sd, enum cpu_idle_type idle,
1977 int *all_pinned)
1934{ 1978{
1935 struct cfs_rq *busy_cfs_rq; 1979 unsigned long total_load_moved = 0, load_moved;
1936 struct rq_iterator cfs_rq_iterator; 1980 int this_best_prio = this_rq->curr->prio;
1937 1981
1938 cfs_rq_iterator.start = load_balance_start_fair; 1982 do {
1939 cfs_rq_iterator.next = load_balance_next_fair; 1983 load_moved = load_balance_fair(this_rq, this_cpu, busiest,
1984 max_load_move - total_load_moved,
1985 sd, idle, all_pinned, &this_best_prio);
1986
1987 total_load_moved += load_moved;
1940 1988
1941 for_each_leaf_cfs_rq(busiest, busy_cfs_rq) { 1989#ifdef CONFIG_PREEMPT
1942 /* 1990 /*
1943 * pass busy_cfs_rq argument into 1991 * NEWIDLE balancing is a source of latency, so preemptible
1944 * load_balance_[start|next]_fair iterators 1992 * kernels will stop after the first task is pulled to minimize
1993 * the critical section.
1945 */ 1994 */
1946 cfs_rq_iterator.arg = busy_cfs_rq; 1995 if (idle == CPU_NEWLY_IDLE && this_rq->nr_running)
1947 if (iter_move_one_task(this_rq, this_cpu, busiest, sd, idle, 1996 break;
1948 &cfs_rq_iterator)) 1997
1949 return 1; 1998 if (raw_spin_is_contended(&this_rq->lock) ||
1999 raw_spin_is_contended(&busiest->lock))
2000 break;
2001#endif
2002 } while (load_moved && max_load_move > total_load_moved);
2003
2004 return total_load_moved > 0;
2005}
2006
2007/********** Helpers for find_busiest_group ************************/
2008/*
2009 * sd_lb_stats - Structure to store the statistics of a sched_domain
2010 * during load balancing.
2011 */
2012struct sd_lb_stats {
2013 struct sched_group *busiest; /* Busiest group in this sd */
2014 struct sched_group *this; /* Local group in this sd */
2015 unsigned long total_load; /* Total load of all groups in sd */
2016 unsigned long total_pwr; /* Total power of all groups in sd */
2017 unsigned long avg_load; /* Average load across all groups in sd */
2018
2019 /** Statistics of this group */
2020 unsigned long this_load;
2021 unsigned long this_load_per_task;
2022 unsigned long this_nr_running;
2023
2024 /* Statistics of the busiest group */
2025 unsigned long max_load;
2026 unsigned long busiest_load_per_task;
2027 unsigned long busiest_nr_running;
2028 unsigned long busiest_group_capacity;
2029
2030 int group_imb; /* Is there imbalance in this sd */
2031#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
2032 int power_savings_balance; /* Is powersave balance needed for this sd */
2033 struct sched_group *group_min; /* Least loaded group in sd */
2034 struct sched_group *group_leader; /* Group which relieves group_min */
2035 unsigned long min_load_per_task; /* load_per_task in group_min */
2036 unsigned long leader_nr_running; /* Nr running of group_leader */
2037 unsigned long min_nr_running; /* Nr running of group_min */
2038#endif
2039};
2040
2041/*
2042 * sg_lb_stats - stats of a sched_group required for load_balancing
2043 */
2044struct sg_lb_stats {
2045 unsigned long avg_load; /*Avg load across the CPUs of the group */
2046 unsigned long group_load; /* Total load over the CPUs of the group */
2047 unsigned long sum_nr_running; /* Nr tasks running in the group */
2048 unsigned long sum_weighted_load; /* Weighted load of group's tasks */
2049 unsigned long group_capacity;
2050 int group_imb; /* Is there an imbalance in the group ? */
2051};
2052
2053/**
2054 * group_first_cpu - Returns the first cpu in the cpumask of a sched_group.
2055 * @group: The group whose first cpu is to be returned.
2056 */
2057static inline unsigned int group_first_cpu(struct sched_group *group)
2058{
2059 return cpumask_first(sched_group_cpus(group));
2060}
2061
2062/**
2063 * get_sd_load_idx - Obtain the load index for a given sched domain.
2064 * @sd: The sched_domain whose load_idx is to be obtained.
2065 * @idle: The Idle status of the CPU for whose sd load_icx is obtained.
2066 */
2067static inline int get_sd_load_idx(struct sched_domain *sd,
2068 enum cpu_idle_type idle)
2069{
2070 int load_idx;
2071
2072 switch (idle) {
2073 case CPU_NOT_IDLE:
2074 load_idx = sd->busy_idx;
2075 break;
2076
2077 case CPU_NEWLY_IDLE:
2078 load_idx = sd->newidle_idx;
2079 break;
2080 default:
2081 load_idx = sd->idle_idx;
2082 break;
2083 }
2084
2085 return load_idx;
2086}
2087
2088
2089#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
2090/**
2091 * init_sd_power_savings_stats - Initialize power savings statistics for
2092 * the given sched_domain, during load balancing.
2093 *
2094 * @sd: Sched domain whose power-savings statistics are to be initialized.
2095 * @sds: Variable containing the statistics for sd.
2096 * @idle: Idle status of the CPU at which we're performing load-balancing.
2097 */
2098static inline void init_sd_power_savings_stats(struct sched_domain *sd,
2099 struct sd_lb_stats *sds, enum cpu_idle_type idle)
2100{
2101 /*
2102 * Busy processors will not participate in power savings
2103 * balance.
2104 */
2105 if (idle == CPU_NOT_IDLE || !(sd->flags & SD_POWERSAVINGS_BALANCE))
2106 sds->power_savings_balance = 0;
2107 else {
2108 sds->power_savings_balance = 1;
2109 sds->min_nr_running = ULONG_MAX;
2110 sds->leader_nr_running = 0;
1950 } 2111 }
2112}
1951 2113
2114/**
2115 * update_sd_power_savings_stats - Update the power saving stats for a
2116 * sched_domain while performing load balancing.
2117 *
2118 * @group: sched_group belonging to the sched_domain under consideration.
2119 * @sds: Variable containing the statistics of the sched_domain
2120 * @local_group: Does group contain the CPU for which we're performing
2121 * load balancing ?
2122 * @sgs: Variable containing the statistics of the group.
2123 */
2124static inline void update_sd_power_savings_stats(struct sched_group *group,
2125 struct sd_lb_stats *sds, int local_group, struct sg_lb_stats *sgs)
2126{
2127
2128 if (!sds->power_savings_balance)
2129 return;
2130
2131 /*
2132 * If the local group is idle or completely loaded
2133 * no need to do power savings balance at this domain
2134 */
2135 if (local_group && (sds->this_nr_running >= sgs->group_capacity ||
2136 !sds->this_nr_running))
2137 sds->power_savings_balance = 0;
2138
2139 /*
2140 * If a group is already running at full capacity or idle,
2141 * don't include that group in power savings calculations
2142 */
2143 if (!sds->power_savings_balance ||
2144 sgs->sum_nr_running >= sgs->group_capacity ||
2145 !sgs->sum_nr_running)
2146 return;
2147
2148 /*
2149 * Calculate the group which has the least non-idle load.
2150 * This is the group from where we need to pick up the load
2151 * for saving power
2152 */
2153 if ((sgs->sum_nr_running < sds->min_nr_running) ||
2154 (sgs->sum_nr_running == sds->min_nr_running &&
2155 group_first_cpu(group) > group_first_cpu(sds->group_min))) {
2156 sds->group_min = group;
2157 sds->min_nr_running = sgs->sum_nr_running;
2158 sds->min_load_per_task = sgs->sum_weighted_load /
2159 sgs->sum_nr_running;
2160 }
2161
2162 /*
2163 * Calculate the group which is almost near its
2164 * capacity but still has some space to pick up some load
2165 * from other group and save more power
2166 */
2167 if (sgs->sum_nr_running + 1 > sgs->group_capacity)
2168 return;
2169
2170 if (sgs->sum_nr_running > sds->leader_nr_running ||
2171 (sgs->sum_nr_running == sds->leader_nr_running &&
2172 group_first_cpu(group) < group_first_cpu(sds->group_leader))) {
2173 sds->group_leader = group;
2174 sds->leader_nr_running = sgs->sum_nr_running;
2175 }
2176}
2177
2178/**
2179 * check_power_save_busiest_group - see if there is potential for some power-savings balance
2180 * @sds: Variable containing the statistics of the sched_domain
2181 * under consideration.
2182 * @this_cpu: Cpu at which we're currently performing load-balancing.
2183 * @imbalance: Variable to store the imbalance.
2184 *
2185 * Description:
2186 * Check if we have potential to perform some power-savings balance.
2187 * If yes, set the busiest group to be the least loaded group in the
2188 * sched_domain, so that it's CPUs can be put to idle.
2189 *
2190 * Returns 1 if there is potential to perform power-savings balance.
2191 * Else returns 0.
2192 */
2193static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
2194 int this_cpu, unsigned long *imbalance)
2195{
2196 if (!sds->power_savings_balance)
2197 return 0;
2198
2199 if (sds->this != sds->group_leader ||
2200 sds->group_leader == sds->group_min)
2201 return 0;
2202
2203 *imbalance = sds->min_load_per_task;
2204 sds->busiest = sds->group_min;
2205
2206 return 1;
2207
2208}
2209#else /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
2210static inline void init_sd_power_savings_stats(struct sched_domain *sd,
2211 struct sd_lb_stats *sds, enum cpu_idle_type idle)
2212{
2213 return;
2214}
2215
2216static inline void update_sd_power_savings_stats(struct sched_group *group,
2217 struct sd_lb_stats *sds, int local_group, struct sg_lb_stats *sgs)
2218{
2219 return;
2220}
2221
2222static inline int check_power_save_busiest_group(struct sd_lb_stats *sds,
2223 int this_cpu, unsigned long *imbalance)
2224{
1952 return 0; 2225 return 0;
1953} 2226}
2227#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
2228
2229
2230unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu)
2231{
2232 return SCHED_LOAD_SCALE;
2233}
2234
2235unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu)
2236{
2237 return default_scale_freq_power(sd, cpu);
2238}
2239
2240unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu)
2241{
2242 unsigned long weight = sd->span_weight;
2243 unsigned long smt_gain = sd->smt_gain;
2244
2245 smt_gain /= weight;
2246
2247 return smt_gain;
2248}
2249
2250unsigned long __weak arch_scale_smt_power(struct sched_domain *sd, int cpu)
2251{
2252 return default_scale_smt_power(sd, cpu);
2253}
2254
2255unsigned long scale_rt_power(int cpu)
2256{
2257 struct rq *rq = cpu_rq(cpu);
2258 u64 total, available;
2259
2260 sched_avg_update(rq);
2261
2262 total = sched_avg_period() + (rq->clock - rq->age_stamp);
2263 available = total - rq->rt_avg;
2264
2265 if (unlikely((s64)total < SCHED_LOAD_SCALE))
2266 total = SCHED_LOAD_SCALE;
2267
2268 total >>= SCHED_LOAD_SHIFT;
2269
2270 return div_u64(available, total);
2271}
2272
2273static void update_cpu_power(struct sched_domain *sd, int cpu)
2274{
2275 unsigned long weight = sd->span_weight;
2276 unsigned long power = SCHED_LOAD_SCALE;
2277 struct sched_group *sdg = sd->groups;
2278
2279 if (sched_feat(ARCH_POWER))
2280 power *= arch_scale_freq_power(sd, cpu);
2281 else
2282 power *= default_scale_freq_power(sd, cpu);
2283
2284 power >>= SCHED_LOAD_SHIFT;
2285
2286 if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
2287 if (sched_feat(ARCH_POWER))
2288 power *= arch_scale_smt_power(sd, cpu);
2289 else
2290 power *= default_scale_smt_power(sd, cpu);
2291
2292 power >>= SCHED_LOAD_SHIFT;
2293 }
2294
2295 power *= scale_rt_power(cpu);
2296 power >>= SCHED_LOAD_SHIFT;
2297
2298 if (!power)
2299 power = 1;
2300
2301 sdg->cpu_power = power;
2302}
2303
2304static void update_group_power(struct sched_domain *sd, int cpu)
2305{
2306 struct sched_domain *child = sd->child;
2307 struct sched_group *group, *sdg = sd->groups;
2308 unsigned long power;
2309
2310 if (!child) {
2311 update_cpu_power(sd, cpu);
2312 return;
2313 }
2314
2315 power = 0;
2316
2317 group = child->groups;
2318 do {
2319 power += group->cpu_power;
2320 group = group->next;
2321 } while (group != child->groups);
2322
2323 sdg->cpu_power = power;
2324}
2325
2326/**
2327 * update_sg_lb_stats - Update sched_group's statistics for load balancing.
2328 * @sd: The sched_domain whose statistics are to be updated.
2329 * @group: sched_group whose statistics are to be updated.
2330 * @this_cpu: Cpu for which load balance is currently performed.
2331 * @idle: Idle status of this_cpu
2332 * @load_idx: Load index of sched_domain of this_cpu for load calc.
2333 * @sd_idle: Idle status of the sched_domain containing group.
2334 * @local_group: Does group contain this_cpu.
2335 * @cpus: Set of cpus considered for load balancing.
2336 * @balance: Should we balance.
2337 * @sgs: variable to hold the statistics for this group.
2338 */
2339static inline void update_sg_lb_stats(struct sched_domain *sd,
2340 struct sched_group *group, int this_cpu,
2341 enum cpu_idle_type idle, int load_idx, int *sd_idle,
2342 int local_group, const struct cpumask *cpus,
2343 int *balance, struct sg_lb_stats *sgs)
2344{
2345 unsigned long load, max_cpu_load, min_cpu_load;
2346 int i;
2347 unsigned int balance_cpu = -1, first_idle_cpu = 0;
2348 unsigned long avg_load_per_task = 0;
2349
2350 if (local_group)
2351 balance_cpu = group_first_cpu(group);
2352
2353 /* Tally up the load of all CPUs in the group */
2354 max_cpu_load = 0;
2355 min_cpu_load = ~0UL;
2356
2357 for_each_cpu_and(i, sched_group_cpus(group), cpus) {
2358 struct rq *rq = cpu_rq(i);
2359
2360 if (*sd_idle && rq->nr_running)
2361 *sd_idle = 0;
2362
2363 /* Bias balancing toward cpus of our domain */
2364 if (local_group) {
2365 if (idle_cpu(i) && !first_idle_cpu) {
2366 first_idle_cpu = 1;
2367 balance_cpu = i;
2368 }
2369
2370 load = target_load(i, load_idx);
2371 } else {
2372 load = source_load(i, load_idx);
2373 if (load > max_cpu_load)
2374 max_cpu_load = load;
2375 if (min_cpu_load > load)
2376 min_cpu_load = load;
2377 }
2378
2379 sgs->group_load += load;
2380 sgs->sum_nr_running += rq->nr_running;
2381 sgs->sum_weighted_load += weighted_cpuload(i);
2382
2383 }
2384
2385 /*
2386 * First idle cpu or the first cpu(busiest) in this sched group
2387 * is eligible for doing load balancing at this and above
2388 * domains. In the newly idle case, we will allow all the cpu's
2389 * to do the newly idle load balance.
2390 */
2391 if (idle != CPU_NEWLY_IDLE && local_group &&
2392 balance_cpu != this_cpu) {
2393 *balance = 0;
2394 return;
2395 }
2396
2397 update_group_power(sd, this_cpu);
2398
2399 /* Adjust by relative CPU power of the group */
2400 sgs->avg_load = (sgs->group_load * SCHED_LOAD_SCALE) / group->cpu_power;
2401
2402 /*
2403 * Consider the group unbalanced when the imbalance is larger
2404 * than the average weight of two tasks.
2405 *
2406 * APZ: with cgroup the avg task weight can vary wildly and
2407 * might not be a suitable number - should we keep a
2408 * normalized nr_running number somewhere that negates
2409 * the hierarchy?
2410 */
2411 if (sgs->sum_nr_running)
2412 avg_load_per_task = sgs->sum_weighted_load / sgs->sum_nr_running;
2413
2414 if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
2415 sgs->group_imb = 1;
2416
2417 sgs->group_capacity =
2418 DIV_ROUND_CLOSEST(group->cpu_power, SCHED_LOAD_SCALE);
2419}
2420
2421/**
2422 * update_sd_lb_stats - Update sched_group's statistics for load balancing.
2423 * @sd: sched_domain whose statistics are to be updated.
2424 * @this_cpu: Cpu for which load balance is currently performed.
2425 * @idle: Idle status of this_cpu
2426 * @sd_idle: Idle status of the sched_domain containing group.
2427 * @cpus: Set of cpus considered for load balancing.
2428 * @balance: Should we balance.
2429 * @sds: variable to hold the statistics for this sched_domain.
2430 */
2431static inline void update_sd_lb_stats(struct sched_domain *sd, int this_cpu,
2432 enum cpu_idle_type idle, int *sd_idle,
2433 const struct cpumask *cpus, int *balance,
2434 struct sd_lb_stats *sds)
2435{
2436 struct sched_domain *child = sd->child;
2437 struct sched_group *group = sd->groups;
2438 struct sg_lb_stats sgs;
2439 int load_idx, prefer_sibling = 0;
2440
2441 if (child && child->flags & SD_PREFER_SIBLING)
2442 prefer_sibling = 1;
2443
2444 init_sd_power_savings_stats(sd, sds, idle);
2445 load_idx = get_sd_load_idx(sd, idle);
2446
2447 do {
2448 int local_group;
2449
2450 local_group = cpumask_test_cpu(this_cpu,
2451 sched_group_cpus(group));
2452 memset(&sgs, 0, sizeof(sgs));
2453 update_sg_lb_stats(sd, group, this_cpu, idle, load_idx, sd_idle,
2454 local_group, cpus, balance, &sgs);
2455
2456 if (local_group && !(*balance))
2457 return;
2458
2459 sds->total_load += sgs.group_load;
2460 sds->total_pwr += group->cpu_power;
2461
2462 /*
2463 * In case the child domain prefers tasks go to siblings
2464 * first, lower the group capacity to one so that we'll try
2465 * and move all the excess tasks away.
2466 */
2467 if (prefer_sibling)
2468 sgs.group_capacity = min(sgs.group_capacity, 1UL);
2469
2470 if (local_group) {
2471 sds->this_load = sgs.avg_load;
2472 sds->this = group;
2473 sds->this_nr_running = sgs.sum_nr_running;
2474 sds->this_load_per_task = sgs.sum_weighted_load;
2475 } else if (sgs.avg_load > sds->max_load &&
2476 (sgs.sum_nr_running > sgs.group_capacity ||
2477 sgs.group_imb)) {
2478 sds->max_load = sgs.avg_load;
2479 sds->busiest = group;
2480 sds->busiest_nr_running = sgs.sum_nr_running;
2481 sds->busiest_group_capacity = sgs.group_capacity;
2482 sds->busiest_load_per_task = sgs.sum_weighted_load;
2483 sds->group_imb = sgs.group_imb;
2484 }
2485
2486 update_sd_power_savings_stats(group, sds, local_group, &sgs);
2487 group = group->next;
2488 } while (group != sd->groups);
2489}
2490
2491/**
2492 * fix_small_imbalance - Calculate the minor imbalance that exists
2493 * amongst the groups of a sched_domain, during
2494 * load balancing.
2495 * @sds: Statistics of the sched_domain whose imbalance is to be calculated.
2496 * @this_cpu: The cpu at whose sched_domain we're performing load-balance.
2497 * @imbalance: Variable to store the imbalance.
2498 */
2499static inline void fix_small_imbalance(struct sd_lb_stats *sds,
2500 int this_cpu, unsigned long *imbalance)
2501{
2502 unsigned long tmp, pwr_now = 0, pwr_move = 0;
2503 unsigned int imbn = 2;
2504 unsigned long scaled_busy_load_per_task;
2505
2506 if (sds->this_nr_running) {
2507 sds->this_load_per_task /= sds->this_nr_running;
2508 if (sds->busiest_load_per_task >
2509 sds->this_load_per_task)
2510 imbn = 1;
2511 } else
2512 sds->this_load_per_task =
2513 cpu_avg_load_per_task(this_cpu);
2514
2515 scaled_busy_load_per_task = sds->busiest_load_per_task
2516 * SCHED_LOAD_SCALE;
2517 scaled_busy_load_per_task /= sds->busiest->cpu_power;
2518
2519 if (sds->max_load - sds->this_load + scaled_busy_load_per_task >=
2520 (scaled_busy_load_per_task * imbn)) {
2521 *imbalance = sds->busiest_load_per_task;
2522 return;
2523 }
2524
2525 /*
2526 * OK, we don't have enough imbalance to justify moving tasks,
2527 * however we may be able to increase total CPU power used by
2528 * moving them.
2529 */
2530
2531 pwr_now += sds->busiest->cpu_power *
2532 min(sds->busiest_load_per_task, sds->max_load);
2533 pwr_now += sds->this->cpu_power *
2534 min(sds->this_load_per_task, sds->this_load);
2535 pwr_now /= SCHED_LOAD_SCALE;
2536
2537 /* Amount of load we'd subtract */
2538 tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
2539 sds->busiest->cpu_power;
2540 if (sds->max_load > tmp)
2541 pwr_move += sds->busiest->cpu_power *
2542 min(sds->busiest_load_per_task, sds->max_load - tmp);
2543
2544 /* Amount of load we'd add */
2545 if (sds->max_load * sds->busiest->cpu_power <
2546 sds->busiest_load_per_task * SCHED_LOAD_SCALE)
2547 tmp = (sds->max_load * sds->busiest->cpu_power) /
2548 sds->this->cpu_power;
2549 else
2550 tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
2551 sds->this->cpu_power;
2552 pwr_move += sds->this->cpu_power *
2553 min(sds->this_load_per_task, sds->this_load + tmp);
2554 pwr_move /= SCHED_LOAD_SCALE;
2555
2556 /* Move if we gain throughput */
2557 if (pwr_move > pwr_now)
2558 *imbalance = sds->busiest_load_per_task;
2559}
2560
2561/**
2562 * calculate_imbalance - Calculate the amount of imbalance present within the
2563 * groups of a given sched_domain during load balance.
2564 * @sds: statistics of the sched_domain whose imbalance is to be calculated.
2565 * @this_cpu: Cpu for which currently load balance is being performed.
2566 * @imbalance: The variable to store the imbalance.
2567 */
2568static inline void calculate_imbalance(struct sd_lb_stats *sds, int this_cpu,
2569 unsigned long *imbalance)
2570{
2571 unsigned long max_pull, load_above_capacity = ~0UL;
2572
2573 sds->busiest_load_per_task /= sds->busiest_nr_running;
2574 if (sds->group_imb) {
2575 sds->busiest_load_per_task =
2576 min(sds->busiest_load_per_task, sds->avg_load);
2577 }
2578
2579 /*
2580 * In the presence of smp nice balancing, certain scenarios can have
2581 * max load less than avg load(as we skip the groups at or below
2582 * its cpu_power, while calculating max_load..)
2583 */
2584 if (sds->max_load < sds->avg_load) {
2585 *imbalance = 0;
2586 return fix_small_imbalance(sds, this_cpu, imbalance);
2587 }
2588
2589 if (!sds->group_imb) {
2590 /*
2591 * Don't want to pull so many tasks that a group would go idle.
2592 */
2593 load_above_capacity = (sds->busiest_nr_running -
2594 sds->busiest_group_capacity);
2595
2596 load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_LOAD_SCALE);
2597
2598 load_above_capacity /= sds->busiest->cpu_power;
2599 }
2600
2601 /*
2602 * We're trying to get all the cpus to the average_load, so we don't
2603 * want to push ourselves above the average load, nor do we wish to
2604 * reduce the max loaded cpu below the average load. At the same time,
2605 * we also don't want to reduce the group load below the group capacity
2606 * (so that we can implement power-savings policies etc). Thus we look
2607 * for the minimum possible imbalance.
2608 * Be careful of negative numbers as they'll appear as very large values
2609 * with unsigned longs.
2610 */
2611 max_pull = min(sds->max_load - sds->avg_load, load_above_capacity);
2612
2613 /* How much load to actually move to equalise the imbalance */
2614 *imbalance = min(max_pull * sds->busiest->cpu_power,
2615 (sds->avg_load - sds->this_load) * sds->this->cpu_power)
2616 / SCHED_LOAD_SCALE;
2617
2618 /*
2619 * if *imbalance is less than the average load per runnable task
2620 * there is no gaurantee that any tasks will be moved so we'll have
2621 * a think about bumping its value to force at least one task to be
2622 * moved
2623 */
2624 if (*imbalance < sds->busiest_load_per_task)
2625 return fix_small_imbalance(sds, this_cpu, imbalance);
2626
2627}
2628/******* find_busiest_group() helpers end here *********************/
2629
2630/**
2631 * find_busiest_group - Returns the busiest group within the sched_domain
2632 * if there is an imbalance. If there isn't an imbalance, and
2633 * the user has opted for power-savings, it returns a group whose
2634 * CPUs can be put to idle by rebalancing those tasks elsewhere, if
2635 * such a group exists.
2636 *
2637 * Also calculates the amount of weighted load which should be moved
2638 * to restore balance.
2639 *
2640 * @sd: The sched_domain whose busiest group is to be returned.
2641 * @this_cpu: The cpu for which load balancing is currently being performed.
2642 * @imbalance: Variable which stores amount of weighted load which should
2643 * be moved to restore balance/put a group to idle.
2644 * @idle: The idle status of this_cpu.
2645 * @sd_idle: The idleness of sd
2646 * @cpus: The set of CPUs under consideration for load-balancing.
2647 * @balance: Pointer to a variable indicating if this_cpu
2648 * is the appropriate cpu to perform load balancing at this_level.
2649 *
2650 * Returns: - the busiest group if imbalance exists.
2651 * - If no imbalance and user has opted for power-savings balance,
2652 * return the least loaded group whose CPUs can be
2653 * put to idle by rebalancing its tasks onto our group.
2654 */
2655static struct sched_group *
2656find_busiest_group(struct sched_domain *sd, int this_cpu,
2657 unsigned long *imbalance, enum cpu_idle_type idle,
2658 int *sd_idle, const struct cpumask *cpus, int *balance)
2659{
2660 struct sd_lb_stats sds;
2661
2662 memset(&sds, 0, sizeof(sds));
2663
2664 /*
2665 * Compute the various statistics relavent for load balancing at
2666 * this level.
2667 */
2668 update_sd_lb_stats(sd, this_cpu, idle, sd_idle, cpus,
2669 balance, &sds);
2670
2671 /* Cases where imbalance does not exist from POV of this_cpu */
2672 /* 1) this_cpu is not the appropriate cpu to perform load balancing
2673 * at this level.
2674 * 2) There is no busy sibling group to pull from.
2675 * 3) This group is the busiest group.
2676 * 4) This group is more busy than the avg busieness at this
2677 * sched_domain.
2678 * 5) The imbalance is within the specified limit.
2679 */
2680 if (!(*balance))
2681 goto ret;
2682
2683 if (!sds.busiest || sds.busiest_nr_running == 0)
2684 goto out_balanced;
2685
2686 if (sds.this_load >= sds.max_load)
2687 goto out_balanced;
2688
2689 sds.avg_load = (SCHED_LOAD_SCALE * sds.total_load) / sds.total_pwr;
2690
2691 if (sds.this_load >= sds.avg_load)
2692 goto out_balanced;
2693
2694 if (100 * sds.max_load <= sd->imbalance_pct * sds.this_load)
2695 goto out_balanced;
2696
2697 /* Looks like there is an imbalance. Compute it */
2698 calculate_imbalance(&sds, this_cpu, imbalance);
2699 return sds.busiest;
2700
2701out_balanced:
2702 /*
2703 * There is no obvious imbalance. But check if we can do some balancing
2704 * to save power.
2705 */
2706 if (check_power_save_busiest_group(&sds, this_cpu, imbalance))
2707 return sds.busiest;
2708ret:
2709 *imbalance = 0;
2710 return NULL;
2711}
2712
2713/*
2714 * find_busiest_queue - find the busiest runqueue among the cpus in group.
2715 */
2716static struct rq *
2717find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
2718 unsigned long imbalance, const struct cpumask *cpus)
2719{
2720 struct rq *busiest = NULL, *rq;
2721 unsigned long max_load = 0;
2722 int i;
2723
2724 for_each_cpu(i, sched_group_cpus(group)) {
2725 unsigned long power = power_of(i);
2726 unsigned long capacity = DIV_ROUND_CLOSEST(power, SCHED_LOAD_SCALE);
2727 unsigned long wl;
2728
2729 if (!cpumask_test_cpu(i, cpus))
2730 continue;
2731
2732 rq = cpu_rq(i);
2733 wl = weighted_cpuload(i);
2734
2735 /*
2736 * When comparing with imbalance, use weighted_cpuload()
2737 * which is not scaled with the cpu power.
2738 */
2739 if (capacity && rq->nr_running == 1 && wl > imbalance)
2740 continue;
2741
2742 /*
2743 * For the load comparisons with the other cpu's, consider
2744 * the weighted_cpuload() scaled with the cpu power, so that
2745 * the load can be moved away from the cpu that is potentially
2746 * running at a lower capacity.
2747 */
2748 wl = (wl * SCHED_LOAD_SCALE) / power;
2749
2750 if (wl > max_load) {
2751 max_load = wl;
2752 busiest = rq;
2753 }
2754 }
2755
2756 return busiest;
2757}
2758
2759/*
2760 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
2761 * so long as it is large enough.
2762 */
2763#define MAX_PINNED_INTERVAL 512
2764
2765/* Working cpumask for load_balance and load_balance_newidle. */
2766static DEFINE_PER_CPU(cpumask_var_t, load_balance_tmpmask);
2767
2768static int need_active_balance(struct sched_domain *sd, int sd_idle, int idle)
2769{
2770 if (idle == CPU_NEWLY_IDLE) {
2771 /*
2772 * The only task running in a non-idle cpu can be moved to this
2773 * cpu in an attempt to completely freeup the other CPU
2774 * package.
2775 *
2776 * The package power saving logic comes from
2777 * find_busiest_group(). If there are no imbalance, then
2778 * f_b_g() will return NULL. However when sched_mc={1,2} then
2779 * f_b_g() will select a group from which a running task may be
2780 * pulled to this cpu in order to make the other package idle.
2781 * If there is no opportunity to make a package idle and if
2782 * there are no imbalance, then f_b_g() will return NULL and no
2783 * action will be taken in load_balance_newidle().
2784 *
2785 * Under normal task pull operation due to imbalance, there
2786 * will be more than one task in the source run queue and
2787 * move_tasks() will succeed. ld_moved will be true and this
2788 * active balance code will not be triggered.
2789 */
2790 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2791 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
2792 return 0;
2793
2794 if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
2795 return 0;
2796 }
2797
2798 return unlikely(sd->nr_balance_failed > sd->cache_nice_tries+2);
2799}
2800
2801static int active_load_balance_cpu_stop(void *data);
2802
2803/*
2804 * Check this_cpu to ensure it is balanced within domain. Attempt to move
2805 * tasks if there is an imbalance.
2806 */
2807static int load_balance(int this_cpu, struct rq *this_rq,
2808 struct sched_domain *sd, enum cpu_idle_type idle,
2809 int *balance)
2810{
2811 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
2812 struct sched_group *group;
2813 unsigned long imbalance;
2814 struct rq *busiest;
2815 unsigned long flags;
2816 struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask);
2817
2818 cpumask_copy(cpus, cpu_active_mask);
2819
2820 /*
2821 * When power savings policy is enabled for the parent domain, idle
2822 * sibling can pick up load irrespective of busy siblings. In this case,
2823 * let the state of idle sibling percolate up as CPU_IDLE, instead of
2824 * portraying it as CPU_NOT_IDLE.
2825 */
2826 if (idle != CPU_NOT_IDLE && sd->flags & SD_SHARE_CPUPOWER &&
2827 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
2828 sd_idle = 1;
2829
2830 schedstat_inc(sd, lb_count[idle]);
2831
2832redo:
2833 update_shares(sd);
2834 group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
2835 cpus, balance);
2836
2837 if (*balance == 0)
2838 goto out_balanced;
2839
2840 if (!group) {
2841 schedstat_inc(sd, lb_nobusyg[idle]);
2842 goto out_balanced;
2843 }
2844
2845 busiest = find_busiest_queue(group, idle, imbalance, cpus);
2846 if (!busiest) {
2847 schedstat_inc(sd, lb_nobusyq[idle]);
2848 goto out_balanced;
2849 }
2850
2851 BUG_ON(busiest == this_rq);
2852
2853 schedstat_add(sd, lb_imbalance[idle], imbalance);
2854
2855 ld_moved = 0;
2856 if (busiest->nr_running > 1) {
2857 /*
2858 * Attempt to move tasks. If find_busiest_group has found
2859 * an imbalance but busiest->nr_running <= 1, the group is
2860 * still unbalanced. ld_moved simply stays zero, so it is
2861 * correctly treated as an imbalance.
2862 */
2863 local_irq_save(flags);
2864 double_rq_lock(this_rq, busiest);
2865 ld_moved = move_tasks(this_rq, this_cpu, busiest,
2866 imbalance, sd, idle, &all_pinned);
2867 double_rq_unlock(this_rq, busiest);
2868 local_irq_restore(flags);
2869
2870 /*
2871 * some other cpu did the load balance for us.
2872 */
2873 if (ld_moved && this_cpu != smp_processor_id())
2874 resched_cpu(this_cpu);
2875
2876 /* All tasks on this runqueue were pinned by CPU affinity */
2877 if (unlikely(all_pinned)) {
2878 cpumask_clear_cpu(cpu_of(busiest), cpus);
2879 if (!cpumask_empty(cpus))
2880 goto redo;
2881 goto out_balanced;
2882 }
2883 }
2884
2885 if (!ld_moved) {
2886 schedstat_inc(sd, lb_failed[idle]);
2887 sd->nr_balance_failed++;
2888
2889 if (need_active_balance(sd, sd_idle, idle)) {
2890 raw_spin_lock_irqsave(&busiest->lock, flags);
2891
2892 /* don't kick the active_load_balance_cpu_stop,
2893 * if the curr task on busiest cpu can't be
2894 * moved to this_cpu
2895 */
2896 if (!cpumask_test_cpu(this_cpu,
2897 &busiest->curr->cpus_allowed)) {
2898 raw_spin_unlock_irqrestore(&busiest->lock,
2899 flags);
2900 all_pinned = 1;
2901 goto out_one_pinned;
2902 }
2903
2904 /*
2905 * ->active_balance synchronizes accesses to
2906 * ->active_balance_work. Once set, it's cleared
2907 * only after active load balance is finished.
2908 */
2909 if (!busiest->active_balance) {
2910 busiest->active_balance = 1;
2911 busiest->push_cpu = this_cpu;
2912 active_balance = 1;
2913 }
2914 raw_spin_unlock_irqrestore(&busiest->lock, flags);
2915
2916 if (active_balance)
2917 stop_one_cpu_nowait(cpu_of(busiest),
2918 active_load_balance_cpu_stop, busiest,
2919 &busiest->active_balance_work);
2920
2921 /*
2922 * We've kicked active balancing, reset the failure
2923 * counter.
2924 */
2925 sd->nr_balance_failed = sd->cache_nice_tries+1;
2926 }
2927 } else
2928 sd->nr_balance_failed = 0;
2929
2930 if (likely(!active_balance)) {
2931 /* We were unbalanced, so reset the balancing interval */
2932 sd->balance_interval = sd->min_interval;
2933 } else {
2934 /*
2935 * If we've begun active balancing, start to back off. This
2936 * case may not be covered by the all_pinned logic if there
2937 * is only 1 task on the busy runqueue (because we don't call
2938 * move_tasks).
2939 */
2940 if (sd->balance_interval < sd->max_interval)
2941 sd->balance_interval *= 2;
2942 }
2943
2944 if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2945 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
2946 ld_moved = -1;
2947
2948 goto out;
2949
2950out_balanced:
2951 schedstat_inc(sd, lb_balanced[idle]);
2952
2953 sd->nr_balance_failed = 0;
2954
2955out_one_pinned:
2956 /* tune up the balancing interval */
2957 if ((all_pinned && sd->balance_interval < MAX_PINNED_INTERVAL) ||
2958 (sd->balance_interval < sd->max_interval))
2959 sd->balance_interval *= 2;
2960
2961 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
2962 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
2963 ld_moved = -1;
2964 else
2965 ld_moved = 0;
2966out:
2967 if (ld_moved)
2968 update_shares(sd);
2969 return ld_moved;
2970}
2971
2972/*
2973 * idle_balance is called by schedule() if this_cpu is about to become
2974 * idle. Attempts to pull tasks from other CPUs.
2975 */
2976static void idle_balance(int this_cpu, struct rq *this_rq)
2977{
2978 struct sched_domain *sd;
2979 int pulled_task = 0;
2980 unsigned long next_balance = jiffies + HZ;
2981
2982 this_rq->idle_stamp = this_rq->clock;
2983
2984 if (this_rq->avg_idle < sysctl_sched_migration_cost)
2985 return;
2986
2987 /*
2988 * Drop the rq->lock, but keep IRQ/preempt disabled.
2989 */
2990 raw_spin_unlock(&this_rq->lock);
2991
2992 for_each_domain(this_cpu, sd) {
2993 unsigned long interval;
2994 int balance = 1;
2995
2996 if (!(sd->flags & SD_LOAD_BALANCE))
2997 continue;
2998
2999 if (sd->flags & SD_BALANCE_NEWIDLE) {
3000 /* If we've pulled tasks over stop searching: */
3001 pulled_task = load_balance(this_cpu, this_rq,
3002 sd, CPU_NEWLY_IDLE, &balance);
3003 }
3004
3005 interval = msecs_to_jiffies(sd->balance_interval);
3006 if (time_after(next_balance, sd->last_balance + interval))
3007 next_balance = sd->last_balance + interval;
3008 if (pulled_task) {
3009 this_rq->idle_stamp = 0;
3010 break;
3011 }
3012 }
3013
3014 raw_spin_lock(&this_rq->lock);
3015
3016 if (pulled_task || time_after(jiffies, this_rq->next_balance)) {
3017 /*
3018 * We are going idle. next_balance may be set based on
3019 * a busy processor. So reset next_balance.
3020 */
3021 this_rq->next_balance = next_balance;
3022 }
3023}
3024
3025/*
3026 * active_load_balance_cpu_stop is run by cpu stopper. It pushes
3027 * running tasks off the busiest CPU onto idle CPUs. It requires at
3028 * least 1 task to be running on each physical CPU where possible, and
3029 * avoids physical / logical imbalances.
3030 */
3031static int active_load_balance_cpu_stop(void *data)
3032{
3033 struct rq *busiest_rq = data;
3034 int busiest_cpu = cpu_of(busiest_rq);
3035 int target_cpu = busiest_rq->push_cpu;
3036 struct rq *target_rq = cpu_rq(target_cpu);
3037 struct sched_domain *sd;
3038
3039 raw_spin_lock_irq(&busiest_rq->lock);
3040
3041 /* make sure the requested cpu hasn't gone down in the meantime */
3042 if (unlikely(busiest_cpu != smp_processor_id() ||
3043 !busiest_rq->active_balance))
3044 goto out_unlock;
3045
3046 /* Is there any task to move? */
3047 if (busiest_rq->nr_running <= 1)
3048 goto out_unlock;
3049
3050 /*
3051 * This condition is "impossible", if it occurs
3052 * we need to fix it. Originally reported by
3053 * Bjorn Helgaas on a 128-cpu setup.
3054 */
3055 BUG_ON(busiest_rq == target_rq);
3056
3057 /* move a task from busiest_rq to target_rq */
3058 double_lock_balance(busiest_rq, target_rq);
3059
3060 /* Search for an sd spanning us and the target CPU. */
3061 for_each_domain(target_cpu, sd) {
3062 if ((sd->flags & SD_LOAD_BALANCE) &&
3063 cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
3064 break;
3065 }
3066
3067 if (likely(sd)) {
3068 schedstat_inc(sd, alb_count);
3069
3070 if (move_one_task(target_rq, target_cpu, busiest_rq,
3071 sd, CPU_IDLE))
3072 schedstat_inc(sd, alb_pushed);
3073 else
3074 schedstat_inc(sd, alb_failed);
3075 }
3076 double_unlock_balance(busiest_rq, target_rq);
3077out_unlock:
3078 busiest_rq->active_balance = 0;
3079 raw_spin_unlock_irq(&busiest_rq->lock);
3080 return 0;
3081}
3082
3083#ifdef CONFIG_NO_HZ
3084static struct {
3085 atomic_t load_balancer;
3086 cpumask_var_t cpu_mask;
3087 cpumask_var_t ilb_grp_nohz_mask;
3088} nohz ____cacheline_aligned = {
3089 .load_balancer = ATOMIC_INIT(-1),
3090};
3091
3092int get_nohz_load_balancer(void)
3093{
3094 return atomic_read(&nohz.load_balancer);
3095}
3096
3097#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
3098/**
3099 * lowest_flag_domain - Return lowest sched_domain containing flag.
3100 * @cpu: The cpu whose lowest level of sched domain is to
3101 * be returned.
3102 * @flag: The flag to check for the lowest sched_domain
3103 * for the given cpu.
3104 *
3105 * Returns the lowest sched_domain of a cpu which contains the given flag.
3106 */
3107static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)
3108{
3109 struct sched_domain *sd;
3110
3111 for_each_domain(cpu, sd)
3112 if (sd && (sd->flags & flag))
3113 break;
3114
3115 return sd;
3116}
3117
3118/**
3119 * for_each_flag_domain - Iterates over sched_domains containing the flag.
3120 * @cpu: The cpu whose domains we're iterating over.
3121 * @sd: variable holding the value of the power_savings_sd
3122 * for cpu.
3123 * @flag: The flag to filter the sched_domains to be iterated.
3124 *
3125 * Iterates over all the scheduler domains for a given cpu that has the 'flag'
3126 * set, starting from the lowest sched_domain to the highest.
3127 */
3128#define for_each_flag_domain(cpu, sd, flag) \
3129 for (sd = lowest_flag_domain(cpu, flag); \
3130 (sd && (sd->flags & flag)); sd = sd->parent)
3131
3132/**
3133 * is_semi_idle_group - Checks if the given sched_group is semi-idle.
3134 * @ilb_group: group to be checked for semi-idleness
3135 *
3136 * Returns: 1 if the group is semi-idle. 0 otherwise.
3137 *
3138 * We define a sched_group to be semi idle if it has atleast one idle-CPU
3139 * and atleast one non-idle CPU. This helper function checks if the given
3140 * sched_group is semi-idle or not.
3141 */
3142static inline int is_semi_idle_group(struct sched_group *ilb_group)
3143{
3144 cpumask_and(nohz.ilb_grp_nohz_mask, nohz.cpu_mask,
3145 sched_group_cpus(ilb_group));
3146
3147 /*
3148 * A sched_group is semi-idle when it has atleast one busy cpu
3149 * and atleast one idle cpu.
3150 */
3151 if (cpumask_empty(nohz.ilb_grp_nohz_mask))
3152 return 0;
3153
3154 if (cpumask_equal(nohz.ilb_grp_nohz_mask, sched_group_cpus(ilb_group)))
3155 return 0;
3156
3157 return 1;
3158}
3159/**
3160 * find_new_ilb - Finds the optimum idle load balancer for nomination.
3161 * @cpu: The cpu which is nominating a new idle_load_balancer.
3162 *
3163 * Returns: Returns the id of the idle load balancer if it exists,
3164 * Else, returns >= nr_cpu_ids.
3165 *
3166 * This algorithm picks the idle load balancer such that it belongs to a
3167 * semi-idle powersavings sched_domain. The idea is to try and avoid
3168 * completely idle packages/cores just for the purpose of idle load balancing
3169 * when there are other idle cpu's which are better suited for that job.
3170 */
3171static int find_new_ilb(int cpu)
3172{
3173 struct sched_domain *sd;
3174 struct sched_group *ilb_group;
3175
3176 /*
3177 * Have idle load balancer selection from semi-idle packages only
3178 * when power-aware load balancing is enabled
3179 */
3180 if (!(sched_smt_power_savings || sched_mc_power_savings))
3181 goto out_done;
3182
3183 /*
3184 * Optimize for the case when we have no idle CPUs or only one
3185 * idle CPU. Don't walk the sched_domain hierarchy in such cases
3186 */
3187 if (cpumask_weight(nohz.cpu_mask) < 2)
3188 goto out_done;
3189
3190 for_each_flag_domain(cpu, sd, SD_POWERSAVINGS_BALANCE) {
3191 ilb_group = sd->groups;
3192
3193 do {
3194 if (is_semi_idle_group(ilb_group))
3195 return cpumask_first(nohz.ilb_grp_nohz_mask);
3196
3197 ilb_group = ilb_group->next;
3198
3199 } while (ilb_group != sd->groups);
3200 }
3201
3202out_done:
3203 return cpumask_first(nohz.cpu_mask);
3204}
3205#else /* (CONFIG_SCHED_MC || CONFIG_SCHED_SMT) */
3206static inline int find_new_ilb(int call_cpu)
3207{
3208 return cpumask_first(nohz.cpu_mask);
3209}
3210#endif
3211
3212/*
3213 * This routine will try to nominate the ilb (idle load balancing)
3214 * owner among the cpus whose ticks are stopped. ilb owner will do the idle
3215 * load balancing on behalf of all those cpus. If all the cpus in the system
3216 * go into this tickless mode, then there will be no ilb owner (as there is
3217 * no need for one) and all the cpus will sleep till the next wakeup event
3218 * arrives...
3219 *
3220 * For the ilb owner, tick is not stopped. And this tick will be used
3221 * for idle load balancing. ilb owner will still be part of
3222 * nohz.cpu_mask..
3223 *
3224 * While stopping the tick, this cpu will become the ilb owner if there
3225 * is no other owner. And will be the owner till that cpu becomes busy
3226 * or if all cpus in the system stop their ticks at which point
3227 * there is no need for ilb owner.
3228 *
3229 * When the ilb owner becomes busy, it nominates another owner, during the
3230 * next busy scheduler_tick()
3231 */
3232int select_nohz_load_balancer(int stop_tick)
3233{
3234 int cpu = smp_processor_id();
3235
3236 if (stop_tick) {
3237 cpu_rq(cpu)->in_nohz_recently = 1;
3238
3239 if (!cpu_active(cpu)) {
3240 if (atomic_read(&nohz.load_balancer) != cpu)
3241 return 0;
3242
3243 /*
3244 * If we are going offline and still the leader,
3245 * give up!
3246 */
3247 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
3248 BUG();
3249
3250 return 0;
3251 }
3252
3253 cpumask_set_cpu(cpu, nohz.cpu_mask);
3254
3255 /* time for ilb owner also to sleep */
3256 if (cpumask_weight(nohz.cpu_mask) == num_active_cpus()) {
3257 if (atomic_read(&nohz.load_balancer) == cpu)
3258 atomic_set(&nohz.load_balancer, -1);
3259 return 0;
3260 }
3261
3262 if (atomic_read(&nohz.load_balancer) == -1) {
3263 /* make me the ilb owner */
3264 if (atomic_cmpxchg(&nohz.load_balancer, -1, cpu) == -1)
3265 return 1;
3266 } else if (atomic_read(&nohz.load_balancer) == cpu) {
3267 int new_ilb;
3268
3269 if (!(sched_smt_power_savings ||
3270 sched_mc_power_savings))
3271 return 1;
3272 /*
3273 * Check to see if there is a more power-efficient
3274 * ilb.
3275 */
3276 new_ilb = find_new_ilb(cpu);
3277 if (new_ilb < nr_cpu_ids && new_ilb != cpu) {
3278 atomic_set(&nohz.load_balancer, -1);
3279 resched_cpu(new_ilb);
3280 return 0;
3281 }
3282 return 1;
3283 }
3284 } else {
3285 if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
3286 return 0;
3287
3288 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3289
3290 if (atomic_read(&nohz.load_balancer) == cpu)
3291 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
3292 BUG();
3293 }
3294 return 0;
3295}
3296#endif
3297
3298static DEFINE_SPINLOCK(balancing);
3299
3300/*
3301 * It checks each scheduling domain to see if it is due to be balanced,
3302 * and initiates a balancing operation if so.
3303 *
3304 * Balancing parameters are set up in arch_init_sched_domains.
3305 */
3306static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3307{
3308 int balance = 1;
3309 struct rq *rq = cpu_rq(cpu);
3310 unsigned long interval;
3311 struct sched_domain *sd;
3312 /* Earliest time when we have to do rebalance again */
3313 unsigned long next_balance = jiffies + 60*HZ;
3314 int update_next_balance = 0;
3315 int need_serialize;
3316
3317 for_each_domain(cpu, sd) {
3318 if (!(sd->flags & SD_LOAD_BALANCE))
3319 continue;
3320
3321 interval = sd->balance_interval;
3322 if (idle != CPU_IDLE)
3323 interval *= sd->busy_factor;
3324
3325 /* scale ms to jiffies */
3326 interval = msecs_to_jiffies(interval);
3327 if (unlikely(!interval))
3328 interval = 1;
3329 if (interval > HZ*NR_CPUS/10)
3330 interval = HZ*NR_CPUS/10;
3331
3332 need_serialize = sd->flags & SD_SERIALIZE;
3333
3334 if (need_serialize) {
3335 if (!spin_trylock(&balancing))
3336 goto out;
3337 }
3338
3339 if (time_after_eq(jiffies, sd->last_balance + interval)) {
3340 if (load_balance(cpu, rq, sd, idle, &balance)) {
3341 /*
3342 * We've pulled tasks over so either we're no
3343 * longer idle, or one of our SMT siblings is
3344 * not idle.
3345 */
3346 idle = CPU_NOT_IDLE;
3347 }
3348 sd->last_balance = jiffies;
3349 }
3350 if (need_serialize)
3351 spin_unlock(&balancing);
3352out:
3353 if (time_after(next_balance, sd->last_balance + interval)) {
3354 next_balance = sd->last_balance + interval;
3355 update_next_balance = 1;
3356 }
3357
3358 /*
3359 * Stop the load balance at this level. There is another
3360 * CPU in our sched group which is doing load balancing more
3361 * actively.
3362 */
3363 if (!balance)
3364 break;
3365 }
3366
3367 /*
3368 * next_balance will be updated only when there is a need.
3369 * When the cpu is attached to null domain for ex, it will not be
3370 * updated.
3371 */
3372 if (likely(update_next_balance))
3373 rq->next_balance = next_balance;
3374}
3375
3376/*
3377 * run_rebalance_domains is triggered when needed from the scheduler tick.
3378 * In CONFIG_NO_HZ case, the idle load balance owner will do the
3379 * rebalancing for all the cpus for whom scheduler ticks are stopped.
3380 */
3381static void run_rebalance_domains(struct softirq_action *h)
3382{
3383 int this_cpu = smp_processor_id();
3384 struct rq *this_rq = cpu_rq(this_cpu);
3385 enum cpu_idle_type idle = this_rq->idle_at_tick ?
3386 CPU_IDLE : CPU_NOT_IDLE;
3387
3388 rebalance_domains(this_cpu, idle);
3389
3390#ifdef CONFIG_NO_HZ
3391 /*
3392 * If this cpu is the owner for idle load balancing, then do the
3393 * balancing on behalf of the other idle cpus whose ticks are
3394 * stopped.
3395 */
3396 if (this_rq->idle_at_tick &&
3397 atomic_read(&nohz.load_balancer) == this_cpu) {
3398 struct rq *rq;
3399 int balance_cpu;
3400
3401 for_each_cpu(balance_cpu, nohz.cpu_mask) {
3402 if (balance_cpu == this_cpu)
3403 continue;
3404
3405 /*
3406 * If this cpu gets work to do, stop the load balancing
3407 * work being done for other cpus. Next load
3408 * balancing owner will pick it up.
3409 */
3410 if (need_resched())
3411 break;
3412
3413 rebalance_domains(balance_cpu, CPU_IDLE);
3414
3415 rq = cpu_rq(balance_cpu);
3416 if (time_after(this_rq->next_balance, rq->next_balance))
3417 this_rq->next_balance = rq->next_balance;
3418 }
3419 }
3420#endif
3421}
3422
3423static inline int on_null_domain(int cpu)
3424{
3425 return !rcu_dereference_sched(cpu_rq(cpu)->sd);
3426}
3427
3428/*
3429 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
3430 *
3431 * In case of CONFIG_NO_HZ, this is the place where we nominate a new
3432 * idle load balancing owner or decide to stop the periodic load balancing,
3433 * if the whole system is idle.
3434 */
3435static inline void trigger_load_balance(struct rq *rq, int cpu)
3436{
3437#ifdef CONFIG_NO_HZ
3438 /*
3439 * If we were in the nohz mode recently and busy at the current
3440 * scheduler tick, then check if we need to nominate new idle
3441 * load balancer.
3442 */
3443 if (rq->in_nohz_recently && !rq->idle_at_tick) {
3444 rq->in_nohz_recently = 0;
3445
3446 if (atomic_read(&nohz.load_balancer) == cpu) {
3447 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3448 atomic_set(&nohz.load_balancer, -1);
3449 }
3450
3451 if (atomic_read(&nohz.load_balancer) == -1) {
3452 int ilb = find_new_ilb(cpu);
3453
3454 if (ilb < nr_cpu_ids)
3455 resched_cpu(ilb);
3456 }
3457 }
3458
3459 /*
3460 * If this cpu is idle and doing idle load balancing for all the
3461 * cpus with ticks stopped, is it time for that to stop?
3462 */
3463 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
3464 cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
3465 resched_cpu(cpu);
3466 return;
3467 }
3468
3469 /*
3470 * If this cpu is idle and the idle load balancing is done by
3471 * someone else, then no need raise the SCHED_SOFTIRQ
3472 */
3473 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
3474 cpumask_test_cpu(cpu, nohz.cpu_mask))
3475 return;
3476#endif
3477 /* Don't need to rebalance while attached to NULL domain */
3478 if (time_after_eq(jiffies, rq->next_balance) &&
3479 likely(!on_null_domain(cpu)))
3480 raise_softirq(SCHED_SOFTIRQ);
3481}
1954 3482
1955static void rq_online_fair(struct rq *rq) 3483static void rq_online_fair(struct rq *rq)
1956{ 3484{
@@ -1962,6 +3490,15 @@ static void rq_offline_fair(struct rq *rq)
1962 update_sysctl(); 3490 update_sysctl();
1963} 3491}
1964 3492
3493#else /* CONFIG_SMP */
3494
3495/*
3496 * on UP we do not need to balance between CPUs:
3497 */
3498static inline void idle_balance(int cpu, struct rq *rq)
3499{
3500}
3501
1965#endif /* CONFIG_SMP */ 3502#endif /* CONFIG_SMP */
1966 3503
1967/* 3504/*
@@ -2076,7 +3613,7 @@ static void moved_group_fair(struct task_struct *p, int on_rq)
2076} 3613}
2077#endif 3614#endif
2078 3615
2079unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) 3616static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task)
2080{ 3617{
2081 struct sched_entity *se = &task->se; 3618 struct sched_entity *se = &task->se;
2082 unsigned int rr_interval = 0; 3619 unsigned int rr_interval = 0;
@@ -2108,8 +3645,6 @@ static const struct sched_class fair_sched_class = {
2108#ifdef CONFIG_SMP 3645#ifdef CONFIG_SMP
2109 .select_task_rq = select_task_rq_fair, 3646 .select_task_rq = select_task_rq_fair,
2110 3647
2111 .load_balance = load_balance_fair,
2112 .move_one_task = move_one_task_fair,
2113 .rq_online = rq_online_fair, 3648 .rq_online = rq_online_fair,
2114 .rq_offline = rq_offline_fair, 3649 .rq_offline = rq_offline_fair,
2115 3650
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-20 11:03:07 -0500