aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChristoph Lameter <clameter@sgi.com>2006-12-10 05:20:25 -0500
committerLinus Torvalds <torvalds@woody.osdl.org>2006-12-10 12:55:42 -0500
commitc9819f4593e8d052b41a89f47140f5c5e7e30582 (patch)
tree85da690ecd1e14506336009281e5c369d8457325
parente418e1c2bf1a253916b569370653414eb28597b6 (diff)
[PATCH] sched: use softirq for load balancing
Call rebalance_tick (renamed to run_rebalance_domains) from a newly introduced softirq. We calculate the earliest time for each layer of sched domains to be rescanned (this is the rescan time for idle) and use the earliest of those to schedule the softirq via a new field "next_balance" added to struct rq. Signed-off-by: Christoph Lameter <clameter@sgi.com> Cc: Peter Williams <pwil3058@bigpond.net.au> Cc: Nick Piggin <nickpiggin@yahoo.com.au> Cc: Christoph Lameter <clameter@sgi.com> Cc: "Siddha, Suresh B" <suresh.b.siddha@intel.com> Cc: "Chen, Kenneth W" <kenneth.w.chen@intel.com> Acked-by: Ingo Molnar <mingo@elte.hu> Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
-rw-r--r--include/linux/interrupt.h3
-rw-r--r--kernel/sched.c22
2 files changed, 19 insertions, 6 deletions
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index de7593f4e895..e36e86c869fb 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -231,7 +231,8 @@ enum
231 NET_TX_SOFTIRQ, 231 NET_TX_SOFTIRQ,
232 NET_RX_SOFTIRQ, 232 NET_RX_SOFTIRQ,
233 BLOCK_SOFTIRQ, 233 BLOCK_SOFTIRQ,
234 TASKLET_SOFTIRQ 234 TASKLET_SOFTIRQ,
235 SCHED_SOFTIRQ,
235}; 236};
236 237
237/* softirq mask and active fields moved to irq_cpustat_t in 238/* softirq mask and active fields moved to irq_cpustat_t in
diff --git a/kernel/sched.c b/kernel/sched.c
index 14a8d9050cd4..0a3e748d737d 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -227,6 +227,7 @@ struct rq {
227 unsigned long expired_timestamp; 227 unsigned long expired_timestamp;
228 unsigned long long timestamp_last_tick; 228 unsigned long long timestamp_last_tick;
229 struct task_struct *curr, *idle; 229 struct task_struct *curr, *idle;
230 unsigned long next_balance;
230 struct mm_struct *prev_mm; 231 struct mm_struct *prev_mm;
231 struct prio_array *active, *expired, arrays[2]; 232 struct prio_array *active, *expired, arrays[2];
232 int best_expired_prio; 233 int best_expired_prio;
@@ -2858,7 +2859,7 @@ static void update_load(struct rq *this_rq)
2858} 2859}
2859 2860
2860/* 2861/*
2861 * rebalance_tick will get called every timer tick, on every CPU. 2862 * run_rebalance_domains is triggered when needed from the scheduler tick.
2862 * 2863 *
2863 * It checks each scheduling domain to see if it is due to be balanced, 2864 * It checks each scheduling domain to see if it is due to be balanced,
2864 * and initiates a balancing operation if so. 2865 * and initiates a balancing operation if so.
@@ -2866,9 +2867,10 @@ static void update_load(struct rq *this_rq)
2866 * Balancing parameters are set up in arch_init_sched_domains. 2867 * Balancing parameters are set up in arch_init_sched_domains.
2867 */ 2868 */
2868 2869
2869static void 2870static void run_rebalance_domains(struct softirq_action *h)
2870rebalance_tick(int this_cpu, struct rq *this_rq)
2871{ 2871{
2872 int this_cpu = smp_processor_id();
2873 struct rq *this_rq = cpu_rq(this_cpu);
2872 unsigned long interval; 2874 unsigned long interval;
2873 struct sched_domain *sd; 2875 struct sched_domain *sd;
2874 /* 2876 /*
@@ -2877,6 +2879,8 @@ rebalance_tick(int this_cpu, struct rq *this_rq)
2877 */ 2879 */
2878 enum idle_type idle = !this_rq->nr_running ? 2880 enum idle_type idle = !this_rq->nr_running ?
2879 SCHED_IDLE : NOT_IDLE; 2881 SCHED_IDLE : NOT_IDLE;
2882 /* Earliest time when we have to call run_rebalance_domains again */
2883 unsigned long next_balance = jiffies + 60*HZ;
2880 2884
2881 for_each_domain(this_cpu, sd) { 2885 for_each_domain(this_cpu, sd) {
2882 if (!(sd->flags & SD_LOAD_BALANCE)) 2886 if (!(sd->flags & SD_LOAD_BALANCE))
@@ -2891,7 +2895,7 @@ rebalance_tick(int this_cpu, struct rq *this_rq)
2891 if (unlikely(!interval)) 2895 if (unlikely(!interval))
2892 interval = 1; 2896 interval = 1;
2893 2897
2894 if (jiffies - sd->last_balance >= interval) { 2898 if (time_after_eq(jiffies, sd->last_balance + interval)) {
2895 if (load_balance(this_cpu, this_rq, sd, idle)) { 2899 if (load_balance(this_cpu, this_rq, sd, idle)) {
2896 /* 2900 /*
2897 * We've pulled tasks over so either we're no 2901 * We've pulled tasks over so either we're no
@@ -2902,7 +2906,10 @@ rebalance_tick(int this_cpu, struct rq *this_rq)
2902 } 2906 }
2903 sd->last_balance += interval; 2907 sd->last_balance += interval;
2904 } 2908 }
2909 if (time_after(next_balance, sd->last_balance + interval))
2910 next_balance = sd->last_balance + interval;
2905 } 2911 }
2912 this_rq->next_balance = next_balance;
2906} 2913}
2907#else 2914#else
2908/* 2915/*
@@ -3155,7 +3162,8 @@ void scheduler_tick(void)
3155 task_running_tick(rq, p); 3162 task_running_tick(rq, p);
3156#ifdef CONFIG_SMP 3163#ifdef CONFIG_SMP
3157 update_load(rq); 3164 update_load(rq);
3158 rebalance_tick(cpu, rq); 3165 if (time_after_eq(jiffies, rq->next_balance))
3166 raise_softirq(SCHED_SOFTIRQ);
3159#endif 3167#endif
3160} 3168}
3161 3169
@@ -6859,6 +6867,10 @@ void __init sched_init(void)
6859 6867
6860 set_load_weight(&init_task); 6868 set_load_weight(&init_task);
6861 6869
6870#ifdef CONFIG_SMP
6871 open_softirq(SCHED_SOFTIRQ, run_rebalance_domains, NULL);
6872#endif
6873
6862#ifdef CONFIG_RT_MUTEXES 6874#ifdef CONFIG_RT_MUTEXES
6863 plist_head_init(&init_task.pi_waiters, &init_task.pi_lock); 6875 plist_head_init(&init_task.pi_waiters, &init_task.pi_lock);
6864#endif 6876#endif