diff options
author | Miao Xie <miaox@cn.fujitsu.com> | 2009-06-16 18:31:47 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2009-06-16 22:47:31 -0400 |
commit | 950592f7b991f267d707d372b90f508bbe72acbc (patch) | |
tree | b57ef015ff5cd54d8ab21005b466c80e9ccfa9e3 | |
parent | f3b39d47ebc51416fc3b690a32dfe030a2035e67 (diff) |
cpusets: update tasks' page/slab spread flags in time
Fix the bug that the kernel didn't spread page cache/slab object evenly
over all the allowed nodes when spread flags were set by updating tasks'
page/slab spread flags in time.
Signed-off-by: Miao Xie <miaox@cn.fujitsu.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christoph Lameter <cl@linux-foundation.org>
Cc: Paul Menage <menage@google.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Yasunori Goto <y-goto@jp.fujitsu.com>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
-rw-r--r-- | kernel/cpuset.c | 59 |
1 files changed, 55 insertions, 4 deletions
diff --git a/kernel/cpuset.c b/kernel/cpuset.c index 66b24d9b6638..af5a83d52187 100644 --- a/kernel/cpuset.c +++ b/kernel/cpuset.c | |||
@@ -406,7 +406,6 @@ void cpuset_update_task_memory_state(void) | |||
406 | cs = task_cs(tsk); /* Maybe changed when task not locked */ | 406 | cs = task_cs(tsk); /* Maybe changed when task not locked */ |
407 | guarantee_online_mems(cs, &tsk->mems_allowed); | 407 | guarantee_online_mems(cs, &tsk->mems_allowed); |
408 | tsk->cpuset_mems_generation = cs->mems_generation; | 408 | tsk->cpuset_mems_generation = cs->mems_generation; |
409 | cpuset_update_task_spread_flag(cs, tsk); | ||
410 | task_unlock(tsk); | 409 | task_unlock(tsk); |
411 | mutex_unlock(&callback_mutex); | 410 | mutex_unlock(&callback_mutex); |
412 | mpol_rebind_task(tsk, &tsk->mems_allowed); | 411 | mpol_rebind_task(tsk, &tsk->mems_allowed); |
@@ -1204,6 +1203,46 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val) | |||
1204 | } | 1203 | } |
1205 | 1204 | ||
1206 | /* | 1205 | /* |
1206 | * cpuset_change_flag - make a task's spread flags the same as its cpuset's | ||
1207 | * @tsk: task to be updated | ||
1208 | * @scan: struct cgroup_scanner containing the cgroup of the task | ||
1209 | * | ||
1210 | * Called by cgroup_scan_tasks() for each task in a cgroup. | ||
1211 | * | ||
1212 | * We don't need to re-check for the cgroup/cpuset membership, since we're | ||
1213 | * holding cgroup_lock() at this point. | ||
1214 | */ | ||
1215 | static void cpuset_change_flag(struct task_struct *tsk, | ||
1216 | struct cgroup_scanner *scan) | ||
1217 | { | ||
1218 | cpuset_update_task_spread_flag(cgroup_cs(scan->cg), tsk); | ||
1219 | } | ||
1220 | |||
1221 | /* | ||
1222 | * update_tasks_flags - update the spread flags of tasks in the cpuset. | ||
1223 | * @cs: the cpuset in which each task's spread flags needs to be changed | ||
1224 | * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks() | ||
1225 | * | ||
1226 | * Called with cgroup_mutex held | ||
1227 | * | ||
1228 | * The cgroup_scan_tasks() function will scan all the tasks in a cgroup, | ||
1229 | * calling callback functions for each. | ||
1230 | * | ||
1231 | * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0 | ||
1232 | * if @heap != NULL. | ||
1233 | */ | ||
1234 | static void update_tasks_flags(struct cpuset *cs, struct ptr_heap *heap) | ||
1235 | { | ||
1236 | struct cgroup_scanner scan; | ||
1237 | |||
1238 | scan.cg = cs->css.cgroup; | ||
1239 | scan.test_task = NULL; | ||
1240 | scan.process_task = cpuset_change_flag; | ||
1241 | scan.heap = heap; | ||
1242 | cgroup_scan_tasks(&scan); | ||
1243 | } | ||
1244 | |||
1245 | /* | ||
1207 | * update_flag - read a 0 or a 1 in a file and update associated flag | 1246 | * update_flag - read a 0 or a 1 in a file and update associated flag |
1208 | * bit: the bit to update (see cpuset_flagbits_t) | 1247 | * bit: the bit to update (see cpuset_flagbits_t) |
1209 | * cs: the cpuset to update | 1248 | * cs: the cpuset to update |
@@ -1216,8 +1255,10 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, | |||
1216 | int turning_on) | 1255 | int turning_on) |
1217 | { | 1256 | { |
1218 | struct cpuset *trialcs; | 1257 | struct cpuset *trialcs; |
1219 | int err; | ||
1220 | int balance_flag_changed; | 1258 | int balance_flag_changed; |
1259 | int spread_flag_changed; | ||
1260 | struct ptr_heap heap; | ||
1261 | int err; | ||
1221 | 1262 | ||
1222 | trialcs = alloc_trial_cpuset(cs); | 1263 | trialcs = alloc_trial_cpuset(cs); |
1223 | if (!trialcs) | 1264 | if (!trialcs) |
@@ -1232,9 +1273,16 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, | |||
1232 | if (err < 0) | 1273 | if (err < 0) |
1233 | goto out; | 1274 | goto out; |
1234 | 1275 | ||
1276 | err = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL); | ||
1277 | if (err < 0) | ||
1278 | goto out; | ||
1279 | |||
1235 | balance_flag_changed = (is_sched_load_balance(cs) != | 1280 | balance_flag_changed = (is_sched_load_balance(cs) != |
1236 | is_sched_load_balance(trialcs)); | 1281 | is_sched_load_balance(trialcs)); |
1237 | 1282 | ||
1283 | spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs)) | ||
1284 | || (is_spread_page(cs) != is_spread_page(trialcs))); | ||
1285 | |||
1238 | mutex_lock(&callback_mutex); | 1286 | mutex_lock(&callback_mutex); |
1239 | cs->flags = trialcs->flags; | 1287 | cs->flags = trialcs->flags; |
1240 | mutex_unlock(&callback_mutex); | 1288 | mutex_unlock(&callback_mutex); |
@@ -1242,6 +1290,9 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, | |||
1242 | if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) | 1290 | if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) |
1243 | async_rebuild_sched_domains(); | 1291 | async_rebuild_sched_domains(); |
1244 | 1292 | ||
1293 | if (spread_flag_changed) | ||
1294 | update_tasks_flags(cs, &heap); | ||
1295 | heap_free(&heap); | ||
1245 | out: | 1296 | out: |
1246 | free_trial_cpuset(trialcs); | 1297 | free_trial_cpuset(trialcs); |
1247 | return err; | 1298 | return err; |
@@ -1392,6 +1443,8 @@ static void cpuset_attach(struct cgroup_subsys *ss, | |||
1392 | if (err) | 1443 | if (err) |
1393 | return; | 1444 | return; |
1394 | 1445 | ||
1446 | cpuset_update_task_spread_flag(cs, tsk); | ||
1447 | |||
1395 | from = oldcs->mems_allowed; | 1448 | from = oldcs->mems_allowed; |
1396 | to = cs->mems_allowed; | 1449 | to = cs->mems_allowed; |
1397 | mm = get_task_mm(tsk); | 1450 | mm = get_task_mm(tsk); |
@@ -1453,11 +1506,9 @@ static int cpuset_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val) | |||
1453 | break; | 1506 | break; |
1454 | case FILE_SPREAD_PAGE: | 1507 | case FILE_SPREAD_PAGE: |
1455 | retval = update_flag(CS_SPREAD_PAGE, cs, val); | 1508 | retval = update_flag(CS_SPREAD_PAGE, cs, val); |
1456 | cs->mems_generation = cpuset_mems_generation++; | ||
1457 | break; | 1509 | break; |
1458 | case FILE_SPREAD_SLAB: | 1510 | case FILE_SPREAD_SLAB: |
1459 | retval = update_flag(CS_SPREAD_SLAB, cs, val); | 1511 | retval = update_flag(CS_SPREAD_SLAB, cs, val); |
1460 | cs->mems_generation = cpuset_mems_generation++; | ||
1461 | break; | 1512 | break; |
1462 | default: | 1513 | default: |
1463 | retval = -EINVAL; | 1514 | retval = -EINVAL; |