aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMiao Xie <miaox@cn.fujitsu.com>2009-06-16 18:31:47 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2009-06-16 22:47:31 -0400
commit950592f7b991f267d707d372b90f508bbe72acbc (patch)
treeb57ef015ff5cd54d8ab21005b466c80e9ccfa9e3
parentf3b39d47ebc51416fc3b690a32dfe030a2035e67 (diff)
cpusets: update tasks' page/slab spread flags in time
Fix the bug that the kernel didn't spread page cache/slab object evenly over all the allowed nodes when spread flags were set by updating tasks' page/slab spread flags in time. Signed-off-by: Miao Xie <miaox@cn.fujitsu.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Christoph Lameter <cl@linux-foundation.org> Cc: Paul Menage <menage@google.com> Cc: Nick Piggin <nickpiggin@yahoo.com.au> Cc: Yasunori Goto <y-goto@jp.fujitsu.com> Cc: Pekka Enberg <penberg@cs.helsinki.fi> Cc: David Rientjes <rientjes@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
-rw-r--r--kernel/cpuset.c59
1 files changed, 55 insertions, 4 deletions
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index 66b24d9b6638..af5a83d52187 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -406,7 +406,6 @@ void cpuset_update_task_memory_state(void)
406 cs = task_cs(tsk); /* Maybe changed when task not locked */ 406 cs = task_cs(tsk); /* Maybe changed when task not locked */
407 guarantee_online_mems(cs, &tsk->mems_allowed); 407 guarantee_online_mems(cs, &tsk->mems_allowed);
408 tsk->cpuset_mems_generation = cs->mems_generation; 408 tsk->cpuset_mems_generation = cs->mems_generation;
409 cpuset_update_task_spread_flag(cs, tsk);
410 task_unlock(tsk); 409 task_unlock(tsk);
411 mutex_unlock(&callback_mutex); 410 mutex_unlock(&callback_mutex);
412 mpol_rebind_task(tsk, &tsk->mems_allowed); 411 mpol_rebind_task(tsk, &tsk->mems_allowed);
@@ -1204,6 +1203,46 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val)
1204} 1203}
1205 1204
1206/* 1205/*
1206 * cpuset_change_flag - make a task's spread flags the same as its cpuset's
1207 * @tsk: task to be updated
1208 * @scan: struct cgroup_scanner containing the cgroup of the task
1209 *
1210 * Called by cgroup_scan_tasks() for each task in a cgroup.
1211 *
1212 * We don't need to re-check for the cgroup/cpuset membership, since we're
1213 * holding cgroup_lock() at this point.
1214 */
1215static void cpuset_change_flag(struct task_struct *tsk,
1216 struct cgroup_scanner *scan)
1217{
1218 cpuset_update_task_spread_flag(cgroup_cs(scan->cg), tsk);
1219}
1220
1221/*
1222 * update_tasks_flags - update the spread flags of tasks in the cpuset.
1223 * @cs: the cpuset in which each task's spread flags needs to be changed
1224 * @heap: if NULL, defer allocating heap memory to cgroup_scan_tasks()
1225 *
1226 * Called with cgroup_mutex held
1227 *
1228 * The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
1229 * calling callback functions for each.
1230 *
1231 * No return value. It's guaranteed that cgroup_scan_tasks() always returns 0
1232 * if @heap != NULL.
1233 */
1234static void update_tasks_flags(struct cpuset *cs, struct ptr_heap *heap)
1235{
1236 struct cgroup_scanner scan;
1237
1238 scan.cg = cs->css.cgroup;
1239 scan.test_task = NULL;
1240 scan.process_task = cpuset_change_flag;
1241 scan.heap = heap;
1242 cgroup_scan_tasks(&scan);
1243}
1244
1245/*
1207 * update_flag - read a 0 or a 1 in a file and update associated flag 1246 * update_flag - read a 0 or a 1 in a file and update associated flag
1208 * bit: the bit to update (see cpuset_flagbits_t) 1247 * bit: the bit to update (see cpuset_flagbits_t)
1209 * cs: the cpuset to update 1248 * cs: the cpuset to update
@@ -1216,8 +1255,10 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
1216 int turning_on) 1255 int turning_on)
1217{ 1256{
1218 struct cpuset *trialcs; 1257 struct cpuset *trialcs;
1219 int err;
1220 int balance_flag_changed; 1258 int balance_flag_changed;
1259 int spread_flag_changed;
1260 struct ptr_heap heap;
1261 int err;
1221 1262
1222 trialcs = alloc_trial_cpuset(cs); 1263 trialcs = alloc_trial_cpuset(cs);
1223 if (!trialcs) 1264 if (!trialcs)
@@ -1232,9 +1273,16 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
1232 if (err < 0) 1273 if (err < 0)
1233 goto out; 1274 goto out;
1234 1275
1276 err = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
1277 if (err < 0)
1278 goto out;
1279
1235 balance_flag_changed = (is_sched_load_balance(cs) != 1280 balance_flag_changed = (is_sched_load_balance(cs) !=
1236 is_sched_load_balance(trialcs)); 1281 is_sched_load_balance(trialcs));
1237 1282
1283 spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
1284 || (is_spread_page(cs) != is_spread_page(trialcs)));
1285
1238 mutex_lock(&callback_mutex); 1286 mutex_lock(&callback_mutex);
1239 cs->flags = trialcs->flags; 1287 cs->flags = trialcs->flags;
1240 mutex_unlock(&callback_mutex); 1288 mutex_unlock(&callback_mutex);
@@ -1242,6 +1290,9 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
1242 if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) 1290 if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
1243 async_rebuild_sched_domains(); 1291 async_rebuild_sched_domains();
1244 1292
1293 if (spread_flag_changed)
1294 update_tasks_flags(cs, &heap);
1295 heap_free(&heap);
1245out: 1296out:
1246 free_trial_cpuset(trialcs); 1297 free_trial_cpuset(trialcs);
1247 return err; 1298 return err;
@@ -1392,6 +1443,8 @@ static void cpuset_attach(struct cgroup_subsys *ss,
1392 if (err) 1443 if (err)
1393 return; 1444 return;
1394 1445
1446 cpuset_update_task_spread_flag(cs, tsk);
1447
1395 from = oldcs->mems_allowed; 1448 from = oldcs->mems_allowed;
1396 to = cs->mems_allowed; 1449 to = cs->mems_allowed;
1397 mm = get_task_mm(tsk); 1450 mm = get_task_mm(tsk);
@@ -1453,11 +1506,9 @@ static int cpuset_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
1453 break; 1506 break;
1454 case FILE_SPREAD_PAGE: 1507 case FILE_SPREAD_PAGE:
1455 retval = update_flag(CS_SPREAD_PAGE, cs, val); 1508 retval = update_flag(CS_SPREAD_PAGE, cs, val);
1456 cs->mems_generation = cpuset_mems_generation++;
1457 break; 1509 break;
1458 case FILE_SPREAD_SLAB: 1510 case FILE_SPREAD_SLAB:
1459 retval = update_flag(CS_SPREAD_SLAB, cs, val); 1511 retval = update_flag(CS_SPREAD_SLAB, cs, val);
1460 cs->mems_generation = cpuset_mems_generation++;
1461 break; 1512 break;
1462 default: 1513 default:
1463 retval = -EINVAL; 1514 retval = -EINVAL;