aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux
diff options
context:
space:
mode:
authorMiao Xie <miaox@cn.fujitsu.com>2009-06-16 18:31:49 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2009-06-16 22:47:31 -0400
commit58568d2a8215cb6f55caf2332017d7bdff954e1c (patch)
treeffcdee457494ac78d6550b0aeac86536ca152e7b /include/linux
parent950592f7b991f267d707d372b90f508bbe72acbc (diff)
cpuset,mm: update tasks' mems_allowed in time
Fix allocating page cache/slab object on the unallowed node when memory spread is set by updating tasks' mems_allowed after its cpuset's mems is changed. In order to update tasks' mems_allowed in time, we must modify the code of memory policy. Because the memory policy is applied in the process's context originally. After applying this patch, one task directly manipulates anothers mems_allowed, and we use alloc_lock in the task_struct to protect mems_allowed and memory policy of the task. But in the fast path, we didn't use lock to protect them, because adding a lock may lead to performance regression. But if we don't add a lock,the task might see no nodes when changing cpuset's mems_allowed to some non-overlapping set. In order to avoid it, we set all new allowed nodes, then clear newly disallowed ones. [lee.schermerhorn@hp.com: The rework of mpol_new() to extract the adjusting of the node mask to apply cpuset and mpol flags "context" breaks set_mempolicy() and mbind() with MPOL_PREFERRED and a NULL nodemask--i.e., explicit local allocation. Fix this by adding the check for MPOL_PREFERRED and empty node mask to mpol_new_mpolicy(). Remove the now unneeded 'nodes = NULL' from mpol_new(). Note that mpol_new_mempolicy() is always called with a non-NULL 'nodes' parameter now that it has been removed from mpol_new(). Therefore, we don't need to test nodes for NULL before testing it for 'empty'. However, just to be extra paranoid, add a VM_BUG_ON() to verify this assumption.] [lee.schermerhorn@hp.com: I don't think the function name 'mpol_new_mempolicy' is descriptive enough to differentiate it from mpol_new(). This function applies cpuset set context, usually constraining nodes to those allowed by the cpuset. However, when the 'RELATIVE_NODES flag is set, it also translates the nodes. So I settled on 'mpol_set_nodemask()', because the comment block for mpol_new() mentions that we need to call this function to "set nodes". Some additional minor line length, whitespace and typo cleanup.] Signed-off-by: Miao Xie <miaox@cn.fujitsu.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Christoph Lameter <cl@linux-foundation.org> Cc: Paul Menage <menage@google.com> Cc: Nick Piggin <nickpiggin@yahoo.com.au> Cc: Yasunori Goto <y-goto@jp.fujitsu.com> Cc: Pekka Enberg <penberg@cs.helsinki.fi> Cc: David Rientjes <rientjes@google.com> Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/cpuset.h13
-rw-r--r--include/linux/sched.h8
2 files changed, 13 insertions, 8 deletions
diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h
index 05ea1dd7d681..a5740fc4d04b 100644
--- a/include/linux/cpuset.h
+++ b/include/linux/cpuset.h
@@ -18,7 +18,6 @@
18 18
19extern int number_of_cpusets; /* How many cpusets are defined in system? */ 19extern int number_of_cpusets; /* How many cpusets are defined in system? */
20 20
21extern int cpuset_init_early(void);
22extern int cpuset_init(void); 21extern int cpuset_init(void);
23extern void cpuset_init_smp(void); 22extern void cpuset_init_smp(void);
24extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); 23extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask);
@@ -27,7 +26,6 @@ extern void cpuset_cpus_allowed_locked(struct task_struct *p,
27extern nodemask_t cpuset_mems_allowed(struct task_struct *p); 26extern nodemask_t cpuset_mems_allowed(struct task_struct *p);
28#define cpuset_current_mems_allowed (current->mems_allowed) 27#define cpuset_current_mems_allowed (current->mems_allowed)
29void cpuset_init_current_mems_allowed(void); 28void cpuset_init_current_mems_allowed(void);
30void cpuset_update_task_memory_state(void);
31int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask); 29int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask);
32 30
33extern int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask); 31extern int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask);
@@ -92,9 +90,13 @@ extern void rebuild_sched_domains(void);
92 90
93extern void cpuset_print_task_mems_allowed(struct task_struct *p); 91extern void cpuset_print_task_mems_allowed(struct task_struct *p);
94 92
93static inline void set_mems_allowed(nodemask_t nodemask)
94{
95 current->mems_allowed = nodemask;
96}
97
95#else /* !CONFIG_CPUSETS */ 98#else /* !CONFIG_CPUSETS */
96 99
97static inline int cpuset_init_early(void) { return 0; }
98static inline int cpuset_init(void) { return 0; } 100static inline int cpuset_init(void) { return 0; }
99static inline void cpuset_init_smp(void) {} 101static inline void cpuset_init_smp(void) {}
100 102
@@ -116,7 +118,6 @@ static inline nodemask_t cpuset_mems_allowed(struct task_struct *p)
116 118
117#define cpuset_current_mems_allowed (node_states[N_HIGH_MEMORY]) 119#define cpuset_current_mems_allowed (node_states[N_HIGH_MEMORY])
118static inline void cpuset_init_current_mems_allowed(void) {} 120static inline void cpuset_init_current_mems_allowed(void) {}
119static inline void cpuset_update_task_memory_state(void) {}
120 121
121static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) 122static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
122{ 123{
@@ -188,6 +189,10 @@ static inline void cpuset_print_task_mems_allowed(struct task_struct *p)
188{ 189{
189} 190}
190 191
192static inline void set_mems_allowed(nodemask_t nodemask)
193{
194}
195
191#endif /* !CONFIG_CPUSETS */ 196#endif /* !CONFIG_CPUSETS */
192 197
193#endif /* _LINUX_CPUSET_H */ 198#endif /* _LINUX_CPUSET_H */
diff --git a/include/linux/sched.h b/include/linux/sched.h
index c900aa530070..1048bf50540a 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1318,7 +1318,8 @@ struct task_struct {
1318/* Thread group tracking */ 1318/* Thread group tracking */
1319 u32 parent_exec_id; 1319 u32 parent_exec_id;
1320 u32 self_exec_id; 1320 u32 self_exec_id;
1321/* Protection of (de-)allocation: mm, files, fs, tty, keyrings */ 1321/* Protection of (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed,
1322 * mempolicy */
1322 spinlock_t alloc_lock; 1323 spinlock_t alloc_lock;
1323 1324
1324#ifdef CONFIG_GENERIC_HARDIRQS 1325#ifdef CONFIG_GENERIC_HARDIRQS
@@ -1386,8 +1387,7 @@ struct task_struct {
1386 cputime_t acct_timexpd; /* stime + utime since last update */ 1387 cputime_t acct_timexpd; /* stime + utime since last update */
1387#endif 1388#endif
1388#ifdef CONFIG_CPUSETS 1389#ifdef CONFIG_CPUSETS
1389 nodemask_t mems_allowed; 1390 nodemask_t mems_allowed; /* Protected by alloc_lock */
1390 int cpuset_mems_generation;
1391 int cpuset_mem_spread_rotor; 1391 int cpuset_mem_spread_rotor;
1392#endif 1392#endif
1393#ifdef CONFIG_CGROUPS 1393#ifdef CONFIG_CGROUPS
@@ -1410,7 +1410,7 @@ struct task_struct {
1410 struct list_head perf_counter_list; 1410 struct list_head perf_counter_list;
1411#endif 1411#endif
1412#ifdef CONFIG_NUMA 1412#ifdef CONFIG_NUMA
1413 struct mempolicy *mempolicy; 1413 struct mempolicy *mempolicy; /* Protected by alloc_lock */
1414 short il_next; 1414 short il_next;
1415#endif 1415#endif
1416 atomic_t fs_excl; /* holding fs exclusive resources */ 1416 atomic_t fs_excl; /* holding fs exclusive resources */