aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
authorSimon Derr <Simon.Derr@bull.net>2005-08-04 22:52:03 -0400
committerLinus Torvalds <torvalds@g5.osdl.org>2005-08-05 00:43:14 -0400
commit2f60f8d3573ff90fe5d75a6d11fd2add1248e7d6 (patch)
tree173980b12459aa55586d3f4f56630abf5af1ffcc /mm
parentb68e9f857271189bd7a59b74c99890de9195b0e1 (diff)
[PATCH] __vm_enough_memory() signedness fix
We have found what seems to be a small bug in __vm_enough_memory() when sysctl_overcommit_memory is set to OVERCOMMIT_NEVER. When this bug occurs the systems fails to boot, with /sbin/init whining about fork() returning ENOMEM. We hunted down the problem to this: The deferred update mecanism used in vm_acct_memory(), on a SMP system, allows the vm_committed_space counter to have a negative value. This should not be a problem since this counter is known to be inaccurate. But in __vm_enough_memory() this counter is compared to the `allowed' variable, which is an unsigned long. This comparison is broken since it will consider the negative values of vm_committed_space to be huge positive values, resulting in a memory allocation failure. Signed-off-by: <Jean-Marc.Saffroy@ext.bull.net> Signed-off-by: <Simon.Derr@bull.net> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'mm')
-rw-r--r--mm/mmap.c6
-rw-r--r--mm/nommu.c6
2 files changed, 10 insertions, 2 deletions
diff --git a/mm/mmap.c b/mm/mmap.c
index da3fa90a0aae..404319477e71 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -143,7 +143,11 @@ int __vm_enough_memory(long pages, int cap_sys_admin)
143 leave 3% of the size of this process for other processes */ 143 leave 3% of the size of this process for other processes */
144 allowed -= current->mm->total_vm / 32; 144 allowed -= current->mm->total_vm / 32;
145 145
146 if (atomic_read(&vm_committed_space) < allowed) 146 /*
147 * cast `allowed' as a signed long because vm_committed_space
148 * sometimes has a negative value
149 */
150 if (atomic_read(&vm_committed_space) < (long)allowed)
147 return 0; 151 return 0;
148 152
149 vm_unacct_memory(pages); 153 vm_unacct_memory(pages);
diff --git a/mm/nommu.c b/mm/nommu.c
index ce74452c02d9..fd4e8df0f02d 100644
--- a/mm/nommu.c
+++ b/mm/nommu.c
@@ -1167,7 +1167,11 @@ int __vm_enough_memory(long pages, int cap_sys_admin)
1167 leave 3% of the size of this process for other processes */ 1167 leave 3% of the size of this process for other processes */
1168 allowed -= current->mm->total_vm / 32; 1168 allowed -= current->mm->total_vm / 32;
1169 1169
1170 if (atomic_read(&vm_committed_space) < allowed) 1170 /*
1171 * cast `allowed' as a signed long because vm_committed_space
1172 * sometimes has a negative value
1173 */
1174 if (atomic_read(&vm_committed_space) < (long)allowed)
1171 return 0; 1175 return 0;
1172 1176
1173 vm_unacct_memory(pages); 1177 vm_unacct_memory(pages);