mm: convert mm->cpu_vm_cpumask into cpumask_var_t

cpumask_t is very big struct and cpu_vm_mask is placed wrong position.
It might lead to reduce cache hit ratio.

This patch has two change.
1) Move the place of cpumask into last of mm_struct. Because usually cpumask
   is accessed only front bits when the system has cpu-hotplug capability
2) Convert cpu_vm_mask into cpumask_var_t. It may help to reduce memory
   footprint if cpumask_size() will use nr_cpumask_bits properly in future.

In addition, this patch change the name of cpu_vm_mask with cpu_vm_mask_var.
It may help to detect out of tree cpu_vm_mask users.

This patch has no functional change.

[akpm@linux-foundation.org: build fix]
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Koichi Yasutake <yasutake.koichi@jp.panasonic.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Chris Metcalf <cmetcalf@tilera.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 34 insertions, 3 deletions

diff --git a/kernel/fork.c b/kernel/fork.c
index 927692734bc..8e7e135d081 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -485,6 +485,20 @@ static void mm_init_aio(struct mm_struct *mm)
 #endif
 }
 
+int mm_init_cpumask(struct mm_struct *mm, struct mm_struct *oldmm)
+{
+#ifdef CONFIG_CPUMASK_OFFSTACK
+        if (!alloc_cpumask_var(&mm->cpu_vm_mask_var, GFP_KERNEL))
+                return -ENOMEM;
+
+        if (oldmm)
+                cpumask_copy(mm_cpumask(mm), mm_cpumask(oldmm));
+        else
+                memset(mm_cpumask(mm), 0, cpumask_size());
+#endif
+        return 0;
+}
+
 static struct mm_struct * mm_init(struct mm_struct * mm, struct task_struct *p)
 {
         atomic_set(&mm->mm_users, 1);
@@ -521,10 +535,20 @@ struct mm_struct * mm_alloc(void)
         struct mm_struct * mm;
 
         mm = allocate_mm();
-        if (mm) {
-                memset(mm, 0, sizeof(*mm));
-                mm = mm_init(mm, current);
+        if (!mm)
+                return NULL;
+
+        memset(mm, 0, sizeof(*mm));
+        mm = mm_init(mm, current);
+        if (!mm)
+                return NULL;
+
+        if (mm_init_cpumask(mm, NULL)) {
+                mm_free_pgd(mm);
+                free_mm(mm);
+                return NULL;
         }
+
         return mm;
 }
 
@@ -536,6 +560,7 @@ struct mm_struct * mm_alloc(void)
 void __mmdrop(struct mm_struct *mm)
 {
         BUG_ON(mm == &init_mm);
+        free_cpumask_var(mm->cpu_vm_mask_var);
         mm_free_pgd(mm);
         destroy_context(mm);
         mmu_notifier_mm_destroy(mm);
@@ -690,6 +715,9 @@ struct mm_struct *dup_mm(struct task_struct *tsk)
         if (!mm_init(mm, tsk))
                 goto fail_nomem;
 
+        if (mm_init_cpumask(mm, oldmm))
+                goto fail_nocpumask;
+
         if (init_new_context(tsk, mm))
                 goto fail_nocontext;
 
@@ -716,6 +744,9 @@ fail_nomem:
         return NULL;
 
 fail_nocontext:
+        free_cpumask_var(mm->cpu_vm_mask_var);
+
+fail_nocpumask:
         /*
          * If init_new_context() failed, we cannot use mmput() to free the mm
          * because it calls destroy_context()