1 files changed, 50 insertions, 24 deletions

diff --git a/mm/mmap.c b/mm/mmap.c
index 456ec6f27889..6128dc8e5ede 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -388,17 +388,23 @@ static inline void
 __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
                 struct vm_area_struct *prev, struct rb_node *rb_parent)
 {
+        struct vm_area_struct *next;
+
+        vma->vm_prev = prev;
         if (prev) {
-                vma->vm_next = prev->vm_next;
+                next = prev->vm_next;
                 prev->vm_next = vma;
         } else {
                 mm->mmap = vma;
                 if (rb_parent)
-                        vma->vm_next = rb_entry(rb_parent,
+                        next = rb_entry(rb_parent,
                                         struct vm_area_struct, vm_rb);
                 else
-                        vma->vm_next = NULL;
+                        next = NULL;
         }
+        vma->vm_next = next;
+        if (next)
+                next->vm_prev = vma;
 }
 
 void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
@@ -452,12 +458,10 @@ static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
                 spin_lock(&mapping->i_mmap_lock);
                 vma->vm_truncate_count = mapping->truncate_count;
         }
-        anon_vma_lock(vma);
 
         __vma_link(mm, vma, prev, rb_link, rb_parent);
         __vma_link_file(vma);
 
-        anon_vma_unlock(vma);
         if (mapping)
                 spin_unlock(&mapping->i_mmap_lock);
 
@@ -485,7 +489,11 @@ static inline void
 __vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
                 struct vm_area_struct *prev)
 {
-        prev->vm_next = vma->vm_next;
+        struct vm_area_struct *next = vma->vm_next;
+
+        prev->vm_next = next;
+        if (next)
+                next->vm_prev = prev;
         rb_erase(&vma->vm_rb, &mm->mm_rb);
         if (mm->mmap_cache == vma)
                 mm->mmap_cache = prev;
@@ -506,6 +514,7 @@ int vma_adjust(struct vm_area_struct *vma, unsigned long start,
         struct vm_area_struct *importer = NULL;
         struct address_space *mapping = NULL;
         struct prio_tree_root *root = NULL;
+        struct anon_vma *anon_vma = NULL;
         struct file *file = vma->vm_file;
         long adjust_next = 0;
         int remove_next = 0;
@@ -578,6 +587,17 @@ again:			remove_next = 1 + (end > next->vm_end);
                 }
         }
 
+        /*
+         * When changing only vma->vm_end, we don't really need anon_vma
+         * lock. This is a fairly rare case by itself, but the anon_vma
+         * lock may be shared between many sibling processes.  Skipping
+         * the lock for brk adjustments makes a difference sometimes.
+         */
+        if (vma->anon_vma && (insert || importer || start != vma->vm_start)) {
+                anon_vma = vma->anon_vma;
+                anon_vma_lock(anon_vma);
+        }
+
         if (root) {
                 flush_dcache_mmap_lock(mapping);
                 vma_prio_tree_remove(vma, root);
@@ -617,6 +637,8 @@ again:			remove_next = 1 + (end > next->vm_end);
                 __insert_vm_struct(mm, insert);
         }
 
+        if (anon_vma)
+                anon_vma_unlock(anon_vma);
         if (mapping)
                 spin_unlock(&mapping->i_mmap_lock);
 
@@ -1694,9 +1716,6 @@ static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, uns
  * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
  * vma is the last one with address > vma->vm_end.  Have to extend vma.
  */
-#ifndef CONFIG_IA64
-static
-#endif
 int expand_upwards(struct vm_area_struct *vma, unsigned long address)
 {
         int error;
@@ -1710,7 +1729,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
          */
         if (unlikely(anon_vma_prepare(vma)))
                 return -ENOMEM;
-        anon_vma_lock(vma);
+        vma_lock_anon_vma(vma);
 
         /*
          * vma->vm_start/vm_end cannot change under us because the caller
@@ -1721,7 +1740,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
         if (address < PAGE_ALIGN(address+4))
                 address = PAGE_ALIGN(address+4);
         else {
-                anon_vma_unlock(vma);
+                vma_unlock_anon_vma(vma);
                 return -ENOMEM;
         }
         error = 0;
@@ -1734,10 +1753,12 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
                 grow = (address - vma->vm_end) >> PAGE_SHIFT;
 
                 error = acct_stack_growth(vma, size, grow);
-                if (!error)
+                if (!error) {
                         vma->vm_end = address;
+                        perf_event_mmap(vma);
+                }
         }
-        anon_vma_unlock(vma);
+        vma_unlock_anon_vma(vma);
         return error;
 }
 #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
@@ -1762,7 +1783,7 @@ static int expand_downwards(struct vm_area_struct *vma,
         if (error)
                 return error;
 
-        anon_vma_lock(vma);
+        vma_lock_anon_vma(vma);
 
         /*
          * vma->vm_start/vm_end cannot change under us because the caller
@@ -1781,9 +1802,10 @@ static int expand_downwards(struct vm_area_struct *vma,
                 if (!error) {
                         vma->vm_start = address;
                         vma->vm_pgoff -= grow;
+                        perf_event_mmap(vma);
                 }
         }
-        anon_vma_unlock(vma);
+        vma_unlock_anon_vma(vma);
         return error;
 }
 
@@ -1900,6 +1922,7 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
         unsigned long addr;
 
         insertion_point = (prev ? &prev->vm_next : &mm->mmap);
+        vma->vm_prev = NULL;
         do {
                 rb_erase(&vma->vm_rb, &mm->mm_rb);
                 mm->map_count--;
@@ -1907,6 +1930,8 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
                 vma = vma->vm_next;
         } while (vma && vma->vm_start < end);
         *insertion_point = vma;
+        if (vma)
+                vma->vm_prev = prev;
         tail_vma->vm_next = NULL;
         if (mm->unmap_area == arch_unmap_area)
                 addr = prev ? prev->vm_end : mm->mmap_base;
@@ -2208,6 +2233,7 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
         vma->vm_page_prot = vm_get_page_prot(flags);
         vma_link(mm, vma, prev, rb_link, rb_parent);
 out:
+        perf_event_mmap(vma);
         mm->total_vm += len >> PAGE_SHIFT;
         if (flags & VM_LOCKED) {
                 if (!mlock_vma_pages_range(vma, addr, addr + len))
@@ -2466,23 +2492,23 @@ static DEFINE_MUTEX(mm_all_locks_mutex);
 
 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
 {
-        if (!test_bit(0, (unsigned long *) &anon_vma->head.next)) {
+        if (!test_bit(0, (unsigned long *) &anon_vma->root->head.next)) {
                 /*
                  * The LSB of head.next can't change from under us
                  * because we hold the mm_all_locks_mutex.
                  */
-                spin_lock_nest_lock(&anon_vma->lock, &mm->mmap_sem);
+                spin_lock_nest_lock(&anon_vma->root->lock, &mm->mmap_sem);
                 /*
                  * We can safely modify head.next after taking the
-                 * anon_vma->lock. If some other vma in this mm shares
+                 * anon_vma->root->lock. If some other vma in this mm shares
                  * the same anon_vma we won't take it again.
                  *
                  * No need of atomic instructions here, head.next
                  * can't change from under us thanks to the
-                 * anon_vma->lock.
+                 * anon_vma->root->lock.
                  */
                 if (__test_and_set_bit(0, (unsigned long *)
-                                       &anon_vma->head.next))
+                                       &anon_vma->root->head.next))
                         BUG();
         }
 }
@@ -2573,7 +2599,7 @@ out_unlock:
 
 static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
 {
-        if (test_bit(0, (unsigned long *) &anon_vma->head.next)) {
+        if (test_bit(0, (unsigned long *) &anon_vma->root->head.next)) {
                 /*
                  * The LSB of head.next can't change to 0 from under
                  * us because we hold the mm_all_locks_mutex.
@@ -2584,12 +2610,12 @@ static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
                  *
                  * No need of atomic instructions here, head.next
                  * can't change from under us until we release the
-                 * anon_vma->lock.
+                 * anon_vma->root->lock.
                  */
                 if (!__test_and_clear_bit(0, (unsigned long *)
-                                          &anon_vma->head.next))
+                                          &anon_vma->root->head.next))
                         BUG();
-                spin_unlock(&anon_vma->lock);
+                anon_vma_unlock(anon_vma);
         }
 }