1 files changed, 93 insertions, 58 deletions

diff --git a/mm/mmap.c b/mm/mmap.c
index ee2298936fe6..f1b4448626bf 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -265,7 +265,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
          * segment grow beyond its set limit the in case where the limit is
          * not page aligned -Ram Gupta
          */
-        rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur;
+        rlim = rlimit(RLIMIT_DATA);
         if (rlim < RLIM_INFINITY && (brk - mm->start_brk) +
                         (mm->end_data - mm->start_data) > rlim)
                 goto out;
@@ -437,7 +437,6 @@ __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
 {
         __vma_link_list(mm, vma, prev, rb_parent);
         __vma_link_rb(mm, vma, rb_link, rb_parent);
-        __anon_vma_link(vma);
 }
 
 static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
@@ -499,7 +498,7 @@ __vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
  * are necessary.  The "insert" vma (if any) is to be inserted
  * before we drop the necessary locks.
  */
-void vma_adjust(struct vm_area_struct *vma, unsigned long start,
+int vma_adjust(struct vm_area_struct *vma, unsigned long start,
         unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
 {
         struct mm_struct *mm = vma->vm_mm;
@@ -542,6 +541,26 @@ again:			remove_next = 1 + (end > next->vm_end);
                 }
         }
 
+        /*
+         * When changing only vma->vm_end, we don't really need anon_vma lock.
+         */
+        if (vma->anon_vma && (insert || importer || start != vma->vm_start))
+                anon_vma = vma->anon_vma;
+        if (anon_vma) {
+                /*
+                 * Easily overlooked: when mprotect shifts the boundary,
+                 * make sure the expanding vma has anon_vma set if the
+                 * shrinking vma had, to cover any anon pages imported.
+                 */
+                if (importer && !importer->anon_vma) {
+                        /* Block reverse map lookups until things are set up. */
+                        if (anon_vma_clone(importer, vma)) {
+                                return -ENOMEM;
+                        }
+                        importer->anon_vma = anon_vma;
+                }
+        }
+
         if (file) {
                 mapping = file->f_mapping;
                 if (!(vma->vm_flags & VM_NONLINEAR))
@@ -567,25 +586,6 @@ again:			remove_next = 1 + (end > next->vm_end);
                 }
         }
 
-        /*
-         * When changing only vma->vm_end, we don't really need
-         * anon_vma lock.
-         */
-        if (vma->anon_vma && (insert || importer || start != vma->vm_start))
-                anon_vma = vma->anon_vma;
-        if (anon_vma) {
-                spin_lock(&anon_vma->lock);
-                /*
-                 * Easily overlooked: when mprotect shifts the boundary,
-                 * make sure the expanding vma has anon_vma set if the
-                 * shrinking vma had, to cover any anon pages imported.
-                 */
-                if (importer && !importer->anon_vma) {
-                        importer->anon_vma = anon_vma;
-                        __anon_vma_link(importer);
-                }
-        }
-
         if (root) {
                 flush_dcache_mmap_lock(mapping);
                 vma_prio_tree_remove(vma, root);
@@ -616,8 +616,6 @@ again:			remove_next = 1 + (end > next->vm_end);
                 __vma_unlink(mm, next, vma);
                 if (file)
                         __remove_shared_vm_struct(next, file, mapping);
-                if (next->anon_vma)
-                        __anon_vma_merge(vma, next);
         } else if (insert) {
                 /*
                  * split_vma has split insert from vma, and needs
@@ -627,8 +625,6 @@ again:			remove_next = 1 + (end > next->vm_end);
                 __insert_vm_struct(mm, insert);
         }
 
-        if (anon_vma)
-                spin_unlock(&anon_vma->lock);
         if (mapping)
                 spin_unlock(&mapping->i_mmap_lock);
 
@@ -638,6 +634,8 @@ again:			remove_next = 1 + (end > next->vm_end);
                         if (next->vm_flags & VM_EXECUTABLE)
                                 removed_exe_file_vma(mm);
                 }
+                if (next->anon_vma)
+                        anon_vma_merge(vma, next);
                 mm->map_count--;
                 mpol_put(vma_policy(next));
                 kmem_cache_free(vm_area_cachep, next);
@@ -653,6 +651,8 @@ again:			remove_next = 1 + (end > next->vm_end);
         }
 
         validate_mm(mm);
+
+        return 0;
 }
 
 /*
@@ -759,6 +759,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
 {
         pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
         struct vm_area_struct *area, *next;
+        int err;
 
         /*
          * We later require that vma->vm_flags == vm_flags,
@@ -792,11 +793,13 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
                                 is_mergeable_anon_vma(prev->anon_vma,
                                                       next->anon_vma)) {
                                                         /* cases 1, 6 */
-                        vma_adjust(prev, prev->vm_start,
+                        err = vma_adjust(prev, prev->vm_start,
                                 next->vm_end, prev->vm_pgoff, NULL);
                 } else                                  /* cases 2, 5, 7 */
-                        vma_adjust(prev, prev->vm_start,
+                        err = vma_adjust(prev, prev->vm_start,
                                 end, prev->vm_pgoff, NULL);
+                if (err)
+                        return NULL;
                 return prev;
         }
 
@@ -808,11 +811,13 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
                         can_vma_merge_before(next, vm_flags,
                                         anon_vma, file, pgoff+pglen)) {
                 if (prev && addr < prev->vm_end)        /* case 4 */
-                        vma_adjust(prev, prev->vm_start,
+                        err = vma_adjust(prev, prev->vm_start,
                                 addr, prev->vm_pgoff, NULL);
                 else                                    /* cases 3, 8 */
-                        vma_adjust(area, addr, next->vm_end,
+                        err = vma_adjust(area, addr, next->vm_end,
                                 next->vm_pgoff - pglen, NULL);
+                if (err)
+                        return NULL;
                 return area;
         }
 
@@ -967,7 +972,7 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
                 unsigned long locked, lock_limit;
                 locked = len >> PAGE_SHIFT;
                 locked += mm->locked_vm;
-                lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
+                lock_limit = rlimit(RLIMIT_MEMLOCK);
                 lock_limit >>= PAGE_SHIFT;
                 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
                         return -EAGAIN;
@@ -1205,6 +1210,7 @@ munmap_back:
         vma->vm_flags = vm_flags;
         vma->vm_page_prot = vm_get_page_prot(vm_flags);
         vma->vm_pgoff = pgoff;
+        INIT_LIST_HEAD(&vma->anon_vma_chain);
 
         if (file) {
                 error = -EINVAL;
@@ -1265,13 +1271,8 @@ out:
         mm->total_vm += len >> PAGE_SHIFT;
         vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
         if (vm_flags & VM_LOCKED) {
-                /*
-                 * makes pages present; downgrades, drops, reacquires mmap_sem
-                 */
-                long nr_pages = mlock_vma_pages_range(vma, addr, addr + len);
-                if (nr_pages < 0)
-                        return nr_pages;        /* vma gone! */
-                mm->locked_vm += (len >> PAGE_SHIFT) - nr_pages;
+                if (!mlock_vma_pages_range(vma, addr, addr + len))
+                        mm->locked_vm += (len >> PAGE_SHIFT);
         } else if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK))
                 make_pages_present(addr, addr + len);
         return addr;
@@ -1599,7 +1600,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, uns
                 return -ENOMEM;
 
         /* Stack limit test */
-        if (size > rlim[RLIMIT_STACK].rlim_cur)
+        if (size > ACCESS_ONCE(rlim[RLIMIT_STACK].rlim_cur))
                 return -ENOMEM;
 
         /* mlock limit tests */
@@ -1607,7 +1608,8 @@ static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, uns
                 unsigned long locked;
                 unsigned long limit;
                 locked = mm->locked_vm + grow;
-                limit = rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
+                limit = ACCESS_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur);
+                limit >>= PAGE_SHIFT;
                 if (locked > limit && !capable(CAP_IPC_LOCK))
                         return -ENOMEM;
         }
@@ -1754,8 +1756,7 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
         if (!prev || expand_stack(prev, addr))
                 return NULL;
         if (prev->vm_flags & VM_LOCKED) {
-                if (mlock_vma_pages_range(prev, addr, prev->vm_end) < 0)
-                        return NULL;    /* vma gone! */
+                mlock_vma_pages_range(prev, addr, prev->vm_end);
         }
         return prev;
 }
@@ -1783,8 +1784,7 @@ find_extend_vma(struct mm_struct * mm, unsigned long addr)
         if (expand_stack(vma, addr))
                 return NULL;
         if (vma->vm_flags & VM_LOCKED) {
-                if (mlock_vma_pages_range(vma, addr, start) < 0)
-                        return NULL;    /* vma gone! */
+                mlock_vma_pages_range(vma, addr, start);
         }
         return vma;
 }
@@ -1871,6 +1871,7 @@ static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
 {
         struct mempolicy *pol;
         struct vm_area_struct *new;
+        int err = -ENOMEM;
 
         if (is_vm_hugetlb_page(vma) && (addr &
                                         ~(huge_page_mask(hstate_vma(vma)))))
@@ -1878,11 +1879,13 @@ static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
 
         new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
         if (!new)
-                return -ENOMEM;
+                goto out_err;
 
         /* most fields are the same, copy all, and then fixup */
         *new = *vma;
 
+        INIT_LIST_HEAD(&new->anon_vma_chain);
+
         if (new_below)
                 new->vm_end = addr;
         else {
@@ -1892,11 +1895,14 @@ static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
 
         pol = mpol_dup(vma_policy(vma));
         if (IS_ERR(pol)) {
-                kmem_cache_free(vm_area_cachep, new);
-                return PTR_ERR(pol);
+                err = PTR_ERR(pol);
+                goto out_free_vma;
         }
         vma_set_policy(new, pol);
 
+        if (anon_vma_clone(new, vma))
+                goto out_free_mpol;
+
         if (new->vm_file) {
                 get_file(new->vm_file);
                 if (vma->vm_flags & VM_EXECUTABLE)
@@ -1907,12 +1913,28 @@ static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
                 new->vm_ops->open(new);
 
         if (new_below)
-                vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
+                err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
                         ((addr - new->vm_start) >> PAGE_SHIFT), new);
         else
-                vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
+                err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
 
-        return 0;
+        /* Success. */
+        if (!err)
+                return 0;
+
+        /* Clean everything up if vma_adjust failed. */
+        new->vm_ops->close(new);
+        if (new->vm_file) {
+                if (vma->vm_flags & VM_EXECUTABLE)
+                        removed_exe_file_vma(mm);
+                fput(new->vm_file);
+        }
+ out_free_mpol:
+        mpol_put(pol);
+ out_free_vma:
+        kmem_cache_free(vm_area_cachep, new);
+ out_err:
+        return err;
 }
 
 /*
@@ -2074,7 +2096,7 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
                 unsigned long locked, lock_limit;
                 locked = len >> PAGE_SHIFT;
                 locked += mm->locked_vm;
-                lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
+                lock_limit = rlimit(RLIMIT_MEMLOCK);
                 lock_limit >>= PAGE_SHIFT;
                 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
                         return -EAGAIN;
@@ -2122,6 +2144,7 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
                 return -ENOMEM;
         }
 
+        INIT_LIST_HEAD(&vma->anon_vma_chain);
         vma->vm_mm = mm;
         vma->vm_start = addr;
         vma->vm_end = addr + len;
@@ -2258,10 +2281,11 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
                 if (new_vma) {
                         *new_vma = *vma;
                         pol = mpol_dup(vma_policy(vma));
-                        if (IS_ERR(pol)) {
-                                kmem_cache_free(vm_area_cachep, new_vma);
-                                return NULL;
-                        }
+                        if (IS_ERR(pol))
+                                goto out_free_vma;
+                        INIT_LIST_HEAD(&new_vma->anon_vma_chain);
+                        if (anon_vma_clone(new_vma, vma))
+                                goto out_free_mempol;
                         vma_set_policy(new_vma, pol);
                         new_vma->vm_start = addr;
                         new_vma->vm_end = addr + len;
@@ -2277,6 +2301,12 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
                 }
         }
         return new_vma;
+
+ out_free_mempol:
+        mpol_put(pol);
+ out_free_vma:
+        kmem_cache_free(vm_area_cachep, new_vma);
+        return NULL;
 }
 
 /*
@@ -2288,7 +2318,7 @@ int may_expand_vm(struct mm_struct *mm, unsigned long npages)
         unsigned long cur = mm->total_vm;       /* pages */
         unsigned long lim;
 
-        lim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
+        lim = rlimit(RLIMIT_AS) >> PAGE_SHIFT;
 
         if (cur + npages > lim)
                 return 0;
@@ -2354,6 +2384,7 @@ int install_special_mapping(struct mm_struct *mm,
         if (unlikely(vma == NULL))
                 return -ENOMEM;
 
+        INIT_LIST_HEAD(&vma->anon_vma_chain);
         vma->vm_mm = mm;
         vma->vm_start = addr;
         vma->vm_end = addr + len;
@@ -2454,6 +2485,7 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
 int mm_take_all_locks(struct mm_struct *mm)
 {
         struct vm_area_struct *vma;
+        struct anon_vma_chain *avc;
         int ret = -EINTR;
 
         BUG_ON(down_read_trylock(&mm->mmap_sem));
@@ -2471,7 +2503,8 @@ int mm_take_all_locks(struct mm_struct *mm)
                 if (signal_pending(current))
                         goto out_unlock;
                 if (vma->anon_vma)
-                        vm_lock_anon_vma(mm, vma->anon_vma);
+                        list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
+                                vm_lock_anon_vma(mm, avc->anon_vma);
         }
 
         ret = 0;
@@ -2526,13 +2559,15 @@ static void vm_unlock_mapping(struct address_space *mapping)
 void mm_drop_all_locks(struct mm_struct *mm)
 {
         struct vm_area_struct *vma;
+        struct anon_vma_chain *avc;
 
         BUG_ON(down_read_trylock(&mm->mmap_sem));
         BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
 
         for (vma = mm->mmap; vma; vma = vma->vm_next) {
                 if (vma->anon_vma)
-                        vm_unlock_anon_vma(vma->anon_vma);
+                        list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
+                                vm_unlock_anon_vma(avc->anon_vma);
                 if (vma->vm_file && vma->vm_file->f_mapping)
                         vm_unlock_mapping(vma->vm_file->f_mapping);
         }