1 files changed, 268 insertions, 130 deletions

diff --git a/mm/mmap.c b/mm/mmap.c
index 73f5e4b64010..456ec6f27889 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -20,7 +20,6 @@
 #include <linux/fs.h>
 #include <linux/personality.h>
 #include <linux/security.h>
-#include <linux/ima.h>
 #include <linux/hugetlb.h>
 #include <linux/profile.h>
 #include <linux/module.h>
@@ -266,7 +265,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
          * segment grow beyond its set limit the in case where the limit is
          * not page aligned -Ram Gupta
          */
-        rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur;
+        rlim = rlimit(RLIMIT_DATA);
         if (rlim < RLIM_INFINITY && (brk - mm->start_brk) +
                         (mm->end_data - mm->start_data) > rlim)
                 goto out;
@@ -438,7 +437,6 @@ __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
 {
         __vma_link_list(mm, vma, prev, rb_parent);
         __vma_link_rb(mm, vma, rb_link, rb_parent);
-        __anon_vma_link(vma);
 }
 
 static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
@@ -500,7 +498,7 @@ __vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
  * are necessary.  The "insert" vma (if any) is to be inserted
  * before we drop the necessary locks.
  */
-void vma_adjust(struct vm_area_struct *vma, unsigned long start,
+int vma_adjust(struct vm_area_struct *vma, unsigned long start,
         unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
 {
         struct mm_struct *mm = vma->vm_mm;
@@ -509,11 +507,12 @@ void vma_adjust(struct vm_area_struct *vma, unsigned long start,
         struct address_space *mapping = NULL;
         struct prio_tree_root *root = NULL;
         struct file *file = vma->vm_file;
-        struct anon_vma *anon_vma = NULL;
         long adjust_next = 0;
         int remove_next = 0;
 
         if (next && !insert) {
+                struct vm_area_struct *exporter = NULL;
+
                 if (end >= next->vm_end) {
                         /*
                          * vma expands, overlapping all the next, and
@@ -521,7 +520,7 @@ void vma_adjust(struct vm_area_struct *vma, unsigned long start,
                          */
 again:                  remove_next = 1 + (end > next->vm_end);
                         end = next->vm_end;
-                        anon_vma = next->anon_vma;
+                        exporter = next;
                         importer = vma;
                 } else if (end > next->vm_start) {
                         /*
@@ -529,7 +528,7 @@ again:			remove_next = 1 + (end > next->vm_end);
                          * mprotect case 5 shifting the boundary up.
                          */
                         adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
-                        anon_vma = next->anon_vma;
+                        exporter = next;
                         importer = vma;
                 } else if (end < vma->vm_end) {
                         /*
@@ -538,9 +537,20 @@ again:			remove_next = 1 + (end > next->vm_end);
                          * mprotect case 4 shifting the boundary down.
                          */
                         adjust_next = - ((vma->vm_end - end) >> PAGE_SHIFT);
-                        anon_vma = next->anon_vma;
+                        exporter = vma;
                         importer = next;
                 }
+
+                /*
+                 * Easily overlooked: when mprotect shifts the boundary,
+                 * make sure the expanding vma has anon_vma set if the
+                 * shrinking vma had, to cover any anon pages imported.
+                 */
+                if (exporter && exporter->anon_vma && !importer->anon_vma) {
+                        if (anon_vma_clone(importer, exporter))
+                                return -ENOMEM;
+                        importer->anon_vma = exporter->anon_vma;
+                }
         }
 
         if (file) {
@@ -568,25 +578,6 @@ again:			remove_next = 1 + (end > next->vm_end);
                 }
         }
 
-        /*
-         * When changing only vma->vm_end, we don't really need
-         * anon_vma lock.
-         */
-        if (vma->anon_vma && (insert || importer || start != vma->vm_start))
-                anon_vma = vma->anon_vma;
-        if (anon_vma) {
-                spin_lock(&anon_vma->lock);
-                /*
-                 * Easily overlooked: when mprotect shifts the boundary,
-                 * make sure the expanding vma has anon_vma set if the
-                 * shrinking vma had, to cover any anon pages imported.
-                 */
-                if (importer && !importer->anon_vma) {
-                        importer->anon_vma = anon_vma;
-                        __anon_vma_link(importer);
-                }
-        }
-
         if (root) {
                 flush_dcache_mmap_lock(mapping);
                 vma_prio_tree_remove(vma, root);
@@ -617,8 +608,6 @@ again:			remove_next = 1 + (end > next->vm_end);
                 __vma_unlink(mm, next, vma);
                 if (file)
                         __remove_shared_vm_struct(next, file, mapping);
-                if (next->anon_vma)
-                        __anon_vma_merge(vma, next);
         } else if (insert) {
                 /*
                  * split_vma has split insert from vma, and needs
@@ -628,8 +617,6 @@ again:			remove_next = 1 + (end > next->vm_end);
                 __insert_vm_struct(mm, insert);
         }
 
-        if (anon_vma)
-                spin_unlock(&anon_vma->lock);
         if (mapping)
                 spin_unlock(&mapping->i_mmap_lock);
 
@@ -639,6 +626,8 @@ again:			remove_next = 1 + (end > next->vm_end);
                         if (next->vm_flags & VM_EXECUTABLE)
                                 removed_exe_file_vma(mm);
                 }
+                if (next->anon_vma)
+                        anon_vma_merge(vma, next);
                 mm->map_count--;
                 mpol_put(vma_policy(next));
                 kmem_cache_free(vm_area_cachep, next);
@@ -654,6 +643,8 @@ again:			remove_next = 1 + (end > next->vm_end);
         }
 
         validate_mm(mm);
+
+        return 0;
 }
 
 /*
@@ -760,6 +751,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
 {
         pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
         struct vm_area_struct *area, *next;
+        int err;
 
         /*
          * We later require that vma->vm_flags == vm_flags,
@@ -793,11 +785,13 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
                                 is_mergeable_anon_vma(prev->anon_vma,
                                                       next->anon_vma)) {
                                                         /* cases 1, 6 */
-                        vma_adjust(prev, prev->vm_start,
+                        err = vma_adjust(prev, prev->vm_start,
                                 next->vm_end, prev->vm_pgoff, NULL);
                 } else                                  /* cases 2, 5, 7 */
-                        vma_adjust(prev, prev->vm_start,
+                        err = vma_adjust(prev, prev->vm_start,
                                 end, prev->vm_pgoff, NULL);
+                if (err)
+                        return NULL;
                 return prev;
         }
 
@@ -809,11 +803,13 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
                         can_vma_merge_before(next, vm_flags,
                                         anon_vma, file, pgoff+pglen)) {
                 if (prev && addr < prev->vm_end)        /* case 4 */
-                        vma_adjust(prev, prev->vm_start,
+                        err = vma_adjust(prev, prev->vm_start,
                                 addr, prev->vm_pgoff, NULL);
                 else                                    /* cases 3, 8 */
-                        vma_adjust(area, addr, next->vm_end,
+                        err = vma_adjust(area, addr, next->vm_end,
                                 next->vm_pgoff - pglen, NULL);
+                if (err)
+                        return NULL;
                 return area;
         }
 
@@ -821,6 +817,61 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
 }
 
 /*
+ * Rough compatbility check to quickly see if it's even worth looking
+ * at sharing an anon_vma.
+ *
+ * They need to have the same vm_file, and the flags can only differ
+ * in things that mprotect may change.
+ *
+ * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
+ * we can merge the two vma's. For example, we refuse to merge a vma if
+ * there is a vm_ops->close() function, because that indicates that the
+ * driver is doing some kind of reference counting. But that doesn't
+ * really matter for the anon_vma sharing case.
+ */
+static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
+{
+        return a->vm_end == b->vm_start &&
+                mpol_equal(vma_policy(a), vma_policy(b)) &&
+                a->vm_file == b->vm_file &&
+                !((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC)) &&
+                b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
+}
+
+/*
+ * Do some basic sanity checking to see if we can re-use the anon_vma
+ * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
+ * the same as 'old', the other will be the new one that is trying
+ * to share the anon_vma.
+ *
+ * NOTE! This runs with mm_sem held for reading, so it is possible that
+ * the anon_vma of 'old' is concurrently in the process of being set up
+ * by another page fault trying to merge _that_. But that's ok: if it
+ * is being set up, that automatically means that it will be a singleton
+ * acceptable for merging, so we can do all of this optimistically. But
+ * we do that ACCESS_ONCE() to make sure that we never re-load the pointer.
+ *
+ * IOW: that the "list_is_singular()" test on the anon_vma_chain only
+ * matters for the 'stable anon_vma' case (ie the thing we want to avoid
+ * is to return an anon_vma that is "complex" due to having gone through
+ * a fork).
+ *
+ * We also make sure that the two vma's are compatible (adjacent,
+ * and with the same memory policies). That's all stable, even with just
+ * a read lock on the mm_sem.
+ */
+static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
+{
+        if (anon_vma_compatible(a, b)) {
+                struct anon_vma *anon_vma = ACCESS_ONCE(old->anon_vma);
+
+                if (anon_vma && list_is_singular(&old->anon_vma_chain))
+                        return anon_vma;
+        }
+        return NULL;
+}
+
+/*
  * find_mergeable_anon_vma is used by anon_vma_prepare, to check
  * neighbouring vmas for a suitable anon_vma, before it goes off
  * to allocate a new anon_vma.  It checks because a repetitive
@@ -830,28 +881,16 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
  */
 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
 {
+        struct anon_vma *anon_vma;
         struct vm_area_struct *near;
-        unsigned long vm_flags;
 
         near = vma->vm_next;
         if (!near)
                 goto try_prev;
 
-        /*
-         * Since only mprotect tries to remerge vmas, match flags
-         * which might be mprotected into each other later on.
-         * Neither mlock nor madvise tries to remerge at present,
-         * so leave their flags as obstructing a merge.
-         */
-        vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);
-        vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);
-
-        if (near->anon_vma && vma->vm_end == near->vm_start &&
-                        mpol_equal(vma_policy(vma), vma_policy(near)) &&
-                        can_vma_merge_before(near, vm_flags,
-                                NULL, vma->vm_file, vma->vm_pgoff +
-                                ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT)))
-                return near->anon_vma;
+        anon_vma = reusable_anon_vma(near, vma, near);
+        if (anon_vma)
+                return anon_vma;
 try_prev:
         /*
          * It is potentially slow to have to call find_vma_prev here.
@@ -864,14 +903,9 @@ try_prev:
         if (!near)
                 goto none;
 
-        vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);
-        vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);
-
-        if (near->anon_vma && near->vm_end == vma->vm_start &&
-                        mpol_equal(vma_policy(near), vma_policy(vma)) &&
-                        can_vma_merge_after(near, vm_flags,
-                                NULL, vma->vm_file, vma->vm_pgoff))
-                return near->anon_vma;
+        anon_vma = reusable_anon_vma(near, near, vma);
+        if (anon_vma)
+                return anon_vma;
 none:
         /*
          * There's no absolute need to look only at touching neighbours:
@@ -932,13 +966,9 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
         if (!(flags & MAP_FIXED))
                 addr = round_hint_to_min(addr);
 
-        error = arch_mmap_check(addr, len, flags);
-        if (error)
-                return error;
-
         /* Careful about overflows.. */
         len = PAGE_ALIGN(len);
-        if (!len || len > TASK_SIZE)
+        if (!len)
                 return -ENOMEM;
 
         /* offset overflow? */
@@ -949,24 +979,6 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
         if (mm->map_count > sysctl_max_map_count)
                 return -ENOMEM;
 
-        if (flags & MAP_HUGETLB) {
-                struct user_struct *user = NULL;
-                if (file)
-                        return -EINVAL;
-
-                /*
-                 * VM_NORESERVE is used because the reservations will be
-                 * taken when vm_ops->mmap() is called
-                 * A dummy user value is used because we are not locking
-                 * memory so no accounting is necessary
-                 */
-                len = ALIGN(len, huge_page_size(&default_hstate));
-                file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, VM_NORESERVE,
-                                                &user, HUGETLB_ANONHUGE_INODE);
-                if (IS_ERR(file))
-                        return PTR_ERR(file);
-        }
-
         /* Obtain the address to map to. we verify (or select) it and ensure
          * that it represents a valid section of the address space.
          */
@@ -990,7 +1002,7 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
                 unsigned long locked, lock_limit;
                 locked = len >> PAGE_SHIFT;
                 locked += mm->locked_vm;
-                lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
+                lock_limit = rlimit(RLIMIT_MEMLOCK);
                 lock_limit >>= PAGE_SHIFT;
                 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
                         return -EAGAIN;
@@ -1061,14 +1073,75 @@ unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
         error = security_file_mmap(file, reqprot, prot, flags, addr, 0);
         if (error)
                 return error;
-        error = ima_file_mmap(file, prot);
-        if (error)
-                return error;
 
         return mmap_region(file, addr, len, flags, vm_flags, pgoff);
 }
 EXPORT_SYMBOL(do_mmap_pgoff);
 
+SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
+                unsigned long, prot, unsigned long, flags,
+                unsigned long, fd, unsigned long, pgoff)
+{
+        struct file *file = NULL;
+        unsigned long retval = -EBADF;
+
+        if (!(flags & MAP_ANONYMOUS)) {
+                if (unlikely(flags & MAP_HUGETLB))
+                        return -EINVAL;
+                file = fget(fd);
+                if (!file)
+                        goto out;
+        } else if (flags & MAP_HUGETLB) {
+                struct user_struct *user = NULL;
+                /*
+                 * VM_NORESERVE is used because the reservations will be
+                 * taken when vm_ops->mmap() is called
+                 * A dummy user value is used because we are not locking
+                 * memory so no accounting is necessary
+                 */
+                len = ALIGN(len, huge_page_size(&default_hstate));
+                file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, VM_NORESERVE,
+                                                &user, HUGETLB_ANONHUGE_INODE);
+                if (IS_ERR(file))
+                        return PTR_ERR(file);
+        }
+
+        flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
+
+        down_write(&current->mm->mmap_sem);
+        retval = do_mmap_pgoff(file, addr, len, prot, flags, pgoff);
+        up_write(&current->mm->mmap_sem);
+
+        if (file)
+                fput(file);
+out:
+        return retval;
+}
+
+#ifdef __ARCH_WANT_SYS_OLD_MMAP
+struct mmap_arg_struct {
+        unsigned long addr;
+        unsigned long len;
+        unsigned long prot;
+        unsigned long flags;
+        unsigned long fd;
+        unsigned long offset;
+};
+
+SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
+{
+        struct mmap_arg_struct a;
+
+        if (copy_from_user(&a, arg, sizeof(a)))
+                return -EFAULT;
+        if (a.offset & ~PAGE_MASK)
+                return -EINVAL;
+
+        return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
+                              a.offset >> PAGE_SHIFT);
+}
+#endif /* __ARCH_WANT_SYS_OLD_MMAP */
+
 /*
  * Some shared mappigns will want the pages marked read-only
  * to track write events. If so, we'll downgrade vm_page_prot
@@ -1191,6 +1264,7 @@ munmap_back:
         vma->vm_flags = vm_flags;
         vma->vm_page_prot = vm_get_page_prot(vm_flags);
         vma->vm_pgoff = pgoff;
+        INIT_LIST_HEAD(&vma->anon_vma_chain);
 
         if (file) {
                 error = -EINVAL;
@@ -1224,8 +1298,20 @@ munmap_back:
                         goto free_vma;
         }
 
-        if (vma_wants_writenotify(vma))
+        if (vma_wants_writenotify(vma)) {
+                pgprot_t pprot = vma->vm_page_prot;
+
+                /* Can vma->vm_page_prot have changed??
+                 *
+                 * Answer: Yes, drivers may have changed it in their
+                 *         f_op->mmap method.
+                 *
+                 * Ensures that vmas marked as uncached stay that way.
+                 */
                 vma->vm_page_prot = vm_get_page_prot(vm_flags & ~VM_SHARED);
+                if (pgprot_val(pprot) == pgprot_val(pgprot_noncached(pprot)))
+                        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+        }
 
         vma_link(mm, vma, prev, rb_link, rb_parent);
         file = vma->vm_file;
@@ -1239,13 +1325,8 @@ out:
         mm->total_vm += len >> PAGE_SHIFT;
         vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
         if (vm_flags & VM_LOCKED) {
-                /*
-                 * makes pages present; downgrades, drops, reacquires mmap_sem
-                 */
-                long nr_pages = mlock_vma_pages_range(vma, addr, addr + len);
-                if (nr_pages < 0)
-                        return nr_pages;        /* vma gone! */
-                mm->locked_vm += (len >> PAGE_SHIFT) - nr_pages;
+                if (!mlock_vma_pages_range(vma, addr, addr + len))
+                        mm->locked_vm += (len >> PAGE_SHIFT);
         } else if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK))
                 make_pages_present(addr, addr + len);
         return addr;
@@ -1459,6 +1540,14 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
         unsigned long (*get_area)(struct file *, unsigned long,
                                   unsigned long, unsigned long, unsigned long);
 
+        unsigned long error = arch_mmap_check(addr, len, flags);
+        if (error)
+                return error;
+
+        /* Careful about overflows.. */
+        if (len > TASK_SIZE)
+                return -ENOMEM;
+
         get_area = current->mm->get_unmapped_area;
         if (file && file->f_op && file->f_op->get_unmapped_area)
                 get_area = file->f_op->get_unmapped_area;
@@ -1565,7 +1654,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, uns
                 return -ENOMEM;
 
         /* Stack limit test */
-        if (size > rlim[RLIMIT_STACK].rlim_cur)
+        if (size > ACCESS_ONCE(rlim[RLIMIT_STACK].rlim_cur))
                 return -ENOMEM;
 
         /* mlock limit tests */
@@ -1573,7 +1662,8 @@ static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, uns
                 unsigned long locked;
                 unsigned long limit;
                 locked = mm->locked_vm + grow;
-                limit = rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
+                limit = ACCESS_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur);
+                limit >>= PAGE_SHIFT;
                 if (locked > limit && !capable(CAP_IPC_LOCK))
                         return -ENOMEM;
         }
@@ -1720,8 +1810,7 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
         if (!prev || expand_stack(prev, addr))
                 return NULL;
         if (prev->vm_flags & VM_LOCKED) {
-                if (mlock_vma_pages_range(prev, addr, prev->vm_end) < 0)
-                        return NULL;    /* vma gone! */
+                mlock_vma_pages_range(prev, addr, prev->vm_end);
         }
         return prev;
 }
@@ -1749,8 +1838,7 @@ find_extend_vma(struct mm_struct * mm, unsigned long addr)
         if (expand_stack(vma, addr))
                 return NULL;
         if (vma->vm_flags & VM_LOCKED) {
-                if (mlock_vma_pages_range(vma, addr, start) < 0)
-                        return NULL;    /* vma gone! */
+                mlock_vma_pages_range(vma, addr, start);
         }
         return vma;
 }
@@ -1829,29 +1917,29 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
 }
 
 /*
- * Split a vma into two pieces at address 'addr', a new vma is allocated
- * either for the first part or the tail.
+ * __split_vma() bypasses sysctl_max_map_count checking.  We use this on the
+ * munmap path where it doesn't make sense to fail.
  */
-int split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
+static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
               unsigned long addr, int new_below)
 {
         struct mempolicy *pol;
         struct vm_area_struct *new;
+        int err = -ENOMEM;
 
         if (is_vm_hugetlb_page(vma) && (addr &
                                         ~(huge_page_mask(hstate_vma(vma)))))
                 return -EINVAL;
 
-        if (mm->map_count >= sysctl_max_map_count)
-                return -ENOMEM;
-
         new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
         if (!new)
-                return -ENOMEM;
+                goto out_err;
 
         /* most fields are the same, copy all, and then fixup */
         *new = *vma;
 
+        INIT_LIST_HEAD(&new->anon_vma_chain);
+
         if (new_below)
                 new->vm_end = addr;
         else {
@@ -1861,11 +1949,14 @@ int split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
 
         pol = mpol_dup(vma_policy(vma));
         if (IS_ERR(pol)) {
-                kmem_cache_free(vm_area_cachep, new);
-                return PTR_ERR(pol);
+                err = PTR_ERR(pol);
+                goto out_free_vma;
         }
         vma_set_policy(new, pol);
 
+        if (anon_vma_clone(new, vma))
+                goto out_free_mpol;
+
         if (new->vm_file) {
                 get_file(new->vm_file);
                 if (vma->vm_flags & VM_EXECUTABLE)
@@ -1876,12 +1967,42 @@ int split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
                 new->vm_ops->open(new);
 
         if (new_below)
-                vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
+                err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
                         ((addr - new->vm_start) >> PAGE_SHIFT), new);
         else
-                vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
+                err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
 
-        return 0;
+        /* Success. */
+        if (!err)
+                return 0;
+
+        /* Clean everything up if vma_adjust failed. */
+        if (new->vm_ops && new->vm_ops->close)
+                new->vm_ops->close(new);
+        if (new->vm_file) {
+                if (vma->vm_flags & VM_EXECUTABLE)
+                        removed_exe_file_vma(mm);
+                fput(new->vm_file);
+        }
+ out_free_mpol:
+        mpol_put(pol);
+ out_free_vma:
+        kmem_cache_free(vm_area_cachep, new);
+ out_err:
+        return err;
+}
+
+/*
+ * Split a vma into two pieces at address 'addr', a new vma is allocated
+ * either for the first part or the tail.
+ */
+int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
+              unsigned long addr, int new_below)
+{
+        if (mm->map_count >= sysctl_max_map_count)
+                return -ENOMEM;
+
+        return __split_vma(mm, vma, addr, new_below);
 }
 
 /* Munmap is split into 2 main parts -- this part which finds
@@ -1919,7 +2040,17 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
          * places tmp vma above, and higher split_vma places tmp vma below.
          */
         if (start > vma->vm_start) {
-                int error = split_vma(mm, vma, start, 0);
+                int error;
+
+                /*
+                 * Make sure that map_count on return from munmap() will
+                 * not exceed its limit; but let map_count go just above
+                 * its limit temporarily, to help free resources as expected.
+                 */
+                if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
+                        return -ENOMEM;
+
+                error = __split_vma(mm, vma, start, 0);
                 if (error)
                         return error;
                 prev = vma;
@@ -1928,7 +2059,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
         /* Does it split the last one? */
         last = find_vma(mm, end);
         if (last && end > last->vm_start) {
-                int error = split_vma(mm, last, end, 1);
+                int error = __split_vma(mm, last, end, 1);
                 if (error)
                         return error;
         }
@@ -2003,20 +2134,14 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
         if (!len)
                 return addr;
 
-        if ((addr + len) > TASK_SIZE || (addr + len) < addr)
-                return -EINVAL;
-
-        if (is_hugepage_only_range(mm, addr, len))
-                return -EINVAL;
-
         error = security_file_mmap(NULL, 0, 0, 0, addr, 1);
         if (error)
                 return error;
 
         flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
 
-        error = arch_mmap_check(addr, len, flags);
-        if (error)
+        error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
+        if (error & ~PAGE_MASK)
                 return error;
 
         /*
@@ -2026,7 +2151,7 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
                 unsigned long locked, lock_limit;
                 locked = len >> PAGE_SHIFT;
                 locked += mm->locked_vm;
-                lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
+                lock_limit = rlimit(RLIMIT_MEMLOCK);
                 lock_limit >>= PAGE_SHIFT;
                 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
                         return -EAGAIN;
@@ -2074,6 +2199,7 @@ unsigned long do_brk(unsigned long addr, unsigned long len)
                 return -ENOMEM;
         }
 
+        INIT_LIST_HEAD(&vma->anon_vma_chain);
         vma->vm_mm = mm;
         vma->vm_start = addr;
         vma->vm_end = addr + len;
@@ -2210,10 +2336,11 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
                 if (new_vma) {
                         *new_vma = *vma;
                         pol = mpol_dup(vma_policy(vma));
-                        if (IS_ERR(pol)) {
-                                kmem_cache_free(vm_area_cachep, new_vma);
-                                return NULL;
-                        }
+                        if (IS_ERR(pol))
+                                goto out_free_vma;
+                        INIT_LIST_HEAD(&new_vma->anon_vma_chain);
+                        if (anon_vma_clone(new_vma, vma))
+                                goto out_free_mempol;
                         vma_set_policy(new_vma, pol);
                         new_vma->vm_start = addr;
                         new_vma->vm_end = addr + len;
@@ -2229,6 +2356,12 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
                 }
         }
         return new_vma;
+
+ out_free_mempol:
+        mpol_put(pol);
+ out_free_vma:
+        kmem_cache_free(vm_area_cachep, new_vma);
+        return NULL;
 }
 
 /*
@@ -2240,7 +2373,7 @@ int may_expand_vm(struct mm_struct *mm, unsigned long npages)
         unsigned long cur = mm->total_vm;       /* pages */
         unsigned long lim;
 
-        lim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
+        lim = rlimit(RLIMIT_AS) >> PAGE_SHIFT;
 
         if (cur + npages > lim)
                 return 0;
@@ -2306,6 +2439,7 @@ int install_special_mapping(struct mm_struct *mm,
         if (unlikely(vma == NULL))
                 return -ENOMEM;
 
+        INIT_LIST_HEAD(&vma->anon_vma_chain);
         vma->vm_mm = mm;
         vma->vm_start = addr;
         vma->vm_end = addr + len;
@@ -2406,6 +2540,7 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
 int mm_take_all_locks(struct mm_struct *mm)
 {
         struct vm_area_struct *vma;
+        struct anon_vma_chain *avc;
         int ret = -EINTR;
 
         BUG_ON(down_read_trylock(&mm->mmap_sem));
@@ -2423,7 +2558,8 @@ int mm_take_all_locks(struct mm_struct *mm)
                 if (signal_pending(current))
                         goto out_unlock;
                 if (vma->anon_vma)
-                        vm_lock_anon_vma(mm, vma->anon_vma);
+                        list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
+                                vm_lock_anon_vma(mm, avc->anon_vma);
         }
 
         ret = 0;
@@ -2478,13 +2614,15 @@ static void vm_unlock_mapping(struct address_space *mapping)
 void mm_drop_all_locks(struct mm_struct *mm)
 {
         struct vm_area_struct *vma;
+        struct anon_vma_chain *avc;
 
         BUG_ON(down_read_trylock(&mm->mmap_sem));
         BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
 
         for (vma = mm->mmap; vma; vma = vma->vm_next) {
                 if (vma->anon_vma)
-                        vm_unlock_anon_vma(vma->anon_vma);
+                        list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
+                                vm_unlock_anon_vma(avc->anon_vma);
                 if (vma->vm_file && vma->vm_file->f_mapping)
                         vm_unlock_mapping(vma->vm_file->f_mapping);
         }