mmap: avoid merging cloned VMAs

Avoid merging a VMA with another VMA which is cloned from the parent process.

The cloned VMA shares the anon_vma lock with the parent process's VMA.  If
we do the merge, more vmas (even the new range is only for current
process) use the perent process's anon_vma lock.  This introduces
scalability issues.  find_mergeable_anon_vma() already considers this
case.

Signed-off-by: Shaohua Li <shaohua.li@intel.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Andi Kleen <andi@firstfloor.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 13 insertions, 5 deletions

diff --git a/mm/mmap.c b/mm/mmap.c
index 15b1fae57efe..4fb5464f7707 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -703,9 +703,17 @@ static inline int is_mergeable_vma(struct vm_area_struct *vma,
 }
 
 static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
-                                        struct anon_vma *anon_vma2)
+                                        struct anon_vma *anon_vma2,
+                                        struct vm_area_struct *vma)
 {
-        return !anon_vma1 || !anon_vma2 || (anon_vma1 == anon_vma2);
+        /*
+         * The list_is_singular() test is to avoid merging VMA cloned from
+         * parents. This can improve scalability caused by anon_vma lock.
+         */
+        if ((!anon_vma1 || !anon_vma2) && (!vma ||
+                list_is_singular(&vma->anon_vma_chain)))
+                return 1;
+        return anon_vma1 == anon_vma2;
 }
 
 /*
@@ -724,7 +732,7 @@ can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
         struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
 {
         if (is_mergeable_vma(vma, file, vm_flags) &&
-            is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
+            is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
                 if (vma->vm_pgoff == vm_pgoff)
                         return 1;
         }
@@ -743,7 +751,7 @@ can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
         struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
 {
         if (is_mergeable_vma(vma, file, vm_flags) &&
-            is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
+            is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
                 pgoff_t vm_pglen;
                 vm_pglen = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
                 if (vma->vm_pgoff + vm_pglen == vm_pgoff)
@@ -821,7 +829,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
                                 can_vma_merge_before(next, vm_flags,
                                         anon_vma, file, pgoff+pglen) &&
                                 is_mergeable_anon_vma(prev->anon_vma,
-                                                      next->anon_vma)) {
+                                                      next->anon_vma, NULL)) {
                                                         /* cases 1, 6 */
                         err = vma_adjust(prev, prev->vm_start,
                                 next->vm_end, prev->vm_pgoff, NULL);