1 files changed, 130 insertions, 26 deletions

diff --git a/mm/rmap.c b/mm/rmap.c
index 5cb47111f79e..be34094e4595 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -62,6 +62,7 @@
 #include "internal.h"
 
 static struct kmem_cache *anon_vma_cachep;
+static struct kmem_cache *anon_vma_chain_cachep;
 
 static inline struct anon_vma *anon_vma_alloc(void)
 {
@@ -73,6 +74,16 @@ void anon_vma_free(struct anon_vma *anon_vma)
         kmem_cache_free(anon_vma_cachep, anon_vma);
 }
 
+static inline struct anon_vma_chain *anon_vma_chain_alloc(void)
+{
+        return kmem_cache_alloc(anon_vma_chain_cachep, GFP_KERNEL);
+}
+
+void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
+{
+        kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
+}
+
 /**
  * anon_vma_prepare - attach an anon_vma to a memory region
  * @vma: the memory region in question
@@ -103,18 +114,23 @@ void anon_vma_free(struct anon_vma *anon_vma)
 int anon_vma_prepare(struct vm_area_struct *vma)
 {
         struct anon_vma *anon_vma = vma->anon_vma;
+        struct anon_vma_chain *avc;
 
         might_sleep();
         if (unlikely(!anon_vma)) {
                 struct mm_struct *mm = vma->vm_mm;
                 struct anon_vma *allocated;
 
+                avc = anon_vma_chain_alloc();
+                if (!avc)
+                        goto out_enomem;
+
                 anon_vma = find_mergeable_anon_vma(vma);
                 allocated = NULL;
                 if (!anon_vma) {
                         anon_vma = anon_vma_alloc();
                         if (unlikely(!anon_vma))
-                                return -ENOMEM;
+                                goto out_enomem_free_avc;
                         allocated = anon_vma;
                 }
                 spin_lock(&anon_vma->lock);
@@ -123,53 +139,113 @@ int anon_vma_prepare(struct vm_area_struct *vma)
                 spin_lock(&mm->page_table_lock);
                 if (likely(!vma->anon_vma)) {
                         vma->anon_vma = anon_vma;
-                        list_add_tail(&vma->anon_vma_node, &anon_vma->head);
+                        avc->anon_vma = anon_vma;
+                        avc->vma = vma;
+                        list_add(&avc->same_vma, &vma->anon_vma_chain);
+                        list_add(&avc->same_anon_vma, &anon_vma->head);
                         allocated = NULL;
                 }
                 spin_unlock(&mm->page_table_lock);
 
                 spin_unlock(&anon_vma->lock);
-                if (unlikely(allocated))
+                if (unlikely(allocated)) {
                         anon_vma_free(allocated);
+                        anon_vma_chain_free(avc);
+                }
         }
         return 0;
+
+ out_enomem_free_avc:
+        anon_vma_chain_free(avc);
+ out_enomem:
+        return -ENOMEM;
 }
 
-void __anon_vma_merge(struct vm_area_struct *vma, struct vm_area_struct *next)
+static void anon_vma_chain_link(struct vm_area_struct *vma,
+                                struct anon_vma_chain *avc,
+                                struct anon_vma *anon_vma)
 {
-        BUG_ON(vma->anon_vma != next->anon_vma);
-        list_del(&next->anon_vma_node);
+        avc->vma = vma;
+        avc->anon_vma = anon_vma;
+        list_add(&avc->same_vma, &vma->anon_vma_chain);
+
+        spin_lock(&anon_vma->lock);
+        list_add_tail(&avc->same_anon_vma, &anon_vma->head);
+        spin_unlock(&anon_vma->lock);
 }
 
-void __anon_vma_link(struct vm_area_struct *vma)
+/*
+ * Attach the anon_vmas from src to dst.
+ * Returns 0 on success, -ENOMEM on failure.
+ */
+int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
 {
-        struct anon_vma *anon_vma = vma->anon_vma;
+        struct anon_vma_chain *avc, *pavc;
+
+        list_for_each_entry(pavc, &src->anon_vma_chain, same_vma) {
+                avc = anon_vma_chain_alloc();
+                if (!avc)
+                        goto enomem_failure;
+                anon_vma_chain_link(dst, avc, pavc->anon_vma);
+        }
+        return 0;
 
-        if (anon_vma)
-                list_add_tail(&vma->anon_vma_node, &anon_vma->head);
+ enomem_failure:
+        unlink_anon_vmas(dst);
+        return -ENOMEM;
 }
 
-void anon_vma_link(struct vm_area_struct *vma)
+/*
+ * Attach vma to its own anon_vma, as well as to the anon_vmas that
+ * the corresponding VMA in the parent process is attached to.
+ * Returns 0 on success, non-zero on failure.
+ */
+int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
 {
-        struct anon_vma *anon_vma = vma->anon_vma;
+        struct anon_vma_chain *avc;
+        struct anon_vma *anon_vma;
 
-        if (anon_vma) {
-                spin_lock(&anon_vma->lock);
-                list_add_tail(&vma->anon_vma_node, &anon_vma->head);
-                spin_unlock(&anon_vma->lock);
-        }
+        /* Don't bother if the parent process has no anon_vma here. */
+        if (!pvma->anon_vma)
+                return 0;
+
+        /*
+         * First, attach the new VMA to the parent VMA's anon_vmas,
+         * so rmap can find non-COWed pages in child processes.
+         */
+        if (anon_vma_clone(vma, pvma))
+                return -ENOMEM;
+
+        /* Then add our own anon_vma. */
+        anon_vma = anon_vma_alloc();
+        if (!anon_vma)
+                goto out_error;
+        avc = anon_vma_chain_alloc();
+        if (!avc)
+                goto out_error_free_anon_vma;
+        anon_vma_chain_link(vma, avc, anon_vma);
+        /* Mark this anon_vma as the one where our new (COWed) pages go. */
+        vma->anon_vma = anon_vma;
+
+        return 0;
+
+ out_error_free_anon_vma:
+        anon_vma_free(anon_vma);
+ out_error:
+        return -ENOMEM;
 }
 
-void anon_vma_unlink(struct vm_area_struct *vma)
+static void anon_vma_unlink(struct anon_vma_chain *anon_vma_chain)
 {
-        struct anon_vma *anon_vma = vma->anon_vma;
+        struct anon_vma *anon_vma = anon_vma_chain->anon_vma;
         int empty;
 
+        /* If anon_vma_fork fails, we can get an empty anon_vma_chain. */
         if (!anon_vma)
                 return;
 
         spin_lock(&anon_vma->lock);
-        list_del(&vma->anon_vma_node);
+        list_del(&anon_vma_chain->same_anon_vma);
 
         /* We must garbage collect the anon_vma if it's empty */
         empty = list_empty(&anon_vma->head) && !ksm_refcount(anon_vma);
@@ -179,6 +255,18 @@ void anon_vma_unlink(struct vm_area_struct *vma)
                 anon_vma_free(anon_vma);
 }
 
+void unlink_anon_vmas(struct vm_area_struct *vma)
+{
+        struct anon_vma_chain *avc, *next;
+
+        /* Unlink each anon_vma chained to the VMA. */
+        list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
+                anon_vma_unlink(avc);
+                list_del(&avc->same_vma);
+                anon_vma_chain_free(avc);
+        }
+}
+
 static void anon_vma_ctor(void *data)
 {
         struct anon_vma *anon_vma = data;
@@ -192,6 +280,7 @@ void __init anon_vma_init(void)
 {
         anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
                         0, SLAB_DESTROY_BY_RCU|SLAB_PANIC, anon_vma_ctor);
+        anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, SLAB_PANIC);
 }
 
 /*
@@ -240,6 +329,18 @@ vma_address(struct page *page, struct vm_area_struct *vma)
                 /* page should be within @vma mapping range */
                 return -EFAULT;
         }
+        if (unlikely(vma->vm_flags & VM_LOCK_RMAP)) {
+                /*
+                 * This VMA is being unlinked or is not yet linked into the
+                 * VMA tree.  Do not try to follow this rmap.  This race
+                 * condition can result in page_referenced() ignoring a
+                 * reference or in try_to_unmap() failing to unmap a page.
+                 * The VMA cannot be freed under us because we hold the
+                 * anon_vma->lock, which the munmap code takes while
+                 * unlinking the anon_vmas from the VMA.
+                 */
+                return -EFAULT;
+        }
         return address;
 }
 
@@ -396,7 +497,7 @@ static int page_referenced_anon(struct page *page,
 {
         unsigned int mapcount;
         struct anon_vma *anon_vma;
-        struct vm_area_struct *vma;
+        struct anon_vma_chain *avc;
         int referenced = 0;
 
         anon_vma = page_lock_anon_vma(page);
@@ -404,7 +505,8 @@ static int page_referenced_anon(struct page *page,
                 return referenced;
 
         mapcount = page_mapcount(page);
-        list_for_each_entry(vma, &anon_vma->head, anon_vma_node) {
+        list_for_each_entry(avc, &anon_vma->head, same_anon_vma) {
+                struct vm_area_struct *vma = avc->vma;
                 unsigned long address = vma_address(page, vma);
                 if (address == -EFAULT)
                         continue;
@@ -1025,14 +1127,15 @@ static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount,
 static int try_to_unmap_anon(struct page *page, enum ttu_flags flags)
 {
         struct anon_vma *anon_vma;
-        struct vm_area_struct *vma;
+        struct anon_vma_chain *avc;
         int ret = SWAP_AGAIN;
 
         anon_vma = page_lock_anon_vma(page);
         if (!anon_vma)
                 return ret;
 
-        list_for_each_entry(vma, &anon_vma->head, anon_vma_node) {
+        list_for_each_entry(avc, &anon_vma->head, same_anon_vma) {
+                struct vm_area_struct *vma = avc->vma;
                 unsigned long address = vma_address(page, vma);
                 if (address == -EFAULT)
                         continue;
@@ -1223,7 +1326,7 @@ static int rmap_walk_anon(struct page *page, int (*rmap_one)(struct page *,
                 struct vm_area_struct *, unsigned long, void *), void *arg)
 {
         struct anon_vma *anon_vma;
-        struct vm_area_struct *vma;
+        struct anon_vma_chain *avc;
         int ret = SWAP_AGAIN;
 
         /*
@@ -1238,7 +1341,8 @@ static int rmap_walk_anon(struct page *page, int (*rmap_one)(struct page *,
         if (!anon_vma)
                 return ret;
         spin_lock(&anon_vma->lock);
-        list_for_each_entry(vma, &anon_vma->head, anon_vma_node) {
+        list_for_each_entry(avc, &anon_vma->head, same_anon_vma) {
+                struct vm_area_struct *vma = avc->vma;
                 unsigned long address = vma_address(page, vma);
                 if (address == -EFAULT)
                         continue;