1 files changed, 88 insertions, 18 deletions

diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index b019ae64e2ab..31b2fd75dcba 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -26,8 +26,18 @@
  */
 struct anon_vma {
         spinlock_t lock;        /* Serialize access to vma list */
-#ifdef CONFIG_KSM
-        atomic_t ksm_refcount;
+        struct anon_vma *root;  /* Root of this anon_vma tree */
+#if defined(CONFIG_KSM) || defined(CONFIG_MIGRATION)
+
+        /*
+         * The external_refcount is taken by either KSM or page migration
+         * to take a reference to an anon_vma when there is no
+         * guarantee that the vma of page tables will exist for
+         * the duration of the operation. A caller that takes
+         * the reference is responsible for clearing up the
+         * anon_vma if they are the last user on release
+         */
+        atomic_t external_refcount;
 #endif
         /*
          * NOTE: the LSB of the head.next is set by
@@ -37,29 +47,64 @@ struct anon_vma {
          * is serialized by a system wide lock only visible to
          * mm_take_all_locks() (mm_all_locks_mutex).
          */
-        struct list_head head;  /* List of private "related" vmas */
+        struct list_head head;  /* Chain of private "related" vmas */
+};
+
+/*
+ * The copy-on-write semantics of fork mean that an anon_vma
+ * can become associated with multiple processes. Furthermore,
+ * each child process will have its own anon_vma, where new
+ * pages for that process are instantiated.
+ *
+ * This structure allows us to find the anon_vmas associated
+ * with a VMA, or the VMAs associated with an anon_vma.
+ * The "same_vma" list contains the anon_vma_chains linking
+ * all the anon_vmas associated with this VMA.
+ * The "same_anon_vma" list contains the anon_vma_chains
+ * which link all the VMAs associated with this anon_vma.
+ */
+struct anon_vma_chain {
+        struct vm_area_struct *vma;
+        struct anon_vma *anon_vma;
+        struct list_head same_vma;   /* locked by mmap_sem & page_table_lock */
+        struct list_head same_anon_vma; /* locked by anon_vma->lock */
 };
 
 #ifdef CONFIG_MMU
-#ifdef CONFIG_KSM
-static inline void ksm_refcount_init(struct anon_vma *anon_vma)
+#if defined(CONFIG_KSM) || defined(CONFIG_MIGRATION)
+static inline void anonvma_external_refcount_init(struct anon_vma *anon_vma)
 {
-        atomic_set(&anon_vma->ksm_refcount, 0);
+        atomic_set(&anon_vma->external_refcount, 0);
 }
 
-static inline int ksm_refcount(struct anon_vma *anon_vma)
+static inline int anonvma_external_refcount(struct anon_vma *anon_vma)
 {
-        return atomic_read(&anon_vma->ksm_refcount);
+        return atomic_read(&anon_vma->external_refcount);
 }
+
+static inline void get_anon_vma(struct anon_vma *anon_vma)
+{
+        atomic_inc(&anon_vma->external_refcount);
+}
+
+void drop_anon_vma(struct anon_vma *);
 #else
-static inline void ksm_refcount_init(struct anon_vma *anon_vma)
+static inline void anonvma_external_refcount_init(struct anon_vma *anon_vma)
 {
 }
 
-static inline int ksm_refcount(struct anon_vma *anon_vma)
+static inline int anonvma_external_refcount(struct anon_vma *anon_vma)
 {
         return 0;
 }
+
+static inline void get_anon_vma(struct anon_vma *anon_vma)
+{
+}
+
+static inline void drop_anon_vma(struct anon_vma *anon_vma)
+{
+}
 #endif /* CONFIG_KSM */
 
 static inline struct anon_vma *page_anon_vma(struct page *page)
@@ -70,18 +115,28 @@ static inline struct anon_vma *page_anon_vma(struct page *page)
         return page_rmapping(page);
 }
 
-static inline void anon_vma_lock(struct vm_area_struct *vma)
+static inline void vma_lock_anon_vma(struct vm_area_struct *vma)
 {
         struct anon_vma *anon_vma = vma->anon_vma;
         if (anon_vma)
-                spin_lock(&anon_vma->lock);
+                spin_lock(&anon_vma->root->lock);
 }
 
-static inline void anon_vma_unlock(struct vm_area_struct *vma)
+static inline void vma_unlock_anon_vma(struct vm_area_struct *vma)
 {
         struct anon_vma *anon_vma = vma->anon_vma;
         if (anon_vma)
-                spin_unlock(&anon_vma->lock);
+                spin_unlock(&anon_vma->root->lock);
+}
+
+static inline void anon_vma_lock(struct anon_vma *anon_vma)
+{
+        spin_lock(&anon_vma->root->lock);
+}
+
+static inline void anon_vma_unlock(struct anon_vma *anon_vma)
+{
+        spin_unlock(&anon_vma->root->lock);
 }
 
 /*
@@ -89,20 +144,35 @@ static inline void anon_vma_unlock(struct vm_area_struct *vma)
  */
 void anon_vma_init(void);       /* create anon_vma_cachep */
 int  anon_vma_prepare(struct vm_area_struct *);
-void __anon_vma_merge(struct vm_area_struct *, struct vm_area_struct *);
-void anon_vma_unlink(struct vm_area_struct *);
-void anon_vma_link(struct vm_area_struct *);
+void unlink_anon_vmas(struct vm_area_struct *);
+int anon_vma_clone(struct vm_area_struct *, struct vm_area_struct *);
+int anon_vma_fork(struct vm_area_struct *, struct vm_area_struct *);
 void __anon_vma_link(struct vm_area_struct *);
 void anon_vma_free(struct anon_vma *);
 
+static inline void anon_vma_merge(struct vm_area_struct *vma,
+                                  struct vm_area_struct *next)
+{
+        VM_BUG_ON(vma->anon_vma != next->anon_vma);
+        unlink_anon_vmas(next);
+}
+
 /*
  * rmap interfaces called when adding or removing pte of page
  */
+void page_move_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);
 void page_add_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);
+void do_page_add_anon_rmap(struct page *, struct vm_area_struct *,
+                           unsigned long, int);
 void page_add_new_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);
 void page_add_file_rmap(struct page *);
 void page_remove_rmap(struct page *);
 
+void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *,
+                            unsigned long);
+void hugepage_add_new_anon_rmap(struct page *, struct vm_area_struct *,
+                                unsigned long);
+
 static inline void page_dup_rmap(struct page *page)
 {
         atomic_inc(&page->_mapcount);
@@ -181,7 +251,7 @@ static inline int page_referenced(struct page *page, int is_locked,
                                   unsigned long *vm_flags)
 {
         *vm_flags = 0;
-        return TestClearPageReferenced(page);
+        return 0;
 }
 
 #define try_to_unmap(page, refs) SWAP_FAIL