[PATCH] mm: rmap optimisation

Optimise rmap functions by minimising atomic operations when we know there
will be no concurrent modifications.

Signed-off-by: Nick Piggin <npiggin@suse.de>
Cc: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

1 files changed, 38 insertions, 11 deletions

diff --git a/mm/rmap.c b/mm/rmap.c
index f853c6def159..4107f64ff749 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -435,6 +435,26 @@ int page_referenced(struct page *page, int is_locked)
 }
 
 /**
+ * page_set_anon_rmap - setup new anonymous rmap
+ * @page:       the page to add the mapping to
+ * @vma:        the vm area in which the mapping is added
+ * @address:    the user virtual address mapped
+ */
+static void __page_set_anon_rmap(struct page *page,
+        struct vm_area_struct *vma, unsigned long address)
+{
+        struct anon_vma *anon_vma = vma->anon_vma;
+
+        BUG_ON(!anon_vma);
+        anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
+        page->mapping = (struct address_space *) anon_vma;
+
+        page->index = linear_page_index(vma, address);
+
+        inc_page_state(nr_mapped);
+}
+
+/**
  * page_add_anon_rmap - add pte mapping to an anonymous page
  * @page:       the page to add the mapping to
  * @vma:        the vm area in which the mapping is added
@@ -445,20 +465,27 @@ int page_referenced(struct page *page, int is_locked)
 void page_add_anon_rmap(struct page *page,
         struct vm_area_struct *vma, unsigned long address)
 {
-        if (atomic_inc_and_test(&page->_mapcount)) {
-                struct anon_vma *anon_vma = vma->anon_vma;
-
-                BUG_ON(!anon_vma);
-                anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
-                page->mapping = (struct address_space *) anon_vma;
-
-                page->index = linear_page_index(vma, address);
-
-                inc_page_state(nr_mapped);
-        }
+        if (atomic_inc_and_test(&page->_mapcount))
+                __page_set_anon_rmap(page, vma, address);
         /* else checking page index and mapping is racy */
 }
 
+/*
+ * page_add_new_anon_rmap - add pte mapping to a new anonymous page
+ * @page:       the page to add the mapping to
+ * @vma:        the vm area in which the mapping is added
+ * @address:    the user virtual address mapped
+ *
+ * Same as page_add_anon_rmap but must only be called on *new* pages.
+ * This means the inc-and-test can be bypassed.
+ */
+void page_add_new_anon_rmap(struct page *page,
+        struct vm_area_struct *vma, unsigned long address)
+{
+        atomic_set(&page->_mapcount, 0); /* elevate count by 1 (starts at -1) */
+        __page_set_anon_rmap(page, vma, address);
+}
+
 /**
  * page_add_file_rmap - add pte mapping to a file page
  * @page: the page to add the mapping to