1 files changed, 55 insertions, 39 deletions

diff --git a/mm/rmap.c b/mm/rmap.c
index 116a5053415b..3e4c7213210c 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -274,6 +274,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
 {
         struct anon_vma_chain *avc;
         struct anon_vma *anon_vma;
+        int error;
 
         /* Don't bother if the parent process has no anon_vma here. */
         if (!pvma->anon_vma)
@@ -283,8 +284,9 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
          * First, attach the new VMA to the parent VMA's anon_vmas,
          * so rmap can find non-COWed pages in child processes.
          */
-        if (anon_vma_clone(vma, pvma))
-                return -ENOMEM;
+        error = anon_vma_clone(vma, pvma);
+        if (error)
+                return error;
 
         /* Then add our own anon_vma. */
         anon_vma = anon_vma_alloc();
@@ -1042,15 +1044,46 @@ void page_add_new_anon_rmap(struct page *page,
  */
 void page_add_file_rmap(struct page *page)
 {
-        bool locked;
+        struct mem_cgroup *memcg;
         unsigned long flags;
+        bool locked;
 
-        mem_cgroup_begin_update_page_stat(page, &locked, &flags);
+        memcg = mem_cgroup_begin_page_stat(page, &locked, &flags);
         if (atomic_inc_and_test(&page->_mapcount)) {
                 __inc_zone_page_state(page, NR_FILE_MAPPED);
-                mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);
+                mem_cgroup_inc_page_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED);
         }
-        mem_cgroup_end_update_page_stat(page, &locked, &flags);
+        mem_cgroup_end_page_stat(memcg, locked, flags);
+}
+
+static void page_remove_file_rmap(struct page *page)
+{
+        struct mem_cgroup *memcg;
+        unsigned long flags;
+        bool locked;
+
+        memcg = mem_cgroup_begin_page_stat(page, &locked, &flags);
+
+        /* page still mapped by someone else? */
+        if (!atomic_add_negative(-1, &page->_mapcount))
+                goto out;
+
+        /* Hugepages are not counted in NR_FILE_MAPPED for now. */
+        if (unlikely(PageHuge(page)))
+                goto out;
+
+        /*
+         * We use the irq-unsafe __{inc|mod}_zone_page_stat because
+         * these counters are not modified in interrupt context, and
+         * pte lock(a spinlock) is held, which implies preemption disabled.
+         */
+        __dec_zone_page_state(page, NR_FILE_MAPPED);
+        mem_cgroup_dec_page_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED);
+
+        if (unlikely(PageMlocked(page)))
+                clear_page_mlock(page);
+out:
+        mem_cgroup_end_page_stat(memcg, locked, flags);
 }
 
 /**
@@ -1061,46 +1094,33 @@ void page_add_file_rmap(struct page *page)
  */
 void page_remove_rmap(struct page *page)
 {
-        bool anon = PageAnon(page);
-        bool locked;
-        unsigned long flags;
-
-        /*
-         * The anon case has no mem_cgroup page_stat to update; but may
-         * uncharge_page() below, where the lock ordering can deadlock if
-         * we hold the lock against page_stat move: so avoid it on anon.
-         */
-        if (!anon)
-                mem_cgroup_begin_update_page_stat(page, &locked, &flags);
+        if (!PageAnon(page)) {
+                page_remove_file_rmap(page);
+                return;
+        }
 
         /* page still mapped by someone else? */
         if (!atomic_add_negative(-1, &page->_mapcount))
-                goto out;
+                return;
+
+        /* Hugepages are not counted in NR_ANON_PAGES for now. */
+        if (unlikely(PageHuge(page)))
+                return;
 
         /*
-         * Hugepages are not counted in NR_ANON_PAGES nor NR_FILE_MAPPED
-         * and not charged by memcg for now.
-         *
          * We use the irq-unsafe __{inc|mod}_zone_page_stat because
          * these counters are not modified in interrupt context, and
-         * these counters are not modified in interrupt context, and
          * pte lock(a spinlock) is held, which implies preemption disabled.
          */
-        if (unlikely(PageHuge(page)))
-                goto out;
-        if (anon) {
-                if (PageTransHuge(page))
-                        __dec_zone_page_state(page,
-                                              NR_ANON_TRANSPARENT_HUGEPAGES);
-                __mod_zone_page_state(page_zone(page), NR_ANON_PAGES,
-                                -hpage_nr_pages(page));
-        } else {
-                __dec_zone_page_state(page, NR_FILE_MAPPED);
-                mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);
-                mem_cgroup_end_update_page_stat(page, &locked, &flags);
-        }
+        if (PageTransHuge(page))
+                __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
+
+        __mod_zone_page_state(page_zone(page), NR_ANON_PAGES,
+                              -hpage_nr_pages(page));
+
         if (unlikely(PageMlocked(page)))
                 clear_page_mlock(page);
+
         /*
          * It would be tidy to reset the PageAnon mapping here,
          * but that might overwrite a racing page_add_anon_rmap
@@ -1110,10 +1130,6 @@ void page_remove_rmap(struct page *page)
          * Leaving it set also helps swapoff to reinstate ptes
          * faster for those pages still in swapcache.
          */
-        return;
-out:
-        if (!anon)
-                mem_cgroup_end_update_page_stat(page, &locked, &flags);
 }
 
 /*