memcg: adjust to support new THP refcounting

As with rmap, with new refcounting we cannot rely on PageTransHuge() to
check if we need to charge size of huge page form the cgroup.  We need
to get information from caller to know whether it was mapped with PMD or
PTE.

We do uncharge when last reference on the page gone.  At that point if
we see PageTransHuge() it means we need to unchange whole huge page.

The tricky part is partial unmap -- when we try to unmap part of huge
page.  We don't do a special handing of this situation, meaning we don't
uncharge the part of huge page unless last user is gone or
split_huge_page() is triggered.  In case of cgroup memory pressure
happens the partial unmapped page will be split through shrinker.  This
should be good enough.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Tested-by: Sasha Levin <sasha.levin@oracle.com>
Tested-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Acked-by: Jerome Marchand <jmarchan@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Steve Capper <steve.capper@linaro.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 14 insertions, 14 deletions

diff --git a/mm/memory.c b/mm/memory.c
index f964d190ce83..a021c295e88d 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2087,7 +2087,7 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
                 cow_user_page(new_page, old_page, address, vma);
         }
 
-        if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg))
+        if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg, false))
                 goto oom_free_new;
 
         __SetPageUptodate(new_page);
@@ -2119,7 +2119,7 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
                  */
                 ptep_clear_flush_notify(vma, address, page_table);
                 page_add_new_anon_rmap(new_page, vma, address, false);
-                mem_cgroup_commit_charge(new_page, memcg, false);
+                mem_cgroup_commit_charge(new_page, memcg, false, false);
                 lru_cache_add_active_or_unevictable(new_page, vma);
                 /*
                  * We call the notify macro here because, when using secondary
@@ -2158,7 +2158,7 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
                 new_page = old_page;
                 page_copied = 1;
         } else {
-                mem_cgroup_cancel_charge(new_page, memcg);
+                mem_cgroup_cancel_charge(new_page, memcg, false);
         }
 
         if (new_page)
@@ -2533,7 +2533,7 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 goto out_page;
         }
 
-        if (mem_cgroup_try_charge(page, mm, GFP_KERNEL, &memcg)) {
+        if (mem_cgroup_try_charge(page, mm, GFP_KERNEL, &memcg, false)) {
                 ret = VM_FAULT_OOM;
                 goto out_page;
         }
@@ -2575,10 +2575,10 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
         set_pte_at(mm, address, page_table, pte);
         if (page == swapcache) {
                 do_page_add_anon_rmap(page, vma, address, exclusive);
-                mem_cgroup_commit_charge(page, memcg, true);
+                mem_cgroup_commit_charge(page, memcg, true, false);
         } else { /* ksm created a completely new copy */
                 page_add_new_anon_rmap(page, vma, address, false);
-                mem_cgroup_commit_charge(page, memcg, false);
+                mem_cgroup_commit_charge(page, memcg, false, false);
                 lru_cache_add_active_or_unevictable(page, vma);
         }
 
@@ -2613,7 +2613,7 @@ unlock:
 out:
         return ret;
 out_nomap:
-        mem_cgroup_cancel_charge(page, memcg);
+        mem_cgroup_cancel_charge(page, memcg, false);
         pte_unmap_unlock(page_table, ptl);
 out_page:
         unlock_page(page);
@@ -2707,7 +2707,7 @@ static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
         if (!page)
                 goto oom;
 
-        if (mem_cgroup_try_charge(page, mm, GFP_KERNEL, &memcg))
+        if (mem_cgroup_try_charge(page, mm, GFP_KERNEL, &memcg, false))
                 goto oom_free_page;
 
         /*
@@ -2728,7 +2728,7 @@ static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
         /* Deliver the page fault to userland, check inside PT lock */
         if (userfaultfd_missing(vma)) {
                 pte_unmap_unlock(page_table, ptl);
-                mem_cgroup_cancel_charge(page, memcg);
+                mem_cgroup_cancel_charge(page, memcg, false);
                 page_cache_release(page);
                 return handle_userfault(vma, address, flags,
                                         VM_UFFD_MISSING);
@@ -2736,7 +2736,7 @@ static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
 
         inc_mm_counter_fast(mm, MM_ANONPAGES);
         page_add_new_anon_rmap(page, vma, address, false);
-        mem_cgroup_commit_charge(page, memcg, false);
+        mem_cgroup_commit_charge(page, memcg, false, false);
         lru_cache_add_active_or_unevictable(page, vma);
 setpte:
         set_pte_at(mm, address, page_table, entry);
@@ -2747,7 +2747,7 @@ unlock:
         pte_unmap_unlock(page_table, ptl);
         return 0;
 release:
-        mem_cgroup_cancel_charge(page, memcg);
+        mem_cgroup_cancel_charge(page, memcg, false);
         page_cache_release(page);
         goto unlock;
 oom_free_page:
@@ -3000,7 +3000,7 @@ static int do_cow_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         if (!new_page)
                 return VM_FAULT_OOM;
 
-        if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg)) {
+        if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg, false)) {
                 page_cache_release(new_page);
                 return VM_FAULT_OOM;
         }
@@ -3029,7 +3029,7 @@ static int do_cow_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                 goto uncharge_out;
         }
         do_set_pte(vma, address, new_page, pte, true, true);
-        mem_cgroup_commit_charge(new_page, memcg, false);
+        mem_cgroup_commit_charge(new_page, memcg, false, false);
         lru_cache_add_active_or_unevictable(new_page, vma);
         pte_unmap_unlock(pte, ptl);
         if (fault_page) {
@@ -3044,7 +3044,7 @@ static int do_cow_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         }
         return ret;
 uncharge_out:
-        mem_cgroup_cancel_charge(new_page, memcg);
+        mem_cgroup_cancel_charge(new_page, memcg, false);
         page_cache_release(new_page);
         return ret;
 }