mm: memcontrol: rewrite charge API

These patches rework memcg charge lifetime to integrate more naturally
with the lifetime of user pages.  This drastically simplifies the code and
reduces charging and uncharging overhead.  The most expensive part of
charging and uncharging is the page_cgroup bit spinlock, which is removed
entirely after this series.

Here are the top-10 profile entries of a stress test that reads a 128G
sparse file on a freshly booted box, without even a dedicated cgroup (i.e.
 executing in the root memcg).  Before:

    15.36%              cat  [kernel.kallsyms]   [k] copy_user_generic_string
    13.31%              cat  [kernel.kallsyms]   [k] memset
    11.48%              cat  [kernel.kallsyms]   [k] do_mpage_readpage
     4.23%              cat  [kernel.kallsyms]   [k] get_page_from_freelist
     2.38%              cat  [kernel.kallsyms]   [k] put_page
     2.32%              cat  [kernel.kallsyms]   [k] __mem_cgroup_commit_charge
     2.18%          kswapd0  [kernel.kallsyms]   [k] __mem_cgroup_uncharge_common
     1.92%          kswapd0  [kernel.kallsyms]   [k] shrink_page_list
     1.86%              cat  [kernel.kallsyms]   [k] __radix_tree_lookup
     1.62%              cat  [kernel.kallsyms]   [k] __pagevec_lru_add_fn

After:

    15.67%           cat  [kernel.kallsyms]   [k] copy_user_generic_string
    13.48%           cat  [kernel.kallsyms]   [k] memset
    11.42%           cat  [kernel.kallsyms]   [k] do_mpage_readpage
     3.98%           cat  [kernel.kallsyms]   [k] get_page_from_freelist
     2.46%           cat  [kernel.kallsyms]   [k] put_page
     2.13%       kswapd0  [kernel.kallsyms]   [k] shrink_page_list
     1.88%           cat  [kernel.kallsyms]   [k] __radix_tree_lookup
     1.67%           cat  [kernel.kallsyms]   [k] __pagevec_lru_add_fn
     1.39%       kswapd0  [kernel.kallsyms]   [k] free_pcppages_bulk
     1.30%           cat  [kernel.kallsyms]   [k] kfree

As you can see, the memcg footprint has shrunk quite a bit.

   text    data     bss     dec     hex filename
  37970    9892     400   48262    bc86 mm/memcontrol.o.old
  35239    9892     400   45531    b1db mm/memcontrol.o

This patch (of 4):

The memcg charge API charges pages before they are rmapped - i.e.  have an
actual "type" - and so every callsite needs its own set of charge and
uncharge functions to know what type is being operated on.  Worse,
uncharge has to happen from a context that is still type-specific, rather
than at the end of the page's lifetime with exclusive access, and so
requires a lot of synchronization.

Rewrite the charge API to provide a generic set of try_charge(),
commit_charge() and cancel_charge() transaction operations, much like
what's currently done for swap-in:

  mem_cgroup_try_charge() attempts to reserve a charge, reclaiming
  pages from the memcg if necessary.

  mem_cgroup_commit_charge() commits the page to the charge once it
  has a valid page->mapping and PageAnon() reliably tells the type.

  mem_cgroup_cancel_charge() aborts the transaction.

This reduces the charge API and enables subsequent patches to
drastically simplify uncharging.

As pages need to be committed after rmap is established but before they
are added to the LRU, page_add_new_anon_rmap() must stop doing LRU
additions again.  Revive lru_cache_add_active_or_unevictable().

[hughd@google.com: fix shmem_unuse]
[hughd@google.com: Add comments on the private use of -EAGAIN]
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Michal Hocko <mhocko@suse.cz>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vladimir Davydov <vdavydov@parallels.com>
Signed-off-by: Hugh Dickins <hughd@google.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 167 insertions, 240 deletions

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 90dc501eaf3f..1cbe1e54ff5f 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2551,17 +2551,8 @@ static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
         return NOTIFY_OK;
 }
 
-/**
- * mem_cgroup_try_charge - try charging a memcg
- * @memcg: memcg to charge
- * @nr_pages: number of pages to charge
- *
- * Returns 0 if @memcg was charged successfully, -EINTR if the charge
- * was bypassed to root_mem_cgroup, and -ENOMEM if the charge failed.
- */
-static int mem_cgroup_try_charge(struct mem_cgroup *memcg,
-                                 gfp_t gfp_mask,
-                                 unsigned int nr_pages)
+static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
+                      unsigned int nr_pages)
 {
         unsigned int batch = max(CHARGE_BATCH, nr_pages);
         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
@@ -2660,41 +2651,7 @@ done:
         return ret;
 }
 
-/**
- * mem_cgroup_try_charge_mm - try charging a mm
- * @mm: mm_struct to charge
- * @nr_pages: number of pages to charge
- * @oom: trigger OOM if reclaim fails
- *
- * Returns the charged mem_cgroup associated with the given mm_struct or
- * NULL the charge failed.
- */
-static struct mem_cgroup *mem_cgroup_try_charge_mm(struct mm_struct *mm,
-                                 gfp_t gfp_mask,
-                                 unsigned int nr_pages)
-
-{
-        struct mem_cgroup *memcg;
-        int ret;
-
-        memcg = get_mem_cgroup_from_mm(mm);
-        ret = mem_cgroup_try_charge(memcg, gfp_mask, nr_pages);
-        css_put(&memcg->css);
-        if (ret == -EINTR)
-                memcg = root_mem_cgroup;
-        else if (ret)
-                memcg = NULL;
-
-        return memcg;
-}
-
-/*
- * Somemtimes we have to undo a charge we got by try_charge().
- * This function is for that and do uncharge, put css's refcnt.
- * gotten by try_charge().
- */
-static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
-                                       unsigned int nr_pages)
+static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
 {
         unsigned long bytes = nr_pages * PAGE_SIZE;
 
@@ -2760,17 +2717,13 @@ struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
         return memcg;
 }
 
-static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
-                                       struct page *page,
-                                       unsigned int nr_pages,
-                                       enum charge_type ctype,
-                                       bool lrucare)
+static void commit_charge(struct page *page, struct mem_cgroup *memcg,
+                          unsigned int nr_pages, bool anon, bool lrucare)
 {
         struct page_cgroup *pc = lookup_page_cgroup(page);
         struct zone *uninitialized_var(zone);
         struct lruvec *lruvec;
         bool was_on_lru = false;
-        bool anon;
 
         lock_page_cgroup(pc);
         VM_BUG_ON_PAGE(PageCgroupUsed(pc), page);
@@ -2807,11 +2760,6 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
                 spin_unlock_irq(&zone->lru_lock);
         }
 
-        if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
-                anon = true;
-        else
-                anon = false;
-
         mem_cgroup_charge_statistics(memcg, page, anon, nr_pages);
         unlock_page_cgroup(pc);
 
@@ -2882,21 +2830,21 @@ static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
         if (ret)
                 return ret;
 
-        ret = mem_cgroup_try_charge(memcg, gfp, size >> PAGE_SHIFT);
+        ret = try_charge(memcg, gfp, size >> PAGE_SHIFT);
         if (ret == -EINTR)  {
                 /*
-                 * mem_cgroup_try_charge() chosed to bypass to root due to
-                 * OOM kill or fatal signal.  Since our only options are to
-                 * either fail the allocation or charge it to this cgroup, do
-                 * it as a temporary condition. But we can't fail. From a
-                 * kmem/slab perspective, the cache has already been selected,
-                 * by mem_cgroup_kmem_get_cache(), so it is too late to change
+                 * try_charge() chose to bypass to root due to OOM kill or
+                 * fatal signal.  Since our only options are to either fail
+                 * the allocation or charge it to this cgroup, do it as a
+                 * temporary condition. But we can't fail. From a kmem/slab
+                 * perspective, the cache has already been selected, by
+                 * mem_cgroup_kmem_get_cache(), so it is too late to change
                  * our minds.
                  *
                  * This condition will only trigger if the task entered
-                 * memcg_charge_kmem in a sane state, but was OOM-killed during
-                 * mem_cgroup_try_charge() above. Tasks that were already
-                 * dying when the allocation triggers should have been already
+                 * memcg_charge_kmem in a sane state, but was OOM-killed
+                 * during try_charge() above. Tasks that were already dying
+                 * when the allocation triggers should have been already
                  * directed to the root cgroup in memcontrol.h
                  */
                 res_counter_charge_nofail(&memcg->res, size, &fail_res);
@@ -3618,164 +3566,6 @@ out:
         return ret;
 }
 
-int mem_cgroup_charge_anon(struct page *page,
-                              struct mm_struct *mm, gfp_t gfp_mask)
-{
-        unsigned int nr_pages = 1;
-        struct mem_cgroup *memcg;
-
-        if (mem_cgroup_disabled())
-                return 0;
-
-        VM_BUG_ON_PAGE(page_mapped(page), page);
-        VM_BUG_ON_PAGE(page->mapping && !PageAnon(page), page);
-        VM_BUG_ON(!mm);
-
-        if (PageTransHuge(page)) {
-                nr_pages <<= compound_order(page);
-                VM_BUG_ON_PAGE(!PageTransHuge(page), page);
-        }
-
-        memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, nr_pages);
-        if (!memcg)
-                return -ENOMEM;
-        __mem_cgroup_commit_charge(memcg, page, nr_pages,
-                                   MEM_CGROUP_CHARGE_TYPE_ANON, false);
-        return 0;
-}
-
-/*
- * While swap-in, try_charge -> commit or cancel, the page is locked.
- * And when try_charge() successfully returns, one refcnt to memcg without
- * struct page_cgroup is acquired. This refcnt will be consumed by
- * "commit()" or removed by "cancel()"
- */
-static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
-                                          struct page *page,
-                                          gfp_t mask,
-                                          struct mem_cgroup **memcgp)
-{
-        struct mem_cgroup *memcg = NULL;
-        struct page_cgroup *pc;
-        int ret;
-
-        pc = lookup_page_cgroup(page);
-        /*
-         * Every swap fault against a single page tries to charge the
-         * page, bail as early as possible.  shmem_unuse() encounters
-         * already charged pages, too.  The USED bit is protected by
-         * the page lock, which serializes swap cache removal, which
-         * in turn serializes uncharging.
-         */
-        if (PageCgroupUsed(pc))
-                goto out;
-        if (do_swap_account)
-                memcg = try_get_mem_cgroup_from_page(page);
-        if (!memcg)
-                memcg = get_mem_cgroup_from_mm(mm);
-        ret = mem_cgroup_try_charge(memcg, mask, 1);
-        css_put(&memcg->css);
-        if (ret == -EINTR)
-                memcg = root_mem_cgroup;
-        else if (ret)
-                return ret;
-out:
-        *memcgp = memcg;
-        return 0;
-}
-
-int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page,
-                                 gfp_t gfp_mask, struct mem_cgroup **memcgp)
-{
-        if (mem_cgroup_disabled()) {
-                *memcgp = NULL;
-                return 0;
-        }
-        /*
-         * A racing thread's fault, or swapoff, may have already
-         * updated the pte, and even removed page from swap cache: in
-         * those cases unuse_pte()'s pte_same() test will fail; but
-         * there's also a KSM case which does need to charge the page.
-         */
-        if (!PageSwapCache(page)) {
-                struct mem_cgroup *memcg;
-
-                memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, 1);
-                if (!memcg)
-                        return -ENOMEM;
-                *memcgp = memcg;
-                return 0;
-        }
-        return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
-}
-
-void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
-{
-        if (mem_cgroup_disabled())
-                return;
-        if (!memcg)
-                return;
-        __mem_cgroup_cancel_charge(memcg, 1);
-}
-
-static void
-__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
-                                        enum charge_type ctype)
-{
-        if (mem_cgroup_disabled())
-                return;
-        if (!memcg)
-                return;
-
-        __mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
-        /*
-         * Now swap is on-memory. This means this page may be
-         * counted both as mem and swap....double count.
-         * Fix it by uncharging from memsw. Basically, this SwapCache is stable
-         * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
-         * may call delete_from_swap_cache() before reach here.
-         */
-        if (do_swap_account && PageSwapCache(page)) {
-                swp_entry_t ent = {.val = page_private(page)};
-                mem_cgroup_uncharge_swap(ent);
-        }
-}
-
-void mem_cgroup_commit_charge_swapin(struct page *page,
-                                     struct mem_cgroup *memcg)
-{
-        __mem_cgroup_commit_charge_swapin(page, memcg,
-                                          MEM_CGROUP_CHARGE_TYPE_ANON);
-}
-
-int mem_cgroup_charge_file(struct page *page, struct mm_struct *mm,
-                                gfp_t gfp_mask)
-{
-        enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
-        struct mem_cgroup *memcg;
-        int ret;
-
-        if (mem_cgroup_disabled())
-                return 0;
-        if (PageCompound(page))
-                return 0;
-
-        if (PageSwapCache(page)) { /* shmem */
-                ret = __mem_cgroup_try_charge_swapin(mm, page,
-                                                     gfp_mask, &memcg);
-                if (ret)
-                        return ret;
-                __mem_cgroup_commit_charge_swapin(page, memcg, type);
-                return 0;
-        }
-
-        memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, 1);
-        if (!memcg)
-                return -ENOMEM;
-        __mem_cgroup_commit_charge(memcg, page, 1, type, false);
-        return 0;
-}
-
 static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
                                    unsigned int nr_pages,
                                    const enum charge_type ctype)
@@ -4122,7 +3912,6 @@ void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
         struct mem_cgroup *memcg = NULL;
         unsigned int nr_pages = 1;
         struct page_cgroup *pc;
-        enum charge_type ctype;
 
         *memcgp = NULL;
 
@@ -4184,16 +3973,12 @@ void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
          * page. In the case new page is migrated but not remapped, new page's
          * mapcount will be finally 0 and we call uncharge in end_migration().
          */
-        if (PageAnon(page))
-                ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
-        else
-                ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
         /*
          * The page is committed to the memcg, but it's not actually
          * charged to the res_counter since we plan on replacing the
          * old one and only one page is going to be left afterwards.
          */
-        __mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false);
+        commit_charge(newpage, memcg, nr_pages, PageAnon(page), false);
 }
 
 /* remove redundant charge if migration failed*/
@@ -4252,7 +4037,6 @@ void mem_cgroup_replace_page_cache(struct page *oldpage,
 {
         struct mem_cgroup *memcg = NULL;
         struct page_cgroup *pc;
-        enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
 
         if (mem_cgroup_disabled())
                 return;
@@ -4278,7 +4062,7 @@ void mem_cgroup_replace_page_cache(struct page *oldpage,
          * the newpage may be on LRU(or pagevec for LRU) already. We lock
          * LRU while we overwrite pc->mem_cgroup.
          */
-        __mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
+        commit_charge(newpage, memcg, 1, false, true);
 }
 
 #ifdef CONFIG_DEBUG_VM
@@ -6319,20 +6103,19 @@ static int mem_cgroup_do_precharge(unsigned long count)
         int ret;
 
         /* Try a single bulk charge without reclaim first */
-        ret = mem_cgroup_try_charge(mc.to, GFP_KERNEL & ~__GFP_WAIT, count);
+        ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_WAIT, count);
         if (!ret) {
                 mc.precharge += count;
                 return ret;
         }
         if (ret == -EINTR) {
-                __mem_cgroup_cancel_charge(root_mem_cgroup, count);
+                cancel_charge(root_mem_cgroup, count);
                 return ret;
         }
 
         /* Try charges one by one with reclaim */
         while (count--) {
-                ret = mem_cgroup_try_charge(mc.to,
-                                            GFP_KERNEL & ~__GFP_NORETRY, 1);
+                ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_NORETRY, 1);
                 /*
                  * In case of failure, any residual charges against
                  * mc.to will be dropped by mem_cgroup_clear_mc()
@@ -6340,7 +6123,7 @@ static int mem_cgroup_do_precharge(unsigned long count)
                  * bypassed to root right away or they'll be lost.
                  */
                 if (ret == -EINTR)
-                        __mem_cgroup_cancel_charge(root_mem_cgroup, 1);
+                        cancel_charge(root_mem_cgroup, 1);
                 if (ret)
                         return ret;
                 mc.precharge++;
@@ -6609,7 +6392,7 @@ static void __mem_cgroup_clear_mc(void)
 
         /* we must uncharge all the leftover precharges from mc.to */
         if (mc.precharge) {
-                __mem_cgroup_cancel_charge(mc.to, mc.precharge);
+                cancel_charge(mc.to, mc.precharge);
                 mc.precharge = 0;
         }
         /*
@@ -6617,7 +6400,7 @@ static void __mem_cgroup_clear_mc(void)
          * we must uncharge here.
          */
         if (mc.moved_charge) {
-                __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
+                cancel_charge(mc.from, mc.moved_charge);
                 mc.moved_charge = 0;
         }
         /* we must fixup refcnts and charges */
@@ -6946,6 +6729,150 @@ static void __init enable_swap_cgroup(void)
 }
 #endif
 
+/**
+ * mem_cgroup_try_charge - try charging a page
+ * @page: page to charge
+ * @mm: mm context of the victim
+ * @gfp_mask: reclaim mode
+ * @memcgp: charged memcg return
+ *
+ * Try to charge @page to the memcg that @mm belongs to, reclaiming
+ * pages according to @gfp_mask if necessary.
+ *
+ * Returns 0 on success, with *@memcgp pointing to the charged memcg.
+ * Otherwise, an error code is returned.
+ *
+ * After page->mapping has been set up, the caller must finalize the
+ * charge with mem_cgroup_commit_charge().  Or abort the transaction
+ * with mem_cgroup_cancel_charge() in case page instantiation fails.
+ */
+int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
+                          gfp_t gfp_mask, struct mem_cgroup **memcgp)
+{
+        struct mem_cgroup *memcg = NULL;
+        unsigned int nr_pages = 1;
+        int ret = 0;
+
+        if (mem_cgroup_disabled())
+                goto out;
+
+        if (PageSwapCache(page)) {
+                struct page_cgroup *pc = lookup_page_cgroup(page);
+                /*
+                 * Every swap fault against a single page tries to charge the
+                 * page, bail as early as possible.  shmem_unuse() encounters
+                 * already charged pages, too.  The USED bit is protected by
+                 * the page lock, which serializes swap cache removal, which
+                 * in turn serializes uncharging.
+                 */
+                if (PageCgroupUsed(pc))
+                        goto out;
+        }
+
+        if (PageTransHuge(page)) {
+                nr_pages <<= compound_order(page);
+                VM_BUG_ON_PAGE(!PageTransHuge(page), page);
+        }
+
+        if (do_swap_account && PageSwapCache(page))
+                memcg = try_get_mem_cgroup_from_page(page);
+        if (!memcg)
+                memcg = get_mem_cgroup_from_mm(mm);
+
+        ret = try_charge(memcg, gfp_mask, nr_pages);
+
+        css_put(&memcg->css);
+
+        if (ret == -EINTR) {
+                memcg = root_mem_cgroup;
+                ret = 0;
+        }
+out:
+        *memcgp = memcg;
+        return ret;
+}
+
+/**
+ * mem_cgroup_commit_charge - commit a page charge
+ * @page: page to charge
+ * @memcg: memcg to charge the page to
+ * @lrucare: page might be on LRU already
+ *
+ * Finalize a charge transaction started by mem_cgroup_try_charge(),
+ * after page->mapping has been set up.  This must happen atomically
+ * as part of the page instantiation, i.e. under the page table lock
+ * for anonymous pages, under the page lock for page and swap cache.
+ *
+ * In addition, the page must not be on the LRU during the commit, to
+ * prevent racing with task migration.  If it might be, use @lrucare.
+ *
+ * Use mem_cgroup_cancel_charge() to cancel the transaction instead.
+ */
+void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
+                              bool lrucare)
+{
+        unsigned int nr_pages = 1;
+
+        VM_BUG_ON_PAGE(!page->mapping, page);
+        VM_BUG_ON_PAGE(PageLRU(page) && !lrucare, page);
+
+        if (mem_cgroup_disabled())
+                return;
+        /*
+         * Swap faults will attempt to charge the same page multiple
+         * times.  But reuse_swap_page() might have removed the page
+         * from swapcache already, so we can't check PageSwapCache().
+         */
+        if (!memcg)
+                return;
+
+        if (PageTransHuge(page)) {
+                nr_pages <<= compound_order(page);
+                VM_BUG_ON_PAGE(!PageTransHuge(page), page);
+        }
+
+        commit_charge(page, memcg, nr_pages, PageAnon(page), lrucare);
+
+        if (do_swap_account && PageSwapCache(page)) {
+                swp_entry_t entry = { .val = page_private(page) };
+                /*
+                 * The swap entry might not get freed for a long time,
+                 * let's not wait for it.  The page already received a
+                 * memory+swap charge, drop the swap entry duplicate.
+                 */
+                mem_cgroup_uncharge_swap(entry);
+        }
+}
+
+/**
+ * mem_cgroup_cancel_charge - cancel a page charge
+ * @page: page to charge
+ * @memcg: memcg to charge the page to
+ *
+ * Cancel a charge transaction started by mem_cgroup_try_charge().
+ */
+void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg)
+{
+        unsigned int nr_pages = 1;
+
+        if (mem_cgroup_disabled())
+                return;
+        /*
+         * Swap faults will attempt to charge the same page multiple
+         * times.  But reuse_swap_page() might have removed the page
+         * from swapcache already, so we can't check PageSwapCache().
+         */
+        if (!memcg)
+                return;
+
+        if (PageTransHuge(page)) {
+                nr_pages <<= compound_order(page);
+                VM_BUG_ON_PAGE(!PageTransHuge(page), page);
+        }
+
+        cancel_charge(memcg, nr_pages);
+}
+
 /*
  * subsys_initcall() for memory controller.
  *