mm: move memcg hierarchy reclaim to generic reclaim code

Memory cgroup limit reclaim and traditional global pressure reclaim will
soon share the same code to reclaim from a hierarchical tree of memory
cgroups.

In preparation of this, move the two right next to each other in
shrink_zone().

The mem_cgroup_hierarchical_reclaim() polymath is split into a soft
limit reclaim function, which still does hierarchy walking on its own,
and a limit (shrinking) reclaim function, which relies on generic
reclaim code to walk the hierarchy.

Signed-off-by: Johannes Weiner <jweiner@redhat.com>
Reviewed-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Reviewed-by: Kirill A. Shutemov <kirill@shutemov.name>
Cc: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp>
Cc: Balbir Singh <bsingharora@gmail.com>
Cc: Ying Han <yinghan@google.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Michel Lespinasse <walken@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 39 insertions, 4 deletions

diff --git a/mm/vmscan.c b/mm/vmscan.c
index e0627d07c3ac..136c7eb0ad88 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2104,12 +2104,43 @@ restart:
 static void shrink_zone(int priority, struct zone *zone,
                         struct scan_control *sc)
 {
-        struct mem_cgroup_zone mz = {
-                .mem_cgroup = sc->target_mem_cgroup,
+        struct mem_cgroup *root = sc->target_mem_cgroup;
+        struct mem_cgroup_reclaim_cookie reclaim = {
                 .zone = zone,
+                .priority = priority,
         };
+        struct mem_cgroup *memcg;
+
+        if (global_reclaim(sc)) {
+                struct mem_cgroup_zone mz = {
+                        .mem_cgroup = NULL,
+                        .zone = zone,
+                };
+
+                shrink_mem_cgroup_zone(priority, &mz, sc);
+                return;
+        }
+
+        memcg = mem_cgroup_iter(root, NULL, &reclaim);
+        do {
+                struct mem_cgroup_zone mz = {
+                        .mem_cgroup = memcg,
+                        .zone = zone,
+                };
 
-        shrink_mem_cgroup_zone(priority, &mz, sc);
+                shrink_mem_cgroup_zone(priority, &mz, sc);
+                /*
+                 * Limit reclaim has historically picked one memcg and
+                 * scanned it with decreasing priority levels until
+                 * nr_to_reclaim had been reclaimed.  This priority
+                 * cycle is thus over after a single memcg.
+                 */
+                if (!global_reclaim(sc)) {
+                        mem_cgroup_iter_break(root, memcg);
+                        break;
+                }
+                memcg = mem_cgroup_iter(root, memcg, &reclaim);
+        } while (memcg);
 }
 
 /*
@@ -2374,6 +2405,10 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *mem,
                 .order = 0,
                 .target_mem_cgroup = mem,
         };
+        struct mem_cgroup_zone mz = {
+                .mem_cgroup = mem,
+                .zone = zone,
+        };
 
         sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
                         (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
@@ -2389,7 +2424,7 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *mem,
          * will pick up pages from other mem cgroup's as well. We hack
          * the priority and make it zero.
          */
-        shrink_zone(0, zone, &sc);
+        shrink_mem_cgroup_zone(0, &mz, &sc);
 
         trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);