mm: have zonelist contains structs with both a zone pointer and zone_idx

Filtering zonelists requires very frequent use of zone_idx().  This is costly
as it involves a lookup of another structure and a substraction operation.  As
the zone_idx is often required, it should be quickly accessible.  The node idx
could also be stored here if it was found that accessing zone->node is
significant which may be the case on workloads where nodemasks are heavily
used.

This patch introduces a struct zoneref to store a zone pointer and a zone
index.  The zonelist then consists of an array of these struct zonerefs which
are looked up as necessary.  Helpers are given for accessing the zone index as
well as the node index.

[kamezawa.hiroyu@jp.fujitsu.com: Suggested struct zoneref instead of embedding information in pointers]
[hugh@veritas.com: mm-have-zonelist: fix memcg ooms]
[hugh@veritas.com: just return do_try_to_free_pages]
[hugh@veritas.com: do_try_to_free_pages gfp_mask redundant]
Signed-off-by: Mel Gorman <mel@csn.ul.ie>
Acked-by: Christoph Lameter <clameter@sgi.com>
Acked-by: David Rientjes <rientjes@google.com>
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 22 insertions, 23 deletions

diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 2c93502cfcb4..e41504aa5da9 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -176,7 +176,7 @@ static inline enum oom_constraint constrained_alloc(struct zonelist *zonelist,
 {
 #ifdef CONFIG_NUMA
         struct zone *zone;
-        struct zone **z;
+        struct zoneref *z;
         enum zone_type high_zoneidx = gfp_zone(gfp_mask);
         nodemask_t nodes = node_states[N_HIGH_MEMORY];
 
@@ -462,29 +462,29 @@ EXPORT_SYMBOL_GPL(unregister_oom_notifier);
  * if a parallel OOM killing is already taking place that includes a zone in
  * the zonelist.  Otherwise, locks all zones in the zonelist and returns 1.
  */
-int try_set_zone_oom(struct zonelist *zonelist)
+int try_set_zone_oom(struct zonelist *zonelist, gfp_t gfp_mask)
 {
-        struct zone **z;
+        struct zoneref *z;
+        struct zone *zone;
         int ret = 1;
 
-        z = zonelist->zones;
-
         spin_lock(&zone_scan_mutex);
-        do {
-                if (zone_is_oom_locked(*z)) {
+        for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) {
+                if (zone_is_oom_locked(zone)) {
                         ret = 0;
                         goto out;
                 }
-        } while (*(++z) != NULL);
+        }
+
+        for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) {
+                /*
+                 * Lock each zone in the zonelist under zone_scan_mutex so a
+                 * parallel invocation of try_set_zone_oom() doesn't succeed
+                 * when it shouldn't.
+                 */
+                zone_set_flag(zone, ZONE_OOM_LOCKED);
+        }
 
-        /*
-         * Lock each zone in the zonelist under zone_scan_mutex so a parallel
-         * invocation of try_set_zone_oom() doesn't succeed when it shouldn't.
-         */
-        z = zonelist->zones;
-        do {
-                zone_set_flag(*z, ZONE_OOM_LOCKED);
-        } while (*(++z) != NULL);
 out:
         spin_unlock(&zone_scan_mutex);
         return ret;
@@ -495,16 +495,15 @@ out:
  * allocation attempts with zonelists containing them may now recall the OOM
  * killer, if necessary.
  */
-void clear_zonelist_oom(struct zonelist *zonelist)
+void clear_zonelist_oom(struct zonelist *zonelist, gfp_t gfp_mask)
 {
-        struct zone **z;
-
-        z = zonelist->zones;
+        struct zoneref *z;
+        struct zone *zone;
 
         spin_lock(&zone_scan_mutex);
-        do {
-                zone_clear_flag(*z, ZONE_OOM_LOCKED);
-        } while (*(++z) != NULL);
+        for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) {
+                zone_clear_flag(zone, ZONE_OOM_LOCKED);
+        }
         spin_unlock(&zone_scan_mutex);
 }