mm, page_alloc: avoid looking up the first zone in a zonelist twice

The allocator fast path looks up the first usable zone in a zonelist and
then get_page_from_freelist does the same job in the zonelist iterator.
This patch preserves the necessary information.

                                             4.6.0-rc2                  4.6.0-rc2
                                        fastmark-v1r20             initonce-v1r20
  Min      alloc-odr0-1               364.00 (  0.00%)           359.00 (  1.37%)
  Min      alloc-odr0-2               262.00 (  0.00%)           260.00 (  0.76%)
  Min      alloc-odr0-4               214.00 (  0.00%)           214.00 (  0.00%)
  Min      alloc-odr0-8               186.00 (  0.00%)           186.00 (  0.00%)
  Min      alloc-odr0-16              173.00 (  0.00%)           173.00 (  0.00%)
  Min      alloc-odr0-32              165.00 (  0.00%)           165.00 (  0.00%)
  Min      alloc-odr0-64              161.00 (  0.00%)           162.00 ( -0.62%)
  Min      alloc-odr0-128             159.00 (  0.00%)           161.00 ( -1.26%)
  Min      alloc-odr0-256             168.00 (  0.00%)           170.00 ( -1.19%)
  Min      alloc-odr0-512             180.00 (  0.00%)           181.00 ( -0.56%)
  Min      alloc-odr0-1024            190.00 (  0.00%)           190.00 (  0.00%)
  Min      alloc-odr0-2048            196.00 (  0.00%)           196.00 (  0.00%)
  Min      alloc-odr0-4096            202.00 (  0.00%)           202.00 (  0.00%)
  Min      alloc-odr0-8192            206.00 (  0.00%)           205.00 (  0.49%)
  Min      alloc-odr0-16384           206.00 (  0.00%)           205.00 (  0.49%)

The benefit is negligible and the results are within the noise but each
cycle counts.

Signed-off-by: Mel Gorman <mgorman@techsingularity.net>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 16 insertions, 18 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 732875b1bdfb..dba8cfd0b2d6 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2704,7 +2704,7 @@ static struct page *
 get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
                                                 const struct alloc_context *ac)
 {
-        struct zoneref *z;
+        struct zoneref *z = ac->preferred_zoneref;
         struct zone *zone;
         bool fair_skipped = false;
         bool apply_fair = (alloc_flags & ALLOC_FAIR);
@@ -2714,7 +2714,7 @@ zonelist_scan:
          * Scan zonelist, looking for a zone with enough free.
          * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
          */
-        for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
+        for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
                                                                 ac->nodemask) {
                 struct page *page;
                 unsigned long mark;
@@ -2734,7 +2734,7 @@ zonelist_scan:
                                 fair_skipped = true;
                                 continue;
                         }
-                        if (!zone_local(ac->preferred_zone, zone)) {
+                        if (!zone_local(ac->preferred_zoneref->zone, zone)) {
                                 if (fair_skipped)
                                         goto reset_fair;
                                 apply_fair = false;
@@ -2780,7 +2780,7 @@ zonelist_scan:
                                 goto try_this_zone;
 
                         if (zone_reclaim_mode == 0 ||
-                            !zone_allows_reclaim(ac->preferred_zone, zone))
+                            !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
                                 continue;
 
                         ret = zone_reclaim(zone, gfp_mask, order);
@@ -2802,7 +2802,7 @@ zonelist_scan:
                 }
 
 try_this_zone:
-                page = buffered_rmqueue(ac->preferred_zone, zone, order,
+                page = buffered_rmqueue(ac->preferred_zoneref->zone, zone, order,
                                 gfp_mask, alloc_flags, ac->migratetype);
                 if (page) {
                         if (prep_new_page(page, order, gfp_mask, alloc_flags))
@@ -2831,7 +2831,7 @@ try_this_zone:
 reset_fair:
                 apply_fair = false;
                 fair_skipped = false;
-                reset_alloc_batches(ac->preferred_zone);
+                reset_alloc_batches(ac->preferred_zoneref->zone);
                 goto zonelist_scan;
         }
 
@@ -3114,7 +3114,7 @@ static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac)
 
         for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
                                                 ac->high_zoneidx, ac->nodemask)
-                wakeup_kswapd(zone, order, zone_idx(ac->preferred_zone));
+                wakeup_kswapd(zone, order, zonelist_zone_idx(ac->preferred_zoneref));
 }
 
 static inline unsigned int
@@ -3332,7 +3332,7 @@ retry:
         if ((did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER) ||
             ((gfp_mask & __GFP_REPEAT) && pages_reclaimed < (1 << order))) {
                 /* Wait for some write requests to complete then retry */
-                wait_iff_congested(ac->preferred_zone, BLK_RW_ASYNC, HZ/50);
+                wait_iff_congested(ac->preferred_zoneref->zone, BLK_RW_ASYNC, HZ/50);
                 goto retry;
         }
 
@@ -3370,7 +3370,6 @@ struct page *
 __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
                         struct zonelist *zonelist, nodemask_t *nodemask)
 {
-        struct zoneref *preferred_zoneref;
         struct page *page;
         unsigned int cpuset_mems_cookie;
         unsigned int alloc_flags = ALLOC_WMARK_LOW|ALLOC_FAIR;
@@ -3416,14 +3415,14 @@ retry_cpuset:
         ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE);
 
         /* The preferred zone is used for statistics later */
-        preferred_zoneref = first_zones_zonelist(ac.zonelist, ac.high_zoneidx,
-                                ac.nodemask, &ac.preferred_zone);
-        if (!ac.preferred_zone) {
+        ac.preferred_zoneref = first_zones_zonelist(ac.zonelist,
+                                        ac.high_zoneidx, ac.nodemask);
+        if (!ac.preferred_zoneref) {
                 page = NULL;
                 goto no_zone;
         }
 
-        ac.classzone_idx = zonelist_zone_idx(preferred_zoneref);
+        ac.classzone_idx = zonelist_zone_idx(ac.preferred_zoneref);
 
         /* First allocation attempt */
         page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
@@ -4462,13 +4461,12 @@ static void build_zonelists(pg_data_t *pgdat)
  */
 int local_memory_node(int node)
 {
-        struct zone *zone;
+        struct zoneref *z;
 
-        (void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
+        z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
                                    gfp_zone(GFP_KERNEL),
-                                   NULL,
-                                   &zone);
-        return zone->node;
+                                   NULL);
+        return z->zone->node;
 }
 #endif