1 files changed, 224 insertions, 57 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index aa6fcc7ca66f..cace22b3ac25 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -83,14 +83,7 @@ int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
 
 EXPORT_SYMBOL(totalram_pages);
 
-/*
- * Used by page_zone() to look up the address of the struct zone whose
- * id is encoded in the upper bits of page->flags
- */
-struct zone *zone_table[1 << ZONETABLE_SHIFT] __read_mostly;
-EXPORT_SYMBOL(zone_table);
-
-static char *zone_names[MAX_NR_ZONES] = {
+static char * const zone_names[MAX_NR_ZONES] = {
          "DMA",
 #ifdef CONFIG_ZONE_DMA32
          "DMA32",
@@ -237,7 +230,7 @@ static void prep_compound_page(struct page *page, unsigned long order)
         int i;
         int nr_pages = 1 << order;
 
-        page[1].lru.next = (void *)free_compound_page;  /* set dtor */
+        set_compound_page_dtor(page, free_compound_page);
         page[1].lru.prev = (void *)order;
         for (i = 0; i < nr_pages; i++) {
                 struct page *p = page + i;
@@ -486,7 +479,7 @@ static void free_one_page(struct zone *zone, struct page *page, int order)
         spin_lock(&zone->lock);
         zone->all_unreclaimable = 0;
         zone->pages_scanned = 0;
-        __free_one_page(page, zone ,order);
+        __free_one_page(page, zone, order);
         spin_unlock(&zone->lock);
 }
 
@@ -605,6 +598,8 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
                         1 << PG_checked | 1 << PG_mappedtodisk);
         set_page_private(page, 0);
         set_page_refcounted(page);
+
+        arch_alloc_page(page, order);
         kernel_map_pages(page, 1 << order, 1);
 
         if (gfp_flags & __GFP_ZERO)
@@ -690,9 +685,15 @@ void drain_node_pages(int nodeid)
 
                         pcp = &pset->pcp[i];
                         if (pcp->count) {
+                                int to_drain;
+
                                 local_irq_save(flags);
-                                free_pages_bulk(zone, pcp->count, &pcp->list, 0);
-                                pcp->count = 0;
+                                if (pcp->count >= pcp->batch)
+                                        to_drain = pcp->batch;
+                                else
+                                        to_drain = pcp->count;
+                                free_pages_bulk(zone, to_drain, &pcp->list, 0);
+                                pcp->count -= to_drain;
                                 local_irq_restore(flags);
                         }
                 }
@@ -700,7 +701,6 @@ void drain_node_pages(int nodeid)
 }
 #endif
 
-#if defined(CONFIG_PM) || defined(CONFIG_HOTPLUG_CPU)
 static void __drain_pages(unsigned int cpu)
 {
         unsigned long flags;
@@ -722,7 +722,6 @@ static void __drain_pages(unsigned int cpu)
                 }
         }
 }
-#endif /* CONFIG_PM || CONFIG_HOTPLUG_CPU */
 
 #ifdef CONFIG_PM
 
@@ -925,31 +924,160 @@ int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
         return 1;
 }
 
+#ifdef CONFIG_NUMA
+/*
+ * zlc_setup - Setup for "zonelist cache".  Uses cached zone data to
+ * skip over zones that are not allowed by the cpuset, or that have
+ * been recently (in last second) found to be nearly full.  See further
+ * comments in mmzone.h.  Reduces cache footprint of zonelist scans
+ * that have to skip over alot of full or unallowed zones.
+ *
+ * If the zonelist cache is present in the passed in zonelist, then
+ * returns a pointer to the allowed node mask (either the current
+ * tasks mems_allowed, or node_online_map.)
+ *
+ * If the zonelist cache is not available for this zonelist, does
+ * nothing and returns NULL.
+ *
+ * If the fullzones BITMAP in the zonelist cache is stale (more than
+ * a second since last zap'd) then we zap it out (clear its bits.)
+ *
+ * We hold off even calling zlc_setup, until after we've checked the
+ * first zone in the zonelist, on the theory that most allocations will
+ * be satisfied from that first zone, so best to examine that zone as
+ * quickly as we can.
+ */
+static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
+{
+        struct zonelist_cache *zlc;     /* cached zonelist speedup info */
+        nodemask_t *allowednodes;       /* zonelist_cache approximation */
+
+        zlc = zonelist->zlcache_ptr;
+        if (!zlc)
+                return NULL;
+
+        if (jiffies - zlc->last_full_zap > 1 * HZ) {
+                bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
+                zlc->last_full_zap = jiffies;
+        }
+
+        allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
+                                        &cpuset_current_mems_allowed :
+                                        &node_online_map;
+        return allowednodes;
+}
+
+/*
+ * Given 'z' scanning a zonelist, run a couple of quick checks to see
+ * if it is worth looking at further for free memory:
+ *  1) Check that the zone isn't thought to be full (doesn't have its
+ *     bit set in the zonelist_cache fullzones BITMAP).
+ *  2) Check that the zones node (obtained from the zonelist_cache
+ *     z_to_n[] mapping) is allowed in the passed in allowednodes mask.
+ * Return true (non-zero) if zone is worth looking at further, or
+ * else return false (zero) if it is not.
+ *
+ * This check -ignores- the distinction between various watermarks,
+ * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ...  If a zone is
+ * found to be full for any variation of these watermarks, it will
+ * be considered full for up to one second by all requests, unless
+ * we are so low on memory on all allowed nodes that we are forced
+ * into the second scan of the zonelist.
+ *
+ * In the second scan we ignore this zonelist cache and exactly
+ * apply the watermarks to all zones, even it is slower to do so.
+ * We are low on memory in the second scan, and should leave no stone
+ * unturned looking for a free page.
+ */
+static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zone **z,
+                                                nodemask_t *allowednodes)
+{
+        struct zonelist_cache *zlc;     /* cached zonelist speedup info */
+        int i;                          /* index of *z in zonelist zones */
+        int n;                          /* node that zone *z is on */
+
+        zlc = zonelist->zlcache_ptr;
+        if (!zlc)
+                return 1;
+
+        i = z - zonelist->zones;
+        n = zlc->z_to_n[i];
+
+        /* This zone is worth trying if it is allowed but not full */
+        return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones);
+}
+
 /*
- * get_page_from_freeliest goes through the zonelist trying to allocate
+ * Given 'z' scanning a zonelist, set the corresponding bit in
+ * zlc->fullzones, so that subsequent attempts to allocate a page
+ * from that zone don't waste time re-examining it.
+ */
+static void zlc_mark_zone_full(struct zonelist *zonelist, struct zone **z)
+{
+        struct zonelist_cache *zlc;     /* cached zonelist speedup info */
+        int i;                          /* index of *z in zonelist zones */
+
+        zlc = zonelist->zlcache_ptr;
+        if (!zlc)
+                return;
+
+        i = z - zonelist->zones;
+
+        set_bit(i, zlc->fullzones);
+}
+
+#else   /* CONFIG_NUMA */
+
+static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
+{
+        return NULL;
+}
+
+static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zone **z,
+                                nodemask_t *allowednodes)
+{
+        return 1;
+}
+
+static void zlc_mark_zone_full(struct zonelist *zonelist, struct zone **z)
+{
+}
+#endif  /* CONFIG_NUMA */
+
+/*
+ * get_page_from_freelist goes through the zonelist trying to allocate
  * a page.
  */
 static struct page *
 get_page_from_freelist(gfp_t gfp_mask, unsigned int order,
                 struct zonelist *zonelist, int alloc_flags)
 {
-        struct zone **z = zonelist->zones;
+        struct zone **z;
         struct page *page = NULL;
-        int classzone_idx = zone_idx(*z);
+        int classzone_idx = zone_idx(zonelist->zones[0]);
         struct zone *zone;
+        nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
+        int zlc_active = 0;             /* set if using zonelist_cache */
+        int did_zlc_setup = 0;          /* just call zlc_setup() one time */
 
+zonelist_scan:
         /*
-         * Go through the zonelist once, looking for a zone with enough free.
+         * Scan zonelist, looking for a zone with enough free.
          * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
          */
+        z = zonelist->zones;
+
         do {
+                if (NUMA_BUILD && zlc_active &&
+                        !zlc_zone_worth_trying(zonelist, z, allowednodes))
+                                continue;
                 zone = *z;
                 if (unlikely(NUMA_BUILD && (gfp_mask & __GFP_THISNODE) &&
                         zone->zone_pgdat != zonelist->zones[0]->zone_pgdat))
                                 break;
                 if ((alloc_flags & ALLOC_CPUSET) &&
-                                !cpuset_zone_allowed(zone, gfp_mask))
-                        continue;
+                        !cpuset_zone_allowed(zone, gfp_mask))
+                                goto try_next_zone;
 
                 if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
                         unsigned long mark;
@@ -959,18 +1087,34 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order,
                                 mark = zone->pages_low;
                         else
                                 mark = zone->pages_high;
-                        if (!zone_watermark_ok(zone , order, mark,
-                                    classzone_idx, alloc_flags))
+                        if (!zone_watermark_ok(zone, order, mark,
+                                    classzone_idx, alloc_flags)) {
                                 if (!zone_reclaim_mode ||
                                     !zone_reclaim(zone, gfp_mask, order))
-                                        continue;
+                                        goto this_zone_full;
+                        }
                 }
 
                 page = buffered_rmqueue(zonelist, zone, order, gfp_mask);
-                if (page) {
+                if (page)
                         break;
+this_zone_full:
+                if (NUMA_BUILD)
+                        zlc_mark_zone_full(zonelist, z);
+try_next_zone:
+                if (NUMA_BUILD && !did_zlc_setup) {
+                        /* we do zlc_setup after the first zone is tried */
+                        allowednodes = zlc_setup(zonelist, alloc_flags);
+                        zlc_active = 1;
+                        did_zlc_setup = 1;
                 }
         } while (*(++z) != NULL);
+
+        if (unlikely(NUMA_BUILD && page == NULL && zlc_active)) {
+                /* Disable zlc cache for second zonelist scan */
+                zlc_active = 0;
+                goto zonelist_scan;
+        }
         return page;
 }
 
@@ -1005,9 +1149,19 @@ restart:
         if (page)
                 goto got_pg;
 
-        do {
+        /*
+         * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
+         * __GFP_NOWARN set) should not cause reclaim since the subsystem
+         * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim
+         * using a larger set of nodes after it has established that the
+         * allowed per node queues are empty and that nodes are
+         * over allocated.
+         */
+        if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
+                goto nopage;
+
+        for (z = zonelist->zones; *z; z++)
                 wakeup_kswapd(*z, order);
-        } while (*(++z));
 
         /*
          * OK, we're below the kswapd watermark and have kicked background
@@ -1041,6 +1195,7 @@ restart:
 
         /* This allocation should allow future memory freeing. */
 
+rebalance:
         if (((p->flags & PF_MEMALLOC) || unlikely(test_thread_flag(TIF_MEMDIE)))
                         && !in_interrupt()) {
                 if (!(gfp_mask & __GFP_NOMEMALLOC)) {
@@ -1062,7 +1217,6 @@ nofail_alloc:
         if (!wait)
                 goto nopage;
 
-rebalance:
         cond_resched();
 
         /* We now go into synchronous reclaim */
@@ -1262,7 +1416,7 @@ unsigned int nr_free_pagecache_pages(void)
 static inline void show_node(struct zone *zone)
 {
         if (NUMA_BUILD)
-                printk("Node %ld ", zone_to_nid(zone));
+                printk("Node %d ", zone_to_nid(zone));
 }
 
 void si_meminfo(struct sysinfo *val)
@@ -1542,6 +1696,24 @@ static void __meminit build_zonelists(pg_data_t *pgdat)
         }
 }
 
+/* Construct the zonelist performance cache - see further mmzone.h */
+static void __meminit build_zonelist_cache(pg_data_t *pgdat)
+{
+        int i;
+
+        for (i = 0; i < MAX_NR_ZONES; i++) {
+                struct zonelist *zonelist;
+                struct zonelist_cache *zlc;
+                struct zone **z;
+
+                zonelist = pgdat->node_zonelists + i;
+                zonelist->zlcache_ptr = zlc = &zonelist->zlcache;
+                bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
+                for (z = zonelist->zones; *z; z++)
+                        zlc->z_to_n[z - zonelist->zones] = zone_to_nid(*z);
+        }
+}
+
 #else   /* CONFIG_NUMA */
 
 static void __meminit build_zonelists(pg_data_t *pgdat)
@@ -1579,14 +1751,26 @@ static void __meminit build_zonelists(pg_data_t *pgdat)
         }
 }
 
+/* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */
+static void __meminit build_zonelist_cache(pg_data_t *pgdat)
+{
+        int i;
+
+        for (i = 0; i < MAX_NR_ZONES; i++)
+                pgdat->node_zonelists[i].zlcache_ptr = NULL;
+}
+
 #endif  /* CONFIG_NUMA */
 
 /* return values int ....just for stop_machine_run() */
 static int __meminit __build_all_zonelists(void *dummy)
 {
         int nid;
-        for_each_online_node(nid)
+
+        for_each_online_node(nid) {
                 build_zonelists(NODE_DATA(nid));
+                build_zonelist_cache(NODE_DATA(nid));
+        }
         return 0;
 }
 
@@ -1715,20 +1899,6 @@ void zone_init_free_lists(struct pglist_data *pgdat, struct zone *zone,
         }
 }
 
-#define ZONETABLE_INDEX(x, zone_nr)     ((x << ZONES_SHIFT) | zone_nr)
-void zonetable_add(struct zone *zone, int nid, enum zone_type zid,
-                unsigned long pfn, unsigned long size)
-{
-        unsigned long snum = pfn_to_section_nr(pfn);
-        unsigned long end = pfn_to_section_nr(pfn + size);
-
-        if (FLAGS_HAS_NODE)
-                zone_table[ZONETABLE_INDEX(nid, zid)] = zone;
-        else
-                for (; snum <= end; snum++)
-                        zone_table[ZONETABLE_INDEX(snum, zid)] = zone;
-}
-
 #ifndef __HAVE_ARCH_MEMMAP_INIT
 #define memmap_init(size, nid, zone, start_pfn) \
         memmap_init_zone((size), (nid), (zone), (start_pfn))
@@ -1881,16 +2051,16 @@ static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb,
         int ret = NOTIFY_OK;
 
         switch (action) {
-                case CPU_UP_PREPARE:
-                        if (process_zones(cpu))
-                                ret = NOTIFY_BAD;
-                        break;
-                case CPU_UP_CANCELED:
-                case CPU_DEAD:
-                        free_zone_pagesets(cpu);
-                        break;
-                default:
-                        break;
+        case CPU_UP_PREPARE:
+                if (process_zones(cpu))
+                        ret = NOTIFY_BAD;
+                break;
+        case CPU_UP_CANCELED:
+        case CPU_DEAD:
+                free_zone_pagesets(cpu);
+                break;
+        default:
+                break;
         }
         return ret;
 }
@@ -2421,7 +2591,6 @@ static void __meminit free_area_init_core(struct pglist_data *pgdat,
                 if (!size)
                         continue;
 
-                zonetable_add(zone, nid, j, zone_start_pfn, size);
                 ret = init_currently_empty_zone(zone, zone_start_pfn, size);
                 BUG_ON(ret);
                 zone_start_pfn += size;
@@ -2736,7 +2905,6 @@ void __init free_area_init(unsigned long *zones_size)
                         __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
 static int page_alloc_cpu_notify(struct notifier_block *self,
                                  unsigned long action, void *hcpu)
 {
@@ -2751,7 +2919,6 @@ static int page_alloc_cpu_notify(struct notifier_block *self,
         }
         return NOTIFY_OK;
 }
-#endif /* CONFIG_HOTPLUG_CPU */
 
 void __init page_alloc_init(void)
 {
@@ -3055,7 +3222,7 @@ void *__init alloc_large_system_hash(const char *tablename,
         /* allow the kernel cmdline to have a say */
         if (!numentries) {
                 /* round applicable memory size up to nearest megabyte */
-                numentries = (flags & HASH_HIGHMEM) ? nr_all_pages : nr_kernel_pages;
+                numentries = nr_kernel_pages;
                 numentries += (1UL << (20 - PAGE_SHIFT)) - 1;
                 numentries >>= 20 - PAGE_SHIFT;
                 numentries <<= 20 - PAGE_SHIFT;