1 files changed, 34 insertions, 28 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index df54e2fc8ee0..791690d7d3fa 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -56,6 +56,7 @@ long nr_swap_pages;
 int percpu_pagelist_fraction;
 
 static void fastcall free_hot_cold_page(struct page *page, int cold);
+static void __free_pages_ok(struct page *page, unsigned int order);
 
 /*
  * results with 256, 32 in the lowmem_reserve sysctl:
@@ -169,20 +170,23 @@ static void bad_page(struct page *page)
  * All pages have PG_compound set.  All pages have their ->private pointing at
  * the head page (even the head page has this).
  *
- * The first tail page's ->mapping, if non-zero, holds the address of the
- * compound page's put_page() function.
- *
- * The order of the allocation is stored in the first tail page's ->index
- * This is only for debug at present.  This usage means that zero-order pages
- * may not be compound.
+ * The first tail page's ->lru.next holds the address of the compound page's
+ * put_page() function.  Its ->lru.prev holds the order of allocation.
+ * This usage means that zero-order pages may not be compound.
  */
+
+static void free_compound_page(struct page *page)
+{
+        __free_pages_ok(page, (unsigned long)page[1].lru.prev);
+}
+
 static void prep_compound_page(struct page *page, unsigned long order)
 {
         int i;
         int nr_pages = 1 << order;
 
-        page[1].mapping = NULL;
-        page[1].index = order;
+        page[1].lru.next = (void *)free_compound_page;  /* set dtor */
+        page[1].lru.prev = (void *)order;
         for (i = 0; i < nr_pages; i++) {
                 struct page *p = page + i;
 
@@ -196,7 +200,7 @@ static void destroy_compound_page(struct page *page, unsigned long order)
         int i;
         int nr_pages = 1 << order;
 
-        if (unlikely(page[1].index != order))
+        if (unlikely((unsigned long)page[1].lru.prev != order))
                 bad_page(page);
 
         for (i = 0; i < nr_pages; i++) {
@@ -1011,7 +1015,7 @@ rebalance:
                 if (page)
                         goto got_pg;
 
-                out_of_memory(gfp_mask, order);
+                out_of_memory(zonelist, gfp_mask, order);
                 goto restart;
         }
 
@@ -1213,18 +1217,21 @@ static void __get_page_state(struct page_state *ret, int nr, cpumask_t *cpumask)
 {
         int cpu = 0;
 
-        memset(ret, 0, sizeof(*ret));
+        memset(ret, 0, nr * sizeof(unsigned long));
         cpus_and(*cpumask, *cpumask, cpu_online_map);
 
         cpu = first_cpu(*cpumask);
         while (cpu < NR_CPUS) {
                 unsigned long *in, *out, off;
 
+                if (!cpu_isset(cpu, *cpumask))
+                        continue;
+
                 in = (unsigned long *)&per_cpu(page_states, cpu);
 
                 cpu = next_cpu(cpu, *cpumask);
 
-                if (cpu < NR_CPUS)
+                if (likely(cpu < NR_CPUS))
                         prefetch(&per_cpu(page_states, cpu));
 
                 out = (unsigned long *)ret;
@@ -1534,29 +1541,29 @@ static int __initdata node_load[MAX_NUMNODES];
  */
 static int __init find_next_best_node(int node, nodemask_t *used_node_mask)
 {
-        int i, n, val;
+        int n, val;
         int min_val = INT_MAX;
         int best_node = -1;
 
-        for_each_online_node(i) {
-                cpumask_t tmp;
+        /* Use the local node if we haven't already */
+        if (!node_isset(node, *used_node_mask)) {
+                node_set(node, *used_node_mask);
+                return node;
+        }
 
-                /* Start from local node */
-                n = (node+i) % num_online_nodes();
+        for_each_online_node(n) {
+                cpumask_t tmp;
 
                 /* Don't want a node to appear more than once */
                 if (node_isset(n, *used_node_mask))
                         continue;
 
-                /* Use the local node if we haven't already */
-                if (!node_isset(node, *used_node_mask)) {
-                        best_node = node;
-                        break;
-                }
-
                 /* Use the distance array to find the distance */
                 val = node_distance(node, n);
 
+                /* Penalize nodes under us ("prefer the next node") */
+                val += (n < node);
+
                 /* Give preference to headless and unused nodes */
                 tmp = node_to_cpumask(n);
                 if (!cpus_empty(tmp))
@@ -1799,7 +1806,7 @@ void zonetable_add(struct zone *zone, int nid, int zid, unsigned long pfn,
         memmap_init_zone((size), (nid), (zone), (start_pfn))
 #endif
 
-static int __meminit zone_batchsize(struct zone *zone)
+static int __cpuinit zone_batchsize(struct zone *zone)
 {
         int batch;
 
@@ -1886,14 +1893,13 @@ static void setup_pagelist_highmark(struct per_cpu_pageset *p,
  * not check if the processor is online before following the pageset pointer.
  * Other parts of the kernel may not check if the zone is available.
  */
-static struct per_cpu_pageset
-        boot_pageset[NR_CPUS];
+static struct per_cpu_pageset boot_pageset[NR_CPUS];
 
 /*
  * Dynamically allocate memory for the
  * per cpu pageset array in struct zone.
  */
-static int __meminit process_zones(int cpu)
+static int __cpuinit process_zones(int cpu)
 {
         struct zone *zone, *dzone;
 
@@ -1934,7 +1940,7 @@ static inline void free_zone_pagesets(int cpu)
         }
 }
 
-static int __meminit pageset_cpuup_callback(struct notifier_block *nfb,
+static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb,
                 unsigned long action,
                 void *hcpu)
 {