12 files changed, 637 insertions, 211 deletions

diff --git a/mm/Kconfig b/mm/Kconfig
index 17b8947aa7da..9c61158308dc 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -115,6 +115,10 @@ config SPARSEMEM_EXTREME
 config SPARSEMEM_VMEMMAP_ENABLE
         bool
 
+config SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
+        def_bool y
+        depends on SPARSEMEM && X86_64
+
 config SPARSEMEM_VMEMMAP
         bool "Sparse Memory virtual memmap"
         depends on SPARSEMEM && SPARSEMEM_VMEMMAP_ENABLE
@@ -195,7 +199,7 @@ config BOUNCE
 config NR_QUICK
         int
         depends on QUICKLIST
-        default "2" if SUPERH || AVR32
+        default "2" if AVR32
         default "1"
 
 config VIRT_TO_BUS
diff --git a/mm/bootmem.c b/mm/bootmem.c
index 7d1486875e1c..d7c791ef0036 100644
--- a/mm/bootmem.c
+++ b/mm/bootmem.c
@@ -13,6 +13,7 @@
 #include <linux/bootmem.h>
 #include <linux/module.h>
 #include <linux/kmemleak.h>
+#include <linux/range.h>
 
 #include <asm/bug.h>
 #include <asm/io.h>
@@ -32,6 +33,7 @@ unsigned long max_pfn;
 unsigned long saved_max_pfn;
 #endif
 
+#ifndef CONFIG_NO_BOOTMEM
 bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
 
 static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
@@ -142,7 +144,7 @@ unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
         min_low_pfn = start;
         return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
 }
-
+#endif
 /*
  * free_bootmem_late - free bootmem pages directly to page allocator
  * @addr: starting address of the range
@@ -167,6 +169,60 @@ void __init free_bootmem_late(unsigned long addr, unsigned long size)
         }
 }
 
+#ifdef CONFIG_NO_BOOTMEM
+static void __init __free_pages_memory(unsigned long start, unsigned long end)
+{
+        int i;
+        unsigned long start_aligned, end_aligned;
+        int order = ilog2(BITS_PER_LONG);
+
+        start_aligned = (start + (BITS_PER_LONG - 1)) & ~(BITS_PER_LONG - 1);
+        end_aligned = end & ~(BITS_PER_LONG - 1);
+
+        if (end_aligned <= start_aligned) {
+#if 1
+                printk(KERN_DEBUG " %lx - %lx\n", start, end);
+#endif
+                for (i = start; i < end; i++)
+                        __free_pages_bootmem(pfn_to_page(i), 0);
+
+                return;
+        }
+
+#if 1
+        printk(KERN_DEBUG " %lx %lx - %lx %lx\n",
+                 start, start_aligned, end_aligned, end);
+#endif
+        for (i = start; i < start_aligned; i++)
+                __free_pages_bootmem(pfn_to_page(i), 0);
+
+        for (i = start_aligned; i < end_aligned; i += BITS_PER_LONG)
+                __free_pages_bootmem(pfn_to_page(i), order);
+
+        for (i = end_aligned; i < end; i++)
+                __free_pages_bootmem(pfn_to_page(i), 0);
+}
+
+unsigned long __init free_all_memory_core_early(int nodeid)
+{
+        int i;
+        u64 start, end;
+        unsigned long count = 0;
+        struct range *range = NULL;
+        int nr_range;
+
+        nr_range = get_free_all_memory_range(&range, nodeid);
+
+        for (i = 0; i < nr_range; i++) {
+                start = range[i].start;
+                end = range[i].end;
+                count += end - start;
+                __free_pages_memory(start, end);
+        }
+
+        return count;
+}
+#else
 static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
 {
         int aligned;
@@ -227,6 +283,7 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
 
         return count;
 }
+#endif
 
 /**
  * free_all_bootmem_node - release a node's free pages to the buddy allocator
@@ -237,7 +294,12 @@ static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
 unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
 {
         register_page_bootmem_info_node(pgdat);
+#ifdef CONFIG_NO_BOOTMEM
+        /* free_all_memory_core_early(MAX_NUMNODES) will be called later */
+        return 0;
+#else
         return free_all_bootmem_core(pgdat->bdata);
+#endif
 }
 
 /**
@@ -247,9 +309,14 @@ unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
  */
 unsigned long __init free_all_bootmem(void)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        return free_all_memory_core_early(NODE_DATA(0)->node_id);
+#else
         return free_all_bootmem_core(NODE_DATA(0)->bdata);
+#endif
 }
 
+#ifndef CONFIG_NO_BOOTMEM
 static void __init __free(bootmem_data_t *bdata,
                         unsigned long sidx, unsigned long eidx)
 {
@@ -344,6 +411,7 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
         }
         BUG();
 }
+#endif
 
 /**
  * free_bootmem_node - mark a page range as usable
@@ -358,6 +426,12 @@ static int __init mark_bootmem(unsigned long start, unsigned long end,
 void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
                               unsigned long size)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        free_early(physaddr, physaddr + size);
+#if 0
+        printk(KERN_DEBUG "free %lx %lx\n", physaddr, size);
+#endif
+#else
         unsigned long start, end;
 
         kmemleak_free_part(__va(physaddr), size);
@@ -366,6 +440,7 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
         end = PFN_DOWN(physaddr + size);
 
         mark_bootmem_node(pgdat->bdata, start, end, 0, 0);
+#endif
 }
 
 /**
@@ -379,6 +454,12 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
  */
 void __init free_bootmem(unsigned long addr, unsigned long size)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        free_early(addr, addr + size);
+#if 0
+        printk(KERN_DEBUG "free %lx %lx\n", addr, size);
+#endif
+#else
         unsigned long start, end;
 
         kmemleak_free_part(__va(addr), size);
@@ -387,6 +468,7 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
         end = PFN_DOWN(addr + size);
 
         mark_bootmem(start, end, 0, 0);
+#endif
 }
 
 /**
@@ -403,12 +485,17 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
 int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
                                  unsigned long size, int flags)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        panic("no bootmem");
+        return 0;
+#else
         unsigned long start, end;
 
         start = PFN_DOWN(physaddr);
         end = PFN_UP(physaddr + size);
 
         return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
+#endif
 }
 
 /**
@@ -424,14 +511,20 @@ int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
 int __init reserve_bootmem(unsigned long addr, unsigned long size,
                             int flags)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        panic("no bootmem");
+        return 0;
+#else
         unsigned long start, end;
 
         start = PFN_DOWN(addr);
         end = PFN_UP(addr + size);
 
         return mark_bootmem(start, end, 1, flags);
+#endif
 }
 
+#ifndef CONFIG_NO_BOOTMEM
 static unsigned long __init align_idx(struct bootmem_data *bdata,
                                       unsigned long idx, unsigned long step)
 {
@@ -582,12 +675,33 @@ static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
 #endif
         return NULL;
 }
+#endif
 
 static void * __init ___alloc_bootmem_nopanic(unsigned long size,
                                         unsigned long align,
                                         unsigned long goal,
                                         unsigned long limit)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        void *ptr;
+
+        if (WARN_ON_ONCE(slab_is_available()))
+                return kzalloc(size, GFP_NOWAIT);
+
+restart:
+
+        ptr = __alloc_memory_core_early(MAX_NUMNODES, size, align, goal, limit);
+
+        if (ptr)
+                return ptr;
+
+        if (goal != 0) {
+                goal = 0;
+                goto restart;
+        }
+
+        return NULL;
+#else
         bootmem_data_t *bdata;
         void *region;
 
@@ -613,6 +727,7 @@ restart:
         }
 
         return NULL;
+#endif
 }
 
 /**
@@ -631,7 +746,13 @@ restart:
 void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
                                         unsigned long goal)
 {
-        return ___alloc_bootmem_nopanic(size, align, goal, 0);
+        unsigned long limit = 0;
+
+#ifdef CONFIG_NO_BOOTMEM
+        limit = -1UL;
+#endif
+
+        return ___alloc_bootmem_nopanic(size, align, goal, limit);
 }
 
 static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
@@ -665,9 +786,16 @@ static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
 void * __init __alloc_bootmem(unsigned long size, unsigned long align,
                               unsigned long goal)
 {
-        return ___alloc_bootmem(size, align, goal, 0);
+        unsigned long limit = 0;
+
+#ifdef CONFIG_NO_BOOTMEM
+        limit = -1UL;
+#endif
+
+        return ___alloc_bootmem(size, align, goal, limit);
 }
 
+#ifndef CONFIG_NO_BOOTMEM
 static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
                                 unsigned long size, unsigned long align,
                                 unsigned long goal, unsigned long limit)
@@ -684,6 +812,7 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
 
         return ___alloc_bootmem(size, align, goal, limit);
 }
+#endif
 
 /**
  * __alloc_bootmem_node - allocate boot memory from a specific node
@@ -706,7 +835,46 @@ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
         if (WARN_ON_ONCE(slab_is_available()))
                 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
 
+#ifdef CONFIG_NO_BOOTMEM
+        return __alloc_memory_core_early(pgdat->node_id, size, align,
+                                         goal, -1ULL);
+#else
         return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
+#endif
+}
+
+void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
+                                   unsigned long align, unsigned long goal)
+{
+#ifdef MAX_DMA32_PFN
+        unsigned long end_pfn;
+
+        if (WARN_ON_ONCE(slab_is_available()))
+                return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
+        /* update goal according ...MAX_DMA32_PFN */
+        end_pfn = pgdat->node_start_pfn + pgdat->node_spanned_pages;
+
+        if (end_pfn > MAX_DMA32_PFN + (128 >> (20 - PAGE_SHIFT)) &&
+            (goal >> PAGE_SHIFT) < MAX_DMA32_PFN) {
+                void *ptr;
+                unsigned long new_goal;
+
+                new_goal = MAX_DMA32_PFN << PAGE_SHIFT;
+#ifdef CONFIG_NO_BOOTMEM
+                ptr =  __alloc_memory_core_early(pgdat->node_id, size, align,
+                                                 new_goal, -1ULL);
+#else
+                ptr = alloc_bootmem_core(pgdat->bdata, size, align,
+                                                 new_goal, 0);
+#endif
+                if (ptr)
+                        return ptr;
+        }
+#endif
+
+        return __alloc_bootmem_node(pgdat, size, align, goal);
+
 }
 
 #ifdef CONFIG_SPARSEMEM
@@ -720,6 +888,16 @@ void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
 void * __init alloc_bootmem_section(unsigned long size,
                                     unsigned long section_nr)
 {
+#ifdef CONFIG_NO_BOOTMEM
+        unsigned long pfn, goal, limit;
+
+        pfn = section_nr_to_pfn(section_nr);
+        goal = pfn << PAGE_SHIFT;
+        limit = section_nr_to_pfn(section_nr + 1) << PAGE_SHIFT;
+
+        return __alloc_memory_core_early(early_pfn_to_nid(pfn), size,
+                                         SMP_CACHE_BYTES, goal, limit);
+#else
         bootmem_data_t *bdata;
         unsigned long pfn, goal, limit;
 
@@ -729,6 +907,7 @@ void * __init alloc_bootmem_section(unsigned long size,
         bdata = &bootmem_node_data[early_pfn_to_nid(pfn)];
 
         return alloc_bootmem_core(bdata, size, SMP_CACHE_BYTES, goal, limit);
+#endif
 }
 #endif
 
@@ -740,11 +919,16 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
         if (WARN_ON_ONCE(slab_is_available()))
                 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
 
+#ifdef CONFIG_NO_BOOTMEM
+        ptr =  __alloc_memory_core_early(pgdat->node_id, size, align,
+                                                 goal, -1ULL);
+#else
         ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
         if (ptr)
                 return ptr;
 
         ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
+#endif
         if (ptr)
                 return ptr;
 
@@ -795,6 +979,11 @@ void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
         if (WARN_ON_ONCE(slab_is_available()))
                 return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
 
+#ifdef CONFIG_NO_BOOTMEM
+        return __alloc_memory_core_early(pgdat->node_id, size, align,
+                                goal, ARCH_LOW_ADDRESS_LIMIT);
+#else
         return ___alloc_bootmem_node(pgdat->bdata, size, align,
                                 goal, ARCH_LOW_ADDRESS_LIMIT);
+#endif
 }
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 2d16fa6b8c2d..3a5aeb37c110 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -2087,7 +2087,7 @@ static void set_huge_ptep_writable(struct vm_area_struct *vma,
 
         entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
         if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) {
-                update_mmu_cache(vma, address, entry);
+                update_mmu_cache(vma, address, ptep);
         }
 }
 
@@ -2558,7 +2558,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
         entry = pte_mkyoung(entry);
         if (huge_ptep_set_access_flags(vma, address, ptep, entry,
                                                 flags & FAULT_FLAG_WRITE))
-                update_mmu_cache(vma, address, entry);
+                update_mmu_cache(vma, address, ptep);
 
 out_page_table_lock:
         spin_unlock(&mm->page_table_lock);
diff --git a/mm/memory.c b/mm/memory.c
index 09e4b1be7b67..72fb5f39bccc 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1593,7 +1593,7 @@ static int insert_pfn(struct vm_area_struct *vma, unsigned long addr,
         /* Ok, finally just insert the thing.. */
         entry = pte_mkspecial(pfn_pte(pfn, prot));
         set_pte_at(mm, addr, pte, entry);
-        update_mmu_cache(vma, addr, entry); /* XXX: why not for insert_page? */
+        update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */
 
         retval = 0;
 out_unlock:
@@ -2116,7 +2116,7 @@ reuse:
                 entry = pte_mkyoung(orig_pte);
                 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
                 if (ptep_set_access_flags(vma, address, page_table, entry,1))
-                        update_mmu_cache(vma, address, entry);
+                        update_mmu_cache(vma, address, page_table);
                 ret |= VM_FAULT_WRITE;
                 goto unlock;
         }
@@ -2185,7 +2185,7 @@ gotten:
                  * new page to be mapped directly into the secondary page table.
                  */
                 set_pte_at_notify(mm, address, page_table, entry);
-                update_mmu_cache(vma, address, entry);
+                update_mmu_cache(vma, address, page_table);
                 if (old_page) {
                         /*
                          * Only after switching the pte to the new page may
@@ -2629,7 +2629,7 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
         }
 
         /* No need to invalidate - it was non-present before */
-        update_mmu_cache(vma, address, pte);
+        update_mmu_cache(vma, address, page_table);
 unlock:
         pte_unmap_unlock(page_table, ptl);
 out:
@@ -2694,7 +2694,7 @@ setpte:
         set_pte_at(mm, address, page_table, entry);
 
         /* No need to invalidate - it was non-present before */
-        update_mmu_cache(vma, address, entry);
+        update_mmu_cache(vma, address, page_table);
 unlock:
         pte_unmap_unlock(page_table, ptl);
         return 0;
@@ -2855,7 +2855,7 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
                 set_pte_at(mm, address, page_table, entry);
 
                 /* no need to invalidate: a not-present page won't be cached */
-                update_mmu_cache(vma, address, entry);
+                update_mmu_cache(vma, address, page_table);
         } else {
                 if (charged)
                         mem_cgroup_uncharge_page(page);
@@ -2992,7 +2992,7 @@ static inline int handle_pte_fault(struct mm_struct *mm,
         }
         entry = pte_mkyoung(entry);
         if (ptep_set_access_flags(vma, address, pte, entry, flags & FAULT_FLAG_WRITE)) {
-                update_mmu_cache(vma, address, entry);
+                update_mmu_cache(vma, address, pte);
         } else {
                 /*
                  * This is needed only for protection faults but the arch code
diff --git a/mm/migrate.c b/mm/migrate.c
index 9a0db5bbabe4..edb6101ed774 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -134,7 +134,7 @@ static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
                 page_add_file_rmap(new);
 
         /* No need to invalidate - it was non-present before */
-        update_mmu_cache(vma, addr, pte);
+        update_mmu_cache(vma, addr, ptep);
 unlock:
         pte_unmap_unlock(ptep, ptl);
 out:
@@ -1002,33 +1002,27 @@ static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
 #define DO_PAGES_STAT_CHUNK_NR 16
         const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
         int chunk_status[DO_PAGES_STAT_CHUNK_NR];
-        unsigned long i, chunk_nr = DO_PAGES_STAT_CHUNK_NR;
-        int err;
 
-        for (i = 0; i < nr_pages; i += chunk_nr) {
-                if (chunk_nr > nr_pages - i)
-                        chunk_nr = nr_pages - i;
+        while (nr_pages) {
+                unsigned long chunk_nr;
 
-                err = copy_from_user(chunk_pages, &pages[i],
-                                     chunk_nr * sizeof(*chunk_pages));
-                if (err) {
-                        err = -EFAULT;
-                        goto out;
-                }
+                chunk_nr = nr_pages;
+                if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
+                        chunk_nr = DO_PAGES_STAT_CHUNK_NR;
+
+                if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
+                        break;
 
                 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
 
-                err = copy_to_user(&status[i], chunk_status,
-                                   chunk_nr * sizeof(*chunk_status));
-                if (err) {
-                        err = -EFAULT;
-                        goto out;
-                }
-        }
-        err = 0;
+                if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
+                        break;
 
-out:
-        return err;
+                pages += chunk_nr;
+                status += chunk_nr;
+                nr_pages -= chunk_nr;
+        }
+        return nr_pages ? -EFAULT : 0;
 }
 
 /*
diff --git a/mm/mmu_context.c b/mm/mmu_context.c
index ded9081f4021..0777654147c9 100644
--- a/mm/mmu_context.c
+++ b/mm/mmu_context.c
@@ -5,6 +5,7 @@
 
 #include <linux/mm.h>
 #include <linux/mmu_context.h>
+#include <linux/module.h>
 #include <linux/sched.h>
 
 #include <asm/mmu_context.h>
@@ -37,6 +38,7 @@ void use_mm(struct mm_struct *mm)
         if (active_mm != mm)
                 mmdrop(active_mm);
 }
+EXPORT_SYMBOL_GPL(use_mm);
 
 /*
  * unuse_mm
@@ -56,3 +58,4 @@ void unuse_mm(struct mm_struct *mm)
         enter_lazy_tlb(mm, tsk);
         task_unlock(tsk);
 }
+EXPORT_SYMBOL_GPL(unuse_mm);
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index f52481b1c1e5..237050478f28 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -459,6 +459,8 @@ static int oom_kill_process(struct task_struct *p, gfp_t gfp_mask, int order,
         list_for_each_entry(c, &p->children, sibling) {
                 if (c->mm == p->mm)
                         continue;
+                if (mem && !task_in_mem_cgroup(c, mem))
+                        continue;
                 if (!oom_kill_task(c))
                         return 0;
         }
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 8deb9d0fd5b1..a6b17aa4740b 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1009,10 +1009,10 @@ static void drain_pages(unsigned int cpu)
                 struct per_cpu_pageset *pset;
                 struct per_cpu_pages *pcp;
 
-                pset = zone_pcp(zone, cpu);
+                local_irq_save(flags);
+                pset = per_cpu_ptr(zone->pageset, cpu);
 
                 pcp = &pset->pcp;
-                local_irq_save(flags);
                 free_pcppages_bulk(zone, pcp->count, pcp);
                 pcp->count = 0;
                 local_irq_restore(flags);
@@ -1096,7 +1096,6 @@ static void free_hot_cold_page(struct page *page, int cold)
         arch_free_page(page, 0);
         kernel_map_pages(page, 1, 0);
 
-        pcp = &zone_pcp(zone, get_cpu())->pcp;
         migratetype = get_pageblock_migratetype(page);
         set_page_private(page, migratetype);
         local_irq_save(flags);
@@ -1119,6 +1118,7 @@ static void free_hot_cold_page(struct page *page, int cold)
                 migratetype = MIGRATE_MOVABLE;
         }
 
+        pcp = &this_cpu_ptr(zone->pageset)->pcp;
         if (cold)
                 list_add_tail(&page->lru, &pcp->lists[migratetype]);
         else
@@ -1131,7 +1131,6 @@ static void free_hot_cold_page(struct page *page, int cold)
 
 out:
         local_irq_restore(flags);
-        put_cpu();
 }
 
 void free_hot_page(struct page *page)
@@ -1181,17 +1180,15 @@ struct page *buffered_rmqueue(struct zone *preferred_zone,
         unsigned long flags;
         struct page *page;
         int cold = !!(gfp_flags & __GFP_COLD);
-        int cpu;
 
 again:
-        cpu  = get_cpu();
         if (likely(order == 0)) {
                 struct per_cpu_pages *pcp;
                 struct list_head *list;
 
-                pcp = &zone_pcp(zone, cpu)->pcp;
-                list = &pcp->lists[migratetype];
                 local_irq_save(flags);
+                pcp = &this_cpu_ptr(zone->pageset)->pcp;
+                list = &pcp->lists[migratetype];
                 if (list_empty(list)) {
                         pcp->count += rmqueue_bulk(zone, 0,
                                         pcp->batch, list,
@@ -1232,7 +1229,6 @@ again:
         __count_zone_vm_events(PGALLOC, zone, 1 << order);
         zone_statistics(preferred_zone, zone);
         local_irq_restore(flags);
-        put_cpu();
 
         VM_BUG_ON(bad_range(zone, page));
         if (prep_new_page(page, order, gfp_flags))
@@ -1241,7 +1237,6 @@ again:
 
 failed:
         local_irq_restore(flags);
-        put_cpu();
         return NULL;
 }
 
@@ -2180,7 +2175,7 @@ void show_free_areas(void)
                 for_each_online_cpu(cpu) {
                         struct per_cpu_pageset *pageset;
 
-                        pageset = zone_pcp(zone, cpu);
+                        pageset = per_cpu_ptr(zone->pageset, cpu);
 
                         printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
                                cpu, pageset->pcp.high,
@@ -2745,10 +2740,29 @@ static void build_zonelist_cache(pg_data_t *pgdat)
 
 #endif  /* CONFIG_NUMA */
 
+/*
+ * Boot pageset table. One per cpu which is going to be used for all
+ * zones and all nodes. The parameters will be set in such a way
+ * that an item put on a list will immediately be handed over to
+ * the buddy list. This is safe since pageset manipulation is done
+ * with interrupts disabled.
+ *
+ * The boot_pagesets must be kept even after bootup is complete for
+ * unused processors and/or zones. They do play a role for bootstrapping
+ * hotplugged processors.
+ *
+ * zoneinfo_show() and maybe other functions do
+ * not check if the processor is online before following the pageset pointer.
+ * Other parts of the kernel may not check if the zone is available.
+ */
+static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
+static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
+
 /* return values int ....just for stop_machine() */
 static int __build_all_zonelists(void *dummy)
 {
         int nid;
+        int cpu;
 
 #ifdef CONFIG_NUMA
         memset(node_load, 0, sizeof(node_load));
@@ -2759,6 +2773,23 @@ static int __build_all_zonelists(void *dummy)
                 build_zonelists(pgdat);
                 build_zonelist_cache(pgdat);
         }
+
+        /*
+         * Initialize the boot_pagesets that are going to be used
+         * for bootstrapping processors. The real pagesets for
+         * each zone will be allocated later when the per cpu
+         * allocator is available.
+         *
+         * boot_pagesets are used also for bootstrapping offline
+         * cpus if the system is already booted because the pagesets
+         * are needed to initialize allocators on a specific cpu too.
+         * F.e. the percpu allocator needs the page allocator which
+         * needs the percpu allocator in order to allocate its pagesets
+         * (a chicken-egg dilemma).
+         */
+        for_each_possible_cpu(cpu)
+                setup_pageset(&per_cpu(boot_pageset, cpu), 0);
+
         return 0;
 }
 
@@ -3096,121 +3127,33 @@ static void setup_pagelist_highmark(struct per_cpu_pageset *p,
                 pcp->batch = PAGE_SHIFT * 8;
 }
 
-
-#ifdef CONFIG_NUMA
-/*
- * Boot pageset table. One per cpu which is going to be used for all
- * zones and all nodes. The parameters will be set in such a way
- * that an item put on a list will immediately be handed over to
- * the buddy list. This is safe since pageset manipulation is done
- * with interrupts disabled.
- *
- * Some NUMA counter updates may also be caught by the boot pagesets.
- *
- * The boot_pagesets must be kept even after bootup is complete for
- * unused processors and/or zones. They do play a role for bootstrapping
- * hotplugged processors.
- *
- * zoneinfo_show() and maybe other functions do
- * not check if the processor is online before following the pageset pointer.
- * Other parts of the kernel may not check if the zone is available.
- */
-static struct per_cpu_pageset boot_pageset[NR_CPUS];
-
 /*
- * Dynamically allocate memory for the
- * per cpu pageset array in struct zone.
+ * Allocate per cpu pagesets and initialize them.
+ * Before this call only boot pagesets were available.
+ * Boot pagesets will no longer be used by this processorr
+ * after setup_per_cpu_pageset().
  */
-static int __cpuinit process_zones(int cpu)
+void __init setup_per_cpu_pageset(void)
 {
-        struct zone *zone, *dzone;
-        int node = cpu_to_node(cpu);
-
-        node_set_state(node, N_CPU);    /* this node has a cpu */
+        struct zone *zone;
+        int cpu;
 
         for_each_populated_zone(zone) {
-                zone_pcp(zone, cpu) = kmalloc_node(sizeof(struct per_cpu_pageset),
-                                         GFP_KERNEL, node);
-                if (!zone_pcp(zone, cpu))
-                        goto bad;
+                zone->pageset = alloc_percpu(struct per_cpu_pageset);
 
-                setup_pageset(zone_pcp(zone, cpu), zone_batchsize(zone));
+                for_each_possible_cpu(cpu) {
+                        struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
 
-                if (percpu_pagelist_fraction)
-                        setup_pagelist_highmark(zone_pcp(zone, cpu),
-                            (zone->present_pages / percpu_pagelist_fraction));
-        }
+                        setup_pageset(pcp, zone_batchsize(zone));
 
-        return 0;
-bad:
-        for_each_zone(dzone) {
-                if (!populated_zone(dzone))
-                        continue;
-                if (dzone == zone)
-                        break;
-                kfree(zone_pcp(dzone, cpu));
-                zone_pcp(dzone, cpu) = &boot_pageset[cpu];
-        }
-        return -ENOMEM;
-}
-
-static inline void free_zone_pagesets(int cpu)
-{
-        struct zone *zone;
-
-        for_each_zone(zone) {
-                struct per_cpu_pageset *pset = zone_pcp(zone, cpu);
-
-                /* Free per_cpu_pageset if it is slab allocated */
-                if (pset != &boot_pageset[cpu])
-                        kfree(pset);
-                zone_pcp(zone, cpu) = &boot_pageset[cpu];
-        }
-}
-
-static int __cpuinit pageset_cpuup_callback(struct notifier_block *nfb,
-                unsigned long action,
-                void *hcpu)
-{
-        int cpu = (long)hcpu;
-        int ret = NOTIFY_OK;
-
-        switch (action) {
-        case CPU_UP_PREPARE:
-        case CPU_UP_PREPARE_FROZEN:
-                if (process_zones(cpu))
-                        ret = NOTIFY_BAD;
-                break;
-        case CPU_UP_CANCELED:
-        case CPU_UP_CANCELED_FROZEN:
-        case CPU_DEAD:
-        case CPU_DEAD_FROZEN:
-                free_zone_pagesets(cpu);
-                break;
-        default:
-                break;
+                        if (percpu_pagelist_fraction)
+                                setup_pagelist_highmark(pcp,
+                                        (zone->present_pages /
+                                                percpu_pagelist_fraction));
+                }
         }
-        return ret;
-}
-
-static struct notifier_block __cpuinitdata pageset_notifier =
-        { &pageset_cpuup_callback, NULL, 0 };
-
-void __init setup_per_cpu_pageset(void)
-{
-        int err;
-
-        /* Initialize per_cpu_pageset for cpu 0.
-         * A cpuup callback will do this for every cpu
-         * as it comes online
-         */
-        err = process_zones(smp_processor_id());
-        BUG_ON(err);
-        register_cpu_notifier(&pageset_notifier);
 }
 
-#endif
-
 static noinline __init_refok
 int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
 {
@@ -3264,7 +3207,7 @@ static int __zone_pcp_update(void *data)
                 struct per_cpu_pageset *pset;
                 struct per_cpu_pages *pcp;
 
-                pset = zone_pcp(zone, cpu);
+                pset = per_cpu_ptr(zone->pageset, cpu);
                 pcp = &pset->pcp;
 
                 local_irq_save(flags);
@@ -3282,21 +3225,17 @@ void zone_pcp_update(struct zone *zone)
 
 static __meminit void zone_pcp_init(struct zone *zone)
 {
-        int cpu;
-        unsigned long batch = zone_batchsize(zone);
+        /*
+         * per cpu subsystem is not up at this point. The following code
+         * relies on the ability of the linker to provide the
+         * offset of a (static) per cpu variable into the per cpu area.
+         */
+        zone->pageset = &boot_pageset;
 
-        for (cpu = 0; cpu < NR_CPUS; cpu++) {
-#ifdef CONFIG_NUMA
-                /* Early boot. Slab allocator not functional yet */
-                zone_pcp(zone, cpu) = &boot_pageset[cpu];
-                setup_pageset(&boot_pageset[cpu],0);
-#else
-                setup_pageset(zone_pcp(zone,cpu), batch);
-#endif
-        }
         if (zone->present_pages)
-                printk(KERN_DEBUG "  %s zone: %lu pages, LIFO batch:%lu\n",
-                        zone->name, zone->present_pages, batch);
+                printk(KERN_DEBUG "  %s zone: %lu pages, LIFO batch:%u\n",
+                        zone->name, zone->present_pages,
+                                         zone_batchsize(zone));
 }
 
 __meminit int init_currently_empty_zone(struct zone *zone,
@@ -3435,6 +3374,61 @@ void __init free_bootmem_with_active_regions(int nid,
         }
 }
 
+int __init add_from_early_node_map(struct range *range, int az,
+                                   int nr_range, int nid)
+{
+        int i;
+        u64 start, end;
+
+        /* need to go over early_node_map to find out good range for node */
+        for_each_active_range_index_in_nid(i, nid) {
+                start = early_node_map[i].start_pfn;
+                end = early_node_map[i].end_pfn;
+                nr_range = add_range(range, az, nr_range, start, end);
+        }
+        return nr_range;
+}
+
+#ifdef CONFIG_NO_BOOTMEM
+void * __init __alloc_memory_core_early(int nid, u64 size, u64 align,
+                                        u64 goal, u64 limit)
+{
+        int i;
+        void *ptr;
+
+        /* need to go over early_node_map to find out good range for node */
+        for_each_active_range_index_in_nid(i, nid) {
+                u64 addr;
+                u64 ei_start, ei_last;
+
+                ei_last = early_node_map[i].end_pfn;
+                ei_last <<= PAGE_SHIFT;
+                ei_start = early_node_map[i].start_pfn;
+                ei_start <<= PAGE_SHIFT;
+                addr = find_early_area(ei_start, ei_last,
+                                         goal, limit, size, align);
+
+                if (addr == -1ULL)
+                        continue;
+
+#if 0
+                printk(KERN_DEBUG "alloc (nid=%d %llx - %llx) (%llx - %llx) %llx %llx => %llx\n",
+                                nid,
+                                ei_start, ei_last, goal, limit, size,
+                                align, addr);
+#endif
+
+                ptr = phys_to_virt(addr);
+                memset(ptr, 0, size);
+                reserve_early_without_check(addr, addr + size, "BOOTMEM");
+                return ptr;
+        }
+
+        return NULL;
+}
+#endif
+
+
 void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data)
 {
         int i;
@@ -4467,7 +4461,11 @@ void __init set_dma_reserve(unsigned long new_dma_reserve)
 }
 
 #ifndef CONFIG_NEED_MULTIPLE_NODES
-struct pglist_data __refdata contig_page_data = { .bdata = &bootmem_node_data[0] };
+struct pglist_data __refdata contig_page_data = {
+#ifndef CONFIG_NO_BOOTMEM
+ .bdata = &bootmem_node_data[0]
+#endif
+ };
 EXPORT_SYMBOL(contig_page_data);
 #endif
 
@@ -4810,10 +4808,11 @@ int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
         if (!write || (ret == -EINVAL))
                 return ret;
         for_each_populated_zone(zone) {
-                for_each_online_cpu(cpu) {
+                for_each_possible_cpu(cpu) {
                         unsigned long  high;
                         high = zone->present_pages / percpu_pagelist_fraction;
-                        setup_pagelist_highmark(zone_pcp(zone, cpu), high);
+                        setup_pagelist_highmark(
+                                per_cpu_ptr(zone->pageset, cpu), high);
                 }
         }
         return 0;
diff --git a/mm/percpu.c b/mm/percpu.c
index 083e7c91e5f6..768419d44ad7 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -80,13 +80,15 @@
 /* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */
 #ifndef __addr_to_pcpu_ptr
 #define __addr_to_pcpu_ptr(addr)                                        \
-        (void *)((unsigned long)(addr) - (unsigned long)pcpu_base_addr  \
-                 + (unsigned long)__per_cpu_start)
+        (void __percpu *)((unsigned long)(addr) -                       \
+                          (unsigned long)pcpu_base_addr +               \
+                          (unsigned long)__per_cpu_start)
 #endif
 #ifndef __pcpu_ptr_to_addr
 #define __pcpu_ptr_to_addr(ptr)                                         \
-        (void *)((unsigned long)(ptr) + (unsigned long)pcpu_base_addr   \
-                 - (unsigned long)__per_cpu_start)
+        (void __force *)((unsigned long)(ptr) +                         \
+                         (unsigned long)pcpu_base_addr -                \
+                         (unsigned long)__per_cpu_start)
 #endif
 
 struct pcpu_chunk {
@@ -913,11 +915,10 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size)
         int rs, re;
 
         /* quick path, check whether it's empty already */
-        pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
-                if (rs == page_start && re == page_end)
-                        return;
-                break;
-        }
+        rs = page_start;
+        pcpu_next_unpop(chunk, &rs, &re, page_end);
+        if (rs == page_start && re == page_end)
+                return;
 
         /* immutable chunks can't be depopulated */
         WARN_ON(chunk->immutable);
@@ -968,11 +969,10 @@ static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
         int rs, re, rc;
 
         /* quick path, check whether all pages are already there */
-        pcpu_for_each_pop_region(chunk, rs, re, page_start, page_end) {
-                if (rs == page_start && re == page_end)
-                        goto clear;
-                break;
-        }
+        rs = page_start;
+        pcpu_next_pop(chunk, &rs, &re, page_end);
+        if (rs == page_start && re == page_end)
+                goto clear;
 
         /* need to allocate and map pages, this chunk can't be immutable */
         WARN_ON(chunk->immutable);
@@ -1067,7 +1067,7 @@ static struct pcpu_chunk *alloc_pcpu_chunk(void)
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
-static void *pcpu_alloc(size_t size, size_t align, bool reserved)
+static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
 {
         static int warn_limit = 10;
         struct pcpu_chunk *chunk;
@@ -1196,7 +1196,7 @@ fail_unlock_mutex:
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
-void *__alloc_percpu(size_t size, size_t align)
+void __percpu *__alloc_percpu(size_t size, size_t align)
 {
         return pcpu_alloc(size, align, false);
 }
@@ -1217,7 +1217,7 @@ EXPORT_SYMBOL_GPL(__alloc_percpu);
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
-void *__alloc_reserved_percpu(size_t size, size_t align)
+void __percpu *__alloc_reserved_percpu(size_t size, size_t align)
 {
         return pcpu_alloc(size, align, true);
 }
@@ -1269,7 +1269,7 @@ static void pcpu_reclaim(struct work_struct *work)
  * CONTEXT:
  * Can be called from atomic context.
  */
-void free_percpu(void *ptr)
+void free_percpu(void __percpu *ptr)
 {
         void *addr;
         struct pcpu_chunk *chunk;
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index d9714bdcb4a3..392b9bb5bc01 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -40,9 +40,11 @@ static void * __init_refok __earlyonly_bootmem_alloc(int node,
                                 unsigned long align,
                                 unsigned long goal)
 {
-        return __alloc_bootmem_node(NODE_DATA(node), size, align, goal);
+        return __alloc_bootmem_node_high(NODE_DATA(node), size, align, goal);
 }
 
+static void *vmemmap_buf;
+static void *vmemmap_buf_end;
 
 void * __meminit vmemmap_alloc_block(unsigned long size, int node)
 {
@@ -64,6 +66,24 @@ void * __meminit vmemmap_alloc_block(unsigned long size, int node)
                                 __pa(MAX_DMA_ADDRESS));
 }
 
+/* need to make sure size is all the same during early stage */
+void * __meminit vmemmap_alloc_block_buf(unsigned long size, int node)
+{
+        void *ptr;
+
+        if (!vmemmap_buf)
+                return vmemmap_alloc_block(size, node);
+
+        /* take the from buf */
+        ptr = (void *)ALIGN((unsigned long)vmemmap_buf, size);
+        if (ptr + size > vmemmap_buf_end)
+                return vmemmap_alloc_block(size, node);
+
+        vmemmap_buf = ptr + size;
+
+        return ptr;
+}
+
 void __meminit vmemmap_verify(pte_t *pte, int node,
                                 unsigned long start, unsigned long end)
 {
@@ -80,7 +100,7 @@ pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node)
         pte_t *pte = pte_offset_kernel(pmd, addr);
         if (pte_none(*pte)) {
                 pte_t entry;
-                void *p = vmemmap_alloc_block(PAGE_SIZE, node);
+                void *p = vmemmap_alloc_block_buf(PAGE_SIZE, node);
                 if (!p)
                         return NULL;
                 entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
@@ -163,3 +183,55 @@ struct page * __meminit sparse_mem_map_populate(unsigned long pnum, int nid)
 
         return map;
 }
+
+void __init sparse_mem_maps_populate_node(struct page **map_map,
+                                          unsigned long pnum_begin,
+                                          unsigned long pnum_end,
+                                          unsigned long map_count, int nodeid)
+{
+        unsigned long pnum;
+        unsigned long size = sizeof(struct page) * PAGES_PER_SECTION;
+        void *vmemmap_buf_start;
+
+        size = ALIGN(size, PMD_SIZE);
+        vmemmap_buf_start = __earlyonly_bootmem_alloc(nodeid, size * map_count,
+                         PMD_SIZE, __pa(MAX_DMA_ADDRESS));
+
+        if (vmemmap_buf_start) {
+                vmemmap_buf = vmemmap_buf_start;
+                vmemmap_buf_end = vmemmap_buf_start + size * map_count;
+        }
+
+        for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
+                struct mem_section *ms;
+
+                if (!present_section_nr(pnum))
+                        continue;
+
+                map_map[pnum] = sparse_mem_map_populate(pnum, nodeid);
+                if (map_map[pnum])
+                        continue;
+                ms = __nr_to_section(pnum);
+                printk(KERN_ERR "%s: sparsemem memory map backing failed "
+                        "some memory will not be available.\n", __func__);
+                ms->section_mem_map = 0;
+        }
+
+        if (vmemmap_buf_start) {
+                /* need to free left buf */
+#ifdef CONFIG_NO_BOOTMEM
+                free_early(__pa(vmemmap_buf_start), __pa(vmemmap_buf_end));
+                if (vmemmap_buf_start < vmemmap_buf) {
+                        char name[15];
+
+                        snprintf(name, sizeof(name), "MEMMAP %d", nodeid);
+                        reserve_early_without_check(__pa(vmemmap_buf_start),
+                                                    __pa(vmemmap_buf), name);
+                }
+#else
+                free_bootmem(__pa(vmemmap_buf), vmemmap_buf_end - vmemmap_buf);
+#endif
+                vmemmap_buf = NULL;
+                vmemmap_buf_end = NULL;
+        }
+}
diff --git a/mm/sparse.c b/mm/sparse.c
index 6ce4aab69e99..22896d589133 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -271,7 +271,8 @@ static unsigned long *__kmalloc_section_usemap(void)
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
 static unsigned long * __init
-sparse_early_usemap_alloc_pgdat_section(struct pglist_data *pgdat)
+sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
+                                         unsigned long count)
 {
         unsigned long section_nr;
 
@@ -286,7 +287,7 @@ sparse_early_usemap_alloc_pgdat_section(struct pglist_data *pgdat)
          * this problem.
          */
         section_nr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT);
-        return alloc_bootmem_section(usemap_size(), section_nr);
+        return alloc_bootmem_section(usemap_size() * count, section_nr);
 }
 
 static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
@@ -329,7 +330,8 @@ static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
 }
 #else
 static unsigned long * __init
-sparse_early_usemap_alloc_pgdat_section(struct pglist_data *pgdat)
+sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
+                                         unsigned long count)
 {
         return NULL;
 }
@@ -339,27 +341,40 @@ static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
 }
 #endif /* CONFIG_MEMORY_HOTREMOVE */
 
-static unsigned long *__init sparse_early_usemap_alloc(unsigned long pnum)
+static void __init sparse_early_usemaps_alloc_node(unsigned long**usemap_map,
+                                 unsigned long pnum_begin,
+                                 unsigned long pnum_end,
+                                 unsigned long usemap_count, int nodeid)
 {
-        unsigned long *usemap;
-        struct mem_section *ms = __nr_to_section(pnum);
-        int nid = sparse_early_nid(ms);
-
-        usemap = sparse_early_usemap_alloc_pgdat_section(NODE_DATA(nid));
-        if (usemap)
-                return usemap;
+        void *usemap;
+        unsigned long pnum;
+        int size = usemap_size();
 
-        usemap = alloc_bootmem_node(NODE_DATA(nid), usemap_size());
+        usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid),
+                                                                 usemap_count);
         if (usemap) {
-                check_usemap_section_nr(nid, usemap);
-                return usemap;
+                for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
+                        if (!present_section_nr(pnum))
+                                continue;
+                        usemap_map[pnum] = usemap;
+                        usemap += size;
+                }
+                return;
         }
 
-        /* Stupid: suppress gcc warning for SPARSEMEM && !NUMA */
-        nid = 0;
+        usemap = alloc_bootmem_node(NODE_DATA(nodeid), size * usemap_count);
+        if (usemap) {
+                for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
+                        if (!present_section_nr(pnum))
+                                continue;
+                        usemap_map[pnum] = usemap;
+                        usemap += size;
+                        check_usemap_section_nr(nodeid, usemap_map[pnum]);
+                }
+                return;
+        }
 
         printk(KERN_WARNING "%s: allocation failed\n", __func__);
-        return NULL;
 }
 
 #ifndef CONFIG_SPARSEMEM_VMEMMAP
@@ -375,8 +390,65 @@ struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid)
                        PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION));
         return map;
 }
+void __init sparse_mem_maps_populate_node(struct page **map_map,
+                                          unsigned long pnum_begin,
+                                          unsigned long pnum_end,
+                                          unsigned long map_count, int nodeid)
+{
+        void *map;
+        unsigned long pnum;
+        unsigned long size = sizeof(struct page) * PAGES_PER_SECTION;
+
+        map = alloc_remap(nodeid, size * map_count);
+        if (map) {
+                for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
+                        if (!present_section_nr(pnum))
+                                continue;
+                        map_map[pnum] = map;
+                        map += size;
+                }
+                return;
+        }
+
+        size = PAGE_ALIGN(size);
+        map = alloc_bootmem_pages_node(NODE_DATA(nodeid), size * map_count);
+        if (map) {
+                for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
+                        if (!present_section_nr(pnum))
+                                continue;
+                        map_map[pnum] = map;
+                        map += size;
+                }
+                return;
+        }
+
+        /* fallback */
+        for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
+                struct mem_section *ms;
+
+                if (!present_section_nr(pnum))
+                        continue;
+                map_map[pnum] = sparse_mem_map_populate(pnum, nodeid);
+                if (map_map[pnum])
+                        continue;
+                ms = __nr_to_section(pnum);
+                printk(KERN_ERR "%s: sparsemem memory map backing failed "
+                        "some memory will not be available.\n", __func__);
+                ms->section_mem_map = 0;
+        }
+}
 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
 
+#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
+static void __init sparse_early_mem_maps_alloc_node(struct page **map_map,
+                                 unsigned long pnum_begin,
+                                 unsigned long pnum_end,
+                                 unsigned long map_count, int nodeid)
+{
+        sparse_mem_maps_populate_node(map_map, pnum_begin, pnum_end,
+                                         map_count, nodeid);
+}
+#else
 static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum)
 {
         struct page *map;
@@ -392,10 +464,12 @@ static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum)
         ms->section_mem_map = 0;
         return NULL;
 }
+#endif
 
 void __attribute__((weak)) __meminit vmemmap_populate_print_last(void)
 {
 }
+
 /*
  * Allocate the accumulated non-linear sections, allocate a mem_map
  * for each and record the physical to section mapping.
@@ -407,6 +481,14 @@ void __init sparse_init(void)
         unsigned long *usemap;
         unsigned long **usemap_map;
         int size;
+        int nodeid_begin = 0;
+        unsigned long pnum_begin = 0;
+        unsigned long usemap_count;
+#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
+        unsigned long map_count;
+        int size2;
+        struct page **map_map;
+#endif
 
         /*
          * map is using big page (aka 2M in x86 64 bit)
@@ -425,10 +507,81 @@ void __init sparse_init(void)
                 panic("can not allocate usemap_map\n");
 
         for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
+                struct mem_section *ms;
+
                 if (!present_section_nr(pnum))
                         continue;
-                usemap_map[pnum] = sparse_early_usemap_alloc(pnum);
+                ms = __nr_to_section(pnum);
+                nodeid_begin = sparse_early_nid(ms);
+                pnum_begin = pnum;
+                break;
         }
+        usemap_count = 1;
+        for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) {
+                struct mem_section *ms;
+                int nodeid;
+
+                if (!present_section_nr(pnum))
+                        continue;
+                ms = __nr_to_section(pnum);
+                nodeid = sparse_early_nid(ms);
+                if (nodeid == nodeid_begin) {
+                        usemap_count++;
+                        continue;
+                }
+                /* ok, we need to take cake of from pnum_begin to pnum - 1*/
+                sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, pnum,
+                                                 usemap_count, nodeid_begin);
+                /* new start, update count etc*/
+                nodeid_begin = nodeid;
+                pnum_begin = pnum;
+                usemap_count = 1;
+        }
+        /* ok, last chunk */
+        sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, NR_MEM_SECTIONS,
+                                         usemap_count, nodeid_begin);
+
+#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
+        size2 = sizeof(struct page *) * NR_MEM_SECTIONS;
+        map_map = alloc_bootmem(size2);
+        if (!map_map)
+                panic("can not allocate map_map\n");
+
+        for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
+                struct mem_section *ms;
+
+                if (!present_section_nr(pnum))
+                        continue;
+                ms = __nr_to_section(pnum);
+                nodeid_begin = sparse_early_nid(ms);
+                pnum_begin = pnum;
+                break;
+        }
+        map_count = 1;
+        for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) {
+                struct mem_section *ms;
+                int nodeid;
+
+                if (!present_section_nr(pnum))
+                        continue;
+                ms = __nr_to_section(pnum);
+                nodeid = sparse_early_nid(ms);
+                if (nodeid == nodeid_begin) {
+                        map_count++;
+                        continue;
+                }
+                /* ok, we need to take cake of from pnum_begin to pnum - 1*/
+                sparse_early_mem_maps_alloc_node(map_map, pnum_begin, pnum,
+                                                 map_count, nodeid_begin);
+                /* new start, update count etc*/
+                nodeid_begin = nodeid;
+                pnum_begin = pnum;
+                map_count = 1;
+        }
+        /* ok, last chunk */
+        sparse_early_mem_maps_alloc_node(map_map, pnum_begin, NR_MEM_SECTIONS,
+                                         map_count, nodeid_begin);
+#endif
 
         for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
                 if (!present_section_nr(pnum))
@@ -438,7 +591,11 @@ void __init sparse_init(void)
                 if (!usemap)
                         continue;
 
+#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
+                map = map_map[pnum];
+#else
                 map = sparse_early_mem_map_alloc(pnum);
+#endif
                 if (!map)
                         continue;
 
@@ -448,6 +605,9 @@ void __init sparse_init(void)
 
         vmemmap_populate_print_last();
 
+#ifdef CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER
+        free_bootmem(__pa(map_map), size2);
+#endif
         free_bootmem(__pa(usemap_map), size);
 }
 
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 6051fbab67ba..fc5aa183bc45 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -139,7 +139,8 @@ static void refresh_zone_stat_thresholds(void)
                 threshold = calculate_threshold(zone);
 
                 for_each_online_cpu(cpu)
-                        zone_pcp(zone, cpu)->stat_threshold = threshold;
+                        per_cpu_ptr(zone->pageset, cpu)->stat_threshold
+                                                        = threshold;
         }
 }
 
@@ -149,7 +150,8 @@ static void refresh_zone_stat_thresholds(void)
 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
                                 int delta)
 {
-        struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
+        struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
+
         s8 *p = pcp->vm_stat_diff + item;
         long x;
 
@@ -202,7 +204,7 @@ EXPORT_SYMBOL(mod_zone_page_state);
  */
 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
 {
-        struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
+        struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
         s8 *p = pcp->vm_stat_diff + item;
 
         (*p)++;
@@ -223,7 +225,7 @@ EXPORT_SYMBOL(__inc_zone_page_state);
 
 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
 {
-        struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
+        struct per_cpu_pageset *pcp = this_cpu_ptr(zone->pageset);
         s8 *p = pcp->vm_stat_diff + item;
 
         (*p)--;
@@ -300,7 +302,7 @@ void refresh_cpu_vm_stats(int cpu)
         for_each_populated_zone(zone) {
                 struct per_cpu_pageset *p;
 
-                p = zone_pcp(zone, cpu);
+                p = per_cpu_ptr(zone->pageset, cpu);
 
                 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
                         if (p->vm_stat_diff[i]) {
@@ -741,7 +743,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
         for_each_online_cpu(i) {
                 struct per_cpu_pageset *pageset;
 
-                pageset = zone_pcp(zone, i);
+                pageset = per_cpu_ptr(zone->pageset, i);
                 seq_printf(m,
                            "\n    cpu: %i"
                            "\n              count: %i"
@@ -906,6 +908,7 @@ static int __cpuinit vmstat_cpuup_callback(struct notifier_block *nfb,
         case CPU_ONLINE:
         case CPU_ONLINE_FROZEN:
                 start_cpu_timer(cpu);
+                node_set_state(cpu_to_node(cpu), N_CPU);
                 break;
         case CPU_DOWN_PREPARE:
         case CPU_DOWN_PREPARE_FROZEN: