4 files changed, 58 insertions, 32 deletions

diff --git a/mm/page_cgroup.c b/mm/page_cgroup.c
index 3dd4a909a1d..11a8a10a390 100644
--- a/mm/page_cgroup.c
+++ b/mm/page_cgroup.c
@@ -47,8 +47,6 @@ static int __init alloc_node_page_cgroup(int nid)
         struct page_cgroup *base, *pc;
         unsigned long table_size;
         unsigned long start_pfn, nr_pages, index;
-        struct page *page;
-        unsigned int order;
 
         start_pfn = NODE_DATA(nid)->node_start_pfn;
         nr_pages = NODE_DATA(nid)->node_spanned_pages;
@@ -57,13 +55,11 @@ static int __init alloc_node_page_cgroup(int nid)
                 return 0;
 
         table_size = sizeof(struct page_cgroup) * nr_pages;
-        order = get_order(table_size);
-        page = alloc_pages_node(nid, GFP_NOWAIT | __GFP_ZERO, order);
-        if (!page)
-                page = alloc_pages_node(-1, GFP_NOWAIT | __GFP_ZERO, order);
-        if (!page)
+
+        base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
+                        table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
+        if (!base)
                 return -ENOMEM;
-        base = page_address(page);
         for (index = 0; index < nr_pages; index++) {
                 pc = base + index;
                 __init_page_cgroup(pc, start_pfn + index);
@@ -73,7 +69,7 @@ static int __init alloc_node_page_cgroup(int nid)
         return 0;
 }
 
-void __init page_cgroup_init(void)
+void __init page_cgroup_init_flatmem(void)
 {
 
         int nid, fail;
@@ -117,16 +113,11 @@ static int __init_refok init_section_page_cgroup(unsigned long pfn)
         if (!section->page_cgroup) {
                 nid = page_to_nid(pfn_to_page(pfn));
                 table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
-                if (slab_is_available()) {
-                        base = kmalloc_node(table_size,
-                                        GFP_KERNEL | __GFP_NOWARN, nid);
-                        if (!base)
-                                base = vmalloc_node(table_size, nid);
-                } else {
-                        base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
-                                table_size,
-                                PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
-                }
+                VM_BUG_ON(!slab_is_available());
+                base = kmalloc_node(table_size,
+                                GFP_KERNEL | __GFP_NOWARN, nid);
+                if (!base)
+                        base = vmalloc_node(table_size, nid);
         } else {
                 /*
                  * We don't have to allocate page_cgroup again, but
diff --git a/mm/slab.c b/mm/slab.c
index f46b65d124e..18e3164de09 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -304,6 +304,12 @@ struct kmem_list3 {
 };
 
 /*
+ * The slab allocator is initialized with interrupts disabled. Therefore, make
+ * sure early boot allocations don't accidentally enable interrupts.
+ */
+static gfp_t slab_gfp_mask __read_mostly = SLAB_GFP_BOOT_MASK;
+
+/*
  * Need this for bootstrapping a per node allocator.
  */
 #define NUM_INIT_LISTS (3 * MAX_NUMNODES)
@@ -753,6 +759,7 @@ static enum {
         NONE,
         PARTIAL_AC,
         PARTIAL_L3,
+        EARLY,
         FULL
 } g_cpucache_up;
 
@@ -761,7 +768,7 @@ static enum {
  */
 int slab_is_available(void)
 {
-        return g_cpucache_up == FULL;
+        return g_cpucache_up >= EARLY;
 }
 
 static DEFINE_PER_CPU(struct delayed_work, reap_work);
@@ -1625,19 +1632,27 @@ void __init kmem_cache_init(void)
                 }
         }
 
-        /* 6) resize the head arrays to their final sizes */
-        {
-                struct kmem_cache *cachep;
-                mutex_lock(&cache_chain_mutex);
-                list_for_each_entry(cachep, &cache_chain, next)
-                        if (enable_cpucache(cachep, GFP_NOWAIT))
-                                BUG();
-                mutex_unlock(&cache_chain_mutex);
-        }
+        g_cpucache_up = EARLY;
 
         /* Annotate slab for lockdep -- annotate the malloc caches */
         init_lock_keys();
+}
+
+void __init kmem_cache_init_late(void)
+{
+        struct kmem_cache *cachep;
+
+        /*
+         * Interrupts are enabled now so all GFP allocations are safe.
+         */
+        slab_gfp_mask = __GFP_BITS_MASK;
 
+        /* 6) resize the head arrays to their final sizes */
+        mutex_lock(&cache_chain_mutex);
+        list_for_each_entry(cachep, &cache_chain, next)
+                if (enable_cpucache(cachep, GFP_NOWAIT))
+                        BUG();
+        mutex_unlock(&cache_chain_mutex);
 
         /* Done! */
         g_cpucache_up = FULL;
@@ -2102,7 +2117,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
                         for_each_online_node(node) {
                                 cachep->nodelists[node] =
                                     kmalloc_node(sizeof(struct kmem_list3),
-                                                GFP_KERNEL, node);
+                                                gfp, node);
                                 BUG_ON(!cachep->nodelists[node]);
                                 kmem_list3_init(cachep->nodelists[node]);
                         }
@@ -3354,6 +3369,8 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
         unsigned long save_flags;
         void *ptr;
 
+        flags &= slab_gfp_mask;
+
         lockdep_trace_alloc(flags);
 
         if (slab_should_failslab(cachep, flags))
@@ -3434,6 +3451,8 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
         unsigned long save_flags;
         void *objp;
 
+        flags &= slab_gfp_mask;
+
         lockdep_trace_alloc(flags);
 
         if (slab_should_failslab(cachep, flags))
diff --git a/mm/slub.c b/mm/slub.c
index 3964d3ce4c1..30354bfeb43 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -178,6 +178,12 @@ static enum {
         SYSFS           /* Sysfs up */
 } slab_state = DOWN;
 
+/*
+ * The slab allocator is initialized with interrupts disabled. Therefore, make
+ * sure early boot allocations don't accidentally enable interrupts.
+ */
+static gfp_t slab_gfp_mask __read_mostly = SLAB_GFP_BOOT_MASK;
+
 /* A list of all slab caches on the system */
 static DECLARE_RWSEM(slub_lock);
 static LIST_HEAD(slab_caches);
@@ -1595,6 +1601,8 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
         unsigned long flags;
         unsigned int objsize;
 
+        gfpflags &= slab_gfp_mask;
+
         lockdep_trace_alloc(gfpflags);
         might_sleep_if(gfpflags & __GFP_WAIT);
 
@@ -3104,6 +3112,14 @@ void __init kmem_cache_init(void)
                 nr_cpu_ids, nr_node_ids);
 }
 
+void __init kmem_cache_init_late(void)
+{
+        /*
+         * Interrupts are enabled now so all GFP allocations are safe.
+         */
+        slab_gfp_mask = __GFP_BITS_MASK;
+}
+
 /*
  * Find a mergeable slab cache
  */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index d254306562c..95c08a8cc2b 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2056,7 +2056,7 @@ unsigned long global_lru_pages(void)
                 + global_page_state(NR_INACTIVE_FILE);
 }
 
-#ifdef CONFIG_PM
+#ifdef CONFIG_HIBERNATION
 /*
  * Helper function for shrink_all_memory().  Tries to reclaim 'nr_pages' pages
  * from LRU lists system-wide, for given pass and priority.
@@ -2196,7 +2196,7 @@ out:
 
         return sc.nr_reclaimed;
 }
-#endif
+#endif /* CONFIG_HIBERNATION */
 
 /* It's optimal to keep kswapds on the same CPUs as their memory, but
    not required for correctness.  So if the last cpu in a node goes