5 files changed, 70 insertions, 57 deletions

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index bb652f5a93fb..a02a14f0f324 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -172,8 +172,9 @@ set_pte_phys(unsigned long vaddr, unsigned long phys, pgprot_t prot)
 }
 
 /*
- * The head.S code sets up the kernel high mapping from:
- * __START_KERNEL_map to __START_KERNEL_map + KERNEL_TEXT_SIZE
+ * The head.S code sets up the kernel high mapping:
+ *
+ *   from __START_KERNEL_map to __START_KERNEL_map + size (== _end-_text)
  *
  * phys_addr holds the negative offset to the kernel, which is added
  * to the compile time generated pmds. This results in invalid pmds up
@@ -515,14 +516,6 @@ void __init mem_init(void)
 
         /* clear_bss() already clear the empty_zero_page */
 
-        /* temporary debugging - double check it's true: */
-        {
-                int i;
-
-                for (i = 0; i < 1024; i++)
-                        WARN_ON_ONCE(empty_zero_page[i]);
-        }
-
         reservedpages = 0;
 
         /* this will put all low memory onto the freelists */
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index 882328efc3db..8fe576baa148 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -134,8 +134,6 @@ static void __iomem *__ioremap(unsigned long phys_addr, unsigned long size,
                         return NULL;
         }
 
-        WARN_ON_ONCE(page_is_ram(pfn));
-
         switch (mode) {
         case IOR_MODE_UNCACHED:
         default:
@@ -162,7 +160,7 @@ static void __iomem *__ioremap(unsigned long phys_addr, unsigned long size,
         area->phys_addr = phys_addr;
         vaddr = (unsigned long) area->addr;
         if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) {
-                remove_vm_area((void *)(vaddr & PAGE_MASK));
+                free_vm_area(area);
                 return NULL;
         }
 
diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c
index 59898fb0a4aa..8ccfee10f5b5 100644
--- a/arch/x86/mm/numa_64.c
+++ b/arch/x86/mm/numa_64.c
@@ -622,13 +622,17 @@ void __init init_cpu_to_node(void)
         int i;
 
         for (i = 0; i < NR_CPUS; i++) {
+                int node;
                 u16 apicid = x86_cpu_to_apicid_init[i];
 
                 if (apicid == BAD_APICID)
                         continue;
-                if (apicid_to_node[apicid] == NUMA_NO_NODE)
+                node = apicid_to_node[apicid];
+                if (node == NUMA_NO_NODE)
                         continue;
-                numa_set_node(i, apicid_to_node[apicid]);
+                if (!node_online(node))
+                        continue;
+                numa_set_node(i, node);
         }
 }
 
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 464d8fc21ce6..14e48b5a94ba 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -44,6 +44,12 @@ static inline unsigned long highmap_end_pfn(void)
 
 #endif
 
+#ifdef CONFIG_DEBUG_PAGEALLOC
+# define debug_pagealloc 1
+#else
+# define debug_pagealloc 0
+#endif
+
 static inline int
 within(unsigned long addr, unsigned long start, unsigned long end)
 {
@@ -355,45 +361,48 @@ out_unlock:
 
 static LIST_HEAD(page_pool);
 static unsigned long pool_size, pool_pages, pool_low;
-static unsigned long pool_used, pool_failed, pool_refill;
+static unsigned long pool_used, pool_failed;
 
-static void cpa_fill_pool(void)
+static void cpa_fill_pool(struct page **ret)
 {
-        struct page *p;
         gfp_t gfp = GFP_KERNEL;
+        unsigned long flags;
+        struct page *p;
 
-        /* Do not allocate from interrupt context */
-        if (in_irq() || irqs_disabled())
-                return;
         /*
-         * Check unlocked. I does not matter when we have one more
-         * page in the pool. The bit lock avoids recursive pool
-         * allocations:
+         * Avoid recursion (on debug-pagealloc) and also signal
+         * our priority to get to these pagetables:
          */
-        if (pool_pages >= pool_size || test_and_set_bit_lock(0, &pool_refill))
+        if (current->flags & PF_MEMALLOC)
                 return;
+        current->flags |= PF_MEMALLOC;
 
-#ifdef CONFIG_DEBUG_PAGEALLOC
         /*
-         * We could do:
-         * gfp = in_atomic() ? GFP_ATOMIC : GFP_KERNEL;
-         * but this fails on !PREEMPT kernels
+         * Allocate atomically from atomic contexts:
          */
-        gfp =  GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
-#endif
+        if (in_atomic() || irqs_disabled() || debug_pagealloc)
+                gfp =  GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
 
-        while (pool_pages < pool_size) {
+        while (pool_pages < pool_size || (ret && !*ret)) {
                 p = alloc_pages(gfp, 0);
                 if (!p) {
                         pool_failed++;
                         break;
                 }
-                spin_lock_irq(&pgd_lock);
+                /*
+                 * If the call site needs a page right now, provide it:
+                 */
+                if (ret && !*ret) {
+                        *ret = p;
+                        continue;
+                }
+                spin_lock_irqsave(&pgd_lock, flags);
                 list_add(&p->lru, &page_pool);
                 pool_pages++;
-                spin_unlock_irq(&pgd_lock);
+                spin_unlock_irqrestore(&pgd_lock, flags);
         }
-        clear_bit_unlock(0, &pool_refill);
+
+        current->flags &= ~PF_MEMALLOC;
 }
 
 #define SHIFT_MB                (20 - PAGE_SHIFT)
@@ -414,11 +423,15 @@ void __init cpa_init(void)
          * GiB. Shift MiB to Gib and multiply the result by
          * POOL_PAGES_PER_GB:
          */
-        gb = ((si.totalram >> SHIFT_MB) + ROUND_MB_GB) >> SHIFT_MB_GB;
-        pool_size = POOL_PAGES_PER_GB * gb;
+        if (debug_pagealloc) {
+                gb = ((si.totalram >> SHIFT_MB) + ROUND_MB_GB) >> SHIFT_MB_GB;
+                pool_size = POOL_PAGES_PER_GB * gb;
+        } else {
+                pool_size = 1;
+        }
         pool_low = pool_size;
 
-        cpa_fill_pool();
+        cpa_fill_pool(NULL);
         printk(KERN_DEBUG
                "CPA: page pool initialized %lu of %lu pages preallocated\n",
                pool_pages, pool_size);
@@ -440,16 +453,20 @@ static int split_large_page(pte_t *kpte, unsigned long address)
         spin_lock_irqsave(&pgd_lock, flags);
         if (list_empty(&page_pool)) {
                 spin_unlock_irqrestore(&pgd_lock, flags);
-                return -ENOMEM;
+                base = NULL;
+                cpa_fill_pool(&base);
+                if (!base)
+                        return -ENOMEM;
+                spin_lock_irqsave(&pgd_lock, flags);
+        } else {
+                base = list_first_entry(&page_pool, struct page, lru);
+                list_del(&base->lru);
+                pool_pages--;
+
+                if (pool_pages < pool_low)
+                        pool_low = pool_pages;
         }
 
-        base = list_first_entry(&page_pool, struct page, lru);
-        list_del(&base->lru);
-        pool_pages--;
-
-        if (pool_pages < pool_low)
-                pool_low = pool_pages;
-
         /*
          * Check for races, another CPU might have split this page
          * up for us already:
@@ -734,7 +751,8 @@ static int change_page_attr_set_clr(unsigned long addr, int numpages,
                 cpa_flush_all(cache);
 
 out:
-        cpa_fill_pool();
+        cpa_fill_pool(NULL);
+
         return ret;
 }
 
@@ -897,7 +915,7 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
          * Try to refill the page pool here. We can do this only after
          * the tlb flush.
          */
-        cpa_fill_pool();
+        cpa_fill_pool(NULL);
 }
 
 #ifdef CONFIG_HIBERNATION
diff --git a/arch/x86/mm/pgtable_32.c b/arch/x86/mm/pgtable_32.c
index 73aba7125203..2f9e9afcb9f4 100644
--- a/arch/x86/mm/pgtable_32.c
+++ b/arch/x86/mm/pgtable_32.c
@@ -342,12 +342,16 @@ static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
 
 pgd_t *pgd_alloc(struct mm_struct *mm)
 {
-        pgd_t *pgd = quicklist_alloc(0, GFP_KERNEL, pgd_ctor);
+        pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
 
-        mm->pgd = pgd;          /* so that alloc_pd can use it */
+        /* so that alloc_pd can use it */
+        mm->pgd = pgd;
+        if (pgd)
+                pgd_ctor(pgd);
 
         if (pgd && !pgd_prepopulate_pmd(mm, pgd)) {
-                quicklist_free(0, pgd_dtor, pgd);
+                pgd_dtor(pgd);
+                free_page((unsigned long)pgd);
                 pgd = NULL;
         }
 
@@ -357,12 +361,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
 void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
         pgd_mop_up_pmds(mm, pgd);
-        quicklist_free(0, pgd_dtor, pgd);
-}
-
-void check_pgt_cache(void)
-{
-        quicklist_trim(0, pgd_dtor, 25, 16);
+        pgd_dtor(pgd);
+        free_page((unsigned long)pgd);
 }
 
 void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte)