1 files changed, 29 insertions, 25 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 6cbde310abed..07efbc3a8656 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1090,10 +1090,10 @@ static void free_pcppages_bulk(struct zone *zone, int count,
 {
         int migratetype = 0;
         int batch_free = 0;
-        unsigned long nr_scanned, flags;
+        unsigned long nr_scanned;
         bool isolated_pageblocks;
 
-        spin_lock_irqsave(&zone->lock, flags);
+        spin_lock(&zone->lock);
         isolated_pageblocks = has_isolate_pageblock(zone);
         nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
         if (nr_scanned)
@@ -1142,7 +1142,7 @@ static void free_pcppages_bulk(struct zone *zone, int count,
                         trace_mm_page_pcpu_drain(page, 0, mt);
                 } while (--count && --batch_free && !list_empty(list));
         }
-        spin_unlock_irqrestore(&zone->lock, flags);
+        spin_unlock(&zone->lock);
 }
 
 static void free_one_page(struct zone *zone,
@@ -1150,9 +1150,8 @@ static void free_one_page(struct zone *zone,
                                 unsigned int order,
                                 int migratetype)
 {
-        unsigned long nr_scanned, flags;
-        spin_lock_irqsave(&zone->lock, flags);
-        __count_vm_events(PGFREE, 1 << order);
+        unsigned long nr_scanned;
+        spin_lock(&zone->lock);
         nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
         if (nr_scanned)
                 __mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);
@@ -1162,7 +1161,7 @@ static void free_one_page(struct zone *zone,
                 migratetype = get_pfnblock_migratetype(page, pfn);
         }
         __free_one_page(page, pfn, zone, order, migratetype);
-        spin_unlock_irqrestore(&zone->lock, flags);
+        spin_unlock(&zone->lock);
 }
 
 static void __meminit __init_single_page(struct page *page, unsigned long pfn,
@@ -1240,6 +1239,7 @@ void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
 
 static void __free_pages_ok(struct page *page, unsigned int order)
 {
+        unsigned long flags;
         int migratetype;
         unsigned long pfn = page_to_pfn(page);
 
@@ -1247,7 +1247,10 @@ static void __free_pages_ok(struct page *page, unsigned int order)
                 return;
 
         migratetype = get_pfnblock_migratetype(page, pfn);
+        local_irq_save(flags);
+        __count_vm_events(PGFREE, 1 << order);
         free_one_page(page_zone(page), page, pfn, order, migratetype);
+        local_irq_restore(flags);
 }
 
 static void __init __free_pages_boot_core(struct page *page, unsigned int order)
@@ -2219,9 +2222,8 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
                         int migratetype, bool cold)
 {
         int i, alloced = 0;
-        unsigned long flags;
 
-        spin_lock_irqsave(&zone->lock, flags);
+        spin_lock(&zone->lock);
         for (i = 0; i < count; ++i) {
                 struct page *page = __rmqueue(zone, order, migratetype);
                 if (unlikely(page == NULL))
@@ -2257,7 +2259,7 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order,
          * pages added to the pcp list.
          */
         __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
-        spin_unlock_irqrestore(&zone->lock, flags);
+        spin_unlock(&zone->lock);
         return alloced;
 }
 
@@ -2373,6 +2375,13 @@ void drain_all_pages(struct zone *zone)
          */
         static cpumask_t cpus_with_pcps;
 
+        /*
+         * Make sure nobody triggers this path before mm_percpu_wq is fully
+         * initialized.
+         */
+        if (WARN_ON_ONCE(!mm_percpu_wq))
+                return;
+
         /* Workqueues cannot recurse */
         if (current->flags & PF_WQ_WORKER)
                 return;
@@ -2422,7 +2431,7 @@ void drain_all_pages(struct zone *zone)
         for_each_cpu(cpu, &cpus_with_pcps) {
                 struct work_struct *work = per_cpu_ptr(&pcpu_drain, cpu);
                 INIT_WORK(work, drain_local_pages_wq);
-                schedule_work_on(cpu, work);
+                queue_work_on(cpu, mm_percpu_wq, work);
         }
         for_each_cpu(cpu, &cpus_with_pcps)
                 flush_work(per_cpu_ptr(&pcpu_drain, cpu));
@@ -2478,20 +2487,17 @@ void free_hot_cold_page(struct page *page, bool cold)
 {
         struct zone *zone = page_zone(page);
         struct per_cpu_pages *pcp;
+        unsigned long flags;
         unsigned long pfn = page_to_pfn(page);
         int migratetype;
 
-        if (in_interrupt()) {
-                __free_pages_ok(page, 0);
-                return;
-        }
-
         if (!free_pcp_prepare(page))
                 return;
 
         migratetype = get_pfnblock_migratetype(page, pfn);
         set_pcppage_migratetype(page, migratetype);
-        preempt_disable();
+        local_irq_save(flags);
+        __count_vm_event(PGFREE);
 
         /*
          * We only track unmovable, reclaimable and movable on pcp lists.
@@ -2508,7 +2514,6 @@ void free_hot_cold_page(struct page *page, bool cold)
                 migratetype = MIGRATE_MOVABLE;
         }
 
-        __count_vm_event(PGFREE);
         pcp = &this_cpu_ptr(zone->pageset)->pcp;
         if (!cold)
                 list_add(&page->lru, &pcp->lists[migratetype]);
@@ -2522,7 +2527,7 @@ void free_hot_cold_page(struct page *page, bool cold)
         }
 
 out:
-        preempt_enable();
+        local_irq_restore(flags);
 }
 
 /*
@@ -2647,8 +2652,6 @@ static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype,
 {
         struct page *page;
 
-        VM_BUG_ON(in_interrupt());
-
         do {
                 if (list_empty(list)) {
                         pcp->count += rmqueue_bulk(zone, 0,
@@ -2679,8 +2682,9 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone,
         struct list_head *list;
         bool cold = ((gfp_flags & __GFP_COLD) != 0);
         struct page *page;
+        unsigned long flags;
 
-        preempt_disable();
+        local_irq_save(flags);
         pcp = &this_cpu_ptr(zone->pageset)->pcp;
         list = &pcp->lists[migratetype];
         page = __rmqueue_pcplist(zone,  migratetype, cold, pcp, list);
@@ -2688,7 +2692,7 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone,
                 __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
                 zone_statistics(preferred_zone, zone);
         }
-        preempt_enable();
+        local_irq_restore(flags);
         return page;
 }
 
@@ -2704,7 +2708,7 @@ struct page *rmqueue(struct zone *preferred_zone,
         unsigned long flags;
         struct page *page;
 
-        if (likely(order == 0) && !in_interrupt()) {
+        if (likely(order == 0)) {
                 page = rmqueue_pcplist(preferred_zone, zone, order,
                                 gfp_flags, migratetype);
                 goto out;
@@ -4519,13 +4523,13 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
                         K(node_page_state(pgdat, NR_FILE_MAPPED)),
                         K(node_page_state(pgdat, NR_FILE_DIRTY)),
                         K(node_page_state(pgdat, NR_WRITEBACK)),
+                        K(node_page_state(pgdat, NR_SHMEM)),
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
                         K(node_page_state(pgdat, NR_SHMEM_THPS) * HPAGE_PMD_NR),
                         K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)
                                         * HPAGE_PMD_NR),
                         K(node_page_state(pgdat, NR_ANON_THPS) * HPAGE_PMD_NR),
 #endif
-                        K(node_page_state(pgdat, NR_SHMEM)),
                         K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
                         K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
                         node_page_state(pgdat, NR_PAGES_SCANNED),