1 files changed, 97 insertions, 35 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 533e2147d14f..3bac76ae4b30 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -205,7 +205,7 @@ static char * const zone_names[MAX_NR_ZONES] = {
 };
 
 int min_free_kbytes = 1024;
-int user_min_free_kbytes;
+int user_min_free_kbytes = -1;
 
 static unsigned long __meminitdata nr_kernel_pages;
 static unsigned long __meminitdata nr_all_pages;
@@ -295,7 +295,7 @@ static inline int bad_range(struct zone *zone, struct page *page)
 }
 #endif
 
-static void bad_page(struct page *page)
+static void bad_page(struct page *page, char *reason, unsigned long bad_flags)
 {
         static unsigned long resume;
         static unsigned long nr_shown;
@@ -329,7 +329,7 @@ static void bad_page(struct page *page)
 
         printk(KERN_ALERT "BUG: Bad page state in process %s  pfn:%05lx\n",
                 current->comm, page_to_pfn(page));
-        dump_page(page);
+        dump_page_badflags(page, reason, bad_flags);
 
         print_modules();
         dump_stack();
@@ -369,9 +369,11 @@ void prep_compound_page(struct page *page, unsigned long order)
         __SetPageHead(page);
         for (i = 1; i < nr_pages; i++) {
                 struct page *p = page + i;
-                __SetPageTail(p);
                 set_page_count(p, 0);
                 p->first_page = page;
+                /* Make sure p->first_page is always valid for PageTail() */
+                smp_wmb();
+                __SetPageTail(p);
         }
 }
 
@@ -383,7 +385,7 @@ static int destroy_compound_page(struct page *page, unsigned long order)
         int bad = 0;
 
         if (unlikely(compound_order(page) != order)) {
-                bad_page(page);
+                bad_page(page, "wrong compound order", 0);
                 bad++;
         }
 
@@ -392,8 +394,11 @@ static int destroy_compound_page(struct page *page, unsigned long order)
         for (i = 1; i < nr_pages; i++) {
                 struct page *p = page + i;
 
-                if (unlikely(!PageTail(p) || (p->first_page != page))) {
-                        bad_page(page);
+                if (unlikely(!PageTail(p))) {
+                        bad_page(page, "PageTail not set", 0);
+                        bad++;
+                } else if (unlikely(p->first_page != page)) {
+                        bad_page(page, "first_page not consistent", 0);
                         bad++;
                 }
                 __ClearPageTail(p);
@@ -506,12 +511,12 @@ static inline int page_is_buddy(struct page *page, struct page *buddy,
                 return 0;
 
         if (page_is_guard(buddy) && page_order(buddy) == order) {
-                VM_BUG_ON(page_count(buddy) != 0);
+                VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
                 return 1;
         }
 
         if (PageBuddy(buddy) && page_order(buddy) == order) {
-                VM_BUG_ON(page_count(buddy) != 0);
+                VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);
                 return 1;
         }
         return 0;
@@ -561,8 +566,8 @@ static inline void __free_one_page(struct page *page,
 
         page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
 
-        VM_BUG_ON(page_idx & ((1 << order) - 1));
-        VM_BUG_ON(bad_range(zone, page));
+        VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
+        VM_BUG_ON_PAGE(bad_range(zone, page), page);
 
         while (order < MAX_ORDER-1) {
                 buddy_idx = __find_buddy_index(page_idx, order);
@@ -618,12 +623,23 @@ out:
 
 static inline int free_pages_check(struct page *page)
 {
-        if (unlikely(page_mapcount(page) |
-                (page->mapping != NULL)  |
-                (atomic_read(&page->_count) != 0) |
-                (page->flags & PAGE_FLAGS_CHECK_AT_FREE) |
-                (mem_cgroup_bad_page_check(page)))) {
-                bad_page(page);
+        char *bad_reason = NULL;
+        unsigned long bad_flags = 0;
+
+        if (unlikely(page_mapcount(page)))
+                bad_reason = "nonzero mapcount";
+        if (unlikely(page->mapping != NULL))
+                bad_reason = "non-NULL mapping";
+        if (unlikely(atomic_read(&page->_count) != 0))
+                bad_reason = "nonzero _count";
+        if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
+                bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
+                bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
+        }
+        if (unlikely(mem_cgroup_bad_page_check(page)))
+                bad_reason = "cgroup check failed";
+        if (unlikely(bad_reason)) {
+                bad_page(page, bad_reason, bad_flags);
                 return 1;
         }
         page_cpupid_reset_last(page);
@@ -813,7 +829,7 @@ static inline void expand(struct zone *zone, struct page *page,
                 area--;
                 high--;
                 size >>= 1;
-                VM_BUG_ON(bad_range(zone, &page[size]));
+                VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
 
 #ifdef CONFIG_DEBUG_PAGEALLOC
                 if (high < debug_guardpage_minorder()) {
@@ -843,12 +859,23 @@ static inline void expand(struct zone *zone, struct page *page,
  */
 static inline int check_new_page(struct page *page)
 {
-        if (unlikely(page_mapcount(page) |
-                (page->mapping != NULL)  |
-                (atomic_read(&page->_count) != 0)  |
-                (page->flags & PAGE_FLAGS_CHECK_AT_PREP) |
-                (mem_cgroup_bad_page_check(page)))) {
-                bad_page(page);
+        char *bad_reason = NULL;
+        unsigned long bad_flags = 0;
+
+        if (unlikely(page_mapcount(page)))
+                bad_reason = "nonzero mapcount";
+        if (unlikely(page->mapping != NULL))
+                bad_reason = "non-NULL mapping";
+        if (unlikely(atomic_read(&page->_count) != 0))
+                bad_reason = "nonzero _count";
+        if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
+                bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
+                bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
+        }
+        if (unlikely(mem_cgroup_bad_page_check(page)))
+                bad_reason = "cgroup check failed";
+        if (unlikely(bad_reason)) {
+                bad_page(page, bad_reason, bad_flags);
                 return 1;
         }
         return 0;
@@ -955,7 +982,7 @@ int move_freepages(struct zone *zone,
 
         for (page = start_page; page <= end_page;) {
                 /* Make sure we are not inadvertently changing nodes */
-                VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone));
+                VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
 
                 if (!pfn_valid_within(page_to_pfn(page))) {
                         page++;
@@ -1211,6 +1238,15 @@ void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
         }
         local_irq_restore(flags);
 }
+static bool gfp_thisnode_allocation(gfp_t gfp_mask)
+{
+        return (gfp_mask & GFP_THISNODE) == GFP_THISNODE;
+}
+#else
+static bool gfp_thisnode_allocation(gfp_t gfp_mask)
+{
+        return false;
+}
 #endif
 
 /*
@@ -1404,8 +1440,8 @@ void split_page(struct page *page, unsigned int order)
 {
         int i;
 
-        VM_BUG_ON(PageCompound(page));
-        VM_BUG_ON(!page_count(page));
+        VM_BUG_ON_PAGE(PageCompound(page), page);
+        VM_BUG_ON_PAGE(!page_count(page), page);
 
 #ifdef CONFIG_KMEMCHECK
         /*
@@ -1547,12 +1583,18 @@ again:
                                           get_pageblock_migratetype(page));
         }
 
-        __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
+        /*
+         * NOTE: GFP_THISNODE allocations do not partake in the kswapd
+         * aging protocol, so they can't be fair.
+         */
+        if (!gfp_thisnode_allocation(gfp_flags))
+                __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
+
         __count_zone_vm_events(PGALLOC, zone, 1 << order);
         zone_statistics(preferred_zone, zone, gfp_flags);
         local_irq_restore(flags);
 
-        VM_BUG_ON(bad_range(zone, page));
+        VM_BUG_ON_PAGE(bad_range(zone, page), page);
         if (prep_new_page(page, order, gfp_flags))
                 goto again;
         return page;
@@ -1919,8 +1961,12 @@ zonelist_scan:
                  * ultimately fall back to remote zones that do not
                  * partake in the fairness round-robin cycle of this
                  * zonelist.
+                 *
+                 * NOTE: GFP_THISNODE allocations do not partake in
+                 * the kswapd aging protocol, so they can't be fair.
                  */
-                if (alloc_flags & ALLOC_WMARK_LOW) {
+                if ((alloc_flags & ALLOC_WMARK_LOW) &&
+                    !gfp_thisnode_allocation(gfp_mask)) {
                         if (zone_page_state(zone, NR_ALLOC_BATCH) <= 0)
                                 continue;
                         if (!zone_local(preferred_zone, zone))
@@ -2476,8 +2522,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
          * allowed per node queues are empty and that nodes are
          * over allocated.
          */
-        if (IS_ENABLED(CONFIG_NUMA) &&
-                        (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
+        if (gfp_thisnode_allocation(gfp_mask))
                 goto nopage;
 
 restart:
@@ -5729,7 +5774,12 @@ module_init(init_per_zone_wmark_min)
 int min_free_kbytes_sysctl_handler(ctl_table *table, int write,
         void __user *buffer, size_t *length, loff_t *ppos)
 {
-        proc_dointvec(table, write, buffer, length, ppos);
+        int rc;
+
+        rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
+        if (rc)
+                return rc;
+
         if (write) {
                 user_min_free_kbytes = min_free_kbytes;
                 setup_per_zone_wmarks();
@@ -5996,7 +6046,7 @@ void set_pageblock_flags_group(struct page *page, unsigned long flags,
         pfn = page_to_pfn(page);
         bitmap = get_pageblock_bitmap(zone, pfn);
         bitidx = pfn_to_bitidx(zone, pfn);
-        VM_BUG_ON(!zone_spans_pfn(zone, pfn));
+        VM_BUG_ON_PAGE(!zone_spans_pfn(zone, pfn), page);
 
         for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
                 if (flags & value)
@@ -6494,12 +6544,24 @@ static void dump_page_flags(unsigned long flags)
         printk(")\n");
 }
 
-void dump_page(struct page *page)
+void dump_page_badflags(struct page *page, char *reason, unsigned long badflags)
 {
         printk(KERN_ALERT
                "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n",
                 page, atomic_read(&page->_count), page_mapcount(page),
                 page->mapping, page->index);
         dump_page_flags(page->flags);
+        if (reason)
+                pr_alert("page dumped because: %s\n", reason);
+        if (page->flags & badflags) {
+                pr_alert("bad because of flags:\n");
+                dump_page_flags(page->flags & badflags);
+        }
         mem_cgroup_print_bad_page(page);
 }
+
+void dump_page(struct page *page, char *reason)
+{
+        dump_page_badflags(page, reason, 0);
+}
+EXPORT_SYMBOL_GPL(dump_page);