1 files changed, 54 insertions, 5 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 83637dfba110..4ba5e37127fc 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -371,8 +371,7 @@ static int destroy_compound_page(struct page *page, unsigned long order)
         int nr_pages = 1 << order;
         int bad = 0;
 
-        if (unlikely(compound_order(page) != order) ||
-            unlikely(!PageHead(page))) {
+        if (unlikely(compound_order(page) != order)) {
                 bad_page(page);
                 bad++;
         }
@@ -611,6 +610,7 @@ static inline int free_pages_check(struct page *page)
                 bad_page(page);
                 return 1;
         }
+        reset_page_last_nid(page);
         if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
                 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
         return 0;
@@ -2612,6 +2612,7 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
         int migratetype = allocflags_to_migratetype(gfp_mask);
         unsigned int cpuset_mems_cookie;
         int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET;
+        struct mem_cgroup *memcg = NULL;
 
         gfp_mask &= gfp_allowed_mask;
 
@@ -2630,6 +2631,13 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
         if (unlikely(!zonelist->_zonerefs->zone))
                 return NULL;
 
+        /*
+         * Will only have any effect when __GFP_KMEMCG is set.  This is
+         * verified in the (always inline) callee
+         */
+        if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order))
+                return NULL;
+
 retry_cpuset:
         cpuset_mems_cookie = get_mems_allowed();
 
@@ -2665,6 +2673,8 @@ out:
         if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
                 goto retry_cpuset;
 
+        memcg_kmem_commit_charge(page, memcg, order);
+
         return page;
 }
 EXPORT_SYMBOL(__alloc_pages_nodemask);
@@ -2717,6 +2727,31 @@ void free_pages(unsigned long addr, unsigned int order)
 
 EXPORT_SYMBOL(free_pages);
 
+/*
+ * __free_memcg_kmem_pages and free_memcg_kmem_pages will free
+ * pages allocated with __GFP_KMEMCG.
+ *
+ * Those pages are accounted to a particular memcg, embedded in the
+ * corresponding page_cgroup. To avoid adding a hit in the allocator to search
+ * for that information only to find out that it is NULL for users who have no
+ * interest in that whatsoever, we provide these functions.
+ *
+ * The caller knows better which flags it relies on.
+ */
+void __free_memcg_kmem_pages(struct page *page, unsigned int order)
+{
+        memcg_kmem_uncharge_pages(page, order);
+        __free_pages(page, order);
+}
+
+void free_memcg_kmem_pages(unsigned long addr, unsigned int order)
+{
+        if (addr != 0) {
+                VM_BUG_ON(!virt_addr_valid((void *)addr));
+                __free_memcg_kmem_pages(virt_to_page((void *)addr), order);
+        }
+}
+
 static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size)
 {
         if (addr) {
@@ -3883,6 +3918,7 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
                 mminit_verify_page_links(page, zone, nid, pfn);
                 init_page_count(page);
                 reset_page_mapcount(page);
+                reset_page_last_nid(page);
                 SetPageReserved(page);
                 /*
                  * Mark the block movable so that blocks are reserved for
@@ -4526,6 +4562,11 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
         int ret;
 
         pgdat_resize_init(pgdat);
+#ifdef CONFIG_NUMA_BALANCING
+        spin_lock_init(&pgdat->numabalancing_migrate_lock);
+        pgdat->numabalancing_migrate_nr_pages = 0;
+        pgdat->numabalancing_migrate_next_window = jiffies;
+#endif
         init_waitqueue_head(&pgdat->kswapd_wait);
         init_waitqueue_head(&pgdat->pfmemalloc_wait);
         pgdat_page_cgroup_init(pgdat);
@@ -5800,7 +5841,8 @@ static int __alloc_contig_migrate_range(struct compact_control *cc,
 
                 ret = migrate_pages(&cc->migratepages,
                                     alloc_migrate_target,
-                                    0, false, MIGRATE_SYNC);
+                                    0, false, MIGRATE_SYNC,
+                                    MR_CMA);
         }
 
         putback_movable_pages(&cc->migratepages);
@@ -5936,8 +5978,15 @@ done:
 
 void free_contig_range(unsigned long pfn, unsigned nr_pages)
 {
-        for (; nr_pages--; ++pfn)
-                __free_page(pfn_to_page(pfn));
+        unsigned int count = 0;
+
+        for (; nr_pages--; pfn++) {
+                struct page *page = pfn_to_page(pfn);
+
+                count += page_count(page) != 1;
+                __free_page(page);
+        }
+        WARN(count != 0, "%d pages are still in use!\n", count);
 }
 #endif