kmemcheck: add hooks for the page allocator

This adds support for tracking the initializedness of memory that
was allocated with the page allocator. Highmem requests are not
tracked.

Cc: Dave Hansen <dave@linux.vnet.ibm.com>
Acked-by: Pekka Enberg <penberg@cs.helsinki.fi>

[build fix for !CONFIG_KMEMCHECK]
Signed-off-by: Ingo Molnar <mingo@elte.hu>

[rebased for mainline inclusion]
Signed-off-by: Vegard Nossum <vegard.nossum@gmail.com>

4 files changed, 78 insertions, 23 deletions

diff --git a/mm/kmemcheck.c b/mm/kmemcheck.c
index eaa41b802611..fd814fd61319 100644
--- a/mm/kmemcheck.c
+++ b/mm/kmemcheck.c
@@ -1,10 +1,10 @@
+#include <linux/gfp.h>
 #include <linux/mm_types.h>
 #include <linux/mm.h>
 #include <linux/slab.h>
 #include <linux/kmemcheck.h>
 
-void kmemcheck_alloc_shadow(struct kmem_cache *s, gfp_t flags, int node,
-                           struct page *page, int order)
+void kmemcheck_alloc_shadow(struct page *page, int order, gfp_t flags, int node)
 {
         struct page *shadow;
         int pages;
@@ -16,7 +16,7 @@ void kmemcheck_alloc_shadow(struct kmem_cache *s, gfp_t flags, int node,
          * With kmemcheck enabled, we need to allocate a memory area for the
          * shadow bits as well.
          */
-        shadow = alloc_pages_node(node, flags, order);
+        shadow = alloc_pages_node(node, flags | __GFP_NOTRACK, order);
         if (!shadow) {
                 if (printk_ratelimit())
                         printk(KERN_ERR "kmemcheck: failed to allocate "
@@ -33,23 +33,17 @@ void kmemcheck_alloc_shadow(struct kmem_cache *s, gfp_t flags, int node,
          * the memory accesses.
          */
         kmemcheck_hide_pages(page, pages);
-
-        /*
-         * Objects from caches that have a constructor don't get
-         * cleared when they're allocated, so we need to do it here.
-         */
-        if (s->ctor)
-                kmemcheck_mark_uninitialized_pages(page, pages);
-        else
-                kmemcheck_mark_unallocated_pages(page, pages);
 }
 
-void kmemcheck_free_shadow(struct kmem_cache *s, struct page *page, int order)
+void kmemcheck_free_shadow(struct page *page, int order)
 {
         struct page *shadow;
         int pages;
         int i;
 
+        if (!kmemcheck_page_is_tracked(page))
+                return;
+
         pages = 1 << order;
 
         kmemcheck_show_pages(page, pages);
@@ -101,3 +95,28 @@ void kmemcheck_slab_free(struct kmem_cache *s, void *object, size_t size)
         if (!s->ctor && !(s->flags & SLAB_DESTROY_BY_RCU))
                 kmemcheck_mark_freed(object, size);
 }
+
+void kmemcheck_pagealloc_alloc(struct page *page, unsigned int order,
+                               gfp_t gfpflags)
+{
+        int pages;
+
+        if (gfpflags & (__GFP_HIGHMEM | __GFP_NOTRACK))
+                return;
+
+        pages = 1 << order;
+
+        /*
+         * NOTE: We choose to track GFP_ZERO pages too; in fact, they
+         * can become uninitialized by copying uninitialized memory
+         * into them.
+         */
+
+        /* XXX: Can use zone->node for node? */
+        kmemcheck_alloc_shadow(page, order, gfpflags, -1);
+
+        if (gfpflags & __GFP_ZERO)
+                kmemcheck_mark_initialized_pages(page, pages);
+        else
+                kmemcheck_mark_uninitialized_pages(page, pages);
+}
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 17d5f539a9aa..0727896a88ac 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -23,6 +23,7 @@
 #include <linux/bootmem.h>
 #include <linux/compiler.h>
 #include <linux/kernel.h>
+#include <linux/kmemcheck.h>
 #include <linux/module.h>
 #include <linux/suspend.h>
 #include <linux/pagevec.h>
@@ -546,6 +547,8 @@ static void __free_pages_ok(struct page *page, unsigned int order)
         int i;
         int bad = 0;
 
+        kmemcheck_free_shadow(page, order);
+
         for (i = 0 ; i < (1 << order) ; ++i)
                 bad += free_pages_check(page + i);
         if (bad)
@@ -994,6 +997,8 @@ static void free_hot_cold_page(struct page *page, int cold)
         struct per_cpu_pages *pcp;
         unsigned long flags;
 
+        kmemcheck_free_shadow(page, 0);
+
         if (PageAnon(page))
                 page->mapping = NULL;
         if (free_pages_check(page))
@@ -1047,6 +1052,16 @@ void split_page(struct page *page, unsigned int order)
 
         VM_BUG_ON(PageCompound(page));
         VM_BUG_ON(!page_count(page));
+
+#ifdef CONFIG_KMEMCHECK
+        /*
+         * Split shadow pages too, because free(page[0]) would
+         * otherwise free the whole shadow.
+         */
+        if (kmemcheck_page_is_tracked(page))
+                split_page(virt_to_page(page[0].shadow), order);
+#endif
+
         for (i = 1; i < (1 << order); i++)
                 set_page_refcounted(page + i);
 }
@@ -1667,7 +1682,10 @@ nopage:
                 dump_stack();
                 show_mem();
         }
+        return page;
 got_pg:
+        if (kmemcheck_enabled)
+                kmemcheck_pagealloc_alloc(page, order, gfp_mask);
         return page;
 }
 EXPORT_SYMBOL(__alloc_pages_internal);
diff --git a/mm/slab.c b/mm/slab.c
index 95b6c5eb40b3..6a1ad0b9a94f 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -1612,7 +1612,7 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
                 flags |= __GFP_RECLAIMABLE;
 
-        page = alloc_pages_node(nodeid, flags, cachep->gfporder);
+        page = alloc_pages_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
         if (!page)
                 return NULL;
 
@@ -1626,8 +1626,14 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
         for (i = 0; i < nr_pages; i++)
                 __SetPageSlab(page + i);
 
-        if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK))
-                kmemcheck_alloc_shadow(cachep, flags, nodeid, page, cachep->gfporder);
+        if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
+                kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
+
+                if (cachep->ctor)
+                        kmemcheck_mark_uninitialized_pages(page, nr_pages);
+                else
+                        kmemcheck_mark_unallocated_pages(page, nr_pages);
+        }
 
         return page_address(page);
 }
@@ -1641,8 +1647,7 @@ static void kmem_freepages(struct kmem_cache *cachep, void *addr)
         struct page *page = virt_to_page(addr);
         const unsigned long nr_freed = i;
 
-        if (kmemcheck_page_is_tracked(page))
-                kmemcheck_free_shadow(cachep, page, cachep->gfporder);
+        kmemcheck_free_shadow(page, cachep->gfporder);
 
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
                 sub_zone_page_state(page_zone(page),
diff --git a/mm/slub.c b/mm/slub.c
index 1cebaa747ad3..898fb5047dcc 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1066,6 +1066,8 @@ static inline struct page *alloc_slab_page(gfp_t flags, int node,
 {
         int order = oo_order(oo);
 
+        flags |= __GFP_NOTRACK;
+
         if (node == -1)
                 return alloc_pages(flags, order);
         else
@@ -1097,7 +1099,18 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
         if (kmemcheck_enabled
                 && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS)))
         {
-                kmemcheck_alloc_shadow(s, flags, node, page, compound_order(page));
+                int pages = 1 << oo_order(oo);
+
+                kmemcheck_alloc_shadow(page, oo_order(oo), flags, node);
+
+                /*
+                 * Objects from caches that have a constructor don't get
+                 * cleared when they're allocated, so we need to do it here.
+                 */
+                if (s->ctor)
+                        kmemcheck_mark_uninitialized_pages(page, pages);
+                else
+                        kmemcheck_mark_unallocated_pages(page, pages);
         }
 
         page->objects = oo_objects(oo);
@@ -1173,8 +1186,7 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
                 __ClearPageSlubDebug(page);
         }
 
-        if (kmemcheck_page_is_tracked(page))
-                kmemcheck_free_shadow(s, page, compound_order(page));
+        kmemcheck_free_shadow(page, compound_order(page));
 
         mod_zone_page_state(page_zone(page),
                 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
@@ -2734,9 +2746,10 @@ EXPORT_SYMBOL(__kmalloc);
 
 static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
 {
-        struct page *page = alloc_pages_node(node, flags | __GFP_COMP,
-                                                get_order(size));
+        struct page *page;
 
+        flags |= __GFP_COMP | __GFP_NOTRACK;
+        page = alloc_pages_node(node, flags, get_order(size));
         if (page)
                 return page_address(page);
         else