9 files changed, 202 insertions, 92 deletions

diff --git a/mm/Kconfig b/mm/Kconfig
index 97d2c88b745e..c948d4ca8bde 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -128,11 +128,11 @@ config SPARSEMEM_VMEMMAP
 config MEMORY_HOTPLUG
         bool "Allow for memory hot-add"
         depends on SPARSEMEM || X86_64_ACPI_NUMA
-        depends on HOTPLUG && !HIBERNATION && ARCH_ENABLE_MEMORY_HOTPLUG
+        depends on HOTPLUG && !(HIBERNATION && !S390) && ARCH_ENABLE_MEMORY_HOTPLUG
         depends on (IA64 || X86 || PPC64 || SUPERH || S390)
 
 comment "Memory hotplug is currently incompatible with Software Suspend"
-        depends on SPARSEMEM && HOTPLUG && HIBERNATION
+        depends on SPARSEMEM && HOTPLUG && HIBERNATION && !S390
 
 config MEMORY_HOTPLUG_SPARSE
         def_bool y
diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug
index bb01e298f260..aa99fd1f7109 100644
--- a/mm/Kconfig.debug
+++ b/mm/Kconfig.debug
@@ -2,6 +2,7 @@ config DEBUG_PAGEALLOC
         bool "Debug page memory allocations"
         depends on DEBUG_KERNEL && ARCH_SUPPORTS_DEBUG_PAGEALLOC
         depends on !HIBERNATION || !PPC && !SPARC
+        depends on !KMEMCHECK
         ---help---
           Unmap pages from the kernel linear mapping after free_pages().
           This results in a large slowdown, but helps to find certain types
diff --git a/mm/Makefile b/mm/Makefile
index cf76be785add..5e0bd6426693 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -28,6 +28,7 @@ obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
 obj-$(CONFIG_PAGE_POISONING) += debug-pagealloc.o
 obj-$(CONFIG_SLAB) += slab.o
 obj-$(CONFIG_SLUB) += slub.o
+obj-$(CONFIG_KMEMCHECK) += kmemcheck.o
 obj-$(CONFIG_FAILSLAB) += failslab.o
 obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o
 obj-$(CONFIG_FS_XIP) += filemap_xip.o
diff --git a/mm/bounce.c b/mm/bounce.c
index 4ebe3ea83795..a2b76a588e34 100644
--- a/mm/bounce.c
+++ b/mm/bounce.c
@@ -13,7 +13,6 @@
 #include <linux/init.h>
 #include <linux/hash.h>
 #include <linux/highmem.h>
-#include <linux/blktrace_api.h>
 #include <asm/tlbflush.h>
 
 #include <trace/events/block.h>
diff --git a/mm/highmem.c b/mm/highmem.c
index 68eb1d9b63fa..25878cc49daa 100644
--- a/mm/highmem.c
+++ b/mm/highmem.c
@@ -26,7 +26,6 @@
 #include <linux/init.h>
 #include <linux/hash.h>
 #include <linux/highmem.h>
-#include <linux/blktrace_api.h>
 #include <asm/tlbflush.h>
 
 /*
diff --git a/mm/kmemcheck.c b/mm/kmemcheck.c
new file mode 100644
index 000000000000..fd814fd61319
--- /dev/null
+++ b/mm/kmemcheck.c
@@ -0,0 +1,122 @@
+#include <linux/gfp.h>
+#include <linux/mm_types.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/kmemcheck.h>
+
+void kmemcheck_alloc_shadow(struct page *page, int order, gfp_t flags, int node)
+{
+        struct page *shadow;
+        int pages;
+        int i;
+
+        pages = 1 << order;
+
+        /*
+         * With kmemcheck enabled, we need to allocate a memory area for the
+         * shadow bits as well.
+         */
+        shadow = alloc_pages_node(node, flags | __GFP_NOTRACK, order);
+        if (!shadow) {
+                if (printk_ratelimit())
+                        printk(KERN_ERR "kmemcheck: failed to allocate "
+                                "shadow bitmap\n");
+                return;
+        }
+
+        for(i = 0; i < pages; ++i)
+                page[i].shadow = page_address(&shadow[i]);
+
+        /*
+         * Mark it as non-present for the MMU so that our accesses to
+         * this memory will trigger a page fault and let us analyze
+         * the memory accesses.
+         */
+        kmemcheck_hide_pages(page, pages);
+}
+
+void kmemcheck_free_shadow(struct page *page, int order)
+{
+        struct page *shadow;
+        int pages;
+        int i;
+
+        if (!kmemcheck_page_is_tracked(page))
+                return;
+
+        pages = 1 << order;
+
+        kmemcheck_show_pages(page, pages);
+
+        shadow = virt_to_page(page[0].shadow);
+
+        for(i = 0; i < pages; ++i)
+                page[i].shadow = NULL;
+
+        __free_pages(shadow, order);
+}
+
+void kmemcheck_slab_alloc(struct kmem_cache *s, gfp_t gfpflags, void *object,
+                          size_t size)
+{
+        /*
+         * Has already been memset(), which initializes the shadow for us
+         * as well.
+         */
+        if (gfpflags & __GFP_ZERO)
+                return;
+
+        /* No need to initialize the shadow of a non-tracked slab. */
+        if (s->flags & SLAB_NOTRACK)
+                return;
+
+        if (!kmemcheck_enabled || gfpflags & __GFP_NOTRACK) {
+                /*
+                 * Allow notracked objects to be allocated from
+                 * tracked caches. Note however that these objects
+                 * will still get page faults on access, they just
+                 * won't ever be flagged as uninitialized. If page
+                 * faults are not acceptable, the slab cache itself
+                 * should be marked NOTRACK.
+                 */
+                kmemcheck_mark_initialized(object, size);
+        } else if (!s->ctor) {
+                /*
+                 * New objects should be marked uninitialized before
+                 * they're returned to the called.
+                 */
+                kmemcheck_mark_uninitialized(object, size);
+        }
+}
+
+void kmemcheck_slab_free(struct kmem_cache *s, void *object, size_t size)
+{
+        /* TODO: RCU freeing is unsupported for now; hide false positives. */
+        if (!s->ctor && !(s->flags & SLAB_DESTROY_BY_RCU))
+                kmemcheck_mark_freed(object, size);
+}
+
+void kmemcheck_pagealloc_alloc(struct page *page, unsigned int order,
+                               gfp_t gfpflags)
+{
+        int pages;
+
+        if (gfpflags & (__GFP_HIGHMEM | __GFP_NOTRACK))
+                return;
+
+        pages = 1 << order;
+
+        /*
+         * NOTE: We choose to track GFP_ZERO pages too; in fact, they
+         * can become uninitialized by copying uninitialized memory
+         * into them.
+         */
+
+        /* XXX: Can use zone->node for node? */
+        kmemcheck_alloc_shadow(page, order, gfpflags, -1);
+
+        if (gfpflags & __GFP_ZERO)
+                kmemcheck_mark_initialized_pages(page, pages);
+        else
+                kmemcheck_mark_uninitialized_pages(page, pages);
+}
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 2f457a756d46..a5f3c278c573 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -23,6 +23,7 @@
 #include <linux/bootmem.h>
 #include <linux/compiler.h>
 #include <linux/kernel.h>
+#include <linux/kmemcheck.h>
 #include <linux/module.h>
 #include <linux/suspend.h>
 #include <linux/pagevec.h>
@@ -558,6 +559,8 @@ static void __free_pages_ok(struct page *page, unsigned int order)
         int bad = 0;
         int clearMlocked = PageMlocked(page);
 
+        kmemcheck_free_shadow(page, order);
+
         for (i = 0 ; i < (1 << order) ; ++i)
                 bad += free_pages_check(page + i);
         if (bad)
@@ -1020,6 +1023,8 @@ static void free_hot_cold_page(struct page *page, int cold)
         unsigned long flags;
         int clearMlocked = PageMlocked(page);
 
+        kmemcheck_free_shadow(page, 0);
+
         if (PageAnon(page))
                 page->mapping = NULL;
         if (free_pages_check(page))
@@ -1076,6 +1081,16 @@ void split_page(struct page *page, unsigned int order)
 
         VM_BUG_ON(PageCompound(page));
         VM_BUG_ON(!page_count(page));
+
+#ifdef CONFIG_KMEMCHECK
+        /*
+         * Split shadow pages too, because free(page[0]) would
+         * otherwise free the whole shadow.
+         */
+        if (kmemcheck_page_is_tracked(page))
+                split_page(virt_to_page(page[0].shadow), order);
+#endif
+
         for (i = 1; i < (1 << order); i++)
                 set_page_refcounted(page + i);
 }
@@ -1828,7 +1843,10 @@ nopage:
                 dump_stack();
                 show_mem();
         }
+        return page;
 got_pg:
+        if (kmemcheck_enabled)
+                kmemcheck_pagealloc_alloc(page, order, gfp_mask);
         return page;
 
 }
diff --git a/mm/slab.c b/mm/slab.c
index 744ab9a665a2..f257d4dd474d 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -114,6 +114,7 @@
 #include        <linux/rtmutex.h>
 #include        <linux/reciprocal_div.h>
 #include        <linux/debugobjects.h>
+#include        <linux/kmemcheck.h>
 
 #include        <asm/cacheflush.h>
 #include        <asm/tlbflush.h>
@@ -179,13 +180,13 @@
                          SLAB_STORE_USER | \
                          SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
                          SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
-                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE)
+                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
 #else
 # define CREATE_MASK    (SLAB_HWCACHE_ALIGN | \
                          SLAB_CACHE_DMA | \
                          SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
                          SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
-                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE)
+                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
 #endif
 
 /*
@@ -380,87 +381,6 @@ static void kmem_list3_init(struct kmem_list3 *parent)
         MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid);  \
         } while (0)
 
-/*
- * struct kmem_cache
- *
- * manages a cache.
- */
-
-struct kmem_cache {
-/* 1) per-cpu data, touched during every alloc/free */
-        struct array_cache *array[NR_CPUS];
-/* 2) Cache tunables. Protected by cache_chain_mutex */
-        unsigned int batchcount;
-        unsigned int limit;
-        unsigned int shared;
-
-        unsigned int buffer_size;
-        u32 reciprocal_buffer_size;
-/* 3) touched by every alloc & free from the backend */
-
-        unsigned int flags;             /* constant flags */
-        unsigned int num;               /* # of objs per slab */
-
-/* 4) cache_grow/shrink */
-        /* order of pgs per slab (2^n) */
-        unsigned int gfporder;
-
-        /* force GFP flags, e.g. GFP_DMA */
-        gfp_t gfpflags;
-
-        size_t colour;                  /* cache colouring range */
-        unsigned int colour_off;        /* colour offset */
-        struct kmem_cache *slabp_cache;
-        unsigned int slab_size;
-        unsigned int dflags;            /* dynamic flags */
-
-        /* constructor func */
-        void (*ctor)(void *obj);
-
-/* 5) cache creation/removal */
-        const char *name;
-        struct list_head next;
-
-/* 6) statistics */
-#if STATS
-        unsigned long num_active;
-        unsigned long num_allocations;
-        unsigned long high_mark;
-        unsigned long grown;
-        unsigned long reaped;
-        unsigned long errors;
-        unsigned long max_freeable;
-        unsigned long node_allocs;
-        unsigned long node_frees;
-        unsigned long node_overflow;
-        atomic_t allochit;
-        atomic_t allocmiss;
-        atomic_t freehit;
-        atomic_t freemiss;
-#endif
-#if DEBUG
-        /*
-         * If debugging is enabled, then the allocator can add additional
-         * fields and/or padding to every object. buffer_size contains the total
-         * object size including these internal fields, the following two
-         * variables contain the offset to the user object and its size.
-         */
-        int obj_offset;
-        int obj_size;
-#endif
-        /*
-         * We put nodelists[] at the end of kmem_cache, because we want to size
-         * this array to nr_node_ids slots instead of MAX_NUMNODES
-         * (see kmem_cache_init())
-         * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
-         * is statically defined, so we reserve the max number of nodes.
-         */
-        struct kmem_list3 *nodelists[MAX_NUMNODES];
-        /*
-         * Do not add fields after nodelists[]
-         */
-};
-
 #define CFLGS_OFF_SLAB          (0x80000000UL)
 #define OFF_SLAB(x)     ((x)->flags & CFLGS_OFF_SLAB)
 
@@ -1704,7 +1624,7 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
                 flags |= __GFP_RECLAIMABLE;
 
-        page = alloc_pages_exact_node(nodeid, flags, cachep->gfporder);
+        page = alloc_pages_exact_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
         if (!page)
                 return NULL;
 
@@ -1717,6 +1637,16 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
                         NR_SLAB_UNRECLAIMABLE, nr_pages);
         for (i = 0; i < nr_pages; i++)
                 __SetPageSlab(page + i);
+
+        if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
+                kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
+
+                if (cachep->ctor)
+                        kmemcheck_mark_uninitialized_pages(page, nr_pages);
+                else
+                        kmemcheck_mark_unallocated_pages(page, nr_pages);
+        }
+
         return page_address(page);
 }
 
@@ -1729,6 +1659,8 @@ static void kmem_freepages(struct kmem_cache *cachep, void *addr)
         struct page *page = virt_to_page(addr);
         const unsigned long nr_freed = i;
 
+        kmemcheck_free_shadow(page, cachep->gfporder);
+
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
                 sub_zone_page_state(page_zone(page),
                                 NR_SLAB_RECLAIMABLE, nr_freed);
@@ -3404,6 +3336,9 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
         kmemleak_alloc_recursive(ptr, obj_size(cachep), 1, cachep->flags,
                                  flags);
 
+        if (likely(ptr))
+                kmemcheck_slab_alloc(cachep, flags, ptr, obj_size(cachep));
+
         if (unlikely((flags & __GFP_ZERO) && ptr))
                 memset(ptr, 0, obj_size(cachep));
 
@@ -3464,6 +3399,9 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
                                  flags);
         prefetchw(objp);
 
+        if (likely(objp))
+                kmemcheck_slab_alloc(cachep, flags, objp, obj_size(cachep));
+
         if (unlikely((flags & __GFP_ZERO) && objp))
                 memset(objp, 0, obj_size(cachep));
 
@@ -3580,6 +3518,8 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
         kmemleak_free_recursive(objp, cachep->flags);
         objp = cache_free_debugcheck(cachep, objp, __builtin_return_address(0));
 
+        kmemcheck_slab_free(cachep, objp, obj_size(cachep));
+
         /*
          * Skip calling cache_free_alien() when the platform is not numa.
          * This will avoid cache misses that happen while accessing slabp (which
diff --git a/mm/slub.c b/mm/slub.c
index 1c950775d6be..2701419b0adc 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -18,6 +18,7 @@
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
 #include <linux/kmemtrace.h>
+#include <linux/kmemcheck.h>
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
 #include <linux/kmemleak.h>
@@ -147,7 +148,7 @@
                 SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE)
 
 #define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
-                SLAB_CACHE_DMA)
+                SLAB_CACHE_DMA | SLAB_NOTRACK)
 
 #ifndef ARCH_KMALLOC_MINALIGN
 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
@@ -1071,6 +1072,8 @@ static inline struct page *alloc_slab_page(gfp_t flags, int node,
 {
         int order = oo_order(oo);
 
+        flags |= __GFP_NOTRACK;
+
         if (node == -1)
                 return alloc_pages(flags, order);
         else
@@ -1098,6 +1101,24 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 
                 stat(get_cpu_slab(s, raw_smp_processor_id()), ORDER_FALLBACK);
         }
+
+        if (kmemcheck_enabled
+                && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS)))
+        {
+                int pages = 1 << oo_order(oo);
+
+                kmemcheck_alloc_shadow(page, oo_order(oo), flags, node);
+
+                /*
+                 * Objects from caches that have a constructor don't get
+                 * cleared when they're allocated, so we need to do it here.
+                 */
+                if (s->ctor)
+                        kmemcheck_mark_uninitialized_pages(page, pages);
+                else
+                        kmemcheck_mark_unallocated_pages(page, pages);
+        }
+
         page->objects = oo_objects(oo);
         mod_zone_page_state(page_zone(page),
                 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
@@ -1171,6 +1192,8 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
                 __ClearPageSlubDebug(page);
         }
 
+        kmemcheck_free_shadow(page, compound_order(page));
+
         mod_zone_page_state(page_zone(page),
                 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
                 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
@@ -1626,7 +1649,9 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
         if (unlikely((gfpflags & __GFP_ZERO) && object))
                 memset(object, 0, objsize);
 
+        kmemcheck_slab_alloc(s, gfpflags, object, c->objsize);
         kmemleak_alloc_recursive(object, objsize, 1, s->flags, gfpflags);
+
         return object;
 }
 
@@ -1759,6 +1784,7 @@ static __always_inline void slab_free(struct kmem_cache *s,
         kmemleak_free_recursive(x, s->flags);
         local_irq_save(flags);
         c = get_cpu_slab(s, smp_processor_id());
+        kmemcheck_slab_free(s, object, c->objsize);
         debug_check_no_locks_freed(object, c->objsize);
         if (!(s->flags & SLAB_DEBUG_OBJECTS))
                 debug_check_no_obj_freed(object, c->objsize);
@@ -2633,7 +2659,8 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags)
 
         if (!s || !text || !kmem_cache_open(s, flags, text,
                         realsize, ARCH_KMALLOC_MINALIGN,
-                        SLAB_CACHE_DMA|__SYSFS_ADD_DEFERRED, NULL)) {
+                        SLAB_CACHE_DMA|SLAB_NOTRACK|__SYSFS_ADD_DEFERRED,
+                        NULL)) {
                 kfree(s);
                 kfree(text);
                 goto unlock_out;
@@ -2727,9 +2754,10 @@ EXPORT_SYMBOL(__kmalloc);
 
 static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
 {
-        struct page *page = alloc_pages_node(node, flags | __GFP_COMP,
-                                                get_order(size));
+        struct page *page;
 
+        flags |= __GFP_COMP | __GFP_NOTRACK;
+        page = alloc_pages_node(node, flags, get_order(size));
         if (page)
                 return page_address(page);
         else
@@ -4412,6 +4440,8 @@ static char *create_unique_id(struct kmem_cache *s)
                 *p++ = 'a';
         if (s->flags & SLAB_DEBUG_FREE)
                 *p++ = 'F';
+        if (!(s->flags & SLAB_NOTRACK))
+                *p++ = 't';
         if (p != name + 1)
                 *p++ = '-';
         p += sprintf(p, "%07d", s->size);