1 files changed, 24 insertions, 84 deletions

diff --git a/mm/slab.c b/mm/slab.c
index 18e3164de09a..af3376d0a833 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -114,6 +114,7 @@
 #include        <linux/rtmutex.h>
 #include        <linux/reciprocal_div.h>
 #include        <linux/debugobjects.h>
+#include        <linux/kmemcheck.h>
 
 #include        <asm/cacheflush.h>
 #include        <asm/tlbflush.h>
@@ -179,13 +180,13 @@
                          SLAB_STORE_USER | \
                          SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
                          SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
-                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE)
+                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
 #else
 # define CREATE_MASK    (SLAB_HWCACHE_ALIGN | \
                          SLAB_CACHE_DMA | \
                          SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
                          SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
-                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE)
+                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
 #endif
 
 /*
@@ -380,87 +381,6 @@ static void kmem_list3_init(struct kmem_list3 *parent)
         MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid);  \
         } while (0)
 
-/*
- * struct kmem_cache
- *
- * manages a cache.
- */
-
-struct kmem_cache {
-/* 1) per-cpu data, touched during every alloc/free */
-        struct array_cache *array[NR_CPUS];
-/* 2) Cache tunables. Protected by cache_chain_mutex */
-        unsigned int batchcount;
-        unsigned int limit;
-        unsigned int shared;
-
-        unsigned int buffer_size;
-        u32 reciprocal_buffer_size;
-/* 3) touched by every alloc & free from the backend */
-
-        unsigned int flags;             /* constant flags */
-        unsigned int num;               /* # of objs per slab */
-
-/* 4) cache_grow/shrink */
-        /* order of pgs per slab (2^n) */
-        unsigned int gfporder;
-
-        /* force GFP flags, e.g. GFP_DMA */
-        gfp_t gfpflags;
-
-        size_t colour;                  /* cache colouring range */
-        unsigned int colour_off;        /* colour offset */
-        struct kmem_cache *slabp_cache;
-        unsigned int slab_size;
-        unsigned int dflags;            /* dynamic flags */
-
-        /* constructor func */
-        void (*ctor)(void *obj);
-
-/* 5) cache creation/removal */
-        const char *name;
-        struct list_head next;
-
-/* 6) statistics */
-#if STATS
-        unsigned long num_active;
-        unsigned long num_allocations;
-        unsigned long high_mark;
-        unsigned long grown;
-        unsigned long reaped;
-        unsigned long errors;
-        unsigned long max_freeable;
-        unsigned long node_allocs;
-        unsigned long node_frees;
-        unsigned long node_overflow;
-        atomic_t allochit;
-        atomic_t allocmiss;
-        atomic_t freehit;
-        atomic_t freemiss;
-#endif
-#if DEBUG
-        /*
-         * If debugging is enabled, then the allocator can add additional
-         * fields and/or padding to every object. buffer_size contains the total
-         * object size including these internal fields, the following two
-         * variables contain the offset to the user object and its size.
-         */
-        int obj_offset;
-        int obj_size;
-#endif
-        /*
-         * We put nodelists[] at the end of kmem_cache, because we want to size
-         * this array to nr_node_ids slots instead of MAX_NUMNODES
-         * (see kmem_cache_init())
-         * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
-         * is statically defined, so we reserve the max number of nodes.
-         */
-        struct kmem_list3 *nodelists[MAX_NUMNODES];
-        /*
-         * Do not add fields after nodelists[]
-         */
-};
-
 #define CFLGS_OFF_SLAB          (0x80000000UL)
 #define OFF_SLAB(x)     ((x)->flags & CFLGS_OFF_SLAB)
 
@@ -1707,7 +1627,7 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
                 flags |= __GFP_RECLAIMABLE;
 
-        page = alloc_pages_node(nodeid, flags, cachep->gfporder);
+        page = alloc_pages_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
         if (!page)
                 return NULL;
 
@@ -1720,6 +1640,16 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
                         NR_SLAB_UNRECLAIMABLE, nr_pages);
         for (i = 0; i < nr_pages; i++)
                 __SetPageSlab(page + i);
+
+        if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
+                kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
+
+                if (cachep->ctor)
+                        kmemcheck_mark_uninitialized_pages(page, nr_pages);
+                else
+                        kmemcheck_mark_unallocated_pages(page, nr_pages);
+        }
+
         return page_address(page);
 }
 
@@ -1732,6 +1662,8 @@ static void kmem_freepages(struct kmem_cache *cachep, void *addr)
         struct page *page = virt_to_page(addr);
         const unsigned long nr_freed = i;
 
+        kmemcheck_free_shadow(page, cachep->gfporder);
+
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
                 sub_zone_page_state(page_zone(page),
                                 NR_SLAB_RECLAIMABLE, nr_freed);
@@ -3407,6 +3339,9 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
         kmemleak_alloc_recursive(ptr, obj_size(cachep), 1, cachep->flags,
                                  flags);
 
+        if (likely(ptr))
+                kmemcheck_slab_alloc(cachep, flags, ptr, obj_size(cachep));
+
         if (unlikely((flags & __GFP_ZERO) && ptr))
                 memset(ptr, 0, obj_size(cachep));
 
@@ -3467,6 +3402,9 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
                                  flags);
         prefetchw(objp);
 
+        if (likely(objp))
+                kmemcheck_slab_alloc(cachep, flags, objp, obj_size(cachep));
+
         if (unlikely((flags & __GFP_ZERO) && objp))
                 memset(objp, 0, obj_size(cachep));
 
@@ -3583,6 +3521,8 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
         kmemleak_free_recursive(objp, cachep->flags);
         objp = cache_free_debugcheck(cachep, objp, __builtin_return_address(0));
 
+        kmemcheck_slab_free(cachep, objp, obj_size(cachep));
+
         /*
          * Skip calling cache_free_alien() when the platform is not numa.
          * This will avoid cache misses that happen while accessing slabp (which