1 files changed, 36 insertions, 90 deletions

diff --git a/mm/slab.c b/mm/slab.c
index 18e3164de09a..d08692303f6e 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -114,6 +114,7 @@
 #include        <linux/rtmutex.h>
 #include        <linux/reciprocal_div.h>
 #include        <linux/debugobjects.h>
+#include        <linux/kmemcheck.h>
 
 #include        <asm/cacheflush.h>
 #include        <asm/tlbflush.h>
@@ -179,13 +180,13 @@
                          SLAB_STORE_USER | \
                          SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
                          SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
-                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE)
+                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
 #else
 # define CREATE_MASK    (SLAB_HWCACHE_ALIGN | \
                          SLAB_CACHE_DMA | \
                          SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
                          SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
-                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE)
+                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
 #endif
 
 /*
@@ -380,87 +381,6 @@ static void kmem_list3_init(struct kmem_list3 *parent)
         MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid);  \
         } while (0)
 
-/*
- * struct kmem_cache
- *
- * manages a cache.
- */
-
-struct kmem_cache {
-/* 1) per-cpu data, touched during every alloc/free */
-        struct array_cache *array[NR_CPUS];
-/* 2) Cache tunables. Protected by cache_chain_mutex */
-        unsigned int batchcount;
-        unsigned int limit;
-        unsigned int shared;
-
-        unsigned int buffer_size;
-        u32 reciprocal_buffer_size;
-/* 3) touched by every alloc & free from the backend */
-
-        unsigned int flags;             /* constant flags */
-        unsigned int num;               /* # of objs per slab */
-
-/* 4) cache_grow/shrink */
-        /* order of pgs per slab (2^n) */
-        unsigned int gfporder;
-
-        /* force GFP flags, e.g. GFP_DMA */
-        gfp_t gfpflags;
-
-        size_t colour;                  /* cache colouring range */
-        unsigned int colour_off;        /* colour offset */
-        struct kmem_cache *slabp_cache;
-        unsigned int slab_size;
-        unsigned int dflags;            /* dynamic flags */
-
-        /* constructor func */
-        void (*ctor)(void *obj);
-
-/* 5) cache creation/removal */
-        const char *name;
-        struct list_head next;
-
-/* 6) statistics */
-#if STATS
-        unsigned long num_active;
-        unsigned long num_allocations;
-        unsigned long high_mark;
-        unsigned long grown;
-        unsigned long reaped;
-        unsigned long errors;
-        unsigned long max_freeable;
-        unsigned long node_allocs;
-        unsigned long node_frees;
-        unsigned long node_overflow;
-        atomic_t allochit;
-        atomic_t allocmiss;
-        atomic_t freehit;
-        atomic_t freemiss;
-#endif
-#if DEBUG
-        /*
-         * If debugging is enabled, then the allocator can add additional
-         * fields and/or padding to every object. buffer_size contains the total
-         * object size including these internal fields, the following two
-         * variables contain the offset to the user object and its size.
-         */
-        int obj_offset;
-        int obj_size;
-#endif
-        /*
-         * We put nodelists[] at the end of kmem_cache, because we want to size
-         * this array to nr_node_ids slots instead of MAX_NUMNODES
-         * (see kmem_cache_init())
-         * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
-         * is statically defined, so we reserve the max number of nodes.
-         */
-        struct kmem_list3 *nodelists[MAX_NUMNODES];
-        /*
-         * Do not add fields after nodelists[]
-         */
-};
-
 #define CFLGS_OFF_SLAB          (0x80000000UL)
 #define OFF_SLAB(x)     ((x)->flags & CFLGS_OFF_SLAB)
 
@@ -898,7 +818,6 @@ static void __slab_error(const char *function, struct kmem_cache *cachep,
   */
 
 static int use_alien_caches __read_mostly = 1;
-static int numa_platform __read_mostly = 1;
 static int __init noaliencache_setup(char *s)
 {
         use_alien_caches = 0;
@@ -1457,10 +1376,8 @@ void __init kmem_cache_init(void)
         int order;
         int node;
 
-        if (num_possible_nodes() == 1) {
+        if (num_possible_nodes() == 1)
                 use_alien_caches = 0;
-                numa_platform = 0;
-        }
 
         for (i = 0; i < NUM_INIT_LISTS; i++) {
                 kmem_list3_init(&initkmem_list3[i]);
@@ -1707,7 +1624,7 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
                 flags |= __GFP_RECLAIMABLE;
 
-        page = alloc_pages_node(nodeid, flags, cachep->gfporder);
+        page = alloc_pages_exact_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
         if (!page)
                 return NULL;
 
@@ -1720,6 +1637,16 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
                         NR_SLAB_UNRECLAIMABLE, nr_pages);
         for (i = 0; i < nr_pages; i++)
                 __SetPageSlab(page + i);
+
+        if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
+                kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
+
+                if (cachep->ctor)
+                        kmemcheck_mark_uninitialized_pages(page, nr_pages);
+                else
+                        kmemcheck_mark_unallocated_pages(page, nr_pages);
+        }
+
         return page_address(page);
 }
 
@@ -1732,6 +1659,8 @@ static void kmem_freepages(struct kmem_cache *cachep, void *addr)
         struct page *page = virt_to_page(addr);
         const unsigned long nr_freed = i;
 
+        kmemcheck_free_shadow(page, cachep->gfporder);
+
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
                 sub_zone_page_state(page_zone(page),
                                 NR_SLAB_RECLAIMABLE, nr_freed);
@@ -2379,6 +2308,15 @@ kmem_cache_create (const char *name, size_t size, size_t align,
                 /* really off slab. No need for manual alignment */
                 slab_size =
                     cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab);
+
+#ifdef CONFIG_PAGE_POISONING
+                /* If we're going to use the generic kernel_map_pages()
+                 * poisoning, then it's going to smash the contents of
+                 * the redzone and userword anyhow, so switch them off.
+                 */
+                if (size % PAGE_SIZE == 0 && flags & SLAB_POISON)
+                        flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
+#endif
         }
 
         cachep->colour_off = cache_line_size();
@@ -3261,7 +3199,7 @@ retry:
                 if (local_flags & __GFP_WAIT)
                         local_irq_enable();
                 kmem_flagcheck(cache, flags);
-                obj = kmem_getpages(cache, local_flags, -1);
+                obj = kmem_getpages(cache, local_flags, numa_node_id());
                 if (local_flags & __GFP_WAIT)
                         local_irq_disable();
                 if (obj) {
@@ -3407,6 +3345,9 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
         kmemleak_alloc_recursive(ptr, obj_size(cachep), 1, cachep->flags,
                                  flags);
 
+        if (likely(ptr))
+                kmemcheck_slab_alloc(cachep, flags, ptr, obj_size(cachep));
+
         if (unlikely((flags & __GFP_ZERO) && ptr))
                 memset(ptr, 0, obj_size(cachep));
 
@@ -3467,6 +3408,9 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
                                  flags);
         prefetchw(objp);
 
+        if (likely(objp))
+                kmemcheck_slab_alloc(cachep, flags, objp, obj_size(cachep));
+
         if (unlikely((flags & __GFP_ZERO) && objp))
                 memset(objp, 0, obj_size(cachep));
 
@@ -3583,6 +3527,8 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
         kmemleak_free_recursive(objp, cachep->flags);
         objp = cache_free_debugcheck(cachep, objp, __builtin_return_address(0));
 
+        kmemcheck_slab_free(cachep, objp, obj_size(cachep));
+
         /*
          * Skip calling cache_free_alien() when the platform is not numa.
          * This will avoid cache misses that happen while accessing slabp (which
@@ -3590,7 +3536,7 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
          * variable to skip the call, which is mostly likely to be present in
          * the cache.
          */
-        if (numa_platform && cache_free_alien(cachep, objp))
+        if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
                 return;
 
         if (likely(ac->avail < ac->limit)) {