1 files changed, 98 insertions, 88 deletions

diff --git a/mm/slab.c b/mm/slab.c
index 7dfa481c96ba..bac0f4fcc216 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -490,7 +490,7 @@ static void **dbg_userword(struct kmem_cache *cachep, void *objp)
 
 #endif
 
-#ifdef CONFIG_KMEMTRACE
+#ifdef CONFIG_TRACING
 size_t slab_buffer_size(struct kmem_cache *cachep)
 {
         return cachep->buffer_size;
@@ -604,6 +604,26 @@ static struct kmem_cache cache_cache = {
 
 #define BAD_ALIEN_MAGIC 0x01020304ul
 
+/*
+ * chicken and egg problem: delay the per-cpu array allocation
+ * until the general caches are up.
+ */
+static enum {
+        NONE,
+        PARTIAL_AC,
+        PARTIAL_L3,
+        EARLY,
+        FULL
+} g_cpucache_up;
+
+/*
+ * used by boot code to determine if it can use slab based allocator
+ */
+int slab_is_available(void)
+{
+        return g_cpucache_up >= EARLY;
+}
+
 #ifdef CONFIG_LOCKDEP
 
 /*
@@ -620,40 +640,52 @@ static struct kmem_cache cache_cache = {
 static struct lock_class_key on_slab_l3_key;
 static struct lock_class_key on_slab_alc_key;
 
-static inline void init_lock_keys(void)
-
+static void init_node_lock_keys(int q)
 {
-        int q;
         struct cache_sizes *s = malloc_sizes;
 
-        while (s->cs_size != ULONG_MAX) {
-                for_each_node(q) {
-                        struct array_cache **alc;
-                        int r;
-                        struct kmem_list3 *l3 = s->cs_cachep->nodelists[q];
-                        if (!l3 || OFF_SLAB(s->cs_cachep))
-                                continue;
-                        lockdep_set_class(&l3->list_lock, &on_slab_l3_key);
-                        alc = l3->alien;
-                        /*
-                         * FIXME: This check for BAD_ALIEN_MAGIC
-                         * should go away when common slab code is taught to
-                         * work even without alien caches.
-                         * Currently, non NUMA code returns BAD_ALIEN_MAGIC
-                         * for alloc_alien_cache,
-                         */
-                        if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC)
-                                continue;
-                        for_each_node(r) {
-                                if (alc[r])
-                                        lockdep_set_class(&alc[r]->lock,
-                                             &on_slab_alc_key);
-                        }
+        if (g_cpucache_up != FULL)
+                return;
+
+        for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) {
+                struct array_cache **alc;
+                struct kmem_list3 *l3;
+                int r;
+
+                l3 = s->cs_cachep->nodelists[q];
+                if (!l3 || OFF_SLAB(s->cs_cachep))
+                        continue;
+                lockdep_set_class(&l3->list_lock, &on_slab_l3_key);
+                alc = l3->alien;
+                /*
+                 * FIXME: This check for BAD_ALIEN_MAGIC
+                 * should go away when common slab code is taught to
+                 * work even without alien caches.
+                 * Currently, non NUMA code returns BAD_ALIEN_MAGIC
+                 * for alloc_alien_cache,
+                 */
+                if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC)
+                        continue;
+                for_each_node(r) {
+                        if (alc[r])
+                                lockdep_set_class(&alc[r]->lock,
+                                        &on_slab_alc_key);
                 }
-                s++;
         }
 }
+
+static inline void init_lock_keys(void)
+{
+        int node;
+
+        for_each_node(node)
+                init_node_lock_keys(node);
+}
 #else
+static void init_node_lock_keys(int q)
+{
+}
+
 static inline void init_lock_keys(void)
 {
 }
@@ -665,27 +697,7 @@ static inline void init_lock_keys(void)
 static DEFINE_MUTEX(cache_chain_mutex);
 static struct list_head cache_chain;
 
-/*
- * chicken and egg problem: delay the per-cpu array allocation
- * until the general caches are up.
- */
-static enum {
-        NONE,
-        PARTIAL_AC,
-        PARTIAL_L3,
-        EARLY,
-        FULL
-} g_cpucache_up;
-
-/*
- * used by boot code to determine if it can use slab based allocator
- */
-int slab_is_available(void)
-{
-        return g_cpucache_up >= EARLY;
-}
-
-static DEFINE_PER_CPU(struct delayed_work, reap_work);
+static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
 
 static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
 {
@@ -826,7 +838,7 @@ __setup("noaliencache", noaliencache_setup);
  * objects freed on different nodes from which they were allocated) and the
  * flushing of remote pcps by calling drain_node_pages.
  */
-static DEFINE_PER_CPU(unsigned long, reap_node);
+static DEFINE_PER_CPU(unsigned long, slab_reap_node);
 
 static void init_reap_node(int cpu)
 {
@@ -836,17 +848,17 @@ static void init_reap_node(int cpu)
         if (node == MAX_NUMNODES)
                 node = first_node(node_online_map);
 
-        per_cpu(reap_node, cpu) = node;
+        per_cpu(slab_reap_node, cpu) = node;
 }
 
 static void next_reap_node(void)
 {
-        int node = __get_cpu_var(reap_node);
+        int node = __get_cpu_var(slab_reap_node);
 
         node = next_node(node, node_online_map);
         if (unlikely(node >= MAX_NUMNODES))
                 node = first_node(node_online_map);
-        __get_cpu_var(reap_node) = node;
+        __get_cpu_var(slab_reap_node) = node;
 }
 
 #else
@@ -863,7 +875,7 @@ static void next_reap_node(void)
  */
 static void __cpuinit start_cpu_timer(int cpu)
 {
-        struct delayed_work *reap_work = &per_cpu(reap_work, cpu);
+        struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
 
         /*
          * When this gets called from do_initcalls via cpucache_init(),
@@ -923,7 +935,6 @@ static int transfer_objects(struct array_cache *to,
 
         from->avail -= nr;
         to->avail += nr;
-        to->touched = 1;
         return nr;
 }
 
@@ -971,13 +982,11 @@ static struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
 
         if (limit > 1)
                 limit = 12;
-        ac_ptr = kmalloc_node(memsize, gfp, node);
+        ac_ptr = kzalloc_node(memsize, gfp, node);
         if (ac_ptr) {
                 for_each_node(i) {
-                        if (i == node || !node_online(i)) {
-                                ac_ptr[i] = NULL;
+                        if (i == node || !node_online(i))
                                 continue;
-                        }
                         ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d, gfp);
                         if (!ac_ptr[i]) {
                                 for (i--; i >= 0; i--)
@@ -1027,7 +1036,7 @@ static void __drain_alien_cache(struct kmem_cache *cachep,
  */
 static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3)
 {
-        int node = __get_cpu_var(reap_node);
+        int node = __get_cpu_var(slab_reap_node);
 
         if (l3->alien) {
                 struct array_cache *ac = l3->alien[node];
@@ -1120,7 +1129,7 @@ static void __cpuinit cpuup_canceled(long cpu)
                 if (nc)
                         free_block(cachep, nc->entry, nc->avail, node);
 
-                if (!cpus_empty(*mask)) {
+                if (!cpumask_empty(mask)) {
                         spin_unlock_irq(&l3->list_lock);
                         goto free_array_cache;
                 }
@@ -1254,6 +1263,8 @@ static int __cpuinit cpuup_prepare(long cpu)
                 kfree(shared);
                 free_alien_cache(alien);
         }
+        init_node_lock_keys(node);
+
         return 0;
 bad:
         cpuup_canceled(cpu);
@@ -1286,9 +1297,9 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb,
                  * anything expensive but will only modify reap_work
                  * and reschedule the timer.
                 */
-                cancel_rearming_delayed_work(&per_cpu(reap_work, cpu));
+                cancel_rearming_delayed_work(&per_cpu(slab_reap_work, cpu));
                 /* Now the cache_reaper is guaranteed to be not running. */
-                per_cpu(reap_work, cpu).work.func = NULL;
+                per_cpu(slab_reap_work, cpu).work.func = NULL;
                 break;
         case CPU_DOWN_FAILED:
         case CPU_DOWN_FAILED_FROZEN:
@@ -2261,9 +2272,11 @@ kmem_cache_create (const char *name, size_t size, size_t align,
         /*
          * Determine if the slab management is 'on' or 'off' slab.
          * (bootstrapping cannot cope with offslab caches so don't do
-         * it too early on.)
+         * it too early on. Always use on-slab management when
+         * SLAB_NOLEAKTRACE to avoid recursive calls into kmemleak)
          */
-        if ((size >= (PAGE_SIZE >> 3)) && !slab_early_init)
+        if ((size >= (PAGE_SIZE >> 3)) && !slab_early_init &&
+            !(flags & SLAB_NOLEAKTRACE))
                 /*
                  * Size is large, assume best to place the slab management obj
                  * off-slab (should allow better packing of objs).
@@ -2582,8 +2595,8 @@ static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp,
                  * kmemleak does not treat the ->s_mem pointer as a reference
                  * to the object. Otherwise we will not report the leak.
                  */
-                kmemleak_scan_area(slabp, offsetof(struct slab, list),
-                                   sizeof(struct list_head), local_flags);
+                kmemleak_scan_area(&slabp->list, sizeof(struct list_head),
+                                   local_flags);
                 if (!slabp)
                         return NULL;
         } else {
@@ -2947,8 +2960,10 @@ retry:
         spin_lock(&l3->list_lock);
 
         /* See if we can refill from the shared array */
-        if (l3->shared && transfer_objects(ac, l3->shared, batchcount))
+        if (l3->shared && transfer_objects(ac, l3->shared, batchcount)) {
+                l3->shared->touched = 1;
                 goto alloc_done;
+        }
 
         while (batchcount > 0) {
                 struct list_head *entry;
@@ -3085,7 +3100,7 @@ static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags)
         if (cachep == &cache_cache)
                 return false;
 
-        return should_failslab(obj_size(cachep), flags);
+        return should_failslab(obj_size(cachep), flags, cachep->flags);
 }
 
 static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
@@ -3103,13 +3118,19 @@ static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
         } else {
                 STATS_INC_ALLOCMISS(cachep);
                 objp = cache_alloc_refill(cachep, flags);
+                /*
+                 * the 'ac' may be updated by cache_alloc_refill(),
+                 * and kmemleak_erase() requires its correct value.
+                 */
+                ac = cpu_cache_get(cachep);
         }
         /*
          * To avoid a false negative, if an object that is in one of the
          * per-CPU caches is leaked, we need to make sure kmemleak doesn't
          * treat the array pointers as a reference to the object.
          */
-        kmemleak_erase(&ac->entry[ac->avail]);
+        if (objp)
+                kmemleak_erase(&ac->entry[ac->avail]);
         return objp;
 }
 
@@ -3306,7 +3327,7 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
         cache_alloc_debugcheck_before(cachep, flags);
         local_irq_save(save_flags);
 
-        if (unlikely(nodeid == -1))
+        if (nodeid == -1)
                 nodeid = numa_node_id();
 
         if (unlikely(!cachep->nodelists[nodeid])) {
@@ -3558,7 +3579,7 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 }
 EXPORT_SYMBOL(kmem_cache_alloc);
 
-#ifdef CONFIG_KMEMTRACE
+#ifdef CONFIG_TRACING
 void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
 {
         return __cache_alloc(cachep, flags, __builtin_return_address(0));
@@ -3581,21 +3602,10 @@ EXPORT_SYMBOL(kmem_cache_alloc_notrace);
  */
 int kmem_ptr_validate(struct kmem_cache *cachep, const void *ptr)
 {
-        unsigned long addr = (unsigned long)ptr;
-        unsigned long min_addr = PAGE_OFFSET;
-        unsigned long align_mask = BYTES_PER_WORD - 1;
         unsigned long size = cachep->buffer_size;
         struct page *page;
 
-        if (unlikely(addr < min_addr))
-                goto out;
-        if (unlikely(addr > (unsigned long)high_memory - size))
-                goto out;
-        if (unlikely(addr & align_mask))
-                goto out;
-        if (unlikely(!kern_addr_valid(addr)))
-                goto out;
-        if (unlikely(!kern_addr_valid(addr + size - 1)))
+        if (unlikely(!kern_ptr_validate(ptr, size)))
                 goto out;
         page = virt_to_page(ptr);
         if (unlikely(!PageSlab(page)))
@@ -3621,7 +3631,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node);
 
-#ifdef CONFIG_KMEMTRACE
+#ifdef CONFIG_TRACING
 void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
                                     gfp_t flags,
                                     int nodeid)
@@ -3649,7 +3659,7 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller)
         return ret;
 }
 
-#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_KMEMTRACE)
+#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING)
 void *__kmalloc_node(size_t size, gfp_t flags, int node)
 {
         return __do_kmalloc_node(size, flags, node,
@@ -3669,7 +3679,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
         return __do_kmalloc_node(size, flags, node, NULL);
 }
 EXPORT_SYMBOL(__kmalloc_node);
-#endif /* CONFIG_DEBUG_SLAB */
+#endif /* CONFIG_DEBUG_SLAB || CONFIG_TRACING */
 #endif /* CONFIG_NUMA */
 
 /**
@@ -3701,7 +3711,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
 }
 
 
-#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_KMEMTRACE)
+#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING)
 void *__kmalloc(size_t size, gfp_t flags)
 {
         return __do_kmalloc(size, flags, __builtin_return_address(0));