1 files changed, 66 insertions, 14 deletions

diff --git a/mm/slub.c b/mm/slub.c
index 0996b3be751a..b2b0c78ae35d 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -17,9 +17,11 @@
 #include <linux/slab.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
-#include <trace/kmemtrace.h>
+#include <linux/kmemtrace.h>
+#include <linux/kmemcheck.h>
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
+#include <linux/kmemleak.h>
 #include <linux/mempolicy.h>
 #include <linux/ctype.h>
 #include <linux/debugobjects.h>
@@ -143,10 +145,10 @@
  * Set of flags that will prevent slab merging
  */
 #define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
-                SLAB_TRACE | SLAB_DESTROY_BY_RCU)
+                SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE)
 
 #define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
-                SLAB_CACHE_DMA)
+                SLAB_CACHE_DMA | SLAB_NOTRACK)
 
 #ifndef ARCH_KMALLOC_MINALIGN
 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
@@ -177,6 +179,12 @@ static enum {
         SYSFS           /* Sysfs up */
 } slab_state = DOWN;
 
+/*
+ * The slab allocator is initialized with interrupts disabled. Therefore, make
+ * sure early boot allocations don't accidentally enable interrupts.
+ */
+static gfp_t slab_gfp_mask __read_mostly = SLAB_GFP_BOOT_MASK;
+
 /* A list of all slab caches on the system */
 static DECLARE_RWSEM(slub_lock);
 static LIST_HEAD(slab_caches);
@@ -1071,6 +1079,8 @@ static inline struct page *alloc_slab_page(gfp_t flags, int node,
 {
         int order = oo_order(oo);
 
+        flags |= __GFP_NOTRACK;
+
         if (node == -1)
                 return alloc_pages(flags, order);
         else
@@ -1098,6 +1108,24 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 
                 stat(get_cpu_slab(s, raw_smp_processor_id()), ORDER_FALLBACK);
         }
+
+        if (kmemcheck_enabled
+                && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS)))
+        {
+                int pages = 1 << oo_order(oo);
+
+                kmemcheck_alloc_shadow(page, oo_order(oo), flags, node);
+
+                /*
+                 * Objects from caches that have a constructor don't get
+                 * cleared when they're allocated, so we need to do it here.
+                 */
+                if (s->ctor)
+                        kmemcheck_mark_uninitialized_pages(page, pages);
+                else
+                        kmemcheck_mark_unallocated_pages(page, pages);
+        }
+
         page->objects = oo_objects(oo);
         mod_zone_page_state(page_zone(page),
                 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
@@ -1171,6 +1199,8 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
                 __ClearPageSlubDebug(page);
         }
 
+        kmemcheck_free_shadow(page, compound_order(page));
+
         mod_zone_page_state(page_zone(page),
                 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
                 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
@@ -1662,6 +1692,8 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
         unsigned long flags;
         unsigned int objsize;
 
+        gfpflags &= slab_gfp_mask;
+
         lockdep_trace_alloc(gfpflags);
         might_sleep_if(gfpflags & __GFP_WAIT);
 
@@ -1685,6 +1717,9 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
         if (unlikely((gfpflags & __GFP_ZERO) && object))
                 memset(object, 0, objsize);
 
+        kmemcheck_slab_alloc(s, gfpflags, object, c->objsize);
+        kmemleak_alloc_recursive(object, objsize, 1, s->flags, gfpflags);
+
         return object;
 }
 
@@ -1814,8 +1849,10 @@ static __always_inline void slab_free(struct kmem_cache *s,
         struct kmem_cache_cpu *c;
         unsigned long flags;
 
+        kmemleak_free_recursive(x, s->flags);
         local_irq_save(flags);
         c = get_cpu_slab(s, smp_processor_id());
+        kmemcheck_slab_free(s, object, c->objsize);
         debug_check_no_locks_freed(object, c->objsize);
         if (!(s->flags & SLAB_DEBUG_OBJECTS))
                 debug_check_no_obj_freed(object, c->objsize);
@@ -2625,13 +2662,16 @@ static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s,
         if (gfp_flags & SLUB_DMA)
                 flags = SLAB_CACHE_DMA;
 
-        down_write(&slub_lock);
+        /*
+         * This function is called with IRQs disabled during early-boot on
+         * single CPU so there's no need to take slub_lock here.
+         */
         if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN,
                                                                 flags, NULL))
                 goto panic;
 
         list_add(&s->list, &slab_caches);
-        up_write(&slub_lock);
+
         if (sysfs_slab_add(s))
                 goto panic;
         return s;
@@ -2687,7 +2727,8 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags)
 
         if (!s || !text || !kmem_cache_open(s, flags, text,
                         realsize, ARCH_KMALLOC_MINALIGN,
-                        SLAB_CACHE_DMA|__SYSFS_ADD_DEFERRED, NULL)) {
+                        SLAB_CACHE_DMA|SLAB_NOTRACK|__SYSFS_ADD_DEFERRED,
+                        NULL)) {
                 kfree(s);
                 kfree(text);
                 goto unlock_out;
@@ -2781,9 +2822,10 @@ EXPORT_SYMBOL(__kmalloc);
 
 static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
 {
-        struct page *page = alloc_pages_node(node, flags | __GFP_COMP,
-                                                get_order(size));
+        struct page *page;
 
+        flags |= __GFP_COMP | __GFP_NOTRACK;
+        page = alloc_pages_node(node, flags, get_order(size));
         if (page)
                 return page_address(page);
         else
@@ -3089,7 +3131,7 @@ void __init kmem_cache_init(void)
          * kmem_cache_open for slab_state == DOWN.
          */
         create_kmalloc_cache(&kmalloc_caches[0], "kmem_cache_node",
-                sizeof(struct kmem_cache_node), GFP_KERNEL);
+                sizeof(struct kmem_cache_node), GFP_NOWAIT);
         kmalloc_caches[0].refcount = -1;
         caches++;
 
@@ -3102,16 +3144,16 @@ void __init kmem_cache_init(void)
         /* Caches that are not of the two-to-the-power-of size */
         if (KMALLOC_MIN_SIZE <= 64) {
                 create_kmalloc_cache(&kmalloc_caches[1],
-                                "kmalloc-96", 96, GFP_KERNEL);
+                                "kmalloc-96", 96, GFP_NOWAIT);
                 caches++;
                 create_kmalloc_cache(&kmalloc_caches[2],
-                                "kmalloc-192", 192, GFP_KERNEL);
+                                "kmalloc-192", 192, GFP_NOWAIT);
                 caches++;
         }
 
         for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
                 create_kmalloc_cache(&kmalloc_caches[i],
-                        "kmalloc", 1 << i, GFP_KERNEL);
+                        "kmalloc", 1 << i, GFP_NOWAIT);
                 caches++;
         }
 
@@ -3148,7 +3190,7 @@ void __init kmem_cache_init(void)
         /* Provide the correct kmalloc names now that the caches are up */
         for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++)
                 kmalloc_caches[i]. name =
-                        kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i);
+                        kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
 
 #ifdef CONFIG_SMP
         register_cpu_notifier(&slab_notifier);
@@ -3166,6 +3208,14 @@ void __init kmem_cache_init(void)
                 nr_cpu_ids, nr_node_ids);
 }
 
+void __init kmem_cache_init_late(void)
+{
+        /*
+         * Interrupts are enabled now so all GFP allocations are safe.
+         */
+        slab_gfp_mask = __GFP_BITS_MASK;
+}
+
 /*
  * Find a mergeable slab cache
  */
@@ -3764,7 +3814,7 @@ static int list_locations(struct kmem_cache *s, char *buf,
                                                  to_cpumask(l->cpus));
                 }
 
-                if (num_online_nodes() > 1 && !nodes_empty(l->nodes) &&
+                if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
                                 len < PAGE_SIZE - 60) {
                         len += sprintf(buf + len, " nodes=");
                         len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50,
@@ -4439,6 +4489,8 @@ static char *create_unique_id(struct kmem_cache *s)
                 *p++ = 'a';
         if (s->flags & SLAB_DEBUG_FREE)
                 *p++ = 'F';
+        if (!(s->flags & SLAB_NOTRACK))
+                *p++ = 't';
         if (p != name + 1)
                 *p++ = '-';
         p += sprintf(p, "%07d", s->size);