1 files changed, 139 insertions, 34 deletions

diff --git a/mm/slub.c b/mm/slub.c
index 7ab54ecbd3f3..b2b0c78ae35d 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -9,6 +9,7 @@
  */
 
 #include <linux/mm.h>
+#include <linux/swap.h> /* struct reclaim_state */
 #include <linux/module.h>
 #include <linux/bit_spinlock.h>
 #include <linux/interrupt.h>
@@ -16,9 +17,11 @@
 #include <linux/slab.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
-#include <trace/kmemtrace.h>
+#include <linux/kmemtrace.h>
+#include <linux/kmemcheck.h>
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
+#include <linux/kmemleak.h>
 #include <linux/mempolicy.h>
 #include <linux/ctype.h>
 #include <linux/debugobjects.h>
@@ -142,10 +145,10 @@
  * Set of flags that will prevent slab merging
  */
 #define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
-                SLAB_TRACE | SLAB_DESTROY_BY_RCU)
+                SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE)
 
 #define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \
-                SLAB_CACHE_DMA)
+                SLAB_CACHE_DMA | SLAB_NOTRACK)
 
 #ifndef ARCH_KMALLOC_MINALIGN
 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
@@ -176,6 +179,12 @@ static enum {
         SYSFS           /* Sysfs up */
 } slab_state = DOWN;
 
+/*
+ * The slab allocator is initialized with interrupts disabled. Therefore, make
+ * sure early boot allocations don't accidentally enable interrupts.
+ */
+static gfp_t slab_gfp_mask __read_mostly = SLAB_GFP_BOOT_MASK;
+
 /* A list of all slab caches on the system */
 static DECLARE_RWSEM(slub_lock);
 static LIST_HEAD(slab_caches);
@@ -831,6 +840,11 @@ static inline unsigned long slabs_node(struct kmem_cache *s, int node)
         return atomic_long_read(&n->nr_slabs);
 }
 
+static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
+{
+        return atomic_long_read(&n->nr_slabs);
+}
+
 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
 {
         struct kmem_cache_node *n = get_node(s, node);
@@ -1049,6 +1063,8 @@ static inline unsigned long kmem_cache_flags(unsigned long objsize,
 
 static inline unsigned long slabs_node(struct kmem_cache *s, int node)
                                                         { return 0; }
+static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
+                                                        { return 0; }
 static inline void inc_slabs_node(struct kmem_cache *s, int node,
                                                         int objects) {}
 static inline void dec_slabs_node(struct kmem_cache *s, int node,
@@ -1063,6 +1079,8 @@ static inline struct page *alloc_slab_page(gfp_t flags, int node,
 {
         int order = oo_order(oo);
 
+        flags |= __GFP_NOTRACK;
+
         if (node == -1)
                 return alloc_pages(flags, order);
         else
@@ -1090,6 +1108,24 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 
                 stat(get_cpu_slab(s, raw_smp_processor_id()), ORDER_FALLBACK);
         }
+
+        if (kmemcheck_enabled
+                && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS)))
+        {
+                int pages = 1 << oo_order(oo);
+
+                kmemcheck_alloc_shadow(page, oo_order(oo), flags, node);
+
+                /*
+                 * Objects from caches that have a constructor don't get
+                 * cleared when they're allocated, so we need to do it here.
+                 */
+                if (s->ctor)
+                        kmemcheck_mark_uninitialized_pages(page, pages);
+                else
+                        kmemcheck_mark_unallocated_pages(page, pages);
+        }
+
         page->objects = oo_objects(oo);
         mod_zone_page_state(page_zone(page),
                 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
@@ -1163,6 +1199,8 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
                 __ClearPageSlubDebug(page);
         }
 
+        kmemcheck_free_shadow(page, compound_order(page));
+
         mod_zone_page_state(page_zone(page),
                 (s->flags & SLAB_RECLAIM_ACCOUNT) ?
                 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
@@ -1170,6 +1208,8 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
 
         __ClearPageSlab(page);
         reset_page_mapcount(page);
+        if (current->reclaim_state)
+                current->reclaim_state->reclaimed_slab += pages;
         __free_pages(page, order);
 }
 
@@ -1481,6 +1521,65 @@ static inline int node_match(struct kmem_cache_cpu *c, int node)
         return 1;
 }
 
+static int count_free(struct page *page)
+{
+        return page->objects - page->inuse;
+}
+
+static unsigned long count_partial(struct kmem_cache_node *n,
+                                        int (*get_count)(struct page *))
+{
+        unsigned long flags;
+        unsigned long x = 0;
+        struct page *page;
+
+        spin_lock_irqsave(&n->list_lock, flags);
+        list_for_each_entry(page, &n->partial, lru)
+                x += get_count(page);
+        spin_unlock_irqrestore(&n->list_lock, flags);
+        return x;
+}
+
+static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
+{
+#ifdef CONFIG_SLUB_DEBUG
+        return atomic_long_read(&n->total_objects);
+#else
+        return 0;
+#endif
+}
+
+static noinline void
+slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
+{
+        int node;
+
+        printk(KERN_WARNING
+                "SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n",
+                nid, gfpflags);
+        printk(KERN_WARNING "  cache: %s, object size: %d, buffer size: %d, "
+                "default order: %d, min order: %d\n", s->name, s->objsize,
+                s->size, oo_order(s->oo), oo_order(s->min));
+
+        for_each_online_node(node) {
+                struct kmem_cache_node *n = get_node(s, node);
+                unsigned long nr_slabs;
+                unsigned long nr_objs;
+                unsigned long nr_free;
+
+                if (!n)
+                        continue;
+
+                nr_free  = count_partial(n, count_free);
+                nr_slabs = node_nr_slabs(n);
+                nr_objs  = node_nr_objs(n);
+
+                printk(KERN_WARNING
+                        "  node %d: slabs: %ld, objs: %ld, free: %ld\n",
+                        node, nr_slabs, nr_objs, nr_free);
+        }
+}
+
 /*
  * Slow path. The lockless freelist is empty or we need to perform
  * debugging duties.
@@ -1562,6 +1661,8 @@ new_slab:
                 c->page = new;
                 goto load_freelist;
         }
+        if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
+                slab_out_of_memory(s, gfpflags, node);
         return NULL;
 debug:
         if (!alloc_debug_processing(s, c->page, object, addr))
@@ -1591,6 +1692,8 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
         unsigned long flags;
         unsigned int objsize;
 
+        gfpflags &= slab_gfp_mask;
+
         lockdep_trace_alloc(gfpflags);
         might_sleep_if(gfpflags & __GFP_WAIT);
 
@@ -1614,6 +1717,9 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
         if (unlikely((gfpflags & __GFP_ZERO) && object))
                 memset(object, 0, objsize);
 
+        kmemcheck_slab_alloc(s, gfpflags, object, c->objsize);
+        kmemleak_alloc_recursive(object, objsize, 1, s->flags, gfpflags);
+
         return object;
 }
 
@@ -1743,8 +1849,10 @@ static __always_inline void slab_free(struct kmem_cache *s,
         struct kmem_cache_cpu *c;
         unsigned long flags;
 
+        kmemleak_free_recursive(x, s->flags);
         local_irq_save(flags);
         c = get_cpu_slab(s, smp_processor_id());
+        kmemcheck_slab_free(s, object, c->objsize);
         debug_check_no_locks_freed(object, c->objsize);
         if (!(s->flags & SLAB_DEBUG_OBJECTS))
                 debug_check_no_obj_freed(object, c->objsize);
@@ -1909,7 +2017,7 @@ static inline int calculate_order(int size)
          * Doh this slab cannot be placed using slub_max_order.
          */
         order = slab_order(size, 1, MAX_ORDER, 1);
-        if (order <= MAX_ORDER)
+        if (order < MAX_ORDER)
                 return order;
         return -ENOSYS;
 }
@@ -2522,6 +2630,7 @@ __setup("slub_min_order=", setup_slub_min_order);
 static int __init setup_slub_max_order(char *str)
 {
         get_option(&str, &slub_max_order);
+        slub_max_order = min(slub_max_order, MAX_ORDER - 1);
 
         return 1;
 }
@@ -2553,13 +2662,16 @@ static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s,
         if (gfp_flags & SLUB_DMA)
                 flags = SLAB_CACHE_DMA;
 
-        down_write(&slub_lock);
+        /*
+         * This function is called with IRQs disabled during early-boot on
+         * single CPU so there's no need to take slub_lock here.
+         */
         if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN,
                                                                 flags, NULL))
                 goto panic;
 
         list_add(&s->list, &slab_caches);
-        up_write(&slub_lock);
+
         if (sysfs_slab_add(s))
                 goto panic;
         return s;
@@ -2615,7 +2727,8 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags)
 
         if (!s || !text || !kmem_cache_open(s, flags, text,
                         realsize, ARCH_KMALLOC_MINALIGN,
-                        SLAB_CACHE_DMA|__SYSFS_ADD_DEFERRED, NULL)) {
+                        SLAB_CACHE_DMA|SLAB_NOTRACK|__SYSFS_ADD_DEFERRED,
+                        NULL)) {
                 kfree(s);
                 kfree(text);
                 goto unlock_out;
@@ -2709,9 +2822,10 @@ EXPORT_SYMBOL(__kmalloc);
 
 static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
 {
-        struct page *page = alloc_pages_node(node, flags | __GFP_COMP,
-                                                get_order(size));
+        struct page *page;
 
+        flags |= __GFP_COMP | __GFP_NOTRACK;
+        page = alloc_pages_node(node, flags, get_order(size));
         if (page)
                 return page_address(page);
         else
@@ -3017,7 +3131,7 @@ void __init kmem_cache_init(void)
          * kmem_cache_open for slab_state == DOWN.
          */
         create_kmalloc_cache(&kmalloc_caches[0], "kmem_cache_node",
-                sizeof(struct kmem_cache_node), GFP_KERNEL);
+                sizeof(struct kmem_cache_node), GFP_NOWAIT);
         kmalloc_caches[0].refcount = -1;
         caches++;
 
@@ -3030,16 +3144,16 @@ void __init kmem_cache_init(void)
         /* Caches that are not of the two-to-the-power-of size */
         if (KMALLOC_MIN_SIZE <= 64) {
                 create_kmalloc_cache(&kmalloc_caches[1],
-                                "kmalloc-96", 96, GFP_KERNEL);
+                                "kmalloc-96", 96, GFP_NOWAIT);
                 caches++;
                 create_kmalloc_cache(&kmalloc_caches[2],
-                                "kmalloc-192", 192, GFP_KERNEL);
+                                "kmalloc-192", 192, GFP_NOWAIT);
                 caches++;
         }
 
         for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
                 create_kmalloc_cache(&kmalloc_caches[i],
-                        "kmalloc", 1 << i, GFP_KERNEL);
+                        "kmalloc", 1 << i, GFP_NOWAIT);
                 caches++;
         }
 
@@ -3076,7 +3190,7 @@ void __init kmem_cache_init(void)
         /* Provide the correct kmalloc names now that the caches are up */
         for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++)
                 kmalloc_caches[i]. name =
-                        kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i);
+                        kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
 
 #ifdef CONFIG_SMP
         register_cpu_notifier(&slab_notifier);
@@ -3094,6 +3208,14 @@ void __init kmem_cache_init(void)
                 nr_cpu_ids, nr_node_ids);
 }
 
+void __init kmem_cache_init_late(void)
+{
+        /*
+         * Interrupts are enabled now so all GFP allocations are safe.
+         */
+        slab_gfp_mask = __GFP_BITS_MASK;
+}
+
 /*
  * Find a mergeable slab cache
  */
@@ -3314,20 +3436,6 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
 }
 
 #ifdef CONFIG_SLUB_DEBUG
-static unsigned long count_partial(struct kmem_cache_node *n,
-                                        int (*get_count)(struct page *))
-{
-        unsigned long flags;
-        unsigned long x = 0;
-        struct page *page;
-
-        spin_lock_irqsave(&n->list_lock, flags);
-        list_for_each_entry(page, &n->partial, lru)
-                x += get_count(page);
-        spin_unlock_irqrestore(&n->list_lock, flags);
-        return x;
-}
-
 static int count_inuse(struct page *page)
 {
         return page->inuse;
@@ -3338,11 +3446,6 @@ static int count_total(struct page *page)
         return page->objects;
 }
 
-static int count_free(struct page *page)
-{
-        return page->objects - page->inuse;
-}
-
 static int validate_slab(struct kmem_cache *s, struct page *page,
                                                 unsigned long *map)
 {
@@ -3711,7 +3814,7 @@ static int list_locations(struct kmem_cache *s, char *buf,
                                                  to_cpumask(l->cpus));
                 }
 
-                if (num_online_nodes() > 1 && !nodes_empty(l->nodes) &&
+                if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
                                 len < PAGE_SIZE - 60) {
                         len += sprintf(buf + len, " nodes=");
                         len += nodelist_scnprintf(buf + len, PAGE_SIZE - len - 50,
@@ -4386,6 +4489,8 @@ static char *create_unique_id(struct kmem_cache *s)
                 *p++ = 'a';
         if (s->flags & SLAB_DEBUG_FREE)
                 *p++ = 'F';
+        if (!(s->flags & SLAB_NOTRACK))
+                *p++ = 't';
         if (p != name + 1)
                 *p++ = '-';
         p += sprintf(p, "%07d", s->size);