2 files changed, 73 insertions, 9 deletions

diff --git a/mm/slab.c b/mm/slab.c
index 29c8716eb7a..e901a36e252 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -1731,6 +1731,52 @@ static int __init cpucache_init(void)
 }
 __initcall(cpucache_init);
 
+static noinline void
+slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
+{
+        struct kmem_list3 *l3;
+        struct slab *slabp;
+        unsigned long flags;
+        int node;
+
+        printk(KERN_WARNING
+                "SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n",
+                nodeid, gfpflags);
+        printk(KERN_WARNING "  cache: %s, object size: %d, order: %d\n",
+                cachep->name, cachep->buffer_size, cachep->gfporder);
+
+        for_each_online_node(node) {
+                unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
+                unsigned long active_slabs = 0, num_slabs = 0;
+
+                l3 = cachep->nodelists[node];
+                if (!l3)
+                        continue;
+
+                spin_lock_irqsave(&l3->list_lock, flags);
+                list_for_each_entry(slabp, &l3->slabs_full, list) {
+                        active_objs += cachep->num;
+                        active_slabs++;
+                }
+                list_for_each_entry(slabp, &l3->slabs_partial, list) {
+                        active_objs += slabp->inuse;
+                        active_slabs++;
+                }
+                list_for_each_entry(slabp, &l3->slabs_free, list)
+                        num_slabs++;
+
+                free_objects += l3->free_objects;
+                spin_unlock_irqrestore(&l3->list_lock, flags);
+
+                num_slabs += active_slabs;
+                num_objs = num_slabs * cachep->num;
+                printk(KERN_WARNING
+                        "  node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n",
+                        node, active_slabs, num_slabs, active_objs, num_objs,
+                        free_objects);
+        }
+}
+
 /*
  * Interface to system's page allocator. No need to hold the cache-lock.
  *
@@ -1757,8 +1803,11 @@ static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid)
                 flags |= __GFP_RECLAIMABLE;
 
         page = alloc_pages_exact_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
-        if (!page)
+        if (!page) {
+                if (!(flags & __GFP_NOWARN) && printk_ratelimit())
+                        slab_out_of_memory(cachep, flags, nodeid);
                 return NULL;
+        }
 
         nr_pages = (1 << cachep->gfporder);
         if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
@@ -3696,13 +3745,12 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp,
 
         if (likely(ac->avail < ac->limit)) {
                 STATS_INC_FREEHIT(cachep);
-                ac->entry[ac->avail++] = objp;
-                return;
         } else {
                 STATS_INC_FREEMISS(cachep);
                 cache_flusharray(cachep, ac);
-                ac->entry[ac->avail++] = objp;
         }
+
+        ac->entry[ac->avail++] = objp;
 }
 
 /**
diff --git a/mm/slub.c b/mm/slub.c
index dcbb1926cb7..ffe13fdf814 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -29,6 +29,7 @@
 #include <linux/math64.h>
 #include <linux/fault-inject.h>
 #include <linux/stacktrace.h>
+#include <linux/prefetch.h>
 
 #include <trace/events/kmem.h>
 
@@ -269,6 +270,11 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object)
         return *(void **)(object + s->offset);
 }
 
+static void prefetch_freepointer(const struct kmem_cache *s, void *object)
+{
+        prefetch(object + s->offset);
+}
+
 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
 {
         void *p;
@@ -1560,6 +1566,7 @@ static void *get_partial_node(struct kmem_cache *s,
                 } else {
                         page->freelist = t;
                         available = put_cpu_partial(s, page, 0);
+                        stat(s, CPU_PARTIAL_NODE);
                 }
                 if (kmem_cache_debug(s) || available > s->cpu_partial / 2)
                         break;
@@ -1983,6 +1990,7 @@ int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
                                 local_irq_restore(flags);
                                 pobjects = 0;
                                 pages = 0;
+                                stat(s, CPU_PARTIAL_DRAIN);
                         }
                 }
 
@@ -1994,7 +2002,6 @@ int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
                 page->next = oldpage;
 
         } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage);
-        stat(s, CPU_PARTIAL_FREE);
         return pobjects;
 }
 
@@ -2327,6 +2334,8 @@ redo:
                 object = __slab_alloc(s, gfpflags, node, addr, c);
 
         else {
+                void *next_object = get_freepointer_safe(s, object);
+
                 /*
                  * The cmpxchg will only match if there was no additional
                  * operation and if we are on the right processor.
@@ -2342,11 +2351,12 @@ redo:
                 if (unlikely(!this_cpu_cmpxchg_double(
                                 s->cpu_slab->freelist, s->cpu_slab->tid,
                                 object, tid,
-                                get_freepointer_safe(s, object), next_tid(tid)))) {
+                                next_object, next_tid(tid)))) {
 
                         note_cmpxchg_failure("slab_alloc", s, tid);
                         goto redo;
                 }
+                prefetch_freepointer(s, next_object);
                 stat(s, ALLOC_FASTPATH);
         }
 
@@ -2483,9 +2493,10 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
                  * If we just froze the page then put it onto the
                  * per cpu partial list.
                  */
-                if (new.frozen && !was_frozen)
+                if (new.frozen && !was_frozen) {
                         put_cpu_partial(s, page, 1);
-
+                        stat(s, CPU_PARTIAL_FREE);
+                }
                 /*
                  * The list lock was not taken therefore no list
                  * activity can be necessary.
@@ -3947,13 +3958,14 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size,
                 if (kmem_cache_open(s, n,
                                 size, align, flags, ctor)) {
                         list_add(&s->list, &slab_caches);
+                        up_write(&slub_lock);
                         if (sysfs_slab_add(s)) {
+                                down_write(&slub_lock);
                                 list_del(&s->list);
                                 kfree(n);
                                 kfree(s);
                                 goto err;
                         }
-                        up_write(&slub_lock);
                         return s;
                 }
                 kfree(n);
@@ -5077,6 +5089,8 @@ STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
 STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
 STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
 STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
+STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
+STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
 #endif
 
 static struct attribute *slab_attrs[] = {
@@ -5142,6 +5156,8 @@ static struct attribute *slab_attrs[] = {
         &cmpxchg_double_cpu_fail_attr.attr,
         &cpu_partial_alloc_attr.attr,
         &cpu_partial_free_attr.attr,
+        &cpu_partial_node_attr.attr,
+        &cpu_partial_drain_attr.attr,
 #endif
 #ifdef CONFIG_FAILSLAB
         &failslab_attr.attr,