2 files changed, 49 insertions, 159 deletions

diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index 1e14beb23f9b..17ebe0f89bf3 100644
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -69,6 +69,7 @@ struct kmem_cache_order_objects {
  * Slab cache management.
  */
 struct kmem_cache {
+        struct kmem_cache_cpu *cpu_slab;
         /* Used for retriving partial slabs etc */
         unsigned long flags;
         int size;               /* The size of an object including meta data */
@@ -104,11 +105,6 @@ struct kmem_cache {
         int remote_node_defrag_ratio;
         struct kmem_cache_node *node[MAX_NUMNODES];
 #endif
-#ifdef CONFIG_SMP
-        struct kmem_cache_cpu *cpu_slab[NR_CPUS];
-#else
-        struct kmem_cache_cpu cpu_slab;
-#endif
 };
 
 /*
diff --git a/mm/slub.c b/mm/slub.c
index 8d71aaf888d7..d6c9ecf629d5 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -242,15 +242,6 @@ static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
 #endif
 }
 
-static inline struct kmem_cache_cpu *get_cpu_slab(struct kmem_cache *s, int cpu)
-{
-#ifdef CONFIG_SMP
-        return s->cpu_slab[cpu];
-#else
-        return &s->cpu_slab;
-#endif
-}
-
 /* Verify that a pointer has an address that is valid within a slab page */
 static inline int check_valid_pointer(struct kmem_cache *s,
                                 struct page *page, const void *object)
@@ -1124,7 +1115,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
                 if (!page)
                         return NULL;
 
-                stat(get_cpu_slab(s, raw_smp_processor_id()), ORDER_FALLBACK);
+                stat(this_cpu_ptr(s->cpu_slab), ORDER_FALLBACK);
         }
 
         if (kmemcheck_enabled
@@ -1422,7 +1413,7 @@ static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node)
 static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
 {
         struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-        struct kmem_cache_cpu *c = get_cpu_slab(s, smp_processor_id());
+        struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab);
 
         __ClearPageSlubFrozen(page);
         if (page->inuse) {
@@ -1454,7 +1445,7 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
                         slab_unlock(page);
                 } else {
                         slab_unlock(page);
-                        stat(get_cpu_slab(s, raw_smp_processor_id()), FREE_SLAB);
+                        stat(__this_cpu_ptr(s->cpu_slab), FREE_SLAB);
                         discard_slab(s, page);
                 }
         }
@@ -1507,7 +1498,7 @@ static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
  */
 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
 {
-        struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
+        struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
 
         if (likely(c && c->page))
                 flush_slab(s, c);
@@ -1673,7 +1664,7 @@ new_slab:
                 local_irq_disable();
 
         if (new) {
-                c = get_cpu_slab(s, smp_processor_id());
+                c = __this_cpu_ptr(s->cpu_slab);
                 stat(c, ALLOC_SLAB);
                 if (c->page)
                         flush_slab(s, c);
@@ -1711,7 +1702,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
         void **object;
         struct kmem_cache_cpu *c;
         unsigned long flags;
-        unsigned int objsize;
+        unsigned long objsize;
 
         gfpflags &= gfp_allowed_mask;
 
@@ -1722,14 +1713,14 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
                 return NULL;
 
         local_irq_save(flags);
-        c = get_cpu_slab(s, smp_processor_id());
+        c = __this_cpu_ptr(s->cpu_slab);
+        object = c->freelist;
         objsize = c->objsize;
-        if (unlikely(!c->freelist || !node_match(c, node)))
+        if (unlikely(!object || !node_match(c, node)))
 
                 object = __slab_alloc(s, gfpflags, node, addr, c);
 
         else {
-                object = c->freelist;
                 c->freelist = object[c->offset];
                 stat(c, ALLOC_FASTPATH);
         }
@@ -1800,7 +1791,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
         void **object = (void *)x;
         struct kmem_cache_cpu *c;
 
-        c = get_cpu_slab(s, raw_smp_processor_id());
+        c = __this_cpu_ptr(s->cpu_slab);
         stat(c, FREE_SLOWPATH);
         slab_lock(page);
 
@@ -1872,7 +1863,7 @@ static __always_inline void slab_free(struct kmem_cache *s,
 
         kmemleak_free_recursive(x, s->flags);
         local_irq_save(flags);
-        c = get_cpu_slab(s, smp_processor_id());
+        c = __this_cpu_ptr(s->cpu_slab);
         kmemcheck_slab_free(s, object, c->objsize);
         debug_check_no_locks_freed(object, c->objsize);
         if (!(s->flags & SLAB_DEBUG_OBJECTS))
@@ -2095,130 +2086,28 @@ init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
 #endif
 }
 
-#ifdef CONFIG_SMP
-/*
- * Per cpu array for per cpu structures.
- *
- * The per cpu array places all kmem_cache_cpu structures from one processor
- * close together meaning that it becomes possible that multiple per cpu
- * structures are contained in one cacheline. This may be particularly
- * beneficial for the kmalloc caches.
- *
- * A desktop system typically has around 60-80 slabs. With 100 here we are
- * likely able to get per cpu structures for all caches from the array defined
- * here. We must be able to cover all kmalloc caches during bootstrap.
- *
- * If the per cpu array is exhausted then fall back to kmalloc
- * of individual cachelines. No sharing is possible then.
- */
-#define NR_KMEM_CACHE_CPU 100
-
-static DEFINE_PER_CPU(struct kmem_cache_cpu [NR_KMEM_CACHE_CPU],
-                      kmem_cache_cpu);
-
-static DEFINE_PER_CPU(struct kmem_cache_cpu *, kmem_cache_cpu_free);
-static DECLARE_BITMAP(kmem_cach_cpu_free_init_once, CONFIG_NR_CPUS);
-
-static struct kmem_cache_cpu *alloc_kmem_cache_cpu(struct kmem_cache *s,
-                                                        int cpu, gfp_t flags)
-{
-        struct kmem_cache_cpu *c = per_cpu(kmem_cache_cpu_free, cpu);
-
-        if (c)
-                per_cpu(kmem_cache_cpu_free, cpu) =
-                                (void *)c->freelist;
-        else {
-                /* Table overflow: So allocate ourselves */
-                c = kmalloc_node(
-                        ALIGN(sizeof(struct kmem_cache_cpu), cache_line_size()),
-                        flags, cpu_to_node(cpu));
+static DEFINE_PER_CPU(struct kmem_cache_cpu, kmalloc_percpu[SLUB_PAGE_SHIFT]);