slab: Common Kmalloc cache determination

Extract the optimized lookup functions from slub and put them into
slab_common.c. Then make slab use these functions as well.

Joonsoo notes that this fixes some issues with constant folding which
also reduces the code size for slub.

https://lkml.org/lkml/2012/10/20/82

Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>

4 files changed, 114 insertions, 142 deletions

diff --git a/mm/slab.c b/mm/slab.c
index 08ba44f81a28..62629b11df38 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -656,40 +656,6 @@ static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
         return cachep->array[smp_processor_id()];
 }
 
-static inline struct kmem_cache *__find_general_cachep(size_t size,
-                                                        gfp_t gfpflags)
-{
-        int i;
-
-#if DEBUG
-        /* This happens if someone tries to call
-         * kmem_cache_create(), or __kmalloc(), before
-         * the generic caches are initialized.
-         */
-        BUG_ON(kmalloc_caches[INDEX_AC] == NULL);
-#endif
-        if (!size)
-                return ZERO_SIZE_PTR;
-
-        i = kmalloc_index(size);
-
-        /*
-         * Really subtle: The last entry with cs->cs_size==ULONG_MAX
-         * has cs_{dma,}cachep==NULL. Thus no special case
-         * for large kmalloc calls required.
-         */
-#ifdef CONFIG_ZONE_DMA
-        if (unlikely(gfpflags & GFP_DMA))
-                return kmalloc_dma_caches[i];
-#endif
-        return kmalloc_caches[i];
-}
-
-static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags)
-{
-        return __find_general_cachep(size, gfpflags);
-}
-
 static size_t slab_mgmt_size(size_t nr_objs, size_t align)
 {
         return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
@@ -2426,7 +2392,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
         cachep->reciprocal_buffer_size = reciprocal_value(size);
 
         if (flags & CFLGS_OFF_SLAB) {
-                cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u);
+                cachep->slabp_cache = kmalloc_slab(slab_size, 0u);
                 /*
                  * This is a possibility for one of the malloc_sizes caches.
                  * But since we go off slab only for object size greater than
@@ -3729,7 +3695,7 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
 {
         struct kmem_cache *cachep;
 
-        cachep = kmem_find_general_cachep(size, flags);
+        cachep = kmalloc_slab(size, flags);
         if (unlikely(ZERO_OR_NULL_PTR(cachep)))
                 return cachep;
         return kmem_cache_alloc_node_trace(cachep, flags, node, size);
@@ -3774,7 +3740,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
          * Then kmalloc uses the uninlined functions instead of the inline
          * functions.
          */
-        cachep = __find_general_cachep(size, flags);
+        cachep = kmalloc_slab(size, flags);
         if (unlikely(ZERO_OR_NULL_PTR(cachep)))
                 return cachep;
         ret = slab_alloc(cachep, flags, caller);
diff --git a/mm/slab.h b/mm/slab.h
index 44c0bd6dc19e..c01bc8921ac5 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -38,6 +38,9 @@ unsigned long calculate_alignment(unsigned long flags,
 #ifndef CONFIG_SLOB
 /* Kmalloc array related functions */
 void create_kmalloc_caches(unsigned long);
+
+/* Find the kmalloc slab corresponding for a certain size */
+struct kmem_cache *kmalloc_slab(size_t, gfp_t);
 #endif
 
 
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 2b0ebb6d071d..6d73f0b7f21c 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -328,6 +328,68 @@ EXPORT_SYMBOL(kmalloc_dma_caches);
 #endif
 
 /*
+ * Conversion table for small slabs sizes / 8 to the index in the
+ * kmalloc array. This is necessary for slabs < 192 since we have non power
+ * of two cache sizes there. The size of larger slabs can be determined using
+ * fls.
+ */
+static s8 size_index[24] = {
+        3,      /* 8 */
+        4,      /* 16 */
+        5,      /* 24 */
+        5,      /* 32 */
+        6,      /* 40 */
+        6,      /* 48 */
+        6,      /* 56 */
+        6,      /* 64 */
+        1,      /* 72 */
+        1,      /* 80 */
+        1,      /* 88 */
+        1,      /* 96 */
+        7,      /* 104 */
+        7,      /* 112 */
+        7,      /* 120 */
+        7,      /* 128 */
+        2,      /* 136 */
+        2,      /* 144 */
+        2,      /* 152 */
+        2,      /* 160 */
+        2,      /* 168 */
+        2,      /* 176 */
+        2,      /* 184 */
+        2       /* 192 */
+};
+
+static inline int size_index_elem(size_t bytes)
+{
+        return (bytes - 1) / 8;
+}
+
+/*
+ * Find the kmem_cache structure that serves a given size of
+ * allocation
+ */
+struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
+{
+        int index;
+
+        if (size <= 192) {
+                if (!size)
+                        return ZERO_SIZE_PTR;
+
+                index = size_index[size_index_elem(size)];
+        } else
+                index = fls(size - 1);
+
+#ifdef CONFIG_ZONE_DMA
+        if (unlikely((flags & SLAB_CACHE_DMA)))
+                return kmalloc_dma_caches[index];
+
+#endif
+        return kmalloc_caches[index];
+}
+
+/*
  * Create the kmalloc array. Some of the regular kmalloc arrays
  * may already have been created because they were needed to
  * enable allocations for slab creation.
@@ -336,6 +398,47 @@ void __init create_kmalloc_caches(unsigned long flags)
 {
         int i;
 
+        /*
+         * Patch up the size_index table if we have strange large alignment
+         * requirements for the kmalloc array. This is only the case for
+         * MIPS it seems. The standard arches will not generate any code here.
+         *
+         * Largest permitted alignment is 256 bytes due to the way we
+         * handle the index determination for the smaller caches.
+         *
+         * Make sure that nothing crazy happens if someone starts tinkering
+         * around with ARCH_KMALLOC_MINALIGN
+         */
+        BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
+                (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
+
+        for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
+                int elem = size_index_elem(i);
+
+                if (elem >= ARRAY_SIZE(size_index))
+                        break;
+                size_index[elem] = KMALLOC_SHIFT_LOW;
+        }
+
+        if (KMALLOC_MIN_SIZE >= 64) {
+                /*
+                 * The 96 byte size cache is not used if the alignment
+                 * is 64 byte.
+                 */
+                for (i = 64 + 8; i <= 96; i += 8)
+                        size_index[size_index_elem(i)] = 7;
+
+        }
+
+        if (KMALLOC_MIN_SIZE >= 128) {
+                /*
+                 * The 192 byte sized cache is not used if the alignment
+                 * is 128 byte. Redirect kmalloc to use the 256 byte cache
+                 * instead.
+                 */
+                for (i = 128 + 8; i <= 192; i += 8)
+                        size_index[size_index_elem(i)] = 8;
+        }
         /* Caches that are not of the two-to-the-power-of size */
         if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1])
                 kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
@@ -379,8 +482,6 @@ void __init create_kmalloc_caches(unsigned long flags)
         }
 #endif
 }
-
-
 #endif /* !CONFIG_SLOB */
 
 
diff --git a/mm/slub.c b/mm/slub.c
index e813c2d30fe0..6184b0821f7e 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2982,7 +2982,7 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
                 s->allocflags |= __GFP_COMP;
 
         if (s->flags & SLAB_CACHE_DMA)
-                s->allocflags |= SLUB_DMA;
+                s->allocflags |= GFP_DMA;
 
         if (s->flags & SLAB_RECLAIM_ACCOUNT)
                 s->allocflags |= __GFP_RECLAIMABLE;
@@ -3210,64 +3210,6 @@ static int __init setup_slub_nomerge(char *str)
 
 __setup("slub_nomerge", setup_slub_nomerge);
 
-/*
- * Conversion table for small slabs sizes / 8 to the index in the
- * kmalloc array. This is necessary for slabs < 192 since we have non power
- * of two cache sizes there. The size of larger slabs can be determined using
- * fls.
- */
-static s8 size_index[24] = {
-        3,      /* 8 */
-        4,      /* 16 */
-        5,      /* 24 */
-        5,      /* 32 */
-        6,      /* 40 */
-        6,      /* 48 */
-        6,      /* 56 */
-        6,      /* 64 */
-        1,      /* 72 */
-        1,      /* 80 */
-        1,      /* 88 */
-        1,      /* 96 */
-        7,      /* 104 */
-        7,      /* 112 */
-        7,      /* 120 */
-        7,      /* 128 */
-        2,      /* 136 */
-        2,      /* 144 */
-        2,      /* 152 */
-        2,      /* 160 */
-        2,      /* 168 */
-        2,      /* 176 */
-        2,      /* 184 */
-        2       /* 192 */
-};
-
-static inline int size_index_elem(size_t bytes)
-{
-        return (bytes - 1) / 8;
-}
-
-static struct kmem_cache *get_slab(size_t size, gfp_t flags)
-{
-        int index;
-
-        if (size <= 192) {
-                if (!size)
-                        return ZERO_SIZE_PTR;
-
-                index = size_index[size_index_elem(size)];
-        } else
-                index = fls(size - 1);
-
-#ifdef CONFIG_ZONE_DMA
-        if (unlikely((flags & SLUB_DMA)))
-                return kmalloc_dma_caches[index];
-
-#endif
-        return kmalloc_caches[index];
-}
-
 void *__kmalloc(size_t size, gfp_t flags)
 {
         struct kmem_cache *s;
@@ -3276,7 +3218,7 @@ void *__kmalloc(size_t size, gfp_t flags)
         if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
                 return kmalloc_large(size, flags);
 
-        s = get_slab(size, flags);
+        s = kmalloc_slab(size, flags);
 
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
@@ -3319,7 +3261,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
                 return ret;
         }
 
-        s = get_slab(size, flags);
+        s = kmalloc_slab(size, flags);
 
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
@@ -3632,7 +3574,6 @@ void __init kmem_cache_init(void)
 {
         static __initdata struct kmem_cache boot_kmem_cache,
                 boot_kmem_cache_node;
-        int i;
 
         if (debug_guardpage_minorder())
                 slub_max_order = 0;
@@ -3663,45 +3604,6 @@ void __init kmem_cache_init(void)
         kmem_cache_node = bootstrap(&boot_kmem_cache_node);
 
         /* Now we can use the kmem_cache to allocate kmalloc slabs */
-
-        /*
-         * Patch up the size_index table if we have strange large alignment
-         * requirements for the kmalloc array. This is only the case for
-         * MIPS it seems. The standard arches will not generate any code here.
-         *
-         * Largest permitted alignment is 256 bytes due to the way we
-         * handle the index determination for the smaller caches.
-         *
-         * Make sure that nothing crazy happens if someone starts tinkering
-         * around with ARCH_KMALLOC_MINALIGN
-         */
-        BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
-                (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
-
-        for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
-                int elem = size_index_elem(i);
-                if (elem >= ARRAY_SIZE(size_index))
-                        break;
-                size_index[elem] = KMALLOC_SHIFT_LOW;
-        }
-
-        if (KMALLOC_MIN_SIZE == 64) {
-                /*
-                 * The 96 byte size cache is not used if the alignment
-                 * is 64 byte.
-                 */
-                for (i = 64 + 8; i <= 96; i += 8)
-                        size_index[size_index_elem(i)] = 7;
-        } else if (KMALLOC_MIN_SIZE == 128) {
-                /*
-                 * The 192 byte sized cache is not used if the alignment
-                 * is 128 byte. Redirect kmalloc to use the 256 byte cache
-                 * instead.
-                 */
-                for (i = 128 + 8; i <= 192; i += 8)
-                        size_index[size_index_elem(i)] = 8;
-        }
-
         create_kmalloc_caches(0);
 
 #ifdef CONFIG_SMP
@@ -3877,7 +3779,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
         if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
                 return kmalloc_large(size, gfpflags);
 
-        s = get_slab(size, gfpflags);
+        s = kmalloc_slab(size, gfpflags);
 
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
@@ -3907,7 +3809,7 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
                 return ret;
         }
 
-        s = get_slab(size, gfpflags);
+        s = kmalloc_slab(size, gfpflags);
 
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;