slab: Common Kmalloc cache determination

Extract the optimized lookup functions from slub and put them into
slab_common.c. Then make slab use these functions as well.

Joonsoo notes that this fixes some issues with constant folding which
also reduces the code size for slub.

https://lkml.org/lkml/2012/10/20/82

Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>

1 files changed, 103 insertions, 2 deletions

diff --git a/mm/slab_common.c b/mm/slab_common.c
index 2b0ebb6d071d..6d73f0b7f21c 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -328,6 +328,68 @@ EXPORT_SYMBOL(kmalloc_dma_caches);
 #endif
 
 /*
+ * Conversion table for small slabs sizes / 8 to the index in the
+ * kmalloc array. This is necessary for slabs < 192 since we have non power
+ * of two cache sizes there. The size of larger slabs can be determined using
+ * fls.
+ */
+static s8 size_index[24] = {
+        3,      /* 8 */
+        4,      /* 16 */
+        5,      /* 24 */
+        5,      /* 32 */
+        6,      /* 40 */
+        6,      /* 48 */
+        6,      /* 56 */
+        6,      /* 64 */
+        1,      /* 72 */
+        1,      /* 80 */
+        1,      /* 88 */
+        1,      /* 96 */
+        7,      /* 104 */
+        7,      /* 112 */
+        7,      /* 120 */
+        7,      /* 128 */
+        2,      /* 136 */
+        2,      /* 144 */
+        2,      /* 152 */
+        2,      /* 160 */
+        2,      /* 168 */
+        2,      /* 176 */
+        2,      /* 184 */
+        2       /* 192 */
+};
+
+static inline int size_index_elem(size_t bytes)
+{
+        return (bytes - 1) / 8;
+}
+
+/*
+ * Find the kmem_cache structure that serves a given size of
+ * allocation
+ */
+struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
+{
+        int index;
+
+        if (size <= 192) {
+                if (!size)
+                        return ZERO_SIZE_PTR;
+
+                index = size_index[size_index_elem(size)];
+        } else
+                index = fls(size - 1);
+
+#ifdef CONFIG_ZONE_DMA
+        if (unlikely((flags & SLAB_CACHE_DMA)))
+                return kmalloc_dma_caches[index];
+
+#endif
+        return kmalloc_caches[index];
+}
+
+/*
  * Create the kmalloc array. Some of the regular kmalloc arrays
  * may already have been created because they were needed to
  * enable allocations for slab creation.
@@ -336,6 +398,47 @@ void __init create_kmalloc_caches(unsigned long flags)
 {
         int i;
 
+        /*
+         * Patch up the size_index table if we have strange large alignment
+         * requirements for the kmalloc array. This is only the case for
+         * MIPS it seems. The standard arches will not generate any code here.
+         *
+         * Largest permitted alignment is 256 bytes due to the way we
+         * handle the index determination for the smaller caches.
+         *
+         * Make sure that nothing crazy happens if someone starts tinkering
+         * around with ARCH_KMALLOC_MINALIGN
+         */
+        BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
+                (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
+
+        for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
+                int elem = size_index_elem(i);
+
+                if (elem >= ARRAY_SIZE(size_index))
+                        break;
+                size_index[elem] = KMALLOC_SHIFT_LOW;
+        }
+
+        if (KMALLOC_MIN_SIZE >= 64) {
+                /*
+                 * The 96 byte size cache is not used if the alignment
+                 * is 64 byte.
+                 */
+                for (i = 64 + 8; i <= 96; i += 8)
+                        size_index[size_index_elem(i)] = 7;
+
+        }
+
+        if (KMALLOC_MIN_SIZE >= 128) {
+                /*
+                 * The 192 byte sized cache is not used if the alignment
+                 * is 128 byte. Redirect kmalloc to use the 256 byte cache
+                 * instead.
+                 */
+                for (i = 128 + 8; i <= 192; i += 8)
+                        size_index[size_index_elem(i)] = 8;
+        }
         /* Caches that are not of the two-to-the-power-of size */
         if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1])
                 kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
@@ -379,8 +482,6 @@ void __init create_kmalloc_caches(unsigned long flags)
         }
 #endif
 }
-
-
 #endif /* !CONFIG_SLOB */