slab: Use the new create_boot_cache function to simplify bootstrap

Simplify setup and reduce code in kmem_cache_init(). This allows us to
get rid of initarray_cache as well as the manual setup code for
the kmem_cache and kmem_cache_node arrays during bootstrap.

We introduce a new bootstrap state "PARTIAL" for slab that signals the
creation of a kmem_cache boot cache.

Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>

1 files changed, 16 insertions, 33 deletions

diff --git a/mm/slab.c b/mm/slab.c
index e351acea6026..e1790e56fd86 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -547,8 +547,6 @@ static struct cache_names __initdata cache_names[] = {
 #undef CACHE
 };
 
-static struct arraycache_init initarray_cache __initdata =
-    { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
 static struct arraycache_init initarray_generic =
     { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
 
@@ -1572,12 +1570,9 @@ static void setup_nodelists_pointer(struct kmem_cache *cachep)
  */
 void __init kmem_cache_init(void)
 {
-        size_t left_over;
         struct cache_sizes *sizes;
         struct cache_names *names;
         int i;
-        int order;
-        int node;
 
         kmem_cache = &kmem_cache_boot;
         setup_nodelists_pointer(kmem_cache);
@@ -1618,36 +1613,16 @@ void __init kmem_cache_init(void)
          * 6) Resize the head arrays of the kmalloc caches to their final sizes.
          */
 
-        node = numa_mem_id();
-
         /* 1) create the kmem_cache */
-        INIT_LIST_HEAD(&slab_caches);
-        list_add(&kmem_cache->list, &slab_caches);
-        kmem_cache->colour_off = cache_line_size();
-        kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
 
         /*
          * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
          */
-        kmem_cache->size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
-                                  nr_node_ids * sizeof(struct kmem_list3 *);
-        kmem_cache->object_size = kmem_cache->size;
-        kmem_cache->size = ALIGN(kmem_cache->object_size,
-                                        cache_line_size());
-        kmem_cache->reciprocal_buffer_size =
-                reciprocal_value(kmem_cache->size);
-
-        for (order = 0; order < MAX_ORDER; order++) {
-                cache_estimate(order, kmem_cache->size,
-                        cache_line_size(), 0, &left_over, &kmem_cache->num);
-                if (kmem_cache->num)
-                        break;
-        }
-        BUG_ON(!kmem_cache->num);
-        kmem_cache->gfporder = order;
-        kmem_cache->colour = left_over / kmem_cache->colour_off;
-        kmem_cache->slab_size = ALIGN(kmem_cache->num * sizeof(kmem_bufctl_t) +
-                                      sizeof(struct slab), cache_line_size());
+        create_boot_cache(kmem_cache, "kmem_cache",
+                offsetof(struct kmem_cache, array[nr_cpu_ids]) +
+                                  nr_node_ids * sizeof(struct kmem_list3 *),
+                                  SLAB_HWCACHE_ALIGN);
+        list_add(&kmem_cache->list, &slab_caches);
 
         /* 2+3) create the kmalloc caches */
         sizes = malloc_sizes;
@@ -1695,7 +1670,6 @@ void __init kmem_cache_init(void)
 
                 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
 
-                BUG_ON(cpu_cache_get(kmem_cache) != &initarray_cache.cache);
                 memcpy(ptr, cpu_cache_get(kmem_cache),
                        sizeof(struct arraycache_init));
                 /*
@@ -2250,7 +2224,15 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
 
         if (slab_state == DOWN) {
                 /*
-                 * Note: the first kmem_cache_create must create the cache
+                 * Note: Creation of first cache (kmem_cache).
+                 * The setup_list3s is taken care
+                 * of by the caller of __kmem_cache_create
+                 */
+                cachep->array[smp_processor_id()] = &initarray_generic.cache;
+                slab_state = PARTIAL;
+        } else if (slab_state == PARTIAL) {
+                /*
+                 * Note: the second kmem_cache_create must create the cache
                  * that's used by kmalloc(24), otherwise the creation of
                  * further caches will BUG().
                  */
@@ -2258,7 +2240,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
 
                 /*
                  * If the cache that's used by kmalloc(sizeof(kmem_list3)) is
-                 * the first cache, then we need to set up all its list3s,
+                 * the second cache, then we need to set up all its list3s,
                  * otherwise the creation of further caches will BUG().
                  */
                 set_up_list3s(cachep, SIZE_AC);
@@ -2267,6 +2249,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
                 else
                         slab_state = PARTIAL_ARRAYCACHE;
         } else {
+                /* Remaining boot caches */
                 cachep->array[smp_processor_id()] =
                         kmalloc(sizeof(struct arraycache_init), gfp);