1 files changed, 70 insertions, 219 deletions

diff --git a/mm/slab.c b/mm/slab.c
index 33d3363658df..2c3a2e0394db 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -162,23 +162,6 @@
  */
 static bool pfmemalloc_active __read_mostly;
 
-/* Legal flag mask for kmem_cache_create(). */
-#if DEBUG
-# define CREATE_MASK    (SLAB_RED_ZONE | \
-                         SLAB_POISON | SLAB_HWCACHE_ALIGN | \
-                         SLAB_CACHE_DMA | \
-                         SLAB_STORE_USER | \
-                         SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
-                         SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
-                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
-#else
-# define CREATE_MASK    (SLAB_HWCACHE_ALIGN | \
-                         SLAB_CACHE_DMA | \
-                         SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
-                         SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
-                         SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
-#endif
-
 /*
  * kmem_bufctl_t:
  *
@@ -564,15 +547,11 @@ static struct cache_names __initdata cache_names[] = {
 #undef CACHE
 };
 
-static struct arraycache_init initarray_cache __initdata =
-    { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
 static struct arraycache_init initarray_generic =
     { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
 
 /* internal cache of cache description objs */
-static struct kmem_list3 *kmem_cache_nodelists[MAX_NUMNODES];
 static struct kmem_cache kmem_cache_boot = {
-        .nodelists = kmem_cache_nodelists,
         .batchcount = 1,
         .limit = BOOT_CPUCACHE_ENTRIES,
         .shared = 1,
@@ -1577,28 +1556,33 @@ static void __init set_up_list3s(struct kmem_cache *cachep, int index)
 }
 
 /*
+ * The memory after the last cpu cache pointer is used for the
+ * the nodelists pointer.
+ */
+static void setup_nodelists_pointer(struct kmem_cache *cachep)
+{
+        cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
+}
+
+/*
  * Initialisation.  Called after the page allocator have been initialised and
  * before smp_init().
  */
 void __init kmem_cache_init(void)
 {
-        size_t left_over;
         struct cache_sizes *sizes;
         struct cache_names *names;
         int i;
-        int order;
-        int node;
 
         kmem_cache = &kmem_cache_boot;
+        setup_nodelists_pointer(kmem_cache);
 
         if (num_possible_nodes() == 1)
                 use_alien_caches = 0;
 
-        for (i = 0; i < NUM_INIT_LISTS; i++) {
+        for (i = 0; i < NUM_INIT_LISTS; i++)
                 kmem_list3_init(&initkmem_list3[i]);
-                if (i < MAX_NUMNODES)
-                        kmem_cache->nodelists[i] = NULL;
-        }
+
         set_up_list3s(kmem_cache, CACHE_CACHE);
 
         /*
@@ -1629,37 +1613,16 @@ void __init kmem_cache_init(void)
          * 6) Resize the head arrays of the kmalloc caches to their final sizes.
          */
 
-        node = numa_mem_id();
-
         /* 1) create the kmem_cache */
-        INIT_LIST_HEAD(&slab_caches);
-        list_add(&kmem_cache->list, &slab_caches);
-        kmem_cache->colour_off = cache_line_size();
-        kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
-        kmem_cache->nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
 
         /*
          * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
          */
-        kmem_cache->size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
-                                  nr_node_ids * sizeof(struct kmem_list3 *);
-        kmem_cache->object_size = kmem_cache->size;
-        kmem_cache->size = ALIGN(kmem_cache->object_size,
-                                        cache_line_size());
-        kmem_cache->reciprocal_buffer_size =
-                reciprocal_value(kmem_cache->size);
-
-        for (order = 0; order < MAX_ORDER; order++) {
-                cache_estimate(order, kmem_cache->size,
-                        cache_line_size(), 0, &left_over, &kmem_cache->num);
-                if (kmem_cache->num)
-                        break;
-        }
-        BUG_ON(!kmem_cache->num);
-        kmem_cache->gfporder = order;
-        kmem_cache->colour = left_over / kmem_cache->colour_off;
-        kmem_cache->slab_size = ALIGN(kmem_cache->num * sizeof(kmem_bufctl_t) +
-                                      sizeof(struct slab), cache_line_size());
+        create_boot_cache(kmem_cache, "kmem_cache",
+                offsetof(struct kmem_cache, array[nr_cpu_ids]) +
+                                  nr_node_ids * sizeof(struct kmem_list3 *),
+                                  SLAB_HWCACHE_ALIGN);
+        list_add(&kmem_cache->list, &slab_caches);
 
         /* 2+3) create the kmalloc caches */
         sizes = malloc_sizes;
@@ -1671,23 +1634,13 @@ void __init kmem_cache_init(void)
          * bug.
          */
 
-        sizes[INDEX_AC].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
-        sizes[INDEX_AC].cs_cachep->name = names[INDEX_AC].name;
-        sizes[INDEX_AC].cs_cachep->size = sizes[INDEX_AC].cs_size;
-        sizes[INDEX_AC].cs_cachep->object_size = sizes[INDEX_AC].cs_size;
-        sizes[INDEX_AC].cs_cachep->align = ARCH_KMALLOC_MINALIGN;
-        __kmem_cache_create(sizes[INDEX_AC].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
-        list_add(&sizes[INDEX_AC].cs_cachep->list, &slab_caches);
-
-        if (INDEX_AC != INDEX_L3) {
-                sizes[INDEX_L3].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
-                sizes[INDEX_L3].cs_cachep->name = names[INDEX_L3].name;
-                sizes[INDEX_L3].cs_cachep->size = sizes[INDEX_L3].cs_size;
-                sizes[INDEX_L3].cs_cachep->object_size = sizes[INDEX_L3].cs_size;
-                sizes[INDEX_L3].cs_cachep->align = ARCH_KMALLOC_MINALIGN;
-                __kmem_cache_create(sizes[INDEX_L3].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
-                list_add(&sizes[INDEX_L3].cs_cachep->list, &slab_caches);
-        }
+        sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name,
+                                        sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS);
+
+        if (INDEX_AC != INDEX_L3)
+                sizes[INDEX_L3].cs_cachep =
+                        create_kmalloc_cache(names[INDEX_L3].name,
+                                sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS);
 
         slab_early_init = 0;
 
@@ -1699,24 +1652,14 @@ void __init kmem_cache_init(void)
                  * Note for systems short on memory removing the alignment will
                  * allow tighter packing of the smaller caches.
                  */
-                if (!sizes->cs_cachep) {
-                        sizes->cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
-                        sizes->cs_cachep->name = names->name;
-                        sizes->cs_cachep->size = sizes->cs_size;
-                        sizes->cs_cachep->object_size = sizes->cs_size;
-                        sizes->cs_cachep->align = ARCH_KMALLOC_MINALIGN;
-                        __kmem_cache_create(sizes->cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
-                        list_add(&sizes->cs_cachep->list, &slab_caches);
-                }
+                if (!sizes->cs_cachep)
+                        sizes->cs_cachep = create_kmalloc_cache(names->name,
+                                        sizes->cs_size, ARCH_KMALLOC_FLAGS);
+
 #ifdef CONFIG_ZONE_DMA
-                sizes->cs_dmacachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
-                sizes->cs_dmacachep->name = names->name_dma;
-                sizes->cs_dmacachep->size = sizes->cs_size;
-                sizes->cs_dmacachep->object_size = sizes->cs_size;
-                sizes->cs_dmacachep->align = ARCH_KMALLOC_MINALIGN;
-                __kmem_cache_create(sizes->cs_dmacachep,
-                               ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA| SLAB_PANIC);
-                list_add(&sizes->cs_dmacachep->list, &slab_caches);
+                sizes->cs_dmacachep = create_kmalloc_cache(
+                        names->name_dma, sizes->cs_size,
+                        SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
 #endif
                 sizes++;
                 names++;
@@ -1727,7 +1670,6 @@ void __init kmem_cache_init(void)
 
                 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
 
-                BUG_ON(cpu_cache_get(kmem_cache) != &initarray_cache.cache);
                 memcpy(ptr, cpu_cache_get(kmem_cache),
                        sizeof(struct arraycache_init));
                 /*
@@ -2282,7 +2224,15 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
 
         if (slab_state == DOWN) {
                 /*
-                 * Note: the first kmem_cache_create must create the cache
+                 * Note: Creation of first cache (kmem_cache).
+                 * The setup_list3s is taken care
+                 * of by the caller of __kmem_cache_create
+                 */
+                cachep->array[smp_processor_id()] = &initarray_generic.cache;
+                slab_state = PARTIAL;
+        } else if (slab_state == PARTIAL) {
+                /*
+                 * Note: the second kmem_cache_create must create the cache
                  * that's used by kmalloc(24), otherwise the creation of
                  * further caches will BUG().
                  */
@@ -2290,7 +2240,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
 
                 /*
                  * If the cache that's used by kmalloc(sizeof(kmem_list3)) is
-                 * the first cache, then we need to set up all its list3s,
+                 * the second cache, then we need to set up all its list3s,
                  * otherwise the creation of further caches will BUG().
                  */
                 set_up_list3s(cachep, SIZE_AC);
@@ -2299,6 +2249,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
                 else
                         slab_state = PARTIAL_ARRAYCACHE;
         } else {
+                /* Remaining boot caches */
                 cachep->array[smp_processor_id()] =
                         kmalloc(sizeof(struct arraycache_init), gfp);
 
@@ -2331,11 +2282,8 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
 
 /**
  * __kmem_cache_create - Create a cache.
- * @name: A string which is used in /proc/slabinfo to identify this cache.
- * @size: The size of objects to be created in this cache.
- * @align: The required alignment for the objects.
+ * @cachep: cache management descriptor
  * @flags: SLAB flags
- * @ctor: A constructor for the objects.
  *
  * Returns a ptr to the cache on success, NULL on failure.
  * Cannot be called within a int, but can be interrupted.
@@ -2378,11 +2326,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
         if (flags & SLAB_DESTROY_BY_RCU)
                 BUG_ON(flags & SLAB_POISON);
 #endif
-        /*
-         * Always checks flags, a caller might be expecting debug support which
-         * isn't available.
-         */
-        BUG_ON(flags & ~CREATE_MASK);
 
         /*
          * Check that size is in terms of words.  This is needed to avoid
@@ -2394,22 +2337,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
                 size &= ~(BYTES_PER_WORD - 1);
         }
 
-        /* calculate the final buffer alignment: */
-
-        /* 1) arch recommendation: can be overridden for debug */
-        if (flags & SLAB_HWCACHE_ALIGN) {
-                /*
-                 * Default alignment: as specified by the arch code.  Except if
-                 * an object is really small, then squeeze multiple objects into
-                 * one cacheline.
-                 */
-                ralign = cache_line_size();
-                while (size <= ralign / 2)
-                        ralign /= 2;
-        } else {
-                ralign = BYTES_PER_WORD;
-        }
-
         /*
          * Redzoning and user store require word alignment or possibly larger.
          * Note this will be overridden by architecture or caller mandated
@@ -2426,10 +2353,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
                 size &= ~(REDZONE_ALIGN - 1);
         }
 
-        /* 2) arch mandated alignment */
-        if (ralign < ARCH_SLAB_MINALIGN) {
-                ralign = ARCH_SLAB_MINALIGN;
-        }
         /* 3) caller mandated alignment */
         if (ralign < cachep->align) {
                 ralign = cachep->align;
@@ -2447,7 +2370,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
         else
                 gfp = GFP_NOWAIT;
 
-        cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
+        setup_nodelists_pointer(cachep);
 #if DEBUG
 
         /*
@@ -3969,12 +3892,6 @@ void kfree(const void *objp)
 }
 EXPORT_SYMBOL(kfree);
 
-unsigned int kmem_cache_size(struct kmem_cache *cachep)
-{
-        return cachep->object_size;
-}
-EXPORT_SYMBOL(kmem_cache_size);
-
 /*
  * This initializes kmem_list3 or resizes various caches for all nodes.
  */
@@ -4276,54 +4193,8 @@ out:
 }
 
 #ifdef CONFIG_SLABINFO
-
-static void print_slabinfo_header(struct seq_file *m)
+void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
 {
-        /*
-         * Output format version, so at least we can change it
-         * without _too_ many complaints.
-         */
-#if STATS
-        seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
-#else
-        seq_puts(m, "slabinfo - version: 2.1\n");
-#endif
-        seq_puts(m, "# name            <active_objs> <num_objs> <objsize> "
-                 "<objperslab> <pagesperslab>");
-        seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
-        seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
-#if STATS
-        seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
-                 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
-        seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
-#endif
-        seq_putc(m, '\n');
-}
-
-static void *s_start(struct seq_file *m, loff_t *pos)
-{
-        loff_t n = *pos;
-
-        mutex_lock(&slab_mutex);
-        if (!n)
-                print_slabinfo_header(m);
-
-        return seq_list_start(&slab_caches, *pos);
-}
-
-static void *s_next(struct seq_file *m, void *p, loff_t *pos)
-{
-        return seq_list_next(p, &slab_caches, pos);
-}
-
-static void s_stop(struct seq_file *m, void *p)
-{
-        mutex_unlock(&slab_mutex);
-}
-
-static int s_show(struct seq_file *m, void *p)
-{
-        struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
         struct slab *slabp;
         unsigned long active_objs;
         unsigned long num_objs;
@@ -4378,13 +4249,20 @@ static int s_show(struct seq_file *m, void *p)
         if (error)
                 printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
 
-        seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
-                   name, active_objs, num_objs, cachep->size,
-                   cachep->num, (1 << cachep->gfporder));
-        seq_printf(m, " : tunables %4u %4u %4u",
-                   cachep->limit, cachep->batchcount, cachep->shared);
-        seq_printf(m, " : slabdata %6lu %6lu %6lu",
-                   active_slabs, num_slabs, shared_avail);
+        sinfo->active_objs = active_objs;
+        sinfo->num_objs = num_objs;
+        sinfo->active_slabs = active_slabs;
+        sinfo->num_slabs = num_slabs;
+        sinfo->shared_avail = shared_avail;
+        sinfo->limit = cachep->limit;
+        sinfo->batchcount = cachep->batchcount;
+        sinfo->shared = cachep->shared;
+        sinfo->objects_per_slab = cachep->num;
+        sinfo->cache_order = cachep->gfporder;
+}
+
+void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
+{
 #if STATS
         {                       /* list3 stats */
                 unsigned long high = cachep->high_mark;
@@ -4414,31 +4292,8 @@ static int s_show(struct seq_file *m, void *p)
                            allochit, allocmiss, freehit, freemiss);
         }
 #endif
-        seq_putc(m, '\n');
-        return 0;
 }
 
-/*
- * slabinfo_op - iterator that generates /proc/slabinfo
- *
- * Output layout:
- * cache-name
- * num-active-objs
- * total-objs
- * object size
- * num-active-slabs
- * total-slabs
- * num-pages-per-slab
- * + further values on SMP and with statistics enabled
- */
-
-static const struct seq_operations slabinfo_op = {
-        .start = s_start,
-        .next = s_next,
-        .stop = s_stop,
-        .show = s_show,
-};
-
 #define MAX_SLABINFO_WRITE 128
 /**
  * slabinfo_write - Tuning for the slab allocator
@@ -4447,7 +4302,7 @@ static const struct seq_operations slabinfo_op = {
  * @count: data length
  * @ppos: unused
  */
-static ssize_t slabinfo_write(struct file *file, const char __user *buffer,
+ssize_t slabinfo_write(struct file *file, const char __user *buffer,
                        size_t count, loff_t *ppos)
 {
         char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
@@ -4490,19 +4345,6 @@ static ssize_t slabinfo_write(struct file *file, const char __user *buffer,
         return res;
 }
 
-static int slabinfo_open(struct inode *inode, struct file *file)
-{
-        return seq_open(file, &slabinfo_op);
-}
-
-static const struct file_operations proc_slabinfo_operations = {
-        .open           = slabinfo_open,
-        .read           = seq_read,
-        .write          = slabinfo_write,
-        .llseek         = seq_lseek,
-        .release        = seq_release,
-};
-
 #ifdef CONFIG_DEBUG_SLAB_LEAK
 
 static void *leaks_start(struct seq_file *m, loff_t *pos)
@@ -4631,6 +4473,16 @@ static int leaks_show(struct seq_file *m, void *p)
         return 0;
 }
 
+static void *s_next(struct seq_file *m, void *p, loff_t *pos)
+{
+        return seq_list_next(p, &slab_caches, pos);
+}
+
+static void s_stop(struct seq_file *m, void *p)
+{
+        mutex_unlock(&slab_mutex);
+}
+
 static const struct seq_operations slabstats_op = {
         .start = leaks_start,
         .next = s_next,
@@ -4665,7 +4517,6 @@ static const struct file_operations proc_slabstats_operations = {
 
 static int __init slab_proc_init(void)
 {
-        proc_create("slabinfo",S_IWUSR|S_IRUSR,NULL,&proc_slabinfo_operations);
 #ifdef CONFIG_DEBUG_SLAB_LEAK
         proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
 #endif