3 files changed, 110 insertions, 69 deletions

diff --git a/Documentation/vm/slabinfo.c b/Documentation/vm/slabinfo.c
index 22d7e3e4d60c..d3ce295bffac 100644
--- a/Documentation/vm/slabinfo.c
+++ b/Documentation/vm/slabinfo.c
@@ -31,7 +31,7 @@ struct slabinfo {
         int hwcache_align, object_size, objs_per_slab;
         int sanity_checks, slab_size, store_user, trace;
         int order, poison, reclaim_account, red_zone;
-        unsigned long partial, objects, slabs;
+        unsigned long partial, objects, slabs, objects_partial, objects_total;
         unsigned long alloc_fastpath, alloc_slowpath;
         unsigned long free_fastpath, free_slowpath;
         unsigned long free_frozen, free_add_partial, free_remove_partial;
@@ -540,7 +540,8 @@ void slabcache(struct slabinfo *s)
                 return;
 
         store_size(size_str, slab_size(s));
-        snprintf(dist_str, 40, "%lu/%lu/%d", s->slabs, s->partial, s->cpu_slabs);
+        snprintf(dist_str, 40, "%lu/%lu/%d", s->slabs - s->cpu_slabs,
+                                                s->partial, s->cpu_slabs);
 
         if (!line++)
                 first_line();
@@ -776,7 +777,6 @@ void totals(void)
                 unsigned long used;
                 unsigned long long wasted;
                 unsigned long long objwaste;
-                long long objects_in_partial_slabs;
                 unsigned long percentage_partial_slabs;
                 unsigned long percentage_partial_objs;
 
@@ -790,18 +790,11 @@ void totals(void)
                 wasted = size - used;
                 objwaste = s->slab_size - s->object_size;
 
-                objects_in_partial_slabs = s->objects -
-                        (s->slabs - s->partial - s ->cpu_slabs) *
-                        s->objs_per_slab;
-
-                if (objects_in_partial_slabs < 0)
-                        objects_in_partial_slabs = 0;
-
                 percentage_partial_slabs = s->partial * 100 / s->slabs;
                 if (percentage_partial_slabs > 100)
                         percentage_partial_slabs = 100;
 
-                percentage_partial_objs = objects_in_partial_slabs * 100
+                percentage_partial_objs = s->objects_partial * 100
                                                         / s->objects;
 
                 if (percentage_partial_objs > 100)
@@ -823,8 +816,8 @@ void totals(void)
                         min_objects = s->objects;
                 if (used < min_used)
                         min_used = used;
-                if (objects_in_partial_slabs < min_partobj)
-                        min_partobj = objects_in_partial_slabs;
+                if (s->objects_partial < min_partobj)
+                        min_partobj = s->objects_partial;
                 if (percentage_partial_slabs < min_ppart)
                         min_ppart = percentage_partial_slabs;
                 if (percentage_partial_objs < min_ppartobj)
@@ -848,8 +841,8 @@ void totals(void)
                         max_objects = s->objects;
                 if (used > max_used)
                         max_used = used;
-                if (objects_in_partial_slabs > max_partobj)
-                        max_partobj = objects_in_partial_slabs;
+                if (s->objects_partial > max_partobj)
+                        max_partobj = s->objects_partial;
                 if (percentage_partial_slabs > max_ppart)
                         max_ppart = percentage_partial_slabs;
                 if (percentage_partial_objs > max_ppartobj)
@@ -864,7 +857,7 @@ void totals(void)
 
                 total_objects += s->objects;
                 total_used += used;
-                total_partobj += objects_in_partial_slabs;
+                total_partobj += s->objects_partial;
                 total_ppart += percentage_partial_slabs;
                 total_ppartobj += percentage_partial_objs;
 
@@ -1160,6 +1153,8 @@ void read_slab_dir(void)
                         slab->hwcache_align = get_obj("hwcache_align");
                         slab->object_size = get_obj("object_size");
                         slab->objects = get_obj("objects");
+                        slab->objects_partial = get_obj("objects_partial");
+                        slab->objects_total = get_obj("objects_total");
                         slab->objs_per_slab = get_obj("objs_per_slab");
                         slab->order = get_obj("order");
                         slab->partial = get_obj("partial");
diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index 4131e5fbd18b..4236b5dee812 100644
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -48,6 +48,7 @@ struct kmem_cache_node {
         struct list_head partial;
 #ifdef CONFIG_SLUB_DEBUG
         atomic_long_t nr_slabs;
+        atomic_long_t total_objects;
         struct list_head full;
 #endif
 };
@@ -79,6 +80,7 @@ struct kmem_cache {
         struct kmem_cache_node local_node;
 
         /* Allocation and freeing of slabs */
+        struct kmem_cache_order_objects max;
         gfp_t allocflags;       /* gfp flags to use on each alloc */
         int refcount;           /* Refcount for slab cache destroy */
         void (*ctor)(struct kmem_cache *, void *);
diff --git a/mm/slub.c b/mm/slub.c
index 0a220df5ed7c..c8514e93ffdf 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -886,7 +886,7 @@ static inline unsigned long slabs_node(struct kmem_cache *s, int node)
         return atomic_long_read(&n->nr_slabs);
 }
 
-static inline void inc_slabs_node(struct kmem_cache *s, int node)
+static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
 {
         struct kmem_cache_node *n = get_node(s, node);
 
@@ -896,14 +896,17 @@ static inline void inc_slabs_node(struct kmem_cache *s, int node)
          * dilemma by deferring the increment of the count during
          * bootstrap (see early_kmem_cache_node_alloc).
          */
-        if (!NUMA_BUILD || n)
+        if (!NUMA_BUILD || n) {
                 atomic_long_inc(&n->nr_slabs);
+                atomic_long_add(objects, &n->total_objects);
+        }
 }
-static inline void dec_slabs_node(struct kmem_cache *s, int node)
+static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
 {
         struct kmem_cache_node *n = get_node(s, node);
 
         atomic_long_dec(&n->nr_slabs);
+        atomic_long_sub(objects, &n->total_objects);
 }
 
 /* Object debug checks for alloc/free paths */
@@ -1101,9 +1104,12 @@ static inline unsigned long kmem_cache_flags(unsigned long objsize,
 
 static inline unsigned long slabs_node(struct kmem_cache *s, int node)
                                                         { return 0; }
-static inline void inc_slabs_node(struct kmem_cache *s, int node) {}
-static inline void dec_slabs_node(struct kmem_cache *s, int node) {}
+static inline void inc_slabs_node(struct kmem_cache *s, int node,
+                                                        int objects) {}
+static inline void dec_slabs_node(struct kmem_cache *s, int node,
+                                                        int objects) {}
 #endif
+
 /*
  * Slab allocation and freeing
  */
@@ -1155,7 +1161,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
         if (!page)
                 goto out;
 
-        inc_slabs_node(s, page_to_nid(page));
+        inc_slabs_node(s, page_to_nid(page), page->objects);
         page->slab = s;
         page->flags |= 1 << PG_slab;
         if (s->flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_POISON |
@@ -1230,7 +1236,7 @@ static void free_slab(struct kmem_cache *s, struct page *page)
 
 static void discard_slab(struct kmem_cache *s, struct page *page)
 {
-        dec_slabs_node(s, page_to_nid(page));
+        dec_slabs_node(s, page_to_nid(page), page->objects);
         free_slab(s, page);
 }
 
@@ -2144,7 +2150,7 @@ static struct kmem_cache_node *early_kmem_cache_node_alloc(gfp_t gfpflags,
         init_tracking(kmalloc_caches, n);
 #endif
         init_kmem_cache_node(n);
-        inc_slabs_node(kmalloc_caches, node);
+        inc_slabs_node(kmalloc_caches, node, page->objects);
 
         /*
          * lockdep requires consistent irq usage for each lock
@@ -2341,6 +2347,8 @@ static int calculate_sizes(struct kmem_cache *s)
          * Determine the number of objects per slab
          */
         s->oo = oo_make(order, size);
+        if (oo_objects(s->oo) > oo_objects(s->max))
+                s->max = s->oo;
 
         return !!oo_objects(s->oo);
 
@@ -2813,7 +2821,7 @@ int kmem_cache_shrink(struct kmem_cache *s)
         struct kmem_cache_node *n;
         struct page *page;
         struct page *t;
-        int objects = oo_objects(s->oo);
+        int objects = oo_objects(s->max);
         struct list_head *slabs_by_inuse =
                 kmalloc(sizeof(struct list_head) * objects, GFP_KERNEL);
         unsigned long flags;
@@ -3276,7 +3284,8 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
 }
 
 #if (defined(CONFIG_SYSFS) && defined(CONFIG_SLUB_DEBUG)) || defined(CONFIG_SLABINFO)
-static unsigned long count_partial(struct kmem_cache_node *n)
+static unsigned long count_partial(struct kmem_cache_node *n,
+                                        int (*get_count)(struct page *))
 {
         unsigned long flags;
         unsigned long x = 0;
@@ -3284,10 +3293,25 @@ static unsigned long count_partial(struct kmem_cache_node *n)
 
         spin_lock_irqsave(&n->list_lock, flags);
         list_for_each_entry(page, &n->partial, lru)
-                x += page->inuse;
+                x += get_count(page);
         spin_unlock_irqrestore(&n->list_lock, flags);
         return x;
 }
+
+static int count_inuse(struct page *page)
+{
+        return page->inuse;
+}
+
+static int count_total(struct page *page)
+{
+        return page->objects;
+}
+
+static int count_free(struct page *page)
+{
+        return page->objects - page->inuse;
+}
 #endif
 
 #if defined(CONFIG_SYSFS) && defined(CONFIG_SLUB_DEBUG)
@@ -3376,7 +3400,7 @@ static long validate_slab_cache(struct kmem_cache *s)
 {
         int node;
         unsigned long count = 0;
-        unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->oo)) *
+        unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) *
                                 sizeof(unsigned long), GFP_KERNEL);
 
         if (!map)
@@ -3676,22 +3700,23 @@ static int list_locations(struct kmem_cache *s, char *buf,
 }
 
 enum slab_stat_type {
-        SL_FULL,
-        SL_PARTIAL,
-        SL_CPU,
-        SL_OBJECTS
+        SL_ALL,                 /* All slabs */
+        SL_PARTIAL,             /* Only partially allocated slabs */
+        SL_CPU,                 /* Only slabs used for cpu caches */
+        SL_OBJECTS,             /* Determine allocated objects not slabs */
+        SL_TOTAL                /* Determine object capacity not slabs */
 };
 
-#define SO_FULL         (1 << SL_FULL)
+#define SO_ALL          (1 << SL_ALL)
 #define SO_PARTIAL      (1 << SL_PARTIAL)
 #define SO_CPU          (1 << SL_CPU)
 #define SO_OBJECTS      (1 << SL_OBJECTS)
+#define SO_TOTAL        (1 << SL_TOTAL)
 
 static ssize_t show_slab_objects(struct kmem_cache *s,
                             char *buf, unsigned long flags)
 {
         unsigned long total = 0;
-        int cpu;
         int node;
         int x;
         unsigned long *nodes;
@@ -3702,56 +3727,60 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
                 return -ENOMEM;
         per_cpu = nodes + nr_node_ids;
 
-        for_each_possible_cpu(cpu) {
-                struct page *page;
-                struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
+        if (flags & SO_CPU) {
+                int cpu;
 
-                if (!c)
-                        continue;
+                for_each_possible_cpu(cpu) {
+                        struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);
 
-                page = c->page;
-                node = c->node;
-                if (node < 0)
-                        continue;
-                if (page) {
-                        if (flags & SO_CPU) {
-                                if (flags & SO_OBJECTS)
-                                        x = page->inuse;
+                        if (!c || c->node < 0)
+                                continue;
+
+                        if (c->page) {
+                                        if (flags & SO_TOTAL)
+                                                x = c->page->objects;
+                                else if (flags & SO_OBJECTS)
+                                        x = c->page->inuse;
                                 else
                                         x = 1;
+
                                 total += x;
-                                nodes[node] += x;
+                                nodes[c->node] += x;
                         }
-                        per_cpu[node]++;
+                        per_cpu[c->node]++;
                 }
         }
 
-        for_each_node_state(node, N_NORMAL_MEMORY) {
-                struct kmem_cache_node *n = get_node(s, node);
+        if (flags & SO_ALL) {
+                for_each_node_state(node, N_NORMAL_MEMORY) {
+                        struct kmem_cache_node *n = get_node(s, node);
+
+                if (flags & SO_TOTAL)
+                        x = atomic_long_read(&n->total_objects);
+                else if (flags & SO_OBJECTS)
+                        x = atomic_long_read(&n->total_objects) -
+                                count_partial(n, count_free);
 
-                if (flags & SO_PARTIAL) {
-                        if (flags & SO_OBJECTS)
-                                x = count_partial(n);
                         else
-                                x = n->nr_partial;
+                                x = atomic_long_read(&n->nr_slabs);
                         total += x;
                         nodes[node] += x;
                 }
 
-                if (flags & SO_FULL) {
-                        int full_slabs = atomic_long_read(&n->nr_slabs)
-                                        - per_cpu[node]
-                                        - n->nr_partial;
+        } else if (flags & SO_PARTIAL) {
+                for_each_node_state(node, N_NORMAL_MEMORY) {
+                        struct kmem_cache_node *n = get_node(s, node);
 
-                        if (flags & SO_OBJECTS)
-                                x = full_slabs * oo_objects(s->oo);
+                        if (flags & SO_TOTAL)
+                                x = count_partial(n, count_total);
+                        else if (flags & SO_OBJECTS)
+                                x = count_partial(n, count_inuse);
                         else
-                                x = full_slabs;
+                                x = n->nr_partial;
                         total += x;
                         nodes[node] += x;
                 }
         }
-
         x = sprintf(buf, "%lu", total);
 #ifdef CONFIG_NUMA
         for_each_node_state(node, N_NORMAL_MEMORY)
@@ -3852,7 +3881,7 @@ SLAB_ATTR_RO(aliases);
 
 static ssize_t slabs_show(struct kmem_cache *s, char *buf)
 {
-        return show_slab_objects(s, buf, SO_FULL|SO_PARTIAL|SO_CPU);
+        return show_slab_objects(s, buf, SO_ALL);
 }
 SLAB_ATTR_RO(slabs);
 
@@ -3870,10 +3899,22 @@ SLAB_ATTR_RO(cpu_slabs);
 
 static ssize_t objects_show(struct kmem_cache *s, char *buf)
 {
-        return show_slab_objects(s, buf, SO_FULL|SO_PARTIAL|SO_CPU|SO_OBJECTS);
+        return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
 }
 SLAB_ATTR_RO(objects);
 
+static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
+{
+        return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
+}
+SLAB_ATTR_RO(objects_partial);
+
+static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
+{
+        return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
+}
+SLAB_ATTR_RO(total_objects);
+
 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
 {
         return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
@@ -4131,6 +4172,8 @@ static struct attribute *slab_attrs[] = {
         &objs_per_slab_attr.attr,
         &order_attr.attr,
         &objects_attr.attr,
+        &objects_partial_attr.attr,
+        &total_objects_attr.attr,
         &slabs_attr.attr,
         &partial_attr.attr,
         &cpu_slabs_attr.attr,
@@ -4459,7 +4502,8 @@ static int s_show(struct seq_file *m, void *p)
         unsigned long nr_partials = 0;
         unsigned long nr_slabs = 0;
         unsigned long nr_inuse = 0;
-        unsigned long nr_objs;
+        unsigned long nr_objs = 0;
+        unsigned long nr_free = 0;
         struct kmem_cache *s;
         int node;
 
@@ -4473,11 +4517,11 @@ static int s_show(struct seq_file *m, void *p)
 
                 nr_partials += n->nr_partial;
                 nr_slabs += atomic_long_read(&n->nr_slabs);
-                nr_inuse += count_partial(n);
+                nr_objs += atomic_long_read(&n->total_objects);
+                nr_free += count_partial(n, count_free);
         }
 
-        nr_objs = nr_slabs * oo_objects(s->oo);
-        nr_inuse += (nr_slabs - nr_partials) * oo_objects(s->oo);
+        nr_inuse = nr_objs - nr_free;
 
         seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, nr_inuse,
                    nr_objs, s->size, oo_objects(s->oo),