1 files changed, 53 insertions, 30 deletions

diff --git a/mm/slub.c b/mm/slub.c
index 0280eee6cf37..c4ea9158c9fb 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -374,14 +374,8 @@ static struct track *get_track(struct kmem_cache *s, void *object,
 static void set_track(struct kmem_cache *s, void *object,
                         enum track_item alloc, unsigned long addr)
 {
-        struct track *p;
-
-        if (s->offset)
-                p = object + s->offset + sizeof(void *);
-        else
-                p = object + s->inuse;
+        struct track *p = get_track(s, object, alloc);
 
-        p += alloc;
         if (addr) {
                 p->addr = addr;
                 p->cpu = smp_processor_id();
@@ -1335,7 +1329,7 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags)
                 n = get_node(s, zone_to_nid(zone));
 
                 if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
-                                n->nr_partial > n->min_partial) {
+                                n->nr_partial > s->min_partial) {
                         page = get_partial_node(n);
                         if (page)
                                 return page;
@@ -1387,7 +1381,7 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
                 slab_unlock(page);
         } else {
                 stat(c, DEACTIVATE_EMPTY);
-                if (n->nr_partial < n->min_partial) {
+                if (n->nr_partial < s->min_partial) {
                         /*
                          * Adding an empty slab to the partial slabs in order
                          * to avoid page allocator overhead. This slab needs
@@ -1596,6 +1590,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
         unsigned long flags;
         unsigned int objsize;
 
+        lockdep_trace_alloc(gfpflags);
         might_sleep_if(gfpflags & __GFP_WAIT);
 
         if (should_failslab(s->objsize, gfpflags))
@@ -1724,7 +1719,7 @@ static __always_inline void slab_free(struct kmem_cache *s,
         c = get_cpu_slab(s, smp_processor_id());
         debug_check_no_locks_freed(object, c->objsize);
         if (!(s->flags & SLAB_DEBUG_OBJECTS))
-                debug_check_no_obj_freed(object, s->objsize);
+                debug_check_no_obj_freed(object, c->objsize);
         if (likely(page == c->page && c->node >= 0)) {
                 object[c->offset] = c->freelist;
                 c->freelist = object;
@@ -1844,6 +1839,7 @@ static inline int calculate_order(int size)
         int order;
         int min_objects;
         int fraction;
+        int max_objects;
 
         /*
          * Attempt to find best configuration for a slab. This
@@ -1856,6 +1852,9 @@ static inline int calculate_order(int size)
         min_objects = slub_min_objects;
         if (!min_objects)
                 min_objects = 4 * (fls(nr_cpu_ids) + 1);
+        max_objects = (PAGE_SIZE << slub_max_order)/size;
+        min_objects = min(min_objects, max_objects);
+
         while (min_objects > 1) {
                 fraction = 16;
                 while (fraction >= 4) {
@@ -1865,7 +1864,7 @@ static inline int calculate_order(int size)
                                 return order;
                         fraction /= 2;
                 }
-                min_objects /= 2;
+                min_objects --;
         }
 
         /*
@@ -1928,17 +1927,6 @@ static void
 init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
 {
         n->nr_partial = 0;
-
-        /*
-         * The larger the object size is, the more pages we want on the partial
-         * list to avoid pounding the page allocator excessively.
-         */
-        n->min_partial = ilog2(s->size);
-        if (n->min_partial < MIN_PARTIAL)
-                n->min_partial = MIN_PARTIAL;
-        else if (n->min_partial > MAX_PARTIAL)
-                n->min_partial = MAX_PARTIAL;
-
         spin_lock_init(&n->list_lock);
         INIT_LIST_HEAD(&n->partial);
 #ifdef CONFIG_SLUB_DEBUG
@@ -2181,6 +2169,15 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
 }
 #endif
 
+static void set_min_partial(struct kmem_cache *s, unsigned long min)
+{
+        if (min < MIN_PARTIAL)
+                min = MIN_PARTIAL;
+        else if (min > MAX_PARTIAL)
+                min = MAX_PARTIAL;
+        s->min_partial = min;
+}
+
 /*
  * calculate_sizes() determines the order and the distribution of data within
  * a slab object.
@@ -2319,6 +2316,11 @@ static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags,
         if (!calculate_sizes(s, -1))
                 goto error;
 
+        /*
+         * The larger the object size is, the more pages we want on the partial
+         * list to avoid pounding the page allocator excessively.
+         */
+        set_min_partial(s, ilog2(s->size));
         s->refcount = 1;
 #ifdef CONFIG_NUMA
         s->remote_node_defrag_ratio = 1000;
@@ -2475,7 +2477,7 @@ EXPORT_SYMBOL(kmem_cache_destroy);
  *              Kmalloc subsystem
  *******************************************************************/
 
-struct kmem_cache kmalloc_caches[PAGE_SHIFT + 1] __cacheline_aligned;
+struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT] __cacheline_aligned;
 EXPORT_SYMBOL(kmalloc_caches);
 
 static int __init setup_slub_min_order(char *str)
@@ -2537,7 +2539,7 @@ panic:
 }
 
 #ifdef CONFIG_ZONE_DMA
-static struct kmem_cache *kmalloc_caches_dma[PAGE_SHIFT + 1];
+static struct kmem_cache *kmalloc_caches_dma[SLUB_PAGE_SHIFT];
 
 static void sysfs_add_func(struct work_struct *w)
 {
@@ -2658,7 +2660,7 @@ void *__kmalloc(size_t size, gfp_t flags)
 {
         struct kmem_cache *s;
 
-        if (unlikely(size > PAGE_SIZE))
+        if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large(size, flags);
 
         s = get_slab(size, flags);
@@ -2686,7 +2688,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
 {
         struct kmem_cache *s;
 
-        if (unlikely(size > PAGE_SIZE))
+        if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large_node(size, flags, node);
 
         s = get_slab(size, flags);
@@ -2986,7 +2988,7 @@ void __init kmem_cache_init(void)
                 caches++;
         }
 
-        for (i = KMALLOC_SHIFT_LOW; i <= PAGE_SHIFT; i++) {
+        for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
                 create_kmalloc_cache(&kmalloc_caches[i],
                         "kmalloc", 1 << i, GFP_KERNEL);
                 caches++;
@@ -3023,7 +3025,7 @@ void __init kmem_cache_init(void)
         slab_state = UP;
 
         /* Provide the correct kmalloc names now that the caches are up */
-        for (i = KMALLOC_SHIFT_LOW; i <= PAGE_SHIFT; i++)
+        for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++)
                 kmalloc_caches[i]. name =
                         kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i);
 
@@ -3223,7 +3225,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
 {
         struct kmem_cache *s;
 
-        if (unlikely(size > PAGE_SIZE))
+        if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large(size, gfpflags);
 
         s = get_slab(size, gfpflags);
@@ -3239,7 +3241,7 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
 {
         struct kmem_cache *s;
 
-        if (unlikely(size > PAGE_SIZE))
+        if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large_node(size, gfpflags, node);
 
         s = get_slab(size, gfpflags);
@@ -3836,6 +3838,26 @@ static ssize_t order_show(struct kmem_cache *s, char *buf)
 }
 SLAB_ATTR(order);
 
+static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
+{
+        return sprintf(buf, "%lu\n", s->min_partial);
+}
+
+static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
+                                 size_t length)
+{
+        unsigned long min;
+        int err;
+
+        err = strict_strtoul(buf, 10, &min);
+        if (err)
+                return err;
+
+        set_min_partial(s, min);
+        return length;
+}
+SLAB_ATTR(min_partial);
+
 static ssize_t ctor_show(struct kmem_cache *s, char *buf)
 {
         if (s->ctor) {
@@ -4151,6 +4173,7 @@ static struct attribute *slab_attrs[] = {
         &object_size_attr.attr,
         &objs_per_slab_attr.attr,
         &order_attr.attr,
+        &min_partial_attr.attr,
         &objects_attr.attr,
         &objects_partial_attr.attr,
         &total_objects_attr.attr,