1 files changed, 42 insertions, 63 deletions

diff --git a/mm/slub.c b/mm/slub.c
index 6d4a49c1ff2f..4f5b96149458 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -102,44 +102,12 @@
  *                      the fast path and disables lockless freelists.
  */
 
-#define FROZEN (1 << PG_active)
-
 #ifdef CONFIG_SLUB_DEBUG
-#define SLABDEBUG (1 << PG_error)
+#define SLABDEBUG 1
 #else
 #define SLABDEBUG 0
 #endif
 
-static inline int SlabFrozen(struct page *page)
-{
-        return page->flags & FROZEN;
-}
-
-static inline void SetSlabFrozen(struct page *page)
-{
-        page->flags |= FROZEN;
-}
-
-static inline void ClearSlabFrozen(struct page *page)
-{
-        page->flags &= ~FROZEN;
-}
-
-static inline int SlabDebug(struct page *page)
-{
-        return page->flags & SLABDEBUG;
-}
-
-static inline void SetSlabDebug(struct page *page)
-{
-        page->flags |= SLABDEBUG;
-}
-
-static inline void ClearSlabDebug(struct page *page)
-{
-        page->flags &= ~SLABDEBUG;
-}
-
 /*
  * Issues still to be resolved:
  *
@@ -971,7 +939,7 @@ static int free_debug_processing(struct kmem_cache *s, struct page *page,
         }
 
         /* Special debug activities for freeing objects */
-        if (!SlabFrozen(page) && !page->freelist)
+        if (!PageSlubFrozen(page) && !page->freelist)
                 remove_full(s, page);
         if (s->flags & SLAB_STORE_USER)
                 set_track(s, object, TRACK_FREE, addr);
@@ -1044,7 +1012,7 @@ __setup("slub_debug", setup_slub_debug);
 
 static unsigned long kmem_cache_flags(unsigned long objsize,
         unsigned long flags, const char *name,
-        void (*ctor)(struct kmem_cache *, void *))
+        void (*ctor)(void *))
 {
         /*
          * Enable debugging if selected on the kernel commandline.
@@ -1072,7 +1040,7 @@ static inline int check_object(struct kmem_cache *s, struct page *page,
 static inline void add_full(struct kmem_cache_node *n, struct page *page) {}
 static inline unsigned long kmem_cache_flags(unsigned long objsize,
         unsigned long flags, const char *name,
-        void (*ctor)(struct kmem_cache *, void *))
+        void (*ctor)(void *))
 {
         return flags;
 }
@@ -1135,7 +1103,7 @@ static void setup_object(struct kmem_cache *s, struct page *page,
 {
         setup_object_debug(s, page, object);
         if (unlikely(s->ctor))
-                s->ctor(s, object);
+                s->ctor(object);
 }
 
 static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
@@ -1157,7 +1125,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
         page->flags |= 1 << PG_slab;
         if (s->flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_POISON |
                         SLAB_STORE_USER | SLAB_TRACE))
-                SetSlabDebug(page);
+                __SetPageSlubDebug(page);
 
         start = page_address(page);
 
@@ -1184,14 +1152,14 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
         int order = compound_order(page);
         int pages = 1 << order;
 
-        if (unlikely(SlabDebug(page))) {
+        if (unlikely(SLABDEBUG && PageSlubDebug(page))) {
                 void *p;
 
                 slab_pad_check(s, page);
                 for_each_object(p, s, page_address(page),
                                                 page->objects)
                         check_object(s, page, p, 0);
-                ClearSlabDebug(page);
+                __ClearPageSlubDebug(page);
         }
 
         mod_zone_page_state(page_zone(page),
@@ -1288,7 +1256,7 @@ static inline int lock_and_freeze_slab(struct kmem_cache_node *n,
         if (slab_trylock(page)) {
                 list_del(&page->lru);
                 n->nr_partial--;
-                SetSlabFrozen(page);
+                __SetPageSlubFrozen(page);
                 return 1;
         }
         return 0;
@@ -1361,7 +1329,7 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags)
                 n = get_node(s, zone_to_nid(zone));
 
                 if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
-                                n->nr_partial > MIN_PARTIAL) {
+                                n->nr_partial > n->min_partial) {
                         page = get_partial_node(n);
                         if (page)
                                 return page;
@@ -1398,7 +1366,7 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
         struct kmem_cache_node *n = get_node(s, page_to_nid(page));
         struct kmem_cache_cpu *c = get_cpu_slab(s, smp_processor_id());
 
-        ClearSlabFrozen(page);
+        __ClearPageSlubFrozen(page);
         if (page->inuse) {
 
                 if (page->freelist) {
@@ -1406,13 +1374,14 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
                         stat(c, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
                 } else {
                         stat(c, DEACTIVATE_FULL);
-                        if (SlabDebug(page) && (s->flags & SLAB_STORE_USER))
+                        if (SLABDEBUG && PageSlubDebug(page) &&
+                                                (s->flags & SLAB_STORE_USER))
                                 add_full(n, page);
                 }
                 slab_unlock(page);
         } else {
                 stat(c, DEACTIVATE_EMPTY);
-                if (n->nr_partial < MIN_PARTIAL) {
+                if (n->nr_partial < n->min_partial) {
                         /*
                          * Adding an empty slab to the partial slabs in order
                          * to avoid page allocator overhead. This slab needs
@@ -1551,7 +1520,7 @@ load_freelist:
         object = c->page->freelist;
         if (unlikely(!object))
                 goto another_slab;
-        if (unlikely(SlabDebug(c->page)))
+        if (unlikely(SLABDEBUG && PageSlubDebug(c->page)))
                 goto debug;
 
         c->freelist = object[c->offset];
@@ -1588,7 +1557,7 @@ new_slab:
                 if (c->page)
                         flush_slab(s, c);
                 slab_lock(new);
-                SetSlabFrozen(new);
+                __SetPageSlubFrozen(new);
                 c->page = new;
                 goto load_freelist;
         }
@@ -1674,7 +1643,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
         stat(c, FREE_SLOWPATH);
         slab_lock(page);
 
-        if (unlikely(SlabDebug(page)))
+        if (unlikely(SLABDEBUG && PageSlubDebug(page)))
                 goto debug;
 
 checks_ok:
@@ -1682,7 +1651,7 @@ checks_ok:
         page->freelist = object;
         page->inuse--;
 
-        if (unlikely(SlabFrozen(page))) {
+        if (unlikely(PageSlubFrozen(page))) {
                 stat(c, FREE_FROZEN);
                 goto out_unlock;
         }
@@ -1944,9 +1913,21 @@ static void init_kmem_cache_cpu(struct kmem_cache *s,
 #endif
 }
 
-static void init_kmem_cache_node(struct kmem_cache_node *n)
+static void
+init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
 {
         n->nr_partial = 0;
+
+        /*
+         * The larger the object size is, the more pages we want on the partial
+         * list to avoid pounding the page allocator excessively.
+         */
+        n->min_partial = ilog2(s->size);
+        if (n->min_partial < MIN_PARTIAL)
+                n->min_partial = MIN_PARTIAL;
+        else if (n->min_partial > MAX_PARTIAL)
+                n->min_partial = MAX_PARTIAL;
+
         spin_lock_init(&n->list_lock);
         INIT_LIST_HEAD(&n->partial);
 #ifdef CONFIG_SLUB_DEBUG
@@ -2118,7 +2099,7 @@ static struct kmem_cache_node *early_kmem_cache_node_alloc(gfp_t gfpflags,
         init_object(kmalloc_caches, n, 1);
         init_tracking(kmalloc_caches, n);
 #endif
-        init_kmem_cache_node(n);
+        init_kmem_cache_node(n, kmalloc_caches);
         inc_slabs_node(kmalloc_caches, node, page->objects);
 
         /*
@@ -2175,7 +2156,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
 
                 }
                 s->node[node] = n;
-                init_kmem_cache_node(n);
+                init_kmem_cache_node(n, s);
         }
         return 1;
 }
@@ -2186,7 +2167,7 @@ static void free_kmem_cache_nodes(struct kmem_cache *s)
 
 static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
 {
-        init_kmem_cache_node(&s->local_node);
+        init_kmem_cache_node(&s->local_node, s);
         return 1;
 }
 #endif
@@ -2317,7 +2298,7 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
 static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags,
                 const char *name, size_t size,
                 size_t align, unsigned long flags,
-                void (*ctor)(struct kmem_cache *, void *))
+                void (*ctor)(void *))
 {
         memset(s, 0, kmem_size);
         s->name = name;
@@ -2746,7 +2727,6 @@ size_t ksize(const void *object)
          */
         return s->size;
 }
-EXPORT_SYMBOL(ksize);
 
 void kfree(const void *x)
 {
@@ -2921,7 +2901,7 @@ static int slab_mem_going_online_callback(void *arg)
                         ret = -ENOMEM;
                         goto out;
                 }
-                init_kmem_cache_node(n);
+                init_kmem_cache_node(n, s);
                 s->node[nid] = n;
         }
 out:
@@ -3073,7 +3053,7 @@ static int slab_unmergeable(struct kmem_cache *s)
 
 static struct kmem_cache *find_mergeable(size_t size,
                 size_t align, unsigned long flags, const char *name,
-                void (*ctor)(struct kmem_cache *, void *))
+                void (*ctor)(void *))
 {
         struct kmem_cache *s;
 
@@ -3113,8 +3093,7 @@ static struct kmem_cache *find_mergeable(size_t size,
 }
 
 struct kmem_cache *kmem_cache_create(const char *name, size_t size,
-                size_t align, unsigned long flags,
-                void (*ctor)(struct kmem_cache *, void *))
+                size_t align, unsigned long flags, void (*ctor)(void *))
 {
         struct kmem_cache *s;
 
@@ -3317,12 +3296,12 @@ static void validate_slab_slab(struct kmem_cache *s, struct page *page,
                         s->name, page);
 
         if (s->flags & DEBUG_DEFAULT_FLAGS) {
-                if (!SlabDebug(page))
-                        printk(KERN_ERR "SLUB %s: SlabDebug not set "
+                if (!PageSlubDebug(page))
+                        printk(KERN_ERR "SLUB %s: SlubDebug not set "
                                 "on slab 0x%p\n", s->name, page);
         } else {
-                if (SlabDebug(page))
-                        printk(KERN_ERR "SLUB %s: SlabDebug set on "
+                if (PageSlubDebug(page))
+                        printk(KERN_ERR "SLUB %s: SlubDebug set on "
                                 "slab 0x%p\n", s->name, page);
         }
 }