1 files changed, 59 insertions, 35 deletions

diff --git a/mm/slub.c b/mm/slub.c
index 744d29b43bf6..9cde663bbb10 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -160,7 +160,7 @@ static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
  */
 #define MAX_PARTIAL 10
 
-#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \
+#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
                                 SLAB_POISON | SLAB_STORE_USER)
 
 /*
@@ -1007,20 +1007,32 @@ static void setup_object_debug(struct kmem_cache *s, struct page *page,
         init_tracking(s, object);
 }
 
-static noinline int alloc_debug_processing(struct kmem_cache *s,
+static inline int alloc_consistency_checks(struct kmem_cache *s,
                                         struct page *page,
                                         void *object, unsigned long addr)
 {
         if (!check_slab(s, page))
-                goto bad;
+                return 0;
 
         if (!check_valid_pointer(s, page, object)) {
                 object_err(s, page, object, "Freelist Pointer check fails");
-                goto bad;
+                return 0;
         }
 
         if (!check_object(s, page, object, SLUB_RED_INACTIVE))
-                goto bad;
+                return 0;
+
+        return 1;
+}
+
+static noinline int alloc_debug_processing(struct kmem_cache *s,
+                                        struct page *page,
+                                        void *object, unsigned long addr)
+{
+        if (s->flags & SLAB_CONSISTENCY_CHECKS) {
+                if (!alloc_consistency_checks(s, page, object, addr))
+                        goto bad;
+        }
 
         /* Success perform special debug activities for allocs */
         if (s->flags & SLAB_STORE_USER)
@@ -1043,39 +1055,21 @@ bad:
         return 0;
 }
 
-/* Supports checking bulk free of a constructed freelist */
-static noinline int free_debug_processing(
-        struct kmem_cache *s, struct page *page,
-        void *head, void *tail, int bulk_cnt,
-        unsigned long addr)
+static inline int free_consistency_checks(struct kmem_cache *s,
+                struct page *page, void *object, unsigned long addr)
 {
-        struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-        void *object = head;
-        int cnt = 0;
-        unsigned long uninitialized_var(flags);
-        int ret = 0;
-
-        spin_lock_irqsave(&n->list_lock, flags);
-        slab_lock(page);
-
-        if (!check_slab(s, page))
-                goto out;
-
-next_object:
-        cnt++;
-
         if (!check_valid_pointer(s, page, object)) {
                 slab_err(s, page, "Invalid object pointer 0x%p", object);
-                goto out;
+                return 0;
         }
 
         if (on_freelist(s, page, object)) {
                 object_err(s, page, object, "Object already free");
-                goto out;
+                return 0;
         }
 
         if (!check_object(s, page, object, SLUB_RED_ACTIVE))
-                goto out;
+                return 0;
 
         if (unlikely(s != page->slab_cache)) {
                 if (!PageSlab(page)) {
@@ -1088,7 +1082,37 @@ next_object:
                 } else
                         object_err(s, page, object,
                                         "page slab pointer corrupt.");
-                goto out;
+                return 0;
+        }
+        return 1;
+}
+
+/* Supports checking bulk free of a constructed freelist */
+static noinline int free_debug_processing(
+        struct kmem_cache *s, struct page *page,
+        void *head, void *tail, int bulk_cnt,
+        unsigned long addr)
+{
+        struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+        void *object = head;
+        int cnt = 0;
+        unsigned long uninitialized_var(flags);
+        int ret = 0;
+
+        spin_lock_irqsave(&n->list_lock, flags);
+        slab_lock(page);
+
+        if (s->flags & SLAB_CONSISTENCY_CHECKS) {
+                if (!check_slab(s, page))
+                        goto out;
+        }
+
+next_object:
+        cnt++;
+
+        if (s->flags & SLAB_CONSISTENCY_CHECKS) {
+                if (!free_consistency_checks(s, page, object, addr))
+                        goto out;
         }
 
         if (s->flags & SLAB_STORE_USER)
@@ -1145,7 +1169,7 @@ static int __init setup_slub_debug(char *str)
         for (; *str && *str != ','; str++) {
                 switch (tolower(*str)) {
                 case 'f':
-                        slub_debug |= SLAB_DEBUG_FREE;
+                        slub_debug |= SLAB_CONSISTENCY_CHECKS;
                         break;
                 case 'z':
                         slub_debug |= SLAB_RED_ZONE;
@@ -1449,7 +1473,7 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
         int order = compound_order(page);
         int pages = 1 << order;
 
-        if (kmem_cache_debug(s)) {
+        if (s->flags & SLAB_CONSISTENCY_CHECKS) {
                 void *p;
 
                 slab_pad_check(s, page);
@@ -4769,16 +4793,16 @@ SLAB_ATTR_RO(total_objects);
 
 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
 {
-        return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE));
+        return sprintf(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
 }
 
 static ssize_t sanity_checks_store(struct kmem_cache *s,
                                 const char *buf, size_t length)
 {
-        s->flags &= ~SLAB_DEBUG_FREE;
+        s->flags &= ~SLAB_CONSISTENCY_CHECKS;
         if (buf[0] == '1') {
                 s->flags &= ~__CMPXCHG_DOUBLE;
-                s->flags |= SLAB_DEBUG_FREE;
+                s->flags |= SLAB_CONSISTENCY_CHECKS;
         }
         return length;
 }
@@ -5313,7 +5337,7 @@ static char *create_unique_id(struct kmem_cache *s)
                 *p++ = 'd';
         if (s->flags & SLAB_RECLAIM_ACCOUNT)
                 *p++ = 'a';
-        if (s->flags & SLAB_DEBUG_FREE)
+        if (s->flags & SLAB_CONSISTENCY_CHECKS)
                 *p++ = 'F';
         if (!(s->flags & SLAB_NOTRACK))
                 *p++ = 't';