6 files changed, 182 insertions, 76 deletions

diff --git a/include/linux/slab.h b/include/linux/slab.h
index 96940772bb92..2037a861e367 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -316,7 +316,7 @@ void kmem_cache_free(struct kmem_cache *, void *);
  * Note that interrupts must be enabled when calling these functions.
  */
 void kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
-bool kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
+int kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
 
 #ifdef CONFIG_NUMA
 void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment;
diff --git a/mm/slab.c b/mm/slab.c
index e0819fa96559..4765c97ce690 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3419,7 +3419,7 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
 }
 EXPORT_SYMBOL(kmem_cache_free_bulk);
 
-bool kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
+int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
                                                                 void **p)
 {
         return __kmem_cache_alloc_bulk(s, flags, size, p);
diff --git a/mm/slab.h b/mm/slab.h
index 27492eb678f7..7b6087197997 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -170,7 +170,7 @@ ssize_t slabinfo_write(struct file *file, const char __user *buffer,
  * may be allocated or freed using these operations.
  */
 void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
-bool __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
+int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
 
 #ifdef CONFIG_MEMCG_KMEM
 /*
diff --git a/mm/slab_common.c b/mm/slab_common.c
index d88e97c10a2e..3c6a86b4ec25 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -112,7 +112,7 @@ void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p)
                 kmem_cache_free(s, p[i]);
 }
 
-bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,
+int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,
                                                                 void **p)
 {
         size_t i;
@@ -121,10 +121,10 @@ bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,
                 void *x = p[i] = kmem_cache_alloc(s, flags);
                 if (!x) {
                         __kmem_cache_free_bulk(s, i, p);
-                        return false;
+                        return 0;
                 }
         }
-        return true;
+        return i;
 }
 
 #ifdef CONFIG_MEMCG_KMEM
diff --git a/mm/slob.c b/mm/slob.c
index 0d7e5df74d1f..17e8f8cc7c53 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -617,7 +617,7 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
 }
 EXPORT_SYMBOL(kmem_cache_free_bulk);
 
-bool kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
+int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
                                                                 void **p)
 {
         return __kmem_cache_alloc_bulk(s, flags, size, p);
diff --git a/mm/slub.c b/mm/slub.c
index a0c1365f6426..46997517406e 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1065,11 +1065,15 @@ bad:
         return 0;
 }
 
+/* Supports checking bulk free of a constructed freelist */
 static noinline struct kmem_cache_node *free_debug_processing(
-        struct kmem_cache *s, struct page *page, void *object,
+        struct kmem_cache *s, struct page *page,
+        void *head, void *tail, int bulk_cnt,
         unsigned long addr, unsigned long *flags)
 {
         struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+        void *object = head;
+        int cnt = 0;
 
         spin_lock_irqsave(&n->list_lock, *flags);
         slab_lock(page);
@@ -1077,6 +1081,9 @@ static noinline struct kmem_cache_node *free_debug_processing(
         if (!check_slab(s, page))
                 goto fail;
 
+next_object:
+        cnt++;
+
         if (!check_valid_pointer(s, page, object)) {
                 slab_err(s, page, "Invalid object pointer 0x%p", object);
                 goto fail;
@@ -1107,8 +1114,19 @@ static noinline struct kmem_cache_node *free_debug_processing(
         if (s->flags & SLAB_STORE_USER)
                 set_track(s, object, TRACK_FREE, addr);
         trace(s, page, object, 0);
+        /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
         init_object(s, object, SLUB_RED_INACTIVE);
+
+        /* Reached end of constructed freelist yet? */
+        if (object != tail) {
+                object = get_freepointer(s, object);
+                goto next_object;
+        }
 out:
+        if (cnt != bulk_cnt)
+                slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n",
+                         bulk_cnt, cnt);
+
         slab_unlock(page);
         /*
          * Keep node_lock to preserve integrity
@@ -1212,7 +1230,8 @@ static inline int alloc_debug_processing(struct kmem_cache *s,
         struct page *page, void *object, unsigned long addr) { return 0; }
 
 static inline struct kmem_cache_node *free_debug_processing(
-        struct kmem_cache *s, struct page *page, void *object,
+        struct kmem_cache *s, struct page *page,
+        void *head, void *tail, int bulk_cnt,
         unsigned long addr, unsigned long *flags) { return NULL; }
 
 static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
@@ -1273,14 +1292,21 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
         return memcg_kmem_get_cache(s, flags);
 }
 
-static inline void slab_post_alloc_hook(struct kmem_cache *s,
-                                        gfp_t flags, void *object)
+static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
+                                        size_t size, void **p)
 {
+        size_t i;
+
         flags &= gfp_allowed_mask;
-        kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
-        kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags);
+        for (i = 0; i < size; i++) {
+                void *object = p[i];
+
+                kmemcheck_slab_alloc(s, flags, object, slab_ksize(s));
+                kmemleak_alloc_recursive(object, s->object_size, 1,
+                                         s->flags, flags);
+                kasan_slab_alloc(s, object);
+        }
         memcg_kmem_put_cache(s);
-        kasan_slab_alloc(s, object);
 }
 
 static inline void slab_free_hook(struct kmem_cache *s, void *x)
@@ -1308,6 +1334,29 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x)
         kasan_slab_free(s, x);
 }
 
+static inline void slab_free_freelist_hook(struct kmem_cache *s,
+                                           void *head, void *tail)
+{
+/*
+ * Compiler cannot detect this function can be removed if slab_free_hook()
+ * evaluates to nothing.  Thus, catch all relevant config debug options here.
+ */
+#if defined(CONFIG_KMEMCHECK) ||                \
+        defined(CONFIG_LOCKDEP) ||              \
+        defined(CONFIG_DEBUG_KMEMLEAK) ||       \
+        defined(CONFIG_DEBUG_OBJECTS_FREE) ||   \
+        defined(CONFIG_KASAN)
+
+        void *object = head;
+        void *tail_obj = tail ? : head;
+
+        do {
+                slab_free_hook(s, object);
+        } while ((object != tail_obj) &&
+                 (object = get_freepointer(s, object)));
+#endif
+}
+
 static void setup_object(struct kmem_cache *s, struct page *page,
                                 void *object)
 {
@@ -2433,7 +2482,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,