1 files changed, 41 insertions, 20 deletions

diff --git a/mm/slub.c b/mm/slub.c
index 36c0befeebd8..dc777761b6b7 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -249,7 +249,18 @@ static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr,
                                  unsigned long ptr_addr)
 {
 #ifdef CONFIG_SLAB_FREELIST_HARDENED
-        return (void *)((unsigned long)ptr ^ s->random ^ ptr_addr);
+        /*
+         * When CONFIG_KASAN_SW_TAGS is enabled, ptr_addr might be tagged.
+         * Normally, this doesn't cause any issues, as both set_freepointer()
+         * and get_freepointer() are called with a pointer with the same tag.
+         * However, there are some issues with CONFIG_SLUB_DEBUG code. For
+         * example, when __free_slub() iterates over objects in a cache, it
+         * passes untagged pointers to check_object(). check_object() in turns
+         * calls get_freepointer() with an untagged pointer, which causes the
+         * freepointer to be restored incorrectly.
+         */
+        return (void *)((unsigned long)ptr ^ s->random ^
+                        (unsigned long)kasan_reset_tag((void *)ptr_addr));
 #else
         return ptr;
 #endif
@@ -303,15 +314,10 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
                 __p < (__addr) + (__objects) * (__s)->size; \
                 __p += (__s)->size)
 
-#define for_each_object_idx(__p, __idx, __s, __addr, __objects) \
-        for (__p = fixup_red_left(__s, __addr), __idx = 1; \
-                __idx <= __objects; \
-                __p += (__s)->size, __idx++)
-
 /* Determine object index from a given position */
 static inline unsigned int slab_index(void *p, struct kmem_cache *s, void *addr)
 {
-        return (p - addr) / s->size;
+        return (kasan_reset_tag(p) - addr) / s->size;
 }
 
 static inline unsigned int order_objects(unsigned int order, unsigned int size)
@@ -507,6 +513,7 @@ static inline int check_valid_pointer(struct kmem_cache *s,
                 return 1;
 
         base = page_address(page);
+        object = kasan_reset_tag(object);
         object = restore_red_left(s, object);
         if (object < base || object >= base + page->objects * s->size ||
                 (object - base) % s->size) {
@@ -1075,6 +1082,16 @@ static void setup_object_debug(struct kmem_cache *s, struct page *page,
         init_tracking(s, object);
 }
 
+static void setup_page_debug(struct kmem_cache *s, void *addr, int order)
+{
+        if (!(s->flags & SLAB_POISON))
+                return;
+
+        metadata_access_enable();
+        memset(addr, POISON_INUSE, PAGE_SIZE << order);
+        metadata_access_disable();
+}
+
 static inline int alloc_consistency_checks(struct kmem_cache *s,
                                         struct page *page,
                                         void *object, unsigned long addr)
@@ -1330,6 +1347,8 @@ slab_flags_t kmem_cache_flags(unsigned int object_size,
 #else /* !CONFIG_SLUB_DEBUG */
 static inline void setup_object_debug(struct kmem_cache *s,
                         struct page *page, void *object) {}
+static inline void setup_page_debug(struct kmem_cache *s,
+                        void *addr, int order) {}
 
 static inline int alloc_debug_processing(struct kmem_cache *s,
         struct page *page, void *object, unsigned long addr) { return 0; }
@@ -1374,8 +1393,10 @@ static inline void dec_slabs_node(struct kmem_cache *s, int node,
  */
 static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
 {
+        ptr = kasan_kmalloc_large(ptr, size, flags);
+        /* As ptr might get tagged, call kmemleak hook after KASAN. */
         kmemleak_alloc(ptr, size, 1, flags);
-        return kasan_kmalloc_large(ptr, size, flags);
+        return ptr;
 }
 
 static __always_inline void kfree_hook(void *x)
@@ -1641,27 +1662,25 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
         if (page_is_pfmemalloc(page))
                 SetPageSlabPfmemalloc(page);
 
-        start = page_address(page);
+        kasan_poison_slab(page);
 
-        if (unlikely(s->flags & SLAB_POISON))
-                memset(start, POISON_INUSE, PAGE_SIZE << order);
+        start = page_address(page);
 
-        kasan_poison_slab(page);
+        setup_page_debug(s, start, order);
 
         shuffle = shuffle_freelist(s, page);
 
         if (!shuffle) {
-                for_each_object_idx(p, idx, s, start, page->objects) {
-                        if (likely(idx < page->objects)) {
-                                next = p + s->size;
-                                next = setup_object(s, page, next);
-                                set_freepointer(s, p, next);
-                        } else
-                                set_freepointer(s, p, NULL);
-                }
                 start = fixup_red_left(s, start);
                 start = setup_object(s, page, start);
                 page->freelist = start;
+                for (idx = 0, p = start; idx < page->objects - 1; idx++) {
+                        next = p + s->size;
+                        next = setup_object(s, page, next);
+                        set_freepointer(s, p, next);
+                        p = next;
+                }
+                set_freepointer(s, p, NULL);
         }
 
         page->inuse = page->objects;
@@ -3846,6 +3865,8 @@ void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
         unsigned int offset;
         size_t object_size;
 
+        ptr = kasan_reset_tag(ptr);
+
         /* Find object and usable object size. */
         s = page->slab_cache;