slub: explicit list_lock taking

The allocator fastpath rework does change the usage of the list_lock.
Remove the list_lock processing from the functions that hide them from the
critical sections and move them into those critical sections.

This in turn simplifies the support functions (no __ variant needed anymore)
and simplifies the lock handling on bootstrap.

Inline add_partial since it becomes pretty simple.

Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>

1 files changed, 49 insertions, 40 deletions

diff --git a/mm/slub.c b/mm/slub.c
index be6715dd0ee..e39be0928a2 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -916,26 +916,27 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x)
 
 /*
  * Tracking of fully allocated slabs for debugging purposes.
+ *
+ * list_lock must be held.
  */
-static void add_full(struct kmem_cache_node *n, struct page *page)
+static void add_full(struct kmem_cache *s,
+        struct kmem_cache_node *n, struct page *page)
 {
-        spin_lock(&n->list_lock);
+        if (!(s->flags & SLAB_STORE_USER))
+                return;
+
         list_add(&page->lru, &n->full);
-        spin_unlock(&n->list_lock);
 }
 
+/*
+ * list_lock must be held.
+ */
 static void remove_full(struct kmem_cache *s, struct page *page)
 {
-        struct kmem_cache_node *n;
-
         if (!(s->flags & SLAB_STORE_USER))
                 return;
 
-        n = get_node(s, page_to_nid(page));
-
-        spin_lock(&n->list_lock);
         list_del(&page->lru);
-        spin_unlock(&n->list_lock);
 }
 
 /* Tracking of the number of slabs for debugging purposes */
@@ -1060,8 +1061,13 @@ static noinline int free_debug_processing(struct kmem_cache *s,
         }
 
         /* Special debug activities for freeing objects */
-        if (!page->frozen && !page->freelist)
+        if (!page->frozen && !page->freelist) {
+                struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+
+                spin_lock(&n->list_lock);
                 remove_full(s, page);
+                spin_unlock(&n->list_lock);
+        }
         if (s->flags & SLAB_STORE_USER)
                 set_track(s, object, TRACK_FREE, addr);
         trace(s, page, object, 0);
@@ -1170,7 +1176,8 @@ static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
                         { return 1; }
 static inline int check_object(struct kmem_cache *s, struct page *page,
                         void *object, u8 val) { return 1; }
-static inline void add_full(struct kmem_cache_node *n, struct page *page) {}
+static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
+                                        struct page *page) {}
 static inline unsigned long kmem_cache_flags(unsigned long objsize,
         unsigned long flags, const char *name,
         void (*ctor)(void *))
@@ -1420,38 +1427,33 @@ static __always_inline int slab_trylock(struct page *page)
 }
 
 /*
- * Management of partially allocated slabs
+ * Management of partially allocated slabs.
+ *
+ * list_lock must be held.
  */
-static void add_partial(struct kmem_cache_node *n,
+static inline void add_partial(struct kmem_cache_node *n,
                                 struct page *page, int tail)
 {
-        spin_lock(&n->list_lock);
         n->nr_partial++;
         if (tail)
                 list_add_tail(&page->lru, &n->partial);
         else
                 list_add(&page->lru, &n->partial);
-        spin_unlock(&n->list_lock);
 }
 
-static inline void __remove_partial(struct kmem_cache_node *n,
+/*
+ * list_lock must be held.
+ */
+static inline void remove_partial(struct kmem_cache_node *n,
                                         struct page *page)
 {
         list_del(&page->lru);
         n->nr_partial--;
 }
 
-static void remove_partial(struct kmem_cache *s, struct page *page)
-{
-        struct kmem_cache_node *n = get_node(s, page_to_nid(page));
-
-        spin_lock(&n->list_lock);
-        __remove_partial(n, page);
-        spin_unlock(&n->list_lock);
-}
-
 /*
- * Lock slab and remove from the partial list.
+ * Lock slab, remove from the partial list and put the object into the
+ * per cpu freelist.
  *
  * Must hold list_lock.
  */
@@ -1459,7 +1461,7 @@ static inline int lock_and_freeze_slab(struct kmem_cache_node *n,
                                                         struct page *page)
 {
         if (slab_trylock(page)) {
-                __remove_partial(n, page);
+                remove_partial(n, page);
                 return 1;
         }
         return 0;
@@ -1576,12 +1578,17 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
         if (page->inuse) {
 
                 if (page->freelist) {
+                        spin_lock(&n->list_lock);
                         add_partial(n, page, tail);
+                        spin_unlock(&n->list_lock);
                         stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD);
                 } else {
                         stat(s, DEACTIVATE_FULL);
-                        if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER))
-                                add_full(n, page);
+                        if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER)) {
+                                spin_lock(&n->list_lock);
+                                add_full(s, n, page);
+                                spin_unlock(&n->list_lock);
+                        }
                 }
                 slab_unlock(page);
         } else {
@@ -1597,7 +1604,9 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail)
                          * kmem_cache_shrink can reclaim any empty slabs from
                          * the partial list.
                          */
+                        spin_lock(&n->list_lock);
                         add_partial(n, page, 1);
+                        spin_unlock(&n->list_lock);
                         slab_unlock(page);
                 } else {
                         slab_unlock(page);
@@ -2099,7 +2108,11 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
          * then add it.
          */
         if (unlikely(!prior)) {
+                struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+
+                spin_lock(&n->list_lock);
                 add_partial(get_node(s, page_to_nid(page)), page, 1);
+                spin_unlock(&n->list_lock);
                 stat(s, FREE_ADD_PARTIAL);
         }
 
@@ -2113,7 +2126,11 @@ slab_empty:
                 /*
                  * Slab still on the partial list.
                  */
-                remove_partial(s, page);
+                struct kmem_cache_node *n = get_node(s, page_to_nid(page));
+
+                spin_lock(&n->list_lock);
+                remove_partial(n, page);
+                spin_unlock(&n->list_lock);
                 stat(s, FREE_REMOVE_PARTIAL);
         }
         slab_unlock(page);
@@ -2391,7 +2408,6 @@ static void early_kmem_cache_node_alloc(int node)
 {
         struct page *page;
         struct kmem_cache_node *n;
-        unsigned long flags;
 
         BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
 
@@ -2418,14 +2434,7 @@ static void early_kmem_cache_node_alloc(int node)
         init_kmem_cache_node(n, kmem_cache_node);
         inc_slabs_node(kmem_cache_node, node, page->objects);
 
-        /*
-         * lockdep requires consistent irq usage for each lock
-         * so even though there cannot be a race this early in
-         * the boot sequence, we still disable irqs.
-         */
-        local_irq_save(flags);
         add_partial(n, page, 0);
-        local_irq_restore(flags);
 }
 
 static void free_kmem_cache_nodes(struct kmem_cache *s)
@@ -2709,7 +2718,7 @@ static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
         spin_lock_irqsave(&n->list_lock, flags);
         list_for_each_entry_safe(page, h, &n->partial, lru) {
                 if (!page->inuse) {
-                        __remove_partial(n, page);
+                        remove_partial(n, page);
                         discard_slab(s, page);
                 } else {
                         list_slab_objects(s, page,
@@ -3047,7 +3056,7 @@ int kmem_cache_shrink(struct kmem_cache *s)
                                  * may have freed the last object and be
                                  * waiting to release the slab.
                                  */
-                                __remove_partial(n, page);
+                                remove_partial(n, page);
                                 slab_unlock(page);
                                 discard_slab(s, page);
                         } else {