inode: convert inode_sb_list_lock to per-sb

The process of reducing contention on per-superblock inode lists
starts with moving the locking to match the per-superblock inode
list. This takes the global lock out of the picture and reduces the
contention problems to within a single filesystem. This doesn't get
rid of contention as the locks still have global CPU scope, but it
does isolate operations on different superblocks form each other.

Signed-off-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Josef Bacik <jbacik@fb.com>
Reviewed-by: Jan Kara <jack@suse.cz>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Tested-by: Dave Chinner <dchinner@redhat.com>

10 files changed, 57 insertions, 54 deletions

diff --git a/fs/block_dev.c b/fs/block_dev.c
index 198243717da5..33b813e04f79 100644
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -1769,7 +1769,7 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
 {
         struct inode *inode, *old_inode = NULL;
 
-        spin_lock(&inode_sb_list_lock);
+        spin_lock(&blockdev_superblock->s_inode_list_lock);
         list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
                 struct address_space *mapping = inode->i_mapping;
 
@@ -1781,13 +1781,13 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
                 }
                 __iget(inode);
                 spin_unlock(&inode->i_lock);
-                spin_unlock(&inode_sb_list_lock);
+                spin_unlock(&blockdev_superblock->s_inode_list_lock);
                 /*
                  * We hold a reference to 'inode' so it couldn't have been
                  * removed from s_inodes list while we dropped the
-                 * inode_sb_list_lock.  We cannot iput the inode now as we can
+                 * s_inode_list_lock  We cannot iput the inode now as we can
                  * be holding the last reference and we cannot iput it under
-                 * inode_sb_list_lock. So we keep the reference and iput it
+                 * s_inode_list_lock. So we keep the reference and iput it
                  * later.
                  */
                 iput(old_inode);
@@ -1795,8 +1795,8 @@ void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
 
                 func(I_BDEV(inode), arg);
 
-                spin_lock(&inode_sb_list_lock);
+                spin_lock(&blockdev_superblock->s_inode_list_lock);
         }
-        spin_unlock(&inode_sb_list_lock);
+        spin_unlock(&blockdev_superblock->s_inode_list_lock);
         iput(old_inode);
 }
diff --git a/fs/drop_caches.c b/fs/drop_caches.c
index 5718cb9f7273..d72d52b90433 100644
--- a/fs/drop_caches.c
+++ b/fs/drop_caches.c
@@ -17,7 +17,7 @@ static void drop_pagecache_sb(struct super_block *sb, void *unused)
 {
         struct inode *inode, *toput_inode = NULL;
 
-        spin_lock(&inode_sb_list_lock);
+        spin_lock(&sb->s_inode_list_lock);
         list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
                 spin_lock(&inode->i_lock);
                 if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
@@ -27,13 +27,15 @@ static void drop_pagecache_sb(struct super_block *sb, void *unused)
                 }
                 __iget(inode);
                 spin_unlock(&inode->i_lock);
-                spin_unlock(&inode_sb_list_lock);
+                spin_unlock(&sb->s_inode_list_lock);
+
                 invalidate_mapping_pages(inode->i_mapping, 0, -1);
                 iput(toput_inode);
                 toput_inode = inode;
-                spin_lock(&inode_sb_list_lock);
+
+                spin_lock(&sb->s_inode_list_lock);
         }
-        spin_unlock(&inode_sb_list_lock);
+        spin_unlock(&sb->s_inode_list_lock);
         iput(toput_inode);
 }
 
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index d98e37bbf417..f45bf876579f 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -2124,7 +2124,7 @@ static void wait_sb_inodes(struct super_block *sb)
          */
         WARN_ON(!rwsem_is_locked(&sb->s_umount));
 
-        spin_lock(&inode_sb_list_lock);
+        spin_lock(&sb->s_inode_list_lock);
 
         /*
          * Data integrity sync. Must wait for all pages under writeback,
@@ -2144,14 +2144,14 @@ static void wait_sb_inodes(struct super_block *sb)
                 }
                 __iget(inode);
                 spin_unlock(&inode->i_lock);
-                spin_unlock(&inode_sb_list_lock);
+                spin_unlock(&sb->s_inode_list_lock);
 
                 /*
                  * We hold a reference to 'inode' so it couldn't have been
                  * removed from s_inodes list while we dropped the
-                 * inode_sb_list_lock.  We cannot iput the inode now as we can
+                 * s_inode_list_lock.  We cannot iput the inode now as we can
                  * be holding the last reference and we cannot iput it under
-                 * inode_sb_list_lock. So we keep the reference and iput it
+                 * s_inode_list_lock. So we keep the reference and iput it
                  * later.
                  */
                 iput(old_inode);
@@ -2161,9 +2161,9 @@ static void wait_sb_inodes(struct super_block *sb)
 
                 cond_resched();
 
-                spin_lock(&inode_sb_list_lock);
+                spin_lock(&sb->s_inode_list_lock);
         }
-        spin_unlock(&inode_sb_list_lock);
+        spin_unlock(&sb->s_inode_list_lock);
         iput(old_inode);
 }
 
diff --git a/fs/inode.c b/fs/inode.c
index d30640f7a193..a2de294f6b77 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -28,8 +28,8 @@
  *   inode->i_state, inode->i_hash, __iget()
  * Inode LRU list locks protect:
  *   inode->i_sb->s_inode_lru, inode->i_lru
- * inode_sb_list_lock protects:
- *   sb->s_inodes, inode->i_sb_list
+ * inode->i_sb->s_inode_list_lock protects:
+ *   inode->i_sb->s_inodes, inode->i_sb_list
  * bdi->wb.list_lock protects:
  *   bdi->wb.b_{dirty,io,more_io,dirty_time}, inode->i_wb_list
  * inode_hash_lock protects:
@@ -37,7 +37,7 @@
  *
  * Lock ordering:
  *
- * inode_sb_list_lock
+ * inode->i_sb->s_inode_list_lock
  *   inode->i_lock
  *     Inode LRU list locks
  *
@@ -45,7 +45,7 @@
  *   inode->i_lock
  *
  * inode_hash_lock
- *   inode_sb_list_lock
+ *   inode->i_sb->s_inode_list_lock
  *   inode->i_lock
  *
  * iunique_lock
@@ -57,8 +57,6 @@ static unsigned int i_hash_shift __read_mostly;
 static struct hlist_head *inode_hashtable __read_mostly;
 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
 
-__cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_sb_list_lock);
-
 /*
  * Empty aops. Can be used for the cases where the user does not
  * define any of the address_space operations.
@@ -426,18 +424,18 @@ static void inode_lru_list_del(struct inode *inode)
  */
 void inode_sb_list_add(struct inode *inode)
 {
-        spin_lock(&inode_sb_list_lock);
+        spin_lock(&inode->i_sb->s_inode_list_lock);
         list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
-        spin_unlock(&inode_sb_list_lock);
+        spin_unlock(&inode->i_sb->s_inode_list_lock);
 }
 EXPORT_SYMBOL_GPL(inode_sb_list_add);
 
 static inline void inode_sb_list_del(struct inode *inode)
 {
         if (!list_empty(&inode->i_sb_list)) {
-                spin_lock(&inode_sb_list_lock);
+                spin_lock(&inode->i_sb->s_inode_list_lock);
                 list_del_init(&inode->i_sb_list);
-                spin_unlock(&inode_sb_list_lock);
+                spin_unlock(&inode->i_sb->s_inode_list_lock);
         }
 }
 
@@ -594,7 +592,7 @@ void evict_inodes(struct super_block *sb)
         struct inode *inode, *next;
         LIST_HEAD(dispose);
 
-        spin_lock(&inode_sb_list_lock);
+        spin_lock(&sb->s_inode_list_lock);
         list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
                 if (atomic_read(&inode->i_count))
                         continue;
@@ -610,7 +608,7 @@ void evict_inodes(struct super_block *sb)
                 spin_unlock(&inode->i_lock);
                 list_add(&inode->i_lru, &dispose);
         }
-        spin_unlock(&inode_sb_list_lock);
+        spin_unlock(&sb->s_inode_list_lock);
 
         dispose_list(&dispose);
 }
@@ -631,7 +629,7 @@ int invalidate_inodes(struct super_block *sb, bool kill_dirty)
         struct inode *inode, *next;
         LIST_HEAD(dispose);
 
-        spin_lock(&inode_sb_list_lock);
+        spin_lock(&sb->s_inode_list_lock);
         list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
                 spin_lock(&inode->i_lock);
                 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
@@ -654,7 +652,7 @@ int invalidate_inodes(struct super_block *sb, bool kill_dirty)
                 spin_unlock(&inode->i_lock);
                 list_add(&inode->i_lru, &dispose);
         }
-        spin_unlock(&inode_sb_list_lock);
+        spin_unlock(&sb->s_inode_list_lock);
 
         dispose_list(&dispose);
 
@@ -890,7 +888,7 @@ struct inode *new_inode(struct super_block *sb)
 {
         struct inode *inode;
 
-        spin_lock_prefetch(&inode_sb_list_lock);
+        spin_lock_prefetch(&sb->s_inode_list_lock);
 
         inode = new_inode_pseudo(sb);
         if (inode)
diff --git a/fs/internal.h b/fs/internal.h
index 4d5af583ab03..ee1209c54eb1 100644
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -112,7 +112,6 @@ extern int vfs_open(const struct path *, struct file *, const struct cred *);
 /*
  * inode.c
  */
-extern spinlock_t inode_sb_list_lock;
 extern long prune_icache_sb(struct super_block *sb, struct shrink_control *sc);
 extern void inode_add_lru(struct inode *inode);
 
diff --git a/fs/notify/inode_mark.c b/fs/notify/inode_mark.c
index 3daf513ee99e..a4e1a8f6c329 100644
--- a/fs/notify/inode_mark.c
+++ b/fs/notify/inode_mark.c
@@ -163,17 +163,17 @@ int fsnotify_add_inode_mark(struct fsnotify_mark *mark,
 
 /**
  * fsnotify_unmount_inodes - an sb is unmounting.  handle any watched inodes.
- * @list: list of inodes being unmounted (sb->s_inodes)
+ * @sb: superblock being unmounted.
  *
  * Called during unmount with no locks held, so needs to be safe against
- * concurrent modifiers. We temporarily drop inode_sb_list_lock and CAN block.
+ * concurrent modifiers. We temporarily drop sb->s_inode_list_lock and CAN block.
  */
-void fsnotify_unmount_inodes(struct list_head *list)
+void fsnotify_unmount_inodes(struct super_block *sb)
 {
         struct inode *inode, *next_i, *need_iput = NULL;
 
-        spin_lock(&inode_sb_list_lock);
-        list_for_each_entry_safe(inode, next_i, list, i_sb_list) {
+        spin_lock(&sb->s_inode_list_lock);
+        list_for_each_entry_safe(inode, next_i, &sb->s_inodes, i_sb_list) {
                 struct inode *need_iput_tmp;
 
                 /*
@@ -209,7 +209,7 @@ void fsnotify_unmount_inodes(struct list_head *list)
                 spin_unlock(&inode->i_lock);
 
                 /* In case the dropping of a reference would nuke next_i. */
-                while (&next_i->i_sb_list != list) {
+                while (&next_i->i_sb_list != &sb->s_inodes) {
                         spin_lock(&next_i->i_lock);
                         if (!(next_i->i_state & (I_FREEING | I_WILL_FREE)) &&
                                                 atomic_read(&next_i->i_count)) {
@@ -224,12 +224,12 @@ void fsnotify_unmount_inodes(struct list_head *list)
                 }
 
                 /*
-                 * We can safely drop inode_sb_list_lock here because either
+                 * We can safely drop s_inode_list_lock here because either
                  * we actually hold references on both inode and next_i or
                  * end of list.  Also no new inodes will be added since the
                  * umount has begun.
                  */
-                spin_unlock(&inode_sb_list_lock);
+                spin_unlock(&sb->s_inode_list_lock);
 
                 if (need_iput_tmp)
                         iput(need_iput_tmp);
@@ -241,7 +241,7 @@ void fsnotify_unmount_inodes(struct list_head *list)
 
                 iput(inode);
 
-                spin_lock(&inode_sb_list_lock);
+                spin_lock(&sb->s_inode_list_lock);
         }
-        spin_unlock(&inode_sb_list_lock);
+        spin_unlock(&sb->s_inode_list_lock);
 }
diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c
index 20d1f74561cf..2863ec6cbadf 100644
--- a/fs/quota/dquot.c
+++ b/fs/quota/dquot.c
@@ -923,7 +923,7 @@ static void add_dquot_ref(struct super_block *sb, int type)
         int reserved = 0;
 #endif
 
-        spin_lock(&inode_sb_list_lock);
+        spin_lock(&sb->s_inode_list_lock);
         list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
                 spin_lock(&inode->i_lock);
                 if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
@@ -934,7 +934,7 @@ static void add_dquot_ref(struct super_block *sb, int type)
                 }
                 __iget(inode);
                 spin_unlock(&inode->i_lock);
-                spin_unlock(&inode_sb_list_lock);
+                spin_unlock(&sb->s_inode_list_lock);
 
 #ifdef CONFIG_QUOTA_DEBUG
                 if (unlikely(inode_get_rsv_space(inode) > 0))
@@ -946,15 +946,15 @@ static void add_dquot_ref(struct super_block *sb, int type)
                 /*
                  * We hold a reference to 'inode' so it couldn't have been
                  * removed from s_inodes list while we dropped the
-                 * inode_sb_list_lock We cannot iput the inode now as we can be
+                 * s_inode_list_lock. We cannot iput the inode now as we can be
                  * holding the last reference and we cannot iput it under
-                 * inode_sb_list_lock. So we keep the reference and iput it
+                 * s_inode_list_lock. So we keep the reference and iput it
                  * later.
                  */
                 old_inode = inode;
-                spin_lock(&inode_sb_list_lock);
+                spin_lock(&sb->s_inode_list_lock);
         }
-        spin_unlock(&inode_sb_list_lock);
+        spin_unlock(&sb->s_inode_list_lock);
         iput(old_inode);
 
 #ifdef CONFIG_QUOTA_DEBUG
@@ -1023,7 +1023,7 @@ static void remove_dquot_ref(struct super_block *sb, int type,
         struct inode *inode;
         int reserved = 0;
 
-        spin_lock(&inode_sb_list_lock);
+        spin_lock(&sb->s_inode_list_lock);
         list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
                 /*
                  *  We have to scan also I_NEW inodes because they can already
@@ -1039,7 +1039,7 @@ static void remove_dquot_ref(struct super_block *sb, int type,
                 }
                 spin_unlock(&dq_data_lock);
         }
-        spin_unlock(&inode_sb_list_lock);
+        spin_unlock(&sb->s_inode_list_lock);
 #ifdef CONFIG_QUOTA_DEBUG
         if (reserved) {
                 printk(KERN_WARNING "VFS (%s): Writes happened after quota"
diff --git a/fs/super.c b/fs/super.c
index b61372354f2b..c808183554a2 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -191,6 +191,7 @@ static struct super_block *alloc_super(struct file_system_type *type, int flags)
         INIT_HLIST_NODE(&s->s_instances);
         INIT_HLIST_BL_HEAD(&s->s_anon);
         INIT_LIST_HEAD(&s->s_inodes);
+        spin_lock_init(&s->s_inode_list_lock);
 
         if (list_lru_init_memcg(&s->s_dentry_lru))
                 goto fail;
@@ -399,7 +400,7 @@ void generic_shutdown_super(struct super_block *sb)
                 sync_filesystem(sb);
                 sb->s_flags &= ~MS_ACTIVE;
 
-                fsnotify_unmount_inodes(&sb->s_inodes);
+                fsnotify_unmount_inodes(sb);
 
                 evict_inodes(sb);
 
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 4a40fa843040..09bbd38485f9 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -1309,7 +1309,6 @@ struct super_block {
 #endif
         const struct xattr_handler **s_xattr;
 
-        struct list_head        s_inodes;       /* all inodes */
         struct hlist_bl_head    s_anon;         /* anonymous dentries for (nfs) exporting */
         struct list_head        s_mounts;       /* list of mounts; _not_ for fs use */
         struct block_device     *s_bdev;
@@ -1380,6 +1379,10 @@ struct super_block {
          * Indicates how deep in a filesystem stack this SB is
          */
         int s_stack_depth;
+
+        /* s_inode_list_lock protects s_inodes */
+        spinlock_t              s_inode_list_lock ____cacheline_aligned_in_smp;
+        struct list_head        s_inodes;       /* all inodes */
 };
 
 extern struct timespec current_fs_time(struct super_block *sb);
diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h
index 65a517dd32f7..0390ee69c439 100644
--- a/include/linux/fsnotify_backend.h
+++ b/include/linux/fsnotify_backend.h
@@ -357,7 +357,7 @@ extern void fsnotify_clear_marks_by_group_flags(struct fsnotify_group *group, un
 extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group);
 extern void fsnotify_get_mark(struct fsnotify_mark *mark);
 extern void fsnotify_put_mark(struct fsnotify_mark *mark);
-extern void fsnotify_unmount_inodes(struct list_head *list);
+extern void fsnotify_unmount_inodes(struct super_block *sb);
 
 /* put here because inotify does some weird stuff when destroying watches */
 extern void fsnotify_init_event(struct fsnotify_event *event,
@@ -393,7 +393,7 @@ static inline u32 fsnotify_get_cookie(void)
         return 0;
 }
 
-static inline void fsnotify_unmount_inodes(struct list_head *list)
+static inline void fsnotify_unmount_inodes(struct super_block *sb)
 {}
 
 #endif  /* CONFIG_FSNOTIFY */