fs-inode_lock-scale-10

Impelemnt lazy inode lru similarly to dcache. This should reduce inode list
lock acquisition (todo: measure).

Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: John Stultz <johnstul@us.ibm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

4 files changed, 42 insertions, 48 deletions

diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index 07d70704078e..df99a99f99a2 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -544,7 +544,7 @@ select_queue:
                         /*
                          * The inode is clean, inuse
                          */
-                        list_move(&inode->i_list, &inode_in_use);
+                        list_del_init(&inode->i_list);
                 } else {
                         /*
                          * The inode is clean, unused
@@ -1151,8 +1151,6 @@ static void wait_sb_inodes(struct super_block *sb)
          */
         WARN_ON(!rwsem_is_locked(&sb->s_umount));
 
-        spin_lock(&sb_inode_list_lock);
-
         /*
          * Data integrity sync. Must wait for all pages under writeback,
          * because there may have been pages dirtied before our sync
@@ -1160,7 +1158,8 @@ static void wait_sb_inodes(struct super_block *sb)
          * In which case, the inode may not be on the dirty list, but
          * we still have to wait for that writeout.
          */
-        list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
+        rcu_read_lock();
+        list_for_each_entry_rcu(inode, &sb->s_inodes, i_sb_list) {
                 struct address_space *mapping;
 
                 mapping = inode->i_mapping;
@@ -1174,13 +1173,13 @@ static void wait_sb_inodes(struct super_block *sb)
                 }
                 __iget(inode);
                 spin_unlock(&inode->i_lock);
-                spin_unlock(&sb_inode_list_lock);
+                rcu_read_unlock();
                 /*
                  * We hold a reference to 'inode' so it couldn't have been
-                 * removed from s_inodes list while we dropped the
-                 * sb_inode_list_lock.  We cannot iput the inode now as we can
-                 * be holding the last reference and we cannot iput it under
-                 * spinlock. So we keep the reference and iput it later.
+                 * removed from s_inodes list while we dropped the i_lock.  We
+                 * cannot iput the inode now as we can be holding the last
+                 * reference and we cannot iput it under spinlock. So we keep
+                 * the reference and iput it later.
                  */
                 iput(old_inode);
                 old_inode = inode;
@@ -1189,9 +1188,9 @@ static void wait_sb_inodes(struct super_block *sb)
 
                 cond_resched();
 
-                spin_lock(&sb_inode_list_lock);
+                rcu_read_lock();
         }
-        spin_unlock(&sb_inode_list_lock);
+        rcu_read_unlock();
         iput(old_inode);
 }
 
diff --git a/fs/inode.c b/fs/inode.c
index 1d7ca6ed7dbd..5039ddf4f0e0 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -74,7 +74,6 @@ static unsigned int i_hash_shift __read_mostly;
  * allowing for low-overhead inode sync() operations.
  */
 
-LIST_HEAD(inode_in_use);
 LIST_HEAD(inode_unused);
 
 struct inode_hash_bucket {
@@ -266,6 +265,7 @@ void inode_init_once(struct inode *inode)
         INIT_HLIST_NODE(&inode->i_hash);
         INIT_LIST_HEAD(&inode->i_dentry);
         INIT_LIST_HEAD(&inode->i_devices);
+        INIT_LIST_HEAD(&inode->i_list);
         INIT_RADIX_TREE(&inode->i_data.page_tree, GFP_ATOMIC);
         spin_lock_init(&inode->i_data.tree_lock);
         spin_lock_init(&inode->i_data.i_mmap_lock);
@@ -291,24 +291,6 @@ static void init_once(void *foo)
         inode_init_once(inode);
 }
 
-/*
- * inode_lock must be held
- */
-void __iget(struct inode *inode)
-{
-        assert_spin_locked(&inode->i_lock);
-        inode->i_count++;
-        if (inode->i_count > 1)
-                return;
-
-        if (!(inode->i_state & (I_DIRTY|I_SYNC))) {
-                spin_lock(&wb_inode_list_lock);
-                list_move(&inode->i_list, &inode_in_use);
-                spin_unlock(&wb_inode_list_lock);
-        }
-        atomic_dec(&inodes_stat.nr_unused);
-}
-
 /**
  * clear_inode - clear an inode
  * @inode: inode to clear
@@ -352,7 +334,7 @@ static void dispose_list(struct list_head *head)
                 struct inode *inode;
 
                 inode = list_first_entry(head, struct inode, i_list);
-                list_del(&inode->i_list);
+                list_del_init(&inode->i_list);
 
                 if (inode->i_data.nrpages)
                         truncate_inode_pages(&inode->i_data, 0);
@@ -405,11 +387,12 @@ static int invalidate_list(struct list_head *head, struct list_head *dispose)
                 invalidate_inode_buffers(inode);
                 if (!inode->i_count) {
                         spin_lock(&wb_inode_list_lock);
-                        list_move(&inode->i_list, dispose);
+                        list_del(&inode->i_list);
                         spin_unlock(&wb_inode_list_lock);
                         WARN_ON(inode->i_state & I_NEW);
                         inode->i_state |= I_FREEING;
                         spin_unlock(&inode->i_lock);
+                        list_add(&inode->i_list, dispose);
                         count++;
                         continue;
                 }
@@ -496,7 +479,13 @@ again:
                         spin_unlock(&wb_inode_list_lock);
                         goto again;
                 }
-                if (inode->i_state || inode->i_count) {
+                if (inode->i_count) {
+                        list_del_init(&inode->i_list);
+                        spin_unlock(&inode->i_lock);
+                        atomic_dec(&inodes_stat.nr_unused);
+                        continue;
+                }
+                if (inode->i_state) {
                         list_move(&inode->i_list, &inode_unused);
                         spin_unlock(&inode->i_lock);
                         continue;
@@ -512,6 +501,7 @@ again:
 again2:
                         spin_lock(&wb_inode_list_lock);
 
+                        /* XXX: may no longer work well */
                         if (inode != list_entry(inode_unused.next,
                                                 struct inode, i_list))
                                 continue;       /* wrong inode or list_empty */
@@ -660,9 +650,6 @@ __inode_add_to_lists(struct super_block *sb, struct inode_hash_bucket *b,
         atomic_inc(&inodes_stat.nr_inodes);
         list_add(&inode->i_sb_list, &sb->s_inodes);
         spin_unlock(&sb_inode_list_lock);
-        spin_lock(&wb_inode_list_lock);
-        list_add(&inode->i_list, &inode_in_use);
-        spin_unlock(&wb_inode_list_lock);
         if (b) {
                 spin_lock(&b->lock);
                 hlist_add_head(&inode->i_hash, &b->head);
@@ -1311,9 +1298,11 @@ void generic_delete_inode(struct inode *inode)
 {
         const struct super_operations *op = inode->i_sb->s_op;
 
-        spin_lock(&wb_inode_list_lock);
-        list_del_init(&inode->i_list);
-        spin_unlock(&wb_inode_list_lock);
+        if (!list_empty(&inode->i_list)) {
+                spin_lock(&wb_inode_list_lock);
+                list_del_init(&inode->i_list);
+                spin_unlock(&wb_inode_list_lock);
+        }
         list_del_init(&inode->i_sb_list);
         spin_unlock(&sb_inode_list_lock);
         WARN_ON(inode->i_state & I_NEW);
@@ -1365,12 +1354,12 @@ int generic_detach_inode(struct inode *inode)
         struct super_block *sb = inode->i_sb;
 
         if (!hlist_unhashed(&inode->i_hash)) {
-                if (!(inode->i_state & (I_DIRTY|I_SYNC))) {
+                if (list_empty(&inode->i_list)) {
                         spin_lock(&wb_inode_list_lock);
-                        list_move(&inode->i_list, &inode_unused);
+                        list_add(&inode->i_list, &inode_unused);
                         spin_unlock(&wb_inode_list_lock);
+                        atomic_inc(&inodes_stat.nr_unused);
                 }
-                atomic_inc(&inodes_stat.nr_unused);
                 if (sb->s_flags & MS_ACTIVE) {
                         spin_unlock(&inode->i_lock);
                         spin_unlock(&sb_inode_list_lock);
@@ -1386,11 +1375,13 @@ int generic_detach_inode(struct inode *inode)
                 WARN_ON(inode->i_state & I_NEW);
                 inode->i_state &= ~I_WILL_FREE;
                 __remove_inode_hash(inode);
+        }
+        if (!list_empty(&inode->i_list)) {
+                spin_lock(&wb_inode_list_lock);
+                list_del_init(&inode->i_list);
+                spin_unlock(&wb_inode_list_lock);
                 atomic_dec(&inodes_stat.nr_unused);
         }
-        spin_lock(&wb_inode_list_lock);
-        list_del_init(&inode->i_list);
-        spin_unlock(&wb_inode_list_lock);
         list_del_init(&inode->i_sb_list);
         spin_unlock(&sb_inode_list_lock);
         WARN_ON(inode->i_state & I_NEW);
@@ -1726,7 +1717,7 @@ void __init inode_init(void)
 
         inode_hashtable =
                 alloc_large_system_hash("Inode-cache",
-                                        sizeof(struct hlist_head),
+                                        sizeof(struct inode_hash_bucket),
                                         ihash_entries,
                                         14,
                                         0,
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 3551528a25f6..b86542c8d68c 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -2174,7 +2174,6 @@ extern int insert_inode_locked4(struct inode *, unsigned long, int (*test)(struc
 extern int insert_inode_locked(struct inode *);
 extern void unlock_new_inode(struct inode *);
 
-extern void __iget(struct inode * inode);
 extern void iget_failed(struct inode *);
 extern void clear_inode(struct inode *);
 extern void destroy_inode(struct inode *);
@@ -2393,6 +2392,12 @@ extern int generic_show_options(struct seq_file *m, struct vfsmount *mnt);
 extern void save_mount_options(struct super_block *sb, char *options);
 extern void replace_mount_options(struct super_block *sb, char *options);
 
+static inline void __iget(struct inode *inode)
+{
+        assert_spin_locked(&inode->i_lock);
+        inode->i_count++;
+}
+
 static inline ino_t parent_ino(struct dentry *dentry)
 {
         ino_t res;
diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index 05479e576a77..15e8bcd90cd1 100644
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -11,7 +11,6 @@ struct backing_dev_info;
 
 extern spinlock_t sb_inode_list_lock;
 extern spinlock_t wb_inode_list_lock;
-extern struct list_head inode_in_use;
 extern struct list_head inode_unused;
 
 /*