fs-inode_rcu

RCU free the struct inode. This will allow:

- sb_inode_list_lock to be moved inside i_lock because sb list walkers who want
  to take i_lock no longer need to take sb_inode_list_lock to walk the list in
  the first place. This will simplify and optimize locking.
- eventually, completely write-free RCU path walking. The inode must be
  consulted for permissions when walking, so a write-free reference (ie.
  RCU is helpful).
- can potentially simplify things a bit in VM land. May not need to take the
  page lock to get back to the page->mapping.
- can remove some nested trylock loops in dcache code

todo: convert all filesystems

Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: John Stultz <johnstul@us.ibm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

13 files changed, 101 insertions, 13 deletions

diff --git a/fs/block_dev.c b/fs/block_dev.c
index bf503be689c3..37a7a1f12329 100644
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -435,13 +435,20 @@ static struct inode *bdev_alloc_inode(struct super_block *sb)
         return &ei->vfs_inode;
 }
 
-static void bdev_destroy_inode(struct inode *inode)
+static void bdev_i_callback(struct rcu_head *head)
 {
+        struct inode *inode = container_of(head, struct inode, i_rcu);
         struct bdev_inode *bdi = BDEV_I(inode);
 
+        INIT_LIST_HEAD(&inode->i_dentry);
         kmem_cache_free(bdev_cachep, bdi);
 }
 
+static void bdev_destroy_inode(struct inode *inode)
+{
+        call_rcu(&inode->i_rcu, bdev_i_callback);
+}
+
 static void init_once(void *foo)
 {
         struct bdev_inode *ei = (struct bdev_inode *) foo;
diff --git a/fs/ext2/super.c b/fs/ext2/super.c
index f9cb54a585ce..d5f19f954fb7 100644
--- a/fs/ext2/super.c
+++ b/fs/ext2/super.c
@@ -157,11 +157,18 @@ static struct inode *ext2_alloc_inode(struct super_block *sb)
         return &ei->vfs_inode;
 }
 
-static void ext2_destroy_inode(struct inode *inode)
+static void ext2_i_callback(struct rcu_head *head)
 {
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        INIT_LIST_HEAD(&inode->i_dentry);
         kmem_cache_free(ext2_inode_cachep, EXT2_I(inode));
 }
 
+static void ext2_destroy_inode(struct inode *inode)
+{
+        call_rcu(&inode->i_rcu, ext2_i_callback);
+}
+
 static void init_once(void *foo)
 {
         struct ext2_inode_info *ei = (struct ext2_inode_info *) foo;
diff --git a/fs/ext3/super.c b/fs/ext3/super.c
index afa2b569da10..29dae329d4f6 100644
--- a/fs/ext3/super.c
+++ b/fs/ext3/super.c
@@ -483,6 +483,13 @@ static struct inode *ext3_alloc_inode(struct super_block *sb)
         return &ei->vfs_inode;
 }
 
+static void ext3_i_callback(struct rcu_head *head)
+{
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        INIT_LIST_HEAD(&inode->i_dentry);
+        kmem_cache_free(ext3_inode_cachep, EXT3_I(inode));
+}
+
 static void ext3_destroy_inode(struct inode *inode)
 {
         if (!list_empty(&(EXT3_I(inode)->i_orphan))) {
@@ -493,7 +500,7 @@ static void ext3_destroy_inode(struct inode *inode)
                                 false);
                 dump_stack();
         }
-        kmem_cache_free(ext3_inode_cachep, EXT3_I(inode));
+        call_rcu(&inode->i_rcu, ext3_i_callback);
 }
 
 static void init_once(void *foo)
diff --git a/fs/fat/inode.c b/fs/fat/inode.c
index 5c5e1cd62414..5220e057664d 100644
--- a/fs/fat/inode.c
+++ b/fs/fat/inode.c
@@ -497,11 +497,18 @@ static struct inode *fat_alloc_inode(struct super_block *sb)
         return &ei->vfs_inode;
 }
 
-static void fat_destroy_inode(struct inode *inode)
+static void fat_i_callback(struct rcu_head *head)
 {
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        INIT_LIST_HEAD(&inode->i_dentry);
         kmem_cache_free(fat_inode_cachep, MSDOS_I(inode));
 }
 
+static void fat_destroy_inode(struct inode *inode)
+{
+        call_rcu(&inode->i_rcu, fat_i_callback);
+}
+
 static void init_once(void *foo)
 {
         struct msdos_inode_info *ei = (struct msdos_inode_info *)foo;
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 5b50a7a2225e..d1a78fc2a78e 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -665,11 +665,18 @@ static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
         return &p->vfs_inode;
 }
 
+static void hugetlbfs_i_callback(struct rcu_head *head)
+{
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        INIT_LIST_HEAD(&inode->i_dentry);
+        kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
+}
+
 static void hugetlbfs_destroy_inode(struct inode *inode)
 {
         hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
         mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
-        kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
+        call_rcu(&inode->i_rcu, hugetlbfs_i_callback);
 }
 
 static const struct address_space_operations hugetlbfs_aops = {
diff --git a/fs/inode.c b/fs/inode.c
index 5039ddf4f0e0..fd57c8371686 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -245,13 +245,20 @@ void __destroy_inode(struct inode *inode)
 }
 EXPORT_SYMBOL(__destroy_inode);
 
+static void i_callback(struct rcu_head *head)
+{
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        INIT_LIST_HEAD(&inode->i_dentry);
+        kmem_cache_free(inode_cachep, inode);
+}
+
 void destroy_inode(struct inode *inode)
 {
         __destroy_inode(inode);
         if (inode->i_sb->s_op->destroy_inode)
                 inode->i_sb->s_op->destroy_inode(inode);
         else
-                kmem_cache_free(inode_cachep, (inode));
+                call_rcu(&inode->i_rcu, i_callback);
 }
 
 /*
diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index 8e7fcc253e43..86eefbafb7b4 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -1395,11 +1395,18 @@ struct inode *nfs_alloc_inode(struct super_block *sb)
         return &nfsi->vfs_inode;
 }
 
-void nfs_destroy_inode(struct inode *inode)
+static void nfs_i_callback(struct rcu_head *head)
 {
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        INIT_LIST_HEAD(&inode->i_dentry);
         kmem_cache_free(nfs_inode_cachep, NFS_I(inode));
 }
 
+void nfs_destroy_inode(struct inode *inode)
+{
+        call_rcu(&inode->i_rcu, nfs_i_callback);
+}
+
 static inline void nfs4_init_once(struct nfs_inode *nfsi)
 {
 #ifdef CONFIG_NFS_V4
diff --git a/fs/proc/inode.c b/fs/proc/inode.c
index 445a02bcaab3..ee0a77b56810 100644
--- a/fs/proc/inode.c
+++ b/fs/proc/inode.c
@@ -65,11 +65,18 @@ static struct inode *proc_alloc_inode(struct super_block *sb)
         return inode;
 }
 
-static void proc_destroy_inode(struct inode *inode)
+static void proc_i_callback(struct rcu_head *head)
 {
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        INIT_LIST_HEAD(&inode->i_dentry);
         kmem_cache_free(proc_inode_cachep, PROC_I(inode));
 }
 
+static void proc_destroy_inode(struct inode *inode)
+{
+        call_rcu(&inode->i_rcu, proc_i_callback);
+}
+
 static void init_once(void *foo)
 {
         struct proc_inode *ei = (struct proc_inode *) foo;
diff --git a/include/linux/fs.h b/include/linux/fs.h
index b86542c8d68c..9c595e725ee1 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -726,7 +726,10 @@ struct inode {
         struct hlist_node       i_hash;
         struct list_head        i_list;         /* backing dev IO list */
         struct list_head        i_sb_list;
-        struct list_head        i_dentry;
+        union {
+                struct list_head        i_dentry;
+                struct rcu_head         i_rcu;
+        };
         unsigned long           i_ino;
         unsigned int            i_count;
         unsigned int            i_nlink;
diff --git a/ipc/mqueue.c b/ipc/mqueue.c
index d72c0fa15e0c..f3a43f7747ca 100644
--- a/ipc/mqueue.c
+++ b/ipc/mqueue.c
@@ -237,11 +237,18 @@ static struct inode *mqueue_alloc_inode(struct super_block *sb)
         return &ei->vfs_inode;
 }
 
-static void mqueue_destroy_inode(struct inode *inode)
+static void mqueue_i_callback(struct rcu_head *head)
 {
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        INIT_LIST_HEAD(&inode->i_dentry);
         kmem_cache_free(mqueue_inode_cachep, MQUEUE_I(inode));
 }
 
+static void mqueue_destroy_inode(struct inode *inode)
+{
+        call_rcu(&inode->i_rcu, mqueue_i_callback);
+}
+
 static void mqueue_delete_inode(struct inode *inode)
 {
         struct mqueue_inode_info *info;
diff --git a/mm/shmem.c b/mm/shmem.c
index c4e04a5da5af..d7905296c2d3 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -2397,13 +2397,20 @@ static struct inode *shmem_alloc_inode(struct super_block *sb)
         return &p->vfs_inode;
 }
 
+static void shmem_i_callback(struct rcu_head *head)
+{
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        INIT_LIST_HEAD(&inode->i_dentry);
+        kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
+}
+
 static void shmem_destroy_inode(struct inode *inode)
 {
         if ((inode->i_mode & S_IFMT) == S_IFREG) {
                 /* only struct inode is valid if it's an inline symlink */
                 mpol_free_shared_policy(&SHMEM_I(inode)->policy);
         }
-        kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
+        call_rcu(&inode->i_rcu, shmem_i_callback);
 }
 
 static void init_once(void *foo)
diff --git a/net/socket.c b/net/socket.c
index 367296ccafbe..371eaf092a31 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -263,12 +263,19 @@ static struct inode *sock_alloc_inode(struct super_block *sb)
         return &ei->vfs_inode;
 }
 
-static void sock_destroy_inode(struct inode *inode)
+static void sock_i_callback(struct rcu_head *head)
 {
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        INIT_LIST_HEAD(&inode->i_dentry);
         kmem_cache_free(sock_inode_cachep,
                         container_of(inode, struct socket_alloc, vfs_inode));
 }
 
+static void sock_destroy_inode(struct inode *inode)
+{
+        call_rcu(&inode->i_rcu, sock_i_callback);
+}
+
 static void init_once(void *foo)
 {
         struct socket_alloc *ei = (struct socket_alloc *)foo;
diff --git a/net/sunrpc/rpc_pipe.c b/net/sunrpc/rpc_pipe.c
index 27a23785a50d..6792d5634fa0 100644
--- a/net/sunrpc/rpc_pipe.c
+++ b/net/sunrpc/rpc_pipe.c
@@ -162,11 +162,19 @@ rpc_alloc_inode(struct super_block *sb)
 }
 
 static void
-rpc_destroy_inode(struct inode *inode)
+rpc_i_callback(struct rcu_head *head)
 {
+        struct inode *inode = container_of(head, struct inode, i_rcu);
+        INIT_LIST_HEAD(&inode->i_dentry);
         kmem_cache_free(rpc_inode_cachep, RPC_I(inode));
 }
 
+static void
+rpc_destroy_inode(struct inode *inode)
+{
+        call_rcu(&inode->i_rcu, rpc_i_callback);
+}
+
 static int
 rpc_pipe_open(struct inode *inode, struct file *filp)
 {