fs: scale inode alias list

Add a new lock, dcache_inode_lock, to protect the inode's i_dentry list
from concurrent modification. d_alias is also protected by d_lock.

Signed-off-by: Nick Piggin <npiggin@kernel.dk>

1 files changed, 58 insertions, 8 deletions

diff --git a/fs/dcache.c b/fs/dcache.c
index a661247a20d5..de38680ee0ed 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -37,6 +37,8 @@
 
 /*
  * Usage:
+ * dcache_inode_lock protects:
+ *   - i_dentry, d_alias, d_inode
  * dcache_hash_lock protects:
  *   - the dcache hash table, s_anon lists
  * dcache_lru_lock protects:
@@ -49,12 +51,14 @@
  *   - d_unhashed()
  *   - d_parent and d_subdirs
  *   - childrens' d_child and d_parent
+ *   - d_alias, d_inode
  *
  * Ordering:
  * dcache_lock
- *   dentry->d_lock
- *     dcache_lru_lock
- *     dcache_hash_lock
+ *   dcache_inode_lock
+ *     dentry->d_lock
+ *       dcache_lru_lock
+ *       dcache_hash_lock
  *
  * If there is an ancestor relationship:
  * dentry->d_parent->...->d_parent->d_lock
@@ -70,11 +74,13 @@
 int sysctl_vfs_cache_pressure __read_mostly = 100;
 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
 
+__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_inode_lock);
 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_hash_lock);
 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock);
 __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock);
 __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
 
+EXPORT_SYMBOL(dcache_inode_lock);
 EXPORT_SYMBOL(dcache_lock);
 
 static struct kmem_cache *dentry_cache __read_mostly;
@@ -154,6 +160,7 @@ static void d_free(struct dentry *dentry)
  */
 static void dentry_iput(struct dentry * dentry)
         __releases(dentry->d_lock)
+        __releases(dcache_inode_lock)
         __releases(dcache_lock)
 {
         struct inode *inode = dentry->d_inode;
@@ -161,6 +168,7 @@ static void dentry_iput(struct dentry * dentry)
                 dentry->d_inode = NULL;
                 list_del_init(&dentry->d_alias);
                 spin_unlock(&dentry->d_lock);
+                spin_unlock(&dcache_inode_lock);
                 spin_unlock(&dcache_lock);
                 if (!inode->i_nlink)
                         fsnotify_inoderemove(inode);
@@ -170,6 +178,7 @@ static void dentry_iput(struct dentry * dentry)
                         iput(inode);
         } else {
                 spin_unlock(&dentry->d_lock);
+                spin_unlock(&dcache_inode_lock);
                 spin_unlock(&dcache_lock);
         }
 }
@@ -231,6 +240,7 @@ static void dentry_lru_move_tail(struct dentry *dentry)
 static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent)
         __releases(dentry->d_lock)
         __releases(parent->d_lock)
+        __releases(dcache_inode_lock)
         __releases(dcache_lock)
 {
         list_del(&dentry->d_u.d_child);
@@ -332,13 +342,18 @@ repeat:
                          * want to reduce dcache_lock anyway so this will
                          * get improved.
                          */
+drop1:
                         spin_unlock(&dentry->d_lock);
                         goto repeat;
                 }
-                if (parent && !spin_trylock(&parent->d_lock)) {
-                        spin_unlock(&dentry->d_lock);
+                if (!spin_trylock(&dcache_inode_lock)) {
+drop2:
                         spin_unlock(&dcache_lock);
-                        goto repeat;
+                        goto drop1;
+                }
+                if (parent && !spin_trylock(&parent->d_lock)) {
+                        spin_unlock(&dcache_inode_lock);
+                        goto drop2;
                 }
         }
         dentry->d_count--;
@@ -369,6 +384,7 @@ repeat:
         spin_unlock(&dentry->d_lock);
         if (parent)
                 spin_unlock(&parent->d_lock);
+        spin_unlock(&dcache_inode_lock);
         spin_unlock(&dcache_lock);
         return;
 
@@ -558,7 +574,9 @@ struct dentry *d_find_alias(struct inode *inode)
 
         if (!list_empty(&inode->i_dentry)) {
                 spin_lock(&dcache_lock);
+                spin_lock(&dcache_inode_lock);
                 de = __d_find_alias(inode, 0);
+                spin_unlock(&dcache_inode_lock);
                 spin_unlock(&dcache_lock);
         }
         return de;
@@ -574,18 +592,21 @@ void d_prune_aliases(struct inode *inode)
         struct dentry *dentry;
 restart:
         spin_lock(&dcache_lock);
+        spin_lock(&dcache_inode_lock);
         list_for_each_entry(dentry, &inode->i_dentry, d_alias) {
                 spin_lock(&dentry->d_lock);
                 if (!dentry->d_count) {
                         __dget_locked_dlock(dentry);
                         __d_drop(dentry);
                         spin_unlock(&dentry->d_lock);
+                        spin_unlock(&dcache_inode_lock);
                         spin_unlock(&dcache_lock);
                         dput(dentry);
                         goto restart;
                 }
                 spin_unlock(&dentry->d_lock);
         }
+        spin_unlock(&dcache_inode_lock);
         spin_unlock(&dcache_lock);
 }
 EXPORT_SYMBOL(d_prune_aliases);
@@ -601,6 +622,7 @@ EXPORT_SYMBOL(d_prune_aliases);
 static void prune_one_dentry(struct dentry *dentry, struct dentry *parent)
         __releases(dentry->d_lock)
         __releases(parent->d_lock)
+        __releases(dcache_inode_lock)
         __releases(dcache_lock)
 {
         __d_drop(dentry);
@@ -612,6 +634,7 @@ static void prune_one_dentry(struct dentry *dentry, struct dentry *parent)
          */
         while (dentry) {
                 spin_lock(&dcache_lock);
+                spin_lock(&dcache_inode_lock);
 again:
                 spin_lock(&dentry->d_lock);
                 if (IS_ROOT(dentry))
@@ -627,6 +650,7 @@ again:
                         if (parent)
                                 spin_unlock(&parent->d_lock);
                         spin_unlock(&dentry->d_lock);
+                        spin_unlock(&dcache_inode_lock);
                         spin_unlock(&dcache_lock);
                         return;
                 }
@@ -676,8 +700,9 @@ relock:
                 spin_unlock(&dcache_lru_lock);
 
                 prune_one_dentry(dentry, parent);
-                /* dcache_lock and dentry->d_lock dropped */
+                /* dcache_lock, dcache_inode_lock and dentry->d_lock dropped */
                 spin_lock(&dcache_lock);
+                spin_lock(&dcache_inode_lock);
                 spin_lock(&dcache_lru_lock);
         }
 }
@@ -699,6 +724,7 @@ static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags)
         int cnt = *count;
 
         spin_lock(&dcache_lock);
+        spin_lock(&dcache_inode_lock);
 relock:
         spin_lock(&dcache_lru_lock);
         while (!list_empty(&sb->s_dentry_lru)) {
@@ -737,8 +763,8 @@ relock:
         if (!list_empty(&referenced))
                 list_splice(&referenced, &sb->s_dentry_lru);
         spin_unlock(&dcache_lru_lock);
+        spin_unlock(&dcache_inode_lock);
         spin_unlock(&dcache_lock);
-
 }
 
 /**
@@ -832,12 +858,14 @@ void shrink_dcache_sb(struct super_block *sb)
         LIST_HEAD(tmp);
 
         spin_lock(&dcache_lock);
+        spin_lock(&dcache_inode_lock);
         spin_lock(&dcache_lru_lock);
         while (!list_empty(&sb->s_dentry_lru)) {
                 list_splice_init(&sb->s_dentry_lru, &tmp);
                 shrink_dentry_list(&tmp);
         }
         spin_unlock(&dcache_lru_lock);
+        spin_unlock(&dcache_inode_lock);
         spin_unlock(&dcache_lock);
 }
 EXPORT_SYMBOL(shrink_dcache_sb);
@@ -1255,9 +1283,11 @@ EXPORT_SYMBOL(d_alloc_name);
 /* the caller must hold dcache_lock */
 static void __d_instantiate(struct dentry *dentry, struct inode *inode)
 {
+        spin_lock(&dentry->d_lock);
         if (inode)
                 list_add(&dentry->d_alias, &inode->i_dentry);
         dentry->d_inode = inode;
+        spin_unlock(&dentry->d_lock);
         fsnotify_d_instantiate(dentry, inode);
 }
 
@@ -1280,7 +1310,9 @@ void d_instantiate(struct dentry *entry, struct inode * inode)
 {
         BUG_ON(!list_empty(&entry->d_alias));
         spin_lock(&dcache_lock);
+        spin_lock(&dcache_inode_lock);
         __d_instantiate(entry, inode);
+        spin_unlock(&dcache_inode_lock);
         spin_unlock(&dcache_lock);
         security_d_instantiate(entry, inode);
 }
@@ -1341,7 +1373,9 @@ struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode)
         BUG_ON(!list_empty(&entry->d_alias));
 
         spin_lock(&dcache_lock);
+        spin_lock(&dcache_inode_lock);
         result = __d_instantiate_unique(entry, inode);
+        spin_unlock(&dcache_inode_lock);
         spin_unlock(&dcache_lock);
 
         if (!result) {
@@ -1432,8 +1466,10 @@ struct dentry *d_obtain_alias(struct inode *inode)
         tmp->d_parent = tmp; /* make sure dput doesn't croak */
 
         spin_lock(&dcache_lock);
+        spin_lock(&dcache_inode_lock);
         res = __d_find_alias(inode, 0);
         if (res) {
+                spin_unlock(&dcache_inode_lock);
                 spin_unlock(&dcache_lock);
                 dput(tmp);
                 goto out_iput;
@@ -1450,6 +1486,7 @@ struct dentry *d_obtain_alias(struct inode *inode)
         hlist_add_head(&tmp->d_hash, &inode->i_sb->s_anon);
         spin_unlock(&dcache_hash_lock);
         spin_unlock(&tmp->d_lock);
+        spin_unlock(&dcache_inode_lock);
 
         spin_unlock(&dcache_lock);
         return tmp;
@@ -1482,9 +1519,11 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
 
         if (inode && S_ISDIR(inode->i_mode)) {
                 spin_lock(&dcache_lock);
+                spin_lock(&dcache_inode_lock);
                 new = __d_find_alias(inode, 1);
                 if (new) {
                         BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED));
+                        spin_unlock(&dcache_inode_lock);
                         spin_unlock(&dcache_lock);
                         security_d_instantiate(new, inode);
                         d_move(new, dentry);
@@ -1492,6 +1531,7 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
                 } else {
                         /* already taking dcache_lock, so d_add() by hand */
                         __d_instantiate(dentry, inode);
+                        spin_unlock(&dcache_inode_lock);
                         spin_unlock(&dcache_lock);
                         security_d_instantiate(dentry, inode);
                         d_rehash(dentry);
@@ -1566,8 +1606,10 @@ struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
          * already has a dentry.
          */
         spin_lock(&dcache_lock);
+        spin_lock(&dcache_inode_lock);
         if (!S_ISDIR(inode->i_mode) || list_empty(&inode->i_dentry)) {
                 __d_instantiate(found, inode);
+                spin_unlock(&dcache_inode_lock);
                 spin_unlock(&dcache_lock);
                 security_d_instantiate(found, inode);
                 return found;
@@ -1579,6 +1621,7 @@ struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
          */
         new = list_entry(inode->i_dentry.next, struct dentry, d_alias);
         dget_locked(new);
+        spin_unlock(&dcache_inode_lock);
         spin_unlock(&dcache_lock);
         security_d_instantiate(found, inode);
         d_move(new, found);
@@ -1797,6 +1840,7 @@ void d_delete(struct dentry * dentry)
          * Are we the only user?
          */
         spin_lock(&dcache_lock);
+        spin_lock(&dcache_inode_lock);
         spin_lock(&dentry->d_lock);
         isdir = S_ISDIR(dentry->d_inode->i_mode);
         if (dentry->d_count == 1) {
@@ -1810,6 +1854,7 @@ void d_delete(struct dentry * dentry)
                 __d_drop(dentry);
 
         spin_unlock(&dentry->d_lock);
+        spin_unlock(&dcache_inode_lock);
         spin_unlock(&dcache_lock);
 
         fsnotify_nameremove(dentry, isdir);
@@ -2067,6 +2112,7 @@ struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
  */
 static struct dentry *__d_unalias(struct dentry *dentry, struct dentry *alias)
         __releases(dcache_lock)
+        __releases(dcache_inode_lock)
 {
         struct mutex *m1 = NULL, *m2 = NULL;
         struct dentry *ret;
@@ -2092,6 +2138,7 @@ out_unalias:
         d_move_locked(alias, dentry);
         ret = alias;
 out_err:
+        spin_unlock(&dcache_inode_lock);
         spin_unlock(&dcache_lock);
         if (m2)
                 mutex_unlock(m2);
@@ -2153,6 +2200,7 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
         BUG_ON(!d_unhashed(dentry));
 
         spin_lock(&dcache_lock);
+        spin_lock(&dcache_inode_lock);
 
         if (!inode) {
                 actual = dentry;
@@ -2196,6 +2244,7 @@ found:
         _d_rehash(actual);
         spin_unlock(&dcache_hash_lock);
         spin_unlock(&actual->d_lock);
+        spin_unlock(&dcache_inode_lock);
         spin_unlock(&dcache_lock);
 out_nolock:
         if (actual == dentry) {
@@ -2207,6 +2256,7 @@ out_nolock:
         return actual;
 
 shouldnt_be_hashed:
+        spin_unlock(&dcache_inode_lock);
         spin_unlock(&dcache_lock);
         BUG();
 }