fs-dcache-scale-d_subdirs

Protect d_subdirs and d_child with d_lock, except in filesystems that aren't
using dcache_lock for these anyway (eg. using i_mutex).

XXX: probably don't need parent lock in inotify (because child lock
should stabilize parent). Also, possibly some filesystems don't need so
much locking (eg. of child dentry when modifying d_child, so long as
parent is locked)... but be on the safe side. Hmm, maybe we should just
say d_child list is protected by d_parent->d_lock. d_parent could remain
protected with d_lock.

Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: John Stultz <johnstul@us.ibm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

3 files changed, 78 insertions, 28 deletions

diff --git a/fs/autofs4/expire.c b/fs/autofs4/expire.c
index 47483ca14017..915f6a93bc48 100644
--- a/fs/autofs4/expire.c
+++ b/fs/autofs4/expire.c
@@ -93,22 +93,59 @@ done:
 /*
  * Calculate next entry in top down tree traversal.
  * From next_mnt in namespace.c - elegant.
+ *
+ * How is this supposed to work if we drop dcache_lock between calls anyway?
+ * How does it cope with renames?
+ * And also callers dput the returned dentry before taking dcache_lock again
+ * so what prevents it from being freed??
  */
-static struct dentry *next_dentry(struct dentry *p, struct dentry *root)
+static struct dentry *get_next_positive_dentry(struct dentry *p,
+                                                struct dentry *root)
 {
-        struct list_head *next = p->d_subdirs.next;
+        struct list_head *next;
+        struct dentry *ret;
 
+        spin_lock(&dcache_lock);
+again:
+        spin_lock(&p->d_lock);
+        next = p->d_subdirs.next;
         if (next == &p->d_subdirs) {
                 while (1) {
-                        if (p == root)
+                        struct dentry *parent;
+
+                        if (p == root) {
+                                spin_unlock(&p->d_lock);
+                                spin_unlock(&dcache_lock);
                                 return NULL;
+                        }
+
+                        parent = p->d_parent;
+                        if (!spin_trylock(&parent->d_lock)) {
+                                spin_unlock(&p->d_lock);
+                                goto again;
+                        }
+                        spin_unlock(&p->d_lock);
                         next = p->d_u.d_child.next;
-                        if (next != &p->d_parent->d_subdirs)
+                        p = parent;
+                        if (next != &parent->d_subdirs)
                                 break;
-                        p = p->d_parent;
                 }
         }
-        return list_entry(next, struct dentry, d_u.d_child);
+        ret = list_entry(next, struct dentry, d_u.d_child);
+
+        spin_lock_nested(&ret->d_lock, DENTRY_D_LOCK_NESTED);
+        /* Negative dentry - try next */
+        if (!simple_positive(ret)) {
+                spin_unlock(&ret->d_lock);
+                p = ret;
+                goto again;
+        }
+        dget_dlock(ret);
+        spin_unlock(&ret->d_lock);
+        spin_unlock(&p->d_lock);
+        spin_unlock(&dcache_lock);
+
+        return ret;
 }
 
 /*
@@ -158,18 +195,11 @@ static int autofs4_tree_busy(struct vfsmount *mnt,
         if (!simple_positive(top))
                 return 1;
 
-        spin_lock(&dcache_lock);
-        for (p = top; p; p = next_dentry(p, top)) {
-                /* Negative dentry - give up */
-                if (!simple_positive(p))
-                        continue;
+        for (p = dget(top); p; p = get_next_positive_dentry(p, top)) {
 
                 DPRINTK("dentry %p %.*s",
                         p, (int) p->d_name.len, p->d_name.name);
 
-                p = dget(p);
-                spin_unlock(&dcache_lock);
-
                 /*
                  * Is someone visiting anywhere in the subtree ?
                  * If there's no mount we need to check the usage
@@ -205,9 +235,7 @@ static int autofs4_tree_busy(struct vfsmount *mnt,
                         }
                 }
                 dput(p);
-                spin_lock(&dcache_lock);
         }
-        spin_unlock(&dcache_lock);
 
         /* Timeout of a tree mount is ultimately determined by its top dentry */
         if (!autofs4_can_expire(top, timeout, do_now))
@@ -226,18 +254,11 @@ static struct dentry *autofs4_check_leaves(struct vfsmount *mnt,
         DPRINTK("parent %p %.*s",
                 parent, (int)parent->d_name.len, parent->d_name.name);
 
-        spin_lock(&dcache_lock);
-        for (p = parent; p; p = next_dentry(p, parent)) {
-                /* Negative dentry - give up */
-                if (!simple_positive(p))
-                        continue;
+        for (p = dget(parent); p; p = get_next_positive_dentry(p, parent)) {
 
                 DPRINTK("dentry %p %.*s",
                         p, (int) p->d_name.len, p->d_name.name);
 
-                p = dget(p);
-                spin_unlock(&dcache_lock);
-
                 if (d_mountpoint(p)) {
                         /* Can we umount this guy */
                         if (autofs4_mount_busy(mnt, p))
@@ -249,9 +270,7 @@ static struct dentry *autofs4_check_leaves(struct vfsmount *mnt,
                 }
 cont:
                 dput(p);
-                spin_lock(&dcache_lock);
         }
-        spin_unlock(&dcache_lock);
         return NULL;
 }
 
@@ -295,6 +314,8 @@ struct dentry *autofs4_expire_direct(struct super_block *sb,
  * A tree is eligible if :-
  *  - it is unused by any user process
  *  - it has been unused for exp_timeout time
+ * This seems to be racy dropping dcache_lock and asking for next->next after
+ * the lock has been dropped.
  */
 struct dentry *autofs4_expire_indirect(struct super_block *sb,
                                        struct vfsmount *mnt,
@@ -317,6 +338,7 @@ struct dentry *autofs4_expire_indirect(struct super_block *sb,
         timeout = sbi->exp_timeout;
 
         spin_lock(&dcache_lock);
+        spin_lock(&root->d_lock);
         next = root->d_subdirs.next;
 
         /* On exit from the loop expire is set to a dgot dentry
@@ -330,7 +352,10 @@ struct dentry *autofs4_expire_indirect(struct super_block *sb,
                         continue;
                 }
 
-                dentry = dget(dentry);
+                spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
+                dentry = dget_dlock(dentry);
+                spin_unlock(&dentry->d_lock);
+                spin_unlock(&root->d_lock);
                 spin_unlock(&dcache_lock);
 
                 spin_lock(&sbi->fs_lock);
@@ -397,8 +422,10 @@ next:
                 spin_unlock(&sbi->fs_lock);
                 dput(dentry);
                 spin_lock(&dcache_lock);
+                spin_lock(&root->d_lock);
                 next = next->next;
         }
+        spin_unlock(&root->d_lock);
         spin_unlock(&dcache_lock);
         return NULL;
 
@@ -411,7 +438,11 @@ found:
         init_completion(&ino->expire_complete);
         spin_unlock(&sbi->fs_lock);
         spin_lock(&dcache_lock);
+        spin_lock(&expired->d_parent->d_lock);
+        spin_lock_nested(&expired->d_lock, DENTRY_D_LOCK_NESTED);
         list_move(&expired->d_parent->d_subdirs, &expired->d_u.d_child);
+        spin_unlock(&expired->d_lock);
+        spin_unlock(&expired->d_parent->d_lock);
         spin_unlock(&dcache_lock);
         return expired;
 }
diff --git a/fs/autofs4/inode.c b/fs/autofs4/inode.c
index d0a3de247458..cf6192583b73 100644
--- a/fs/autofs4/inode.c
+++ b/fs/autofs4/inode.c
@@ -113,6 +113,7 @@ static void autofs4_force_release(struct autofs_sb_info *sbi)
 
         spin_lock(&dcache_lock);
 repeat:
+        spin_lock(&this_parent->d_lock);
         next = this_parent->d_subdirs.next;
 resume:
         while (next != &this_parent->d_subdirs) {
@@ -125,11 +126,13 @@ resume:
                 }
 
                 if (!list_empty(&dentry->d_subdirs)) {
+                        spin_unlock(&this_parent->d_lock);
                         this_parent = dentry;
                         goto repeat;
                 }
 
                 next = next->next;
+                spin_unlock(&this_parent->d_lock);
                 spin_unlock(&dcache_lock);
 
                 DPRINTK("dentry %p %.*s",
@@ -137,20 +140,24 @@ resume:
 
                 dput(dentry);
                 spin_lock(&dcache_lock);
+                spin_lock(&this_parent->d_lock);
         }
 
         if (this_parent != sbi->sb->s_root) {
                 struct dentry *dentry = this_parent;
 
                 next = this_parent->d_u.d_child.next;
+                spin_unlock(&this_parent->d_lock);
                 this_parent = this_parent->d_parent;
                 spin_unlock(&dcache_lock);
                 DPRINTK("parent dentry %p %.*s",
                         dentry, (int)dentry->d_name.len, dentry->d_name.name);
                 dput(dentry);
                 spin_lock(&dcache_lock);
+                spin_lock(&this_parent->d_lock);
                 goto resume;
         }
+        spin_unlock(&this_parent->d_lock);
         spin_unlock(&dcache_lock);
 }
 
diff --git a/fs/autofs4/root.c b/fs/autofs4/root.c
index 5d5c6d4600bb..9fc3409f9dd9 100644
--- a/fs/autofs4/root.c
+++ b/fs/autofs4/root.c
@@ -226,10 +226,13 @@ static int autofs4_dir_open(struct inode *inode, struct file *file)
          * it.
          */
         spin_lock(&dcache_lock);
+        spin_lock(&dentry->d_lock);
         if (!d_mountpoint(dentry) && list_empty(&dentry->d_subdirs)) {
+                spin_unlock(&dentry->d_lock);
                 spin_unlock(&dcache_lock);
                 return -ENOENT;
         }
+        spin_unlock(&dentry->d_lock);
         spin_unlock(&dcache_lock);
 
 out:
@@ -311,9 +314,12 @@ static void *autofs4_follow_link(struct dentry *dentry, struct nameidata *nd)
          * multi-mount with no root mount offset. So don't try to
          * mount it again.
          */
+        spin_lock(&dcache_lock);
+        spin_lock(&dentry->d_lock);
         if (ino->flags & AUTOFS_INF_PENDING ||
             (!d_mountpoint(dentry) && list_empty(&dentry->d_subdirs))) {
                 ino->flags |= AUTOFS_INF_PENDING;
+                spin_unlock(&dentry->d_lock);
                 spin_unlock(&dcache_lock);
                 spin_unlock(&sbi->fs_lock);
 
@@ -328,6 +334,7 @@ static void *autofs4_follow_link(struct dentry *dentry, struct nameidata *nd)
 
                 goto follow;
         }
+        spin_unlock(&dentry->d_lock);
         spin_unlock(&dcache_lock);
         spin_unlock(&sbi->fs_lock);
 follow:
@@ -931,11 +938,16 @@ static int autofs4_dir_rmdir(struct inode *dir, struct dentry *dentry)
                 return -EACCES;
 
         spin_lock(&dcache_lock);
+        spin_lock(&dentry->d_lock);
         if (!list_empty(&dentry->d_subdirs)) {
+                spin_unlock(&dentry->d_lock);
                 spin_unlock(&dcache_lock);
                 return -ENOTEMPTY;
         }
-        spin_lock(&dentry->d_lock);
+        spin_lock(&sbi->lookup_lock);
+        if (list_empty(&ino->expiring))
+                list_add(&ino->expiring, &sbi->expiring_list);
+        spin_unlock(&sbi->lookup_lock);
         __d_drop(dentry);
         spin_unlock(&dentry->d_lock);
         spin_unlock(&dcache_lock);