4 files changed, 89 insertions, 53 deletions

diff --git a/fs/dcache.c b/fs/dcache.c
index 07d1f6862dc7..9f04e1ba75b7 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -33,14 +33,18 @@
 #include <linux/bootmem.h>
 #include <linux/fs_struct.h>
 #include <linux/hardirq.h>
+#include <linux/bit_spinlock.h>
+#include <linux/rculist_bl.h>
 #include "internal.h"
 
 /*
  * Usage:
  * dcache_inode_lock protects:
  *   - i_dentry, d_alias, d_inode
- * dcache_hash_lock protects:
- *   - the dcache hash table, s_anon lists
+ * dcache_hash_bucket lock protects:
+ *   - the dcache hash table
+ * s_anon bl list spinlock protects:
+ *   - the s_anon list (see __d_drop)
  * dcache_lru_lock protects:
  *   - the dcache lru lists and counters
  * d_lock protects:
@@ -57,7 +61,8 @@
  * dcache_inode_lock
  *   dentry->d_lock
  *     dcache_lru_lock
- *     dcache_hash_lock
+ *     dcache_hash_bucket lock
+ *     s_anon lock
  *
  * If there is an ancestor relationship:
  * dentry->d_parent->...->d_parent->d_lock
@@ -74,7 +79,6 @@ int sysctl_vfs_cache_pressure __read_mostly = 100;
 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
 
 __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_inode_lock);
-static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_hash_lock);
 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock);
 __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
 
@@ -96,7 +100,29 @@ static struct kmem_cache *dentry_cache __read_mostly;
 
 static unsigned int d_hash_mask __read_mostly;
 static unsigned int d_hash_shift __read_mostly;
-static struct hlist_head *dentry_hashtable __read_mostly;
+
+struct dcache_hash_bucket {
+        struct hlist_bl_head head;
+};
+static struct dcache_hash_bucket *dentry_hashtable __read_mostly;
+
+static inline struct dcache_hash_bucket *d_hash(struct dentry *parent,
+                                        unsigned long hash)
+{
+        hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES;
+        hash = hash ^ ((hash ^ GOLDEN_RATIO_PRIME) >> D_HASHBITS);
+        return dentry_hashtable + (hash & D_HASHMASK);
+}
+
+static inline void spin_lock_bucket(struct dcache_hash_bucket *b)
+{
+        bit_spin_lock(0, (unsigned long *)&b->head.first);
+}
+
+static inline void spin_unlock_bucket(struct dcache_hash_bucket *b)
+{
+        __bit_spin_unlock(0, (unsigned long *)&b->head.first);
+}
 
 /* Statistics gathering. */
 struct dentry_stat_t dentry_stat = {
@@ -144,7 +170,7 @@ static void d_free(struct dentry *dentry)
                 dentry->d_op->d_release(dentry);
 
         /* if dentry was never inserted into hash, immediate free is OK */
-        if (hlist_unhashed(&dentry->d_hash))
+        if (hlist_bl_unhashed(&dentry->d_hash))
                 __d_free(&dentry->d_u.d_rcu);
         else
                 call_rcu(&dentry->d_u.d_rcu, __d_free);
@@ -302,11 +328,27 @@ static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent)
 void __d_drop(struct dentry *dentry)
 {
         if (!(dentry->d_flags & DCACHE_UNHASHED)) {
-                dentry->d_flags |= DCACHE_UNHASHED;
-                spin_lock(&dcache_hash_lock);
-                hlist_del_rcu(&dentry->d_hash);
-                spin_unlock(&dcache_hash_lock);
-                dentry_rcuwalk_barrier(dentry);
+                if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED)) {
+                        bit_spin_lock(0,
+                                (unsigned long *)&dentry->d_sb->s_anon.first);
+                        dentry->d_flags |= DCACHE_UNHASHED;
+                        hlist_bl_del_init(&dentry->d_hash);
+                        __bit_spin_unlock(0,
+                                (unsigned long *)&dentry->d_sb->s_anon.first);
+                } else {
+                        struct dcache_hash_bucket *b;
+                        b = d_hash(dentry->d_parent, dentry->d_name.hash);
+                        spin_lock_bucket(b);
+                        /*
+                         * We may not actually need to put DCACHE_UNHASHED
+                         * manipulations under the hash lock, but follow
+                         * the principle of least surprise.
+                         */
+                        dentry->d_flags |= DCACHE_UNHASHED;
+                        hlist_bl_del_rcu(&dentry->d_hash);
+                        spin_unlock_bucket(b);
+                        dentry_rcuwalk_barrier(dentry);
+                }
         }
 }
 EXPORT_SYMBOL(__d_drop);
@@ -961,8 +1003,8 @@ void shrink_dcache_for_umount(struct super_block *sb)
         spin_unlock(&dentry->d_lock);
         shrink_dcache_for_umount_subtree(dentry);
 
-        while (!hlist_empty(&sb->s_anon)) {
-                dentry = hlist_entry(sb->s_anon.first, struct dentry, d_hash);
+        while (!hlist_bl_empty(&sb->s_anon)) {
+                dentry = hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash);
                 shrink_dcache_for_umount_subtree(dentry);
         }
 }
@@ -1263,7 +1305,7 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
         dentry->d_sb = NULL;
         dentry->d_op = NULL;
         dentry->d_fsdata = NULL;
-        INIT_HLIST_NODE(&dentry->d_hash);
+        INIT_HLIST_BL_NODE(&dentry->d_hash);
         INIT_LIST_HEAD(&dentry->d_lru);
         INIT_LIST_HEAD(&dentry->d_subdirs);
         INIT_LIST_HEAD(&dentry->d_alias);
@@ -1459,14 +1501,6 @@ struct dentry * d_alloc_root(struct inode * root_inode)
 }
 EXPORT_SYMBOL(d_alloc_root);
 
-static inline struct hlist_head *d_hash(struct dentry *parent,
-                                        unsigned long hash)
-{
-        hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES;
-        hash = hash ^ ((hash ^ GOLDEN_RATIO_PRIME) >> D_HASHBITS);
-        return dentry_hashtable + (hash & D_HASHMASK);
-}
-
 /**
  * d_obtain_alias - find or allocate a dentry for a given inode
  * @inode: inode to allocate the dentry for
@@ -1521,11 +1555,11 @@ struct dentry *d_obtain_alias(struct inode *inode)
         tmp->d_sb = inode->i_sb;
         tmp->d_inode = inode;
         tmp->d_flags |= DCACHE_DISCONNECTED;
-        tmp->d_flags &= ~DCACHE_UNHASHED;
         list_add(&tmp->d_alias, &inode->i_dentry);
-        spin_lock(&dcache_hash_lock);
-        hlist_add_head(&tmp->d_hash, &inode->i_sb->s_anon);
-        spin_unlock(&dcache_hash_lock);
+        bit_spin_lock(0, (unsigned long *)&tmp->d_sb->s_anon.first);
+        tmp->d_flags &= ~DCACHE_UNHASHED;
+        hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon);
+        __bit_spin_unlock(0, (unsigned long *)&tmp->d_sb->s_anon.first);
         spin_unlock(&tmp->d_lock);
         spin_unlock(&dcache_inode_lock);
 
@@ -1567,7 +1601,7 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
                         d_move(new, dentry);
                         iput(inode);
                 } else {
-                        /* already taking dcache_inode_lock, so d_add() by hand */
+                        /* already got dcache_inode_lock, so d_add() by hand */
                         __d_instantiate(dentry, inode);
                         spin_unlock(&dcache_inode_lock);
                         security_d_instantiate(dentry, inode);
@@ -1702,8 +1736,8 @@ struct dentry *__d_lookup_rcu(struct dentry *parent, struct qstr *name,
         unsigned int len = name->len;
         unsigned int hash = name->hash;