1 files changed, 470 insertions, 292 deletions

diff --git a/fs/dcache.c b/fs/dcache.c
index 83cfb834db03..99d4d7226203 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -88,6 +88,35 @@ EXPORT_SYMBOL(rename_lock);
 
 static struct kmem_cache *dentry_cache __read_mostly;
 
+/**
+ * read_seqbegin_or_lock - begin a sequence number check or locking block
+ * @lock: sequence lock
+ * @seq : sequence number to be checked
+ *
+ * First try it once optimistically without taking the lock. If that fails,
+ * take the lock. The sequence number is also used as a marker for deciding
+ * whether to be a reader (even) or writer (odd).
+ * N.B. seq must be initialized to an even number to begin with.
+ */
+static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq)
+{
+        if (!(*seq & 1))        /* Even */
+                *seq = read_seqbegin(lock);
+        else                    /* Odd */
+                read_seqlock_excl(lock);
+}
+
+static inline int need_seqretry(seqlock_t *lock, int seq)
+{
+        return !(seq & 1) && read_seqretry(lock, seq);
+}
+
+static inline void done_seqretry(seqlock_t *lock, int seq)
+{
+        if (seq & 1)
+                read_sequnlock_excl(lock);
+}
+
 /*
  * This is the single most critical data structure when it comes
  * to the dcache: the hashtable for lookups. Somebody should try
@@ -229,7 +258,7 @@ static void __d_free(struct rcu_head *head)
  */
 static void d_free(struct dentry *dentry)
 {
-        BUG_ON(dentry->d_count);
+        BUG_ON((int)dentry->d_lockref.count > 0);
         this_cpu_dec(nr_dentry);
         if (dentry->d_op && dentry->d_op->d_release)
                 dentry->d_op->d_release(dentry);
@@ -308,8 +337,9 @@ static void dentry_unlink_inode(struct dentry * dentry)
  */
 static void dentry_lru_add(struct dentry *dentry)
 {
-        if (list_empty(&dentry->d_lru)) {
+        if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST))) {
                 spin_lock(&dcache_lru_lock);
+                dentry->d_flags |= DCACHE_LRU_LIST;
                 list_add(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
                 dentry->d_sb->s_nr_dentry_unused++;
                 dentry_stat.nr_unused++;
@@ -320,7 +350,7 @@ static void dentry_lru_add(struct dentry *dentry)
 static void __dentry_lru_del(struct dentry *dentry)
 {
         list_del_init(&dentry->d_lru);
-        dentry->d_flags &= ~DCACHE_SHRINK_LIST;
+        dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
         dentry->d_sb->s_nr_dentry_unused--;
         dentry_stat.nr_unused--;
 }
@@ -341,6 +371,7 @@ static void dentry_lru_move_list(struct dentry *dentry, struct list_head *list)
 {
         spin_lock(&dcache_lru_lock);
         if (list_empty(&dentry->d_lru)) {
+                dentry->d_flags |= DCACHE_LRU_LIST;
                 list_add_tail(&dentry->d_lru, list);
                 dentry->d_sb->s_nr_dentry_unused++;
                 dentry_stat.nr_unused++;
@@ -443,7 +474,7 @@ EXPORT_SYMBOL(d_drop);
  * If ref is non-zero, then decrement the refcount too.
  * Returns dentry requiring refcount drop, or NULL if we're done.
  */
-static inline struct dentry *dentry_kill(struct dentry *dentry, int ref)
+static inline struct dentry *dentry_kill(struct dentry *dentry)
         __releases(dentry->d_lock)
 {
         struct inode *inode;
@@ -466,13 +497,16 @@ relock:
                 goto relock;
         }
 
-        if (ref)
-                dentry->d_count--;
+        /*
+         * The dentry is now unrecoverably dead to the world.
+         */
+        lockref_mark_dead(&dentry->d_lockref);
+
         /*
          * inform the fs via d_prune that this dentry is about to be
          * unhashed and destroyed.
          */
-        if (dentry->d_flags & DCACHE_OP_PRUNE)
+        if ((dentry->d_flags & DCACHE_OP_PRUNE) && !d_unhashed(dentry))
                 dentry->d_op->d_prune(dentry);
 
         dentry_lru_del(dentry);
@@ -509,38 +543,31 @@ relock:
  */
 void dput(struct dentry *dentry)
 {
-        if (!dentry)
+        if (unlikely(!dentry))
                 return;
 
 repeat:
-        if (dentry->d_count == 1)
-                might_sleep();
-        spin_lock(&dentry->d_lock);
-        BUG_ON(!dentry->d_count);
-        if (dentry->d_count > 1) {
-                dentry->d_count--;
-                spin_unlock(&dentry->d_lock);
+        if (lockref_put_or_lock(&dentry->d_lockref))
                 return;
-        }
 
-        if (dentry->d_flags & DCACHE_OP_DELETE) {
+        /* Unreachable? Get rid of it */
+        if (unlikely(d_unhashed(dentry)))
+                goto kill_it;
+
+        if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
                 if (dentry->d_op->d_delete(dentry))
                         goto kill_it;
         }
 
-        /* Unreachable? Get rid of it */
-        if (d_unhashed(dentry))
-                goto kill_it;
-
         dentry->d_flags |= DCACHE_REFERENCED;
         dentry_lru_add(dentry);
 
-        dentry->d_count--;
+        dentry->d_lockref.count--;
         spin_unlock(&dentry->d_lock);
         return;
 
 kill_it:
-        dentry = dentry_kill(dentry, 1);
+        dentry = dentry_kill(dentry);
         if (dentry)
                 goto repeat;
 }
@@ -590,7 +617,7 @@ int d_invalidate(struct dentry * dentry)
          * We also need to leave mountpoints alone,
          * directory or not.
          */
-        if (dentry->d_count > 1 && dentry->d_inode) {
+        if (dentry->d_lockref.count > 1 && dentry->d_inode) {
                 if (S_ISDIR(dentry->d_inode->i_mode) || d_mountpoint(dentry)) {
                         spin_unlock(&dentry->d_lock);
                         return -EBUSY;
@@ -606,20 +633,33 @@ EXPORT_SYMBOL(d_invalidate);
 /* This must be called with d_lock held */
 static inline void __dget_dlock(struct dentry *dentry)
 {
-        dentry->d_count++;
+        dentry->d_lockref.count++;
 }
 
 static inline void __dget(struct dentry *dentry)
 {
-        spin_lock(&dentry->d_lock);
-        __dget_dlock(dentry);
-        spin_unlock(&dentry->d_lock);
+        lockref_get(&dentry->d_lockref);
 }
 
 struct dentry *dget_parent(struct dentry *dentry)
 {
+        int gotref;
         struct dentry *ret;
 
+        /*
+         * Do optimistic parent lookup without any
+         * locking.
+         */
+        rcu_read_lock();
+        ret = ACCESS_ONCE(dentry->d_parent);
+        gotref = lockref_get_not_zero(&ret->d_lockref);
+        rcu_read_unlock();
+        if (likely(gotref)) {
+                if (likely(ret == ACCESS_ONCE(dentry->d_parent)))
+                        return ret;
+                dput(ret);
+        }
+
 repeat:
         /*
          * Don't need rcu_dereference because we re-check it was correct under
@@ -634,8 +674,8 @@ repeat:
                 goto repeat;
         }
         rcu_read_unlock();
-        BUG_ON(!ret->d_count);
-        ret->d_count++;
+        BUG_ON(!ret->d_lockref.count);
+        ret->d_lockref.count++;
         spin_unlock(&ret->d_lock);
         return ret;
 }
@@ -718,7 +758,15 @@ restart:
         spin_lock(&inode->i_lock);
         hlist_for_each_entry(dentry, &inode->i_dentry, d_alias) {
                 spin_lock(&dentry->d_lock);
-                if (!dentry->d_count) {
+                if (!dentry->d_lockref.count) {
+                        /*
+                         * inform the fs via d_prune that this dentry
+                         * is about to be unhashed and destroyed.
+                         */
+                        if ((dentry->d_flags & DCACHE_OP_PRUNE) &&
+                            !d_unhashed(dentry))
+                                dentry->d_op->d_prune(dentry);
+
                         __dget_dlock(dentry);
                         __d_drop(dentry);
                         spin_unlock(&dentry->d_lock);
@@ -744,7 +792,7 @@ static void try_prune_one_dentry(struct dentry *dentry)
 {
         struct dentry *parent;
 
-        parent = dentry_kill(dentry, 0);
+        parent = dentry_kill(dentry);
         /*
          * If dentry_kill returns NULL, we have nothing more to do.
          * if it returns the same dentry, trylocks failed. In either
@@ -763,13 +811,9 @@ static void try_prune_one_dentry(struct dentry *dentry)
         /* Prune ancestors. */
         dentry = parent;
         while (dentry) {
-                spin_lock(&dentry->d_lock);
-                if (dentry->d_count > 1) {
-                        dentry->d_count--;
-                        spin_unlock(&dentry->d_lock);
+                if (lockref_put_or_lock(&dentry->d_lockref))
                         return;
-                }
-                dentry = dentry_kill(dentry, 1);
+                dentry = dentry_kill(dentry);
         }
 }
 
@@ -793,7 +837,7 @@ static void shrink_dentry_list(struct list_head *list)
                  * the LRU because of laziness during lookup.  Do not free
                  * it - just keep it off the LRU list.
                  */
-                if (dentry->d_count) {
+                if (dentry->d_lockref.count) {
                         dentry_lru_del(dentry);
                         spin_unlock(&dentry->d_lock);
                         continue;
@@ -907,13 +951,14 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
                          * inform the fs that this dentry is about to be
                          * unhashed and destroyed.
                          */
-                        if (dentry->d_flags & DCACHE_OP_PRUNE)
+                        if ((dentry->d_flags & DCACHE_OP_PRUNE) &&
+                            !d_unhashed(dentry))
                                 dentry->d_op->d_prune(dentry);
 
                         dentry_lru_del(dentry);
                         __d_shrink(dentry);
 
-                        if (dentry->d_count != 0) {
+                        if (dentry->d_lockref.count != 0) {
                                 printk(KERN_ERR
                                        "BUG: Dentry %p{i=%lx,n=%s}"
                                        " still in use (%d)"
@@ -922,7 +967,7 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
                                        dentry->d_inode ?
                                        dentry->d_inode->i_ino : 0UL,
                                        dentry->d_name.name,
-                                       dentry->d_count,
+                                       dentry->d_lockref.count,
                                        dentry->d_sb->s_type->name,
                                        dentry->d_sb->s_id);
                                 BUG();
@@ -933,7 +978,7 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
                                 list_del(&dentry->d_u.d_child);
                         } else {
                                 parent = dentry->d_parent;
-                                parent->d_count--;
+                                parent->d_lockref.count--;
                                 list_del(&dentry->d_u.d_child);
                         }
 
@@ -981,7 +1026,7 @@ void shrink_dcache_for_umount(struct super_block *sb)
 
         dentry = sb->s_root;
         sb->s_root = NULL;
-        dentry->d_count--;
+        dentry->d_lockref.count--;
         shrink_dcache_for_umount_subtree(dentry);
 
         while (!hlist_bl_empty(&sb->s_anon)) {
@@ -996,7 +1041,7 @@ void shrink_dcache_for_umount(struct super_block *sb)
  * the parenthood after dropping the lock and check
  * that the sequence number still matches.
  */
-static struct dentry *try_to_ascend(struct dentry *old, int locked, unsigned seq)
+static struct dentry *try_to_ascend(struct dentry *old, unsigned seq)
 {
         struct dentry *new = old->d_parent;
 
@@ -1010,7 +1055,7 @@ static struct dentry *try_to_ascend(struct dentry *old, int locked, unsigned seq
          */
         if (new != old->d_parent ||
                  (old->d_flags & DCACHE_DENTRY_KILLED) ||
-                 (!locked && read_seqretry(&rename_lock, seq))) {
+                 need_seqretry(&rename_lock, seq)) {
                 spin_unlock(&new->d_lock);
                 new = NULL;
         }
@@ -1018,34 +1063,55 @@ static struct dentry *try_to_ascend(struct dentry *old, int locked, unsigned seq
         return new;
 }
 
+/**
+ * enum d_walk_ret - action to talke during tree walk
+ * @D_WALK_CONTINUE:    contrinue walk
+ * @D_WALK_QUIT:        quit walk
+ * @D_WALK_NORETRY:     quit when retry is needed
+ * @D_WALK_SKIP:        skip this dentry and its children
+ */
+enum d_walk_ret {
+        D_WALK_CONTINUE,
+        D_WALK_QUIT,
+        D_WALK_NORETRY,
+        D_WALK_SKIP,
+};
 
-/*
- * Search for at least 1 mount point in the dentry's subdirs.
- * We descend to the next level whenever the d_subdirs
- * list is non-empty and continue searching.
- */
- 
 /**
- * have_submounts - check for mounts over a dentry
- * @parent: dentry to check.
+ * d_walk - walk the dentry tree
+ * @parent:     start of walk
+ * @data:       data passed to @enter() and @finish()
+ * @enter:      callback when first entering the dentry
+ * @finish:     callback when successfully finished the walk
  *
- * Return true if the parent or its subdirectories contain
- * a mount point
+ * The @enter() and @finish() callbacks are called with d_lock held.
  */
-int have_submounts(struct dentry *parent)
+static void d_walk(struct dentry *parent, void *data,
+                   enum d_walk_ret (*enter)(void *, struct dentry *),
+                   void (*finish)(void *))
 {
         struct dentry *this_parent;
         struct list_head *next;
-        unsigned seq;
-        int locked = 0;
+        unsigned seq = 0;
+        enum d_walk_ret ret;
+        bool retry = true;
 
-        seq = read_seqbegin(&rename_lock);
 again:
+        read_seqbegin_or_lock(&rename_lock, &seq);
         this_parent = parent;
-
-        if (d_mountpoint(parent))
-                goto positive;
         spin_lock(&this_parent->d_lock);
+
+        ret = enter(data, this_parent);
+        switch (ret) {
+        case D_WALK_CONTINUE:
+                break;
+        case D_WALK_QUIT:
+        case D_WALK_SKIP:
+                goto out_unlock;
+        case D_WALK_NORETRY:
+                retry = false;
+                break;
+        }
 repeat:
         next = this_parent->d_subdirs.next;
 resume:
@@ -1055,12 +1121,22 @@ resume:
                 next = tmp->next;
 
                 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
-                /* Have we found a mount point ? */
-                if (d_mountpoint(dentry)) {
+
+                ret = enter(data, dentry);
+                switch (ret) {
+                case D_WALK_CONTINUE:
+                        break;
+                case D_WALK_QUIT:
                         spin_unlock(&dentry->d_lock);
-                        spin_unlock(&this_parent->d_lock);
-                        goto positive;
+                        goto out_unlock;
+                case D_WALK_NORETRY:
+                        retry = false;
+                        break;
+                case D_WALK_SKIP:
+                        spin_unlock(&dentry->d_lock);
+                        continue;
                 }
+
                 if (!list_empty(&dentry->d_subdirs)) {
                         spin_unlock(&this_parent->d_lock);
                         spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
@@ -1075,35 +1151,99 @@ resume:
          */
         if (this_parent != parent) {
                 struct dentry *child = this_parent;
-                this_parent = try_to_ascend(this_parent, locked, seq);
+                this_parent = try_to_ascend(this_parent, seq);
                 if (!this_parent)
                         goto rename_retry;
                 next = child->d_u.d_child.next;
                 goto resume;
         }
-        spin_unlock(&this_parent->d_lock);
-        if (!locked && read_seqretry(&rename_lock, seq))
-                goto rename_retry;
-        if (locked)
-                write_sequnlock(&rename_lock);
-        return 0; /* No mount points found in tree */
-positive:
-        if (!locked && read_seqretry(&rename_lock, seq))
+        if (need_seqretry(&rename_lock, seq)) {
+                spin_unlock(&this_parent->d_lock);
                 goto rename_retry;
-        if (locked)
-                write_sequnlock(&rename_lock);
-        return 1;
+        }
+        if (finish)
+                finish(data);
+
+out_unlock:
+        spin_unlock(&this_parent->d_lock);
+        done_seqretry(&rename_lock, seq);
+        return;
 
 rename_retry:
-        if (locked)
-                goto again;
-        locked = 1;
-        write_seqlock(&rename_lock);
+        if (!retry)
+                return;
+        seq = 1;
         goto again;
 }
+
+/*
+ * Search for at least 1 mount point in the dentry's subdirs.
+ * We descend to the next level whenever the d_subdirs
+ * list is non-empty and continue searching.
+ */
+
+/**
+ * have_submounts - check for mounts over a dentry
+ * @parent: dentry to check.
+ *
+ * Return true if the parent or its subdirectories contain
+ * a mount point
+ */
+
+static enum d_walk_ret check_mount(void *data, struct dentry *dentry)
+{
+        int *ret = data;
+        if (d_mountpoint(dentry)) {
+                *ret = 1;
+                return D_WALK_QUIT;
+        }
+        return D_WALK_CONTINUE;
+}
+
+int have_submounts(struct dentry *parent)
+{
+        int ret = 0;
+
+        d_walk(parent, &ret, check_mount, NULL);
+
+        return ret;
+}
 EXPORT_SYMBOL(have_submounts);
 
 /*
+ * Called by mount code to set a mountpoint and check if the mountpoint is
+ * reachable (e.g. NFS can unhash a directory dentry and then the complete
+ * subtree can become unreachable).
+ *
+ * Only one of check_submounts_and_drop() and d_set_mounted() must succeed.  For
+ * this reason take rename_lock and d_lock on dentry and ancestors.
+ */
+int d_set_mounted(struct dentry *dentry)
+{
+        struct dentry *p;
+        int ret = -ENOENT;
+        write_seqlock(&rename_lock);
+        for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
+                /* Need exclusion wrt. check_submounts_and_drop() */
+                spin_lock(&p->d_lock);
+                if (unlikely(d_unhashed(p))) {
+                        spin_unlock(&p->d_lock);
+                        goto out;
+                }
+                spin_unlock(&p->d_lock);
+        }
+        spin_lock(&dentry->d_lock);
+        if (!d_unlinked(dentry)) {
+                dentry->d_flags |= DCACHE_MOUNTED;
+                ret = 0;
+        }
+        spin_unlock(&dentry->d_lock);
+out:
+        write_sequnlock(&rename_lock);
+        return ret;
+}
+
+/*
  * Search the dentry child list of the specified parent,
  * and move any unused dentries to the end of the unused
  * list for prune_dcache(). We descend to the next level
@@ -1117,93 +1257,46 @@ EXPORT_SYMBOL(have_submounts);
  * drop the lock and return early due to latency
  * constraints.
  */
-static int select_parent(struct dentry *parent, struct list_head *dispose)
-{
-        struct dentry *this_parent;
-        struct list_head *next;
-        unsigned seq;
-        int found = 0;
-        int locked = 0;
-
-        seq = read_seqbegin(&rename_lock);
-again:
-        this_parent = parent;
-        spin_lock(&this_parent->d_lock);
-repeat:
-        next = this_parent->d_subdirs.next;
-resume:
-        while (next != &this_parent->d_subdirs) {
-                struct list_head *tmp = next;
-                struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
-                next = tmp->next;
 
-                spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
+struct select_data {
+        struct dentry *start;
+        struct list_head dispose;
+        int found;
+};
 
-                /*
-                 * move only zero ref count dentries to the dispose list.
-                 *
-                 * Those which are presently on the shrink list, being processed
-                 * by shrink_dentry_list(), shouldn't be moved.  Otherwise the
-                 * loop in shrink_dcache_parent() might not make any progress
-                 * and loop forever.
-                 */
-                if (dentry->d_count) {
-                        dentry_lru_del(dentry);
-                } else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
-                        dentry_lru_move_list(dentry, dispose);
-                        dentry->d_flags |= DCACHE_SHRINK_LIST;
-                        found++;
-                }
-                /*
-                 * We can return to the caller if we have found some (this
-                 * ensures forward progress). We'll be coming back to find
-                 * the rest.
-                 */
-                if (found && need_resched()) {
-                        spin_unlock(&dentry->d_lock);
-                        goto out;
-                }
+static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
+{
+        struct select_data *data = _data;
+        enum d_walk_ret ret = D_WALK_CONTINUE;
 
-                /*
-                 * Descend a level if the d_subdirs list is non-empty.
-                 */
-                if (!list_empty(&dentry->d_subdirs)) {
-                        spin_unlock(&this_parent->d_lock);
-                        spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
-                        this_parent = dentry;
-                        spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
-                        goto repeat;
-                }
+        if (data->start == dentry)
+                goto out;
 
-                spin_unlock(&dentry->d_lock);
-        }
         /*
-         * All done at this level ... ascend and resume the search.
+         * move only zero ref count dentries to the dispose list.
+         *
+         * Those which are presently on the shrink list, being processed
+         * by shrink_dentry_list(), shouldn't be moved.  Otherwise the
+         * loop in shrink_dcache_parent() might not make any progress
+         * and loop forever.
          */
-        if (this_parent != parent) {
-                struct dentry *child = this_parent;
-                this_parent = try_to_ascend(this_parent, locked, seq);
-                if (!this_parent)
-                        goto rename_retry;
-                next = child->d_u.d_child.next;
-                goto resume;
+        if (dentry->d_lockref.count) {
+                dentry_lru_del(dentry);
+        } else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) {
+                dentry_lru_move_list(dentry, &data->dispose);
+                dentry->d_flags |= DCACHE_SHRINK_LIST;
+                data->found++;
+                ret = D_WALK_NORETRY;
         }
+        /*
+         * We can return to the caller if we have found some (this
+         * ensures forward progress). We'll be coming back to find
+         * the rest.
+         */
+        if (data->found && need_resched())
+                ret = D_WALK_QUIT;
 out:
-        spin_unlock(&this_parent->d_lock);
-        if (!locked && read_seqretry(&rename_lock, seq))
-                goto rename_retry;
-        if (locked)
-                write_sequnlock(&rename_lock);
-        return found;
-
-rename_retry:
-        if (found)
-                return found;
-        if (locked)
-                goto again;
-        locked = 1;
-        write_seqlock(&rename_lock);
-        goto again;
+        return ret;
 }
 
 /**
@@ -1212,18 +1305,90 @@ rename_retry:
  *
  * Prune the dcache to remove unused children of the parent dentry.
  */
-void shrink_dcache_parent(struct dentry * parent)
+void shrink_dcache_parent(struct dentry *parent)
 {
-        LIST_HEAD(dispose);
-        int found;
+        for (;;) {
+                struct select_data data;
+
+                INIT_LIST_HEAD(&data.dispose);
+                data.start = parent;
+                data.found = 0;
 
-        while ((found = select_parent(parent, &dispose)) != 0) {
-                shrink_dentry_list(&dispose);
+                d_walk(parent, &data, select_collect, NULL);
+                if (!data.found)
+                        break;
+
+                shrink_dentry_list(&data.dispose);
                 cond_resched();
         }
 }
 EXPORT_SYMBOL(shrink_dcache_parent);
 
+static enum d_walk_ret check_and_collect(void *_data, struct dentry *dentry)
+{
+        struct select_data *data = _data;
+
+        if (d_mountpoint(dentry)) {
+                data->found = -EBUSY;
+                return D_WALK_QUIT;
+        }
+
+        return select_collect(_data, dentry);
+}
+
+static void check_and_drop(void *_data)
+{
+        struct select_data *data = _data;
+
+        if (d_mountpoint(data->start))
+                data->found = -EBUSY;
+        if (!data->found)
+                __d_drop(data->start);
+}
+
+/**
+ * check_submounts_and_drop - prune dcache, check for submounts and drop
+ *
+ * All done as a single atomic operation relative to has_unlinked_ancestor().
+ * Returns 0 if successfully unhashed @parent.  If there were submounts then
+ * return -EBUSY.
+ *
+ * @dentry: dentry to prune and drop
+ */
+int check_submounts_and_drop(struct dentry *dentry)
+{
+        int ret = 0;
+
+        /* Negative dentries can be dropped without further checks */
+        if (!dentry->d_inode) {
+                d_drop(dentry);
+                goto out;
+        }
+
+        for (;;) {
+                struct select_data data;
+
+                INIT_LIST_HEAD(&data.dispose);
+                data.start = dentry;
+                data.found = 0;
+
+                d_walk(dentry, &data, check_and_collect, check_and_drop);
+                ret = data.found;
+
+                if (!list_empty(&data.dispose))
+                        shrink_dentry_list(&data.dispose);
+
+                if (ret <= 0)
+                        break;
+
+                cond_resched();
+        }
+
+out:
+        return ret;
+}
+EXPORT_SYMBOL(check_submounts_and_drop);
+
 /**
  * __d_alloc    -       allocate a dcache entry
  * @sb: filesystem it will belong to
@@ -1269,7 +1434,7 @@ struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
         smp_wmb();
         dentry->d_name.name = dname;
 
-        dentry->d_count = 1;
+        dentry->d_lockref.count = 1;
         dentry->d_flags = 0;
         spin_lock_init(&dentry->d_lock);
         seqcount_init(&dentry->d_seq);
@@ -1782,7 +1947,7 @@ static noinline enum slow_d_compare slow_dentry_cmp(
  * without taking d_lock and checking d_seq sequence count against @seq
  * returned here.
  *
- * A refcount may be taken on the found dentry with the __d_rcu_to_refcount
+ * A refcount may be taken on the found dentry with the d_rcu_to_refcount
  * function.
  *
  * Alternatively, __d_lookup_rcu may be called again to look up the child of
@@ -1970,7 +2135,7 @@ struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
                                 goto next;
                 }
 
-                dentry->d_count++;
+                dentry->d_lockref.count++;
                 found = dentry;
                 spin_unlock(&dentry->d_lock);
                 break;
@@ -2069,7 +2234,7 @@ again:
         spin_lock(&dentry->d_lock);
         inode = dentry->d_inode;
         isdir = S_ISDIR(inode->i_mode);
-        if (dentry->d_count == 1) {
+        if (dentry->d_lockref.count == 1) {
                 if (!spin_trylock(&inode->i_lock)) {
                         spin_unlock(&dentry->d_lock);
                         cpu_relax();
@@ -2506,9 +2671,39 @@ static int prepend(char **buffer, int *buflen, const char *str, int namelen)
         return 0;
 }
 
+/**
+ * prepend_name - prepend a pathname in front of current buffer pointer
+ * @buffer: buffer pointer
+ * @buflen: allocated length of the buffer
+ * @name:   name string and length qstr structure
+ *
+ * With RCU path tracing, it may race with d_move(). Use ACCESS_ONCE() to
+ * make sure that either the old or the new name pointer and length are
+ * fetched. However, there may be mismatch between length and pointer.
+ * The length cannot be trusted, we need to copy it byte-by-byte until
+ * the length is reached or a null byte is found. It also prepends "/" at
+ * the beginning of the name. The sequence number check at the caller will
+ * retry it again when a d_move() does happen. So any garbage in the buffer
+ * due to mismatched pointer and length will be discarded.
+ */
 static int prepend_name(char **buffer, int *buflen, struct qstr *name)
 {
-        return prepend(buffer, buflen, name->name, name->len);
+        const char *dname = ACCESS_ONCE(name->name);
+        u32 dlen = ACCESS_ONCE(name->len);
+        char *p;
+
+        if (*buflen < dlen + 1)
+                return -ENAMETOOLONG;
+        *buflen -= dlen + 1;
+        p = *buffer -= dlen + 1;
+        *p++ = '/';
+        while (dlen--) {
+                char c = *dname++;
+                if (!c)
+                        break;
+                *p++ = c;
+        }
+        return 0;
 }
 
 /**
@@ -2518,7 +2713,15 @@ static int prepend_name(char **buffer, int *buflen, struct qstr *name)
  * @buffer: pointer to the end of the buffer
  * @buflen: pointer to buffer length
  *
- * Caller holds the rename_lock.
+ * The function will first try to write out the pathname without taking any
+ * lock other than the RCU read lock to make sure that dentries won't go away.
+ * It only checks the sequence number of the global rename_lock as any change
+ * in the dentry's d_seq will be preceded by changes in the rename_lock
+ * sequence number. If the sequence number had been changed, it will restart
+ * the whole pathname back-tracing sequence again by taking the rename_lock.
+ * In this case, there is no need to take the RCU read lock as the recursive
+ * parent pointer references will keep the dentry chain alive as long as no
+ * rename operation is performed.
  */
 static int prepend_path(const struct path *path,
                         const struct path *root,
@@ -2527,54 +2730,66 @@ static int prepend_path(const struct path *path,
         struct dentry *dentry = path->dentry;
         struct vfsmount *vfsmnt = path->mnt;
         struct mount *mnt = real_mount(vfsmnt);
-        bool slash = false;
         int error = 0;
+        unsigned seq = 0;
+        char *bptr;
+        int blen;
 
+        rcu_read_lock();
+restart:
+        bptr = *buffer;
+        blen = *buflen;
+        read_seqbegin_or_lock(&rename_lock, &seq);
         while (dentry != root->dentry || vfsmnt != root->mnt) {
                 struct dentry * parent;
 
                 if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
                         /* Global root? */
-                        if (!mnt_has_parent(mnt))
-                                goto global_root;
-                        dentry = mnt->mnt_mountpoint;
-                        mnt = mnt->mnt_parent;
-                        vfsmnt = &mnt->mnt;
-                        continue;
+                        if (mnt_has_parent(mnt)) {
+                                dentry = mnt->mnt_mountpoint;
+                                mnt = mnt->mnt_parent;
+                                vfsmnt = &mnt->mnt;
+                                continue;
+                        }
+                        /*
+                         * Filesystems needing to implement special "root names"
+                         * should do so with ->d_dname()
+                         */
+                        if (IS_ROOT(dentry) &&
+                           (dentry->d_name.len != 1 ||
+                            dentry->d_name.name[0] != '/')) {
+                                WARN(1, "Root dentry has weird name <%.*s>\n",
+                                     (int) dentry->d_name.len,
+                                     dentry->d_name.name);
+                        }
+                        if (!error)
+                                error = is_mounted(vfsmnt) ? 1 : 2;
+                        break;
                 }
                 parent = dentry->d_parent;
                 prefetch(parent);
-                spin_lock(&dentry->d_lock);
-                error = prepend_name(buffer, buflen, &dentry->d_name);
-                spin_unlock(&dentry->d_lock);
-                if (!error)
-                        error = prepend(buffer, buflen, "/", 1);
+                error = prepend_name(&bptr, &blen, &dentry->d_name);
                 if (error)
                         break;
 
-                slash = true;
                 dentry = parent;
         }
+        if (!(seq & 1))
+                rcu_read_unlock();
+        if (need_seqretry(&rename_lock, seq)) {
+                seq = 1;
+                goto restart;
+        }
+        done_seqretry(&rename_lock, seq);
 
-        if (!error && !slash)
-                error = prepend(buffer, buflen, "/", 1);
-
-        return error;
-
-global_root:
-        /*
-         * Filesystems needing to implement special "root names"
-         * should do so with ->d_dname()
-         */
-        if (IS_ROOT(dentry) &&
-            (dentry->d_name.len != 1 || dentry->d_name.name[0] != '/')) {
-                WARN(1, "Root dentry has weird name <%.*s>\n",
-                     (int) dentry->d_name.len, dentry->d_name.name);
-        }
-        if (!slash)
-                error = prepend(buffer, buflen, "/", 1);
-        if (!error)
-                error = is_mounted(vfsmnt) ? 1 : 2;
+        if (error >= 0 && bptr == *buffer) {
+                if (--blen < 0)
+                        error = -ENAMETOOLONG;
+                else
+                        *--bptr = '/';
+        }
+        *buffer = bptr;
+        *buflen = blen;
         return error;
 }
 
@@ -2603,9 +2818,7 @@ char *__d_path(const struct path *path,
 
         prepend(&res, &buflen, "\0", 1);
         br_read_lock(&vfsmount_lock);
-        write_seqlock(&rename_lock);
         error = prepend_path(path, root, &res, &buflen);
-        write_sequnlock(&rename_lock);
         br_read_unlock(&vfsmount_lock);
 
         if (error < 0)
@@ -2624,9 +2837,7 @@ char *d_absolute_path(const struct path *path,
 
         prepend(&res, &buflen, "\0", 1);
         br_read_lock(&vfsmount_lock);
-        write_seqlock(&rename_lock);
         error = prepend_path(path, &root, &res, &buflen);
-        write_sequnlock(&rename_lock);
         br_read_unlock(&vfsmount_lock);
 
         if (error > 1)
@@ -2692,9 +2903,7 @@ char *d_path(const struct path *path, char *buf, int buflen)
 
         get_fs_root(current->fs, &root);
         br_read_lock(&vfsmount_lock);
-        write_seqlock(&rename_lock);
         error = path_with_deleted(path, &root, &res, &buflen);
-        write_sequnlock(&rename_lock);
         br_read_unlock(&vfsmount_lock);
         if (error < 0)
                 res = ERR_PTR(error);
@@ -2729,10 +2938,10 @@ char *simple_dname(struct dentry *dentry, char *buffer, int buflen)
         char *end = buffer + buflen;
         /* these dentries are never renamed, so d_lock is not needed */
         if (prepend(&end, &buflen, " (deleted)", 11) ||
-            prepend_name(&end, &buflen, &dentry->d_name) ||
+            prepend(&end, &buflen, dentry->d_name.name, dentry->d_name.len) ||
             prepend(&end, &buflen, "/", 1))  
                 end = ERR_PTR(-ENAMETOOLONG);
-        return end;  
+        return end;
 }
 
 /*
@@ -2740,30 +2949,42 @@ char *simple_dname(struct dentry *dentry, char *buffer, int buflen)
  */
 static char *__dentry_path(struct dentry *dentry, char *buf, int buflen)
 {
-        char *end = buf + buflen;
-        char *retval;
+        char *end, *retval;
+        int len, seq = 0;
+        int error = 0;
 
-        prepend(&end, &buflen, "\0", 1);
+        rcu_read_lock();
+restart:
+        end = buf + buflen;
+        len = buflen;
+        prepend(&end, &len, "\0", 1);
         if (buflen < 1)
                 goto Elong;
         /* Get '/' right */
         retval = end-1;
         *retval = '/';
-
+        read_seqbegin_or_lock(&rename_lock, &seq);
         while (!IS_ROOT(dentry)) {
                 struct dentry *parent = dentry->d_parent;
                 int error;
 
                 prefetch(parent);
-                spin_lock(&dentry->d_lock);
-                error = prepend_name(&end, &buflen, &dentry->d_name);
-                spin_unlock(&dentry->d_lock);
-                if (error != 0 || prepend(&end, &buflen, "/", 1) != 0)
-                        goto Elong;
+                error = prepend_name(&end, &len, &dentry->d_name);
+                if (error)
+                        break;
 
                 retval = end;
                 dentry = parent;
         }
+        if (!(seq & 1))
+                rcu_read_unlock();
+        if (need_seqretry(&rename_lock, seq)) {
+                seq = 1;
+                goto restart;
+        }
+        done_seqretry(&rename_lock, seq);
+        if (error)
+                goto Elong;
         return retval;
 Elong:
         return ERR_PTR(-ENAMETOOLONG);
@@ -2771,13 +2992,7 @@ Elong:
 
 char *dentry_path_raw(struct dentry *dentry, char *buf, int buflen)
 {
-        char *retval;
-
-        write_seqlock(&rename_lock);
-        retval = __dentry_path(dentry, buf, buflen);
-        write_sequnlock(&rename_lock);
-
-        return retval;
+        return __dentry_path(dentry, buf, buflen);
 }
 EXPORT_SYMBOL(dentry_path_raw);
 
@@ -2786,7 +3001,6 @@ char *dentry_path(struct dentry *dentry, char *buf, int buflen)
         char *p = NULL;
         char *retval;
 
-        write_seqlock(&rename_lock);
         if (d_unlinked(dentry)) {
                 p = buf + buflen;
                 if (prepend(&p, &buflen, "//deleted", 10) != 0)
@@ -2794,7 +3008,6 @@ char *dentry_path(struct dentry *dentry, char *buf, int buflen)
                 buflen++;
         }
         retval = __dentry_path(dentry, buf, buflen);
-        write_sequnlock(&rename_lock);
         if (!IS_ERR(retval) && p)
                 *p = '/';       /* restore '/' overriden with '\0' */
         return retval;
@@ -2802,6 +3015,18 @@ Elong:
         return ERR_PTR(-ENAMETOOLONG);
 }
 
+static void get_fs_root_and_pwd_rcu(struct fs_struct *fs, struct path *root,
+                                    struct path *pwd)
+{
+        unsigned seq;
+
+        do {
+                seq = read_seqcount_begin(&fs->seq);
+                *root = fs->root;
+                *pwd = fs->pwd;
+        } while (read_seqcount_retry(&fs->seq, seq));
+}
+
 /*
  * NOTE! The user-level library version returns a
  * character pointer. The kernel system call just
@@ -2829,11 +3054,11 @@ SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
         if (!page)
                 return -ENOMEM;
 
-        get_fs_root_and_pwd(current->fs, &root, &pwd);
+        rcu_read_lock();
+        get_fs_root_and_pwd_rcu(current->fs, &root, &pwd);
 
         error = -ENOENT;
         br_read_lock(&vfsmount_lock);
-        write_seqlock(&rename_lock);
         if (!d_unlinked(pwd.dentry)) {
                 unsigned long len;
                 char *cwd = page + PAGE_SIZE;
@@ -2841,7 +3066,6 @@ SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
 
                 prepend(&cwd, &buflen, "\0", 1);
                 error = prepend_path(&pwd, &root, &cwd, &buflen);
-                write_sequnlock(&rename_lock);
                 br_read_unlock(&vfsmount_lock);
 
                 if (error < 0)
@@ -2862,13 +3086,11 @@ SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
                                 error = -EFAULT;
                 }
         } else {
-                write_sequnlock(&rename_lock);
                 br_read_unlock(&vfsmount_lock);
         }
 
 out:
-        path_put(&pwd);
-        path_put(&root);
+        rcu_read_unlock();
         free_page((unsigned long) page);
         return error;
 }
@@ -2915,68 +3137,24 @@ int is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
         return result;
 }
 
-void d_genocide(struct dentry *root)
+static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
 {
-        struct dentry *this_parent;
-        struct list_head *next;
-        unsigned seq;
-        int locked = 0;
-
-        seq = read_seqbegin(&rename_lock);
-again:
-        this_parent = root;
-        spin_lock(&this_parent->d_lock);
-repeat:
-        next = this_parent->d_subdirs.next;
-resume:
-        while (next != &this_parent->d_subdirs) {
-                struct list_head *tmp = next;
-                struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
-                next = tmp->next;
+        struct dentry *root = data;
+        if (dentry != root) {
+                if (d_unhashed(dentry) || !dentry->d_inode)
+                        return D_WALK_SKIP;
 
-                spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
-                if (d_unhashed(dentry) || !dentry->d_inode) {
-                        spin_unlock(&dentry->d_lock);
-                        continue;
-                }
-                if (!list_empty(&dentry->d_subdirs)) {
-                        spin_unlock(&this_parent->d_lock);
-                        spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
-                        this_parent = dentry;
-                        spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
-                        goto repeat;
-                }
                 if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
                         dentry->d_flags |= DCACHE_GENOCIDE;
-                        dentry->d_count--;
-                }
-                spin_unlock(&dentry->d_lock);
-        }
-        if (this_parent != root) {
-                struct dentry *child = this_parent;
-                if (!(this_parent->d_flags & DCACHE_GENOCIDE)) {
-                        this_parent->d_flags |= DCACHE_GENOCIDE;
-                        this_parent->d_count--;
+                        dentry->d_lockref.count--;
                 }
-                this_parent = try_to_ascend(this_parent, locked, seq);
-                if (!this_parent)
-                        goto rename_retry;
-                next = child->d_u.d_child.next;
-                goto resume;
         }
-        spin_unlock(&this_parent->d_lock);
-        if (!locked && read_seqretry(&rename_lock, seq))
-                goto rename_retry;
-        if (locked)
-                write_sequnlock(&rename_lock);
-        return;
+        return D_WALK_CONTINUE;
+}
 
-rename_retry:
-        if (locked)
-                goto again;
-        locked = 1;
-        write_seqlock(&rename_lock);
-        goto again;
+void d_genocide(struct dentry *parent)
+{
+        d_walk(parent, parent, d_genocide_kill, NULL);
 }
 
 void d_tmpfile(struct dentry *dentry, struct inode *inode)