1 files changed, 312 insertions, 344 deletions

diff --git a/fs/inode.c b/fs/inode.c
index 0b3da4a77704..05a1f75ae791 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -26,6 +26,38 @@
 #include <linux/posix_acl.h>
 #include <linux/ima.h>
 #include <linux/cred.h>
+#include "internal.h"
+
+/*
+ * inode locking rules.
+ *
+ * inode->i_lock protects:
+ *   inode->i_state, inode->i_hash, __iget()
+ * inode_lru_lock protects:
+ *   inode_lru, inode->i_lru
+ * inode_sb_list_lock protects:
+ *   sb->s_inodes, inode->i_sb_list
+ * inode_wb_list_lock protects:
+ *   bdi->wb.b_{dirty,io,more_io}, inode->i_wb_list
+ * inode_hash_lock protects:
+ *   inode_hashtable, inode->i_hash
+ *
+ * Lock ordering:
+ *
+ * inode_sb_list_lock
+ *   inode->i_lock
+ *     inode_lru_lock
+ *
+ * inode_wb_list_lock
+ *   inode->i_lock
+ *
+ * inode_hash_lock
+ *   inode_sb_list_lock
+ *   inode->i_lock
+ *
+ * iunique_lock
+ *   inode_hash_lock
+ */
 
 /*
  * This is needed for the following functions:
@@ -60,6 +92,8 @@
 
 static unsigned int i_hash_mask __read_mostly;
 static unsigned int i_hash_shift __read_mostly;
+static struct hlist_head *inode_hashtable __read_mostly;
+static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock);
 
 /*
  * Each inode can be on two separate lists. One is
@@ -74,15 +108,10 @@ static unsigned int i_hash_shift __read_mostly;
  */
 
 static LIST_HEAD(inode_lru);
-static struct hlist_head *inode_hashtable __read_mostly;
+static DEFINE_SPINLOCK(inode_lru_lock);
 
-/*
- * A simple spinlock to protect the list manipulations.
- *
- * NOTE! You also have to own the lock if you change
- * the i_state of an inode while it is in use..
- */
-DEFINE_SPINLOCK(inode_lock);
+__cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_sb_list_lock);
+__cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_wb_list_lock);
 
 /*
  * iprune_sem provides exclusion between the icache shrinking and the
@@ -137,15 +166,6 @@ int proc_nr_inodes(ctl_table *table, int write,
 }
 #endif
 
-static void wake_up_inode(struct inode *inode)
-{
-        /*
-         * Prevent speculative execution through spin_unlock(&inode_lock);
-         */
-        smp_mb();
-        wake_up_bit(&inode->i_state, __I_NEW);
-}
-
 /**
  * inode_init_always - perform inode structure intialisation
  * @sb: superblock inode belongs to
@@ -336,7 +356,7 @@ static void init_once(void *foo)
 }
 
 /*
- * inode_lock must be held
+ * inode->i_lock must be held
  */
 void __iget(struct inode *inode)
 {
@@ -354,23 +374,22 @@ EXPORT_SYMBOL(ihold);
 
 static void inode_lru_list_add(struct inode *inode)
 {
+        spin_lock(&inode_lru_lock);
         if (list_empty(&inode->i_lru)) {
                 list_add(&inode->i_lru, &inode_lru);
                 inodes_stat.nr_unused++;
         }
+        spin_unlock(&inode_lru_lock);
 }
 
 static void inode_lru_list_del(struct inode *inode)
 {
+        spin_lock(&inode_lru_lock);
         if (!list_empty(&inode->i_lru)) {
                 list_del_init(&inode->i_lru);
                 inodes_stat.nr_unused--;
         }
-}
-
-static inline void __inode_sb_list_add(struct inode *inode)
-{
-        list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
+        spin_unlock(&inode_lru_lock);
 }
 
 /**
@@ -379,15 +398,17 @@ static inline void __inode_sb_list_add(struct inode *inode)
  */
 void inode_sb_list_add(struct inode *inode)
 {
-        spin_lock(&inode_lock);
-        __inode_sb_list_add(inode);
-        spin_unlock(&inode_lock);
+        spin_lock(&inode_sb_list_lock);
+        list_add(&inode->i_sb_list, &inode->i_sb->s_inodes);
+        spin_unlock(&inode_sb_list_lock);
 }
 EXPORT_SYMBOL_GPL(inode_sb_list_add);
 
-static inline void __inode_sb_list_del(struct inode *inode)
+static inline void inode_sb_list_del(struct inode *inode)
 {
+        spin_lock(&inode_sb_list_lock);
         list_del_init(&inode->i_sb_list);
+        spin_unlock(&inode_sb_list_lock);
 }
 
 static unsigned long hash(struct super_block *sb, unsigned long hashval)
@@ -412,24 +433,15 @@ void __insert_inode_hash(struct inode *inode, unsigned long hashval)
 {
         struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval);
 
-        spin_lock(&inode_lock);
+        spin_lock(&inode_hash_lock);
+        spin_lock(&inode->i_lock);
         hlist_add_head(&inode->i_hash, b);
-        spin_unlock(&inode_lock);
+        spin_unlock(&inode->i_lock);
+        spin_unlock(&inode_hash_lock);
 }
 EXPORT_SYMBOL(__insert_inode_hash);
 
 /**
- *      __remove_inode_hash - remove an inode from the hash
- *      @inode: inode to unhash
- *
- *      Remove an inode from the superblock.
- */
-static void __remove_inode_hash(struct inode *inode)
-{
-        hlist_del_init(&inode->i_hash);
-}
-
-/**
  *      remove_inode_hash - remove an inode from the hash
  *      @inode: inode to unhash
  *
@@ -437,9 +449,11 @@ static void __remove_inode_hash(struct inode *inode)
  */
 void remove_inode_hash(struct inode *inode)
 {
-        spin_lock(&inode_lock);
+        spin_lock(&inode_hash_lock);
+        spin_lock(&inode->i_lock);
         hlist_del_init(&inode->i_hash);
-        spin_unlock(&inode_lock);
+        spin_unlock(&inode->i_lock);
+        spin_unlock(&inode_hash_lock);
 }
 EXPORT_SYMBOL(remove_inode_hash);
 
@@ -456,10 +470,29 @@ void end_writeback(struct inode *inode)
 }
 EXPORT_SYMBOL(end_writeback);
 
+/*
+ * Free the inode passed in, removing it from the lists it is still connected
+ * to. We remove any pages still attached to the inode and wait for any IO that
+ * is still in progress before finally destroying the inode.
+ *
+ * An inode must already be marked I_FREEING so that we avoid the inode being
+ * moved back onto lists if we race with other code that manipulates the lists
+ * (e.g. writeback_single_inode). The caller is responsible for setting this.
+ *
+ * An inode must already be removed from the LRU list before being evicted from
+ * the cache. This should occur atomically with setting the I_FREEING state
+ * flag, so no inodes here should ever be on the LRU when being evicted.
+ */
 static void evict(struct inode *inode)
 {
         const struct super_operations *op = inode->i_sb->s_op;
 
+        BUG_ON(!(inode->i_state & I_FREEING));
+        BUG_ON(!list_empty(&inode->i_lru));
+
+        inode_wb_list_del(inode);
+        inode_sb_list_del(inode);
+
         if (op->evict_inode) {
                 op->evict_inode(inode);
         } else {
@@ -471,6 +504,15 @@ static void evict(struct inode *inode)
                 bd_forget(inode);
         if (S_ISCHR(inode->i_mode) && inode->i_cdev)
                 cd_forget(inode);
+
+        remove_inode_hash(inode);
+
+        spin_lock(&inode->i_lock);
+        wake_up_bit(&inode->i_state, __I_NEW);
+        BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
+        spin_unlock(&inode->i_lock);
+
+        destroy_inode(inode);
 }
 
 /*
@@ -489,14 +531,6 @@ static void dispose_list(struct list_head *head)
                 list_del_init(&inode->i_lru);
 
                 evict(inode);
-
-                spin_lock(&inode_lock);
-                __remove_inode_hash(inode);
-                __inode_sb_list_del(inode);
-                spin_unlock(&inode_lock);
-
-                wake_up_inode(inode);
-                destroy_inode(inode);
         }
 }
 
@@ -514,25 +548,23 @@ void evict_inodes(struct super_block *sb)
         struct inode *inode, *next;
         LIST_HEAD(dispose);
 
-        spin_lock(&inode_lock);
+        spin_lock(&inode_sb_list_lock);
         list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
                 if (atomic_read(&inode->i_count))
                         continue;
-                if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE))
+
+                spin_lock(&inode->i_lock);
+                if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
+                        spin_unlock(&inode->i_lock);
                         continue;
+                }
 
                 inode->i_state |= I_FREEING;
-
-                /*
-                 * Move the inode off the IO lists and LRU once I_FREEING is
-                 * set so that it won't get moved back on there if it is dirty.
-                 */
-                list_move(&inode->i_lru, &dispose);
-                list_del_init(&inode->i_wb_list);
-                if (!(inode->i_state & (I_DIRTY | I_SYNC)))
-                        inodes_stat.nr_unused--;
+                inode_lru_list_del(inode);
+                spin_unlock(&inode->i_lock);
+                list_add(&inode->i_lru, &dispose);
         }
-        spin_unlock(&inode_lock);
+        spin_unlock(&inode_sb_list_lock);
 
         dispose_list(&dispose);
 
@@ -561,31 +593,30 @@ int invalidate_inodes(struct super_block *sb, bool kill_dirty)
         struct inode *inode, *next;
         LIST_HEAD(dispose);
 
-        spin_lock(&inode_lock);
+        spin_lock(&inode_sb_list_lock);
         list_for_each_entry_safe(inode, next, &sb->s_inodes, i_sb_list) {
-                if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE))
+                spin_lock(&inode->i_lock);
+                if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
+                        spin_unlock(&inode->i_lock);
                         continue;
+                }
                 if (inode->i_state & I_DIRTY && !kill_dirty) {
+                        spin_unlock(&inode->i_lock);
                         busy = 1;
                         continue;
                 }
                 if (atomic_read(&inode->i_count)) {
+                        spin_unlock(&inode->i_lock);
                         busy = 1;
                         continue;
                 }
 
                 inode->i_state |= I_FREEING;
-
-                /*
-                 * Move the inode off the IO lists and LRU once I_FREEING is
-                 * set so that it won't get moved back on there if it is dirty.
-                 */
-                list_move(&inode->i_lru, &dispose);
-                list_del_init(&inode->i_wb_list);
-                if (!(inode->i_state & (I_DIRTY | I_SYNC)))
-                        inodes_stat.nr_unused--;
+                inode_lru_list_del(inode);
+                spin_unlock(&inode->i_lock);
+                list_add(&inode->i_lru, &dispose);
         }
-        spin_unlock(&inode_lock);
+        spin_unlock(&inode_sb_list_lock);
 
         dispose_list(&dispose);
 
@@ -607,7 +638,7 @@ static int can_unuse(struct inode *inode)
 
 /*
  * Scan `goal' inodes on the unused list for freeable ones. They are moved to a
- * temporary list and then are freed outside inode_lock by dispose_list().
+ * temporary list and then are freed outside inode_lru_lock by dispose_list().
  *
  * Any inodes which are pinned purely because of attached pagecache have their
  * pagecache removed.  If the inode has metadata buffers attached to
@@ -628,7 +659,7 @@ static void prune_icache(int nr_to_scan)
         unsigned long reap = 0;
 
         down_read(&iprune_sem);
-        spin_lock(&inode_lock);
+        spin_lock(&inode_lru_lock);
         for (nr_scanned = 0; nr_scanned < nr_to_scan; nr_scanned++) {
                 struct inode *inode;
 
@@ -638,53 +669,67 @@ static void prune_icache(int nr_to_scan)
                 inode = list_entry(inode_lru.prev, struct inode, i_lru);
 
                 /*
+                 * we are inverting the inode_lru_lock/inode->i_lock here,
+                 * so use a trylock. If we fail to get the lock, just move the
+                 * inode to the back of the list so we don't spin on it.
+                 */
+                if (!spin_trylock(&inode->i_lock)) {
+                        list_move(&inode->i_lru, &inode_lru);
+                        continue;
+                }
+
+                /*
                  * Referenced or dirty inodes are still in use. Give them
                  * another pass through the LRU as we canot reclaim them now.
                  */
                 if (atomic_read(&inode->i_count) ||
                     (inode->i_state & ~I_REFERENCED)) {
                         list_del_init(&inode->i_lru);
+                        spin_unlock(&inode->i_lock);
                         inodes_stat.nr_unused--;
                         continue;
                 }
 
                 /* recently referenced inodes get one more pass */
                 if (inode->i_state & I_REFERENCED) {
-                        list_move(&inode->i_lru, &inode_lru);
                         inode->i_state &= ~I_REFERENCED;
+                        list_move(&inode->i_lru, &inode_lru);
+                        spin_unlock(&inode->i_lock);
                         continue;
                 }
                 if (inode_has_buffers(inode) || inode->i_data.nrpages) {
                         __iget(inode);
-                        spin_unlock(&inode_lock);
+                        spin_unlock(&inode->i_lock);
+                        spin_unlock(&inode_lru_lock);
                         if (remove_inode_buffers(inode))
                                 reap += invalidate_mapping_pages(&inode->i_data,
                                                                 0, -1);
                         iput(inode);
-                        spin_lock(&inode_lock);
+                        spin_lock(&inode_lru_lock);
 
                         if (inode != list_entry(inode_lru.next,
                                                 struct inode, i_lru))
                                 continue;       /* wrong inode or list_empty */
-                        if (!can_unuse(inode))
+                        /* avoid lock inversions with trylock */
+                        if (!spin_trylock(&inode->i_lock))
+                                continue;
+                        if (!can_unuse(inode)) {
+                                spin_unlock(&inode->i_lock);
                                 continue;
+                        }
                 }
                 WARN_ON(inode->i_state & I_NEW);
                 inode->i_state |= I_FREEING;
+                spin_unlock(&inode->i_lock);
 
-                /*
-                 * Move the inode off the IO lists and LRU once I_FREEING is
-                 * set so that it won't get moved back on there if it is dirty.
-                 */
                 list_move(&inode->i_lru, &freeable);
-                list_del_init(&inode->i_wb_list);
                 inodes_stat.nr_unused--;
         }
         if (current_is_kswapd())
                 __count_vm_events(KSWAPD_INODESTEAL, reap);
         else
                 __count_vm_events(PGINODESTEAL, reap);
-        spin_unlock(&inode_lock);
+        spin_unlock(&inode_lru_lock);
 
         dispose_list(&freeable);
         up_read(&iprune_sem);
@@ -733,15 +778,21 @@ static struct inode *find_inode(struct super_block *sb,
 
 repeat:
         hlist_for_each_entry(inode, node, head, i_hash) {
-                if (inode->i_sb != sb)
+                spin_lock(&inode->i_lock);
+                if (inode->i_sb != sb) {
+                        spin_unlock(&inode->i_lock);
                         continue;
-                if (!test(inode, data))
+                }
+                if (!test(inode, data)) {
+                        spin_unlock(&inode->i_lock);
                         continue;
+                }
                 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
                         __wait_on_freeing_inode(inode);
                         goto repeat;
                 }
                 __iget(inode);
+                spin_unlock(&inode->i_lock);
                 return inode;
         }
         return NULL;
@@ -759,15 +810,21 @@ static struct inode *find_inode_fast(struct super_block *sb,
 
 repeat:
         hlist_for_each_entry(inode, node, head, i_hash) {
-                if (inode->i_ino != ino)
+                spin_lock(&inode->i_lock);
+                if (inode->i_ino != ino) {
+                        spin_unlock(&inode->i_lock);
                         continue;
-                if (inode->i_sb != sb)
+                }
+                if (inode->i_sb != sb) {
+                        spin_unlock(&inode->i_lock);
                         continue;
+                }
                 if (inode->i_state & (I_FREEING|I_WILL_FREE)) {
                         __wait_on_freeing_inode(inode);
                         goto repeat;
                 }
                 __iget(inode);
+                spin_unlock(&inode->i_lock);
                 return inode;
         }
         return NULL;
@@ -827,19 +884,26 @@ struct inode *new_inode(struct super_block *sb)
 {
         struct inode *inode;
 
-        spin_lock_prefetch(&inode_lock);
+        spin_lock_prefetch(&inode_sb_list_lock);
 
         inode = alloc_inode(sb);
         if (inode) {
-                spin_lock(&inode_lock);
-                __inode_sb_list_add(inode);
+                spin_lock(&inode->i_lock);
                 inode->i_state = 0;
-                spin_unlock(&inode_lock);
+                spin_unlock(&inode->i_lock);
+                inode_sb_list_add(inode);
         }
         return inode;
 }
 EXPORT_SYMBOL(new_inode);
 
+/**
+ * unlock_new_inode - clear the I_NEW state and wake up any waiters
+ * @inode:      new inode to unlock
+ *
+ * Called when the inode is fully initialised to clear the new state of the
+ * inode and wake up anyone waiting for the inode to finish initialisation.
+ */
 void unlock_new_inode(struct inode *inode)
 {
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
@@ -859,51 +923,67 @@ void unlock_new_inode(struct inode *inode)
                 }
         }
 #endif
-        /*
-         * This is special!  We do not need the spinlock when clearing I_NEW,
-         * because we're guaranteed that nobody else tries to do anything about
-         * the state of the inode when it is locked, as we just created it (so
-         * there can be no old holders that haven't tested I_NEW).
-         * However we must emit the memory barrier so that other CPUs reliably
-         * see the clearing of I_NEW after the other inode initialisation has
-         * completed.
-         */
-        smp_mb();
+        spin_lock(&inode->i_lock);
         WARN_ON(!(inode->i_state & I_NEW));
         inode->i_state &= ~I_NEW;
-        wake_up_inode(inode);
+        wake_up_bit(&inode->i_state, __I_NEW);
+        spin_unlock(&inode->i_lock);
 }
 EXPORT_SYMBOL(unlock_new_inode);
 
-/*
- * This is called without the inode lock held.. Be careful.
+/**
+ * iget5_locked - obtain an inode from a mounted file system
+ * @sb:         super block of file system
+ * @hashval:    hash value (usually inode number) to get
+ * @test:       callback used for comparisons between inodes
+ * @set:        callback used to initialize a new struct inode
+ * @data:       opaque data pointer to pass to @test and @set
+ *
+ * Search for the inode specified by @hashval and @data in the inode cache,
+ * and if present it is return it with an increased reference count. This is
+ * a generalized version of iget_locked() for file systems where the inode
+ * number is not sufficient for unique identification of an inode.
  *
- * We no longer cache the sb_flags in i_flags - see fs.h
- *      -- rmk@arm.uk.linux.org
+ * If the inode is not in cache, allocate a new inode and return it locked,
+ * hashed, and with the I_NEW flag set. The file system gets to fill it in
+ * before unlocking it via unlock_new_inode().
+ *
+ * Note both @test and @set are called with the inode_hash_lock held, so can't
+ * sleep.
  */
-static struct inode *get_new_inode(struct super_block *sb,
-                                struct hlist_head *head,
-                                int (*test)(struct inode *, void *),
-                                int (*set)(struct inode *, void *),
-                                void *data)
+struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
+                int (*test)(struct inode *, void *),
+                int (*set)(struct inode *, void *), void *data)
 {
+        struct hlist_head *head = inode_hashtable + hash(sb, hashval);
         struct inode *inode;
 
+        spin_lock(&inode_hash_lock);
+        inode = find_inode(sb, head, test, data);
+        spin_unlock(&inode_hash_lock);
+
+        if (inode) {
+                wait_on_inode(inode);
+                return inode;
+        }
+
         inode = alloc_inode(sb);
         if (inode) {
                 struct inode *old;
 
-                spin_lock(&inode_lock);
+                spin_lock(&inode_hash_lock);
                 /* We released the lock, so.. */
                 old = find_inode(sb, head, test, data);
                 if (!old) {
                         if (set(inode, data))
                                 goto set_failed;
 
-                        hlist_add_head(&inode->i_hash, head);
-                        __inode_sb_list_add(inode);
+                        spin_lock(&inode->i_lock);
                         inode->i_state = I_NEW;
-                        spin_unlock(&inode_lock);
+                        hlist_add_head(&inode->i_hash, head);
+                        spin_unlock(&inode->i_lock);
+                        inode_sb_list_add(inode);
+                        spin_unlock(&inode_hash_lock);
 
                         /* Return the locked inode with I_NEW set, the
                          * caller is responsible for filling in the contents
@@ -916,7 +996,7 @@ static struct inode *get_new_inode(struct super_block *sb,
                  * us. Use the old inode instead of the one we just
                  * allocated.
                  */
-                spin_unlock(&inode_lock);
+                spin_unlock(&inode_hash_lock);
                 destroy_inode(inode);
                 inode = old;
                 wait_on_inode(inode);
@@ -924,33 +1004,53 @@ static struct inode *get_new_inode(struct super_block *sb,
         return inode;
 
 set_failed:
-        spin_unlock(&inode_lock);
+        spin_unlock(&inode_hash_lock);
         destroy_inode(inode);
         return NULL;
 }
+EXPORT_SYMBOL(iget5_locked);
 
-/*
- * get_new_inode_fast is the fast path version of get_new_inode, see the
- * comment at iget_locked for details.
+/**
+ * iget_locked - obtain an inode from a mounted file system
+ * @sb:         super block of file system
+ * @ino:        inode number to get
+ *
+ * Search for the inode specified by @ino in the inode cache and if present
+ * return it with an increased reference count. This is for file systems
+ * where the inode number is sufficient for unique identification of an inode.
+ *
+ * If the inode is not in cache, allocate a new inode and return it locked,
+ * hashed, and with the I_NEW flag set.  The file system gets to fill it in
+ * before unlocking it via unlock_new_inode().
  */
-static struct inode *get_new_inode_fast(struct super_block *sb,
-                                struct hlist_head *head, unsigned long ino)
+struct inode *iget_locked(struct super_block *sb, unsigned long ino)
 {
+        struct hlist_head *head = inode_hashtable + hash(sb, ino);
         struct inode *inode;
 
+        spin_lock(&inode_hash_lock);
+        inode = find_inode_fast(sb, head, ino);
+        spin_unlock(&inode_hash_lock);
+        if (inode) {
+                wait_on_inode(inode);
+                return inode;
+        }
+
         inode = alloc_inode(sb);
         if (inode) {
                 struct inode *old;
 
-                spin_lock(&inode_lock);
+                spin_lock(&inode_hash_lock);
                 /* We released the lock, so.. */
                 old = find_inode_fast(sb, head, ino);
                 if (!old) {
                         inode->i_ino = ino;
-                        hlist_add_head(&inode->i_hash, head);
-                        __inode_sb_list_add(inode);
+                        spin_lock(&inode->i_lock);
                         inode->i_state = I_NEW;
-                        spin_unlock(&inode_lock);
+                        hlist_add_head(&inode->i_hash, head);
+                        spin_unlock(&inode->i_lock);
+                        inode_sb_list_add(inode);
+                        spin_unlock(&inode_hash_lock);
 
                         /* Return the locked inode with I_NEW set, the
                          * caller is responsible for filling in the contents
@@ -963,13 +1063,14 @@ static struct inode *get_new_inode_fast(struct super_block *sb,
                  * us. Use the old inode instead of the one we just
                  * allocated.
                  */
-                spin_unlock(&inode_lock);
+                spin_unlock(&inode_hash_lock);
                 destroy_inode(inode);
                 inode = old;
                 wait_on_inode(inode);
         }
         return inode;
 }
+EXPORT_SYMBOL(iget_locked);
 
 /*
  * search the inode cache for a matching inode number.
@@ -984,10 +1085,14 @@ static int test_inode_iunique(struct super_block *sb, unsigned long ino)
         struct hlist_node *node;
         struct inode *inode;
 
+        spin_lock(&inode_hash_lock);
         hlist_for_each_entry(inode, node, b, i_hash) {
-                if (inode->i_ino == ino && inode->i_sb == sb)
+                if (inode->i_ino == ino && inode->i_sb == sb) {
+                        spin_unlock(&inode_hash_lock);
                         return 0;
+                }
         }
+        spin_unlock(&inode_hash_lock);
 
         return 1;
 }
@@ -1017,7 +1122,6 @@ ino_t iunique(struct super_block *sb, ino_t max_reserved)
         static unsigned int counter;
         ino_t res;
 
-        spin_lock(&inode_lock);
         spin_lock(&iunique_lock);
         do {
                 if (counter <= max_reserved)
@@ -1025,7 +1129,6 @@ ino_t iunique(struct super_block *sb, ino_t max_reserved)
                 res = counter++;
         } while (!test_inode_iunique(sb, res));
         spin_unlock(&iunique_lock);
-        spin_unlock(&inode_lock);
 
         return res;
 }
@@ -1033,116 +1136,50 @@ EXPORT_SYMBOL(iunique);
 
 struct inode *igrab(struct inode *inode)
 {
-        spin_lock(&inode_lock);
-        if (!(inode->i_state & (I_FREEING|I_WILL_FREE)))
+        spin_lock(&inode->i_lock);
+        if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) {
                 __iget(inode);
-        else
+                spin_unlock(&inode->i_lock);
+        } else {
+                spin_unlock(&inode->i_lock);
                 /*
                  * Handle the case where s_op->clear_inode is not been
                  * called yet, and somebody is calling igrab
                  * while the inode is getting freed.
                  */
                 inode = NULL;
-        spin_unlock(&inode_lock);
+        }
         return inode;
 }
 EXPORT_SYMBOL(igrab);
 
 /**
- * ifind - internal function, you want ilookup5() or iget5().
- * @sb:         super block of file system to search
- * @head:       the head of the list to search
- * @test:       callback used for comparisons between inodes
- * @data:       opaque data pointer to pass to @test
- * @wait:       if true wait for the inode to be unlocked, if false do not
- *
- * ifind() searches for the inode specified by @data in the inode
- * cache. This is a generalized version of ifind_fast() for file systems where
- * the inode number is not sufficient for unique identification of an inode.
- *
- * If the inode is in the cache, the inode is returned with an incremented
- * reference count.
- *
- * Otherwise NULL is returned.
- *
- * Note, @test is called with the inode_lock held, so can't sleep.
- */
-static struct inode *ifind(struct super_block *sb,
-                struct hlist_head *head, int (*test)(struct inode *, void *),
-                void *data, const int wait)
-{
-        struct inode *inode;
-
-        spin_lock(&inode_lock);
-        inode = find_inode(sb, head, test, data);
-        if (inode) {
-                spin_unlock(&inode_lock);
-                if (likely(wait))
-                        wait_on_inode(inode);
-                return inode;
-        }
-        spin_unlock(&inode_lock);
-        return NULL;
-}
-
-/**
- * ifind_fast - internal function, you want ilookup() or iget().
- * @sb:         super block of file system to search
- * @head:       head of the list to search
- * @ino:        inode number to search for
- *
- * ifind_fast() searches for the inode @ino in the inode cache. This is for
- * file systems where the inode number is sufficient for unique identification
- * of an inode.
- *
- * If the inode is in the cache, the inode is returned with an incremented
- * reference count.
- *
- * Otherwise NULL is returned.
- */
-static struct inode *ifind_fast(struct super_block *sb,
-                struct hlist_head *head, unsigned long ino)
-{
-        struct inode *inode;
-
-        spin_lock(&inode_lock);
-        inode = find_inode_fast(sb, head, ino);
-        if (inode) {
-                spin_unlock(&inode_lock);
-                wait_on_inode(inode);
-                return inode;
-        }
-        spin_unlock(&inode_lock);
-        return NULL;
-}
-
-/**
  * ilookup5_nowait - search for an inode in the inode cache
  * @sb:         super block of file system to search
  * @hashval:    hash value (usually inode number) to search for
  * @test:       callback used for comparisons between inodes
  * @data:       opaque data pointer to pass to @test
  *
- * ilookup5() uses ifind() to search for the inode specified by @hashval and
- * @data in the inode cache. This is a generalized version of ilookup() for
- * file systems where the inode number is not sufficient for unique
- * identification of an inode.
- *
+ * Search for the inode specified by @hashval and @data in the inode cache.
  * If the inode is in the cache, the inode is returned with an incremented
- * reference count.  Note, the inode lock is not waited upon so you have to be
- * very careful what you do with the returned inode.  You probably should be
- * using ilookup5() instead.
+ * reference count.
  *
- * Otherwise NULL is returned.
+ * Note: I_NEW is not waited upon so you have to be very careful what you do
+ * with the returned inode.  You probably should be using ilookup5() instead.
  *
- * Note, @test is called with the inode_lock held, so can't sleep.
+ * Note: @test is called with the inode_hash_lock held, so can't sleep.
  */
 struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval,
                 int (*test)(struct inode *, void *), void *data)
 {
         struct hlist_head *head = inode_hashtable + hash(sb, hashval);
+        struct inode *inode;
+
+        spin_lock(&inode_hash_lock);
+        inode = find_inode(sb, head, test, data);
+        spin_unlock(&inode_hash_lock);
 
-        return ifind(sb, head, test, data, 0);
+        return inode;
 }
 EXPORT_SYMBOL(ilookup5_nowait);
 
@@ -1153,24 +1190,24 @@ EXPORT_SYMBOL(ilookup5_nowait);
  * @test:       callback used for comparisons between inodes
  * @data:       opaque data pointer to pass to @test
  *
- * ilookup5() uses ifind() to search for the inode specified by @hashval and
- * @data in the inode cache. This is a generalized version of ilookup() for
- * file systems where the inode number is not sufficient for unique
- * identification of an inode.
- *
- * If the inode is in the cache, the inode lock is waited upon and the inode is
+ * Search for the inode specified by @hashval and @data in the inode cache,
+ * and if the inode is in the cache, return the inode with an incremented
+ * reference count.  Waits on I_NEW before returning the inode.
  * returned with an incremented reference count.
  *
- * Otherwise NULL is returned.
+ * This is a generalized version of ilookup() for file systems where the
+ * inode number is not sufficient for unique identification of an inode.
  *
- * Note, @test is called with the inode_lock held, so can't sleep.
+ * Note: @test is called with the inode_hash_lock held, so can't sleep.
  */
 struct inode *ilookup5(struct super_block *sb, unsigned long hashval,
                 int (*test)(struct inode *, void *), void *data)
 {
-        struct hlist_head *head = inode_hashtable + hash(sb, hashval);
+        struct inode *inode = ilookup5_nowait(sb, hashval, test, data);
 
-        return ifind(sb, head, test, data, 1);
+        if (inode)
+                wait_on_inode(inode);
+        return inode;
 }
 EXPORT_SYMBOL(ilookup5);
 
@@ -1179,91 +1216,23 @@ EXPORT_SYMBOL(ilookup5);
  * @sb:         super block of file system to search
  * @ino:        inode number to search for
  *
- * ilookup() uses ifind_fast() to search for the inode @ino in the inode cache.
- * This is for file systems where the inode number is sufficient for unique
- * identification of an inode.
- *
- * If the inode is in the cache, the inode is returned with an incremented
- * reference count.
- *
- * Otherwise NULL is returned.
+ * Search for the inode @ino in the inode cache, and if the inode is in the
+ * cache, the inode is returned with an incremented reference count.
  */
 struct inode *ilookup(struct super_block *sb, unsigned long ino)
 {
         struct hlist_head *head = inode_hashtable + hash(sb, ino);
-
-        return ifind_fast(sb, head, ino);
-}
-EXPORT_SYMBOL(ilookup);
-
-/**
- * iget5_locked - obtain an inode from a mounted file system
- * @sb:         super block of file system
- * @hashval:    hash value (usually inode number) to get
- * @test:       callback used for comparisons between inodes
- * @set:        callback used to initialize a new struct inode
- * @data:       opaque data pointer to pass to @test and @set
- *
- * iget5_locked() uses ifind() to search for the inode specified by @hashval
- * and @data in the inode cache and if present it is returned with an increased
- * reference count. This is a generalized version of iget_locked() for file
- * systems where the inode number is not sufficient for unique identification
- * of an inode.
- *
- * If the inode is not in cache, get_new_inode() is called to allocate a new
- * inode and this is returned locked, hashed, and with the I_NEW flag set. The
- * file system gets to fill it in before unlocking it via unlock_new_inode().
- *
- * Note both @test and @set are called with the inode_lock held, so can't sleep.
- */
-struct inode *iget5_locked(struct super_block *sb, unsigned long hashval,
-                int (*test)(struct inode *, void *),
-                int (*set)(struct inode *, void *), void *data)
-{
-        struct hlist_head *head = inode_hashtable + hash(sb, hashval);
         struct inode *inode;
 
-        inode = ifind(sb, head, test, data, 1);
-        if (inode)
-                return inode;
-        /*
-         * get_new_inode() will do the right thing, re-trying the search
-         * in case it had to block at any point.
-         */
-        return get_new_inode(sb, head, test, set, data);
-}
-EXPORT_SYMBOL(iget5_locked);
-
-/**
- * iget_locked - obtain an inode from a mounted file system
- * @sb:         super block of file system
- * @ino:        inode number to get
- *
- * iget_locked() uses ifind_fast() to search for the inode specified by @ino in
- * the inode cache and if present it is returned with an increased reference
- * count. This is for file systems where the inode number is sufficient for
- * unique identification of an inode.
- *
- * If the inode is not in cache, get_new_inode_fast() is called to allocate a
- * new inode and this is returned locked, hashed, and with the I_NEW flag set.
- * The file system gets to fill it in before unlocking it via
- * unlock_new_inode().
- */
-struct inode *iget_locked(struct super_block *sb, unsigned long ino)
-{
-        struct hlist_head *head = inode_hashtable + hash(sb, ino);
-        struct inode *inode;
+        spin_lock(&inode_hash_lock);
+        inode = find_inode_fast(sb, head, ino);
+        spin_unlock(&inode_hash_lock);
 
-        inode = ifind_fast(sb, head, ino);
         if (inode)
-                return inode;
-        /*
-         * get_new_inode_fast() will do the right thing, re-trying the search
-         * in case it had to block at any point.
-         */
-        return get_new_inode_fast(sb, head, ino);
+                wait_on_inode(inode);
+        return inode;
 }
-EXPORT_SYMBOL(iget_locked);
+EXPORT_SYMBOL(ilookup);
 
 int insert_inode_locked(struct inode *inode)
 {
@@ -1271,27 +1240,33 @@ int insert_inode_locked(struct inode *inode)
         ino_t ino = inode->i_ino;
         struct hlist_head *head = inode_hashtable + hash(sb, ino);
 
-        inode->i_state |= I_NEW;
         while (1) {
                 struct hlist_node *node;
                 struct inode *old = NULL;
-                spin_lock(&inode_lock);
+                spin_lock(&inode_hash_lock);
                 hlist_for_each_entry(old, node, head, i_hash) {
                         if (old->i_ino != ino)
                                 continue;
                         if (old->i_sb != sb)
                                 continue;
-                        if (old->i_state & (I_FREEING|I_WILL_FREE))
+                        spin_lock(&old->i_lock);
+                        if (old->i_state & (I_FREEING|I_WILL_FREE)) {
+                                spin_unlock(&old->i_lock);
                                 continue;
+                        }
                         break;
                 }
                 if (likely(!node)) {
+                        spin_lock(&inode->i_lock);
+                        inode->i_state |= I_NEW;
                         hlist_add_head(&inode->i_hash, head);
-                        spin_unlock(&inode_lock);
+                        spin_unlock(&inode->i_lock);
+                        spin_unlock(&inode_hash_lock);
                         return 0;
                 }
                 __iget(old);
-                spin_unlock(&inode_lock);
+                spin_unlock(&old->i_lock);
+                spin_unlock(&inode_hash_lock);
                 wait_on_inode(old);
                 if (unlikely(!inode_unhashed(old))) {
                         iput(old);
@@ -1308,29 +1283,34 @@ int insert_inode_locked4(struct inode *inode, unsigned long hashval,
         struct super_block *sb = inode->i_sb;
         struct hlist_head *head = inode_hashtable + hash(sb, hashval);
 
-        inode->i_state |= I_NEW;
-
         while (1) {
                 struct hlist_node *node;
                 struct inode *old = NULL;
 
-                spin_lock(&inode_lock);
+                spin_lock(&inode_hash_lock);
                 hlist_for_each_entry(old, node, head, i_hash) {
                         if (old->i_sb != sb)
                                 continue;
                         if (!test(old, data))
                                 continue;
-                        if (old->i_state & (I_FREEING|I_WILL_FREE))
+                        spin_lock(&old->i_lock);
+                        if (old->i_state & (I_FREEING|I_WILL_FREE)) {
+                                spin_unlock(&old->i_lock);
                                 continue;
+                        }
                         break;
                 }
                 if (likely(!node)) {
+                        spin_lock(&inode->i_lock);
+                        inode->i_state |= I_NEW;
                         hlist_add_head(&inode->i_hash, head);
-                        spin_unlock(&inode_lock);
+                        spin_unlock(&inode->i_lock);
+                        spin_unlock(&inode_hash_lock);
                         return 0;
                 }
                 __iget(old);
-                spin_unlock(&inode_lock);
+                spin_unlock(&old->i_lock);
+                spin_unlock(&inode_hash_lock);
                 wait_on_inode(old);
                 if (unlikely(!inode_unhashed(old))) {
                         iput(old);
@@ -1375,47 +1355,35 @@ static void iput_final(struct inode *inode)
         const struct super_operations *op = inode->i_sb->s_op;
         int drop;
 
+        WARN_ON(inode->i_state & I_NEW);
+
         if (op && op->drop_inode)
                 drop = op->drop_inode(inode);
         else
                 drop = generic_drop_inode(inode);
 
+        if (!drop && (sb->s_flags & MS_ACTIVE)) {
+                inode->i_state |= I_REFERENCED;
+                if (!(inode->i_state & (I_DIRTY|I_SYNC)))
+                        inode_lru_list_add(inode);
+                spin_unlock(&inode->i_lock);
+                return;
+        }
+
         if (!drop) {
-                if (sb->s_flags & MS_ACTIVE) {
-                        inode->i_state |= I_REFERENCED;
-                        if (!(inode->i_state & (I_DIRTY|I_SYNC))) {
-                                inode_lru_list_add(inode);
-                        }
-                        spin_unlock(&inode_lock);
-                        return;
-                }
-                WARN_ON(inode->i_state & I_NEW);
                 inode->i_state |= I_WILL_FREE;
-                spin_unlock(&inode_lock);
+                spin_unlock(&inode->i_lock);
                 write_inode_now(inode, 1);
-                spin_lock(&inode_lock);
+                spin_lock(&inode->i_lock);
                 WARN_ON(inode->i_state & I_NEW);
                 inode->i_state &= ~I_WILL_FREE;
-                __remove_inode_hash(inode);
         }
 
-        WARN_ON(inode->i_state & I_NEW);
         inode->i_state |= I_FREEING;
-
-        /*
-         * Move the inode off the IO lists and LRU once I_FREEING is
-         * set so that it won't get moved back on there if it is dirty.
-         */
         inode_lru_list_del(inode);
-        list_del_init(&inode->i_wb_list);
+        spin_unlock(&inode->i_lock);
 
-        __inode_sb_list_del(inode);
-        spin_unlock(&inode_lock);
         evict(inode);
-        remove_inode_hash(inode);
-        wake_up_inode(inode);
-        BUG_ON(inode->i_state != (I_FREEING | I_CLEAR));
-        destroy_inode(inode);
 }
 
 /**
@@ -1432,7 +1400,7 @@ void iput(struct inode *inode)
         if (inode) {
                 BUG_ON(inode->i_state & I_CLEAR);
 
-                if (atomic_dec_and_lock(&inode->i_count, &inode_lock))
+                if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock))
                         iput_final(inode);
         }
 }
@@ -1611,9 +1579,8 @@ EXPORT_SYMBOL(inode_wait);
  * to recheck inode state.
  *
  * It doesn't matter if I_NEW is not set initially, a call to
- * wake_up_inode() after removing from the hash list will DTRT.
- *
- * This is called with inode_lock held.
+ * wake_up_bit(&inode->i_state, __I_NEW) after removing from the hash list
+ * will DTRT.
  */
 static void __wait_on_freeing_inode(struct inode *inode)
 {
@@ -1621,10 +1588,11 @@ static void __wait_on_freeing_inode(struct inode *inode)
         DEFINE_WAIT_BIT(wait, &inode->i_state, __I_NEW);
         wq = bit_waitqueue(&inode->i_state, __I_NEW);
         prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
-        spin_unlock(&inode_lock);
+        spin_unlock(&inode->i_lock);
+        spin_unlock(&inode_hash_lock);
         schedule();
         finish_wait(wq, &wait.wait);
-        spin_lock(&inode_lock);
+        spin_lock(&inode_hash_lock);
 }
 
 static __initdata unsigned long ihash_entries;