diff options
Diffstat (limited to 'fs/dcache.c')
-rw-r--r-- | fs/dcache.c | 89 |
1 files changed, 33 insertions, 56 deletions
diff --git a/fs/dcache.c b/fs/dcache.c index ad25c4cec7d5..22a0ef41bad1 100644 --- a/fs/dcache.c +++ b/fs/dcache.c | |||
@@ -99,12 +99,9 @@ static struct kmem_cache *dentry_cache __read_mostly; | |||
99 | static unsigned int d_hash_mask __read_mostly; | 99 | static unsigned int d_hash_mask __read_mostly; |
100 | static unsigned int d_hash_shift __read_mostly; | 100 | static unsigned int d_hash_shift __read_mostly; |
101 | 101 | ||
102 | struct dcache_hash_bucket { | 102 | static struct hlist_bl_head *dentry_hashtable __read_mostly; |
103 | struct hlist_bl_head head; | ||
104 | }; | ||
105 | static struct dcache_hash_bucket *dentry_hashtable __read_mostly; | ||
106 | 103 | ||
107 | static inline struct dcache_hash_bucket *d_hash(struct dentry *parent, | 104 | static inline struct hlist_bl_head *d_hash(struct dentry *parent, |
108 | unsigned long hash) | 105 | unsigned long hash) |
109 | { | 106 | { |
110 | hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES; | 107 | hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES; |
@@ -112,16 +109,6 @@ static inline struct dcache_hash_bucket *d_hash(struct dentry *parent, | |||
112 | return dentry_hashtable + (hash & D_HASHMASK); | 109 | return dentry_hashtable + (hash & D_HASHMASK); |
113 | } | 110 | } |
114 | 111 | ||
115 | static inline void spin_lock_bucket(struct dcache_hash_bucket *b) | ||
116 | { | ||
117 | bit_spin_lock(0, (unsigned long *)&b->head.first); | ||
118 | } | ||
119 | |||
120 | static inline void spin_unlock_bucket(struct dcache_hash_bucket *b) | ||
121 | { | ||
122 | __bit_spin_unlock(0, (unsigned long *)&b->head.first); | ||
123 | } | ||
124 | |||
125 | /* Statistics gathering. */ | 112 | /* Statistics gathering. */ |
126 | struct dentry_stat_t dentry_stat = { | 113 | struct dentry_stat_t dentry_stat = { |
127 | .age_limit = 45, | 114 | .age_limit = 45, |
@@ -167,8 +154,8 @@ static void d_free(struct dentry *dentry) | |||
167 | if (dentry->d_op && dentry->d_op->d_release) | 154 | if (dentry->d_op && dentry->d_op->d_release) |
168 | dentry->d_op->d_release(dentry); | 155 | dentry->d_op->d_release(dentry); |
169 | 156 | ||
170 | /* if dentry was never inserted into hash, immediate free is OK */ | 157 | /* if dentry was never visible to RCU, immediate free is OK */ |
171 | if (hlist_bl_unhashed(&dentry->d_hash)) | 158 | if (!(dentry->d_flags & DCACHE_RCUACCESS)) |
172 | __d_free(&dentry->d_u.d_rcu); | 159 | __d_free(&dentry->d_u.d_rcu); |
173 | else | 160 | else |
174 | call_rcu(&dentry->d_u.d_rcu, __d_free); | 161 | call_rcu(&dentry->d_u.d_rcu, __d_free); |
@@ -330,28 +317,19 @@ static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent) | |||
330 | */ | 317 | */ |
331 | void __d_drop(struct dentry *dentry) | 318 | void __d_drop(struct dentry *dentry) |
332 | { | 319 | { |
333 | if (!(dentry->d_flags & DCACHE_UNHASHED)) { | 320 | if (!d_unhashed(dentry)) { |
334 | if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED)) { | 321 | struct hlist_bl_head *b; |
335 | bit_spin_lock(0, | 322 | if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED)) |
336 | (unsigned long *)&dentry->d_sb->s_anon.first); | 323 | b = &dentry->d_sb->s_anon; |
337 | dentry->d_flags |= DCACHE_UNHASHED; | 324 | else |
338 | hlist_bl_del_init(&dentry->d_hash); | ||
339 | __bit_spin_unlock(0, | ||
340 | (unsigned long *)&dentry->d_sb->s_anon.first); | ||
341 | } else { | ||
342 | struct dcache_hash_bucket *b; | ||
343 | b = d_hash(dentry->d_parent, dentry->d_name.hash); | 325 | b = d_hash(dentry->d_parent, dentry->d_name.hash); |
344 | spin_lock_bucket(b); | 326 | |
345 | /* | 327 | hlist_bl_lock(b); |
346 | * We may not actually need to put DCACHE_UNHASHED | 328 | __hlist_bl_del(&dentry->d_hash); |
347 | * manipulations under the hash lock, but follow | 329 | dentry->d_hash.pprev = NULL; |
348 | * the principle of least surprise. | 330 | hlist_bl_unlock(b); |
349 | */ | 331 | |
350 | dentry->d_flags |= DCACHE_UNHASHED; | 332 | dentry_rcuwalk_barrier(dentry); |
351 | hlist_bl_del_rcu(&dentry->d_hash); | ||
352 | spin_unlock_bucket(b); | ||
353 | dentry_rcuwalk_barrier(dentry); | ||
354 | } | ||
355 | } | 333 | } |
356 | } | 334 | } |
357 | EXPORT_SYMBOL(__d_drop); | 335 | EXPORT_SYMBOL(__d_drop); |
@@ -1304,7 +1282,7 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) | |||
1304 | dname[name->len] = 0; | 1282 | dname[name->len] = 0; |
1305 | 1283 | ||
1306 | dentry->d_count = 1; | 1284 | dentry->d_count = 1; |
1307 | dentry->d_flags = DCACHE_UNHASHED; | 1285 | dentry->d_flags = 0; |
1308 | spin_lock_init(&dentry->d_lock); | 1286 | spin_lock_init(&dentry->d_lock); |
1309 | seqcount_init(&dentry->d_seq); | 1287 | seqcount_init(&dentry->d_seq); |
1310 | dentry->d_inode = NULL; | 1288 | dentry->d_inode = NULL; |
@@ -1606,10 +1584,9 @@ struct dentry *d_obtain_alias(struct inode *inode) | |||
1606 | tmp->d_inode = inode; | 1584 | tmp->d_inode = inode; |
1607 | tmp->d_flags |= DCACHE_DISCONNECTED; | 1585 | tmp->d_flags |= DCACHE_DISCONNECTED; |
1608 | list_add(&tmp->d_alias, &inode->i_dentry); | 1586 | list_add(&tmp->d_alias, &inode->i_dentry); |
1609 | bit_spin_lock(0, (unsigned long *)&tmp->d_sb->s_anon.first); | 1587 | hlist_bl_lock(&tmp->d_sb->s_anon); |
1610 | tmp->d_flags &= ~DCACHE_UNHASHED; | ||
1611 | hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon); | 1588 | hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon); |
1612 | __bit_spin_unlock(0, (unsigned long *)&tmp->d_sb->s_anon.first); | 1589 | hlist_bl_unlock(&tmp->d_sb->s_anon); |
1613 | spin_unlock(&tmp->d_lock); | 1590 | spin_unlock(&tmp->d_lock); |
1614 | spin_unlock(&inode->i_lock); | 1591 | spin_unlock(&inode->i_lock); |
1615 | security_d_instantiate(tmp, inode); | 1592 | security_d_instantiate(tmp, inode); |
@@ -1789,7 +1766,7 @@ struct dentry *__d_lookup_rcu(struct dentry *parent, struct qstr *name, | |||
1789 | unsigned int len = name->len; | 1766 | unsigned int len = name->len; |
1790 | unsigned int hash = name->hash; | 1767 | unsigned int hash = name->hash; |
1791 | const unsigned char *str = name->name; | 1768 | const unsigned char *str = name->name; |
1792 | struct dcache_hash_bucket *b = d_hash(parent, hash); | 1769 | struct hlist_bl_head *b = d_hash(parent, hash); |
1793 | struct hlist_bl_node *node; | 1770 | struct hlist_bl_node *node; |
1794 | struct dentry *dentry; | 1771 | struct dentry *dentry; |
1795 | 1772 | ||
@@ -1813,7 +1790,7 @@ struct dentry *__d_lookup_rcu(struct dentry *parent, struct qstr *name, | |||
1813 | * | 1790 | * |
1814 | * See Documentation/filesystems/path-lookup.txt for more details. | 1791 | * See Documentation/filesystems/path-lookup.txt for more details. |
1815 | */ | 1792 | */ |
1816 | hlist_bl_for_each_entry_rcu(dentry, node, &b->head, d_hash) { | 1793 | hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { |
1817 | struct inode *i; | 1794 | struct inode *i; |
1818 | const char *tname; | 1795 | const char *tname; |
1819 | int tlen; | 1796 | int tlen; |
@@ -1908,7 +1885,7 @@ struct dentry *__d_lookup(struct dentry *parent, struct qstr *name) | |||
1908 | unsigned int len = name->len; | 1885 | unsigned int len = name->len; |
1909 | unsigned int hash = name->hash; | 1886 | unsigned int hash = name->hash; |
1910 | const unsigned char *str = name->name; | 1887 | const unsigned char *str = name->name; |
1911 | struct dcache_hash_bucket *b = d_hash(parent, hash); | 1888 | struct hlist_bl_head *b = d_hash(parent, hash); |
1912 | struct hlist_bl_node *node; | 1889 | struct hlist_bl_node *node; |
1913 | struct dentry *found = NULL; | 1890 | struct dentry *found = NULL; |
1914 | struct dentry *dentry; | 1891 | struct dentry *dentry; |
@@ -1935,7 +1912,7 @@ struct dentry *__d_lookup(struct dentry *parent, struct qstr *name) | |||
1935 | */ | 1912 | */ |
1936 | rcu_read_lock(); | 1913 | rcu_read_lock(); |
1937 | 1914 | ||
1938 | hlist_bl_for_each_entry_rcu(dentry, node, &b->head, d_hash) { | 1915 | hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { |
1939 | const char *tname; | 1916 | const char *tname; |
1940 | int tlen; | 1917 | int tlen; |
1941 | 1918 | ||
@@ -2086,13 +2063,13 @@ again: | |||
2086 | } | 2063 | } |
2087 | EXPORT_SYMBOL(d_delete); | 2064 | EXPORT_SYMBOL(d_delete); |
2088 | 2065 | ||
2089 | static void __d_rehash(struct dentry * entry, struct dcache_hash_bucket *b) | 2066 | static void __d_rehash(struct dentry * entry, struct hlist_bl_head *b) |
2090 | { | 2067 | { |
2091 | BUG_ON(!d_unhashed(entry)); | 2068 | BUG_ON(!d_unhashed(entry)); |
2092 | spin_lock_bucket(b); | 2069 | hlist_bl_lock(b); |
2093 | entry->d_flags &= ~DCACHE_UNHASHED; | 2070 | entry->d_flags |= DCACHE_RCUACCESS; |
2094 | hlist_bl_add_head_rcu(&entry->d_hash, &b->head); | 2071 | hlist_bl_add_head_rcu(&entry->d_hash, b); |
2095 | spin_unlock_bucket(b); | 2072 | hlist_bl_unlock(b); |
2096 | } | 2073 | } |
2097 | 2074 | ||
2098 | static void _d_rehash(struct dentry * entry) | 2075 | static void _d_rehash(struct dentry * entry) |
@@ -2131,7 +2108,7 @@ EXPORT_SYMBOL(d_rehash); | |||
2131 | */ | 2108 | */ |
2132 | void dentry_update_name_case(struct dentry *dentry, struct qstr *name) | 2109 | void dentry_update_name_case(struct dentry *dentry, struct qstr *name) |
2133 | { | 2110 | { |
2134 | BUG_ON(!mutex_is_locked(&dentry->d_inode->i_mutex)); | 2111 | BUG_ON(!mutex_is_locked(&dentry->d_parent->d_inode->i_mutex)); |
2135 | BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */ | 2112 | BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */ |
2136 | 2113 | ||
2137 | spin_lock(&dentry->d_lock); | 2114 | spin_lock(&dentry->d_lock); |
@@ -3025,7 +3002,7 @@ static void __init dcache_init_early(void) | |||
3025 | 3002 | ||
3026 | dentry_hashtable = | 3003 | dentry_hashtable = |
3027 | alloc_large_system_hash("Dentry cache", | 3004 | alloc_large_system_hash("Dentry cache", |
3028 | sizeof(struct dcache_hash_bucket), | 3005 | sizeof(struct hlist_bl_head), |
3029 | dhash_entries, | 3006 | dhash_entries, |
3030 | 13, | 3007 | 13, |
3031 | HASH_EARLY, | 3008 | HASH_EARLY, |
@@ -3034,7 +3011,7 @@ static void __init dcache_init_early(void) | |||
3034 | 0); | 3011 | 0); |
3035 | 3012 | ||
3036 | for (loop = 0; loop < (1 << d_hash_shift); loop++) | 3013 | for (loop = 0; loop < (1 << d_hash_shift); loop++) |
3037 | INIT_HLIST_BL_HEAD(&dentry_hashtable[loop].head); | 3014 | INIT_HLIST_BL_HEAD(dentry_hashtable + loop); |
3038 | } | 3015 | } |
3039 | 3016 | ||
3040 | static void __init dcache_init(void) | 3017 | static void __init dcache_init(void) |
@@ -3057,7 +3034,7 @@ static void __init dcache_init(void) | |||
3057 | 3034 | ||
3058 | dentry_hashtable = | 3035 | dentry_hashtable = |
3059 | alloc_large_system_hash("Dentry cache", | 3036 | alloc_large_system_hash("Dentry cache", |
3060 | sizeof(struct dcache_hash_bucket), | 3037 | sizeof(struct hlist_bl_head), |
3061 | dhash_entries, | 3038 | dhash_entries, |
3062 | 13, | 3039 | 13, |
3063 | 0, | 3040 | 0, |
@@ -3066,7 +3043,7 @@ static void __init dcache_init(void) | |||
3066 | 0); | 3043 | 0); |
3067 | 3044 | ||
3068 | for (loop = 0; loop < (1 << d_hash_shift); loop++) | 3045 | for (loop = 0; loop < (1 << d_hash_shift); loop++) |
3069 | INIT_HLIST_BL_HEAD(&dentry_hashtable[loop].head); | 3046 | INIT_HLIST_BL_HEAD(dentry_hashtable + loop); |
3070 | } | 3047 | } |
3071 | 3048 | ||
3072 | /* SLAB cache for __getname() consumers */ | 3049 | /* SLAB cache for __getname() consumers */ |