aboutsummaryrefslogtreecommitdiffstats
path: root/fs/dcache.c
diff options
context:
space:
mode:
Diffstat (limited to 'fs/dcache.c')
-rw-r--r--fs/dcache.c1530
1 files changed, 1068 insertions, 462 deletions
diff --git a/fs/dcache.c b/fs/dcache.c
index 83293be48149..2a6bd9a4ae97 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -33,20 +33,58 @@
33#include <linux/bootmem.h> 33#include <linux/bootmem.h>
34#include <linux/fs_struct.h> 34#include <linux/fs_struct.h>
35#include <linux/hardirq.h> 35#include <linux/hardirq.h>
36#include <linux/bit_spinlock.h>
37#include <linux/rculist_bl.h>
36#include "internal.h" 38#include "internal.h"
37 39
40/*
41 * Usage:
42 * dcache->d_inode->i_lock protects:
43 * - i_dentry, d_alias, d_inode of aliases
44 * dcache_hash_bucket lock protects:
45 * - the dcache hash table
46 * s_anon bl list spinlock protects:
47 * - the s_anon list (see __d_drop)
48 * dcache_lru_lock protects:
49 * - the dcache lru lists and counters
50 * d_lock protects:
51 * - d_flags
52 * - d_name
53 * - d_lru
54 * - d_count
55 * - d_unhashed()
56 * - d_parent and d_subdirs
57 * - childrens' d_child and d_parent
58 * - d_alias, d_inode
59 *
60 * Ordering:
61 * dentry->d_inode->i_lock
62 * dentry->d_lock
63 * dcache_lru_lock
64 * dcache_hash_bucket lock
65 * s_anon lock
66 *
67 * If there is an ancestor relationship:
68 * dentry->d_parent->...->d_parent->d_lock
69 * ...
70 * dentry->d_parent->d_lock
71 * dentry->d_lock
72 *
73 * If no ancestor relationship:
74 * if (dentry1 < dentry2)
75 * dentry1->d_lock
76 * dentry2->d_lock
77 */
38int sysctl_vfs_cache_pressure __read_mostly = 100; 78int sysctl_vfs_cache_pressure __read_mostly = 100;
39EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure); 79EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
40 80
41 __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock); 81static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock);
42__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock); 82__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
43 83
44EXPORT_SYMBOL(dcache_lock); 84EXPORT_SYMBOL(rename_lock);
45 85
46static struct kmem_cache *dentry_cache __read_mostly; 86static struct kmem_cache *dentry_cache __read_mostly;
47 87
48#define DNAME_INLINE_LEN (sizeof(struct dentry)-offsetof(struct dentry,d_iname))
49
50/* 88/*
51 * This is the single most critical data structure when it comes 89 * This is the single most critical data structure when it comes
52 * to the dcache: the hashtable for lookups. Somebody should try 90 * to the dcache: the hashtable for lookups. Somebody should try
@@ -60,56 +98,111 @@ static struct kmem_cache *dentry_cache __read_mostly;
60 98
61static unsigned int d_hash_mask __read_mostly; 99static unsigned int d_hash_mask __read_mostly;
62static unsigned int d_hash_shift __read_mostly; 100static unsigned int d_hash_shift __read_mostly;
63static struct hlist_head *dentry_hashtable __read_mostly; 101
102struct dcache_hash_bucket {
103 struct hlist_bl_head head;
104};
105static struct dcache_hash_bucket *dentry_hashtable __read_mostly;
106
107static inline struct dcache_hash_bucket *d_hash(struct dentry *parent,
108 unsigned long hash)
109{
110 hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES;
111 hash = hash ^ ((hash ^ GOLDEN_RATIO_PRIME) >> D_HASHBITS);
112 return dentry_hashtable + (hash & D_HASHMASK);
113}
114
115static inline void spin_lock_bucket(struct dcache_hash_bucket *b)
116{
117 bit_spin_lock(0, (unsigned long *)&b->head.first);
118}
119
120static inline void spin_unlock_bucket(struct dcache_hash_bucket *b)
121{
122 __bit_spin_unlock(0, (unsigned long *)&b->head.first);
123}
64 124
65/* Statistics gathering. */ 125/* Statistics gathering. */
66struct dentry_stat_t dentry_stat = { 126struct dentry_stat_t dentry_stat = {
67 .age_limit = 45, 127 .age_limit = 45,
68}; 128};
69 129
70static void __d_free(struct dentry *dentry) 130static DEFINE_PER_CPU(unsigned int, nr_dentry);
131
132#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
133static int get_nr_dentry(void)
134{
135 int i;
136 int sum = 0;
137 for_each_possible_cpu(i)
138 sum += per_cpu(nr_dentry, i);
139 return sum < 0 ? 0 : sum;
140}
141
142int proc_nr_dentry(ctl_table *table, int write, void __user *buffer,
143 size_t *lenp, loff_t *ppos)
71{ 144{
145 dentry_stat.nr_dentry = get_nr_dentry();
146 return proc_dointvec(table, write, buffer, lenp, ppos);
147}
148#endif
149
150static void __d_free(struct rcu_head *head)
151{
152 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
153
72 WARN_ON(!list_empty(&dentry->d_alias)); 154 WARN_ON(!list_empty(&dentry->d_alias));
73 if (dname_external(dentry)) 155 if (dname_external(dentry))
74 kfree(dentry->d_name.name); 156 kfree(dentry->d_name.name);
75 kmem_cache_free(dentry_cache, dentry); 157 kmem_cache_free(dentry_cache, dentry);
76} 158}
77 159
78static void d_callback(struct rcu_head *head)
79{
80 struct dentry * dentry = container_of(head, struct dentry, d_u.d_rcu);
81 __d_free(dentry);
82}
83
84/* 160/*
85 * no dcache_lock, please. The caller must decrement dentry_stat.nr_dentry 161 * no locks, please.
86 * inside dcache_lock.
87 */ 162 */
88static void d_free(struct dentry *dentry) 163static void d_free(struct dentry *dentry)
89{ 164{
165 BUG_ON(dentry->d_count);
166 this_cpu_dec(nr_dentry);
90 if (dentry->d_op && dentry->d_op->d_release) 167 if (dentry->d_op && dentry->d_op->d_release)
91 dentry->d_op->d_release(dentry); 168 dentry->d_op->d_release(dentry);
169
92 /* if dentry was never inserted into hash, immediate free is OK */ 170 /* if dentry was never inserted into hash, immediate free is OK */
93 if (hlist_unhashed(&dentry->d_hash)) 171 if (hlist_bl_unhashed(&dentry->d_hash))
94 __d_free(dentry); 172 __d_free(&dentry->d_u.d_rcu);
95 else 173 else
96 call_rcu(&dentry->d_u.d_rcu, d_callback); 174 call_rcu(&dentry->d_u.d_rcu, __d_free);
175}
176
177/**
178 * dentry_rcuwalk_barrier - invalidate in-progress rcu-walk lookups
179 * @dentry: the target dentry
180 * After this call, in-progress rcu-walk path lookup will fail. This
181 * should be called after unhashing, and after changing d_inode (if
182 * the dentry has not already been unhashed).
183 */
184static inline void dentry_rcuwalk_barrier(struct dentry *dentry)
185{
186 assert_spin_locked(&dentry->d_lock);
187 /* Go through a barrier */
188 write_seqcount_barrier(&dentry->d_seq);
97} 189}
98 190
99/* 191/*
100 * Release the dentry's inode, using the filesystem 192 * Release the dentry's inode, using the filesystem
101 * d_iput() operation if defined. 193 * d_iput() operation if defined. Dentry has no refcount
194 * and is unhashed.
102 */ 195 */
103static void dentry_iput(struct dentry * dentry) 196static void dentry_iput(struct dentry * dentry)
104 __releases(dentry->d_lock) 197 __releases(dentry->d_lock)
105 __releases(dcache_lock) 198 __releases(dentry->d_inode->i_lock)
106{ 199{
107 struct inode *inode = dentry->d_inode; 200 struct inode *inode = dentry->d_inode;
108 if (inode) { 201 if (inode) {
109 dentry->d_inode = NULL; 202 dentry->d_inode = NULL;
110 list_del_init(&dentry->d_alias); 203 list_del_init(&dentry->d_alias);
111 spin_unlock(&dentry->d_lock); 204 spin_unlock(&dentry->d_lock);
112 spin_unlock(&dcache_lock); 205 spin_unlock(&inode->i_lock);
113 if (!inode->i_nlink) 206 if (!inode->i_nlink)
114 fsnotify_inoderemove(inode); 207 fsnotify_inoderemove(inode);
115 if (dentry->d_op && dentry->d_op->d_iput) 208 if (dentry->d_op && dentry->d_op->d_iput)
@@ -118,69 +211,191 @@ static void dentry_iput(struct dentry * dentry)
118 iput(inode); 211 iput(inode);
119 } else { 212 } else {
120 spin_unlock(&dentry->d_lock); 213 spin_unlock(&dentry->d_lock);
121 spin_unlock(&dcache_lock);
122 } 214 }
123} 215}
124 216
125/* 217/*
126 * dentry_lru_(add|add_tail|del|del_init) must be called with dcache_lock held. 218 * Release the dentry's inode, using the filesystem
219 * d_iput() operation if defined. dentry remains in-use.
220 */
221static void dentry_unlink_inode(struct dentry * dentry)
222 __releases(dentry->d_lock)
223 __releases(dentry->d_inode->i_lock)
224{
225 struct inode *inode = dentry->d_inode;
226 dentry->d_inode = NULL;
227 list_del_init(&dentry->d_alias);
228 dentry_rcuwalk_barrier(dentry);
229 spin_unlock(&dentry->d_lock);
230 spin_unlock(&inode->i_lock);
231 if (!inode->i_nlink)
232 fsnotify_inoderemove(inode);
233 if (dentry->d_op && dentry->d_op->d_iput)
234 dentry->d_op->d_iput(dentry, inode);
235 else
236 iput(inode);
237}
238
239/*
240 * dentry_lru_(add|del|move_tail) must be called with d_lock held.
127 */ 241 */
128static void dentry_lru_add(struct dentry *dentry) 242static void dentry_lru_add(struct dentry *dentry)
129{ 243{
130 list_add(&dentry->d_lru, &dentry->d_sb->s_dentry_lru); 244 if (list_empty(&dentry->d_lru)) {
131 dentry->d_sb->s_nr_dentry_unused++; 245 spin_lock(&dcache_lru_lock);
132 dentry_stat.nr_unused++; 246 list_add(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
247 dentry->d_sb->s_nr_dentry_unused++;
248 dentry_stat.nr_unused++;
249 spin_unlock(&dcache_lru_lock);
250 }
133} 251}
134 252
135static void dentry_lru_add_tail(struct dentry *dentry) 253static void __dentry_lru_del(struct dentry *dentry)
136{ 254{
137 list_add_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru); 255 list_del_init(&dentry->d_lru);
138 dentry->d_sb->s_nr_dentry_unused++; 256 dentry->d_sb->s_nr_dentry_unused--;
139 dentry_stat.nr_unused++; 257 dentry_stat.nr_unused--;
140} 258}
141 259
142static void dentry_lru_del(struct dentry *dentry) 260static void dentry_lru_del(struct dentry *dentry)
143{ 261{
144 if (!list_empty(&dentry->d_lru)) { 262 if (!list_empty(&dentry->d_lru)) {
145 list_del(&dentry->d_lru); 263 spin_lock(&dcache_lru_lock);
146 dentry->d_sb->s_nr_dentry_unused--; 264 __dentry_lru_del(dentry);
147 dentry_stat.nr_unused--; 265 spin_unlock(&dcache_lru_lock);
148 } 266 }
149} 267}
150 268
151static void dentry_lru_del_init(struct dentry *dentry) 269static void dentry_lru_move_tail(struct dentry *dentry)
152{ 270{
153 if (likely(!list_empty(&dentry->d_lru))) { 271 spin_lock(&dcache_lru_lock);
154 list_del_init(&dentry->d_lru); 272 if (list_empty(&dentry->d_lru)) {
155 dentry->d_sb->s_nr_dentry_unused--; 273 list_add_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
156 dentry_stat.nr_unused--; 274 dentry->d_sb->s_nr_dentry_unused++;
275 dentry_stat.nr_unused++;
276 } else {
277 list_move_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
157 } 278 }
279 spin_unlock(&dcache_lru_lock);
158} 280}
159 281
160/** 282/**
161 * d_kill - kill dentry and return parent 283 * d_kill - kill dentry and return parent
162 * @dentry: dentry to kill 284 * @dentry: dentry to kill
285 * @parent: parent dentry
163 * 286 *
164 * The dentry must already be unhashed and removed from the LRU. 287 * The dentry must already be unhashed and removed from the LRU.
165 * 288 *
166 * If this is the root of the dentry tree, return NULL. 289 * If this is the root of the dentry tree, return NULL.
290 *
291 * dentry->d_lock and parent->d_lock must be held by caller, and are dropped by
292 * d_kill.
167 */ 293 */
168static struct dentry *d_kill(struct dentry *dentry) 294static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent)
169 __releases(dentry->d_lock) 295 __releases(dentry->d_lock)
170 __releases(dcache_lock) 296 __releases(parent->d_lock)
297 __releases(dentry->d_inode->i_lock)
171{ 298{
172 struct dentry *parent; 299 dentry->d_parent = NULL;
173
174 list_del(&dentry->d_u.d_child); 300 list_del(&dentry->d_u.d_child);
175 dentry_stat.nr_dentry--; /* For d_free, below */ 301 if (parent)
176 /*drops the locks, at that point nobody can reach this dentry */ 302 spin_unlock(&parent->d_lock);
177 dentry_iput(dentry); 303 dentry_iput(dentry);
304 /*
305 * dentry_iput drops the locks, at which point nobody (except
306 * transient RCU lookups) can reach this dentry.
307 */
308 d_free(dentry);
309 return parent;
310}
311
312/**
313 * d_drop - drop a dentry
314 * @dentry: dentry to drop
315 *
316 * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
317 * be found through a VFS lookup any more. Note that this is different from
318 * deleting the dentry - d_delete will try to mark the dentry negative if
319 * possible, giving a successful _negative_ lookup, while d_drop will
320 * just make the cache lookup fail.
321 *
322 * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
323 * reason (NFS timeouts or autofs deletes).
324 *
325 * __d_drop requires dentry->d_lock.
326 */
327void __d_drop(struct dentry *dentry)
328{
329 if (!(dentry->d_flags & DCACHE_UNHASHED)) {
330 if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED)) {
331 bit_spin_lock(0,
332 (unsigned long *)&dentry->d_sb->s_anon.first);
333 dentry->d_flags |= DCACHE_UNHASHED;
334 hlist_bl_del_init(&dentry->d_hash);
335 __bit_spin_unlock(0,
336 (unsigned long *)&dentry->d_sb->s_anon.first);
337 } else {
338 struct dcache_hash_bucket *b;
339 b = d_hash(dentry->d_parent, dentry->d_name.hash);
340 spin_lock_bucket(b);
341 /*
342 * We may not actually need to put DCACHE_UNHASHED
343 * manipulations under the hash lock, but follow
344 * the principle of least surprise.
345 */
346 dentry->d_flags |= DCACHE_UNHASHED;
347 hlist_bl_del_rcu(&dentry->d_hash);
348 spin_unlock_bucket(b);
349 dentry_rcuwalk_barrier(dentry);
350 }
351 }
352}
353EXPORT_SYMBOL(__d_drop);
354
355void d_drop(struct dentry *dentry)
356{
357 spin_lock(&dentry->d_lock);
358 __d_drop(dentry);
359 spin_unlock(&dentry->d_lock);
360}
361EXPORT_SYMBOL(d_drop);
362
363/*
364 * Finish off a dentry we've decided to kill.
365 * dentry->d_lock must be held, returns with it unlocked.
366 * If ref is non-zero, then decrement the refcount too.
367 * Returns dentry requiring refcount drop, or NULL if we're done.
368 */
369static inline struct dentry *dentry_kill(struct dentry *dentry, int ref)
370 __releases(dentry->d_lock)
371{
372 struct inode *inode;
373 struct dentry *parent;
374
375 inode = dentry->d_inode;
376 if (inode && !spin_trylock(&inode->i_lock)) {
377relock:
378 spin_unlock(&dentry->d_lock);
379 cpu_relax();
380 return dentry; /* try again with same dentry */
381 }
178 if (IS_ROOT(dentry)) 382 if (IS_ROOT(dentry))
179 parent = NULL; 383 parent = NULL;
180 else 384 else
181 parent = dentry->d_parent; 385 parent = dentry->d_parent;
182 d_free(dentry); 386 if (parent && !spin_trylock(&parent->d_lock)) {
183 return parent; 387 if (inode)
388 spin_unlock(&inode->i_lock);
389 goto relock;
390 }
391
392 if (ref)
393 dentry->d_count--;
394 /* if dentry was on the d_lru list delete it from there */
395 dentry_lru_del(dentry);
396 /* if it was on the hash then remove it */
397 __d_drop(dentry);
398 return d_kill(dentry, parent);
184} 399}
185 400
186/* 401/*
@@ -208,52 +423,42 @@ static struct dentry *d_kill(struct dentry *dentry)
208 * call the dentry unlink method as well as removing it from the queues and 423 * call the dentry unlink method as well as removing it from the queues and
209 * releasing its resources. If the parent dentries were scheduled for release 424 * releasing its resources. If the parent dentries were scheduled for release
210 * they too may now get deleted. 425 * they too may now get deleted.
211 *
212 * no dcache lock, please.
213 */ 426 */
214
215void dput(struct dentry *dentry) 427void dput(struct dentry *dentry)
216{ 428{
217 if (!dentry) 429 if (!dentry)
218 return; 430 return;
219 431
220repeat: 432repeat:
221 if (atomic_read(&dentry->d_count) == 1) 433 if (dentry->d_count == 1)
222 might_sleep(); 434 might_sleep();
223 if (!atomic_dec_and_lock(&dentry->d_count, &dcache_lock))
224 return;
225
226 spin_lock(&dentry->d_lock); 435 spin_lock(&dentry->d_lock);
227 if (atomic_read(&dentry->d_count)) { 436 BUG_ON(!dentry->d_count);
437 if (dentry->d_count > 1) {
438 dentry->d_count--;
228 spin_unlock(&dentry->d_lock); 439 spin_unlock(&dentry->d_lock);
229 spin_unlock(&dcache_lock);
230 return; 440 return;
231 } 441 }
232 442
233 /* 443 if (dentry->d_flags & DCACHE_OP_DELETE) {
234 * AV: ->d_delete() is _NOT_ allowed to block now.
235 */
236 if (dentry->d_op && dentry->d_op->d_delete) {
237 if (dentry->d_op->d_delete(dentry)) 444 if (dentry->d_op->d_delete(dentry))
238 goto unhash_it; 445 goto kill_it;
239 } 446 }
447
240 /* Unreachable? Get rid of it */ 448 /* Unreachable? Get rid of it */
241 if (d_unhashed(dentry)) 449 if (d_unhashed(dentry))
242 goto kill_it; 450 goto kill_it;
243 if (list_empty(&dentry->d_lru)) { 451
244 dentry->d_flags |= DCACHE_REFERENCED; 452 /* Otherwise leave it cached and ensure it's on the LRU */
245 dentry_lru_add(dentry); 453 dentry->d_flags |= DCACHE_REFERENCED;
246 } 454 dentry_lru_add(dentry);
247 spin_unlock(&dentry->d_lock); 455
248 spin_unlock(&dcache_lock); 456 dentry->d_count--;
457 spin_unlock(&dentry->d_lock);
249 return; 458 return;
250 459
251unhash_it:
252 __d_drop(dentry);
253kill_it: 460kill_it:
254 /* if dentry was on the d_lru list delete it from there */ 461 dentry = dentry_kill(dentry, 1);
255 dentry_lru_del(dentry);
256 dentry = d_kill(dentry);
257 if (dentry) 462 if (dentry)
258 goto repeat; 463 goto repeat;
259} 464}
@@ -276,9 +481,9 @@ int d_invalidate(struct dentry * dentry)
276 /* 481 /*
277 * If it's already been dropped, return OK. 482 * If it's already been dropped, return OK.
278 */ 483 */
279 spin_lock(&dcache_lock); 484 spin_lock(&dentry->d_lock);
280 if (d_unhashed(dentry)) { 485 if (d_unhashed(dentry)) {
281 spin_unlock(&dcache_lock); 486 spin_unlock(&dentry->d_lock);
282 return 0; 487 return 0;
283 } 488 }
284 /* 489 /*
@@ -286,9 +491,9 @@ int d_invalidate(struct dentry * dentry)
286 * to get rid of unused child entries. 491 * to get rid of unused child entries.
287 */ 492 */
288 if (!list_empty(&dentry->d_subdirs)) { 493 if (!list_empty(&dentry->d_subdirs)) {
289 spin_unlock(&dcache_lock); 494 spin_unlock(&dentry->d_lock);
290 shrink_dcache_parent(dentry); 495 shrink_dcache_parent(dentry);
291 spin_lock(&dcache_lock); 496 spin_lock(&dentry->d_lock);
292 } 497 }
293 498
294 /* 499 /*
@@ -301,36 +506,61 @@ int d_invalidate(struct dentry * dentry)
301 * we might still populate it if it was a 506 * we might still populate it if it was a
302 * working directory or similar). 507 * working directory or similar).
303 */ 508 */
304 spin_lock(&dentry->d_lock); 509 if (dentry->d_count > 1) {
305 if (atomic_read(&dentry->d_count) > 1) {
306 if (dentry->d_inode && S_ISDIR(dentry->d_inode->i_mode)) { 510 if (dentry->d_inode && S_ISDIR(dentry->d_inode->i_mode)) {
307 spin_unlock(&dentry->d_lock); 511 spin_unlock(&dentry->d_lock);
308 spin_unlock(&dcache_lock);
309 return -EBUSY; 512 return -EBUSY;
310 } 513 }
311 } 514 }
312 515
313 __d_drop(dentry); 516 __d_drop(dentry);
314 spin_unlock(&dentry->d_lock); 517 spin_unlock(&dentry->d_lock);
315 spin_unlock(&dcache_lock);
316 return 0; 518 return 0;
317} 519}
318EXPORT_SYMBOL(d_invalidate); 520EXPORT_SYMBOL(d_invalidate);
319 521
320/* This should be called _only_ with dcache_lock held */ 522/* This must be called with d_lock held */
523static inline void __dget_dlock(struct dentry *dentry)
524{
525 dentry->d_count++;
526}
321 527
322static inline struct dentry * __dget_locked(struct dentry *dentry) 528static inline void __dget(struct dentry *dentry)
323{ 529{
324 atomic_inc(&dentry->d_count); 530 spin_lock(&dentry->d_lock);
325 dentry_lru_del_init(dentry); 531 __dget_dlock(dentry);
326 return dentry; 532 spin_unlock(&dentry->d_lock);
327} 533}
328 534
329struct dentry * dget_locked(struct dentry *dentry) 535struct dentry *dget_parent(struct dentry *dentry)
330{ 536{
331 return __dget_locked(dentry); 537 struct dentry *ret;
538
539repeat:
540 /*
541 * Don't need rcu_dereference because we re-check it was correct under
542 * the lock.
543 */
544 rcu_read_lock();
545 ret = dentry->d_parent;
546 if (!ret) {
547 rcu_read_unlock();
548 goto out;
549 }
550 spin_lock(&ret->d_lock);
551 if (unlikely(ret != dentry->d_parent)) {
552 spin_unlock(&ret->d_lock);
553 rcu_read_unlock();
554 goto repeat;
555 }
556 rcu_read_unlock();
557 BUG_ON(!ret->d_count);
558 ret->d_count++;
559 spin_unlock(&ret->d_lock);
560out:
561 return ret;
332} 562}
333EXPORT_SYMBOL(dget_locked); 563EXPORT_SYMBOL(dget_parent);
334 564
335/** 565/**
336 * d_find_alias - grab a hashed alias of inode 566 * d_find_alias - grab a hashed alias of inode
@@ -348,42 +578,51 @@ EXPORT_SYMBOL(dget_locked);
348 * any other hashed alias over that one unless @want_discon is set, 578 * any other hashed alias over that one unless @want_discon is set,
349 * in which case only return an IS_ROOT, DCACHE_DISCONNECTED alias. 579 * in which case only return an IS_ROOT, DCACHE_DISCONNECTED alias.
350 */ 580 */
351 581static struct dentry *__d_find_alias(struct inode *inode, int want_discon)
352static struct dentry * __d_find_alias(struct inode *inode, int want_discon)
353{ 582{
354 struct list_head *head, *next, *tmp; 583 struct dentry *alias, *discon_alias;
355 struct dentry *alias, *discon_alias=NULL;
356 584
357 head = &inode->i_dentry; 585again:
358 next = inode->i_dentry.next; 586 discon_alias = NULL;
359 while (next != head) { 587 list_for_each_entry(alias, &inode->i_dentry, d_alias) {
360 tmp = next; 588 spin_lock(&alias->d_lock);
361 next = tmp->next;
362 prefetch(next);
363 alias = list_entry(tmp, struct dentry, d_alias);
364 if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) { 589 if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
365 if (IS_ROOT(alias) && 590 if (IS_ROOT(alias) &&
366 (alias->d_flags & DCACHE_DISCONNECTED)) 591 (alias->d_flags & DCACHE_DISCONNECTED)) {
367 discon_alias = alias; 592 discon_alias = alias;
368 else if (!want_discon) { 593 } else if (!want_discon) {
369 __dget_locked(alias); 594 __dget_dlock(alias);
595 spin_unlock(&alias->d_lock);
596 return alias;
597 }
598 }
599 spin_unlock(&alias->d_lock);
600 }
601 if (discon_alias) {
602 alias = discon_alias;
603 spin_lock(&alias->d_lock);
604 if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) {
605 if (IS_ROOT(alias) &&
606 (alias->d_flags & DCACHE_DISCONNECTED)) {
607 __dget_dlock(alias);
608 spin_unlock(&alias->d_lock);
370 return alias; 609 return alias;
371 } 610 }
372 } 611 }
612 spin_unlock(&alias->d_lock);
613 goto again;
373 } 614 }
374 if (discon_alias) 615 return NULL;
375 __dget_locked(discon_alias);
376 return discon_alias;
377} 616}
378 617
379struct dentry * d_find_alias(struct inode *inode) 618struct dentry *d_find_alias(struct inode *inode)
380{ 619{
381 struct dentry *de = NULL; 620 struct dentry *de = NULL;
382 621
383 if (!list_empty(&inode->i_dentry)) { 622 if (!list_empty(&inode->i_dentry)) {
384 spin_lock(&dcache_lock); 623 spin_lock(&inode->i_lock);
385 de = __d_find_alias(inode, 0); 624 de = __d_find_alias(inode, 0);
386 spin_unlock(&dcache_lock); 625 spin_unlock(&inode->i_lock);
387 } 626 }
388 return de; 627 return de;
389} 628}
@@ -397,132 +636,153 @@ void d_prune_aliases(struct inode *inode)
397{ 636{
398 struct dentry *dentry; 637 struct dentry *dentry;
399restart: 638restart:
400 spin_lock(&dcache_lock); 639 spin_lock(&inode->i_lock);
401 list_for_each_entry(dentry, &inode->i_dentry, d_alias) { 640 list_for_each_entry(dentry, &inode->i_dentry, d_alias) {
402 spin_lock(&dentry->d_lock); 641 spin_lock(&dentry->d_lock);
403 if (!atomic_read(&dentry->d_count)) { 642 if (!dentry->d_count) {
404 __dget_locked(dentry); 643 __dget_dlock(dentry);
405 __d_drop(dentry); 644 __d_drop(dentry);
406 spin_unlock(&dentry->d_lock); 645 spin_unlock(&dentry->d_lock);
407 spin_unlock(&dcache_lock); 646 spin_unlock(&inode->i_lock);
408 dput(dentry); 647 dput(dentry);
409 goto restart; 648 goto restart;
410 } 649 }
411 spin_unlock(&dentry->d_lock); 650 spin_unlock(&dentry->d_lock);
412 } 651 }
413 spin_unlock(&dcache_lock); 652 spin_unlock(&inode->i_lock);
414} 653}
415EXPORT_SYMBOL(d_prune_aliases); 654EXPORT_SYMBOL(d_prune_aliases);
416 655
417/* 656/*
418 * Throw away a dentry - free the inode, dput the parent. This requires that 657 * Try to throw away a dentry - free the inode, dput the parent.
419 * the LRU list has already been removed. 658 * Requires dentry->d_lock is held, and dentry->d_count == 0.
659 * Releases dentry->d_lock.
420 * 660 *
421 * Try to prune ancestors as well. This is necessary to prevent 661 * This may fail if locks cannot be acquired no problem, just try again.
422 * quadratic behavior of shrink_dcache_parent(), but is also expected
423 * to be beneficial in reducing dentry cache fragmentation.
424 */ 662 */
425static void prune_one_dentry(struct dentry * dentry) 663static void try_prune_one_dentry(struct dentry *dentry)
426 __releases(dentry->d_lock) 664 __releases(dentry->d_lock)
427 __releases(dcache_lock)
428 __acquires(dcache_lock)
429{ 665{
430 __d_drop(dentry); 666 struct dentry *parent;
431 dentry = d_kill(dentry);
432 667
668 parent = dentry_kill(dentry, 0);
433 /* 669 /*
434 * Prune ancestors. Locking is simpler than in dput(), 670 * If dentry_kill returns NULL, we have nothing more to do.
435 * because dcache_lock needs to be taken anyway. 671 * if it returns the same dentry, trylocks failed. In either
672 * case, just loop again.
673 *
674 * Otherwise, we need to prune ancestors too. This is necessary
675 * to prevent quadratic behavior of shrink_dcache_parent(), but
676 * is also expected to be beneficial in reducing dentry cache
677 * fragmentation.
436 */ 678 */
437 spin_lock(&dcache_lock); 679 if (!parent)
680 return;
681 if (parent == dentry)
682 return;
683
684 /* Prune ancestors. */
685 dentry = parent;
438 while (dentry) { 686 while (dentry) {
439 if (!atomic_dec_and_lock(&dentry->d_count, &dentry->d_lock)) 687 spin_lock(&dentry->d_lock);
688 if (dentry->d_count > 1) {
689 dentry->d_count--;
690 spin_unlock(&dentry->d_lock);
440 return; 691 return;
441 692 }
442 if (dentry->d_op && dentry->d_op->d_delete) 693 dentry = dentry_kill(dentry, 1);
443 dentry->d_op->d_delete(dentry);
444 dentry_lru_del_init(dentry);
445 __d_drop(dentry);
446 dentry = d_kill(dentry);
447 spin_lock(&dcache_lock);
448 } 694 }
449} 695}
450 696
451/* 697static void shrink_dentry_list(struct list_head *list)
452 * Shrink the dentry LRU on a given superblock.
453 * @sb : superblock to shrink dentry LRU.
454 * @count: If count is NULL, we prune all dentries on superblock.
455 * @flags: If flags is non-zero, we need to do special processing based on
456 * which flags are set. This means we don't need to maintain multiple
457 * similar copies of this loop.
458 */
459static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags)
460{ 698{
461 LIST_HEAD(referenced);
462 LIST_HEAD(tmp);
463 struct dentry *dentry; 699 struct dentry *dentry;
464 int cnt = 0;
465
466 BUG_ON(!sb);
467 BUG_ON((flags & DCACHE_REFERENCED) && count == NULL);
468 spin_lock(&dcache_lock);
469 if (count != NULL)
470 /* called from prune_dcache() and shrink_dcache_parent() */
471 cnt = *count;
472restart:
473 if (count == NULL)
474 list_splice_init(&sb->s_dentry_lru, &tmp);
475 else {
476 while (!list_empty(&sb->s_dentry_lru)) {
477 dentry = list_entry(sb->s_dentry_lru.prev,
478 struct dentry, d_lru);
479 BUG_ON(dentry->d_sb != sb);
480 700
481 spin_lock(&dentry->d_lock); 701 rcu_read_lock();
482 /* 702 for (;;) {
483 * If we are honouring the DCACHE_REFERENCED flag and 703 dentry = list_entry_rcu(list->prev, struct dentry, d_lru);
484 * the dentry has this flag set, don't free it. Clear 704 if (&dentry->d_lru == list)
485 * the flag and put it back on the LRU. 705 break; /* empty */
486 */
487 if ((flags & DCACHE_REFERENCED)
488 && (dentry->d_flags & DCACHE_REFERENCED)) {
489 dentry->d_flags &= ~DCACHE_REFERENCED;
490 list_move(&dentry->d_lru, &referenced);
491 spin_unlock(&dentry->d_lock);
492 } else {
493 list_move_tail(&dentry->d_lru, &tmp);
494 spin_unlock(&dentry->d_lock);
495 cnt--;
496 if (!cnt)
497 break;
498 }
499 cond_resched_lock(&dcache_lock);
500 }
501 }
502 while (!list_empty(&tmp)) {
503 dentry = list_entry(tmp.prev, struct dentry, d_lru);
504 dentry_lru_del_init(dentry);
505 spin_lock(&dentry->d_lock); 706 spin_lock(&dentry->d_lock);
707 if (dentry != list_entry(list->prev, struct dentry, d_lru)) {
708 spin_unlock(&dentry->d_lock);
709 continue;
710 }
711
506 /* 712 /*
507 * We found an inuse dentry which was not removed from 713 * We found an inuse dentry which was not removed from
508 * the LRU because of laziness during lookup. Do not free 714 * the LRU because of laziness during lookup. Do not free
509 * it - just keep it off the LRU list. 715 * it - just keep it off the LRU list.
510 */ 716 */
511 if (atomic_read(&dentry->d_count)) { 717 if (dentry->d_count) {
718 dentry_lru_del(dentry);
512 spin_unlock(&dentry->d_lock); 719 spin_unlock(&dentry->d_lock);
513 continue; 720 continue;
514 } 721 }
515 prune_one_dentry(dentry); 722
516 /* dentry->d_lock was dropped in prune_one_dentry() */ 723 rcu_read_unlock();
517 cond_resched_lock(&dcache_lock); 724
518 } 725 try_prune_one_dentry(dentry);
519 if (count == NULL && !list_empty(&sb->s_dentry_lru)) 726
520 goto restart; 727 rcu_read_lock();
521 if (count != NULL) 728 }
522 *count = cnt; 729 rcu_read_unlock();
730}
731
732/**
733 * __shrink_dcache_sb - shrink the dentry LRU on a given superblock
734 * @sb: superblock to shrink dentry LRU.
735 * @count: number of entries to prune
736 * @flags: flags to control the dentry processing
737 *
738 * If flags contains DCACHE_REFERENCED reference dentries will not be pruned.
739 */
740static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags)
741{
742 /* called from prune_dcache() and shrink_dcache_parent() */
743 struct dentry *dentry;
744 LIST_HEAD(referenced);
745 LIST_HEAD(tmp);
746 int cnt = *count;
747
748relock:
749 spin_lock(&dcache_lru_lock);
750 while (!list_empty(&sb->s_dentry_lru)) {
751 dentry = list_entry(sb->s_dentry_lru.prev,
752 struct dentry, d_lru);
753 BUG_ON(dentry->d_sb != sb);
754
755 if (!spin_trylock(&dentry->d_lock)) {
756 spin_unlock(&dcache_lru_lock);
757 cpu_relax();
758 goto relock;
759 }
760
761 /*
762 * If we are honouring the DCACHE_REFERENCED flag and the
763 * dentry has this flag set, don't free it. Clear the flag
764 * and put it back on the LRU.
765 */
766 if (flags & DCACHE_REFERENCED &&
767 dentry->d_flags & DCACHE_REFERENCED) {
768 dentry->d_flags &= ~DCACHE_REFERENCED;
769 list_move(&dentry->d_lru, &referenced);
770 spin_unlock(&dentry->d_lock);
771 } else {
772 list_move_tail(&dentry->d_lru, &tmp);
773 spin_unlock(&dentry->d_lock);
774 if (!--cnt)
775 break;
776 }
777 cond_resched_lock(&dcache_lru_lock);
778 }
523 if (!list_empty(&referenced)) 779 if (!list_empty(&referenced))
524 list_splice(&referenced, &sb->s_dentry_lru); 780 list_splice(&referenced, &sb->s_dentry_lru);
525 spin_unlock(&dcache_lock); 781 spin_unlock(&dcache_lru_lock);
782
783 shrink_dentry_list(&tmp);
784
785 *count = cnt;
526} 786}
527 787
528/** 788/**
@@ -544,7 +804,6 @@ static void prune_dcache(int count)
544 804
545 if (unused == 0 || count == 0) 805 if (unused == 0 || count == 0)
546 return; 806 return;
547 spin_lock(&dcache_lock);
548 if (count >= unused) 807 if (count >= unused)
549 prune_ratio = 1; 808 prune_ratio = 1;
550 else 809 else
@@ -581,11 +840,9 @@ static void prune_dcache(int count)
581 if (down_read_trylock(&sb->s_umount)) { 840 if (down_read_trylock(&sb->s_umount)) {
582 if ((sb->s_root != NULL) && 841 if ((sb->s_root != NULL) &&
583 (!list_empty(&sb->s_dentry_lru))) { 842 (!list_empty(&sb->s_dentry_lru))) {
584 spin_unlock(&dcache_lock);
585 __shrink_dcache_sb(sb, &w_count, 843 __shrink_dcache_sb(sb, &w_count,
586 DCACHE_REFERENCED); 844 DCACHE_REFERENCED);
587 pruned -= w_count; 845 pruned -= w_count;
588 spin_lock(&dcache_lock);
589 } 846 }
590 up_read(&sb->s_umount); 847 up_read(&sb->s_umount);
591 } 848 }
@@ -601,20 +858,27 @@ static void prune_dcache(int count)
601 if (p) 858 if (p)
602 __put_super(p); 859 __put_super(p);
603 spin_unlock(&sb_lock); 860 spin_unlock(&sb_lock);
604 spin_unlock(&dcache_lock);
605} 861}
606 862
607/** 863/**
608 * shrink_dcache_sb - shrink dcache for a superblock 864 * shrink_dcache_sb - shrink dcache for a superblock
609 * @sb: superblock 865 * @sb: superblock
610 * 866 *
611 * Shrink the dcache for the specified super block. This 867 * Shrink the dcache for the specified super block. This is used to free
612 * is used to free the dcache before unmounting a file 868 * the dcache before unmounting a file system.
613 * system
614 */ 869 */
615void shrink_dcache_sb(struct super_block * sb) 870void shrink_dcache_sb(struct super_block *sb)
616{ 871{
617 __shrink_dcache_sb(sb, NULL, 0); 872 LIST_HEAD(tmp);
873
874 spin_lock(&dcache_lru_lock);
875 while (!list_empty(&sb->s_dentry_lru)) {
876 list_splice_init(&sb->s_dentry_lru, &tmp);
877 spin_unlock(&dcache_lru_lock);
878 shrink_dentry_list(&tmp);
879 spin_lock(&dcache_lru_lock);
880 }
881 spin_unlock(&dcache_lru_lock);
618} 882}
619EXPORT_SYMBOL(shrink_dcache_sb); 883EXPORT_SYMBOL(shrink_dcache_sb);
620 884
@@ -631,10 +895,10 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
631 BUG_ON(!IS_ROOT(dentry)); 895 BUG_ON(!IS_ROOT(dentry));
632 896
633 /* detach this root from the system */ 897 /* detach this root from the system */
634 spin_lock(&dcache_lock); 898 spin_lock(&dentry->d_lock);
635 dentry_lru_del_init(dentry); 899 dentry_lru_del(dentry);
636 __d_drop(dentry); 900 __d_drop(dentry);
637 spin_unlock(&dcache_lock); 901 spin_unlock(&dentry->d_lock);
638 902
639 for (;;) { 903 for (;;) {
640 /* descend to the first leaf in the current subtree */ 904 /* descend to the first leaf in the current subtree */
@@ -643,14 +907,16 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
643 907
644 /* this is a branch with children - detach all of them 908 /* this is a branch with children - detach all of them
645 * from the system in one go */ 909 * from the system in one go */
646 spin_lock(&dcache_lock); 910 spin_lock(&dentry->d_lock);
647 list_for_each_entry(loop, &dentry->d_subdirs, 911 list_for_each_entry(loop, &dentry->d_subdirs,
648 d_u.d_child) { 912 d_u.d_child) {
649 dentry_lru_del_init(loop); 913 spin_lock_nested(&loop->d_lock,
914 DENTRY_D_LOCK_NESTED);
915 dentry_lru_del(loop);
650 __d_drop(loop); 916 __d_drop(loop);
651 cond_resched_lock(&dcache_lock); 917 spin_unlock(&loop->d_lock);
652 } 918 }
653 spin_unlock(&dcache_lock); 919 spin_unlock(&dentry->d_lock);
654 920
655 /* move to the first child */ 921 /* move to the first child */
656 dentry = list_entry(dentry->d_subdirs.next, 922 dentry = list_entry(dentry->d_subdirs.next,
@@ -662,7 +928,7 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
662 do { 928 do {
663 struct inode *inode; 929 struct inode *inode;
664 930
665 if (atomic_read(&dentry->d_count) != 0) { 931 if (dentry->d_count != 0) {
666 printk(KERN_ERR 932 printk(KERN_ERR
667 "BUG: Dentry %p{i=%lx,n=%s}" 933 "BUG: Dentry %p{i=%lx,n=%s}"
668 " still in use (%d)" 934 " still in use (%d)"
@@ -671,20 +937,23 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
671 dentry->d_inode ? 937 dentry->d_inode ?
672 dentry->d_inode->i_ino : 0UL, 938 dentry->d_inode->i_ino : 0UL,
673 dentry->d_name.name, 939 dentry->d_name.name,
674 atomic_read(&dentry->d_count), 940 dentry->d_count,
675 dentry->d_sb->s_type->name, 941 dentry->d_sb->s_type->name,
676 dentry->d_sb->s_id); 942 dentry->d_sb->s_id);
677 BUG(); 943 BUG();
678 } 944 }
679 945
680 if (IS_ROOT(dentry)) 946 if (IS_ROOT(dentry)) {
681 parent = NULL; 947 parent = NULL;
682 else { 948 list_del(&dentry->d_u.d_child);
949 } else {
683 parent = dentry->d_parent; 950 parent = dentry->d_parent;
684 atomic_dec(&parent->d_count); 951 spin_lock(&parent->d_lock);
952 parent->d_count--;
953 list_del(&dentry->d_u.d_child);
954 spin_unlock(&parent->d_lock);
685 } 955 }
686 956
687 list_del(&dentry->d_u.d_child);
688 detached++; 957 detached++;
689 958
690 inode = dentry->d_inode; 959 inode = dentry->d_inode;
@@ -703,26 +972,18 @@ static void shrink_dcache_for_umount_subtree(struct dentry *dentry)
703 * otherwise we ascend to the parent and move to the 972 * otherwise we ascend to the parent and move to the
704 * next sibling if there is one */ 973 * next sibling if there is one */
705 if (!parent) 974 if (!parent)
706 goto out; 975 return;
707
708 dentry = parent; 976 dentry = parent;
709
710 } while (list_empty(&dentry->d_subdirs)); 977 } while (list_empty(&dentry->d_subdirs));
711 978
712 dentry = list_entry(dentry->d_subdirs.next, 979 dentry = list_entry(dentry->d_subdirs.next,
713 struct dentry, d_u.d_child); 980 struct dentry, d_u.d_child);
714 } 981 }
715out:
716 /* several dentries were freed, need to correct nr_dentry */
717 spin_lock(&dcache_lock);
718 dentry_stat.nr_dentry -= detached;
719 spin_unlock(&dcache_lock);
720} 982}
721 983
722/* 984/*
723 * destroy the dentries attached to a superblock on unmounting 985 * destroy the dentries attached to a superblock on unmounting
724 * - we don't need to use dentry->d_lock, and only need dcache_lock when 986 * - we don't need to use dentry->d_lock because:
725 * removing the dentry from the system lists and hashes because:
726 * - the superblock is detached from all mountings and open files, so the 987 * - the superblock is detached from all mountings and open files, so the
727 * dentry trees will not be rearranged by the VFS 988 * dentry trees will not be rearranged by the VFS
728 * - s_umount is write-locked, so the memory pressure shrinker will ignore 989 * - s_umount is write-locked, so the memory pressure shrinker will ignore
@@ -739,11 +1000,13 @@ void shrink_dcache_for_umount(struct super_block *sb)
739 1000
740 dentry = sb->s_root; 1001 dentry = sb->s_root;
741 sb->s_root = NULL; 1002 sb->s_root = NULL;
742 atomic_dec(&dentry->d_count); 1003 spin_lock(&dentry->d_lock);
1004 dentry->d_count--;
1005 spin_unlock(&dentry->d_lock);
743 shrink_dcache_for_umount_subtree(dentry); 1006 shrink_dcache_for_umount_subtree(dentry);
744 1007
745 while (!hlist_empty(&sb->s_anon)) { 1008 while (!hlist_bl_empty(&sb->s_anon)) {
746 dentry = hlist_entry(sb->s_anon.first, struct dentry, d_hash); 1009 dentry = hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash);
747 shrink_dcache_for_umount_subtree(dentry); 1010 shrink_dcache_for_umount_subtree(dentry);
748 } 1011 }
749} 1012}
@@ -761,15 +1024,20 @@ void shrink_dcache_for_umount(struct super_block *sb)
761 * Return true if the parent or its subdirectories contain 1024 * Return true if the parent or its subdirectories contain
762 * a mount point 1025 * a mount point
763 */ 1026 */
764
765int have_submounts(struct dentry *parent) 1027int have_submounts(struct dentry *parent)
766{ 1028{
767 struct dentry *this_parent = parent; 1029 struct dentry *this_parent;
768 struct list_head *next; 1030 struct list_head *next;
1031 unsigned seq;
1032 int locked = 0;
1033
1034 seq = read_seqbegin(&rename_lock);
1035again:
1036 this_parent = parent;
769 1037
770 spin_lock(&dcache_lock);
771 if (d_mountpoint(parent)) 1038 if (d_mountpoint(parent))
772 goto positive; 1039 goto positive;
1040 spin_lock(&this_parent->d_lock);
773repeat: 1041repeat:
774 next = this_parent->d_subdirs.next; 1042 next = this_parent->d_subdirs.next;
775resume: 1043resume:
@@ -777,27 +1045,65 @@ resume:
777 struct list_head *tmp = next; 1045 struct list_head *tmp = next;
778 struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child); 1046 struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
779 next = tmp->next; 1047 next = tmp->next;
1048
1049 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
780 /* Have we found a mount point ? */ 1050 /* Have we found a mount point ? */
781 if (d_mountpoint(dentry)) 1051 if (d_mountpoint(dentry)) {
1052 spin_unlock(&dentry->d_lock);
1053 spin_unlock(&this_parent->d_lock);
782 goto positive; 1054 goto positive;
1055 }
783 if (!list_empty(&dentry->d_subdirs)) { 1056 if (!list_empty(&dentry->d_subdirs)) {
1057 spin_unlock(&this_parent->d_lock);
1058 spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
784 this_parent = dentry; 1059 this_parent = dentry;
1060 spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
785 goto repeat; 1061 goto repeat;
786 } 1062 }
1063 spin_unlock(&dentry->d_lock);
787 } 1064 }
788 /* 1065 /*
789 * All done at this level ... ascend and resume the search. 1066 * All done at this level ... ascend and resume the search.
790 */ 1067 */
791 if (this_parent != parent) { 1068 if (this_parent != parent) {
792 next = this_parent->d_u.d_child.next; 1069 struct dentry *tmp;
793 this_parent = this_parent->d_parent; 1070 struct dentry *child;
1071
1072 tmp = this_parent->d_parent;
1073 rcu_read_lock();
1074 spin_unlock(&this_parent->d_lock);
1075 child = this_parent;
1076 this_parent = tmp;
1077 spin_lock(&this_parent->d_lock);
1078 /* might go back up the wrong parent if we have had a rename
1079 * or deletion */
1080 if (this_parent != child->d_parent ||
1081 (!locked && read_seqretry(&rename_lock, seq))) {
1082 spin_unlock(&this_parent->d_lock);
1083 rcu_read_unlock();
1084 goto rename_retry;
1085 }
1086 rcu_read_unlock();
1087 next = child->d_u.d_child.next;
794 goto resume; 1088 goto resume;
795 } 1089 }
796 spin_unlock(&dcache_lock); 1090 spin_unlock(&this_parent->d_lock);
1091 if (!locked && read_seqretry(&rename_lock, seq))
1092 goto rename_retry;
1093 if (locked)
1094 write_sequnlock(&rename_lock);
797 return 0; /* No mount points found in tree */ 1095 return 0; /* No mount points found in tree */
798positive: 1096positive:
799 spin_unlock(&dcache_lock); 1097 if (!locked && read_seqretry(&rename_lock, seq))
1098 goto rename_retry;
1099 if (locked)
1100 write_sequnlock(&rename_lock);
800 return 1; 1101 return 1;
1102
1103rename_retry:
1104 locked = 1;
1105 write_seqlock(&rename_lock);
1106 goto again;
801} 1107}
802EXPORT_SYMBOL(have_submounts); 1108EXPORT_SYMBOL(have_submounts);
803 1109
@@ -817,11 +1123,16 @@ EXPORT_SYMBOL(have_submounts);
817 */ 1123 */
818static int select_parent(struct dentry * parent) 1124static int select_parent(struct dentry * parent)
819{ 1125{
820 struct dentry *this_parent = parent; 1126 struct dentry *this_parent;
821 struct list_head *next; 1127 struct list_head *next;
1128 unsigned seq;
822 int found = 0; 1129 int found = 0;
1130 int locked = 0;
823 1131
824 spin_lock(&dcache_lock); 1132 seq = read_seqbegin(&rename_lock);
1133again:
1134 this_parent = parent;
1135 spin_lock(&this_parent->d_lock);
825repeat: 1136repeat:
826 next = this_parent->d_subdirs.next; 1137 next = this_parent->d_subdirs.next;
827resume: 1138resume:
@@ -830,14 +1141,17 @@ resume:
830 struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child); 1141 struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
831 next = tmp->next; 1142 next = tmp->next;
832 1143
833 dentry_lru_del_init(dentry); 1144 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1145
834 /* 1146 /*
835 * move only zero ref count dentries to the end 1147 * move only zero ref count dentries to the end
836 * of the unused list for prune_dcache 1148 * of the unused list for prune_dcache
837 */ 1149 */
838 if (!atomic_read(&dentry->d_count)) { 1150 if (!dentry->d_count) {
839 dentry_lru_add_tail(dentry); 1151 dentry_lru_move_tail(dentry);
840 found++; 1152 found++;
1153 } else {
1154 dentry_lru_del(dentry);
841 } 1155 }
842 1156
843 /* 1157 /*
@@ -845,28 +1159,63 @@ resume:
845 * ensures forward progress). We'll be coming back to find 1159 * ensures forward progress). We'll be coming back to find
846 * the rest. 1160 * the rest.
847 */ 1161 */
848 if (found && need_resched()) 1162 if (found && need_resched()) {
1163 spin_unlock(&dentry->d_lock);
849 goto out; 1164 goto out;
1165 }
850 1166
851 /* 1167 /*
852 * Descend a level if the d_subdirs list is non-empty. 1168 * Descend a level if the d_subdirs list is non-empty.
853 */ 1169 */
854 if (!list_empty(&dentry->d_subdirs)) { 1170 if (!list_empty(&dentry->d_subdirs)) {
1171 spin_unlock(&this_parent->d_lock);
1172 spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
855 this_parent = dentry; 1173 this_parent = dentry;
1174 spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
856 goto repeat; 1175 goto repeat;
857 } 1176 }
1177
1178 spin_unlock(&dentry->d_lock);
858 } 1179 }
859 /* 1180 /*
860 * All done at this level ... ascend and resume the search. 1181 * All done at this level ... ascend and resume the search.
861 */ 1182 */
862 if (this_parent != parent) { 1183 if (this_parent != parent) {
863 next = this_parent->d_u.d_child.next; 1184 struct dentry *tmp;
864 this_parent = this_parent->d_parent; 1185 struct dentry *child;
1186
1187 tmp = this_parent->d_parent;
1188 rcu_read_lock();
1189 spin_unlock(&this_parent->d_lock);
1190 child = this_parent;
1191 this_parent = tmp;
1192 spin_lock(&this_parent->d_lock);
1193 /* might go back up the wrong parent if we have had a rename
1194 * or deletion */
1195 if (this_parent != child->d_parent ||
1196 (!locked && read_seqretry(&rename_lock, seq))) {
1197 spin_unlock(&this_parent->d_lock);
1198 rcu_read_unlock();
1199 goto rename_retry;
1200 }
1201 rcu_read_unlock();
1202 next = child->d_u.d_child.next;
865 goto resume; 1203 goto resume;
866 } 1204 }
867out: 1205out:
868 spin_unlock(&dcache_lock); 1206 spin_unlock(&this_parent->d_lock);
1207 if (!locked && read_seqretry(&rename_lock, seq))
1208 goto rename_retry;
1209 if (locked)
1210 write_sequnlock(&rename_lock);
869 return found; 1211 return found;
1212
1213rename_retry:
1214 if (found)
1215 return found;
1216 locked = 1;
1217 write_seqlock(&rename_lock);
1218 goto again;
870} 1219}
871 1220
872/** 1221/**
@@ -905,6 +1254,7 @@ static int shrink_dcache_memory(struct shrinker *shrink, int nr, gfp_t gfp_mask)
905 return -1; 1254 return -1;
906 prune_dcache(nr); 1255 prune_dcache(nr);
907 } 1256 }
1257
908 return (dentry_stat.nr_unused / 100) * sysctl_vfs_cache_pressure; 1258 return (dentry_stat.nr_unused / 100) * sysctl_vfs_cache_pressure;
909} 1259}
910 1260
@@ -948,37 +1298,54 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
948 memcpy(dname, name->name, name->len); 1298 memcpy(dname, name->name, name->len);
949 dname[name->len] = 0; 1299 dname[name->len] = 0;
950 1300
951 atomic_set(&dentry->d_count, 1); 1301 dentry->d_count = 1;
952 dentry->d_flags = DCACHE_UNHASHED; 1302 dentry->d_flags = DCACHE_UNHASHED;
953 spin_lock_init(&dentry->d_lock); 1303 spin_lock_init(&dentry->d_lock);
1304 seqcount_init(&dentry->d_seq);
954 dentry->d_inode = NULL; 1305 dentry->d_inode = NULL;
955 dentry->d_parent = NULL; 1306 dentry->d_parent = NULL;
956 dentry->d_sb = NULL; 1307 dentry->d_sb = NULL;
957 dentry->d_op = NULL; 1308 dentry->d_op = NULL;
958 dentry->d_fsdata = NULL; 1309 dentry->d_fsdata = NULL;
959 dentry->d_mounted = 0; 1310 INIT_HLIST_BL_NODE(&dentry->d_hash);
960 INIT_HLIST_NODE(&dentry->d_hash);
961 INIT_LIST_HEAD(&dentry->d_lru); 1311 INIT_LIST_HEAD(&dentry->d_lru);
962 INIT_LIST_HEAD(&dentry->d_subdirs); 1312 INIT_LIST_HEAD(&dentry->d_subdirs);
963 INIT_LIST_HEAD(&dentry->d_alias); 1313 INIT_LIST_HEAD(&dentry->d_alias);
1314 INIT_LIST_HEAD(&dentry->d_u.d_child);
964 1315
965 if (parent) { 1316 if (parent) {
966 dentry->d_parent = dget(parent); 1317 spin_lock(&parent->d_lock);
1318 /*
1319 * don't need child lock because it is not subject
1320 * to concurrency here
1321 */
1322 __dget_dlock(parent);
1323 dentry->d_parent = parent;
967 dentry->d_sb = parent->d_sb; 1324 dentry->d_sb = parent->d_sb;
968 } else { 1325 d_set_d_op(dentry, dentry->d_sb->s_d_op);
969 INIT_LIST_HEAD(&dentry->d_u.d_child); 1326 list_add(&dentry->d_u.d_child, &parent->d_subdirs);
1327 spin_unlock(&parent->d_lock);
970 } 1328 }
971 1329
972 spin_lock(&dcache_lock); 1330 this_cpu_inc(nr_dentry);
973 if (parent)
974 list_add(&dentry->d_u.d_child, &parent->d_subdirs);
975 dentry_stat.nr_dentry++;
976 spin_unlock(&dcache_lock);
977 1331
978 return dentry; 1332 return dentry;
979} 1333}
980EXPORT_SYMBOL(d_alloc); 1334EXPORT_SYMBOL(d_alloc);
981 1335
1336struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
1337{
1338 struct dentry *dentry = d_alloc(NULL, name);
1339 if (dentry) {
1340 dentry->d_sb = sb;
1341 d_set_d_op(dentry, dentry->d_sb->s_d_op);
1342 dentry->d_parent = dentry;
1343 dentry->d_flags |= DCACHE_DISCONNECTED;
1344 }
1345 return dentry;
1346}
1347EXPORT_SYMBOL(d_alloc_pseudo);
1348
982struct dentry *d_alloc_name(struct dentry *parent, const char *name) 1349struct dentry *d_alloc_name(struct dentry *parent, const char *name)
983{ 1350{
984 struct qstr q; 1351 struct qstr q;
@@ -990,12 +1357,39 @@ struct dentry *d_alloc_name(struct dentry *parent, const char *name)
990} 1357}
991EXPORT_SYMBOL(d_alloc_name); 1358EXPORT_SYMBOL(d_alloc_name);
992 1359
993/* the caller must hold dcache_lock */ 1360void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
1361{
1362 WARN_ON_ONCE(dentry->d_op);
1363 WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
1364 DCACHE_OP_COMPARE |
1365 DCACHE_OP_REVALIDATE |
1366 DCACHE_OP_DELETE ));
1367 dentry->d_op = op;
1368 if (!op)
1369 return;
1370 if (op->d_hash)
1371 dentry->d_flags |= DCACHE_OP_HASH;
1372 if (op->d_compare)
1373 dentry->d_flags |= DCACHE_OP_COMPARE;
1374 if (op->d_revalidate)
1375 dentry->d_flags |= DCACHE_OP_REVALIDATE;
1376 if (op->d_delete)
1377 dentry->d_flags |= DCACHE_OP_DELETE;
1378
1379}
1380EXPORT_SYMBOL(d_set_d_op);
1381
994static void __d_instantiate(struct dentry *dentry, struct inode *inode) 1382static void __d_instantiate(struct dentry *dentry, struct inode *inode)
995{ 1383{
996 if (inode) 1384 spin_lock(&dentry->d_lock);
1385 if (inode) {
1386 if (unlikely(IS_AUTOMOUNT(inode)))
1387 dentry->d_flags |= DCACHE_NEED_AUTOMOUNT;
997 list_add(&dentry->d_alias, &inode->i_dentry); 1388 list_add(&dentry->d_alias, &inode->i_dentry);
1389 }
998 dentry->d_inode = inode; 1390 dentry->d_inode = inode;
1391 dentry_rcuwalk_barrier(dentry);
1392 spin_unlock(&dentry->d_lock);
999 fsnotify_d_instantiate(dentry, inode); 1393 fsnotify_d_instantiate(dentry, inode);
1000} 1394}
1001 1395
@@ -1017,9 +1411,11 @@ static void __d_instantiate(struct dentry *dentry, struct inode *inode)
1017void d_instantiate(struct dentry *entry, struct inode * inode) 1411void d_instantiate(struct dentry *entry, struct inode * inode)
1018{ 1412{
1019 BUG_ON(!list_empty(&entry->d_alias)); 1413 BUG_ON(!list_empty(&entry->d_alias));
1020 spin_lock(&dcache_lock); 1414 if (inode)
1415 spin_lock(&inode->i_lock);
1021 __d_instantiate(entry, inode); 1416 __d_instantiate(entry, inode);
1022 spin_unlock(&dcache_lock); 1417 if (inode)
1418 spin_unlock(&inode->i_lock);
1023 security_d_instantiate(entry, inode); 1419 security_d_instantiate(entry, inode);
1024} 1420}
1025EXPORT_SYMBOL(d_instantiate); 1421EXPORT_SYMBOL(d_instantiate);
@@ -1056,15 +1452,18 @@ static struct dentry *__d_instantiate_unique(struct dentry *entry,
1056 list_for_each_entry(alias, &inode->i_dentry, d_alias) { 1452 list_for_each_entry(alias, &inode->i_dentry, d_alias) {
1057 struct qstr *qstr = &alias->d_name; 1453 struct qstr *qstr = &alias->d_name;
1058 1454
1455 /*
1456 * Don't need alias->d_lock here, because aliases with
1457 * d_parent == entry->d_parent are not subject to name or
1458 * parent changes, because the parent inode i_mutex is held.
1459 */
1059 if (qstr->hash != hash) 1460 if (qstr->hash != hash)
1060 continue; 1461 continue;
1061 if (alias->d_parent != entry->d_parent) 1462 if (alias->d_parent != entry->d_parent)
1062 continue; 1463 continue;
1063 if (qstr->len != len) 1464 if (dentry_cmp(qstr->name, qstr->len, name, len))
1064 continue; 1465 continue;
1065 if (memcmp(qstr->name, name, len)) 1466 __dget(alias);
1066 continue;
1067 dget_locked(alias);
1068 return alias; 1467 return alias;
1069 } 1468 }
1070 1469
@@ -1078,9 +1477,11 @@ struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode)
1078 1477
1079 BUG_ON(!list_empty(&entry->d_alias)); 1478 BUG_ON(!list_empty(&entry->d_alias));
1080 1479
1081 spin_lock(&dcache_lock); 1480 if (inode)
1481 spin_lock(&inode->i_lock);
1082 result = __d_instantiate_unique(entry, inode); 1482 result = __d_instantiate_unique(entry, inode);
1083 spin_unlock(&dcache_lock); 1483 if (inode)
1484 spin_unlock(&inode->i_lock);
1084 1485
1085 if (!result) { 1486 if (!result) {
1086 security_d_instantiate(entry, inode); 1487 security_d_instantiate(entry, inode);
@@ -1113,6 +1514,7 @@ struct dentry * d_alloc_root(struct inode * root_inode)
1113 res = d_alloc(NULL, &name); 1514 res = d_alloc(NULL, &name);
1114 if (res) { 1515 if (res) {
1115 res->d_sb = root_inode->i_sb; 1516 res->d_sb = root_inode->i_sb;
1517 d_set_d_op(res, res->d_sb->s_d_op);
1116 res->d_parent = res; 1518 res->d_parent = res;
1117 d_instantiate(res, root_inode); 1519 d_instantiate(res, root_inode);
1118 } 1520 }
@@ -1121,14 +1523,6 @@ struct dentry * d_alloc_root(struct inode * root_inode)
1121} 1523}
1122EXPORT_SYMBOL(d_alloc_root); 1524EXPORT_SYMBOL(d_alloc_root);
1123 1525
1124static inline struct hlist_head *d_hash(struct dentry *parent,
1125 unsigned long hash)
1126{
1127 hash += ((unsigned long) parent ^ GOLDEN_RATIO_PRIME) / L1_CACHE_BYTES;
1128 hash = hash ^ ((hash ^ GOLDEN_RATIO_PRIME) >> D_HASHBITS);
1129 return dentry_hashtable + (hash & D_HASHMASK);
1130}
1131
1132/** 1526/**
1133 * d_obtain_alias - find or allocate a dentry for a given inode 1527 * d_obtain_alias - find or allocate a dentry for a given inode
1134 * @inode: inode to allocate the dentry for 1528 * @inode: inode to allocate the dentry for
@@ -1169,10 +1563,11 @@ struct dentry *d_obtain_alias(struct inode *inode)
1169 } 1563 }
1170 tmp->d_parent = tmp; /* make sure dput doesn't croak */ 1564 tmp->d_parent = tmp; /* make sure dput doesn't croak */
1171 1565
1172 spin_lock(&dcache_lock); 1566
1567 spin_lock(&inode->i_lock);
1173 res = __d_find_alias(inode, 0); 1568 res = __d_find_alias(inode, 0);
1174 if (res) { 1569 if (res) {
1175 spin_unlock(&dcache_lock); 1570 spin_unlock(&inode->i_lock);
1176 dput(tmp); 1571 dput(tmp);
1177 goto out_iput; 1572 goto out_iput;
1178 } 1573 }
@@ -1180,14 +1575,17 @@ struct dentry *d_obtain_alias(struct inode *inode)
1180 /* attach a disconnected dentry */ 1575 /* attach a disconnected dentry */
1181 spin_lock(&tmp->d_lock); 1576 spin_lock(&tmp->d_lock);
1182 tmp->d_sb = inode->i_sb; 1577 tmp->d_sb = inode->i_sb;
1578 d_set_d_op(tmp, tmp->d_sb->s_d_op);
1183 tmp->d_inode = inode; 1579 tmp->d_inode = inode;
1184 tmp->d_flags |= DCACHE_DISCONNECTED; 1580 tmp->d_flags |= DCACHE_DISCONNECTED;
1185 tmp->d_flags &= ~DCACHE_UNHASHED;
1186 list_add(&tmp->d_alias, &inode->i_dentry); 1581 list_add(&tmp->d_alias, &inode->i_dentry);
1187 hlist_add_head(&tmp->d_hash, &inode->i_sb->s_anon); 1582 bit_spin_lock(0, (unsigned long *)&tmp->d_sb->s_anon.first);
1583 tmp->d_flags &= ~DCACHE_UNHASHED;
1584 hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon);
1585 __bit_spin_unlock(0, (unsigned long *)&tmp->d_sb->s_anon.first);
1188 spin_unlock(&tmp->d_lock); 1586 spin_unlock(&tmp->d_lock);
1587 spin_unlock(&inode->i_lock);
1189 1588
1190 spin_unlock(&dcache_lock);
1191 return tmp; 1589 return tmp;
1192 1590
1193 out_iput: 1591 out_iput:
@@ -1217,18 +1615,18 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
1217 struct dentry *new = NULL; 1615 struct dentry *new = NULL;
1218 1616
1219 if (inode && S_ISDIR(inode->i_mode)) { 1617 if (inode && S_ISDIR(inode->i_mode)) {
1220 spin_lock(&dcache_lock); 1618 spin_lock(&inode->i_lock);
1221 new = __d_find_alias(inode, 1); 1619 new = __d_find_alias(inode, 1);
1222 if (new) { 1620 if (new) {
1223 BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED)); 1621 BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED));
1224 spin_unlock(&dcache_lock); 1622 spin_unlock(&inode->i_lock);
1225 security_d_instantiate(new, inode); 1623 security_d_instantiate(new, inode);
1226 d_move(new, dentry); 1624 d_move(new, dentry);
1227 iput(inode); 1625 iput(inode);
1228 } else { 1626 } else {
1229 /* already taking dcache_lock, so d_add() by hand */ 1627 /* already taking inode->i_lock, so d_add() by hand */
1230 __d_instantiate(dentry, inode); 1628 __d_instantiate(dentry, inode);
1231 spin_unlock(&dcache_lock); 1629 spin_unlock(&inode->i_lock);
1232 security_d_instantiate(dentry, inode); 1630 security_d_instantiate(dentry, inode);
1233 d_rehash(dentry); 1631 d_rehash(dentry);
1234 } 1632 }
@@ -1301,10 +1699,10 @@ struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
1301 * Negative dentry: instantiate it unless the inode is a directory and 1699 * Negative dentry: instantiate it unless the inode is a directory and
1302 * already has a dentry. 1700 * already has a dentry.
1303 */ 1701 */
1304 spin_lock(&dcache_lock); 1702 spin_lock(&inode->i_lock);
1305 if (!S_ISDIR(inode->i_mode) || list_empty(&inode->i_dentry)) { 1703 if (!S_ISDIR(inode->i_mode) || list_empty(&inode->i_dentry)) {
1306 __d_instantiate(found, inode); 1704 __d_instantiate(found, inode);
1307 spin_unlock(&dcache_lock); 1705 spin_unlock(&inode->i_lock);
1308 security_d_instantiate(found, inode); 1706 security_d_instantiate(found, inode);
1309 return found; 1707 return found;
1310 } 1708 }
@@ -1314,8 +1712,8 @@ struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
1314 * reference to it, move it in place and use it. 1712 * reference to it, move it in place and use it.
1315 */ 1713 */
1316 new = list_entry(inode->i_dentry.next, struct dentry, d_alias); 1714 new = list_entry(inode->i_dentry.next, struct dentry, d_alias);
1317 dget_locked(new); 1715 __dget(new);
1318 spin_unlock(&dcache_lock); 1716 spin_unlock(&inode->i_lock);
1319 security_d_instantiate(found, inode); 1717 security_d_instantiate(found, inode);
1320 d_move(new, found); 1718 d_move(new, found);
1321 iput(inode); 1719 iput(inode);
@@ -1329,6 +1727,112 @@ err_out:
1329EXPORT_SYMBOL(d_add_ci); 1727EXPORT_SYMBOL(d_add_ci);
1330 1728
1331/** 1729/**
1730 * __d_lookup_rcu - search for a dentry (racy, store-free)
1731 * @parent: parent dentry
1732 * @name: qstr of name we wish to find
1733 * @seq: returns d_seq value at the point where the dentry was found
1734 * @inode: returns dentry->d_inode when the inode was found valid.
1735 * Returns: dentry, or NULL
1736 *
1737 * __d_lookup_rcu is the dcache lookup function for rcu-walk name
1738 * resolution (store-free path walking) design described in
1739 * Documentation/filesystems/path-lookup.txt.
1740 *
1741 * This is not to be used outside core vfs.
1742 *
1743 * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
1744 * held, and rcu_read_lock held. The returned dentry must not be stored into
1745 * without taking d_lock and checking d_seq sequence count against @seq
1746 * returned here.
1747 *
1748 * A refcount may be taken on the found dentry with the __d_rcu_to_refcount
1749 * function.
1750 *
1751 * Alternatively, __d_lookup_rcu may be called again to look up the child of
1752 * the returned dentry, so long as its parent's seqlock is checked after the
1753 * child is looked up. Thus, an interlocking stepping of sequence lock checks
1754 * is formed, giving integrity down the path walk.
1755 */
1756struct dentry *__d_lookup_rcu(struct dentry *parent, struct qstr *name,
1757 unsigned *seq, struct inode **inode)
1758{
1759 unsigned int len = name->len;
1760 unsigned int hash = name->hash;
1761 const unsigned char *str = name->name;
1762 struct dcache_hash_bucket *b = d_hash(parent, hash);
1763 struct hlist_bl_node *node;
1764 struct dentry *dentry;
1765
1766 /*
1767 * Note: There is significant duplication with __d_lookup_rcu which is
1768 * required to prevent single threaded performance regressions
1769 * especially on architectures where smp_rmb (in seqcounts) are costly.
1770 * Keep the two functions in sync.
1771 */
1772
1773 /*
1774 * The hash list is protected using RCU.
1775 *
1776 * Carefully use d_seq when comparing a candidate dentry, to avoid
1777 * races with d_move().
1778 *
1779 * It is possible that concurrent renames can mess up our list
1780 * walk here and result in missing our dentry, resulting in the
1781 * false-negative result. d_lookup() protects against concurrent
1782 * renames using rename_lock seqlock.
1783 *
1784 * See Documentation/vfs/dcache-locking.txt for more details.
1785 */
1786 hlist_bl_for_each_entry_rcu(dentry, node, &b->head, d_hash) {
1787 struct inode *i;
1788 const char *tname;
1789 int tlen;
1790
1791 if (dentry->d_name.hash != hash)
1792 continue;
1793
1794seqretry:
1795 *seq = read_seqcount_begin(&dentry->d_seq);
1796 if (dentry->d_parent != parent)
1797 continue;
1798 if (d_unhashed(dentry))
1799 continue;
1800 tlen = dentry->d_name.len;
1801 tname = dentry->d_name.name;
1802 i = dentry->d_inode;
1803 prefetch(tname);
1804 if (i)
1805 prefetch(i);
1806 /*
1807 * This seqcount check is required to ensure name and
1808 * len are loaded atomically, so as not to walk off the
1809 * edge of memory when walking. If we could load this
1810 * atomically some other way, we could drop this check.
1811 */
1812 if (read_seqcount_retry(&dentry->d_seq, *seq))
1813 goto seqretry;
1814 if (parent->d_flags & DCACHE_OP_COMPARE) {
1815 if (parent->d_op->d_compare(parent, *inode,
1816 dentry, i,
1817 tlen, tname, name))
1818 continue;
1819 } else {
1820 if (dentry_cmp(tname, tlen, str, len))
1821 continue;
1822 }
1823 /*
1824 * No extra seqcount check is required after the name
1825 * compare. The caller must perform a seqcount check in
1826 * order to do anything useful with the returned dentry
1827 * anyway.
1828 */
1829 *inode = i;
1830 return dentry;
1831 }
1832 return NULL;
1833}
1834
1835/**
1332 * d_lookup - search for a dentry 1836 * d_lookup - search for a dentry
1333 * @parent: parent dentry 1837 * @parent: parent dentry
1334 * @name: qstr of name we wish to find 1838 * @name: qstr of name we wish to find
@@ -1339,10 +1843,10 @@ EXPORT_SYMBOL(d_add_ci);
1339 * dentry is returned. The caller must use dput to free the entry when it has 1843 * dentry is returned. The caller must use dput to free the entry when it has
1340 * finished using it. %NULL is returned if the dentry does not exist. 1844 * finished using it. %NULL is returned if the dentry does not exist.
1341 */ 1845 */
1342struct dentry * d_lookup(struct dentry * parent, struct qstr * name) 1846struct dentry *d_lookup(struct dentry *parent, struct qstr *name)
1343{ 1847{
1344 struct dentry * dentry = NULL; 1848 struct dentry *dentry;
1345 unsigned long seq; 1849 unsigned seq;
1346 1850
1347 do { 1851 do {
1348 seq = read_seqbegin(&rename_lock); 1852 seq = read_seqbegin(&rename_lock);
@@ -1354,7 +1858,7 @@ struct dentry * d_lookup(struct dentry * parent, struct qstr * name)
1354} 1858}
1355EXPORT_SYMBOL(d_lookup); 1859EXPORT_SYMBOL(d_lookup);
1356 1860
1357/* 1861/**
1358 * __d_lookup - search for a dentry (racy) 1862 * __d_lookup - search for a dentry (racy)
1359 * @parent: parent dentry 1863 * @parent: parent dentry
1360 * @name: qstr of name we wish to find 1864 * @name: qstr of name we wish to find
@@ -1369,17 +1873,24 @@ EXPORT_SYMBOL(d_lookup);
1369 * 1873 *
1370 * __d_lookup callers must be commented. 1874 * __d_lookup callers must be commented.
1371 */ 1875 */
1372struct dentry * __d_lookup(struct dentry * parent, struct qstr * name) 1876struct dentry *__d_lookup(struct dentry *parent, struct qstr *name)
1373{ 1877{
1374 unsigned int len = name->len; 1878 unsigned int len = name->len;
1375 unsigned int hash = name->hash; 1879 unsigned int hash = name->hash;
1376 const unsigned char *str = name->name; 1880 const unsigned char *str = name->name;
1377 struct hlist_head *head = d_hash(parent,hash); 1881 struct dcache_hash_bucket *b = d_hash(parent, hash);
1882 struct hlist_bl_node *node;
1378 struct dentry *found = NULL; 1883 struct dentry *found = NULL;
1379 struct hlist_node *node;
1380 struct dentry *dentry; 1884 struct dentry *dentry;
1381 1885
1382 /* 1886 /*
1887 * Note: There is significant duplication with __d_lookup_rcu which is
1888 * required to prevent single threaded performance regressions
1889 * especially on architectures where smp_rmb (in seqcounts) are costly.
1890 * Keep the two functions in sync.
1891 */
1892
1893 /*
1383 * The hash list is protected using RCU. 1894 * The hash list is protected using RCU.
1384 * 1895 *
1385 * Take d_lock when comparing a candidate dentry, to avoid races 1896 * Take d_lock when comparing a candidate dentry, to avoid races
@@ -1394,25 +1905,16 @@ struct dentry * __d_lookup(struct dentry * parent, struct qstr * name)
1394 */ 1905 */
1395 rcu_read_lock(); 1906 rcu_read_lock();
1396 1907
1397 hlist_for_each_entry_rcu(dentry, node, head, d_hash) { 1908 hlist_bl_for_each_entry_rcu(dentry, node, &b->head, d_hash) {
1398 struct qstr *qstr; 1909 const char *tname;
1910 int tlen;
1399 1911
1400 if (dentry->d_name.hash != hash) 1912 if (dentry->d_name.hash != hash)
1401 continue; 1913 continue;
1402 if (dentry->d_parent != parent)
1403 continue;
1404 1914
1405 spin_lock(&dentry->d_lock); 1915 spin_lock(&dentry->d_lock);
1406
1407 /*
1408 * Recheck the dentry after taking the lock - d_move may have
1409 * changed things. Don't bother checking the hash because
1410 * we're about to compare the whole name anyway.
1411 */
1412 if (dentry->d_parent != parent) 1916 if (dentry->d_parent != parent)
1413 goto next; 1917 goto next;
1414
1415 /* non-existing due to RCU? */
1416 if (d_unhashed(dentry)) 1918 if (d_unhashed(dentry))
1417 goto next; 1919 goto next;
1418 1920
@@ -1420,18 +1922,19 @@ struct dentry * __d_lookup(struct dentry * parent, struct qstr * name)
1420 * It is safe to compare names since d_move() cannot 1922 * It is safe to compare names since d_move() cannot
1421 * change the qstr (protected by d_lock). 1923 * change the qstr (protected by d_lock).
1422 */ 1924 */
1423 qstr = &dentry->d_name; 1925 tlen = dentry->d_name.len;
1424 if (parent->d_op && parent->d_op->d_compare) { 1926 tname = dentry->d_name.name;
1425 if (parent->d_op->d_compare(parent, qstr, name)) 1927 if (parent->d_flags & DCACHE_OP_COMPARE) {
1928 if (parent->d_op->d_compare(parent, parent->d_inode,
1929 dentry, dentry->d_inode,
1930 tlen, tname, name))
1426 goto next; 1931 goto next;
1427 } else { 1932 } else {
1428 if (qstr->len != len) 1933 if (dentry_cmp(tname, tlen, str, len))
1429 goto next;
1430 if (memcmp(qstr->name, str, len))
1431 goto next; 1934 goto next;
1432 } 1935 }
1433 1936
1434 atomic_inc(&dentry->d_count); 1937 dentry->d_count++;
1435 found = dentry; 1938 found = dentry;
1436 spin_unlock(&dentry->d_lock); 1939 spin_unlock(&dentry->d_lock);
1437 break; 1940 break;
@@ -1460,8 +1963,8 @@ struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
1460 * routine may choose to leave the hash value unchanged. 1963 * routine may choose to leave the hash value unchanged.
1461 */ 1964 */
1462 name->hash = full_name_hash(name->name, name->len); 1965 name->hash = full_name_hash(name->name, name->len);
1463 if (dir->d_op && dir->d_op->d_hash) { 1966 if (dir->d_flags & DCACHE_OP_HASH) {
1464 if (dir->d_op->d_hash(dir, name) < 0) 1967 if (dir->d_op->d_hash(dir, dir->d_inode, name) < 0)
1465 goto out; 1968 goto out;
1466 } 1969 }
1467 dentry = d_lookup(dir, name); 1970 dentry = d_lookup(dir, name);
@@ -1470,41 +1973,32 @@ out:
1470} 1973}
1471 1974
1472/** 1975/**
1473 * d_validate - verify dentry provided from insecure source 1976 * d_validate - verify dentry provided from insecure source (deprecated)
1474 * @dentry: The dentry alleged to be valid child of @dparent 1977 * @dentry: The dentry alleged to be valid child of @dparent
1475 * @dparent: The parent dentry (known to be valid) 1978 * @dparent: The parent dentry (known to be valid)
1476 * 1979 *
1477 * An insecure source has sent us a dentry, here we verify it and dget() it. 1980 * An insecure source has sent us a dentry, here we verify it and dget() it.
1478 * This is used by ncpfs in its readdir implementation. 1981 * This is used by ncpfs in its readdir implementation.
1479 * Zero is returned in the dentry is invalid. 1982 * Zero is returned in the dentry is invalid.
1983 *
1984 * This function is slow for big directories, and deprecated, do not use it.
1480 */ 1985 */
1481
1482int d_validate(struct dentry *dentry, struct dentry *dparent) 1986int d_validate(struct dentry *dentry, struct dentry *dparent)
1483{ 1987{
1484 struct hlist_head *base; 1988 struct dentry *child;
1485 struct hlist_node *lhp;
1486
1487 /* Check whether the ptr might be valid at all.. */
1488 if (!kmem_ptr_validate(dentry_cache, dentry))
1489 goto out;
1490
1491 if (dentry->d_parent != dparent)
1492 goto out;
1493 1989
1494 spin_lock(&dcache_lock); 1990 spin_lock(&dparent->d_lock);
1495 base = d_hash(dparent, dentry->d_name.hash); 1991 list_for_each_entry(child, &dparent->d_subdirs, d_u.d_child) {
1496 hlist_for_each(lhp,base) { 1992 if (dentry == child) {
1497 /* hlist_for_each_entry_rcu() not required for d_hash list 1993 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1498 * as it is parsed under dcache_lock 1994 __dget_dlock(dentry);
1499 */ 1995 spin_unlock(&dentry->d_lock);
1500 if (dentry == hlist_entry(lhp, struct dentry, d_hash)) { 1996 spin_unlock(&dparent->d_lock);
1501 __dget_locked(dentry);
1502 spin_unlock(&dcache_lock);
1503 return 1; 1997 return 1;
1504 } 1998 }
1505 } 1999 }
1506 spin_unlock(&dcache_lock); 2000 spin_unlock(&dparent->d_lock);
1507out: 2001
1508 return 0; 2002 return 0;
1509} 2003}
1510EXPORT_SYMBOL(d_validate); 2004EXPORT_SYMBOL(d_validate);
@@ -1532,16 +2026,23 @@ EXPORT_SYMBOL(d_validate);
1532 2026
1533void d_delete(struct dentry * dentry) 2027void d_delete(struct dentry * dentry)
1534{ 2028{
2029 struct inode *inode;
1535 int isdir = 0; 2030 int isdir = 0;
1536 /* 2031 /*
1537 * Are we the only user? 2032 * Are we the only user?
1538 */ 2033 */
1539 spin_lock(&dcache_lock); 2034again:
1540 spin_lock(&dentry->d_lock); 2035 spin_lock(&dentry->d_lock);
1541 isdir = S_ISDIR(dentry->d_inode->i_mode); 2036 inode = dentry->d_inode;
1542 if (atomic_read(&dentry->d_count) == 1) { 2037 isdir = S_ISDIR(inode->i_mode);
2038 if (dentry->d_count == 1) {
2039 if (inode && !spin_trylock(&inode->i_lock)) {
2040 spin_unlock(&dentry->d_lock);
2041 cpu_relax();
2042 goto again;
2043 }
1543 dentry->d_flags &= ~DCACHE_CANT_MOUNT; 2044 dentry->d_flags &= ~DCACHE_CANT_MOUNT;
1544 dentry_iput(dentry); 2045 dentry_unlink_inode(dentry);
1545 fsnotify_nameremove(dentry, isdir); 2046 fsnotify_nameremove(dentry, isdir);
1546 return; 2047 return;
1547 } 2048 }
@@ -1550,17 +2051,18 @@ void d_delete(struct dentry * dentry)
1550 __d_drop(dentry); 2051 __d_drop(dentry);
1551 2052
1552 spin_unlock(&dentry->d_lock); 2053 spin_unlock(&dentry->d_lock);
1553 spin_unlock(&dcache_lock);
1554 2054
1555 fsnotify_nameremove(dentry, isdir); 2055 fsnotify_nameremove(dentry, isdir);
1556} 2056}
1557EXPORT_SYMBOL(d_delete); 2057EXPORT_SYMBOL(d_delete);
1558 2058
1559static void __d_rehash(struct dentry * entry, struct hlist_head *list) 2059static void __d_rehash(struct dentry * entry, struct dcache_hash_bucket *b)
1560{ 2060{
1561 2061 BUG_ON(!d_unhashed(entry));
2062 spin_lock_bucket(b);
1562 entry->d_flags &= ~DCACHE_UNHASHED; 2063 entry->d_flags &= ~DCACHE_UNHASHED;
1563 hlist_add_head_rcu(&entry->d_hash, list); 2064 hlist_bl_add_head_rcu(&entry->d_hash, &b->head);
2065 spin_unlock_bucket(b);
1564} 2066}
1565 2067
1566static void _d_rehash(struct dentry * entry) 2068static void _d_rehash(struct dentry * entry)
@@ -1577,25 +2079,39 @@ static void _d_rehash(struct dentry * entry)
1577 2079
1578void d_rehash(struct dentry * entry) 2080void d_rehash(struct dentry * entry)
1579{ 2081{
1580 spin_lock(&dcache_lock);
1581 spin_lock(&entry->d_lock); 2082 spin_lock(&entry->d_lock);
1582 _d_rehash(entry); 2083 _d_rehash(entry);
1583 spin_unlock(&entry->d_lock); 2084 spin_unlock(&entry->d_lock);
1584 spin_unlock(&dcache_lock);
1585} 2085}
1586EXPORT_SYMBOL(d_rehash); 2086EXPORT_SYMBOL(d_rehash);
1587 2087
1588/* 2088/**
1589 * When switching names, the actual string doesn't strictly have to 2089 * dentry_update_name_case - update case insensitive dentry with a new name
1590 * be preserved in the target - because we're dropping the target 2090 * @dentry: dentry to be updated
1591 * anyway. As such, we can just do a simple memcpy() to copy over 2091 * @name: new name
1592 * the new name before we switch.
1593 * 2092 *
1594 * Note that we have to be a lot more careful about getting the hash 2093 * Update a case insensitive dentry with new case of name.
1595 * switched - we have to switch the hash value properly even if it 2094 *
1596 * then no longer matches the actual (corrupted) string of the target. 2095 * dentry must have been returned by d_lookup with name @name. Old and new
1597 * The hash value has to match the hash queue that the dentry is on.. 2096 * name lengths must match (ie. no d_compare which allows mismatched name
2097 * lengths).
2098 *
2099 * Parent inode i_mutex must be held over d_lookup and into this call (to
2100 * keep renames and concurrent inserts, and readdir(2) away).
1598 */ 2101 */
2102void dentry_update_name_case(struct dentry *dentry, struct qstr *name)
2103{
2104 BUG_ON(!mutex_is_locked(&dentry->d_inode->i_mutex));
2105 BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */
2106
2107 spin_lock(&dentry->d_lock);
2108 write_seqcount_begin(&dentry->d_seq);
2109 memcpy((unsigned char *)dentry->d_name.name, name->name, name->len);
2110 write_seqcount_end(&dentry->d_seq);
2111 spin_unlock(&dentry->d_lock);
2112}
2113EXPORT_SYMBOL(dentry_update_name_case);
2114
1599static void switch_names(struct dentry *dentry, struct dentry *target) 2115static void switch_names(struct dentry *dentry, struct dentry *target)
1600{ 2116{
1601 if (dname_external(target)) { 2117 if (dname_external(target)) {
@@ -1637,54 +2153,84 @@ static void switch_names(struct dentry *dentry, struct dentry *target)
1637 swap(dentry->d_name.len, target->d_name.len); 2153 swap(dentry->d_name.len, target->d_name.len);
1638} 2154}
1639 2155
2156static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target)
2157{
2158 /*
2159 * XXXX: do we really need to take target->d_lock?
2160 */
2161 if (IS_ROOT(dentry) || dentry->d_parent == target->d_parent)
2162 spin_lock(&target->d_parent->d_lock);
2163 else {
2164 if (d_ancestor(dentry->d_parent, target->d_parent)) {
2165 spin_lock(&dentry->d_parent->d_lock);
2166 spin_lock_nested(&target->d_parent->d_lock,
2167 DENTRY_D_LOCK_NESTED);
2168 } else {
2169 spin_lock(&target->d_parent->d_lock);
2170 spin_lock_nested(&dentry->d_parent->d_lock,
2171 DENTRY_D_LOCK_NESTED);
2172 }
2173 }
2174 if (target < dentry) {
2175 spin_lock_nested(&target->d_lock, 2);
2176 spin_lock_nested(&dentry->d_lock, 3);
2177 } else {
2178 spin_lock_nested(&dentry->d_lock, 2);
2179 spin_lock_nested(&target->d_lock, 3);
2180 }
2181}
2182
2183static void dentry_unlock_parents_for_move(struct dentry *dentry,
2184 struct dentry *target)
2185{
2186 if (target->d_parent != dentry->d_parent)
2187 spin_unlock(&dentry->d_parent->d_lock);
2188 if (target->d_parent != target)
2189 spin_unlock(&target->d_parent->d_lock);
2190}
2191
1640/* 2192/*
1641 * We cannibalize "target" when moving dentry on top of it, 2193 * When switching names, the actual string doesn't strictly have to
1642 * because it's going to be thrown away anyway. We could be more 2194 * be preserved in the target - because we're dropping the target
1643 * polite about it, though. 2195 * anyway. As such, we can just do a simple memcpy() to copy over
1644 * 2196 * the new name before we switch.
1645 * This forceful removal will result in ugly /proc output if 2197 *
1646 * somebody holds a file open that got deleted due to a rename. 2198 * Note that we have to be a lot more careful about getting the hash
1647 * We could be nicer about the deleted file, and let it show 2199 * switched - we have to switch the hash value properly even if it
1648 * up under the name it had before it was deleted rather than 2200 * then no longer matches the actual (corrupted) string of the target.
1649 * under the original name of the file that was moved on top of it. 2201 * The hash value has to match the hash queue that the dentry is on..
1650 */ 2202 */
1651
1652/* 2203/*
1653 * d_move_locked - move a dentry 2204 * d_move - move a dentry
1654 * @dentry: entry to move 2205 * @dentry: entry to move
1655 * @target: new dentry 2206 * @target: new dentry
1656 * 2207 *
1657 * Update the dcache to reflect the move of a file name. Negative 2208 * Update the dcache to reflect the move of a file name. Negative
1658 * dcache entries should not be moved in this way. 2209 * dcache entries should not be moved in this way.
1659 */ 2210 */
1660static void d_move_locked(struct dentry * dentry, struct dentry * target) 2211void d_move(struct dentry * dentry, struct dentry * target)
1661{ 2212{
1662 struct hlist_head *list;
1663
1664 if (!dentry->d_inode) 2213 if (!dentry->d_inode)
1665 printk(KERN_WARNING "VFS: moving negative dcache entry\n"); 2214 printk(KERN_WARNING "VFS: moving negative dcache entry\n");
1666 2215
2216 BUG_ON(d_ancestor(dentry, target));
2217 BUG_ON(d_ancestor(target, dentry));
2218
1667 write_seqlock(&rename_lock); 2219 write_seqlock(&rename_lock);
1668 /*
1669 * XXXX: do we really need to take target->d_lock?
1670 */
1671 if (target < dentry) {
1672 spin_lock(&target->d_lock);
1673 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1674 } else {
1675 spin_lock(&dentry->d_lock);
1676 spin_lock_nested(&target->d_lock, DENTRY_D_LOCK_NESTED);
1677 }
1678 2220
1679 /* Move the dentry to the target hash queue, if on different bucket */ 2221 dentry_lock_for_move(dentry, target);
1680 if (d_unhashed(dentry))
1681 goto already_unhashed;
1682 2222
1683 hlist_del_rcu(&dentry->d_hash); 2223 write_seqcount_begin(&dentry->d_seq);
2224 write_seqcount_begin(&target->d_seq);
1684 2225
1685already_unhashed: 2226 /* __d_drop does write_seqcount_barrier, but they're OK to nest. */
1686 list = d_hash(target->d_parent, target->d_name.hash); 2227
1687 __d_rehash(dentry, list); 2228 /*
2229 * Move the dentry to the target hash queue. Don't bother checking
2230 * for the same hash queue because of how unlikely it is.
2231 */
2232 __d_drop(dentry);
2233 __d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash));
1688 2234
1689 /* Unhash the target: dput() will then get rid of it */ 2235 /* Unhash the target: dput() will then get rid of it */
1690 __d_drop(target); 2236 __d_drop(target);
@@ -1709,27 +2255,16 @@ already_unhashed:
1709 } 2255 }
1710 2256
1711 list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs); 2257 list_add(&dentry->d_u.d_child, &dentry->d_parent->d_subdirs);
2258
2259 write_seqcount_end(&target->d_seq);
2260 write_seqcount_end(&dentry->d_seq);
2261
2262 dentry_unlock_parents_for_move(dentry, target);
1712 spin_unlock(&target->d_lock); 2263 spin_unlock(&target->d_lock);
1713 fsnotify_d_move(dentry); 2264 fsnotify_d_move(dentry);
1714 spin_unlock(&dentry->d_lock); 2265 spin_unlock(&dentry->d_lock);
1715 write_sequnlock(&rename_lock); 2266 write_sequnlock(&rename_lock);
1716} 2267}
1717
1718/**
1719 * d_move - move a dentry
1720 * @dentry: entry to move
1721 * @target: new dentry
1722 *
1723 * Update the dcache to reflect the move of a file name. Negative
1724 * dcache entries should not be moved in this way.
1725 */
1726
1727void d_move(struct dentry * dentry, struct dentry * target)
1728{
1729 spin_lock(&dcache_lock);
1730 d_move_locked(dentry, target);
1731 spin_unlock(&dcache_lock);
1732}
1733EXPORT_SYMBOL(d_move); 2268EXPORT_SYMBOL(d_move);
1734 2269
1735/** 2270/**
@@ -1755,13 +2290,13 @@ struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
1755 * This helper attempts to cope with remotely renamed directories 2290 * This helper attempts to cope with remotely renamed directories
1756 * 2291 *
1757 * It assumes that the caller is already holding 2292 * It assumes that the caller is already holding
1758 * dentry->d_parent->d_inode->i_mutex and the dcache_lock 2293 * dentry->d_parent->d_inode->i_mutex and the inode->i_lock
1759 * 2294 *
1760 * Note: If ever the locking in lock_rename() changes, then please 2295 * Note: If ever the locking in lock_rename() changes, then please
1761 * remember to update this too... 2296 * remember to update this too...
1762 */ 2297 */
1763static struct dentry *__d_unalias(struct dentry *dentry, struct dentry *alias) 2298static struct dentry *__d_unalias(struct inode *inode,
1764 __releases(dcache_lock) 2299 struct dentry *dentry, struct dentry *alias)
1765{ 2300{
1766 struct mutex *m1 = NULL, *m2 = NULL; 2301 struct mutex *m1 = NULL, *m2 = NULL;
1767 struct dentry *ret; 2302 struct dentry *ret;
@@ -1784,10 +2319,10 @@ static struct dentry *__d_unalias(struct dentry *dentry, struct dentry *alias)
1784 goto out_err; 2319 goto out_err;
1785 m2 = &alias->d_parent->d_inode->i_mutex; 2320 m2 = &alias->d_parent->d_inode->i_mutex;
1786out_unalias: 2321out_unalias:
1787 d_move_locked(alias, dentry); 2322 d_move(alias, dentry);
1788 ret = alias; 2323 ret = alias;
1789out_err: 2324out_err:
1790 spin_unlock(&dcache_lock); 2325 spin_unlock(&inode->i_lock);
1791 if (m2) 2326 if (m2)
1792 mutex_unlock(m2); 2327 mutex_unlock(m2);
1793 if (m1) 2328 if (m1)
@@ -1798,17 +2333,23 @@ out_err:
1798/* 2333/*
1799 * Prepare an anonymous dentry for life in the superblock's dentry tree as a 2334 * Prepare an anonymous dentry for life in the superblock's dentry tree as a
1800 * named dentry in place of the dentry to be replaced. 2335 * named dentry in place of the dentry to be replaced.
2336 * returns with anon->d_lock held!
1801 */ 2337 */
1802static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon) 2338static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon)
1803{ 2339{
1804 struct dentry *dparent, *aparent; 2340 struct dentry *dparent, *aparent;
1805 2341
1806 switch_names(dentry, anon); 2342 dentry_lock_for_move(anon, dentry);
1807 swap(dentry->d_name.hash, anon->d_name.hash); 2343
2344 write_seqcount_begin(&dentry->d_seq);
2345 write_seqcount_begin(&anon->d_seq);
1808 2346
1809 dparent = dentry->d_parent; 2347 dparent = dentry->d_parent;
1810 aparent = anon->d_parent; 2348 aparent = anon->d_parent;
1811 2349
2350 switch_names(dentry, anon);
2351 swap(dentry->d_name.hash, anon->d_name.hash);
2352
1812 dentry->d_parent = (aparent == anon) ? dentry : aparent; 2353 dentry->d_parent = (aparent == anon) ? dentry : aparent;
1813 list_del(&dentry->d_u.d_child); 2354 list_del(&dentry->d_u.d_child);
1814 if (!IS_ROOT(dentry)) 2355 if (!IS_ROOT(dentry))
@@ -1823,6 +2364,13 @@ static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon)
1823 else 2364 else
1824 INIT_LIST_HEAD(&anon->d_u.d_child); 2365 INIT_LIST_HEAD(&anon->d_u.d_child);
1825 2366
2367 write_seqcount_end(&dentry->d_seq);
2368 write_seqcount_end(&anon->d_seq);
2369
2370 dentry_unlock_parents_for_move(anon, dentry);
2371 spin_unlock(&dentry->d_lock);
2372
2373 /* anon->d_lock still locked, returns locked */
1826 anon->d_flags &= ~DCACHE_DISCONNECTED; 2374 anon->d_flags &= ~DCACHE_DISCONNECTED;
1827} 2375}
1828 2376
@@ -1840,14 +2388,15 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
1840 2388
1841 BUG_ON(!d_unhashed(dentry)); 2389 BUG_ON(!d_unhashed(dentry));
1842 2390
1843 spin_lock(&dcache_lock);
1844
1845 if (!inode) { 2391 if (!inode) {
1846 actual = dentry; 2392 actual = dentry;
1847 __d_instantiate(dentry, NULL); 2393 __d_instantiate(dentry, NULL);
1848 goto found_lock; 2394 d_rehash(actual);
2395 goto out_nolock;
1849 } 2396 }
1850 2397
2398 spin_lock(&inode->i_lock);
2399
1851 if (S_ISDIR(inode->i_mode)) { 2400 if (S_ISDIR(inode->i_mode)) {
1852 struct dentry *alias; 2401 struct dentry *alias;
1853 2402
@@ -1858,13 +2407,12 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
1858 /* Is this an anonymous mountpoint that we could splice 2407 /* Is this an anonymous mountpoint that we could splice
1859 * into our tree? */ 2408 * into our tree? */
1860 if (IS_ROOT(alias)) { 2409 if (IS_ROOT(alias)) {
1861 spin_lock(&alias->d_lock);
1862 __d_materialise_dentry(dentry, alias); 2410 __d_materialise_dentry(dentry, alias);
1863 __d_drop(alias); 2411 __d_drop(alias);
1864 goto found; 2412 goto found;
1865 } 2413 }
1866 /* Nope, but we must(!) avoid directory aliasing */ 2414 /* Nope, but we must(!) avoid directory aliasing */
1867 actual = __d_unalias(dentry, alias); 2415 actual = __d_unalias(inode, dentry, alias);
1868 if (IS_ERR(actual)) 2416 if (IS_ERR(actual))
1869 dput(alias); 2417 dput(alias);
1870 goto out_nolock; 2418 goto out_nolock;
@@ -1875,15 +2423,14 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
1875 actual = __d_instantiate_unique(dentry, inode); 2423 actual = __d_instantiate_unique(dentry, inode);
1876 if (!actual) 2424 if (!actual)
1877 actual = dentry; 2425 actual = dentry;
1878 else if (unlikely(!d_unhashed(actual))) 2426 else
1879 goto shouldnt_be_hashed; 2427 BUG_ON(!d_unhashed(actual));
1880 2428
1881found_lock:
1882 spin_lock(&actual->d_lock); 2429 spin_lock(&actual->d_lock);
1883found: 2430found:
1884 _d_rehash(actual); 2431 _d_rehash(actual);
1885 spin_unlock(&actual->d_lock); 2432 spin_unlock(&actual->d_lock);
1886 spin_unlock(&dcache_lock); 2433 spin_unlock(&inode->i_lock);
1887out_nolock: 2434out_nolock:
1888 if (actual == dentry) { 2435 if (actual == dentry) {
1889 security_d_instantiate(dentry, inode); 2436 security_d_instantiate(dentry, inode);
@@ -1892,10 +2439,6 @@ out_nolock:
1892 2439
1893 iput(inode); 2440 iput(inode);
1894 return actual; 2441 return actual;
1895
1896shouldnt_be_hashed:
1897 spin_unlock(&dcache_lock);
1898 BUG();
1899} 2442}
1900EXPORT_SYMBOL_GPL(d_materialise_unique); 2443EXPORT_SYMBOL_GPL(d_materialise_unique);
1901 2444
@@ -1915,14 +2458,13 @@ static int prepend_name(char **buffer, int *buflen, struct qstr *name)
1915} 2458}
1916 2459
1917/** 2460/**
1918 * Prepend path string to a buffer 2461 * prepend_path - Prepend path string to a buffer
1919 *
1920 * @path: the dentry/vfsmount to report 2462 * @path: the dentry/vfsmount to report
1921 * @root: root vfsmnt/dentry (may be modified by this function) 2463 * @root: root vfsmnt/dentry (may be modified by this function)
1922 * @buffer: pointer to the end of the buffer 2464 * @buffer: pointer to the end of the buffer
1923 * @buflen: pointer to buffer length 2465 * @buflen: pointer to buffer length
1924 * 2466 *
1925 * Caller holds the dcache_lock. 2467 * Caller holds the rename_lock.
1926 * 2468 *
1927 * If path is not reachable from the supplied root, then the value of 2469 * If path is not reachable from the supplied root, then the value of
1928 * root is changed (without modifying refcounts). 2470 * root is changed (without modifying refcounts).
@@ -1950,7 +2492,9 @@ static int prepend_path(const struct path *path, struct path *root,
1950 } 2492 }
1951 parent = dentry->d_parent; 2493 parent = dentry->d_parent;
1952 prefetch(parent); 2494 prefetch(parent);
2495 spin_lock(&dentry->d_lock);
1953 error = prepend_name(buffer, buflen, &dentry->d_name); 2496 error = prepend_name(buffer, buflen, &dentry->d_name);
2497 spin_unlock(&dentry->d_lock);
1954 if (!error) 2498 if (!error)
1955 error = prepend(buffer, buflen, "/", 1); 2499 error = prepend(buffer, buflen, "/", 1);
1956 if (error) 2500 if (error)
@@ -1994,7 +2538,7 @@ global_root:
1994 * Returns a pointer into the buffer or an error code if the 2538 * Returns a pointer into the buffer or an error code if the
1995 * path was too long. 2539 * path was too long.
1996 * 2540 *
1997 * "buflen" should be positive. Caller holds the dcache_lock. 2541 * "buflen" should be positive.
1998 * 2542 *
1999 * If path is not reachable from the supplied root, then the value of 2543 * If path is not reachable from the supplied root, then the value of
2000 * root is changed (without modifying refcounts). 2544 * root is changed (without modifying refcounts).
@@ -2006,10 +2550,12 @@ char *__d_path(const struct path *path, struct path *root,
2006 int error; 2550 int error;
2007 2551
2008 prepend(&res, &buflen, "\0", 1); 2552 prepend(&res, &buflen, "\0", 1);
2553 write_seqlock(&rename_lock);
2009 error = prepend_path(path, root, &res, &buflen); 2554 error = prepend_path(path, root, &res, &buflen);
2555 write_sequnlock(&rename_lock);
2556
2010 if (error) 2557 if (error)
2011 return ERR_PTR(error); 2558 return ERR_PTR(error);
2012
2013 return res; 2559 return res;
2014} 2560}
2015 2561
@@ -2068,12 +2614,12 @@ char *d_path(const struct path *path, char *buf, int buflen)
2068 return path->dentry->d_op->d_dname(path->dentry, buf, buflen); 2614 return path->dentry->d_op->d_dname(path->dentry, buf, buflen);
2069 2615
2070 get_fs_root(current->fs, &root); 2616 get_fs_root(current->fs, &root);
2071 spin_lock(&dcache_lock); 2617 write_seqlock(&rename_lock);
2072 tmp = root; 2618 tmp = root;
2073 error = path_with_deleted(path, &tmp, &res, &buflen); 2619 error = path_with_deleted(path, &tmp, &res, &buflen);
2074 if (error) 2620 if (error)
2075 res = ERR_PTR(error); 2621 res = ERR_PTR(error);
2076 spin_unlock(&dcache_lock); 2622 write_sequnlock(&rename_lock);
2077 path_put(&root); 2623 path_put(&root);
2078 return res; 2624 return res;
2079} 2625}
@@ -2099,12 +2645,12 @@ char *d_path_with_unreachable(const struct path *path, char *buf, int buflen)
2099 return path->dentry->d_op->d_dname(path->dentry, buf, buflen); 2645 return path->dentry->d_op->d_dname(path->dentry, buf, buflen);
2100 2646
2101 get_fs_root(current->fs, &root); 2647 get_fs_root(current->fs, &root);
2102 spin_lock(&dcache_lock); 2648 write_seqlock(&rename_lock);
2103 tmp = root; 2649 tmp = root;
2104 error = path_with_deleted(path, &tmp, &res, &buflen); 2650 error = path_with_deleted(path, &tmp, &res, &buflen);
2105 if (!error && !path_equal(&tmp, &root)) 2651 if (!error && !path_equal(&tmp, &root))
2106 error = prepend_unreachable(&res, &buflen); 2652 error = prepend_unreachable(&res, &buflen);
2107 spin_unlock(&dcache_lock); 2653 write_sequnlock(&rename_lock);
2108 path_put(&root); 2654 path_put(&root);
2109 if (error) 2655 if (error)
2110 res = ERR_PTR(error); 2656 res = ERR_PTR(error);
@@ -2136,7 +2682,7 @@ char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen,
2136/* 2682/*
2137 * Write full pathname from the root of the filesystem into the buffer. 2683 * Write full pathname from the root of the filesystem into the buffer.
2138 */ 2684 */
2139char *__dentry_path(struct dentry *dentry, char *buf, int buflen) 2685static char *__dentry_path(struct dentry *dentry, char *buf, int buflen)
2140{ 2686{
2141 char *end = buf + buflen; 2687 char *end = buf + buflen;
2142 char *retval; 2688 char *retval;
@@ -2150,10 +2696,13 @@ char *__dentry_path(struct dentry *dentry, char *buf, int buflen)
2150 2696
2151 while (!IS_ROOT(dentry)) { 2697 while (!IS_ROOT(dentry)) {
2152 struct dentry *parent = dentry->d_parent; 2698 struct dentry *parent = dentry->d_parent;
2699 int error;
2153 2700
2154 prefetch(parent); 2701 prefetch(parent);
2155 if ((prepend_name(&end, &buflen, &dentry->d_name) != 0) || 2702 spin_lock(&dentry->d_lock);
2156 (prepend(&end, &buflen, "/", 1) != 0)) 2703 error = prepend_name(&end, &buflen, &dentry->d_name);
2704 spin_unlock(&dentry->d_lock);
2705 if (error != 0 || prepend(&end, &buflen, "/", 1) != 0)
2157 goto Elong; 2706 goto Elong;
2158 2707
2159 retval = end; 2708 retval = end;
@@ -2163,14 +2712,25 @@ char *__dentry_path(struct dentry *dentry, char *buf, int buflen)
2163Elong: 2712Elong:
2164 return ERR_PTR(-ENAMETOOLONG); 2713 return ERR_PTR(-ENAMETOOLONG);
2165} 2714}
2166EXPORT_SYMBOL(__dentry_path); 2715
2716char *dentry_path_raw(struct dentry *dentry, char *buf, int buflen)
2717{
2718 char *retval;
2719
2720 write_seqlock(&rename_lock);
2721 retval = __dentry_path(dentry, buf, buflen);
2722 write_sequnlock(&rename_lock);
2723
2724 return retval;
2725}
2726EXPORT_SYMBOL(dentry_path_raw);
2167 2727
2168char *dentry_path(struct dentry *dentry, char *buf, int buflen) 2728char *dentry_path(struct dentry *dentry, char *buf, int buflen)
2169{ 2729{
2170 char *p = NULL; 2730 char *p = NULL;
2171 char *retval; 2731 char *retval;
2172 2732
2173 spin_lock(&dcache_lock); 2733 write_seqlock(&rename_lock);
2174 if (d_unlinked(dentry)) { 2734 if (d_unlinked(dentry)) {
2175 p = buf + buflen; 2735 p = buf + buflen;
2176 if (prepend(&p, &buflen, "//deleted", 10) != 0) 2736 if (prepend(&p, &buflen, "//deleted", 10) != 0)
@@ -2178,12 +2738,11 @@ char *dentry_path(struct dentry *dentry, char *buf, int buflen)
2178 buflen++; 2738 buflen++;
2179 } 2739 }
2180 retval = __dentry_path(dentry, buf, buflen); 2740 retval = __dentry_path(dentry, buf, buflen);
2181 spin_unlock(&dcache_lock); 2741 write_sequnlock(&rename_lock);
2182 if (!IS_ERR(retval) && p) 2742 if (!IS_ERR(retval) && p)
2183 *p = '/'; /* restore '/' overriden with '\0' */ 2743 *p = '/'; /* restore '/' overriden with '\0' */
2184 return retval; 2744 return retval;
2185Elong: 2745Elong:
2186 spin_unlock(&dcache_lock);
2187 return ERR_PTR(-ENAMETOOLONG); 2746 return ERR_PTR(-ENAMETOOLONG);
2188} 2747}
2189 2748
@@ -2217,7 +2776,7 @@ SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
2217 get_fs_root_and_pwd(current->fs, &root, &pwd); 2776 get_fs_root_and_pwd(current->fs, &root, &pwd);
2218 2777
2219 error = -ENOENT; 2778 error = -ENOENT;
2220 spin_lock(&dcache_lock); 2779 write_seqlock(&rename_lock);
2221 if (!d_unlinked(pwd.dentry)) { 2780 if (!d_unlinked(pwd.dentry)) {
2222 unsigned long len; 2781 unsigned long len;
2223 struct path tmp = root; 2782 struct path tmp = root;
@@ -2226,7 +2785,7 @@ SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
2226 2785
2227 prepend(&cwd, &buflen, "\0", 1); 2786 prepend(&cwd, &buflen, "\0", 1);
2228 error = prepend_path(&pwd, &tmp, &cwd, &buflen); 2787 error = prepend_path(&pwd, &tmp, &cwd, &buflen);
2229 spin_unlock(&dcache_lock); 2788 write_sequnlock(&rename_lock);
2230 2789
2231 if (error) 2790 if (error)
2232 goto out; 2791 goto out;
@@ -2245,8 +2804,9 @@ SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size)
2245 if (copy_to_user(buf, cwd, len)) 2804 if (copy_to_user(buf, cwd, len))
2246 error = -EFAULT; 2805 error = -EFAULT;
2247 } 2806 }
2248 } else 2807 } else {
2249 spin_unlock(&dcache_lock); 2808 write_sequnlock(&rename_lock);
2809 }
2250 2810
2251out: 2811out:
2252 path_put(&pwd); 2812 path_put(&pwd);
@@ -2274,25 +2834,25 @@ out:
2274int is_subdir(struct dentry *new_dentry, struct dentry *old_dentry) 2834int is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
2275{ 2835{
2276 int result; 2836 int result;
2277 unsigned long seq; 2837 unsigned seq;
2278 2838
2279 if (new_dentry == old_dentry) 2839 if (new_dentry == old_dentry)
2280 return 1; 2840 return 1;
2281 2841
2282 /*
2283 * Need rcu_readlock to protect against the d_parent trashing
2284 * due to d_move
2285 */
2286 rcu_read_lock();
2287 do { 2842 do {
2288 /* for restarting inner loop in case of seq retry */ 2843 /* for restarting inner loop in case of seq retry */
2289 seq = read_seqbegin(&rename_lock); 2844 seq = read_seqbegin(&rename_lock);
2845 /*
2846 * Need rcu_readlock to protect against the d_parent trashing
2847 * due to d_move
2848 */
2849 rcu_read_lock();
2290 if (d_ancestor(old_dentry, new_dentry)) 2850 if (d_ancestor(old_dentry, new_dentry))
2291 result = 1; 2851 result = 1;
2292 else 2852 else
2293 result = 0; 2853 result = 0;
2854 rcu_read_unlock();
2294 } while (read_seqretry(&rename_lock, seq)); 2855 } while (read_seqretry(&rename_lock, seq));
2295 rcu_read_unlock();
2296 2856
2297 return result; 2857 return result;
2298} 2858}
@@ -2324,10 +2884,15 @@ EXPORT_SYMBOL(path_is_under);
2324 2884
2325void d_genocide(struct dentry *root) 2885void d_genocide(struct dentry *root)
2326{ 2886{
2327 struct dentry *this_parent = root; 2887 struct dentry *this_parent;
2328 struct list_head *next; 2888 struct list_head *next;
2889 unsigned seq;
2890 int locked = 0;
2329 2891
2330 spin_lock(&dcache_lock); 2892 seq = read_seqbegin(&rename_lock);
2893again:
2894 this_parent = root;
2895 spin_lock(&this_parent->d_lock);
2331repeat: 2896repeat:
2332 next = this_parent->d_subdirs.next; 2897 next = this_parent->d_subdirs.next;
2333resume: 2898resume:
@@ -2335,21 +2900,62 @@ resume:
2335 struct list_head *tmp = next; 2900 struct list_head *tmp = next;
2336 struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child); 2901 struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
2337 next = tmp->next; 2902 next = tmp->next;
2338 if (d_unhashed(dentry)||!dentry->d_inode) 2903
2904 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
2905 if (d_unhashed(dentry) || !dentry->d_inode) {
2906 spin_unlock(&dentry->d_lock);
2339 continue; 2907 continue;
2908 }
2340 if (!list_empty(&dentry->d_subdirs)) { 2909 if (!list_empty(&dentry->d_subdirs)) {
2910 spin_unlock(&this_parent->d_lock);
2911 spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
2341 this_parent = dentry; 2912 this_parent = dentry;
2913 spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
2342 goto repeat; 2914 goto repeat;
2343 } 2915 }
2344 atomic_dec(&dentry->d_count); 2916 if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
2917 dentry->d_flags |= DCACHE_GENOCIDE;
2918 dentry->d_count--;
2919 }
2920 spin_unlock(&dentry->d_lock);
2345 } 2921 }
2346 if (this_parent != root) { 2922 if (this_parent != root) {
2347 next = this_parent->d_u.d_child.next; 2923 struct dentry *tmp;
2348 atomic_dec(&this_parent->d_count); 2924 struct dentry *child;
2349 this_parent = this_parent->d_parent; 2925
2926 tmp = this_parent->d_parent;
2927 if (!(this_parent->d_flags & DCACHE_GENOCIDE)) {
2928 this_parent->d_flags |= DCACHE_GENOCIDE;
2929 this_parent->d_count--;
2930 }
2931 rcu_read_lock();
2932 spin_unlock(&this_parent->d_lock);
2933 child = this_parent;
2934 this_parent = tmp;
2935 spin_lock(&this_parent->d_lock);
2936 /* might go back up the wrong parent if we have had a rename
2937 * or deletion */
2938 if (this_parent != child->d_parent ||
2939 (!locked && read_seqretry(&rename_lock, seq))) {
2940 spin_unlock(&this_parent->d_lock);
2941 rcu_read_unlock();
2942 goto rename_retry;
2943 }
2944 rcu_read_unlock();
2945 next = child->d_u.d_child.next;
2350 goto resume; 2946 goto resume;
2351 } 2947 }
2352 spin_unlock(&dcache_lock); 2948 spin_unlock(&this_parent->d_lock);
2949 if (!locked && read_seqretry(&rename_lock, seq))
2950 goto rename_retry;
2951 if (locked)
2952 write_sequnlock(&rename_lock);
2953 return;
2954
2955rename_retry:
2956 locked = 1;
2957 write_seqlock(&rename_lock);
2958 goto again;
2353} 2959}
2354 2960
2355/** 2961/**
@@ -2403,7 +3009,7 @@ static void __init dcache_init_early(void)
2403 3009
2404 dentry_hashtable = 3010 dentry_hashtable =
2405 alloc_large_system_hash("Dentry cache", 3011 alloc_large_system_hash("Dentry cache",
2406 sizeof(struct hlist_head), 3012 sizeof(struct dcache_hash_bucket),
2407 dhash_entries, 3013 dhash_entries,
2408 13, 3014 13,
2409 HASH_EARLY, 3015 HASH_EARLY,
@@ -2412,7 +3018,7 @@ static void __init dcache_init_early(void)
2412 0); 3018 0);
2413 3019
2414 for (loop = 0; loop < (1 << d_hash_shift); loop++) 3020 for (loop = 0; loop < (1 << d_hash_shift); loop++)
2415 INIT_HLIST_HEAD(&dentry_hashtable[loop]); 3021 INIT_HLIST_BL_HEAD(&dentry_hashtable[loop].head);
2416} 3022}
2417 3023
2418static void __init dcache_init(void) 3024static void __init dcache_init(void)
@@ -2435,7 +3041,7 @@ static void __init dcache_init(void)
2435 3041
2436 dentry_hashtable = 3042 dentry_hashtable =
2437 alloc_large_system_hash("Dentry cache", 3043 alloc_large_system_hash("Dentry cache",
2438 sizeof(struct hlist_head), 3044 sizeof(struct dcache_hash_bucket),
2439 dhash_entries, 3045 dhash_entries,
2440 13, 3046 13,
2441 0, 3047 0,
@@ -2444,7 +3050,7 @@ static void __init dcache_init(void)
2444 0); 3050 0);
2445 3051
2446 for (loop = 0; loop < (1 << d_hash_shift); loop++) 3052 for (loop = 0; loop < (1 << d_hash_shift); loop++)
2447 INIT_HLIST_HEAD(&dentry_hashtable[loop]); 3053 INIT_HLIST_BL_HEAD(&dentry_hashtable[loop].head);
2448} 3054}
2449 3055
2450/* SLAB cache for __getname() consumers */ 3056/* SLAB cache for __getname() consumers */
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-22 00:44:22 -0400