aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2017-02-15 22:29:51 -0500
committerDavid S. Miller <davem@davemloft.net>2017-02-15 22:29:51 -0500
commitbf3f14d6342cfb37eab8f0cddd0e4d4063fd9fc9 (patch)
tree2907cdfb8da2c43bf4c599bc3d398274e21dfc91
parent75224c93fa985f4a6fb983f53208f5c5aa555fbf (diff)
rhashtable: Revert nested table changes.
This reverts commits: 6a25478077d987edc5e2f880590a2bc5fcab4441 9dbbfb0ab6680c6a85609041011484e6658e7d3c 40137906c5f55c252194ef5834130383e639536f It's too risky to put in this late in the release cycle. We'll put these changes into the next merge window instead. Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r--fs/gfs2/glock.c28
-rw-r--r--include/linux/rhashtable.h78
-rw-r--r--lib/rhashtable.c270
-rw-r--r--net/tipc/net.c4
-rw-r--r--net/tipc/socket.c30
5 files changed, 94 insertions, 316 deletions
diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index 70e94170af85..94f50cac91c6 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -1420,32 +1420,26 @@ static struct shrinker glock_shrinker = {
1420 * @sdp: the filesystem 1420 * @sdp: the filesystem
1421 * @bucket: the bucket 1421 * @bucket: the bucket
1422 * 1422 *
1423 * Note that the function can be called multiple times on the same
1424 * object. So the user must ensure that the function can cope with
1425 * that.
1426 */ 1423 */
1427 1424
1428static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp) 1425static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
1429{ 1426{
1430 struct gfs2_glock *gl; 1427 struct gfs2_glock *gl;
1431 struct rhashtable_iter iter; 1428 struct rhash_head *pos;
1432 1429 const struct bucket_table *tbl;
1433 rhashtable_walk_enter(&gl_hash_table, &iter); 1430 int i;
1434
1435 do {
1436 gl = ERR_PTR(rhashtable_walk_start(&iter));
1437 if (gl)
1438 continue;
1439 1431
1440 while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl)) 1432 rcu_read_lock();
1433 tbl = rht_dereference_rcu(gl_hash_table.tbl, &gl_hash_table);
1434 for (i = 0; i < tbl->size; i++) {
1435 rht_for_each_entry_rcu(gl, pos, tbl, i, gl_node) {
1441 if ((gl->gl_name.ln_sbd == sdp) && 1436 if ((gl->gl_name.ln_sbd == sdp) &&
1442 lockref_get_not_dead(&gl->gl_lockref)) 1437 lockref_get_not_dead(&gl->gl_lockref))
1443 examiner(gl); 1438 examiner(gl);
1444 1439 }
1445 rhashtable_walk_stop(&iter); 1440 }
1446 } while (cond_resched(), gl == ERR_PTR(-EAGAIN)); 1441 rcu_read_unlock();
1447 1442 cond_resched();
1448 rhashtable_walk_exit(&iter);
1449} 1443}
1450 1444
1451/** 1445/**
diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h
index f2e12a845910..5c132d3188be 100644
--- a/include/linux/rhashtable.h
+++ b/include/linux/rhashtable.h
@@ -61,7 +61,6 @@ struct rhlist_head {
61/** 61/**
62 * struct bucket_table - Table of hash buckets 62 * struct bucket_table - Table of hash buckets
63 * @size: Number of hash buckets 63 * @size: Number of hash buckets
64 * @nest: Number of bits of first-level nested table.
65 * @rehash: Current bucket being rehashed 64 * @rehash: Current bucket being rehashed
66 * @hash_rnd: Random seed to fold into hash 65 * @hash_rnd: Random seed to fold into hash
67 * @locks_mask: Mask to apply before accessing locks[] 66 * @locks_mask: Mask to apply before accessing locks[]
@@ -69,12 +68,10 @@ struct rhlist_head {
69 * @walkers: List of active walkers 68 * @walkers: List of active walkers
70 * @rcu: RCU structure for freeing the table 69 * @rcu: RCU structure for freeing the table
71 * @future_tbl: Table under construction during rehashing 70 * @future_tbl: Table under construction during rehashing
72 * @ntbl: Nested table used when out of memory.
73 * @buckets: size * hash buckets 71 * @buckets: size * hash buckets
74 */ 72 */
75struct bucket_table { 73struct bucket_table {
76 unsigned int size; 74 unsigned int size;
77 unsigned int nest;
78 unsigned int rehash; 75 unsigned int rehash;
79 u32 hash_rnd; 76 u32 hash_rnd;
80 unsigned int locks_mask; 77 unsigned int locks_mask;
@@ -84,7 +81,7 @@ struct bucket_table {
84 81
85 struct bucket_table __rcu *future_tbl; 82 struct bucket_table __rcu *future_tbl;
86 83
87 struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp; 84 struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp;
88}; 85};
89 86
90/** 87/**
@@ -377,12 +374,6 @@ void rhashtable_free_and_destroy(struct rhashtable *ht,
377 void *arg); 374 void *arg);
378void rhashtable_destroy(struct rhashtable *ht); 375void rhashtable_destroy(struct rhashtable *ht);
379 376
380struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl,
381 unsigned int hash);
382struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht,
383 struct bucket_table *tbl,
384 unsigned int hash);
385
386#define rht_dereference(p, ht) \ 377#define rht_dereference(p, ht) \
387 rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht)) 378 rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht))
388 379
@@ -398,27 +389,6 @@ struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht,
398#define rht_entry(tpos, pos, member) \ 389#define rht_entry(tpos, pos, member) \
399 ({ tpos = container_of(pos, typeof(*tpos), member); 1; }) 390 ({ tpos = container_of(pos, typeof(*tpos), member); 1; })
400 391
401static inline struct rhash_head __rcu *const *rht_bucket(
402 const struct bucket_table *tbl, unsigned int hash)
403{
404 return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) :
405 &tbl->buckets[hash];
406}
407
408static inline struct rhash_head __rcu **rht_bucket_var(
409 struct bucket_table *tbl, unsigned int hash)
410{
411 return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) :
412 &tbl->buckets[hash];
413}
414
415static inline struct rhash_head __rcu **rht_bucket_insert(
416 struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)
417{
418 return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) :
419 &tbl->buckets[hash];
420}
421
422/** 392/**
423 * rht_for_each_continue - continue iterating over hash chain 393 * rht_for_each_continue - continue iterating over hash chain
424 * @pos: the &struct rhash_head to use as a loop cursor. 394 * @pos: the &struct rhash_head to use as a loop cursor.
@@ -438,7 +408,7 @@ static inline struct rhash_head __rcu **rht_bucket_insert(
438 * @hash: the hash value / bucket index 408 * @hash: the hash value / bucket index
439 */ 409 */
440#define rht_for_each(pos, tbl, hash) \ 410#define rht_for_each(pos, tbl, hash) \
441 rht_for_each_continue(pos, *rht_bucket(tbl, hash), tbl, hash) 411 rht_for_each_continue(pos, (tbl)->buckets[hash], tbl, hash)
442 412
443/** 413/**
444 * rht_for_each_entry_continue - continue iterating over hash chain 414 * rht_for_each_entry_continue - continue iterating over hash chain
@@ -463,7 +433,7 @@ static inline struct rhash_head __rcu **rht_bucket_insert(
463 * @member: name of the &struct rhash_head within the hashable struct. 433 * @member: name of the &struct rhash_head within the hashable struct.
464 */ 434 */
465#define rht_for_each_entry(tpos, pos, tbl, hash, member) \ 435#define rht_for_each_entry(tpos, pos, tbl, hash, member) \
466 rht_for_each_entry_continue(tpos, pos, *rht_bucket(tbl, hash), \ 436 rht_for_each_entry_continue(tpos, pos, (tbl)->buckets[hash], \
467 tbl, hash, member) 437 tbl, hash, member)
468 438
469/** 439/**
@@ -478,13 +448,13 @@ static inline struct rhash_head __rcu **rht_bucket_insert(
478 * This hash chain list-traversal primitive allows for the looped code to 448 * This hash chain list-traversal primitive allows for the looped code to
479 * remove the loop cursor from the list. 449 * remove the loop cursor from the list.
480 */ 450 */
481#define rht_for_each_entry_safe(tpos, pos, next, tbl, hash, member) \ 451#define rht_for_each_entry_safe(tpos, pos, next, tbl, hash, member) \
482 for (pos = rht_dereference_bucket(*rht_bucket(tbl, hash), tbl, hash), \ 452 for (pos = rht_dereference_bucket((tbl)->buckets[hash], tbl, hash), \
483 next = !rht_is_a_nulls(pos) ? \ 453 next = !rht_is_a_nulls(pos) ? \
484 rht_dereference_bucket(pos->next, tbl, hash) : NULL; \ 454 rht_dereference_bucket(pos->next, tbl, hash) : NULL; \
485 (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member); \ 455 (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member); \
486 pos = next, \ 456 pos = next, \
487 next = !rht_is_a_nulls(pos) ? \ 457 next = !rht_is_a_nulls(pos) ? \
488 rht_dereference_bucket(pos->next, tbl, hash) : NULL) 458 rht_dereference_bucket(pos->next, tbl, hash) : NULL)
489 459
490/** 460/**
@@ -515,7 +485,7 @@ static inline struct rhash_head __rcu **rht_bucket_insert(
515 * traversal is guarded by rcu_read_lock(). 485 * traversal is guarded by rcu_read_lock().
516 */ 486 */
517#define rht_for_each_rcu(pos, tbl, hash) \ 487#define rht_for_each_rcu(pos, tbl, hash) \
518 rht_for_each_rcu_continue(pos, *rht_bucket(tbl, hash), tbl, hash) 488 rht_for_each_rcu_continue(pos, (tbl)->buckets[hash], tbl, hash)
519 489
520/** 490/**
521 * rht_for_each_entry_rcu_continue - continue iterating over rcu hash chain 491 * rht_for_each_entry_rcu_continue - continue iterating over rcu hash chain
@@ -548,8 +518,8 @@ static inline struct rhash_head __rcu **rht_bucket_insert(
548 * the _rcu mutation primitives such as rhashtable_insert() as long as the 518 * the _rcu mutation primitives such as rhashtable_insert() as long as the
549 * traversal is guarded by rcu_read_lock(). 519 * traversal is guarded by rcu_read_lock().
550 */ 520 */
551#define rht_for_each_entry_rcu(tpos, pos, tbl, hash, member) \ 521#define rht_for_each_entry_rcu(tpos, pos, tbl, hash, member) \
552 rht_for_each_entry_rcu_continue(tpos, pos, *rht_bucket(tbl, hash), \ 522 rht_for_each_entry_rcu_continue(tpos, pos, (tbl)->buckets[hash],\
553 tbl, hash, member) 523 tbl, hash, member)
554 524
555/** 525/**
@@ -595,7 +565,7 @@ static inline struct rhash_head *__rhashtable_lookup(
595 .ht = ht, 565 .ht = ht,
596 .key = key, 566 .key = key,
597 }; 567 };
598 struct bucket_table *tbl; 568 const struct bucket_table *tbl;
599 struct rhash_head *he; 569 struct rhash_head *he;
600 unsigned int hash; 570 unsigned int hash;
601 571
@@ -727,12 +697,8 @@ slow_path:
727 } 697 }
728 698
729 elasticity = ht->elasticity; 699 elasticity = ht->elasticity;
730 pprev = rht_bucket_insert(ht, tbl, hash); 700 pprev = &tbl->buckets[hash];
731 data = ERR_PTR(-ENOMEM); 701 rht_for_each(head, tbl, hash) {
732 if (!pprev)
733 goto out;
734
735 rht_for_each_continue(head, *pprev, tbl, hash) {
736 struct rhlist_head *plist; 702 struct rhlist_head *plist;
737 struct rhlist_head *list; 703 struct rhlist_head *list;
738 704
@@ -770,7 +736,7 @@ slow_path:
770 if (unlikely(rht_grow_above_100(ht, tbl))) 736 if (unlikely(rht_grow_above_100(ht, tbl)))
771 goto slow_path; 737 goto slow_path;
772 738
773 head = rht_dereference_bucket(*pprev, tbl, hash); 739 head = rht_dereference_bucket(tbl->buckets[hash], tbl, hash);
774 740
775 RCU_INIT_POINTER(obj->next, head); 741 RCU_INIT_POINTER(obj->next, head);
776 if (rhlist) { 742 if (rhlist) {
@@ -780,7 +746,7 @@ slow_path:
780 RCU_INIT_POINTER(list->next, NULL); 746 RCU_INIT_POINTER(list->next, NULL);
781 } 747 }
782 748
783 rcu_assign_pointer(*pprev, obj); 749 rcu_assign_pointer(tbl->buckets[hash], obj);
784 750
785 atomic_inc(&ht->nelems); 751 atomic_inc(&ht->nelems);
786 if (rht_grow_above_75(ht, tbl)) 752 if (rht_grow_above_75(ht, tbl))
@@ -989,8 +955,8 @@ static inline int __rhashtable_remove_fast_one(
989 955
990 spin_lock_bh(lock); 956 spin_lock_bh(lock);
991 957
992 pprev = rht_bucket_var(tbl, hash); 958 pprev = &tbl->buckets[hash];
993 rht_for_each_continue(he, *pprev, tbl, hash) { 959 rht_for_each(he, tbl, hash) {
994 struct rhlist_head *list; 960 struct rhlist_head *list;
995 961
996 list = container_of(he, struct rhlist_head, rhead); 962 list = container_of(he, struct rhlist_head, rhead);
@@ -1141,8 +1107,8 @@ static inline int __rhashtable_replace_fast(
1141 1107
1142 spin_lock_bh(lock); 1108 spin_lock_bh(lock);
1143 1109
1144 pprev = rht_bucket_var(tbl, hash); 1110 pprev = &tbl->buckets[hash];
1145 rht_for_each_continue(he, *pprev, tbl, hash) { 1111 rht_for_each(he, tbl, hash) {
1146 if (he != obj_old) { 1112 if (he != obj_old) {
1147 pprev = &he->next; 1113 pprev = &he->next;
1148 continue; 1114 continue;
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index 172454e6b979..32d0ad058380 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -32,11 +32,6 @@
32#define HASH_MIN_SIZE 4U 32#define HASH_MIN_SIZE 4U
33#define BUCKET_LOCKS_PER_CPU 32UL 33#define BUCKET_LOCKS_PER_CPU 32UL
34 34
35union nested_table {
36 union nested_table __rcu *table;
37 struct rhash_head __rcu *bucket;
38};
39
40static u32 head_hashfn(struct rhashtable *ht, 35static u32 head_hashfn(struct rhashtable *ht,
41 const struct bucket_table *tbl, 36 const struct bucket_table *tbl,
42 const struct rhash_head *he) 37 const struct rhash_head *he)
@@ -81,9 +76,6 @@ static int alloc_bucket_locks(struct rhashtable *ht, struct bucket_table *tbl,
81 /* Never allocate more than 0.5 locks per bucket */ 76 /* Never allocate more than 0.5 locks per bucket */
82 size = min_t(unsigned int, size, tbl->size >> 1); 77 size = min_t(unsigned int, size, tbl->size >> 1);
83 78
84 if (tbl->nest)
85 size = min(size, 1U << tbl->nest);
86
87 if (sizeof(spinlock_t) != 0) { 79 if (sizeof(spinlock_t) != 0) {
88 tbl->locks = NULL; 80 tbl->locks = NULL;
89#ifdef CONFIG_NUMA 81#ifdef CONFIG_NUMA
@@ -107,45 +99,8 @@ static int alloc_bucket_locks(struct rhashtable *ht, struct bucket_table *tbl,
107 return 0; 99 return 0;
108} 100}
109 101
110static void nested_table_free(union nested_table *ntbl, unsigned int size)
111{
112 const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
113 const unsigned int len = 1 << shift;
114 unsigned int i;
115
116 ntbl = rcu_dereference_raw(ntbl->table);
117 if (!ntbl)
118 return;
119
120 if (size > len) {
121 size >>= shift;
122 for (i = 0; i < len; i++)
123 nested_table_free(ntbl + i, size);
124 }
125
126 kfree(ntbl);
127}
128
129static void nested_bucket_table_free(const struct bucket_table *tbl)
130{
131 unsigned int size = tbl->size >> tbl->nest;
132 unsigned int len = 1 << tbl->nest;
133 union nested_table *ntbl;
134 unsigned int i;
135
136 ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]);
137
138 for (i = 0; i < len; i++)
139 nested_table_free(ntbl + i, size);
140
141 kfree(ntbl);
142}
143
144static void bucket_table_free(const struct bucket_table *tbl) 102static void bucket_table_free(const struct bucket_table *tbl)
145{ 103{
146 if (tbl->nest)
147 nested_bucket_table_free(tbl);
148
149 if (tbl) 104 if (tbl)
150 kvfree(tbl->locks); 105 kvfree(tbl->locks);
151 106
@@ -157,59 +112,6 @@ static void bucket_table_free_rcu(struct rcu_head *head)
157 bucket_table_free(container_of(head, struct bucket_table, rcu)); 112 bucket_table_free(container_of(head, struct bucket_table, rcu));
158} 113}
159 114
160static union nested_table *nested_table_alloc(struct rhashtable *ht,
161 union nested_table __rcu **prev,
162 unsigned int shifted,
163 unsigned int nhash)
164{
165 union nested_table *ntbl;
166 int i;
167
168 ntbl = rcu_dereference(*prev);
169 if (ntbl)
170 return ntbl;
171
172 ntbl = kzalloc(PAGE_SIZE, GFP_ATOMIC);
173
174 if (ntbl && shifted) {
175 for (i = 0; i < PAGE_SIZE / sizeof(ntbl[0].bucket); i++)
176 INIT_RHT_NULLS_HEAD(ntbl[i].bucket, ht,
177 (i << shifted) | nhash);
178 }
179
180 rcu_assign_pointer(*prev, ntbl);
181
182 return ntbl;
183}
184
185static struct bucket_table *nested_bucket_table_alloc(struct rhashtable *ht,
186 size_t nbuckets,
187 gfp_t gfp)
188{
189 const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
190 struct bucket_table *tbl;
191 size_t size;
192
193 if (nbuckets < (1 << (shift + 1)))
194 return NULL;
195
196 size = sizeof(*tbl) + sizeof(tbl->buckets[0]);
197
198 tbl = kzalloc(size, gfp);
199 if (!tbl)
200 return NULL;
201
202 if (!nested_table_alloc(ht, (union nested_table __rcu **)tbl->buckets,
203 0, 0)) {
204 kfree(tbl);
205 return NULL;
206 }
207
208 tbl->nest = (ilog2(nbuckets) - 1) % shift + 1;
209
210 return tbl;
211}
212
213static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, 115static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
214 size_t nbuckets, 116 size_t nbuckets,
215 gfp_t gfp) 117 gfp_t gfp)
@@ -224,17 +126,10 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
224 tbl = kzalloc(size, gfp | __GFP_NOWARN | __GFP_NORETRY); 126 tbl = kzalloc(size, gfp | __GFP_NOWARN | __GFP_NORETRY);
225 if (tbl == NULL && gfp == GFP_KERNEL) 127 if (tbl == NULL && gfp == GFP_KERNEL)
226 tbl = vzalloc(size); 128 tbl = vzalloc(size);
227
228 size = nbuckets;
229
230 if (tbl == NULL && gfp != GFP_KERNEL) {
231 tbl = nested_bucket_table_alloc(ht, nbuckets, gfp);
232 nbuckets = 0;
233 }
234 if (tbl == NULL) 129 if (tbl == NULL)
235 return NULL; 130 return NULL;
236 131
237 tbl->size = size; 132 tbl->size = nbuckets;
238 133
239 if (alloc_bucket_locks(ht, tbl, gfp) < 0) { 134 if (alloc_bucket_locks(ht, tbl, gfp) < 0) {
240 bucket_table_free(tbl); 135 bucket_table_free(tbl);
@@ -269,17 +164,12 @@ static int rhashtable_rehash_one(struct rhashtable *ht, unsigned int old_hash)
269 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); 164 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
270 struct bucket_table *new_tbl = rhashtable_last_table(ht, 165 struct bucket_table *new_tbl = rhashtable_last_table(ht,
271 rht_dereference_rcu(old_tbl->future_tbl, ht)); 166 rht_dereference_rcu(old_tbl->future_tbl, ht));
272 struct rhash_head __rcu **pprev = rht_bucket_var(old_tbl, old_hash); 167 struct rhash_head __rcu **pprev = &old_tbl->buckets[old_hash];
273 int err = -EAGAIN; 168 int err = -ENOENT;
274 struct rhash_head *head, *next, *entry; 169 struct rhash_head *head, *next, *entry;
275 spinlock_t *new_bucket_lock; 170 spinlock_t *new_bucket_lock;
276 unsigned int new_hash; 171 unsigned int new_hash;
277 172
278 if (new_tbl->nest)
279 goto out;
280
281 err = -ENOENT;
282
283 rht_for_each(entry, old_tbl, old_hash) { 173 rht_for_each(entry, old_tbl, old_hash) {
284 err = 0; 174 err = 0;
285 next = rht_dereference_bucket(entry->next, old_tbl, old_hash); 175 next = rht_dereference_bucket(entry->next, old_tbl, old_hash);
@@ -312,26 +202,19 @@ out:
312 return err; 202 return err;
313} 203}
314 204
315static int rhashtable_rehash_chain(struct rhashtable *ht, 205static void rhashtable_rehash_chain(struct rhashtable *ht,
316 unsigned int old_hash) 206 unsigned int old_hash)
317{ 207{
318 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); 208 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
319 spinlock_t *old_bucket_lock; 209 spinlock_t *old_bucket_lock;
320 int err;
321 210
322 old_bucket_lock = rht_bucket_lock(old_tbl, old_hash); 211 old_bucket_lock = rht_bucket_lock(old_tbl, old_hash);
323 212
324 spin_lock_bh(old_bucket_lock); 213 spin_lock_bh(old_bucket_lock);
325 while (!(err = rhashtable_rehash_one(ht, old_hash))) 214 while (!rhashtable_rehash_one(ht, old_hash))
326 ; 215 ;
327 216 old_tbl->rehash++;
328 if (err == -ENOENT) {
329 old_tbl->rehash++;
330 err = 0;
331 }
332 spin_unlock_bh(old_bucket_lock); 217 spin_unlock_bh(old_bucket_lock);
333
334 return err;
335} 218}
336 219
337static int rhashtable_rehash_attach(struct rhashtable *ht, 220static int rhashtable_rehash_attach(struct rhashtable *ht,
@@ -363,17 +246,13 @@ static int rhashtable_rehash_table(struct rhashtable *ht)
363 struct bucket_table *new_tbl; 246 struct bucket_table *new_tbl;
364 struct rhashtable_walker *walker; 247 struct rhashtable_walker *walker;
365 unsigned int old_hash; 248 unsigned int old_hash;
366 int err;
367 249
368 new_tbl = rht_dereference(old_tbl->future_tbl, ht); 250 new_tbl = rht_dereference(old_tbl->future_tbl, ht);
369 if (!new_tbl) 251 if (!new_tbl)
370 return 0; 252 return 0;
371 253
372 for (old_hash = 0; old_hash < old_tbl->size; old_hash++) { 254 for (old_hash = 0; old_hash < old_tbl->size; old_hash++)
373 err = rhashtable_rehash_chain(ht, old_hash); 255 rhashtable_rehash_chain(ht, old_hash);
374 if (err)
375 return err;
376 }
377 256
378 /* Publish the new table pointer. */ 257 /* Publish the new table pointer. */
379 rcu_assign_pointer(ht->tbl, new_tbl); 258 rcu_assign_pointer(ht->tbl, new_tbl);
@@ -392,16 +271,31 @@ static int rhashtable_rehash_table(struct rhashtable *ht)
392 return rht_dereference(new_tbl->future_tbl, ht) ? -EAGAIN : 0; 271 return rht_dereference(new_tbl->future_tbl, ht) ? -EAGAIN : 0;
393} 272}
394 273
395static int rhashtable_rehash_alloc(struct rhashtable *ht, 274/**
396 struct bucket_table *old_tbl, 275 * rhashtable_expand - Expand hash table while allowing concurrent lookups
397 unsigned int size) 276 * @ht: the hash table to expand
277 *
278 * A secondary bucket array is allocated and the hash entries are migrated.
279 *
280 * This function may only be called in a context where it is safe to call
281 * synchronize_rcu(), e.g. not within a rcu_read_lock() section.
282 *
283 * The caller must ensure that no concurrent resizing occurs by holding
284 * ht->mutex.
285 *
286 * It is valid to have concurrent insertions and deletions protected by per
287 * bucket locks or concurrent RCU protected lookups and traversals.
288 */
289static int rhashtable_expand(struct rhashtable *ht)
398{ 290{
399 struct bucket_table *new_tbl; 291 struct bucket_table *new_tbl, *old_tbl = rht_dereference(ht->tbl, ht);
400 int err; 292 int err;
401 293
402 ASSERT_RHT_MUTEX(ht); 294 ASSERT_RHT_MUTEX(ht);
403 295
404 new_tbl = bucket_table_alloc(ht, size, GFP_KERNEL); 296 old_tbl = rhashtable_last_table(ht, old_tbl);
297
298 new_tbl = bucket_table_alloc(ht, old_tbl->size * 2, GFP_KERNEL);
405 if (new_tbl == NULL) 299 if (new_tbl == NULL)
406 return -ENOMEM; 300 return -ENOMEM;
407 301
@@ -430,9 +324,12 @@ static int rhashtable_rehash_alloc(struct rhashtable *ht,
430 */ 324 */
431static int rhashtable_shrink(struct rhashtable *ht) 325static int rhashtable_shrink(struct rhashtable *ht)
432{ 326{
433 struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); 327 struct bucket_table *new_tbl, *old_tbl = rht_dereference(ht->tbl, ht);
434 unsigned int nelems = atomic_read(&ht->nelems); 328 unsigned int nelems = atomic_read(&ht->nelems);
435 unsigned int size = 0; 329 unsigned int size = 0;
330 int err;
331
332 ASSERT_RHT_MUTEX(ht);
436 333
437 if (nelems) 334 if (nelems)
438 size = roundup_pow_of_two(nelems * 3 / 2); 335 size = roundup_pow_of_two(nelems * 3 / 2);
@@ -445,7 +342,15 @@ static int rhashtable_shrink(struct rhashtable *ht)
445 if (rht_dereference(old_tbl->future_tbl, ht)) 342 if (rht_dereference(old_tbl->future_tbl, ht))
446 return -EEXIST; 343 return -EEXIST;
447 344
448 return rhashtable_rehash_alloc(ht, old_tbl, size); 345 new_tbl = bucket_table_alloc(ht, size, GFP_KERNEL);
346 if (new_tbl == NULL)
347 return -ENOMEM;
348
349 err = rhashtable_rehash_attach(ht, old_tbl, new_tbl);
350 if (err)
351 bucket_table_free(new_tbl);
352
353 return err;
449} 354}
450 355
451static void rht_deferred_worker(struct work_struct *work) 356static void rht_deferred_worker(struct work_struct *work)
@@ -461,14 +366,11 @@ static void rht_deferred_worker(struct work_struct *work)
461 tbl = rhashtable_last_table(ht, tbl); 366 tbl = rhashtable_last_table(ht, tbl);
462 367
463 if (rht_grow_above_75(ht, tbl)) 368 if (rht_grow_above_75(ht, tbl))
464 err = rhashtable_rehash_alloc(ht, tbl, tbl->size * 2); 369 rhashtable_expand(ht);
465 else if (ht->p.automatic_shrinking && rht_shrink_below_30(ht, tbl)) 370 else if (ht->p.automatic_shrinking && rht_shrink_below_30(ht, tbl))
466 err = rhashtable_shrink(ht); 371 rhashtable_shrink(ht);
467 else if (tbl->nest)
468 err = rhashtable_rehash_alloc(ht, tbl, tbl->size);
469 372
470 if (!err) 373 err = rhashtable_rehash_table(ht);
471 err = rhashtable_rehash_table(ht);
472 374
473 mutex_unlock(&ht->mutex); 375 mutex_unlock(&ht->mutex);
474 376
@@ -537,8 +439,8 @@ static void *rhashtable_lookup_one(struct rhashtable *ht,
537 int elasticity; 439 int elasticity;
538 440
539 elasticity = ht->elasticity; 441 elasticity = ht->elasticity;
540 pprev = rht_bucket_var(tbl, hash); 442 pprev = &tbl->buckets[hash];
541 rht_for_each_continue(head, *pprev, tbl, hash) { 443 rht_for_each(head, tbl, hash) {
542 struct rhlist_head *list; 444 struct rhlist_head *list;
543 struct rhlist_head *plist; 445 struct rhlist_head *plist;
544 446
@@ -575,7 +477,6 @@ static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht,
575 struct rhash_head *obj, 477 struct rhash_head *obj,
576 void *data) 478 void *data)
577{ 479{
578 struct rhash_head __rcu **pprev;
579 struct bucket_table *new_tbl; 480 struct bucket_table *new_tbl;
580 struct rhash_head *head; 481 struct rhash_head *head;
581 482
@@ -598,11 +499,7 @@ static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht,
598 if (unlikely(rht_grow_above_100(ht, tbl))) 499 if (unlikely(rht_grow_above_100(ht, tbl)))
599 return ERR_PTR(-EAGAIN); 500 return ERR_PTR(-EAGAIN);
600 501
601 pprev = rht_bucket_insert(ht, tbl, hash); 502 head = rht_dereference_bucket(tbl->buckets[hash], tbl, hash);
602 if (!pprev)
603 return ERR_PTR(-ENOMEM);
604
605 head = rht_dereference_bucket(*pprev, tbl, hash);
606 503
607 RCU_INIT_POINTER(obj->next, head); 504 RCU_INIT_POINTER(obj->next, head);
608 if (ht->rhlist) { 505 if (ht->rhlist) {
@@ -612,7 +509,7 @@ static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht,
612 RCU_INIT_POINTER(list->next, NULL); 509 RCU_INIT_POINTER(list->next, NULL);
613 } 510 }
614 511
615 rcu_assign_pointer(*pprev, obj); 512 rcu_assign_pointer(tbl->buckets[hash], obj);
616 513
617 atomic_inc(&ht->nelems); 514 atomic_inc(&ht->nelems);
618 if (rht_grow_above_75(ht, tbl)) 515 if (rht_grow_above_75(ht, tbl))
@@ -1078,7 +975,7 @@ void rhashtable_free_and_destroy(struct rhashtable *ht,
1078 void (*free_fn)(void *ptr, void *arg), 975 void (*free_fn)(void *ptr, void *arg),
1079 void *arg) 976 void *arg)
1080{ 977{
1081 struct bucket_table *tbl; 978 const struct bucket_table *tbl;
1082 unsigned int i; 979 unsigned int i;
1083 980
1084 cancel_work_sync(&ht->run_work); 981 cancel_work_sync(&ht->run_work);
@@ -1089,7 +986,7 @@ void rhashtable_free_and_destroy(struct rhashtable *ht,
1089 for (i = 0; i < tbl->size; i++) { 986 for (i = 0; i < tbl->size; i++) {
1090 struct rhash_head *pos, *next; 987 struct rhash_head *pos, *next;
1091 988
1092 for (pos = rht_dereference(*rht_bucket(tbl, i), ht), 989 for (pos = rht_dereference(tbl->buckets[i], ht),
1093 next = !rht_is_a_nulls(pos) ? 990 next = !rht_is_a_nulls(pos) ?
1094 rht_dereference(pos->next, ht) : NULL; 991 rht_dereference(pos->next, ht) : NULL;
1095 !rht_is_a_nulls(pos); 992 !rht_is_a_nulls(pos);
@@ -1110,70 +1007,3 @@ void rhashtable_destroy(struct rhashtable *ht)
1110 return rhashtable_free_and_destroy(ht, NULL, NULL); 1007 return rhashtable_free_and_destroy(ht, NULL, NULL);
1111} 1008}
1112EXPORT_SYMBOL_GPL(rhashtable_destroy); 1009EXPORT_SYMBOL_GPL(rhashtable_destroy);
1113
1114struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl,
1115 unsigned int hash)
1116{
1117 const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
1118 static struct rhash_head __rcu *rhnull =
1119 (struct rhash_head __rcu *)NULLS_MARKER(0);
1120 unsigned int index = hash & ((1 << tbl->nest) - 1);
1121 unsigned int size = tbl->size >> tbl->nest;
1122 unsigned int subhash = hash;
1123 union nested_table *ntbl;
1124
1125 ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]);
1126 ntbl = rht_dereference_bucket(ntbl[index].table, tbl, hash);
1127 subhash >>= tbl->nest;
1128
1129 while (ntbl && size > (1 << shift)) {
1130 index = subhash & ((1 << shift) - 1);
1131 ntbl = rht_dereference_bucket(ntbl[index].table, tbl, hash);
1132 size >>= shift;
1133 subhash >>= shift;
1134 }
1135
1136 if (!ntbl)
1137 return &rhnull;
1138
1139 return &ntbl[subhash].bucket;
1140
1141}
1142EXPORT_SYMBOL_GPL(rht_bucket_nested);
1143
1144struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht,
1145 struct bucket_table *tbl,
1146 unsigned int hash)
1147{
1148 const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
1149 unsigned int index = hash & ((1 << tbl->nest) - 1);
1150 unsigned int size = tbl->size >> tbl->nest;
1151 union nested_table *ntbl;
1152 unsigned int shifted;
1153 unsigned int nhash;
1154
1155 ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]);
1156 hash >>= tbl->nest;
1157 nhash = index;
1158 shifted = tbl->nest;
1159 ntbl = nested_table_alloc(ht, &ntbl[index].table,
1160 size <= (1 << shift) ? shifted : 0, nhash);
1161
1162 while (ntbl && size > (1 << shift)) {
1163 index = hash & ((1 << shift) - 1);
1164 size >>= shift;
1165 hash >>= shift;
1166 nhash |= index << shifted;
1167 shifted += shift;
1168 ntbl = nested_table_alloc(ht, &ntbl[index].table,
1169 size <= (1 << shift) ? shifted : 0,
1170 nhash);
1171 }
1172
1173 if (!ntbl)
1174 return NULL;
1175
1176 return &ntbl[hash].bucket;
1177
1178}
1179EXPORT_SYMBOL_GPL(rht_bucket_nested_insert);
diff --git a/net/tipc/net.c b/net/tipc/net.c
index ab8a2d5d1e32..28bf4feeb81c 100644
--- a/net/tipc/net.c
+++ b/net/tipc/net.c
@@ -110,10 +110,6 @@ int tipc_net_start(struct net *net, u32 addr)
110 char addr_string[16]; 110 char addr_string[16];
111 111
112 tn->own_addr = addr; 112 tn->own_addr = addr;
113
114 /* Ensure that the new address is visible before we reinit. */
115 smp_mb();
116
117 tipc_named_reinit(net); 113 tipc_named_reinit(net);
118 tipc_sk_reinit(net); 114 tipc_sk_reinit(net);
119 115
diff --git a/net/tipc/socket.c b/net/tipc/socket.c
index 370a5912bcb5..800caaa699a1 100644
--- a/net/tipc/socket.c
+++ b/net/tipc/socket.c
@@ -384,6 +384,8 @@ static int tipc_sk_create(struct net *net, struct socket *sock,
384 INIT_LIST_HEAD(&tsk->publications); 384 INIT_LIST_HEAD(&tsk->publications);
385 msg = &tsk->phdr; 385 msg = &tsk->phdr;
386 tn = net_generic(sock_net(sk), tipc_net_id); 386 tn = net_generic(sock_net(sk), tipc_net_id);
387 tipc_msg_init(tn->own_addr, msg, TIPC_LOW_IMPORTANCE, TIPC_NAMED_MSG,
388 NAMED_H_SIZE, 0);
387 389
388 /* Finish initializing socket data structures */ 390 /* Finish initializing socket data structures */
389 sock->ops = ops; 391 sock->ops = ops;
@@ -393,13 +395,6 @@ static int tipc_sk_create(struct net *net, struct socket *sock,
393 pr_warn("Socket create failed; port number exhausted\n"); 395 pr_warn("Socket create failed; port number exhausted\n");
394 return -EINVAL; 396 return -EINVAL;
395 } 397 }
396
397 /* Ensure tsk is visible before we read own_addr. */
398 smp_mb();
399
400 tipc_msg_init(tn->own_addr, msg, TIPC_LOW_IMPORTANCE, TIPC_NAMED_MSG,
401 NAMED_H_SIZE, 0);
402
403 msg_set_origport(msg, tsk->portid); 398 msg_set_origport(msg, tsk->portid);
404 setup_timer(&sk->sk_timer, tipc_sk_timeout, (unsigned long)tsk); 399 setup_timer(&sk->sk_timer, tipc_sk_timeout, (unsigned long)tsk);
405 sk->sk_shutdown = 0; 400 sk->sk_shutdown = 0;
@@ -2274,27 +2269,24 @@ static int tipc_sk_withdraw(struct tipc_sock *tsk, uint scope,
2274void tipc_sk_reinit(struct net *net) 2269void tipc_sk_reinit(struct net *net)
2275{ 2270{
2276 struct tipc_net *tn = net_generic(net, tipc_net_id); 2271 struct tipc_net *tn = net_generic(net, tipc_net_id);
2277 struct rhashtable_iter iter; 2272 const struct bucket_table *tbl;
2273 struct rhash_head *pos;
2278 struct tipc_sock *tsk; 2274 struct tipc_sock *tsk;
2279 struct tipc_msg *msg; 2275 struct tipc_msg *msg;
2276 int i;
2280 2277
2281 rhashtable_walk_enter(&tn->sk_rht, &iter); 2278 rcu_read_lock();
2282 2279 tbl = rht_dereference_rcu((&tn->sk_rht)->tbl, &tn->sk_rht);
2283 do { 2280 for (i = 0; i < tbl->size; i++) {
2284 tsk = ERR_PTR(rhashtable_walk_start(&iter)); 2281 rht_for_each_entry_rcu(tsk, pos, tbl, i, node) {
2285 if (tsk)
2286 continue;
2287
2288 while ((tsk = rhashtable_walk_next(&iter)) && !IS_ERR(tsk)) {
2289 spin_lock_bh(&tsk->sk.sk_lock.slock); 2282 spin_lock_bh(&tsk->sk.sk_lock.slock);
2290 msg = &tsk->phdr; 2283 msg = &tsk->phdr;
2291 msg_set_prevnode(msg, tn->own_addr); 2284 msg_set_prevnode(msg, tn->own_addr);
2292 msg_set_orignode(msg, tn->own_addr); 2285 msg_set_orignode(msg, tn->own_addr);
2293 spin_unlock_bh(&tsk->sk.sk_lock.slock); 2286 spin_unlock_bh(&tsk->sk.sk_lock.slock);
2294 } 2287 }
2295 2288 }
2296 rhashtable_walk_stop(&iter); 2289 rcu_read_unlock();
2297 } while (tsk == ERR_PTR(-EAGAIN));
2298} 2290}
2299 2291
2300static struct tipc_sock *tipc_sk_lookup(struct net *net, u32 portid) 2292static struct tipc_sock *tipc_sk_lookup(struct net *net, u32 portid)