diff options
| author | David S. Miller <davem@davemloft.net> | 2017-02-15 22:29:51 -0500 |
|---|---|---|
| committer | David S. Miller <davem@davemloft.net> | 2017-02-15 22:29:51 -0500 |
| commit | bf3f14d6342cfb37eab8f0cddd0e4d4063fd9fc9 (patch) | |
| tree | 2907cdfb8da2c43bf4c599bc3d398274e21dfc91 | |
| parent | 75224c93fa985f4a6fb983f53208f5c5aa555fbf (diff) | |
rhashtable: Revert nested table changes.
This reverts commits:
6a25478077d987edc5e2f880590a2bc5fcab4441
9dbbfb0ab6680c6a85609041011484e6658e7d3c
40137906c5f55c252194ef5834130383e639536f
It's too risky to put in this late in the release
cycle. We'll put these changes into the next merge
window instead.
Signed-off-by: David S. Miller <davem@davemloft.net>
| -rw-r--r-- | fs/gfs2/glock.c | 28 | ||||
| -rw-r--r-- | include/linux/rhashtable.h | 78 | ||||
| -rw-r--r-- | lib/rhashtable.c | 270 | ||||
| -rw-r--r-- | net/tipc/net.c | 4 | ||||
| -rw-r--r-- | net/tipc/socket.c | 30 |
5 files changed, 94 insertions, 316 deletions
diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index 70e94170af85..94f50cac91c6 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c | |||
| @@ -1420,32 +1420,26 @@ static struct shrinker glock_shrinker = { | |||
| 1420 | * @sdp: the filesystem | 1420 | * @sdp: the filesystem |
| 1421 | * @bucket: the bucket | 1421 | * @bucket: the bucket |
| 1422 | * | 1422 | * |
| 1423 | * Note that the function can be called multiple times on the same | ||
| 1424 | * object. So the user must ensure that the function can cope with | ||
| 1425 | * that. | ||
| 1426 | */ | 1423 | */ |
| 1427 | 1424 | ||
| 1428 | static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp) | 1425 | static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp) |
| 1429 | { | 1426 | { |
| 1430 | struct gfs2_glock *gl; | 1427 | struct gfs2_glock *gl; |
| 1431 | struct rhashtable_iter iter; | 1428 | struct rhash_head *pos; |
| 1432 | 1429 | const struct bucket_table *tbl; | |
| 1433 | rhashtable_walk_enter(&gl_hash_table, &iter); | 1430 | int i; |
| 1434 | |||
| 1435 | do { | ||
| 1436 | gl = ERR_PTR(rhashtable_walk_start(&iter)); | ||
| 1437 | if (gl) | ||
| 1438 | continue; | ||
| 1439 | 1431 | ||
| 1440 | while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl)) | 1432 | rcu_read_lock(); |
| 1433 | tbl = rht_dereference_rcu(gl_hash_table.tbl, &gl_hash_table); | ||
| 1434 | for (i = 0; i < tbl->size; i++) { | ||
| 1435 | rht_for_each_entry_rcu(gl, pos, tbl, i, gl_node) { | ||
| 1441 | if ((gl->gl_name.ln_sbd == sdp) && | 1436 | if ((gl->gl_name.ln_sbd == sdp) && |
| 1442 | lockref_get_not_dead(&gl->gl_lockref)) | 1437 | lockref_get_not_dead(&gl->gl_lockref)) |
| 1443 | examiner(gl); | 1438 | examiner(gl); |
| 1444 | 1439 | } | |
| 1445 | rhashtable_walk_stop(&iter); | 1440 | } |
| 1446 | } while (cond_resched(), gl == ERR_PTR(-EAGAIN)); | 1441 | rcu_read_unlock(); |
| 1447 | 1442 | cond_resched(); | |
| 1448 | rhashtable_walk_exit(&iter); | ||
| 1449 | } | 1443 | } |
| 1450 | 1444 | ||
| 1451 | /** | 1445 | /** |
diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h index f2e12a845910..5c132d3188be 100644 --- a/include/linux/rhashtable.h +++ b/include/linux/rhashtable.h | |||
| @@ -61,7 +61,6 @@ struct rhlist_head { | |||
| 61 | /** | 61 | /** |
| 62 | * struct bucket_table - Table of hash buckets | 62 | * struct bucket_table - Table of hash buckets |
| 63 | * @size: Number of hash buckets | 63 | * @size: Number of hash buckets |
| 64 | * @nest: Number of bits of first-level nested table. | ||
| 65 | * @rehash: Current bucket being rehashed | 64 | * @rehash: Current bucket being rehashed |
| 66 | * @hash_rnd: Random seed to fold into hash | 65 | * @hash_rnd: Random seed to fold into hash |
| 67 | * @locks_mask: Mask to apply before accessing locks[] | 66 | * @locks_mask: Mask to apply before accessing locks[] |
| @@ -69,12 +68,10 @@ struct rhlist_head { | |||
| 69 | * @walkers: List of active walkers | 68 | * @walkers: List of active walkers |
| 70 | * @rcu: RCU structure for freeing the table | 69 | * @rcu: RCU structure for freeing the table |
| 71 | * @future_tbl: Table under construction during rehashing | 70 | * @future_tbl: Table under construction during rehashing |
| 72 | * @ntbl: Nested table used when out of memory. | ||
| 73 | * @buckets: size * hash buckets | 71 | * @buckets: size * hash buckets |
| 74 | */ | 72 | */ |
| 75 | struct bucket_table { | 73 | struct bucket_table { |
| 76 | unsigned int size; | 74 | unsigned int size; |
| 77 | unsigned int nest; | ||
| 78 | unsigned int rehash; | 75 | unsigned int rehash; |
| 79 | u32 hash_rnd; | 76 | u32 hash_rnd; |
| 80 | unsigned int locks_mask; | 77 | unsigned int locks_mask; |
| @@ -84,7 +81,7 @@ struct bucket_table { | |||
| 84 | 81 | ||
| 85 | struct bucket_table __rcu *future_tbl; | 82 | struct bucket_table __rcu *future_tbl; |
| 86 | 83 | ||
| 87 | struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp; | 84 | struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp; |
| 88 | }; | 85 | }; |
| 89 | 86 | ||
| 90 | /** | 87 | /** |
| @@ -377,12 +374,6 @@ void rhashtable_free_and_destroy(struct rhashtable *ht, | |||
| 377 | void *arg); | 374 | void *arg); |
| 378 | void rhashtable_destroy(struct rhashtable *ht); | 375 | void rhashtable_destroy(struct rhashtable *ht); |
| 379 | 376 | ||
| 380 | struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, | ||
| 381 | unsigned int hash); | ||
| 382 | struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, | ||
| 383 | struct bucket_table *tbl, | ||
| 384 | unsigned int hash); | ||
| 385 | |||
| 386 | #define rht_dereference(p, ht) \ | 377 | #define rht_dereference(p, ht) \ |
| 387 | rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht)) | 378 | rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht)) |
| 388 | 379 | ||
| @@ -398,27 +389,6 @@ struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, | |||
| 398 | #define rht_entry(tpos, pos, member) \ | 389 | #define rht_entry(tpos, pos, member) \ |
| 399 | ({ tpos = container_of(pos, typeof(*tpos), member); 1; }) | 390 | ({ tpos = container_of(pos, typeof(*tpos), member); 1; }) |
| 400 | 391 | ||
| 401 | static inline struct rhash_head __rcu *const *rht_bucket( | ||
| 402 | const struct bucket_table *tbl, unsigned int hash) | ||
| 403 | { | ||
| 404 | return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) : | ||
| 405 | &tbl->buckets[hash]; | ||
| 406 | } | ||
| 407 | |||
| 408 | static inline struct rhash_head __rcu **rht_bucket_var( | ||
| 409 | struct bucket_table *tbl, unsigned int hash) | ||
| 410 | { | ||
| 411 | return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) : | ||
| 412 | &tbl->buckets[hash]; | ||
| 413 | } | ||
| 414 | |||
| 415 | static inline struct rhash_head __rcu **rht_bucket_insert( | ||
| 416 | struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash) | ||
| 417 | { | ||
| 418 | return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) : | ||
| 419 | &tbl->buckets[hash]; | ||
| 420 | } | ||
| 421 | |||
| 422 | /** | 392 | /** |
| 423 | * rht_for_each_continue - continue iterating over hash chain | 393 | * rht_for_each_continue - continue iterating over hash chain |
| 424 | * @pos: the &struct rhash_head to use as a loop cursor. | 394 | * @pos: the &struct rhash_head to use as a loop cursor. |
| @@ -438,7 +408,7 @@ static inline struct rhash_head __rcu **rht_bucket_insert( | |||
| 438 | * @hash: the hash value / bucket index | 408 | * @hash: the hash value / bucket index |
| 439 | */ | 409 | */ |
| 440 | #define rht_for_each(pos, tbl, hash) \ | 410 | #define rht_for_each(pos, tbl, hash) \ |
| 441 | rht_for_each_continue(pos, *rht_bucket(tbl, hash), tbl, hash) | 411 | rht_for_each_continue(pos, (tbl)->buckets[hash], tbl, hash) |
| 442 | 412 | ||
| 443 | /** | 413 | /** |
| 444 | * rht_for_each_entry_continue - continue iterating over hash chain | 414 | * rht_for_each_entry_continue - continue iterating over hash chain |
| @@ -463,7 +433,7 @@ static inline struct rhash_head __rcu **rht_bucket_insert( | |||
| 463 | * @member: name of the &struct rhash_head within the hashable struct. | 433 | * @member: name of the &struct rhash_head within the hashable struct. |
| 464 | */ | 434 | */ |
| 465 | #define rht_for_each_entry(tpos, pos, tbl, hash, member) \ | 435 | #define rht_for_each_entry(tpos, pos, tbl, hash, member) \ |
| 466 | rht_for_each_entry_continue(tpos, pos, *rht_bucket(tbl, hash), \ | 436 | rht_for_each_entry_continue(tpos, pos, (tbl)->buckets[hash], \ |
| 467 | tbl, hash, member) | 437 | tbl, hash, member) |
| 468 | 438 | ||
| 469 | /** | 439 | /** |
| @@ -478,13 +448,13 @@ static inline struct rhash_head __rcu **rht_bucket_insert( | |||
| 478 | * This hash chain list-traversal primitive allows for the looped code to | 448 | * This hash chain list-traversal primitive allows for the looped code to |
| 479 | * remove the loop cursor from the list. | 449 | * remove the loop cursor from the list. |
| 480 | */ | 450 | */ |
| 481 | #define rht_for_each_entry_safe(tpos, pos, next, tbl, hash, member) \ | 451 | #define rht_for_each_entry_safe(tpos, pos, next, tbl, hash, member) \ |
| 482 | for (pos = rht_dereference_bucket(*rht_bucket(tbl, hash), tbl, hash), \ | 452 | for (pos = rht_dereference_bucket((tbl)->buckets[hash], tbl, hash), \ |
| 483 | next = !rht_is_a_nulls(pos) ? \ | 453 | next = !rht_is_a_nulls(pos) ? \ |
| 484 | rht_dereference_bucket(pos->next, tbl, hash) : NULL; \ | 454 | rht_dereference_bucket(pos->next, tbl, hash) : NULL; \ |
| 485 | (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member); \ | 455 | (!rht_is_a_nulls(pos)) && rht_entry(tpos, pos, member); \ |
| 486 | pos = next, \ | 456 | pos = next, \ |
| 487 | next = !rht_is_a_nulls(pos) ? \ | 457 | next = !rht_is_a_nulls(pos) ? \ |
| 488 | rht_dereference_bucket(pos->next, tbl, hash) : NULL) | 458 | rht_dereference_bucket(pos->next, tbl, hash) : NULL) |
| 489 | 459 | ||
| 490 | /** | 460 | /** |
| @@ -515,7 +485,7 @@ static inline struct rhash_head __rcu **rht_bucket_insert( | |||
| 515 | * traversal is guarded by rcu_read_lock(). | 485 | * traversal is guarded by rcu_read_lock(). |
| 516 | */ | 486 | */ |
| 517 | #define rht_for_each_rcu(pos, tbl, hash) \ | 487 | #define rht_for_each_rcu(pos, tbl, hash) \ |
| 518 | rht_for_each_rcu_continue(pos, *rht_bucket(tbl, hash), tbl, hash) | 488 | rht_for_each_rcu_continue(pos, (tbl)->buckets[hash], tbl, hash) |
| 519 | 489 | ||
| 520 | /** | 490 | /** |
| 521 | * rht_for_each_entry_rcu_continue - continue iterating over rcu hash chain | 491 | * rht_for_each_entry_rcu_continue - continue iterating over rcu hash chain |
| @@ -548,8 +518,8 @@ static inline struct rhash_head __rcu **rht_bucket_insert( | |||
| 548 | * the _rcu mutation primitives such as rhashtable_insert() as long as the | 518 | * the _rcu mutation primitives such as rhashtable_insert() as long as the |
| 549 | * traversal is guarded by rcu_read_lock(). | 519 | * traversal is guarded by rcu_read_lock(). |
| 550 | */ | 520 | */ |
| 551 | #define rht_for_each_entry_rcu(tpos, pos, tbl, hash, member) \ | 521 | #define rht_for_each_entry_rcu(tpos, pos, tbl, hash, member) \ |
| 552 | rht_for_each_entry_rcu_continue(tpos, pos, *rht_bucket(tbl, hash), \ | 522 | rht_for_each_entry_rcu_continue(tpos, pos, (tbl)->buckets[hash],\ |
| 553 | tbl, hash, member) | 523 | tbl, hash, member) |
| 554 | 524 | ||
| 555 | /** | 525 | /** |
| @@ -595,7 +565,7 @@ static inline struct rhash_head *__rhashtable_lookup( | |||
| 595 | .ht = ht, | 565 | .ht = ht, |
| 596 | .key = key, | 566 | .key = key, |
| 597 | }; | 567 | }; |
| 598 | struct bucket_table *tbl; | 568 | const struct bucket_table *tbl; |
| 599 | struct rhash_head *he; | 569 | struct rhash_head *he; |
| 600 | unsigned int hash; | 570 | unsigned int hash; |
| 601 | 571 | ||
| @@ -727,12 +697,8 @@ slow_path: | |||
| 727 | } | 697 | } |
| 728 | 698 | ||
| 729 | elasticity = ht->elasticity; | 699 | elasticity = ht->elasticity; |
| 730 | pprev = rht_bucket_insert(ht, tbl, hash); | 700 | pprev = &tbl->buckets[hash]; |
| 731 | data = ERR_PTR(-ENOMEM); | 701 | rht_for_each(head, tbl, hash) { |
| 732 | if (!pprev) | ||
| 733 | goto out; | ||
| 734 | |||
| 735 | rht_for_each_continue(head, *pprev, tbl, hash) { | ||
| 736 | struct rhlist_head *plist; | 702 | struct rhlist_head *plist; |
| 737 | struct rhlist_head *list; | 703 | struct rhlist_head *list; |
| 738 | 704 | ||
| @@ -770,7 +736,7 @@ slow_path: | |||
| 770 | if (unlikely(rht_grow_above_100(ht, tbl))) | 736 | if (unlikely(rht_grow_above_100(ht, tbl))) |
| 771 | goto slow_path; | 737 | goto slow_path; |
| 772 | 738 | ||
| 773 | head = rht_dereference_bucket(*pprev, tbl, hash); | 739 | head = rht_dereference_bucket(tbl->buckets[hash], tbl, hash); |
| 774 | 740 | ||
| 775 | RCU_INIT_POINTER(obj->next, head); | 741 | RCU_INIT_POINTER(obj->next, head); |
| 776 | if (rhlist) { | 742 | if (rhlist) { |
| @@ -780,7 +746,7 @@ slow_path: | |||
| 780 | RCU_INIT_POINTER(list->next, NULL); | 746 | RCU_INIT_POINTER(list->next, NULL); |
| 781 | } | 747 | } |
| 782 | 748 | ||
| 783 | rcu_assign_pointer(*pprev, obj); | 749 | rcu_assign_pointer(tbl->buckets[hash], obj); |
| 784 | 750 | ||
| 785 | atomic_inc(&ht->nelems); | 751 | atomic_inc(&ht->nelems); |
| 786 | if (rht_grow_above_75(ht, tbl)) | 752 | if (rht_grow_above_75(ht, tbl)) |
| @@ -989,8 +955,8 @@ static inline int __rhashtable_remove_fast_one( | |||
| 989 | 955 | ||
| 990 | spin_lock_bh(lock); | 956 | spin_lock_bh(lock); |
| 991 | 957 | ||
| 992 | pprev = rht_bucket_var(tbl, hash); | 958 | pprev = &tbl->buckets[hash]; |
| 993 | rht_for_each_continue(he, *pprev, tbl, hash) { | 959 | rht_for_each(he, tbl, hash) { |
| 994 | struct rhlist_head *list; | 960 | struct rhlist_head *list; |
| 995 | 961 | ||
| 996 | list = container_of(he, struct rhlist_head, rhead); | 962 | list = container_of(he, struct rhlist_head, rhead); |
| @@ -1141,8 +1107,8 @@ static inline int __rhashtable_replace_fast( | |||
| 1141 | 1107 | ||
| 1142 | spin_lock_bh(lock); | 1108 | spin_lock_bh(lock); |
| 1143 | 1109 | ||
| 1144 | pprev = rht_bucket_var(tbl, hash); | 1110 | pprev = &tbl->buckets[hash]; |
| 1145 | rht_for_each_continue(he, *pprev, tbl, hash) { | 1111 | rht_for_each(he, tbl, hash) { |
| 1146 | if (he != obj_old) { | 1112 | if (he != obj_old) { |
| 1147 | pprev = &he->next; | 1113 | pprev = &he->next; |
| 1148 | continue; | 1114 | continue; |
diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 172454e6b979..32d0ad058380 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c | |||
| @@ -32,11 +32,6 @@ | |||
| 32 | #define HASH_MIN_SIZE 4U | 32 | #define HASH_MIN_SIZE 4U |
| 33 | #define BUCKET_LOCKS_PER_CPU 32UL | 33 | #define BUCKET_LOCKS_PER_CPU 32UL |
| 34 | 34 | ||
| 35 | union nested_table { | ||
| 36 | union nested_table __rcu *table; | ||
| 37 | struct rhash_head __rcu *bucket; | ||
| 38 | }; | ||
| 39 | |||
| 40 | static u32 head_hashfn(struct rhashtable *ht, | 35 | static u32 head_hashfn(struct rhashtable *ht, |
| 41 | const struct bucket_table *tbl, | 36 | const struct bucket_table *tbl, |
| 42 | const struct rhash_head *he) | 37 | const struct rhash_head *he) |
| @@ -81,9 +76,6 @@ static int alloc_bucket_locks(struct rhashtable *ht, struct bucket_table *tbl, | |||
| 81 | /* Never allocate more than 0.5 locks per bucket */ | 76 | /* Never allocate more than 0.5 locks per bucket */ |
| 82 | size = min_t(unsigned int, size, tbl->size >> 1); | 77 | size = min_t(unsigned int, size, tbl->size >> 1); |
| 83 | 78 | ||
| 84 | if (tbl->nest) | ||
| 85 | size = min(size, 1U << tbl->nest); | ||
| 86 | |||
| 87 | if (sizeof(spinlock_t) != 0) { | 79 | if (sizeof(spinlock_t) != 0) { |
| 88 | tbl->locks = NULL; | 80 | tbl->locks = NULL; |
| 89 | #ifdef CONFIG_NUMA | 81 | #ifdef CONFIG_NUMA |
| @@ -107,45 +99,8 @@ static int alloc_bucket_locks(struct rhashtable *ht, struct bucket_table *tbl, | |||
| 107 | return 0; | 99 | return 0; |
| 108 | } | 100 | } |
| 109 | 101 | ||
| 110 | static void nested_table_free(union nested_table *ntbl, unsigned int size) | ||
| 111 | { | ||
| 112 | const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); | ||
| 113 | const unsigned int len = 1 << shift; | ||
| 114 | unsigned int i; | ||
| 115 | |||
| 116 | ntbl = rcu_dereference_raw(ntbl->table); | ||
| 117 | if (!ntbl) | ||
| 118 | return; | ||
| 119 | |||
| 120 | if (size > len) { | ||
| 121 | size >>= shift; | ||
| 122 | for (i = 0; i < len; i++) | ||
| 123 | nested_table_free(ntbl + i, size); | ||
| 124 | } | ||
| 125 | |||
| 126 | kfree(ntbl); | ||
| 127 | } | ||
| 128 | |||
| 129 | static void nested_bucket_table_free(const struct bucket_table *tbl) | ||
| 130 | { | ||
| 131 | unsigned int size = tbl->size >> tbl->nest; | ||
| 132 | unsigned int len = 1 << tbl->nest; | ||
| 133 | union nested_table *ntbl; | ||
| 134 | unsigned int i; | ||
| 135 | |||
| 136 | ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]); | ||
| 137 | |||
| 138 | for (i = 0; i < len; i++) | ||
| 139 | nested_table_free(ntbl + i, size); | ||
| 140 | |||
| 141 | kfree(ntbl); | ||
| 142 | } | ||
| 143 | |||
| 144 | static void bucket_table_free(const struct bucket_table *tbl) | 102 | static void bucket_table_free(const struct bucket_table *tbl) |
| 145 | { | 103 | { |
| 146 | if (tbl->nest) | ||
| 147 | nested_bucket_table_free(tbl); | ||
| 148 | |||
| 149 | if (tbl) | 104 | if (tbl) |
| 150 | kvfree(tbl->locks); | 105 | kvfree(tbl->locks); |
| 151 | 106 | ||
| @@ -157,59 +112,6 @@ static void bucket_table_free_rcu(struct rcu_head *head) | |||
| 157 | bucket_table_free(container_of(head, struct bucket_table, rcu)); | 112 | bucket_table_free(container_of(head, struct bucket_table, rcu)); |
| 158 | } | 113 | } |
| 159 | 114 | ||
| 160 | static union nested_table *nested_table_alloc(struct rhashtable *ht, | ||
| 161 | union nested_table __rcu **prev, | ||
| 162 | unsigned int shifted, | ||
| 163 | unsigned int nhash) | ||
| 164 | { | ||
| 165 | union nested_table *ntbl; | ||
| 166 | int i; | ||
| 167 | |||
| 168 | ntbl = rcu_dereference(*prev); | ||
| 169 | if (ntbl) | ||
| 170 | return ntbl; | ||
| 171 | |||
| 172 | ntbl = kzalloc(PAGE_SIZE, GFP_ATOMIC); | ||
| 173 | |||
| 174 | if (ntbl && shifted) { | ||
| 175 | for (i = 0; i < PAGE_SIZE / sizeof(ntbl[0].bucket); i++) | ||
| 176 | INIT_RHT_NULLS_HEAD(ntbl[i].bucket, ht, | ||
| 177 | (i << shifted) | nhash); | ||
| 178 | } | ||
| 179 | |||
| 180 | rcu_assign_pointer(*prev, ntbl); | ||
| 181 | |||
| 182 | return ntbl; | ||
| 183 | } | ||
| 184 | |||
| 185 | static struct bucket_table *nested_bucket_table_alloc(struct rhashtable *ht, | ||
| 186 | size_t nbuckets, | ||
| 187 | gfp_t gfp) | ||
| 188 | { | ||
| 189 | const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); | ||
| 190 | struct bucket_table *tbl; | ||
| 191 | size_t size; | ||
| 192 | |||
| 193 | if (nbuckets < (1 << (shift + 1))) | ||
| 194 | return NULL; | ||
| 195 | |||
| 196 | size = sizeof(*tbl) + sizeof(tbl->buckets[0]); | ||
| 197 | |||
| 198 | tbl = kzalloc(size, gfp); | ||
| 199 | if (!tbl) | ||
| 200 | return NULL; | ||
| 201 | |||
| 202 | if (!nested_table_alloc(ht, (union nested_table __rcu **)tbl->buckets, | ||
| 203 | 0, 0)) { | ||
| 204 | kfree(tbl); | ||
| 205 | return NULL; | ||
| 206 | } | ||
| 207 | |||
| 208 | tbl->nest = (ilog2(nbuckets) - 1) % shift + 1; | ||
| 209 | |||
| 210 | return tbl; | ||
| 211 | } | ||
| 212 | |||
| 213 | static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, | 115 | static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, |
| 214 | size_t nbuckets, | 116 | size_t nbuckets, |
| 215 | gfp_t gfp) | 117 | gfp_t gfp) |
| @@ -224,17 +126,10 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht, | |||
| 224 | tbl = kzalloc(size, gfp | __GFP_NOWARN | __GFP_NORETRY); | 126 | tbl = kzalloc(size, gfp | __GFP_NOWARN | __GFP_NORETRY); |
| 225 | if (tbl == NULL && gfp == GFP_KERNEL) | 127 | if (tbl == NULL && gfp == GFP_KERNEL) |
| 226 | tbl = vzalloc(size); | 128 | tbl = vzalloc(size); |
| 227 | |||
| 228 | size = nbuckets; | ||
| 229 | |||
| 230 | if (tbl == NULL && gfp != GFP_KERNEL) { | ||
| 231 | tbl = nested_bucket_table_alloc(ht, nbuckets, gfp); | ||
| 232 | nbuckets = 0; | ||
| 233 | } | ||
| 234 | if (tbl == NULL) | 129 | if (tbl == NULL) |
| 235 | return NULL; | 130 | return NULL; |
| 236 | 131 | ||
| 237 | tbl->size = size; | 132 | tbl->size = nbuckets; |
| 238 | 133 | ||
| 239 | if (alloc_bucket_locks(ht, tbl, gfp) < 0) { | 134 | if (alloc_bucket_locks(ht, tbl, gfp) < 0) { |
| 240 | bucket_table_free(tbl); | 135 | bucket_table_free(tbl); |
| @@ -269,17 +164,12 @@ static int rhashtable_rehash_one(struct rhashtable *ht, unsigned int old_hash) | |||
| 269 | struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); | 164 | struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); |
| 270 | struct bucket_table *new_tbl = rhashtable_last_table(ht, | 165 | struct bucket_table *new_tbl = rhashtable_last_table(ht, |
| 271 | rht_dereference_rcu(old_tbl->future_tbl, ht)); | 166 | rht_dereference_rcu(old_tbl->future_tbl, ht)); |
| 272 | struct rhash_head __rcu **pprev = rht_bucket_var(old_tbl, old_hash); | 167 | struct rhash_head __rcu **pprev = &old_tbl->buckets[old_hash]; |
| 273 | int err = -EAGAIN; | 168 | int err = -ENOENT; |
| 274 | struct rhash_head *head, *next, *entry; | 169 | struct rhash_head *head, *next, *entry; |
| 275 | spinlock_t *new_bucket_lock; | 170 | spinlock_t *new_bucket_lock; |
| 276 | unsigned int new_hash; | 171 | unsigned int new_hash; |
| 277 | 172 | ||
| 278 | if (new_tbl->nest) | ||
| 279 | goto out; | ||
| 280 | |||
| 281 | err = -ENOENT; | ||
| 282 | |||
| 283 | rht_for_each(entry, old_tbl, old_hash) { | 173 | rht_for_each(entry, old_tbl, old_hash) { |
| 284 | err = 0; | 174 | err = 0; |
| 285 | next = rht_dereference_bucket(entry->next, old_tbl, old_hash); | 175 | next = rht_dereference_bucket(entry->next, old_tbl, old_hash); |
| @@ -312,26 +202,19 @@ out: | |||
| 312 | return err; | 202 | return err; |
| 313 | } | 203 | } |
| 314 | 204 | ||
| 315 | static int rhashtable_rehash_chain(struct rhashtable *ht, | 205 | static void rhashtable_rehash_chain(struct rhashtable *ht, |
| 316 | unsigned int old_hash) | 206 | unsigned int old_hash) |
| 317 | { | 207 | { |
| 318 | struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); | 208 | struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); |
| 319 | spinlock_t *old_bucket_lock; | 209 | spinlock_t *old_bucket_lock; |
| 320 | int err; | ||
| 321 | 210 | ||
| 322 | old_bucket_lock = rht_bucket_lock(old_tbl, old_hash); | 211 | old_bucket_lock = rht_bucket_lock(old_tbl, old_hash); |
| 323 | 212 | ||
| 324 | spin_lock_bh(old_bucket_lock); | 213 | spin_lock_bh(old_bucket_lock); |
| 325 | while (!(err = rhashtable_rehash_one(ht, old_hash))) | 214 | while (!rhashtable_rehash_one(ht, old_hash)) |
| 326 | ; | 215 | ; |
| 327 | 216 | old_tbl->rehash++; | |
| 328 | if (err == -ENOENT) { | ||
| 329 | old_tbl->rehash++; | ||
| 330 | err = 0; | ||
| 331 | } | ||
| 332 | spin_unlock_bh(old_bucket_lock); | 217 | spin_unlock_bh(old_bucket_lock); |
| 333 | |||
| 334 | return err; | ||
| 335 | } | 218 | } |
| 336 | 219 | ||
| 337 | static int rhashtable_rehash_attach(struct rhashtable *ht, | 220 | static int rhashtable_rehash_attach(struct rhashtable *ht, |
| @@ -363,17 +246,13 @@ static int rhashtable_rehash_table(struct rhashtable *ht) | |||
| 363 | struct bucket_table *new_tbl; | 246 | struct bucket_table *new_tbl; |
| 364 | struct rhashtable_walker *walker; | 247 | struct rhashtable_walker *walker; |
| 365 | unsigned int old_hash; | 248 | unsigned int old_hash; |
| 366 | int err; | ||
| 367 | 249 | ||
| 368 | new_tbl = rht_dereference(old_tbl->future_tbl, ht); | 250 | new_tbl = rht_dereference(old_tbl->future_tbl, ht); |
| 369 | if (!new_tbl) | 251 | if (!new_tbl) |
| 370 | return 0; | 252 | return 0; |
| 371 | 253 | ||
| 372 | for (old_hash = 0; old_hash < old_tbl->size; old_hash++) { | 254 | for (old_hash = 0; old_hash < old_tbl->size; old_hash++) |
| 373 | err = rhashtable_rehash_chain(ht, old_hash); | 255 | rhashtable_rehash_chain(ht, old_hash); |
| 374 | if (err) | ||
| 375 | return err; | ||
| 376 | } | ||
| 377 | 256 | ||
| 378 | /* Publish the new table pointer. */ | 257 | /* Publish the new table pointer. */ |
| 379 | rcu_assign_pointer(ht->tbl, new_tbl); | 258 | rcu_assign_pointer(ht->tbl, new_tbl); |
| @@ -392,16 +271,31 @@ static int rhashtable_rehash_table(struct rhashtable *ht) | |||
| 392 | return rht_dereference(new_tbl->future_tbl, ht) ? -EAGAIN : 0; | 271 | return rht_dereference(new_tbl->future_tbl, ht) ? -EAGAIN : 0; |
| 393 | } | 272 | } |
| 394 | 273 | ||
| 395 | static int rhashtable_rehash_alloc(struct rhashtable *ht, | 274 | /** |
| 396 | struct bucket_table *old_tbl, | 275 | * rhashtable_expand - Expand hash table while allowing concurrent lookups |
| 397 | unsigned int size) | 276 | * @ht: the hash table to expand |
| 277 | * | ||
| 278 | * A secondary bucket array is allocated and the hash entries are migrated. | ||
| 279 | * | ||
| 280 | * This function may only be called in a context where it is safe to call | ||
| 281 | * synchronize_rcu(), e.g. not within a rcu_read_lock() section. | ||
| 282 | * | ||
| 283 | * The caller must ensure that no concurrent resizing occurs by holding | ||
| 284 | * ht->mutex. | ||
| 285 | * | ||
| 286 | * It is valid to have concurrent insertions and deletions protected by per | ||
| 287 | * bucket locks or concurrent RCU protected lookups and traversals. | ||
| 288 | */ | ||
| 289 | static int rhashtable_expand(struct rhashtable *ht) | ||
| 398 | { | 290 | { |
| 399 | struct bucket_table *new_tbl; | 291 | struct bucket_table *new_tbl, *old_tbl = rht_dereference(ht->tbl, ht); |
| 400 | int err; | 292 | int err; |
| 401 | 293 | ||
| 402 | ASSERT_RHT_MUTEX(ht); | 294 | ASSERT_RHT_MUTEX(ht); |
| 403 | 295 | ||
| 404 | new_tbl = bucket_table_alloc(ht, size, GFP_KERNEL); | 296 | old_tbl = rhashtable_last_table(ht, old_tbl); |
| 297 | |||
| 298 | new_tbl = bucket_table_alloc(ht, old_tbl->size * 2, GFP_KERNEL); | ||
| 405 | if (new_tbl == NULL) | 299 | if (new_tbl == NULL) |
| 406 | return -ENOMEM; | 300 | return -ENOMEM; |
| 407 | 301 | ||
| @@ -430,9 +324,12 @@ static int rhashtable_rehash_alloc(struct rhashtable *ht, | |||
| 430 | */ | 324 | */ |
| 431 | static int rhashtable_shrink(struct rhashtable *ht) | 325 | static int rhashtable_shrink(struct rhashtable *ht) |
| 432 | { | 326 | { |
| 433 | struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); | 327 | struct bucket_table *new_tbl, *old_tbl = rht_dereference(ht->tbl, ht); |
| 434 | unsigned int nelems = atomic_read(&ht->nelems); | 328 | unsigned int nelems = atomic_read(&ht->nelems); |
| 435 | unsigned int size = 0; | 329 | unsigned int size = 0; |
| 330 | int err; | ||
| 331 | |||
| 332 | ASSERT_RHT_MUTEX(ht); | ||
| 436 | 333 | ||
| 437 | if (nelems) | 334 | if (nelems) |
| 438 | size = roundup_pow_of_two(nelems * 3 / 2); | 335 | size = roundup_pow_of_two(nelems * 3 / 2); |
| @@ -445,7 +342,15 @@ static int rhashtable_shrink(struct rhashtable *ht) | |||
| 445 | if (rht_dereference(old_tbl->future_tbl, ht)) | 342 | if (rht_dereference(old_tbl->future_tbl, ht)) |
| 446 | return -EEXIST; | 343 | return -EEXIST; |
| 447 | 344 | ||
| 448 | return rhashtable_rehash_alloc(ht, old_tbl, size); | 345 | new_tbl = bucket_table_alloc(ht, size, GFP_KERNEL); |
| 346 | if (new_tbl == NULL) | ||
| 347 | return -ENOMEM; | ||
| 348 | |||
| 349 | err = rhashtable_rehash_attach(ht, old_tbl, new_tbl); | ||
| 350 | if (err) | ||
| 351 | bucket_table_free(new_tbl); | ||
| 352 | |||
| 353 | return err; | ||
| 449 | } | 354 | } |
| 450 | 355 | ||
| 451 | static void rht_deferred_worker(struct work_struct *work) | 356 | static void rht_deferred_worker(struct work_struct *work) |
| @@ -461,14 +366,11 @@ static void rht_deferred_worker(struct work_struct *work) | |||
| 461 | tbl = rhashtable_last_table(ht, tbl); | 366 | tbl = rhashtable_last_table(ht, tbl); |
| 462 | 367 | ||
| 463 | if (rht_grow_above_75(ht, tbl)) | 368 | if (rht_grow_above_75(ht, tbl)) |
| 464 | err = rhashtable_rehash_alloc(ht, tbl, tbl->size * 2); | 369 | rhashtable_expand(ht); |
| 465 | else if (ht->p.automatic_shrinking && rht_shrink_below_30(ht, tbl)) | 370 | else if (ht->p.automatic_shrinking && rht_shrink_below_30(ht, tbl)) |
| 466 | err = rhashtable_shrink(ht); | 371 | rhashtable_shrink(ht); |
| 467 | else if (tbl->nest) | ||
| 468 | err = rhashtable_rehash_alloc(ht, tbl, tbl->size); | ||
| 469 | 372 | ||
| 470 | if (!err) | 373 | err = rhashtable_rehash_table(ht); |
| 471 | err = rhashtable_rehash_table(ht); | ||
| 472 | 374 | ||
| 473 | mutex_unlock(&ht->mutex); | 375 | mutex_unlock(&ht->mutex); |
| 474 | 376 | ||
| @@ -537,8 +439,8 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, | |||
| 537 | int elasticity; | 439 | int elasticity; |
| 538 | 440 | ||
| 539 | elasticity = ht->elasticity; | 441 | elasticity = ht->elasticity; |
| 540 | pprev = rht_bucket_var(tbl, hash); | 442 | pprev = &tbl->buckets[hash]; |
| 541 | rht_for_each_continue(head, *pprev, tbl, hash) { | 443 | rht_for_each(head, tbl, hash) { |
| 542 | struct rhlist_head *list; | 444 | struct rhlist_head *list; |
| 543 | struct rhlist_head *plist; | 445 | struct rhlist_head *plist; |
| 544 | 446 | ||
| @@ -575,7 +477,6 @@ static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht, | |||
| 575 | struct rhash_head *obj, | 477 | struct rhash_head *obj, |
| 576 | void *data) | 478 | void *data) |
| 577 | { | 479 | { |
| 578 | struct rhash_head __rcu **pprev; | ||
| 579 | struct bucket_table *new_tbl; | 480 | struct bucket_table *new_tbl; |
| 580 | struct rhash_head *head; | 481 | struct rhash_head *head; |
| 581 | 482 | ||
| @@ -598,11 +499,7 @@ static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht, | |||
| 598 | if (unlikely(rht_grow_above_100(ht, tbl))) | 499 | if (unlikely(rht_grow_above_100(ht, tbl))) |
| 599 | return ERR_PTR(-EAGAIN); | 500 | return ERR_PTR(-EAGAIN); |
| 600 | 501 | ||
| 601 | pprev = rht_bucket_insert(ht, tbl, hash); | 502 | head = rht_dereference_bucket(tbl->buckets[hash], tbl, hash); |
| 602 | if (!pprev) | ||
| 603 | return ERR_PTR(-ENOMEM); | ||
| 604 | |||
| 605 | head = rht_dereference_bucket(*pprev, tbl, hash); | ||
| 606 | 503 | ||
| 607 | RCU_INIT_POINTER(obj->next, head); | 504 | RCU_INIT_POINTER(obj->next, head); |
| 608 | if (ht->rhlist) { | 505 | if (ht->rhlist) { |
| @@ -612,7 +509,7 @@ static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht, | |||
| 612 | RCU_INIT_POINTER(list->next, NULL); | 509 | RCU_INIT_POINTER(list->next, NULL); |
| 613 | } | 510 | } |
| 614 | 511 | ||
| 615 | rcu_assign_pointer(*pprev, obj); | 512 | rcu_assign_pointer(tbl->buckets[hash], obj); |
| 616 | 513 | ||
| 617 | atomic_inc(&ht->nelems); | 514 | atomic_inc(&ht->nelems); |
| 618 | if (rht_grow_above_75(ht, tbl)) | 515 | if (rht_grow_above_75(ht, tbl)) |
| @@ -1078,7 +975,7 @@ void rhashtable_free_and_destroy(struct rhashtable *ht, | |||
| 1078 | void (*free_fn)(void *ptr, void *arg), | 975 | void (*free_fn)(void *ptr, void *arg), |
| 1079 | void *arg) | 976 | void *arg) |
| 1080 | { | 977 | { |
| 1081 | struct bucket_table *tbl; | 978 | const struct bucket_table *tbl; |
| 1082 | unsigned int i; | 979 | unsigned int i; |
| 1083 | 980 | ||
| 1084 | cancel_work_sync(&ht->run_work); | 981 | cancel_work_sync(&ht->run_work); |
| @@ -1089,7 +986,7 @@ void rhashtable_free_and_destroy(struct rhashtable *ht, | |||
| 1089 | for (i = 0; i < tbl->size; i++) { | 986 | for (i = 0; i < tbl->size; i++) { |
| 1090 | struct rhash_head *pos, *next; | 987 | struct rhash_head *pos, *next; |
| 1091 | 988 | ||
| 1092 | for (pos = rht_dereference(*rht_bucket(tbl, i), ht), | 989 | for (pos = rht_dereference(tbl->buckets[i], ht), |
| 1093 | next = !rht_is_a_nulls(pos) ? | 990 | next = !rht_is_a_nulls(pos) ? |
| 1094 | rht_dereference(pos->next, ht) : NULL; | 991 | rht_dereference(pos->next, ht) : NULL; |
| 1095 | !rht_is_a_nulls(pos); | 992 | !rht_is_a_nulls(pos); |
| @@ -1110,70 +1007,3 @@ void rhashtable_destroy(struct rhashtable *ht) | |||
| 1110 | return rhashtable_free_and_destroy(ht, NULL, NULL); | 1007 | return rhashtable_free_and_destroy(ht, NULL, NULL); |
| 1111 | } | 1008 | } |
| 1112 | EXPORT_SYMBOL_GPL(rhashtable_destroy); | 1009 | EXPORT_SYMBOL_GPL(rhashtable_destroy); |
| 1113 | |||
| 1114 | struct rhash_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, | ||
| 1115 | unsigned int hash) | ||
| 1116 | { | ||
| 1117 | const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); | ||
| 1118 | static struct rhash_head __rcu *rhnull = | ||
| 1119 | (struct rhash_head __rcu *)NULLS_MARKER(0); | ||
| 1120 | unsigned int index = hash & ((1 << tbl->nest) - 1); | ||
| 1121 | unsigned int size = tbl->size >> tbl->nest; | ||
| 1122 | unsigned int subhash = hash; | ||
| 1123 | union nested_table *ntbl; | ||
| 1124 | |||
| 1125 | ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]); | ||
| 1126 | ntbl = rht_dereference_bucket(ntbl[index].table, tbl, hash); | ||
| 1127 | subhash >>= tbl->nest; | ||
| 1128 | |||
| 1129 | while (ntbl && size > (1 << shift)) { | ||
| 1130 | index = subhash & ((1 << shift) - 1); | ||
| 1131 | ntbl = rht_dereference_bucket(ntbl[index].table, tbl, hash); | ||
| 1132 | size >>= shift; | ||
| 1133 | subhash >>= shift; | ||
| 1134 | } | ||
| 1135 | |||
| 1136 | if (!ntbl) | ||
| 1137 | return &rhnull; | ||
| 1138 | |||
| 1139 | return &ntbl[subhash].bucket; | ||
| 1140 | |||
| 1141 | } | ||
| 1142 | EXPORT_SYMBOL_GPL(rht_bucket_nested); | ||
| 1143 | |||
| 1144 | struct rhash_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, | ||
| 1145 | struct bucket_table *tbl, | ||
| 1146 | unsigned int hash) | ||
| 1147 | { | ||
| 1148 | const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); | ||
| 1149 | unsigned int index = hash & ((1 << tbl->nest) - 1); | ||
| 1150 | unsigned int size = tbl->size >> tbl->nest; | ||
| 1151 | union nested_table *ntbl; | ||
| 1152 | unsigned int shifted; | ||
| 1153 | unsigned int nhash; | ||
| 1154 | |||
| 1155 | ntbl = (union nested_table *)rcu_dereference_raw(tbl->buckets[0]); | ||
| 1156 | hash >>= tbl->nest; | ||
| 1157 | nhash = index; | ||
| 1158 | shifted = tbl->nest; | ||
| 1159 | ntbl = nested_table_alloc(ht, &ntbl[index].table, | ||
| 1160 | size <= (1 << shift) ? shifted : 0, nhash); | ||
| 1161 | |||
| 1162 | while (ntbl && size > (1 << shift)) { | ||
| 1163 | index = hash & ((1 << shift) - 1); | ||
| 1164 | size >>= shift; | ||
| 1165 | hash >>= shift; | ||
| 1166 | nhash |= index << shifted; | ||
| 1167 | shifted += shift; | ||
| 1168 | ntbl = nested_table_alloc(ht, &ntbl[index].table, | ||
| 1169 | size <= (1 << shift) ? shifted : 0, | ||
| 1170 | nhash); | ||
| 1171 | } | ||
| 1172 | |||
| 1173 | if (!ntbl) | ||
| 1174 | return NULL; | ||
| 1175 | |||
| 1176 | return &ntbl[hash].bucket; | ||
| 1177 | |||
| 1178 | } | ||
| 1179 | EXPORT_SYMBOL_GPL(rht_bucket_nested_insert); | ||
diff --git a/net/tipc/net.c b/net/tipc/net.c index ab8a2d5d1e32..28bf4feeb81c 100644 --- a/net/tipc/net.c +++ b/net/tipc/net.c | |||
| @@ -110,10 +110,6 @@ int tipc_net_start(struct net *net, u32 addr) | |||
| 110 | char addr_string[16]; | 110 | char addr_string[16]; |
| 111 | 111 | ||
| 112 | tn->own_addr = addr; | 112 | tn->own_addr = addr; |
| 113 | |||
| 114 | /* Ensure that the new address is visible before we reinit. */ | ||
| 115 | smp_mb(); | ||
| 116 | |||
| 117 | tipc_named_reinit(net); | 113 | tipc_named_reinit(net); |
| 118 | tipc_sk_reinit(net); | 114 | tipc_sk_reinit(net); |
| 119 | 115 | ||
diff --git a/net/tipc/socket.c b/net/tipc/socket.c index 370a5912bcb5..800caaa699a1 100644 --- a/net/tipc/socket.c +++ b/net/tipc/socket.c | |||
| @@ -384,6 +384,8 @@ static int tipc_sk_create(struct net *net, struct socket *sock, | |||
| 384 | INIT_LIST_HEAD(&tsk->publications); | 384 | INIT_LIST_HEAD(&tsk->publications); |
| 385 | msg = &tsk->phdr; | 385 | msg = &tsk->phdr; |
| 386 | tn = net_generic(sock_net(sk), tipc_net_id); | 386 | tn = net_generic(sock_net(sk), tipc_net_id); |
| 387 | tipc_msg_init(tn->own_addr, msg, TIPC_LOW_IMPORTANCE, TIPC_NAMED_MSG, | ||
| 388 | NAMED_H_SIZE, 0); | ||
| 387 | 389 | ||
| 388 | /* Finish initializing socket data structures */ | 390 | /* Finish initializing socket data structures */ |
| 389 | sock->ops = ops; | 391 | sock->ops = ops; |
| @@ -393,13 +395,6 @@ static int tipc_sk_create(struct net *net, struct socket *sock, | |||
| 393 | pr_warn("Socket create failed; port number exhausted\n"); | 395 | pr_warn("Socket create failed; port number exhausted\n"); |
| 394 | return -EINVAL; | 396 | return -EINVAL; |
| 395 | } | 397 | } |
| 396 | |||
| 397 | /* Ensure tsk is visible before we read own_addr. */ | ||
| 398 | smp_mb(); | ||
| 399 | |||
| 400 | tipc_msg_init(tn->own_addr, msg, TIPC_LOW_IMPORTANCE, TIPC_NAMED_MSG, | ||
| 401 | NAMED_H_SIZE, 0); | ||
| 402 | |||
| 403 | msg_set_origport(msg, tsk->portid); | 398 | msg_set_origport(msg, tsk->portid); |
| 404 | setup_timer(&sk->sk_timer, tipc_sk_timeout, (unsigned long)tsk); | 399 | setup_timer(&sk->sk_timer, tipc_sk_timeout, (unsigned long)tsk); |
| 405 | sk->sk_shutdown = 0; | 400 | sk->sk_shutdown = 0; |
| @@ -2274,27 +2269,24 @@ static int tipc_sk_withdraw(struct tipc_sock *tsk, uint scope, | |||
| 2274 | void tipc_sk_reinit(struct net *net) | 2269 | void tipc_sk_reinit(struct net *net) |
| 2275 | { | 2270 | { |
| 2276 | struct tipc_net *tn = net_generic(net, tipc_net_id); | 2271 | struct tipc_net *tn = net_generic(net, tipc_net_id); |
| 2277 | struct rhashtable_iter iter; | 2272 | const struct bucket_table *tbl; |
| 2273 | struct rhash_head *pos; | ||
| 2278 | struct tipc_sock *tsk; | 2274 | struct tipc_sock *tsk; |
| 2279 | struct tipc_msg *msg; | 2275 | struct tipc_msg *msg; |
| 2276 | int i; | ||
| 2280 | 2277 | ||
| 2281 | rhashtable_walk_enter(&tn->sk_rht, &iter); | 2278 | rcu_read_lock(); |
| 2282 | 2279 | tbl = rht_dereference_rcu((&tn->sk_rht)->tbl, &tn->sk_rht); | |
| 2283 | do { | 2280 | for (i = 0; i < tbl->size; i++) { |
| 2284 | tsk = ERR_PTR(rhashtable_walk_start(&iter)); | 2281 | rht_for_each_entry_rcu(tsk, pos, tbl, i, node) { |
| 2285 | if (tsk) | ||
| 2286 | continue; | ||
| 2287 | |||
| 2288 | while ((tsk = rhashtable_walk_next(&iter)) && !IS_ERR(tsk)) { | ||
| 2289 | spin_lock_bh(&tsk->sk.sk_lock.slock); | 2282 | spin_lock_bh(&tsk->sk.sk_lock.slock); |
| 2290 | msg = &tsk->phdr; | 2283 | msg = &tsk->phdr; |
| 2291 | msg_set_prevnode(msg, tn->own_addr); | 2284 | msg_set_prevnode(msg, tn->own_addr); |
| 2292 | msg_set_orignode(msg, tn->own_addr); | 2285 | msg_set_orignode(msg, tn->own_addr); |
| 2293 | spin_unlock_bh(&tsk->sk.sk_lock.slock); | 2286 | spin_unlock_bh(&tsk->sk.sk_lock.slock); |
| 2294 | } | 2287 | } |
| 2295 | 2288 | } | |
| 2296 | rhashtable_walk_stop(&iter); | 2289 | rcu_read_unlock(); |
| 2297 | } while (tsk == ERR_PTR(-EAGAIN)); | ||
| 2298 | } | 2290 | } |
| 2299 | 2291 | ||
| 2300 | static struct tipc_sock *tipc_sk_lookup(struct net *net, u32 portid) | 2292 | static struct tipc_sock *tipc_sk_lookup(struct net *net, u32 portid) |
