diff options
Diffstat (limited to 'include/linux/rhashtable.h')
| -rw-r--r-- | include/linux/rhashtable.h | 164 |
1 files changed, 7 insertions, 157 deletions
diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h index 4e1f535c2034..eb7111039247 100644 --- a/include/linux/rhashtable.h +++ b/include/linux/rhashtable.h | |||
| @@ -1,3 +1,4 @@ | |||
| 1 | /* SPDX-License-Identifier: GPL-2.0 */ | ||
| 1 | /* | 2 | /* |
| 2 | * Resizable, Scalable, Concurrent Hash Table | 3 | * Resizable, Scalable, Concurrent Hash Table |
| 3 | * | 4 | * |
| @@ -17,37 +18,18 @@ | |||
| 17 | #ifndef _LINUX_RHASHTABLE_H | 18 | #ifndef _LINUX_RHASHTABLE_H |
| 18 | #define _LINUX_RHASHTABLE_H | 19 | #define _LINUX_RHASHTABLE_H |
| 19 | 20 | ||
| 20 | #include <linux/atomic.h> | ||
| 21 | #include <linux/compiler.h> | ||
| 22 | #include <linux/err.h> | 21 | #include <linux/err.h> |
| 23 | #include <linux/errno.h> | 22 | #include <linux/errno.h> |
| 24 | #include <linux/jhash.h> | 23 | #include <linux/jhash.h> |
| 25 | #include <linux/list_nulls.h> | 24 | #include <linux/list_nulls.h> |
| 26 | #include <linux/workqueue.h> | 25 | #include <linux/workqueue.h> |
| 27 | #include <linux/mutex.h> | ||
| 28 | #include <linux/rculist.h> | 26 | #include <linux/rculist.h> |
| 29 | 27 | ||
| 28 | #include <linux/rhashtable-types.h> | ||
| 30 | /* | 29 | /* |
| 31 | * The end of the chain is marked with a special nulls marks which has | 30 | * The end of the chain is marked with a special nulls marks which has |
| 32 | * the following format: | 31 | * the least significant bit set. |
| 33 | * | ||
| 34 | * +-------+-----------------------------------------------------+-+ | ||
| 35 | * | Base | Hash |1| | ||
| 36 | * +-------+-----------------------------------------------------+-+ | ||
| 37 | * | ||
| 38 | * Base (4 bits) : Reserved to distinguish between multiple tables. | ||
| 39 | * Specified via &struct rhashtable_params.nulls_base. | ||
| 40 | * Hash (27 bits): Full hash (unmasked) of first element added to bucket | ||
| 41 | * 1 (1 bit) : Nulls marker (always set) | ||
| 42 | * | ||
| 43 | * The remaining bits of the next pointer remain unused for now. | ||
| 44 | */ | 32 | */ |
| 45 | #define RHT_BASE_BITS 4 | ||
| 46 | #define RHT_HASH_BITS 27 | ||
| 47 | #define RHT_BASE_SHIFT RHT_HASH_BITS | ||
| 48 | |||
| 49 | /* Base bits plus 1 bit for nulls marker */ | ||
| 50 | #define RHT_HASH_RESERVED_SPACE (RHT_BASE_BITS + 1) | ||
| 51 | 33 | ||
| 52 | /* Maximum chain length before rehash | 34 | /* Maximum chain length before rehash |
| 53 | * | 35 | * |
| @@ -64,15 +46,6 @@ | |||
| 64 | */ | 46 | */ |
| 65 | #define RHT_ELASTICITY 16u | 47 | #define RHT_ELASTICITY 16u |
| 66 | 48 | ||
| 67 | struct rhash_head { | ||
| 68 | struct rhash_head __rcu *next; | ||
| 69 | }; | ||
| 70 | |||
| 71 | struct rhlist_head { | ||
| 72 | struct rhash_head rhead; | ||
| 73 | struct rhlist_head __rcu *next; | ||
| 74 | }; | ||
| 75 | |||
| 76 | /** | 49 | /** |
| 77 | * struct bucket_table - Table of hash buckets | 50 | * struct bucket_table - Table of hash buckets |
| 78 | * @size: Number of hash buckets | 51 | * @size: Number of hash buckets |
| @@ -102,132 +75,14 @@ struct bucket_table { | |||
| 102 | struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp; | 75 | struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp; |
| 103 | }; | 76 | }; |
| 104 | 77 | ||
| 105 | /** | 78 | #define INIT_RHT_NULLS_HEAD(ptr) \ |
| 106 | * struct rhashtable_compare_arg - Key for the function rhashtable_compare | 79 | ((ptr) = (typeof(ptr)) NULLS_MARKER(0)) |
| 107 | * @ht: Hash table | ||
| 108 | * @key: Key to compare against | ||
| 109 | */ | ||
| 110 | struct rhashtable_compare_arg { | ||
| 111 | struct rhashtable *ht; | ||
| 112 | const void *key; | ||
| 113 | }; | ||
| 114 | |||
| 115 | typedef u32 (*rht_hashfn_t)(const void *data, u32 len, u32 seed); | ||
| 116 | typedef u32 (*rht_obj_hashfn_t)(const void *data, u32 len, u32 seed); | ||
| 117 | typedef int (*rht_obj_cmpfn_t)(struct rhashtable_compare_arg *arg, | ||
| 118 | const void *obj); | ||
| 119 | |||
| 120 | struct rhashtable; | ||
| 121 | |||
| 122 | /** | ||
| 123 | * struct rhashtable_params - Hash table construction parameters | ||
| 124 | * @nelem_hint: Hint on number of elements, should be 75% of desired size | ||
| 125 | * @key_len: Length of key | ||
| 126 | * @key_offset: Offset of key in struct to be hashed | ||
| 127 | * @head_offset: Offset of rhash_head in struct to be hashed | ||
| 128 | * @max_size: Maximum size while expanding | ||
| 129 | * @min_size: Minimum size while shrinking | ||
| 130 | * @locks_mul: Number of bucket locks to allocate per cpu (default: 32) | ||
| 131 | * @automatic_shrinking: Enable automatic shrinking of tables | ||
| 132 | * @nulls_base: Base value to generate nulls marker | ||
| 133 | * @hashfn: Hash function (default: jhash2 if !(key_len % 4), or jhash) | ||
| 134 | * @obj_hashfn: Function to hash object | ||
| 135 | * @obj_cmpfn: Function to compare key with object | ||
| 136 | */ | ||
| 137 | struct rhashtable_params { | ||
| 138 | u16 nelem_hint; | ||
| 139 | u16 key_len; | ||
| 140 | u16 key_offset; | ||
| 141 | u16 head_offset; | ||
| 142 | unsigned int max_size; | ||
| 143 | u16 min_size; | ||
| 144 | bool automatic_shrinking; | ||
| 145 | u8 locks_mul; | ||
| 146 | u32 nulls_base; | ||
| 147 | rht_hashfn_t hashfn; | ||
| 148 | rht_obj_hashfn_t obj_hashfn; | ||
| 149 | rht_obj_cmpfn_t obj_cmpfn; | ||
| 150 | }; | ||
| 151 | |||
| 152 | /** | ||
| 153 | * struct rhashtable - Hash table handle | ||
| 154 | * @tbl: Bucket table | ||
| 155 | * @key_len: Key length for hashfn | ||
| 156 | * @max_elems: Maximum number of elements in table | ||
| 157 | * @p: Configuration parameters | ||
| 158 | * @rhlist: True if this is an rhltable | ||
| 159 | * @run_work: Deferred worker to expand/shrink asynchronously | ||
| 160 | * @mutex: Mutex to protect current/future table swapping | ||
| 161 | * @lock: Spin lock to protect walker list | ||
| 162 | * @nelems: Number of elements in table | ||
| 163 | */ | ||
| 164 | struct rhashtable { | ||
| 165 | struct bucket_table __rcu *tbl; | ||
| 166 | unsigned int key_len; | ||
| 167 | unsigned int max_elems; | ||
| 168 | struct rhashtable_params p; | ||
| 169 | bool rhlist; | ||
| 170 | struct work_struct run_work; | ||
| 171 | struct mutex mutex; | ||
| 172 | spinlock_t lock; | ||
| 173 | atomic_t nelems; | ||
| 174 | }; | ||
| 175 | |||
| 176 | /** | ||
| 177 | * struct rhltable - Hash table with duplicate objects in a list | ||
| 178 | * @ht: Underlying rhtable | ||
| 179 | */ | ||
| 180 | struct rhltable { | ||
| 181 | struct rhashtable ht; | ||
| 182 | }; | ||
| 183 | |||
| 184 | /** | ||
| 185 | * struct rhashtable_walker - Hash table walker | ||
| 186 | * @list: List entry on list of walkers | ||
| 187 | * @tbl: The table that we were walking over | ||
| 188 | */ | ||
| 189 | struct rhashtable_walker { | ||
| 190 | struct list_head list; | ||
| 191 | struct bucket_table *tbl; | ||
| 192 | }; | ||
| 193 | |||
| 194 | /** | ||
| 195 | * struct rhashtable_iter - Hash table iterator | ||
| 196 | * @ht: Table to iterate through | ||
| 197 | * @p: Current pointer | ||
| 198 | * @list: Current hash list pointer | ||
| 199 | * @walker: Associated rhashtable walker | ||
| 200 | * @slot: Current slot | ||
| 201 | * @skip: Number of entries to skip in slot | ||
| 202 | */ | ||
| 203 | struct rhashtable_iter { | ||
| 204 | struct rhashtable *ht; | ||
| 205 | struct rhash_head *p; | ||
| 206 | struct rhlist_head *list; | ||
| 207 | struct rhashtable_walker walker; | ||
| 208 | unsigned int slot; | ||
| 209 | unsigned int skip; | ||
| 210 | bool end_of_table; | ||
| 211 | }; | ||
| 212 | |||
| 213 | static inline unsigned long rht_marker(const struct rhashtable *ht, u32 hash) | ||
| 214 | { | ||
| 215 | return NULLS_MARKER(ht->p.nulls_base + hash); | ||
| 216 | } | ||
| 217 | |||
| 218 | #define INIT_RHT_NULLS_HEAD(ptr, ht, hash) \ | ||
| 219 | ((ptr) = (typeof(ptr)) rht_marker(ht, hash)) | ||
| 220 | 80 | ||
| 221 | static inline bool rht_is_a_nulls(const struct rhash_head *ptr) | 81 | static inline bool rht_is_a_nulls(const struct rhash_head *ptr) |
| 222 | { | 82 | { |
| 223 | return ((unsigned long) ptr & 1); | 83 | return ((unsigned long) ptr & 1); |
| 224 | } | 84 | } |
| 225 | 85 | ||
| 226 | static inline unsigned long rht_get_nulls_value(const struct rhash_head *ptr) | ||
| 227 | { | ||
| 228 | return ((unsigned long) ptr) >> 1; | ||
| 229 | } | ||
| 230 | |||
| 231 | static inline void *rht_obj(const struct rhashtable *ht, | 86 | static inline void *rht_obj(const struct rhashtable *ht, |
| 232 | const struct rhash_head *he) | 87 | const struct rhash_head *he) |
| 233 | { | 88 | { |
| @@ -237,7 +92,7 @@ static inline void *rht_obj(const struct rhashtable *ht, | |||
| 237 | static inline unsigned int rht_bucket_index(const struct bucket_table *tbl, | 92 | static inline unsigned int rht_bucket_index(const struct bucket_table *tbl, |
| 238 | unsigned int hash) | 93 | unsigned int hash) |
| 239 | { | 94 | { |
| 240 | return (hash >> RHT_HASH_RESERVED_SPACE) & (tbl->size - 1); | 95 | return hash & (tbl->size - 1); |
| 241 | } | 96 | } |
| 242 | 97 | ||
| 243 | static inline unsigned int rht_key_get_hash(struct rhashtable *ht, | 98 | static inline unsigned int rht_key_get_hash(struct rhashtable *ht, |
| @@ -376,11 +231,6 @@ static inline int lockdep_rht_bucket_is_held(const struct bucket_table *tbl, | |||
| 376 | } | 231 | } |
| 377 | #endif /* CONFIG_PROVE_LOCKING */ | 232 | #endif /* CONFIG_PROVE_LOCKING */ |
| 378 | 233 | ||
| 379 | int rhashtable_init(struct rhashtable *ht, | ||
| 380 | const struct rhashtable_params *params); | ||
| 381 | int rhltable_init(struct rhltable *hlt, | ||
| 382 | const struct rhashtable_params *params); | ||
| 383 | |||
| 384 | void *rhashtable_insert_slow(struct rhashtable *ht, const void *key, | 234 | void *rhashtable_insert_slow(struct rhashtable *ht, const void *key, |
| 385 | struct rhash_head *obj); | 235 | struct rhash_head *obj); |
| 386 | 236 | ||
| @@ -745,7 +595,7 @@ static inline void *__rhashtable_insert_fast( | |||
| 745 | lock = rht_bucket_lock(tbl, hash); | 595 | lock = rht_bucket_lock(tbl, hash); |
| 746 | spin_lock_bh(lock); | 596 | spin_lock_bh(lock); |
| 747 | 597 | ||
| 748 | if (unlikely(rht_dereference_bucket(tbl->future_tbl, tbl, hash))) { | 598 | if (unlikely(rcu_access_pointer(tbl->future_tbl))) { |
| 749 | slow_path: | 599 | slow_path: |
| 750 | spin_unlock_bh(lock); | 600 | spin_unlock_bh(lock); |
| 751 | rcu_read_unlock(); | 601 | rcu_read_unlock(); |
