aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux/rhashtable.h
diff options
context:
space:
mode:
Diffstat (limited to 'include/linux/rhashtable.h')
-rw-r--r--include/linux/rhashtable.h164
1 files changed, 7 insertions, 157 deletions
diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h
index 4e1f535c2034..eb7111039247 100644
--- a/include/linux/rhashtable.h
+++ b/include/linux/rhashtable.h
@@ -1,3 +1,4 @@
1/* SPDX-License-Identifier: GPL-2.0 */
1/* 2/*
2 * Resizable, Scalable, Concurrent Hash Table 3 * Resizable, Scalable, Concurrent Hash Table
3 * 4 *
@@ -17,37 +18,18 @@
17#ifndef _LINUX_RHASHTABLE_H 18#ifndef _LINUX_RHASHTABLE_H
18#define _LINUX_RHASHTABLE_H 19#define _LINUX_RHASHTABLE_H
19 20
20#include <linux/atomic.h>
21#include <linux/compiler.h>
22#include <linux/err.h> 21#include <linux/err.h>
23#include <linux/errno.h> 22#include <linux/errno.h>
24#include <linux/jhash.h> 23#include <linux/jhash.h>
25#include <linux/list_nulls.h> 24#include <linux/list_nulls.h>
26#include <linux/workqueue.h> 25#include <linux/workqueue.h>
27#include <linux/mutex.h>
28#include <linux/rculist.h> 26#include <linux/rculist.h>
29 27
28#include <linux/rhashtable-types.h>
30/* 29/*
31 * The end of the chain is marked with a special nulls marks which has 30 * The end of the chain is marked with a special nulls marks which has
32 * the following format: 31 * the least significant bit set.
33 *
34 * +-------+-----------------------------------------------------+-+
35 * | Base | Hash |1|
36 * +-------+-----------------------------------------------------+-+
37 *
38 * Base (4 bits) : Reserved to distinguish between multiple tables.
39 * Specified via &struct rhashtable_params.nulls_base.
40 * Hash (27 bits): Full hash (unmasked) of first element added to bucket
41 * 1 (1 bit) : Nulls marker (always set)
42 *
43 * The remaining bits of the next pointer remain unused for now.
44 */ 32 */
45#define RHT_BASE_BITS 4
46#define RHT_HASH_BITS 27
47#define RHT_BASE_SHIFT RHT_HASH_BITS
48
49/* Base bits plus 1 bit for nulls marker */
50#define RHT_HASH_RESERVED_SPACE (RHT_BASE_BITS + 1)
51 33
52/* Maximum chain length before rehash 34/* Maximum chain length before rehash
53 * 35 *
@@ -64,15 +46,6 @@
64 */ 46 */
65#define RHT_ELASTICITY 16u 47#define RHT_ELASTICITY 16u
66 48
67struct rhash_head {
68 struct rhash_head __rcu *next;
69};
70
71struct rhlist_head {
72 struct rhash_head rhead;
73 struct rhlist_head __rcu *next;
74};
75
76/** 49/**
77 * struct bucket_table - Table of hash buckets 50 * struct bucket_table - Table of hash buckets
78 * @size: Number of hash buckets 51 * @size: Number of hash buckets
@@ -102,132 +75,14 @@ struct bucket_table {
102 struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp; 75 struct rhash_head __rcu *buckets[] ____cacheline_aligned_in_smp;
103}; 76};
104 77
105/** 78#define INIT_RHT_NULLS_HEAD(ptr) \
106 * struct rhashtable_compare_arg - Key for the function rhashtable_compare 79 ((ptr) = (typeof(ptr)) NULLS_MARKER(0))
107 * @ht: Hash table
108 * @key: Key to compare against
109 */
110struct rhashtable_compare_arg {
111 struct rhashtable *ht;
112 const void *key;
113};
114
115typedef u32 (*rht_hashfn_t)(const void *data, u32 len, u32 seed);
116typedef u32 (*rht_obj_hashfn_t)(const void *data, u32 len, u32 seed);
117typedef int (*rht_obj_cmpfn_t)(struct rhashtable_compare_arg *arg,
118 const void *obj);
119
120struct rhashtable;
121
122/**
123 * struct rhashtable_params - Hash table construction parameters
124 * @nelem_hint: Hint on number of elements, should be 75% of desired size
125 * @key_len: Length of key
126 * @key_offset: Offset of key in struct to be hashed
127 * @head_offset: Offset of rhash_head in struct to be hashed
128 * @max_size: Maximum size while expanding
129 * @min_size: Minimum size while shrinking
130 * @locks_mul: Number of bucket locks to allocate per cpu (default: 32)
131 * @automatic_shrinking: Enable automatic shrinking of tables
132 * @nulls_base: Base value to generate nulls marker
133 * @hashfn: Hash function (default: jhash2 if !(key_len % 4), or jhash)
134 * @obj_hashfn: Function to hash object
135 * @obj_cmpfn: Function to compare key with object
136 */
137struct rhashtable_params {
138 u16 nelem_hint;
139 u16 key_len;
140 u16 key_offset;
141 u16 head_offset;
142 unsigned int max_size;
143 u16 min_size;
144 bool automatic_shrinking;
145 u8 locks_mul;
146 u32 nulls_base;
147 rht_hashfn_t hashfn;
148 rht_obj_hashfn_t obj_hashfn;
149 rht_obj_cmpfn_t obj_cmpfn;
150};
151
152/**
153 * struct rhashtable - Hash table handle
154 * @tbl: Bucket table
155 * @key_len: Key length for hashfn
156 * @max_elems: Maximum number of elements in table
157 * @p: Configuration parameters
158 * @rhlist: True if this is an rhltable
159 * @run_work: Deferred worker to expand/shrink asynchronously
160 * @mutex: Mutex to protect current/future table swapping
161 * @lock: Spin lock to protect walker list
162 * @nelems: Number of elements in table
163 */
164struct rhashtable {
165 struct bucket_table __rcu *tbl;
166 unsigned int key_len;
167 unsigned int max_elems;
168 struct rhashtable_params p;
169 bool rhlist;
170 struct work_struct run_work;
171 struct mutex mutex;
172 spinlock_t lock;
173 atomic_t nelems;
174};
175
176/**
177 * struct rhltable - Hash table with duplicate objects in a list
178 * @ht: Underlying rhtable
179 */
180struct rhltable {
181 struct rhashtable ht;
182};
183
184/**
185 * struct rhashtable_walker - Hash table walker
186 * @list: List entry on list of walkers
187 * @tbl: The table that we were walking over
188 */
189struct rhashtable_walker {
190 struct list_head list;
191 struct bucket_table *tbl;
192};
193
194/**
195 * struct rhashtable_iter - Hash table iterator
196 * @ht: Table to iterate through
197 * @p: Current pointer
198 * @list: Current hash list pointer
199 * @walker: Associated rhashtable walker
200 * @slot: Current slot
201 * @skip: Number of entries to skip in slot
202 */
203struct rhashtable_iter {
204 struct rhashtable *ht;
205 struct rhash_head *p;
206 struct rhlist_head *list;
207 struct rhashtable_walker walker;
208 unsigned int slot;
209 unsigned int skip;
210 bool end_of_table;
211};
212
213static inline unsigned long rht_marker(const struct rhashtable *ht, u32 hash)
214{
215 return NULLS_MARKER(ht->p.nulls_base + hash);
216}
217
218#define INIT_RHT_NULLS_HEAD(ptr, ht, hash) \
219 ((ptr) = (typeof(ptr)) rht_marker(ht, hash))
220 80
221static inline bool rht_is_a_nulls(const struct rhash_head *ptr) 81static inline bool rht_is_a_nulls(const struct rhash_head *ptr)
222{ 82{
223 return ((unsigned long) ptr & 1); 83 return ((unsigned long) ptr & 1);
224} 84}
225 85
226static inline unsigned long rht_get_nulls_value(const struct rhash_head *ptr)
227{
228 return ((unsigned long) ptr) >> 1;
229}
230
231static inline void *rht_obj(const struct rhashtable *ht, 86static inline void *rht_obj(const struct rhashtable *ht,
232 const struct rhash_head *he) 87 const struct rhash_head *he)
233{ 88{
@@ -237,7 +92,7 @@ static inline void *rht_obj(const struct rhashtable *ht,
237static inline unsigned int rht_bucket_index(const struct bucket_table *tbl, 92static inline unsigned int rht_bucket_index(const struct bucket_table *tbl,
238 unsigned int hash) 93 unsigned int hash)
239{ 94{
240 return (hash >> RHT_HASH_RESERVED_SPACE) & (tbl->size - 1); 95 return hash & (tbl->size - 1);
241} 96}
242 97
243static inline unsigned int rht_key_get_hash(struct rhashtable *ht, 98static inline unsigned int rht_key_get_hash(struct rhashtable *ht,
@@ -376,11 +231,6 @@ static inline int lockdep_rht_bucket_is_held(const struct bucket_table *tbl,
376} 231}
377#endif /* CONFIG_PROVE_LOCKING */ 232#endif /* CONFIG_PROVE_LOCKING */
378 233
379int rhashtable_init(struct rhashtable *ht,
380 const struct rhashtable_params *params);
381int rhltable_init(struct rhltable *hlt,
382 const struct rhashtable_params *params);
383
384void *rhashtable_insert_slow(struct rhashtable *ht, const void *key, 234void *rhashtable_insert_slow(struct rhashtable *ht, const void *key,
385 struct rhash_head *obj); 235 struct rhash_head *obj);
386 236
@@ -745,7 +595,7 @@ static inline void *__rhashtable_insert_fast(
745 lock = rht_bucket_lock(tbl, hash); 595 lock = rht_bucket_lock(tbl, hash);
746 spin_lock_bh(lock); 596 spin_lock_bh(lock);
747 597
748 if (unlikely(rht_dereference_bucket(tbl->future_tbl, tbl, hash))) { 598 if (unlikely(rcu_access_pointer(tbl->future_tbl))) {
749slow_path: 599slow_path:
750 spin_unlock_bh(lock); 600 spin_unlock_bh(lock);
751 rcu_read_unlock(); 601 rcu_read_unlock();