diff options
Diffstat (limited to 'lib/assoc_array.c')
-rw-r--r-- | lib/assoc_array.c | 37 |
1 files changed, 12 insertions, 25 deletions
diff --git a/lib/assoc_array.c b/lib/assoc_array.c index b77d51da8c73..c6659cb37033 100644 --- a/lib/assoc_array.c +++ b/lib/assoc_array.c | |||
@@ -38,12 +38,10 @@ begin_node: | |||
38 | if (assoc_array_ptr_is_shortcut(cursor)) { | 38 | if (assoc_array_ptr_is_shortcut(cursor)) { |
39 | /* Descend through a shortcut */ | 39 | /* Descend through a shortcut */ |
40 | shortcut = assoc_array_ptr_to_shortcut(cursor); | 40 | shortcut = assoc_array_ptr_to_shortcut(cursor); |
41 | smp_read_barrier_depends(); | 41 | cursor = READ_ONCE(shortcut->next_node); /* Address dependency. */ |
42 | cursor = READ_ONCE(shortcut->next_node); | ||
43 | } | 42 | } |
44 | 43 | ||
45 | node = assoc_array_ptr_to_node(cursor); | 44 | node = assoc_array_ptr_to_node(cursor); |
46 | smp_read_barrier_depends(); | ||
47 | slot = 0; | 45 | slot = 0; |
48 | 46 | ||
49 | /* We perform two passes of each node. | 47 | /* We perform two passes of each node. |
@@ -55,15 +53,12 @@ begin_node: | |||
55 | */ | 53 | */ |
56 | has_meta = 0; | 54 | has_meta = 0; |
57 | for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) { | 55 | for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) { |
58 | ptr = READ_ONCE(node->slots[slot]); | 56 | ptr = READ_ONCE(node->slots[slot]); /* Address dependency. */ |
59 | has_meta |= (unsigned long)ptr; | 57 | has_meta |= (unsigned long)ptr; |
60 | if (ptr && assoc_array_ptr_is_leaf(ptr)) { | 58 | if (ptr && assoc_array_ptr_is_leaf(ptr)) { |
61 | /* We need a barrier between the read of the pointer | 59 | /* We need a barrier between the read of the pointer, |
62 | * and dereferencing the pointer - but only if we are | 60 | * which is supplied by the above READ_ONCE(). |
63 | * actually going to dereference it. | ||
64 | */ | 61 | */ |
65 | smp_read_barrier_depends(); | ||
66 | |||
67 | /* Invoke the callback */ | 62 | /* Invoke the callback */ |
68 | ret = iterator(assoc_array_ptr_to_leaf(ptr), | 63 | ret = iterator(assoc_array_ptr_to_leaf(ptr), |
69 | iterator_data); | 64 | iterator_data); |
@@ -86,10 +81,8 @@ begin_node: | |||
86 | 81 | ||
87 | continue_node: | 82 | continue_node: |
88 | node = assoc_array_ptr_to_node(cursor); | 83 | node = assoc_array_ptr_to_node(cursor); |
89 | smp_read_barrier_depends(); | ||
90 | |||
91 | for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) { | 84 | for (; slot < ASSOC_ARRAY_FAN_OUT; slot++) { |
92 | ptr = READ_ONCE(node->slots[slot]); | 85 | ptr = READ_ONCE(node->slots[slot]); /* Address dependency. */ |
93 | if (assoc_array_ptr_is_meta(ptr)) { | 86 | if (assoc_array_ptr_is_meta(ptr)) { |
94 | cursor = ptr; | 87 | cursor = ptr; |
95 | goto begin_node; | 88 | goto begin_node; |
@@ -98,16 +91,15 @@ continue_node: | |||
98 | 91 | ||
99 | finished_node: | 92 | finished_node: |
100 | /* Move up to the parent (may need to skip back over a shortcut) */ | 93 | /* Move up to the parent (may need to skip back over a shortcut) */ |
101 | parent = READ_ONCE(node->back_pointer); | 94 | parent = READ_ONCE(node->back_pointer); /* Address dependency. */ |
102 | slot = node->parent_slot; | 95 | slot = node->parent_slot; |
103 | if (parent == stop) | 96 | if (parent == stop) |
104 | return 0; | 97 | return 0; |
105 | 98 | ||
106 | if (assoc_array_ptr_is_shortcut(parent)) { | 99 | if (assoc_array_ptr_is_shortcut(parent)) { |
107 | shortcut = assoc_array_ptr_to_shortcut(parent); | 100 | shortcut = assoc_array_ptr_to_shortcut(parent); |
108 | smp_read_barrier_depends(); | ||
109 | cursor = parent; | 101 | cursor = parent; |
110 | parent = READ_ONCE(shortcut->back_pointer); | 102 | parent = READ_ONCE(shortcut->back_pointer); /* Address dependency. */ |
111 | slot = shortcut->parent_slot; | 103 | slot = shortcut->parent_slot; |
112 | if (parent == stop) | 104 | if (parent == stop) |
113 | return 0; | 105 | return 0; |
@@ -147,7 +139,7 @@ int assoc_array_iterate(const struct assoc_array *array, | |||
147 | void *iterator_data), | 139 | void *iterator_data), |
148 | void *iterator_data) | 140 | void *iterator_data) |
149 | { | 141 | { |
150 | struct assoc_array_ptr *root = READ_ONCE(array->root); | 142 | struct assoc_array_ptr *root = READ_ONCE(array->root); /* Address dependency. */ |
151 | 143 | ||
152 | if (!root) | 144 | if (!root) |
153 | return 0; | 145 | return 0; |
@@ -194,7 +186,7 @@ assoc_array_walk(const struct assoc_array *array, | |||
194 | 186 | ||
195 | pr_devel("-->%s()\n", __func__); | 187 | pr_devel("-->%s()\n", __func__); |
196 | 188 | ||
197 | cursor = READ_ONCE(array->root); | 189 | cursor = READ_ONCE(array->root); /* Address dependency. */ |
198 | if (!cursor) | 190 | if (!cursor) |
199 | return assoc_array_walk_tree_empty; | 191 | return assoc_array_walk_tree_empty; |
200 | 192 | ||
@@ -216,11 +208,9 @@ jumped: | |||
216 | 208 | ||
217 | consider_node: | 209 | consider_node: |
218 | node = assoc_array_ptr_to_node(cursor); | 210 | node = assoc_array_ptr_to_node(cursor); |
219 | smp_read_barrier_depends(); | ||
220 | |||
221 | slot = segments >> (level & ASSOC_ARRAY_KEY_CHUNK_MASK); | 211 | slot = segments >> (level & ASSOC_ARRAY_KEY_CHUNK_MASK); |
222 | slot &= ASSOC_ARRAY_FAN_MASK; | 212 | slot &= ASSOC_ARRAY_FAN_MASK; |
223 | ptr = READ_ONCE(node->slots[slot]); | 213 | ptr = READ_ONCE(node->slots[slot]); /* Address dependency. */ |
224 | 214 | ||
225 | pr_devel("consider slot %x [ix=%d type=%lu]\n", | 215 | pr_devel("consider slot %x [ix=%d type=%lu]\n", |
226 | slot, level, (unsigned long)ptr & 3); | 216 | slot, level, (unsigned long)ptr & 3); |
@@ -254,7 +244,6 @@ consider_node: | |||
254 | cursor = ptr; | 244 | cursor = ptr; |
255 | follow_shortcut: | 245 | follow_shortcut: |
256 | shortcut = assoc_array_ptr_to_shortcut(cursor); | 246 | shortcut = assoc_array_ptr_to_shortcut(cursor); |
257 | smp_read_barrier_depends(); | ||
258 | pr_devel("shortcut to %d\n", shortcut->skip_to_level); | 247 | pr_devel("shortcut to %d\n", shortcut->skip_to_level); |
259 | sc_level = level + ASSOC_ARRAY_LEVEL_STEP; | 248 | sc_level = level + ASSOC_ARRAY_LEVEL_STEP; |
260 | BUG_ON(sc_level > shortcut->skip_to_level); | 249 | BUG_ON(sc_level > shortcut->skip_to_level); |
@@ -294,7 +283,7 @@ follow_shortcut: | |||
294 | } while (sc_level < shortcut->skip_to_level); | 283 | } while (sc_level < shortcut->skip_to_level); |
295 | 284 | ||
296 | /* The shortcut matches the leaf's index to this point. */ | 285 | /* The shortcut matches the leaf's index to this point. */ |
297 | cursor = READ_ONCE(shortcut->next_node); | 286 | cursor = READ_ONCE(shortcut->next_node); /* Address dependency. */ |
298 | if (((level ^ sc_level) & ~ASSOC_ARRAY_KEY_CHUNK_MASK) != 0) { | 287 | if (((level ^ sc_level) & ~ASSOC_ARRAY_KEY_CHUNK_MASK) != 0) { |
299 | level = sc_level; | 288 | level = sc_level; |
300 | goto jumped; | 289 | goto jumped; |
@@ -331,20 +320,18 @@ void *assoc_array_find(const struct assoc_array *array, | |||
331 | return NULL; | 320 | return NULL; |
332 | 321 | ||
333 | node = result.terminal_node.node; | 322 | node = result.terminal_node.node; |
334 | smp_read_barrier_depends(); | ||
335 | 323 | ||
336 | /* If the target key is available to us, it's has to be pointed to by | 324 | /* If the target key is available to us, it's has to be pointed to by |
337 | * the terminal node. | 325 | * the terminal node. |
338 | */ | 326 | */ |
339 | for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) { | 327 | for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) { |
340 | ptr = READ_ONCE(node->slots[slot]); | 328 | ptr = READ_ONCE(node->slots[slot]); /* Address dependency. */ |
341 | if (ptr && assoc_array_ptr_is_leaf(ptr)) { | 329 | if (ptr && assoc_array_ptr_is_leaf(ptr)) { |
342 | /* We need a barrier between the read of the pointer | 330 | /* We need a barrier between the read of the pointer |
343 | * and dereferencing the pointer - but only if we are | 331 | * and dereferencing the pointer - but only if we are |
344 | * actually going to dereference it. | 332 | * actually going to dereference it. |
345 | */ | 333 | */ |
346 | leaf = assoc_array_ptr_to_leaf(ptr); | 334 | leaf = assoc_array_ptr_to_leaf(ptr); |
347 | smp_read_barrier_depends(); | ||
348 | if (ops->compare_object(leaf, index_key)) | 335 | if (ops->compare_object(leaf, index_key)) |
349 | return (void *)leaf; | 336 | return (void *)leaf; |
350 | } | 337 | } |