diff options
author | Ingo Molnar <mingo@elte.hu> | 2012-02-24 02:31:31 -0500 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2012-02-24 04:05:59 -0500 |
commit | c5905afb0ee6550b42c49213da1c22d67316c194 (patch) | |
tree | 253fdb322e6e5b257ffda3b9b66bce90a473a6f7 /kernel/jump_label.c | |
parent | 1cfa60dc7d7c7cc774a44eee47ff135a644a1f31 (diff) |
static keys: Introduce 'struct static_key', static_key_true()/false() and static_key_slow_[inc|dec]()
So here's a boot tested patch on top of Jason's series that does
all the cleanups I talked about and turns jump labels into a
more intuitive to use facility. It should also address the
various misconceptions and confusions that surround jump labels.
Typical usage scenarios:
#include <linux/static_key.h>
struct static_key key = STATIC_KEY_INIT_TRUE;
if (static_key_false(&key))
do unlikely code
else
do likely code
Or:
if (static_key_true(&key))
do likely code
else
do unlikely code
The static key is modified via:
static_key_slow_inc(&key);
...
static_key_slow_dec(&key);
The 'slow' prefix makes it abundantly clear that this is an
expensive operation.
I've updated all in-kernel code to use this everywhere. Note
that I (intentionally) have not pushed through the rename
blindly through to the lowest levels: the actual jump-label
patching arch facility should be named like that, so we want to
decouple jump labels from the static-key facility a bit.
On non-jump-label enabled architectures static keys default to
likely()/unlikely() branches.
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Acked-by: Jason Baron <jbaron@redhat.com>
Acked-by: Steven Rostedt <rostedt@goodmis.org>
Cc: a.p.zijlstra@chello.nl
Cc: mathieu.desnoyers@efficios.com
Cc: davem@davemloft.net
Cc: ddaney.cavm@gmail.com
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/20120222085809.GA26397@elte.hu
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'kernel/jump_label.c')
-rw-r--r-- | kernel/jump_label.c | 128 |
1 files changed, 74 insertions, 54 deletions
diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 543782e7cdd..bf9dcadbb53 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c | |||
@@ -12,7 +12,7 @@ | |||
12 | #include <linux/slab.h> | 12 | #include <linux/slab.h> |
13 | #include <linux/sort.h> | 13 | #include <linux/sort.h> |
14 | #include <linux/err.h> | 14 | #include <linux/err.h> |
15 | #include <linux/jump_label.h> | 15 | #include <linux/static_key.h> |
16 | 16 | ||
17 | #ifdef HAVE_JUMP_LABEL | 17 | #ifdef HAVE_JUMP_LABEL |
18 | 18 | ||
@@ -29,10 +29,11 @@ void jump_label_unlock(void) | |||
29 | mutex_unlock(&jump_label_mutex); | 29 | mutex_unlock(&jump_label_mutex); |
30 | } | 30 | } |
31 | 31 | ||
32 | bool jump_label_enabled(struct jump_label_key *key) | 32 | bool static_key_enabled(struct static_key *key) |
33 | { | 33 | { |
34 | return !!atomic_read(&key->enabled); | 34 | return (atomic_read(&key->enabled) > 0); |
35 | } | 35 | } |
36 | EXPORT_SYMBOL_GPL(static_key_enabled); | ||
36 | 37 | ||
37 | static int jump_label_cmp(const void *a, const void *b) | 38 | static int jump_label_cmp(const void *a, const void *b) |
38 | { | 39 | { |
@@ -58,22 +59,26 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) | |||
58 | sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); | 59 | sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); |
59 | } | 60 | } |
60 | 61 | ||
61 | static void jump_label_update(struct jump_label_key *key, int enable); | 62 | static void jump_label_update(struct static_key *key, int enable); |
62 | 63 | ||
63 | void jump_label_inc(struct jump_label_key *key) | 64 | void static_key_slow_inc(struct static_key *key) |
64 | { | 65 | { |
65 | if (atomic_inc_not_zero(&key->enabled)) | 66 | if (atomic_inc_not_zero(&key->enabled)) |
66 | return; | 67 | return; |
67 | 68 | ||
68 | jump_label_lock(); | 69 | jump_label_lock(); |
69 | if (atomic_read(&key->enabled) == 0) | 70 | if (atomic_read(&key->enabled) == 0) { |
70 | jump_label_update(key, JUMP_LABEL_ENABLE); | 71 | if (!jump_label_get_branch_default(key)) |
72 | jump_label_update(key, JUMP_LABEL_ENABLE); | ||
73 | else | ||
74 | jump_label_update(key, JUMP_LABEL_DISABLE); | ||
75 | } | ||
71 | atomic_inc(&key->enabled); | 76 | atomic_inc(&key->enabled); |
72 | jump_label_unlock(); | 77 | jump_label_unlock(); |
73 | } | 78 | } |
74 | EXPORT_SYMBOL_GPL(jump_label_inc); | 79 | EXPORT_SYMBOL_GPL(static_key_slow_inc); |
75 | 80 | ||
76 | static void __jump_label_dec(struct jump_label_key *key, | 81 | static void __static_key_slow_dec(struct static_key *key, |
77 | unsigned long rate_limit, struct delayed_work *work) | 82 | unsigned long rate_limit, struct delayed_work *work) |
78 | { | 83 | { |
79 | if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) { | 84 | if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) { |
@@ -85,32 +90,35 @@ static void __jump_label_dec(struct jump_label_key *key, | |||
85 | if (rate_limit) { | 90 | if (rate_limit) { |
86 | atomic_inc(&key->enabled); | 91 | atomic_inc(&key->enabled); |
87 | schedule_delayed_work(work, rate_limit); | 92 | schedule_delayed_work(work, rate_limit); |
88 | } else | 93 | } else { |
89 | jump_label_update(key, JUMP_LABEL_DISABLE); | 94 | if (!jump_label_get_branch_default(key)) |
90 | 95 | jump_label_update(key, JUMP_LABEL_DISABLE); | |
96 | else | ||
97 | jump_label_update(key, JUMP_LABEL_ENABLE); | ||
98 | } | ||
91 | jump_label_unlock(); | 99 | jump_label_unlock(); |
92 | } | 100 | } |
93 | EXPORT_SYMBOL_GPL(jump_label_dec); | ||
94 | 101 | ||
95 | static void jump_label_update_timeout(struct work_struct *work) | 102 | static void jump_label_update_timeout(struct work_struct *work) |
96 | { | 103 | { |
97 | struct jump_label_key_deferred *key = | 104 | struct static_key_deferred *key = |
98 | container_of(work, struct jump_label_key_deferred, work.work); | 105 | container_of(work, struct static_key_deferred, work.work); |
99 | __jump_label_dec(&key->key, 0, NULL); | 106 | __static_key_slow_dec(&key->key, 0, NULL); |
100 | } | 107 | } |
101 | 108 | ||
102 | void jump_label_dec(struct jump_label_key *key) | 109 | void static_key_slow_dec(struct static_key *key) |
103 | { | 110 | { |
104 | __jump_label_dec(key, 0, NULL); | 111 | __static_key_slow_dec(key, 0, NULL); |
105 | } | 112 | } |
113 | EXPORT_SYMBOL_GPL(static_key_slow_dec); | ||
106 | 114 | ||
107 | void jump_label_dec_deferred(struct jump_label_key_deferred *key) | 115 | void static_key_slow_dec_deferred(struct static_key_deferred *key) |
108 | { | 116 | { |
109 | __jump_label_dec(&key->key, key->timeout, &key->work); | 117 | __static_key_slow_dec(&key->key, key->timeout, &key->work); |
110 | } | 118 | } |
119 | EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred); | ||
111 | 120 | ||
112 | 121 | void jump_label_rate_limit(struct static_key_deferred *key, | |
113 | void jump_label_rate_limit(struct jump_label_key_deferred *key, | ||
114 | unsigned long rl) | 122 | unsigned long rl) |
115 | { | 123 | { |
116 | key->timeout = rl; | 124 | key->timeout = rl; |
@@ -153,7 +161,7 @@ void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry | |||
153 | arch_jump_label_transform(entry, type); | 161 | arch_jump_label_transform(entry, type); |
154 | } | 162 | } |
155 | 163 | ||
156 | static void __jump_label_update(struct jump_label_key *key, | 164 | static void __jump_label_update(struct static_key *key, |
157 | struct jump_entry *entry, | 165 | struct jump_entry *entry, |
158 | struct jump_entry *stop, int enable) | 166 | struct jump_entry *stop, int enable) |
159 | { | 167 | { |
@@ -170,27 +178,40 @@ static void __jump_label_update(struct jump_label_key *key, | |||
170 | } | 178 | } |
171 | } | 179 | } |
172 | 180 | ||
181 | static enum jump_label_type jump_label_type(struct static_key *key) | ||
182 | { | ||
183 | bool true_branch = jump_label_get_branch_default(key); | ||
184 | bool state = static_key_enabled(key); | ||
185 | |||
186 | if ((!true_branch && state) || (true_branch && !state)) | ||
187 | return JUMP_LABEL_ENABLE; | ||
188 | |||
189 | return JUMP_LABEL_DISABLE; | ||
190 | } | ||
191 | |||
173 | void __init jump_label_init(void) | 192 | void __init jump_label_init(void) |
174 | { | 193 | { |
175 | struct jump_entry *iter_start = __start___jump_table; | 194 | struct jump_entry *iter_start = __start___jump_table; |
176 | struct jump_entry *iter_stop = __stop___jump_table; | 195 | struct jump_entry *iter_stop = __stop___jump_table; |
177 | struct jump_label_key *key = NULL; | 196 | struct static_key *key = NULL; |
178 | struct jump_entry *iter; | 197 | struct jump_entry *iter; |
179 | 198 | ||
180 | jump_label_lock(); | 199 | jump_label_lock(); |
181 | jump_label_sort_entries(iter_start, iter_stop); | 200 | jump_label_sort_entries(iter_start, iter_stop); |
182 | 201 | ||
183 | for (iter = iter_start; iter < iter_stop; iter++) { | 202 | for (iter = iter_start; iter < iter_stop; iter++) { |
184 | struct jump_label_key *iterk; | 203 | struct static_key *iterk; |
185 | 204 | ||
186 | iterk = (struct jump_label_key *)(unsigned long)iter->key; | 205 | iterk = (struct static_key *)(unsigned long)iter->key; |
187 | arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ? | 206 | arch_jump_label_transform_static(iter, jump_label_type(iterk)); |
188 | JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); | ||
189 | if (iterk == key) | 207 | if (iterk == key) |
190 | continue; | 208 | continue; |
191 | 209 | ||
192 | key = iterk; | 210 | key = iterk; |
193 | key->entries = iter; | 211 | /* |
212 | * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH. | ||
213 | */ | ||
214 | *((unsigned long *)&key->entries) += (unsigned long)iter; | ||
194 | #ifdef CONFIG_MODULES | 215 | #ifdef CONFIG_MODULES |
195 | key->next = NULL; | 216 | key->next = NULL; |
196 | #endif | 217 | #endif |
@@ -200,8 +221,8 @@ void __init jump_label_init(void) | |||
200 | 221 | ||
201 | #ifdef CONFIG_MODULES | 222 | #ifdef CONFIG_MODULES |
202 | 223 | ||
203 | struct jump_label_mod { | 224 | struct static_key_mod { |
204 | struct jump_label_mod *next; | 225 | struct static_key_mod *next; |
205 | struct jump_entry *entries; | 226 | struct jump_entry *entries; |
206 | struct module *mod; | 227 | struct module *mod; |
207 | }; | 228 | }; |
@@ -221,9 +242,9 @@ static int __jump_label_mod_text_reserved(void *start, void *end) | |||
221 | start, end); | 242 | start, end); |
222 | } | 243 | } |
223 | 244 | ||
224 | static void __jump_label_mod_update(struct jump_label_key *key, int enable) | 245 | static void __jump_label_mod_update(struct static_key *key, int enable) |
225 | { | 246 | { |
226 | struct jump_label_mod *mod = key->next; | 247 | struct static_key_mod *mod = key->next; |
227 | 248 | ||
228 | while (mod) { | 249 | while (mod) { |
229 | struct module *m = mod->mod; | 250 | struct module *m = mod->mod; |
@@ -254,11 +275,7 @@ void jump_label_apply_nops(struct module *mod) | |||
254 | return; | 275 | return; |
255 | 276 | ||
256 | for (iter = iter_start; iter < iter_stop; iter++) { | 277 | for (iter = iter_start; iter < iter_stop; iter++) { |
257 | struct jump_label_key *iterk; | 278 | arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE); |
258 | |||
259 | iterk = (struct jump_label_key *)(unsigned long)iter->key; | ||
260 | arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ? | ||
261 | JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); | ||
262 | } | 279 | } |
263 | } | 280 | } |
264 | 281 | ||
@@ -267,8 +284,8 @@ static int jump_label_add_module(struct module *mod) | |||
267 | struct jump_entry *iter_start = mod->jump_entries; | 284 | struct jump_entry *iter_start = mod->jump_entries; |
268 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; | 285 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; |
269 | struct jump_entry *iter; | 286 | struct jump_entry *iter; |
270 | struct jump_label_key *key = NULL; | 287 | struct static_key *key = NULL; |
271 | struct jump_label_mod *jlm; | 288 | struct static_key_mod *jlm; |
272 | 289 | ||
273 | /* if the module doesn't have jump label entries, just return */ | 290 | /* if the module doesn't have jump label entries, just return */ |
274 | if (iter_start == iter_stop) | 291 | if (iter_start == iter_stop) |
@@ -277,28 +294,30 @@ static int jump_label_add_module(struct module *mod) | |||
277 | jump_label_sort_entries(iter_start, iter_stop); | 294 | jump_label_sort_entries(iter_start, iter_stop); |
278 | 295 | ||
279 | for (iter = iter_start; iter < iter_stop; iter++) { | 296 | for (iter = iter_start; iter < iter_stop; iter++) { |
280 | if (iter->key == (jump_label_t)(unsigned long)key) | 297 | struct static_key *iterk; |
281 | continue; | ||
282 | 298 | ||
283 | key = (struct jump_label_key *)(unsigned long)iter->key; | 299 | iterk = (struct static_key *)(unsigned long)iter->key; |
300 | if (iterk == key) | ||
301 | continue; | ||
284 | 302 | ||
303 | key = iterk; | ||
285 | if (__module_address(iter->key) == mod) { | 304 | if (__module_address(iter->key) == mod) { |
286 | atomic_set(&key->enabled, 0); | 305 | /* |
287 | key->entries = iter; | 306 | * Set key->entries to iter, but preserve JUMP_LABEL_TRUE_BRANCH. |
307 | */ | ||
308 | *((unsigned long *)&key->entries) += (unsigned long)iter; | ||
288 | key->next = NULL; | 309 | key->next = NULL; |
289 | continue; | 310 | continue; |
290 | } | 311 | } |
291 | 312 | jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL); | |
292 | jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL); | ||
293 | if (!jlm) | 313 | if (!jlm) |
294 | return -ENOMEM; | 314 | return -ENOMEM; |
295 | |||
296 | jlm->mod = mod; | 315 | jlm->mod = mod; |
297 | jlm->entries = iter; | 316 | jlm->entries = iter; |
298 | jlm->next = key->next; | 317 | jlm->next = key->next; |
299 | key->next = jlm; | 318 | key->next = jlm; |
300 | 319 | ||
301 | if (jump_label_enabled(key)) | 320 | if (jump_label_type(key) == JUMP_LABEL_ENABLE) |
302 | __jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE); | 321 | __jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE); |
303 | } | 322 | } |
304 | 323 | ||
@@ -310,14 +329,14 @@ static void jump_label_del_module(struct module *mod) | |||
310 | struct jump_entry *iter_start = mod->jump_entries; | 329 | struct jump_entry *iter_start = mod->jump_entries; |
311 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; | 330 | struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; |
312 | struct jump_entry *iter; | 331 | struct jump_entry *iter; |
313 | struct jump_label_key *key = NULL; | 332 | struct static_key *key = NULL; |
314 | struct jump_label_mod *jlm, **prev; | 333 | struct static_key_mod *jlm, **prev; |
315 | 334 | ||
316 | for (iter = iter_start; iter < iter_stop; iter++) { | 335 | for (iter = iter_start; iter < iter_stop; iter++) { |
317 | if (iter->key == (jump_label_t)(unsigned long)key) | 336 | if (iter->key == (jump_label_t)(unsigned long)key) |
318 | continue; | 337 | continue; |
319 | 338 | ||
320 | key = (struct jump_label_key *)(unsigned long)iter->key; | 339 | key = (struct static_key *)(unsigned long)iter->key; |
321 | 340 | ||
322 | if (__module_address(iter->key) == mod) | 341 | if (__module_address(iter->key) == mod) |
323 | continue; | 342 | continue; |
@@ -419,9 +438,10 @@ int jump_label_text_reserved(void *start, void *end) | |||
419 | return ret; | 438 | return ret; |
420 | } | 439 | } |
421 | 440 | ||
422 | static void jump_label_update(struct jump_label_key *key, int enable) | 441 | static void jump_label_update(struct static_key *key, int enable) |
423 | { | 442 | { |
424 | struct jump_entry *entry = key->entries, *stop = __stop___jump_table; | 443 | struct jump_entry *stop = __stop___jump_table; |
444 | struct jump_entry *entry = jump_label_get_entries(key); | ||
425 | 445 | ||
426 | #ifdef CONFIG_MODULES | 446 | #ifdef CONFIG_MODULES |
427 | struct module *mod = __module_address((unsigned long)key); | 447 | struct module *mod = __module_address((unsigned long)key); |