aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/jump_label.c
diff options
context:
space:
mode:
authorJason Baron <jbaron@redhat.com>2010-09-17 11:09:00 -0400
committerSteven Rostedt <rostedt@goodmis.org>2010-09-22 16:29:41 -0400
commitbf5438fca2950b03c21ad868090cc1a8fcd49536 (patch)
tree9fc5693763263704de8d8ba1c37a84172dbe5eb7 /kernel/jump_label.c
parentfa6f2cc77081792e4edca9168420a3422299ef15 (diff)
jump label: Base patch for jump label
base patch to implement 'jump labeling'. Based on a new 'asm goto' inline assembly gcc mechanism, we can now branch to labels from an 'asm goto' statment. This allows us to create a 'no-op' fastpath, which can subsequently be patched with a jump to the slowpath code. This is useful for code which might be rarely used, but which we'd like to be able to call, if needed. Tracepoints are the current usecase that these are being implemented for. Acked-by: David S. Miller <davem@davemloft.net> Signed-off-by: Jason Baron <jbaron@redhat.com> LKML-Reference: <ee8b3595967989fdaf84e698dc7447d315ce972a.1284733808.git.jbaron@redhat.com> [ cleaned up some formating ] Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Diffstat (limited to 'kernel/jump_label.c')
-rw-r--r--kernel/jump_label.c346
1 files changed, 346 insertions, 0 deletions
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
new file mode 100644
index 000000000000..460fd40112b3
--- /dev/null
+++ b/kernel/jump_label.c
@@ -0,0 +1,346 @@
1/*
2 * jump label support
3 *
4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
5 *
6 */
7#include <linux/jump_label.h>
8#include <linux/memory.h>
9#include <linux/uaccess.h>
10#include <linux/module.h>
11#include <linux/list.h>
12#include <linux/jhash.h>
13#include <linux/slab.h>
14#include <linux/sort.h>
15#include <linux/err.h>
16
17#ifdef HAVE_JUMP_LABEL
18
19#define JUMP_LABEL_HASH_BITS 6
20#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
21static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
22
23/* mutex to protect coming/going of the the jump_label table */
24static DEFINE_MUTEX(jump_label_mutex);
25
26struct jump_label_entry {
27 struct hlist_node hlist;
28 struct jump_entry *table;
29 int nr_entries;
30 /* hang modules off here */
31 struct hlist_head modules;
32 unsigned long key;
33};
34
35struct jump_label_module_entry {
36 struct hlist_node hlist;
37 struct jump_entry *table;
38 int nr_entries;
39 struct module *mod;
40};
41
42static int jump_label_cmp(const void *a, const void *b)
43{
44 const struct jump_entry *jea = a;
45 const struct jump_entry *jeb = b;
46
47 if (jea->key < jeb->key)
48 return -1;
49
50 if (jea->key > jeb->key)
51 return 1;
52
53 return 0;
54}
55
56static void
57sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
58{
59 unsigned long size;
60
61 size = (((unsigned long)stop - (unsigned long)start)
62 / sizeof(struct jump_entry));
63 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
64}
65
66static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
67{
68 struct hlist_head *head;
69 struct hlist_node *node;
70 struct jump_label_entry *e;
71 u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
72
73 head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
74 hlist_for_each_entry(e, node, head, hlist) {
75 if (key == e->key)
76 return e;
77 }
78 return NULL;
79}
80
81static struct jump_label_entry *
82add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
83{
84 struct hlist_head *head;
85 struct jump_label_entry *e;
86 u32 hash;
87
88 e = get_jump_label_entry(key);
89 if (e)
90 return ERR_PTR(-EEXIST);
91
92 e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
93 if (!e)
94 return ERR_PTR(-ENOMEM);
95
96 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
97 head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
98 e->key = key;
99 e->table = table;
100 e->nr_entries = nr_entries;
101 INIT_HLIST_HEAD(&(e->modules));
102 hlist_add_head(&e->hlist, head);
103 return e;
104}
105
106static int
107build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
108{
109 struct jump_entry *iter, *iter_begin;
110 struct jump_label_entry *entry;
111 int count;
112
113 sort_jump_label_entries(start, stop);
114 iter = start;
115 while (iter < stop) {
116 entry = get_jump_label_entry(iter->key);
117 if (!entry) {
118 iter_begin = iter;
119 count = 0;
120 while ((iter < stop) &&
121 (iter->key == iter_begin->key)) {
122 iter++;
123 count++;
124 }
125 entry = add_jump_label_entry(iter_begin->key,
126 count, iter_begin);
127 if (IS_ERR(entry))
128 return PTR_ERR(entry);
129 } else {
130 WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
131 return -1;
132 }
133 }
134 return 0;
135}
136
137/***
138 * jump_label_update - update jump label text
139 * @key - key value associated with a a jump label
140 * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
141 *
142 * Will enable/disable the jump for jump label @key, depending on the
143 * value of @type.
144 *
145 */
146
147void jump_label_update(unsigned long key, enum jump_label_type type)
148{
149 struct jump_entry *iter;
150 struct jump_label_entry *entry;
151 struct hlist_node *module_node;
152 struct jump_label_module_entry *e_module;
153 int count;
154
155 mutex_lock(&jump_label_mutex);
156 entry = get_jump_label_entry((jump_label_t)key);
157 if (entry) {
158 count = entry->nr_entries;
159 iter = entry->table;
160 while (count--) {
161 if (kernel_text_address(iter->code))
162 arch_jump_label_transform(iter, type);
163 iter++;
164 }
165 /* eanble/disable jump labels in modules */
166 hlist_for_each_entry(e_module, module_node, &(entry->modules),
167 hlist) {
168 count = e_module->nr_entries;
169 iter = e_module->table;
170 while (count--) {
171 if (kernel_text_address(iter->code))
172 arch_jump_label_transform(iter, type);
173 iter++;
174 }
175 }
176 }
177 mutex_unlock(&jump_label_mutex);
178}
179
180static __init int init_jump_label(void)
181{
182 int ret;
183 struct jump_entry *iter_start = __start___jump_table;
184 struct jump_entry *iter_stop = __stop___jump_table;
185 struct jump_entry *iter;
186
187 mutex_lock(&jump_label_mutex);
188 ret = build_jump_label_hashtable(__start___jump_table,
189 __stop___jump_table);
190 iter = iter_start;
191 while (iter < iter_stop) {
192 arch_jump_label_text_poke_early(iter->code);
193 iter++;
194 }
195 mutex_unlock(&jump_label_mutex);
196 return ret;
197}
198early_initcall(init_jump_label);
199
200#ifdef CONFIG_MODULES
201
202static struct jump_label_module_entry *
203add_jump_label_module_entry(struct jump_label_entry *entry,
204 struct jump_entry *iter_begin,
205 int count, struct module *mod)
206{
207 struct jump_label_module_entry *e;
208
209 e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
210 if (!e)
211 return ERR_PTR(-ENOMEM);
212 e->mod = mod;
213 e->nr_entries = count;
214 e->table = iter_begin;
215 hlist_add_head(&e->hlist, &entry->modules);
216 return e;
217}
218
219static int add_jump_label_module(struct module *mod)
220{
221 struct jump_entry *iter, *iter_begin;
222 struct jump_label_entry *entry;
223 struct jump_label_module_entry *module_entry;
224 int count;
225
226 /* if the module doesn't have jump label entries, just return */
227 if (!mod->num_jump_entries)
228 return 0;
229
230 sort_jump_label_entries(mod->jump_entries,
231 mod->jump_entries + mod->num_jump_entries);
232 iter = mod->jump_entries;
233 while (iter < mod->jump_entries + mod->num_jump_entries) {
234 entry = get_jump_label_entry(iter->key);
235 iter_begin = iter;
236 count = 0;
237 while ((iter < mod->jump_entries + mod->num_jump_entries) &&
238 (iter->key == iter_begin->key)) {
239 iter++;
240 count++;
241 }
242 if (!entry) {
243 entry = add_jump_label_entry(iter_begin->key, 0, NULL);
244 if (IS_ERR(entry))
245 return PTR_ERR(entry);
246 }
247 module_entry = add_jump_label_module_entry(entry, iter_begin,
248 count, mod);
249 if (IS_ERR(module_entry))
250 return PTR_ERR(module_entry);
251 }
252 return 0;
253}
254
255static void remove_jump_label_module(struct module *mod)
256{
257 struct hlist_head *head;
258 struct hlist_node *node, *node_next, *module_node, *module_node_next;
259 struct jump_label_entry *e;
260 struct jump_label_module_entry *e_module;
261 int i;
262
263 /* if the module doesn't have jump label entries, just return */
264 if (!mod->num_jump_entries)
265 return;
266
267 for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
268 head = &jump_label_table[i];
269 hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
270 hlist_for_each_entry_safe(e_module, module_node,
271 module_node_next,
272 &(e->modules), hlist) {
273 if (e_module->mod == mod) {
274 hlist_del(&e_module->hlist);
275 kfree(e_module);
276 }
277 }
278 if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
279 hlist_del(&e->hlist);
280 kfree(e);
281 }
282 }
283 }
284}
285
286static int
287jump_label_module_notify(struct notifier_block *self, unsigned long val,
288 void *data)
289{
290 struct module *mod = data;
291 int ret = 0;
292
293 switch (val) {
294 case MODULE_STATE_COMING:
295 mutex_lock(&jump_label_mutex);
296 ret = add_jump_label_module(mod);
297 if (ret)
298 remove_jump_label_module(mod);
299 mutex_unlock(&jump_label_mutex);
300 break;
301 case MODULE_STATE_GOING:
302 mutex_lock(&jump_label_mutex);
303 remove_jump_label_module(mod);
304 mutex_unlock(&jump_label_mutex);
305 break;
306 }
307 return ret;
308}
309
310/***
311 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
312 * @mod: module to patch
313 *
314 * Allow for run-time selection of the optimal nops. Before the module
315 * loads patch these with arch_get_jump_label_nop(), which is specified by
316 * the arch specific jump label code.
317 */
318void jump_label_apply_nops(struct module *mod)
319{
320 struct jump_entry *iter;
321
322 /* if the module doesn't have jump label entries, just return */
323 if (!mod->num_jump_entries)
324 return;
325
326 iter = mod->jump_entries;
327 while (iter < mod->jump_entries + mod->num_jump_entries) {
328 arch_jump_label_text_poke_early(iter->code);
329 iter++;
330 }
331}
332
333struct notifier_block jump_label_module_nb = {
334 .notifier_call = jump_label_module_notify,
335 .priority = 0,
336};
337
338static __init int init_jump_label_module(void)
339{
340 return register_module_notifier(&jump_label_module_nb);
341}
342early_initcall(init_jump_label_module);
343
344#endif /* CONFIG_MODULES */
345
346#endif
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-01 20:17:20 -0500