aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/Makefile2
-rw-r--r--kernel/jump_label.c346
-rw-r--r--kernel/kprobes.c1
-rw-r--r--kernel/module.c6
4 files changed, 354 insertions, 1 deletions
diff --git a/kernel/Makefile b/kernel/Makefile
index 0b72d1a74be0..d52b473c99a1 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -10,7 +10,7 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o \
10 kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ 10 kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
11 hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \ 11 hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
12 notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \ 12 notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \
13 async.o range.o 13 async.o range.o jump_label.o
14obj-$(CONFIG_HAVE_EARLY_RES) += early_res.o 14obj-$(CONFIG_HAVE_EARLY_RES) += early_res.o
15obj-y += groups.o 15obj-y += groups.o
16 16
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
new file mode 100644
index 000000000000..460fd40112b3
--- /dev/null
+++ b/kernel/jump_label.c
@@ -0,0 +1,346 @@
1/*
2 * jump label support
3 *
4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
5 *
6 */
7#include <linux/jump_label.h>
8#include <linux/memory.h>
9#include <linux/uaccess.h>
10#include <linux/module.h>
11#include <linux/list.h>
12#include <linux/jhash.h>
13#include <linux/slab.h>
14#include <linux/sort.h>
15#include <linux/err.h>
16
17#ifdef HAVE_JUMP_LABEL
18
19#define JUMP_LABEL_HASH_BITS 6
20#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
21static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
22
23/* mutex to protect coming/going of the the jump_label table */
24static DEFINE_MUTEX(jump_label_mutex);
25
26struct jump_label_entry {
27 struct hlist_node hlist;
28 struct jump_entry *table;
29 int nr_entries;
30 /* hang modules off here */
31 struct hlist_head modules;
32 unsigned long key;
33};
34
35struct jump_label_module_entry {
36 struct hlist_node hlist;
37 struct jump_entry *table;
38 int nr_entries;
39 struct module *mod;
40};
41
42static int jump_label_cmp(const void *a, const void *b)
43{
44 const struct jump_entry *jea = a;
45 const struct jump_entry *jeb = b;
46
47 if (jea->key < jeb->key)
48 return -1;
49
50 if (jea->key > jeb->key)
51 return 1;
52
53 return 0;
54}
55
56static void
57sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
58{
59 unsigned long size;
60
61 size = (((unsigned long)stop - (unsigned long)start)
62 / sizeof(struct jump_entry));
63 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
64}
65
66static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
67{
68 struct hlist_head *head;
69 struct hlist_node *node;
70 struct jump_label_entry *e;
71 u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
72
73 head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
74 hlist_for_each_entry(e, node, head, hlist) {
75 if (key == e->key)
76 return e;
77 }
78 return NULL;
79}
80
81static struct jump_label_entry *
82add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
83{
84 struct hlist_head *head;
85 struct jump_label_entry *e;
86 u32 hash;
87
88 e = get_jump_label_entry(key);
89 if (e)
90 return ERR_PTR(-EEXIST);
91
92 e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
93 if (!e)
94 return ERR_PTR(-ENOMEM);
95
96 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
97 head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
98 e->key = key;
99 e->table = table;
100 e->nr_entries = nr_entries;
101 INIT_HLIST_HEAD(&(e->modules));
102 hlist_add_head(&e->hlist, head);
103 return e;
104}
105
106static int
107build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
108{
109 struct jump_entry *iter, *iter_begin;
110 struct jump_label_entry *entry;
111 int count;
112
113 sort_jump_label_entries(start, stop);
114 iter = start;
115 while (iter < stop) {
116 entry = get_jump_label_entry(iter->key);
117 if (!entry) {
118 iter_begin = iter;
119 count = 0;
120 while ((iter < stop) &&
121 (iter->key == iter_begin->key)) {
122 iter++;
123 count++;
124 }
125 entry = add_jump_label_entry(iter_begin->key,
126 count, iter_begin);
127 if (IS_ERR(entry))
128 return PTR_ERR(entry);
129 } else {
130 WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
131 return -1;
132 }
133 }
134 return 0;
135}
136
137/***
138 * jump_label_update - update jump label text
139 * @key - key value associated with a a jump label
140 * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
141 *
142 * Will enable/disable the jump for jump label @key, depending on the
143 * value of @type.
144 *
145 */
146
147void jump_label_update(unsigned long key, enum jump_label_type type)
148{
149 struct jump_entry *iter;
150 struct jump_label_entry *entry;
151 struct hlist_node *module_node;
152 struct jump_label_module_entry *e_module;
153 int count;
154
155 mutex_lock(&jump_label_mutex);
156 entry = get_jump_label_entry((jump_label_t)key);
157 if (entry) {
158 count = entry->nr_entries;
159 iter = entry->table;
160 while (count--) {
161 if (kernel_text_address(iter->code))
162 arch_jump_label_transform(iter, type);
163 iter++;
164 }
165 /* eanble/disable jump labels in modules */
166 hlist_for_each_entry(e_module, module_node, &(entry->modules),
167 hlist) {
168 count = e_module->nr_entries;
169 iter = e_module->table;
170 while (count--) {
171 if (kernel_text_address(iter->code))
172 arch_jump_label_transform(iter, type);
173 iter++;
174 }
175 }
176 }
177 mutex_unlock(&jump_label_mutex);
178}
179
180static __init int init_jump_label(void)
181{
182 int ret;
183 struct jump_entry *iter_start = __start___jump_table;
184 struct jump_entry *iter_stop = __stop___jump_table;
185 struct jump_entry *iter;
186
187 mutex_lock(&jump_label_mutex);
188 ret = build_jump_label_hashtable(__start___jump_table,
189 __stop___jump_table);
190 iter = iter_start;
191 while (iter < iter_stop) {
192 arch_jump_label_text_poke_early(iter->code);
193 iter++;
194 }
195 mutex_unlock(&jump_label_mutex);
196 return ret;
197}
198early_initcall(init_jump_label);
199
200#ifdef CONFIG_MODULES
201
202static struct jump_label_module_entry *
203add_jump_label_module_entry(struct jump_label_entry *entry,
204 struct jump_entry *iter_begin,
205 int count, struct module *mod)
206{
207 struct jump_label_module_entry *e;
208
209 e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
210 if (!e)
211 return ERR_PTR(-ENOMEM);
212 e->mod = mod;
213 e->nr_entries = count;
214 e->table = iter_begin;
215 hlist_add_head(&e->hlist, &entry->modules);
216 return e;
217}
218
219static int add_jump_label_module(struct module *mod)
220{
221 struct jump_entry *iter, *iter_begin;
222 struct jump_label_entry *entry;
223 struct jump_label_module_entry *module_entry;
224 int count;
225
226 /* if the module doesn't have jump label entries, just return */
227 if (!mod->num_jump_entries)
228 return 0;
229
230 sort_jump_label_entries(mod->jump_entries,
231 mod->jump_entries + mod->num_jump_entries);
232 iter = mod->jump_entries;
233 while (iter < mod->jump_entries + mod->num_jump_entries) {
234 entry = get_jump_label_entry(iter->key);
235 iter_begin = iter;
236 count = 0;
237 while ((iter < mod->jump_entries + mod->num_jump_entries) &&
238 (iter->key == iter_begin->key)) {
239 iter++;
240 count++;
241 }
242 if (!entry) {
243 entry = add_jump_label_entry(iter_begin->key, 0, NULL);
244 if (IS_ERR(entry))
245 return PTR_ERR(entry);
246 }
247 module_entry = add_jump_label_module_entry(entry, iter_begin,
248 count, mod);
249 if (IS_ERR(module_entry))
250 return PTR_ERR(module_entry);
251 }
252 return 0;
253}
254
255static void remove_jump_label_module(struct module *mod)
256{
257 struct hlist_head *head;
258 struct hlist_node *node, *node_next, *module_node, *module_node_next;
259 struct jump_label_entry *e;
260 struct jump_label_module_entry *e_module;
261 int i;
262
263 /* if the module doesn't have jump label entries, just return */
264 if (!mod->num_jump_entries)
265 return;
266
267 for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
268 head = &jump_label_table[i];
269 hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
270 hlist_for_each_entry_safe(e_module, module_node,
271 module_node_next,
272 &(e->modules), hlist) {
273 if (e_module->mod == mod) {
274 hlist_del(&e_module->hlist);
275 kfree(e_module);
276 }
277 }
278 if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
279 hlist_del(&e->hlist);
280 kfree(e);
281 }
282 }
283 }
284}
285
286static int
287jump_label_module_notify(struct notifier_block *self, unsigned long val,
288 void *data)
289{
290 struct module *mod = data;
291 int ret = 0;
292
293 switch (val) {
294 case MODULE_STATE_COMING:
295 mutex_lock(&jump_label_mutex);
296 ret = add_jump_label_module(mod);
297 if (ret)
298 remove_jump_label_module(mod);
299 mutex_unlock(&jump_label_mutex);
300 break;
301 case MODULE_STATE_GOING:
302 mutex_lock(&jump_label_mutex);
303 remove_jump_label_module(mod);
304 mutex_unlock(&jump_label_mutex);
305 break;
306 }
307 return ret;
308}
309
310/***
311 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
312 * @mod: module to patch
313 *
314 * Allow for run-time selection of the optimal nops. Before the module
315 * loads patch these with arch_get_jump_label_nop(), which is specified by
316 * the arch specific jump label code.
317 */
318void jump_label_apply_nops(struct module *mod)
319{
320 struct jump_entry *iter;
321
322 /* if the module doesn't have jump label entries, just return */
323 if (!mod->num_jump_entries)
324 return;
325
326 iter = mod->jump_entries;
327 while (iter < mod->jump_entries + mod->num_jump_entries) {
328 arch_jump_label_text_poke_early(iter->code);
329 iter++;
330 }
331}
332
333struct notifier_block jump_label_module_nb = {
334 .notifier_call = jump_label_module_notify,
335 .priority = 0,
336};
337
338static __init int init_jump_label_module(void)
339{
340 return register_module_notifier(&jump_label_module_nb);
341}
342early_initcall(init_jump_label_module);
343
344#endif /* CONFIG_MODULES */
345
346#endif
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 6dd5359e1f0e..18904e42a918 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -47,6 +47,7 @@
47#include <linux/memory.h> 47#include <linux/memory.h>
48#include <linux/ftrace.h> 48#include <linux/ftrace.h>
49#include <linux/cpu.h> 49#include <linux/cpu.h>
50#include <linux/jump_label.h>
50 51
51#include <asm-generic/sections.h> 52#include <asm-generic/sections.h>
52#include <asm/cacheflush.h> 53#include <asm/cacheflush.h>
diff --git a/kernel/module.c b/kernel/module.c
index d0b5f8db11b4..eba134157ef6 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -55,6 +55,7 @@
55#include <linux/async.h> 55#include <linux/async.h>
56#include <linux/percpu.h> 56#include <linux/percpu.h>
57#include <linux/kmemleak.h> 57#include <linux/kmemleak.h>
58#include <linux/jump_label.h>
58 59
59#define CREATE_TRACE_POINTS 60#define CREATE_TRACE_POINTS
60#include <trace/events/module.h> 61#include <trace/events/module.h>
@@ -2308,6 +2309,11 @@ static void find_module_sections(struct module *mod, struct load_info *info)
2308 sizeof(*mod->tracepoints), 2309 sizeof(*mod->tracepoints),
2309 &mod->num_tracepoints); 2310 &mod->num_tracepoints);
2310#endif 2311#endif
2312#ifdef HAVE_JUMP_LABEL
2313 mod->jump_entries = section_objs(info, "__jump_table",
2314 sizeof(*mod->jump_entries),
2315 &mod->num_jump_entries);
2316#endif
2311#ifdef CONFIG_EVENT_TRACING 2317#ifdef CONFIG_EVENT_TRACING
2312 mod->trace_events = section_objs(info, "_ftrace_events", 2318 mod->trace_events = section_objs(info, "_ftrace_events",
2313 sizeof(*mod->trace_events), 2319 sizeof(*mod->trace_events),
generated by cgit v1.2.2 (git 2.25.0) at 2025-11-01 20:59:03 -0400