aboutsummaryrefslogtreecommitdiffstats
path: root/arch/i386/kernel/alternative.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/i386/kernel/alternative.c')
-rw-r--r--arch/i386/kernel/alternative.c321
1 files changed, 321 insertions, 0 deletions
diff --git a/arch/i386/kernel/alternative.c b/arch/i386/kernel/alternative.c
new file mode 100644
index 000000000000..5cbd6f99fb2a
--- /dev/null
+++ b/arch/i386/kernel/alternative.c
@@ -0,0 +1,321 @@
1#include <linux/module.h>
2#include <linux/spinlock.h>
3#include <linux/list.h>
4#include <asm/alternative.h>
5#include <asm/sections.h>
6
7#define DEBUG 0
8#if DEBUG
9# define DPRINTK(fmt, args...) printk(fmt, args)
10#else
11# define DPRINTK(fmt, args...)
12#endif
13
14/* Use inline assembly to define this because the nops are defined
15 as inline assembly strings in the include files and we cannot
16 get them easily into strings. */
17asm("\t.data\nintelnops: "
18 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
19 GENERIC_NOP7 GENERIC_NOP8);
20asm("\t.data\nk8nops: "
21 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
22 K8_NOP7 K8_NOP8);
23asm("\t.data\nk7nops: "
24 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
25 K7_NOP7 K7_NOP8);
26
27extern unsigned char intelnops[], k8nops[], k7nops[];
28static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
29 NULL,
30 intelnops,
31 intelnops + 1,
32 intelnops + 1 + 2,
33 intelnops + 1 + 2 + 3,
34 intelnops + 1 + 2 + 3 + 4,
35 intelnops + 1 + 2 + 3 + 4 + 5,
36 intelnops + 1 + 2 + 3 + 4 + 5 + 6,
37 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
38};
39static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
40 NULL,
41 k8nops,
42 k8nops + 1,
43 k8nops + 1 + 2,
44 k8nops + 1 + 2 + 3,
45 k8nops + 1 + 2 + 3 + 4,
46 k8nops + 1 + 2 + 3 + 4 + 5,
47 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
48 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
49};
50static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
51 NULL,
52 k7nops,
53 k7nops + 1,
54 k7nops + 1 + 2,
55 k7nops + 1 + 2 + 3,
56 k7nops + 1 + 2 + 3 + 4,
57 k7nops + 1 + 2 + 3 + 4 + 5,
58 k7nops + 1 + 2 + 3 + 4 + 5 + 6,
59 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
60};
61static struct nop {
62 int cpuid;
63 unsigned char **noptable;
64} noptypes[] = {
65 { X86_FEATURE_K8, k8_nops },
66 { X86_FEATURE_K7, k7_nops },
67 { -1, NULL }
68};
69
70
71extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
72extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];
73extern u8 *__smp_locks[], *__smp_locks_end[];
74
75extern u8 __smp_alt_begin[], __smp_alt_end[];
76
77
78static unsigned char** find_nop_table(void)
79{
80 unsigned char **noptable = intel_nops;
81 int i;
82
83 for (i = 0; noptypes[i].cpuid >= 0; i++) {
84 if (boot_cpu_has(noptypes[i].cpuid)) {
85 noptable = noptypes[i].noptable;
86 break;
87 }
88 }
89 return noptable;
90}
91
92/* Replace instructions with better alternatives for this CPU type.
93 This runs before SMP is initialized to avoid SMP problems with
94 self modifying code. This implies that assymetric systems where
95 APs have less capabilities than the boot processor are not handled.
96 Tough. Make sure you disable such features by hand. */
97
98void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
99{
100 unsigned char **noptable = find_nop_table();
101 struct alt_instr *a;
102 int diff, i, k;
103
104 DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
105 for (a = start; a < end; a++) {
106 BUG_ON(a->replacementlen > a->instrlen);
107 if (!boot_cpu_has(a->cpuid))
108 continue;
109 memcpy(a->instr, a->replacement, a->replacementlen);
110 diff = a->instrlen - a->replacementlen;
111 /* Pad the rest with nops */
112 for (i = a->replacementlen; diff > 0; diff -= k, i += k) {
113 k = diff;
114 if (k > ASM_NOP_MAX)
115 k = ASM_NOP_MAX;
116 memcpy(a->instr + i, noptable[k], k);
117 }
118 }
119}
120
121static void alternatives_smp_save(struct alt_instr *start, struct alt_instr *end)
122{
123 struct alt_instr *a;
124
125 DPRINTK("%s: alt table %p-%p\n", __FUNCTION__, start, end);
126 for (a = start; a < end; a++) {
127 memcpy(a->replacement + a->replacementlen,
128 a->instr,
129 a->instrlen);
130 }
131}
132
133static void alternatives_smp_apply(struct alt_instr *start, struct alt_instr *end)
134{
135 struct alt_instr *a;
136
137 for (a = start; a < end; a++) {
138 memcpy(a->instr,
139 a->replacement + a->replacementlen,
140 a->instrlen);
141 }
142}
143
144static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
145{
146 u8 **ptr;
147
148 for (ptr = start; ptr < end; ptr++) {
149 if (*ptr < text)
150 continue;
151 if (*ptr > text_end)
152 continue;
153 **ptr = 0xf0; /* lock prefix */
154 };
155}
156
157static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
158{
159 unsigned char **noptable = find_nop_table();
160 u8 **ptr;
161
162 for (ptr = start; ptr < end; ptr++) {
163 if (*ptr < text)
164 continue;
165 if (*ptr > text_end)
166 continue;
167 **ptr = noptable[1][0];
168 };
169}
170
171struct smp_alt_module {
172 /* what is this ??? */
173 struct module *mod;
174 char *name;
175
176 /* ptrs to lock prefixes */
177 u8 **locks;
178 u8 **locks_end;
179
180 /* .text segment, needed to avoid patching init code ;) */
181 u8 *text;
182 u8 *text_end;
183
184 struct list_head next;
185};
186static LIST_HEAD(smp_alt_modules);
187static DEFINE_SPINLOCK(smp_alt);
188
189static int smp_alt_once = 0;
190static int __init bootonly(char *str)
191{
192 smp_alt_once = 1;
193 return 1;
194}
195__setup("smp-alt-boot", bootonly);
196
197void alternatives_smp_module_add(struct module *mod, char *name,
198 void *locks, void *locks_end,
199 void *text, void *text_end)
200{
201 struct smp_alt_module *smp;
202 unsigned long flags;
203
204 if (smp_alt_once) {
205 if (boot_cpu_has(X86_FEATURE_UP))
206 alternatives_smp_unlock(locks, locks_end,
207 text, text_end);
208 return;
209 }
210
211 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
212 if (NULL == smp)
213 return; /* we'll run the (safe but slow) SMP code then ... */
214
215 smp->mod = mod;
216 smp->name = name;
217 smp->locks = locks;
218 smp->locks_end = locks_end;
219 smp->text = text;
220 smp->text_end = text_end;
221 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
222 __FUNCTION__, smp->locks, smp->locks_end,
223 smp->text, smp->text_end, smp->name);
224
225 spin_lock_irqsave(&smp_alt, flags);
226 list_add_tail(&smp->next, &smp_alt_modules);
227 if (boot_cpu_has(X86_FEATURE_UP))
228 alternatives_smp_unlock(smp->locks, smp->locks_end,
229 smp->text, smp->text_end);
230 spin_unlock_irqrestore(&smp_alt, flags);
231}
232
233void alternatives_smp_module_del(struct module *mod)
234{
235 struct smp_alt_module *item;
236 unsigned long flags;
237
238 if (smp_alt_once)
239 return;
240
241 spin_lock_irqsave(&smp_alt, flags);
242 list_for_each_entry(item, &smp_alt_modules, next) {
243 if (mod != item->mod)
244 continue;
245 list_del(&item->next);
246 spin_unlock_irqrestore(&smp_alt, flags);
247 DPRINTK("%s: %s\n", __FUNCTION__, item->name);
248 kfree(item);
249 return;
250 }
251 spin_unlock_irqrestore(&smp_alt, flags);
252}
253
254void alternatives_smp_switch(int smp)
255{
256 struct smp_alt_module *mod;
257 unsigned long flags;
258
259 if (smp_alt_once)
260 return;
261 BUG_ON(!smp && (num_online_cpus() > 1));
262
263 spin_lock_irqsave(&smp_alt, flags);
264 if (smp) {
265 printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
266 clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
267 clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
268 alternatives_smp_apply(__smp_alt_instructions,
269 __smp_alt_instructions_end);
270 list_for_each_entry(mod, &smp_alt_modules, next)
271 alternatives_smp_lock(mod->locks, mod->locks_end,
272 mod->text, mod->text_end);
273 } else {
274 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
275 set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
276 set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
277 apply_alternatives(__smp_alt_instructions,
278 __smp_alt_instructions_end);
279 list_for_each_entry(mod, &smp_alt_modules, next)
280 alternatives_smp_unlock(mod->locks, mod->locks_end,
281 mod->text, mod->text_end);
282 }
283 spin_unlock_irqrestore(&smp_alt, flags);
284}
285
286void __init alternative_instructions(void)
287{
288 apply_alternatives(__alt_instructions, __alt_instructions_end);
289
290 /* switch to patch-once-at-boottime-only mode and free the
291 * tables in case we know the number of CPUs will never ever
292 * change */
293#ifdef CONFIG_HOTPLUG_CPU
294 if (num_possible_cpus() < 2)
295 smp_alt_once = 1;
296#else
297 smp_alt_once = 1;
298#endif
299
300 if (smp_alt_once) {
301 if (1 == num_possible_cpus()) {
302 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
303 set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
304 set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
305 apply_alternatives(__smp_alt_instructions,
306 __smp_alt_instructions_end);
307 alternatives_smp_unlock(__smp_locks, __smp_locks_end,
308 _text, _etext);
309 }
310 free_init_pages("SMP alternatives",
311 (unsigned long)__smp_alt_begin,
312 (unsigned long)__smp_alt_end);
313 } else {
314 alternatives_smp_save(__smp_alt_instructions,
315 __smp_alt_instructions_end);
316 alternatives_smp_module_add(NULL, "core kernel",
317 __smp_locks, __smp_locks_end,
318 _text, _etext);
319 alternatives_smp_switch(0);
320 }
321}
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-24 20:36:01 -0400