aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/i386/kernel/alternative.c118
-rw-r--r--arch/x86_64/kernel/Makefile4
-rw-r--r--arch/x86_64/kernel/module.c38
-rw-r--r--arch/x86_64/kernel/setup.c76
-rw-r--r--arch/x86_64/kernel/smpboot.c4
-rw-r--r--arch/x86_64/kernel/vmlinux.lds.S20
-rw-r--r--arch/x86_64/mm/init.c27
-rw-r--r--include/asm-i386/alternative.h2
-rw-r--r--include/asm-x86_64/alternative.h146
-rw-r--r--include/asm-x86_64/atomic.h42
-rw-r--r--include/asm-x86_64/bitops.h7
-rw-r--r--include/asm-x86_64/cpufeature.h2
-rw-r--r--include/asm-x86_64/mutex.h4
-rw-r--r--include/asm-x86_64/rwlock.h8
-rw-r--r--include/asm-x86_64/semaphore.h8
-rw-r--r--include/asm-x86_64/spinlock.h10
-rw-r--r--include/asm-x86_64/system.h86
17 files changed, 344 insertions, 258 deletions
diff --git a/arch/i386/kernel/alternative.c b/arch/i386/kernel/alternative.c
index 5cbd6f99fb2a..50eb0e03777e 100644
--- a/arch/i386/kernel/alternative.c
+++ b/arch/i386/kernel/alternative.c
@@ -4,27 +4,41 @@
4#include <asm/alternative.h> 4#include <asm/alternative.h>
5#include <asm/sections.h> 5#include <asm/sections.h>
6 6
7#define DEBUG 0 7static int no_replacement = 0;
8#if DEBUG 8static int smp_alt_once = 0;
9# define DPRINTK(fmt, args...) printk(fmt, args) 9static int debug_alternative = 0;
10#else 10
11# define DPRINTK(fmt, args...) 11static int __init noreplacement_setup(char *s)
12#endif 12{
13 no_replacement = 1;
14 return 1;
15}
16static int __init bootonly(char *str)
17{
18 smp_alt_once = 1;
19 return 1;
20}
21static int __init debug_alt(char *str)
22{
23 debug_alternative = 1;
24 return 1;
25}
13 26
27__setup("noreplacement", noreplacement_setup);
28__setup("smp-alt-boot", bootonly);
29__setup("debug-alternative", debug_alt);
30
31#define DPRINTK(fmt, args...) if (debug_alternative) \
32 printk(KERN_DEBUG fmt, args)
33
34#ifdef GENERIC_NOP1
14/* Use inline assembly to define this because the nops are defined 35/* Use inline assembly to define this because the nops are defined
15 as inline assembly strings in the include files and we cannot 36 as inline assembly strings in the include files and we cannot
16 get them easily into strings. */ 37 get them easily into strings. */
17asm("\t.data\nintelnops: " 38asm("\t.data\nintelnops: "
18 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6 39 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
19 GENERIC_NOP7 GENERIC_NOP8); 40 GENERIC_NOP7 GENERIC_NOP8);
20asm("\t.data\nk8nops: " 41extern unsigned char intelnops[];
21 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
22 K8_NOP7 K8_NOP8);
23asm("\t.data\nk7nops: "
24 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
25 K7_NOP7 K7_NOP8);
26
27extern unsigned char intelnops[], k8nops[], k7nops[];
28static unsigned char *intel_nops[ASM_NOP_MAX+1] = { 42static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
29 NULL, 43 NULL,
30 intelnops, 44 intelnops,
@@ -36,6 +50,13 @@ static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
36 intelnops + 1 + 2 + 3 + 4 + 5 + 6, 50 intelnops + 1 + 2 + 3 + 4 + 5 + 6,
37 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 51 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
38}; 52};
53#endif
54
55#ifdef K8_NOP1
56asm("\t.data\nk8nops: "
57 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
58 K8_NOP7 K8_NOP8);
59extern unsigned char k8nops[];
39static unsigned char *k8_nops[ASM_NOP_MAX+1] = { 60static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
40 NULL, 61 NULL,
41 k8nops, 62 k8nops,
@@ -47,6 +68,13 @@ static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
47 k8nops + 1 + 2 + 3 + 4 + 5 + 6, 68 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
48 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 69 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
49}; 70};
71#endif
72
73#ifdef K7_NOP1
74asm("\t.data\nk7nops: "
75 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
76 K7_NOP7 K7_NOP8);
77extern unsigned char k7nops[];
50static unsigned char *k7_nops[ASM_NOP_MAX+1] = { 78static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
51 NULL, 79 NULL,
52 k7nops, 80 k7nops,
@@ -58,6 +86,18 @@ static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
58 k7nops + 1 + 2 + 3 + 4 + 5 + 6, 86 k7nops + 1 + 2 + 3 + 4 + 5 + 6,
59 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 87 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
60}; 88};
89#endif
90
91#ifdef CONFIG_X86_64
92
93extern char __vsyscall_0;
94static inline unsigned char** find_nop_table(void)
95{
96 return k8_nops;
97}
98
99#else /* CONFIG_X86_64 */
100
61static struct nop { 101static struct nop {
62 int cpuid; 102 int cpuid;
63 unsigned char **noptable; 103 unsigned char **noptable;
@@ -67,14 +107,6 @@ static struct nop {
67 { -1, NULL } 107 { -1, NULL }
68}; 108};
69 109
70
71extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
72extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];
73extern u8 *__smp_locks[], *__smp_locks_end[];
74
75extern u8 __smp_alt_begin[], __smp_alt_end[];
76
77
78static unsigned char** find_nop_table(void) 110static unsigned char** find_nop_table(void)
79{ 111{
80 unsigned char **noptable = intel_nops; 112 unsigned char **noptable = intel_nops;
@@ -89,6 +121,14 @@ static unsigned char** find_nop_table(void)
89 return noptable; 121 return noptable;
90} 122}
91 123
124#endif /* CONFIG_X86_64 */
125
126extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
127extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];
128extern u8 *__smp_locks[], *__smp_locks_end[];
129
130extern u8 __smp_alt_begin[], __smp_alt_end[];
131
92/* Replace instructions with better alternatives for this CPU type. 132/* Replace instructions with better alternatives for this CPU type.
93 This runs before SMP is initialized to avoid SMP problems with 133 This runs before SMP is initialized to avoid SMP problems with
94 self modifying code. This implies that assymetric systems where 134 self modifying code. This implies that assymetric systems where
@@ -99,6 +139,7 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
99{ 139{
100 unsigned char **noptable = find_nop_table(); 140 unsigned char **noptable = find_nop_table();
101 struct alt_instr *a; 141 struct alt_instr *a;
142 u8 *instr;
102 int diff, i, k; 143 int diff, i, k;
103 144
104 DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end); 145 DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
@@ -106,7 +147,16 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
106 BUG_ON(a->replacementlen > a->instrlen); 147 BUG_ON(a->replacementlen > a->instrlen);
107 if (!boot_cpu_has(a->cpuid)) 148 if (!boot_cpu_has(a->cpuid))
108 continue; 149 continue;
109 memcpy(a->instr, a->replacement, a->replacementlen); 150 instr = a->instr;
151#ifdef CONFIG_X86_64
152 /* vsyscall code is not mapped yet. resolve it manually. */
153 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
154 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
155 DPRINTK("%s: vsyscall fixup: %p => %p\n",
156 __FUNCTION__, a->instr, instr);
157 }
158#endif
159 memcpy(instr, a->replacement, a->replacementlen);
110 diff = a->instrlen - a->replacementlen; 160 diff = a->instrlen - a->replacementlen;
111 /* Pad the rest with nops */ 161 /* Pad the rest with nops */
112 for (i = a->replacementlen; diff > 0; diff -= k, i += k) { 162 for (i = a->replacementlen; diff > 0; diff -= k, i += k) {
@@ -186,14 +236,6 @@ struct smp_alt_module {
186static LIST_HEAD(smp_alt_modules); 236static LIST_HEAD(smp_alt_modules);
187static DEFINE_SPINLOCK(smp_alt); 237static DEFINE_SPINLOCK(smp_alt);
188 238
189static int smp_alt_once = 0;
190static int __init bootonly(char *str)
191{
192 smp_alt_once = 1;
193 return 1;
194}
195__setup("smp-alt-boot", bootonly);
196
197void alternatives_smp_module_add(struct module *mod, char *name, 239void alternatives_smp_module_add(struct module *mod, char *name,
198 void *locks, void *locks_end, 240 void *locks, void *locks_end,
199 void *text, void *text_end) 241 void *text, void *text_end)
@@ -201,6 +243,9 @@ void alternatives_smp_module_add(struct module *mod, char *name,
201 struct smp_alt_module *smp; 243 struct smp_alt_module *smp;
202 unsigned long flags; 244 unsigned long flags;
203 245
246 if (no_replacement)
247 return;
248
204 if (smp_alt_once) { 249 if (smp_alt_once) {
205 if (boot_cpu_has(X86_FEATURE_UP)) 250 if (boot_cpu_has(X86_FEATURE_UP))
206 alternatives_smp_unlock(locks, locks_end, 251 alternatives_smp_unlock(locks, locks_end,
@@ -235,7 +280,7 @@ void alternatives_smp_module_del(struct module *mod)
235 struct smp_alt_module *item; 280 struct smp_alt_module *item;
236 unsigned long flags; 281 unsigned long flags;
237 282
238 if (smp_alt_once) 283 if (no_replacement || smp_alt_once)
239 return; 284 return;
240 285
241 spin_lock_irqsave(&smp_alt, flags); 286 spin_lock_irqsave(&smp_alt, flags);
@@ -256,7 +301,7 @@ void alternatives_smp_switch(int smp)
256 struct smp_alt_module *mod; 301 struct smp_alt_module *mod;
257 unsigned long flags; 302 unsigned long flags;
258 303
259 if (smp_alt_once) 304 if (no_replacement || smp_alt_once)
260 return; 305 return;
261 BUG_ON(!smp && (num_online_cpus() > 1)); 306 BUG_ON(!smp && (num_online_cpus() > 1));
262 307
@@ -285,6 +330,13 @@ void alternatives_smp_switch(int smp)
285 330
286void __init alternative_instructions(void) 331void __init alternative_instructions(void)
287{ 332{
333 if (no_replacement) {
334 printk(KERN_INFO "(SMP-)alternatives turned off\n");
335 free_init_pages("SMP alternatives",
336 (unsigned long)__smp_alt_begin,
337 (unsigned long)__smp_alt_end);
338 return;
339 }
288 apply_alternatives(__alt_instructions, __alt_instructions_end); 340 apply_alternatives(__alt_instructions, __alt_instructions_end);
289 341
290 /* switch to patch-once-at-boottime-only mode and free the 342 /* switch to patch-once-at-boottime-only mode and free the
diff --git a/arch/x86_64/kernel/Makefile b/arch/x86_64/kernel/Makefile
index 059c88313f4e..381bc6ad743e 100644
--- a/arch/x86_64/kernel/Makefile
+++ b/arch/x86_64/kernel/Makefile
@@ -8,7 +8,7 @@ obj-y := process.o signal.o entry.o traps.o irq.o \
8 ptrace.o time.o ioport.o ldt.o setup.o i8259.o sys_x86_64.o \ 8 ptrace.o time.o ioport.o ldt.o setup.o i8259.o sys_x86_64.o \
9 x8664_ksyms.o i387.o syscall.o vsyscall.o \ 9 x8664_ksyms.o i387.o syscall.o vsyscall.o \
10 setup64.o bootflag.o e820.o reboot.o quirks.o i8237.o \ 10 setup64.o bootflag.o e820.o reboot.o quirks.o i8237.o \
11 pci-dma.o pci-nommu.o 11 pci-dma.o pci-nommu.o alternative.o
12 12
13obj-$(CONFIG_X86_MCE) += mce.o 13obj-$(CONFIG_X86_MCE) += mce.o
14obj-$(CONFIG_X86_MCE_INTEL) += mce_intel.o 14obj-$(CONFIG_X86_MCE_INTEL) += mce_intel.o
@@ -49,3 +49,5 @@ intel_cacheinfo-y += ../../i386/kernel/cpu/intel_cacheinfo.o
49quirks-y += ../../i386/kernel/quirks.o 49quirks-y += ../../i386/kernel/quirks.o
50i8237-y += ../../i386/kernel/i8237.o 50i8237-y += ../../i386/kernel/i8237.o
51msr-$(subst m,y,$(CONFIG_X86_MSR)) += ../../i386/kernel/msr.o 51msr-$(subst m,y,$(CONFIG_X86_MSR)) += ../../i386/kernel/msr.o
52alternative-y += ../../i386/kernel/alternative.o
53
diff --git a/arch/x86_64/kernel/module.c b/arch/x86_64/kernel/module.c
index bac195c74bcc..9d0958ff547f 100644
--- a/arch/x86_64/kernel/module.c
+++ b/arch/x86_64/kernel/module.c
@@ -145,26 +145,38 @@ int apply_relocate(Elf_Shdr *sechdrs,
145 return -ENOSYS; 145 return -ENOSYS;
146} 146}
147 147
148extern void apply_alternatives(void *start, void *end);
149
150int module_finalize(const Elf_Ehdr *hdr, 148int module_finalize(const Elf_Ehdr *hdr,
151 const Elf_Shdr *sechdrs, 149 const Elf_Shdr *sechdrs,
152 struct module *me) 150 struct module *me)
153{ 151{
154 const Elf_Shdr *s; 152 const Elf_Shdr *s, *text = NULL, *alt = NULL, *locks = NULL;
155 char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 153 char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
156 154
157 /* look for .altinstructions to patch */ 155 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
158 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 156 if (!strcmp(".text", secstrings + s->sh_name))
159 void *seg; 157 text = s;
160 if (strcmp(".altinstructions", secstrings + s->sh_name)) 158 if (!strcmp(".altinstructions", secstrings + s->sh_name))
161 continue; 159 alt = s;
162 seg = (void *)s->sh_addr; 160 if (!strcmp(".smp_locks", secstrings + s->sh_name))
163 apply_alternatives(seg, seg + s->sh_size); 161 locks= s;
164 } 162 }
163
164 if (alt) {
165 /* patch .altinstructions */
166 void *aseg = (void *)alt->sh_addr;
167 apply_alternatives(aseg, aseg + alt->sh_size);
168 }
169 if (locks && text) {
170 void *lseg = (void *)locks->sh_addr;
171 void *tseg = (void *)text->sh_addr;
172 alternatives_smp_module_add(me, me->name,
173 lseg, lseg + locks->sh_size,
174 tseg, tseg + text->sh_size);
175 }
165 return 0; 176 return 0;
166} 177}
167 178
168void module_arch_cleanup(struct module *mod) 179void module_arch_cleanup(struct module *mod)
169{ 180{
181 alternatives_smp_module_del(mod);
170} 182}
diff --git a/arch/x86_64/kernel/setup.c b/arch/x86_64/kernel/setup.c
index 4b7e02216970..64640c8f5eed 100644
--- a/arch/x86_64/kernel/setup.c
+++ b/arch/x86_64/kernel/setup.c
@@ -473,80 +473,6 @@ contig_initmem_init(unsigned long start_pfn, unsigned long end_pfn)
473} 473}
474#endif 474#endif
475 475
476/* Use inline assembly to define this because the nops are defined
477 as inline assembly strings in the include files and we cannot
478 get them easily into strings. */
479asm("\t.data\nk8nops: "
480 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
481 K8_NOP7 K8_NOP8);
482
483extern unsigned char k8nops[];
484static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
485 NULL,
486 k8nops,
487 k8nops + 1,
488 k8nops + 1 + 2,
489 k8nops + 1 + 2 + 3,
490 k8nops + 1 + 2 + 3 + 4,
491 k8nops + 1 + 2 + 3 + 4 + 5,
492 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
493 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
494};
495
496extern char __vsyscall_0;
497
498/* Replace instructions with better alternatives for this CPU type.
499
500 This runs before SMP is initialized to avoid SMP problems with
501 self modifying code. This implies that assymetric systems where
502 APs have less capabilities than the boot processor are not handled.
503 In this case boot with "noreplacement". */
504void apply_alternatives(void *start, void *end)
505{
506 struct alt_instr *a;
507 int diff, i, k;
508 for (a = start; (void *)a < end; a++) {
509 u8 *instr;
510
511 if (!boot_cpu_has(a->cpuid))
512 continue;
513
514 BUG_ON(a->replacementlen > a->instrlen);
515 instr = a->instr;
516 /* vsyscall code is not mapped yet. resolve it manually. */
517 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END)
518 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
519 __inline_memcpy(instr, a->replacement, a->replacementlen);
520 diff = a->instrlen - a->replacementlen;
521
522 /* Pad the rest with nops */
523 for (i = a->replacementlen; diff > 0; diff -= k, i += k) {
524 k = diff;
525 if (k > ASM_NOP_MAX)
526 k = ASM_NOP_MAX;
527 __inline_memcpy(instr + i, k8_nops[k], k);
528 }
529 }
530}
531
532static int no_replacement __initdata = 0;
533
534void __init alternative_instructions(void)
535{
536 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
537 if (no_replacement)
538 return;
539 apply_alternatives(__alt_instructions, __alt_instructions_end);
540}
541
542static int __init noreplacement_setup(char *s)
543{
544 no_replacement = 1;
545 return 1;
546}
547
548__setup("noreplacement", noreplacement_setup);
549
550#if defined(CONFIG_EDD) || defined(CONFIG_EDD_MODULE) 476#if defined(CONFIG_EDD) || defined(CONFIG_EDD_MODULE)
551struct edd edd; 477struct edd edd;
552#ifdef CONFIG_EDD_MODULE 478#ifdef CONFIG_EDD_MODULE
@@ -1303,7 +1229,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
1303 /* Other (Linux-defined) */ 1229 /* Other (Linux-defined) */
1304 "cxmmx", NULL, "cyrix_arr", "centaur_mcr", NULL, 1230 "cxmmx", NULL, "cyrix_arr", "centaur_mcr", NULL,
1305 "constant_tsc", NULL, NULL, 1231 "constant_tsc", NULL, NULL,
1306 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 1232 "up", NULL, NULL, NULL, NULL, NULL, NULL, NULL,
1307 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 1233 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
1308 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 1234 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
1309 1235
diff --git a/arch/x86_64/kernel/smpboot.c b/arch/x86_64/kernel/smpboot.c
index 71a7222cf9ce..06535e7687ce 100644
--- a/arch/x86_64/kernel/smpboot.c
+++ b/arch/x86_64/kernel/smpboot.c
@@ -797,6 +797,8 @@ static int __cpuinit do_boot_cpu(int cpu, int apicid)
797 } 797 }
798 798
799 799
800 alternatives_smp_switch(1);
801
800 c_idle.idle = get_idle_for_cpu(cpu); 802 c_idle.idle = get_idle_for_cpu(cpu);
801 803
802 if (c_idle.idle) { 804 if (c_idle.idle) {
@@ -1259,6 +1261,8 @@ void __cpu_die(unsigned int cpu)
1259 /* They ack this in play_dead by setting CPU_DEAD */ 1261 /* They ack this in play_dead by setting CPU_DEAD */
1260 if (per_cpu(cpu_state, cpu) == CPU_DEAD) { 1262 if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
1261 printk ("CPU %d is now offline\n", cpu); 1263 printk ("CPU %d is now offline\n", cpu);
1264 if (1 == num_online_cpus())
1265 alternatives_smp_switch(0);
1262 return; 1266 return;
1263 } 1267 }
1264 msleep(100); 1268 msleep(100);
diff --git a/arch/x86_64/kernel/vmlinux.lds.S b/arch/x86_64/kernel/vmlinux.lds.S
index b81f473c4a19..5968c2415da9 100644
--- a/arch/x86_64/kernel/vmlinux.lds.S
+++ b/arch/x86_64/kernel/vmlinux.lds.S
@@ -131,6 +131,26 @@ SECTIONS
131 *(.data.page_aligned) 131 *(.data.page_aligned)
132 } 132 }
133 133
134 /* might get freed after init */
135 . = ALIGN(4096);
136 __smp_alt_begin = .;
137 __smp_alt_instructions = .;
138 .smp_altinstructions : AT(ADDR(.smp_altinstructions) - LOAD_OFFSET) {
139 *(.smp_altinstructions)
140 }
141 __smp_alt_instructions_end = .;
142 . = ALIGN(8);
143 __smp_locks = .;
144 .smp_locks : AT(ADDR(.smp_locks) - LOAD_OFFSET) {
145 *(.smp_locks)
146 }
147 __smp_locks_end = .;
148 .smp_altinstr_replacement : AT(ADDR(.smp_altinstr_replacement) - LOAD_OFFSET) {
149 *(.smp_altinstr_replacement)
150 }
151 . = ALIGN(4096);
152 __smp_alt_end = .;
153
134 . = ALIGN(4096); /* Init code and data */ 154 . = ALIGN(4096); /* Init code and data */
135 __init_begin = .; 155 __init_begin = .;
136 .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) { 156 .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) {
diff --git a/arch/x86_64/mm/init.c b/arch/x86_64/mm/init.c
index 4ba34e95d835..a70a8e3e312d 100644
--- a/arch/x86_64/mm/init.c
+++ b/arch/x86_64/mm/init.c
@@ -644,20 +644,29 @@ void __init mem_init(void)
644#endif 644#endif
645} 645}
646 646
647void free_initmem(void) 647void free_init_pages(char *what, unsigned long begin, unsigned long end)
648{ 648{
649 unsigned long addr; 649 unsigned long addr;
650 650
651 addr = (unsigned long)(&__init_begin); 651 if (begin >= end)
652 for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) { 652 return;
653
654 printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
655 for (addr = begin; addr < end; addr += PAGE_SIZE) {
653 ClearPageReserved(virt_to_page(addr)); 656 ClearPageReserved(virt_to_page(addr));
654 init_page_count(virt_to_page(addr)); 657 init_page_count(virt_to_page(addr));
655 memset((void *)(addr & ~(PAGE_SIZE-1)), 0xcc, PAGE_SIZE); 658 memset((void *)(addr & ~(PAGE_SIZE-1)), 0xcc, PAGE_SIZE);
656 free_page(addr); 659 free_page(addr);
657 totalram_pages++; 660 totalram_pages++;
658 } 661 }
662}
663
664void free_initmem(void)
665{
659 memset(__initdata_begin, 0xba, __initdata_end - __initdata_begin); 666 memset(__initdata_begin, 0xba, __initdata_end - __initdata_begin);
660 printk ("Freeing unused kernel memory: %luk freed\n", (__init_end - __init_begin) >> 10); 667 free_init_pages("unused kernel memory",
668 (unsigned long)(&__init_begin),
669 (unsigned long)(&__init_end));
661} 670}
662 671
663#ifdef CONFIG_DEBUG_RODATA 672#ifdef CONFIG_DEBUG_RODATA
@@ -686,15 +695,7 @@ void mark_rodata_ro(void)
686#ifdef CONFIG_BLK_DEV_INITRD 695#ifdef CONFIG_BLK_DEV_INITRD
687void free_initrd_mem(unsigned long start, unsigned long end) 696void free_initrd_mem(unsigned long start, unsigned long end)
688{ 697{
689 if (start >= end) 698 free_init_pages("initrd memory", start, end);
690 return;
691 printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
692 for (; start < end; start += PAGE_SIZE) {
693 ClearPageReserved(virt_to_page(start));
694 init_page_count(virt_to_page(start));
695 free_page(start);
696 totalram_pages++;
697 }
698} 699}
699#endif 700#endif
700 701
diff --git a/include/asm-i386/alternative.h b/include/asm-i386/alternative.h
index d79e9ee10fd7..c61bd1a17f37 100644
--- a/include/asm-i386/alternative.h
+++ b/include/asm-i386/alternative.h
@@ -5,6 +5,8 @@
5 5
6#include <asm/types.h> 6#include <asm/types.h>
7 7
8#include <linux/types.h>
9
8struct alt_instr { 10struct alt_instr {
9 u8 *instr; /* original instruction */ 11 u8 *instr; /* original instruction */
10 u8 *replacement; 12 u8 *replacement;
diff --git a/include/asm-x86_64/alternative.h b/include/asm-x86_64/alternative.h
new file mode 100644
index 000000000000..387c8f66af7d
--- /dev/null
+++ b/include/asm-x86_64/alternative.h
@@ -0,0 +1,146 @@
1#ifndef _X86_64_ALTERNATIVE_H
2#define _X86_64_ALTERNATIVE_H
3
4#ifdef __KERNEL__
5
6#include <linux/types.h>
7
8struct alt_instr {
9 u8 *instr; /* original instruction */
10 u8 *replacement;
11 u8 cpuid; /* cpuid bit set for replacement */
12 u8 instrlen; /* length of original instruction */
13 u8 replacementlen; /* length of new instruction, <= instrlen */
14 u8 pad[5];
15};
16
17extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
18
19struct module;
20extern void alternatives_smp_module_add(struct module *mod, char *name,
21 void *locks, void *locks_end,
22 void *text, void *text_end);
23extern void alternatives_smp_module_del(struct module *mod);
24extern void alternatives_smp_switch(int smp);
25
26#endif
27
28/*
29 * Alternative instructions for different CPU types or capabilities.
30 *
31 * This allows to use optimized instructions even on generic binary
32 * kernels.
33 *
34 * length of oldinstr must be longer or equal the length of newinstr
35 * It can be padded with nops as needed.
36 *
37 * For non barrier like inlines please define new variants
38 * without volatile and memory clobber.
39 */
40#define alternative(oldinstr, newinstr, feature) \
41 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
42 ".section .altinstructions,\"a\"\n" \
43 " .align 8\n" \
44 " .quad 661b\n" /* label */ \
45 " .quad 663f\n" /* new instruction */ \
46 " .byte %c0\n" /* feature bit */ \
47 " .byte 662b-661b\n" /* sourcelen */ \
48 " .byte 664f-663f\n" /* replacementlen */ \
49 ".previous\n" \
50 ".section .altinstr_replacement,\"ax\"\n" \
51 "663:\n\t" newinstr "\n664:\n" /* replacement */ \
52 ".previous" :: "i" (feature) : "memory")
53
54/*
55 * Alternative inline assembly with input.
56 *
57 * Pecularities:
58 * No memory clobber here.
59 * Argument numbers start with 1.
60 * Best is to use constraints that are fixed size (like (%1) ... "r")
61 * If you use variable sized constraints like "m" or "g" in the
62 * replacement make sure to pad to the worst case length.
63 */
64#define alternative_input(oldinstr, newinstr, feature, input...) \
65 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
66 ".section .altinstructions,\"a\"\n" \
67 " .align 8\n" \
68 " .quad 661b\n" /* label */ \
69 " .quad 663f\n" /* new instruction */ \
70 " .byte %c0\n" /* feature bit */ \
71 " .byte 662b-661b\n" /* sourcelen */ \
72 " .byte 664f-663f\n" /* replacementlen */ \
73 ".previous\n" \
74 ".section .altinstr_replacement,\"ax\"\n" \
75 "663:\n\t" newinstr "\n664:\n" /* replacement */ \
76 ".previous" :: "i" (feature), ##input)
77
78/* Like alternative_input, but with a single output argument */
79#define alternative_io(oldinstr, newinstr, feature, output, input...) \
80 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
81 ".section .altinstructions,\"a\"\n" \
82 " .align 8\n" \
83 " .quad 661b\n" /* label */ \
84 " .quad 663f\n" /* new instruction */ \
85 " .byte %c[feat]\n" /* feature bit */ \
86 " .byte 662b-661b\n" /* sourcelen */ \
87 " .byte 664f-663f\n" /* replacementlen */ \
88 ".previous\n" \
89 ".section .altinstr_replacement,\"ax\"\n" \
90 "663:\n\t" newinstr "\n664:\n" /* replacement */ \
91 ".previous" : output : [feat] "i" (feature), ##input)
92
93/*
94 * Alternative inline assembly for SMP.
95 *
96 * alternative_smp() takes two versions (SMP first, UP second) and is
97 * for more complex stuff such as spinlocks.
98 *
99 * The LOCK_PREFIX macro defined here replaces the LOCK and
100 * LOCK_PREFIX macros used everywhere in the source tree.
101 *
102 * SMP alternatives use the same data structures as the other
103 * alternatives and the X86_FEATURE_UP flag to indicate the case of a
104 * UP system running a SMP kernel. The existing apply_alternatives()
105 * works fine for patching a SMP kernel for UP.
106 *
107 * The SMP alternative tables can be kept after boot and contain both
108 * UP and SMP versions of the instructions to allow switching back to
109 * SMP at runtime, when hotplugging in a new CPU, which is especially
110 * useful in virtualized environments.
111 *
112 * The very common lock prefix is handled as special case in a
113 * separate table which is a pure address list without replacement ptr
114 * and size information. That keeps the table sizes small.
115 */
116
117#ifdef CONFIG_SMP
118#define alternative_smp(smpinstr, upinstr, args...) \
119 asm volatile ("661:\n\t" smpinstr "\n662:\n" \
120 ".section .smp_altinstructions,\"a\"\n" \
121 " .align 8\n" \
122 " .quad 661b\n" /* label */ \
123 " .quad 663f\n" /* new instruction */ \
124 " .byte 0x66\n" /* X86_FEATURE_UP */ \
125 " .byte 662b-661b\n" /* sourcelen */ \
126 " .byte 664f-663f\n" /* replacementlen */ \
127 ".previous\n" \
128 ".section .smp_altinstr_replacement,\"awx\"\n" \
129 "663:\n\t" upinstr "\n" /* replacement */ \
130 "664:\n\t.fill 662b-661b,1,0x42\n" /* space for original */ \
131 ".previous" : args)
132
133#define LOCK_PREFIX \
134 ".section .smp_locks,\"a\"\n" \
135 " .align 8\n" \
136 " .quad 661f\n" /* address */ \
137 ".previous\n" \
138 "661:\n\tlock; "
139
140#else /* ! CONFIG_SMP */
141#define alternative_smp(smpinstr, upinstr, args...) \
142 asm volatile (upinstr : args)
143#define LOCK_PREFIX ""
144#endif
145
146#endif /* _X86_64_ALTERNATIVE_H */
diff --git a/include/asm-x86_64/atomic.h b/include/asm-x86_64/atomic.h
index bd3fa67ed835..007e88d6d43f 100644
--- a/include/asm-x86_64/atomic.h
+++ b/include/asm-x86_64/atomic.h
@@ -1,7 +1,7 @@
1#ifndef __ARCH_X86_64_ATOMIC__ 1#ifndef __ARCH_X86_64_ATOMIC__
2#define __ARCH_X86_64_ATOMIC__ 2#define __ARCH_X86_64_ATOMIC__
3 3
4#include <asm/types.h> 4#include <asm/alternative.h>
5 5
6/* atomic_t should be 32 bit signed type */ 6/* atomic_t should be 32 bit signed type */
7 7
@@ -52,7 +52,7 @@ typedef struct { volatile int counter; } atomic_t;
52static __inline__ void atomic_add(int i, atomic_t *v) 52static __inline__ void atomic_add(int i, atomic_t *v)
53{ 53{
54 __asm__ __volatile__( 54 __asm__ __volatile__(
55 LOCK "addl %1,%0" 55 LOCK_PREFIX "addl %1,%0"
56 :"=m" (v->counter) 56 :"=m" (v->counter)
57 :"ir" (i), "m" (v->counter)); 57 :"ir" (i), "m" (v->counter));
58} 58}
@@ -67,7 +67,7 @@ static __inline__ void atomic_add(int i, atomic_t *v)
67static __inline__ void atomic_sub(int i, atomic_t *v) 67static __inline__ void atomic_sub(int i, atomic_t *v)
68{ 68{
69 __asm__ __volatile__( 69 __asm__ __volatile__(
70 LOCK "subl %1,%0" 70 LOCK_PREFIX "subl %1,%0"
71 :"=m" (v->counter) 71 :"=m" (v->counter)
72 :"ir" (i), "m" (v->counter)); 72 :"ir" (i), "m" (v->counter));
73} 73}
@@ -86,7 +86,7 @@ static __inline__ int atomic_sub_and_test(int i, atomic_t *v)
86 unsigned char c; 86 unsigned char c;
87 87
88 __asm__ __volatile__( 88 __asm__ __volatile__(
89 LOCK "subl %2,%0; sete %1" 89 LOCK_PREFIX "subl %2,%0; sete %1"
90 :"=m" (v->counter), "=qm" (c) 90 :"=m" (v->counter), "=qm" (c)
91 :"ir" (i), "m" (v->counter) : "memory"); 91 :"ir" (i), "m" (v->counter) : "memory");
92 return c; 92 return c;
@@ -101,7 +101,7 @@ static __inline__ int atomic_sub_and_test(int i, atomic_t *v)
101static __inline__ void atomic_inc(atomic_t *v) 101static __inline__ void atomic_inc(atomic_t *v)
102{ 102{
103 __asm__ __volatile__( 103 __asm__ __volatile__(
104 LOCK "incl %0" 104 LOCK_PREFIX "incl %0"
105 :"=m" (v->counter) 105 :"=m" (v->counter)
106 :"m" (v->counter)); 106 :"m" (v->counter));
107} 107}
@@ -115,7 +115,7 @@ static __inline__ void atomic_inc(atomic_t *v)
115static __inline__ void atomic_dec(atomic_t *v) 115static __inline__ void atomic_dec(atomic_t *v)
116{ 116{
117 __asm__ __volatile__( 117 __asm__ __volatile__(
118 LOCK "decl %0" 118 LOCK_PREFIX "decl %0"
119 :"=m" (v->counter) 119 :"=m" (v->counter)
120 :"m" (v->counter)); 120 :"m" (v->counter));
121} 121}
@@ -133,7 +133,7 @@ static __inline__ int atomic_dec_and_test(atomic_t *v)
133 unsigned char c; 133 unsigned char c;
134 134
135 __asm__ __volatile__( 135 __asm__ __volatile__(
136 LOCK "decl %0; sete %1" 136 LOCK_PREFIX "decl %0; sete %1"
137 :"=m" (v->counter), "=qm" (c) 137 :"=m" (v->counter), "=qm" (c)
138 :"m" (v->counter) : "memory"); 138 :"m" (v->counter) : "memory");
139 return c != 0; 139 return c != 0;
@@ -152,7 +152,7 @@ static __inline__ int atomic_inc_and_test(atomic_t *v)
152 unsigned char c; 152 unsigned char c;
153 153
154 __asm__ __volatile__( 154 __asm__ __volatile__(
155 LOCK "incl %0; sete %1" 155 LOCK_PREFIX "incl %0; sete %1"
156 :"=m" (v->counter), "=qm" (c) 156 :"=m" (v->counter), "=qm" (c)
157 :"m" (v->counter) : "memory"); 157 :"m" (v->counter) : "memory");
158 return c != 0; 158 return c != 0;
@@ -172,7 +172,7 @@ static __inline__ int atomic_add_negative(int i, atomic_t *v)
172 unsigned char c; 172 unsigned char c;
173 173
174 __asm__ __volatile__( 174 __asm__ __volatile__(
175 LOCK "addl %2,%0; sets %1" 175 LOCK_PREFIX "addl %2,%0; sets %1"
176 :"=m" (v->counter), "=qm" (c) 176 :"=m" (v->counter), "=qm" (c)
177 :"ir" (i), "m" (v->counter) : "memory"); 177 :"ir" (i), "m" (v->counter) : "memory");
178 return c; 178 return c;
@@ -189,7 +189,7 @@ static __inline__ int atomic_add_return(int i, atomic_t *v)
189{ 189{
190 int __i = i; 190 int __i = i;
191 __asm__ __volatile__( 191 __asm__ __volatile__(
192 LOCK "xaddl %0, %1;" 192 LOCK_PREFIX "xaddl %0, %1;"
193 :"=r"(i) 193 :"=r"(i)
194 :"m"(v->counter), "0"(i)); 194 :"m"(v->counter), "0"(i));
195 return i + __i; 195 return i + __i;
@@ -237,7 +237,7 @@ typedef struct { volatile long counter; } atomic64_t;
237static __inline__ void atomic64_add(long i, atomic64_t *v) 237static __inline__ void atomic64_add(long i, atomic64_t *v)
238{ 238{
239 __asm__ __volatile__( 239 __asm__ __volatile__(
240 LOCK "addq %1,%0" 240 LOCK_PREFIX "addq %1,%0"
241 :"=m" (v->counter) 241 :"=m" (v->counter)
242 :"ir" (i), "m" (v->counter)); 242 :"ir" (i), "m" (v->counter));
243} 243}
@@ -252,7 +252,7 @@ static __inline__ void atomic64_add(long i, atomic64_t *v)
252static __inline__ void atomic64_sub(long i, atomic64_t *v) 252static __inline__ void atomic64_sub(long i, atomic64_t *v)
253{ 253{
254 __asm__ __volatile__( 254 __asm__ __volatile__(
255 LOCK "subq %1,%0" 255 LOCK_PREFIX "subq %1,%0"
256 :"=m" (v->counter) 256 :"=m" (v->counter)
257 :"ir" (i), "m" (v->counter)); 257 :"ir" (i), "m" (v->counter));
258} 258}
@@ -271,7 +271,7 @@ static __inline__ int atomic64_sub_and_test(long i, atomic64_t *v)
271 unsigned char c; 271 unsigned char c;
272 272
273 __asm__ __volatile__( 273 __asm__ __volatile__(
274 LOCK "subq %2,%0; sete %1" 274 LOCK_PREFIX "subq %2,%0; sete %1"
275 :"=m" (v->counter), "=qm" (c) 275 :"=m" (v->counter), "=qm" (c)
276 :"ir" (i), "m" (v->counter) : "memory"); 276 :"ir" (i), "m" (v->counter) : "memory");
277 return c; 277 return c;
@@ -286,7 +286,7 @@ static __inline__ int atomic64_sub_and_test(long i, atomic64_t *v)
286static __inline__ void atomic64_inc(atomic64_t *v) 286static __inline__ void atomic64_inc(atomic64_t *v)
287{ 287{
288 __asm__ __volatile__( 288 __asm__ __volatile__(
289 LOCK "incq %0" 289 LOCK_PREFIX "incq %0"
290 :"=m" (v->counter) 290 :"=m" (v->counter)
291 :"m" (v->counter)); 291 :"m" (v->counter));
292} 292}
@@ -300,7 +300,7 @@ static __inline__ void atomic64_inc(atomic64_t *v)
300static __inline__ void atomic64_dec(atomic64_t *v) 300static __inline__ void atomic64_dec(atomic64_t *v)
301{ 301{
302 __asm__ __volatile__( 302 __asm__ __volatile__(
303 LOCK "decq %0" 303 LOCK_PREFIX "decq %0"
304 :"=m" (v->counter) 304 :"=m" (v->counter)
305 :"m" (v->counter)); 305 :"m" (v->counter));
306} 306}
@@ -318,7 +318,7 @@ static __inline__ int atomic64_dec_and_test(atomic64_t *v)
318 unsigned char c; 318 unsigned char c;
319 319
320 __asm__ __volatile__( 320 __asm__ __volatile__(
321 LOCK "decq %0; sete %1" 321 LOCK_PREFIX "decq %0; sete %1"
322 :"=m" (v->counter), "=qm" (c) 322 :"=m" (v->counter), "=qm" (c)
323 :"m" (v->counter) : "memory"); 323 :"m" (v->counter) : "memory");
324 return c != 0; 324 return c != 0;
@@ -337,7 +337,7 @@ static __inline__ int atomic64_inc_and_test(atomic64_t *v)
337 unsigned char c; 337 unsigned char c;
338 338
339 __asm__ __volatile__( 339 __asm__ __volatile__(
340 LOCK "incq %0; sete %1" 340 LOCK_PREFIX "incq %0; sete %1"
341 :"=m" (v->counter), "=qm" (c) 341 :"=m" (v->counter), "=qm" (c)
342 :"m" (v->counter) : "memory"); 342 :"m" (v->counter) : "memory");
343 return c != 0; 343 return c != 0;
@@ -357,7 +357,7 @@ static __inline__ int atomic64_add_negative(long i, atomic64_t *v)
357 unsigned char c; 357 unsigned char c;
358 358
359 __asm__ __volatile__( 359 __asm__ __volatile__(
360 LOCK "addq %2,%0; sets %1" 360 LOCK_PREFIX "addq %2,%0; sets %1"
361 :"=m" (v->counter), "=qm" (c) 361 :"=m" (v->counter), "=qm" (c)
362 :"ir" (i), "m" (v->counter) : "memory"); 362 :"ir" (i), "m" (v->counter) : "memory");
363 return c; 363 return c;
@@ -374,7 +374,7 @@ static __inline__ long atomic64_add_return(long i, atomic64_t *v)
374{ 374{
375 long __i = i; 375 long __i = i;
376 __asm__ __volatile__( 376 __asm__ __volatile__(
377 LOCK "xaddq %0, %1;" 377 LOCK_PREFIX "xaddq %0, %1;"
378 :"=r"(i) 378 :"=r"(i)
379 :"m"(v->counter), "0"(i)); 379 :"m"(v->counter), "0"(i));
380 return i + __i; 380 return i + __i;
@@ -418,11 +418,11 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v)
418 418
419/* These are x86-specific, used by some header files */ 419/* These are x86-specific, used by some header files */
420#define atomic_clear_mask(mask, addr) \ 420#define atomic_clear_mask(mask, addr) \
421__asm__ __volatile__(LOCK "andl %0,%1" \ 421__asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \
422: : "r" (~(mask)),"m" (*addr) : "memory") 422: : "r" (~(mask)),"m" (*addr) : "memory")
423 423
424#define atomic_set_mask(mask, addr) \ 424#define atomic_set_mask(mask, addr) \
425__asm__ __volatile__(LOCK "orl %0,%1" \ 425__asm__ __volatile__(LOCK_PREFIX "orl %0,%1" \
426: : "r" ((unsigned)mask),"m" (*(addr)) : "memory") 426: : "r" ((unsigned)mask),"m" (*(addr)) : "memory")
427 427
428/* Atomic operations are already serializing on x86 */ 428/* Atomic operations are already serializing on x86 */
diff --git a/include/asm-x86_64/bitops.h b/include/asm-x86_64/bitops.h
index e9bf933d25d0..f7ba57b1cc08 100644
--- a/include/asm-x86_64/bitops.h
+++ b/include/asm-x86_64/bitops.h
@@ -5,12 +5,7 @@
5 * Copyright 1992, Linus Torvalds. 5 * Copyright 1992, Linus Torvalds.
6 */ 6 */
7 7
8 8#include <asm/alternative.h>
9#ifdef CONFIG_SMP
10#define LOCK_PREFIX "lock ; "
11#else
12#define LOCK_PREFIX ""
13#endif
14 9
15#define ADDR (*(volatile long *) addr) 10#define ADDR (*(volatile long *) addr)
16 11
diff --git a/include/asm-x86_64/cpufeature.h b/include/asm-x86_64/cpufeature.h
index 662964b74e34..afc44e557400 100644
--- a/include/asm-x86_64/cpufeature.h
+++ b/include/asm-x86_64/cpufeature.h
@@ -65,6 +65,8 @@
65#define X86_FEATURE_CONSTANT_TSC (3*32+5) /* TSC runs at constant rate */ 65#define X86_FEATURE_CONSTANT_TSC (3*32+5) /* TSC runs at constant rate */
66#define X86_FEATURE_SYNC_RDTSC (3*32+6) /* RDTSC syncs CPU core */ 66#define X86_FEATURE_SYNC_RDTSC (3*32+6) /* RDTSC syncs CPU core */
67#define X86_FEATURE_FXSAVE_LEAK (3*32+7) /* FIP/FOP/FDP leaks through FXSAVE */ 67#define X86_FEATURE_FXSAVE_LEAK (3*32+7) /* FIP/FOP/FDP leaks through FXSAVE */
68#define X86_FEATURE_UP (3*32+8) /* SMP kernel running on UP */
69
68 70
69/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ 71/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
70#define X86_FEATURE_XMM3 (4*32+ 0) /* Streaming SIMD Extensions-3 */ 72#define X86_FEATURE_XMM3 (4*32+ 0) /* Streaming SIMD Extensions-3 */
diff --git a/include/asm-x86_64/mutex.h b/include/asm-x86_64/mutex.h
index 11fbee2bd6c0..06fab6de2a88 100644
--- a/include/asm-x86_64/mutex.h
+++ b/include/asm-x86_64/mutex.h
@@ -24,7 +24,7 @@ do { \
24 typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \ 24 typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \
25 \ 25 \
26 __asm__ __volatile__( \ 26 __asm__ __volatile__( \
27 LOCK " decl (%%rdi) \n" \ 27 LOCK_PREFIX " decl (%%rdi) \n" \
28 " js 2f \n" \ 28 " js 2f \n" \
29 "1: \n" \ 29 "1: \n" \
30 \ 30 \
@@ -74,7 +74,7 @@ do { \
74 typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \ 74 typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \
75 \ 75 \
76 __asm__ __volatile__( \ 76 __asm__ __volatile__( \
77 LOCK " incl (%%rdi) \n" \ 77 LOCK_PREFIX " incl (%%rdi) \n" \
78 " jle 2f \n" \ 78 " jle 2f \n" \
79 "1: \n" \ 79 "1: \n" \
80 \ 80 \
diff --git a/include/asm-x86_64/rwlock.h b/include/asm-x86_64/rwlock.h
index 9942cc393064..dea0e9459264 100644
--- a/include/asm-x86_64/rwlock.h
+++ b/include/asm-x86_64/rwlock.h
@@ -24,7 +24,7 @@
24#define RW_LOCK_BIAS_STR "0x01000000" 24#define RW_LOCK_BIAS_STR "0x01000000"
25 25
26#define __build_read_lock_ptr(rw, helper) \ 26#define __build_read_lock_ptr(rw, helper) \
27 asm volatile(LOCK "subl $1,(%0)\n\t" \ 27 asm volatile(LOCK_PREFIX "subl $1,(%0)\n\t" \
28 "js 2f\n" \ 28 "js 2f\n" \
29 "1:\n" \ 29 "1:\n" \
30 LOCK_SECTION_START("") \ 30 LOCK_SECTION_START("") \
@@ -34,7 +34,7 @@
34 ::"a" (rw) : "memory") 34 ::"a" (rw) : "memory")
35 35
36#define __build_read_lock_const(rw, helper) \ 36#define __build_read_lock_const(rw, helper) \
37 asm volatile(LOCK "subl $1,%0\n\t" \ 37 asm volatile(LOCK_PREFIX "subl $1,%0\n\t" \
38 "js 2f\n" \ 38 "js 2f\n" \
39 "1:\n" \ 39 "1:\n" \
40 LOCK_SECTION_START("") \ 40 LOCK_SECTION_START("") \
@@ -54,7 +54,7 @@
54 } while (0) 54 } while (0)
55 55
56#define __build_write_lock_ptr(rw, helper) \ 56#define __build_write_lock_ptr(rw, helper) \
57 asm volatile(LOCK "subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \ 57 asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \
58 "jnz 2f\n" \ 58 "jnz 2f\n" \
59 "1:\n" \ 59 "1:\n" \
60 LOCK_SECTION_START("") \ 60 LOCK_SECTION_START("") \
@@ -64,7 +64,7 @@
64 ::"a" (rw) : "memory") 64 ::"a" (rw) : "memory")
65 65
66#define __build_write_lock_const(rw, helper) \ 66#define __build_write_lock_const(rw, helper) \
67 asm volatile(LOCK "subl $" RW_LOCK_BIAS_STR ",%0\n\t" \ 67 asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",%0\n\t" \
68 "jnz 2f\n" \ 68 "jnz 2f\n" \
69 "1:\n" \ 69 "1:\n" \
70 LOCK_SECTION_START("") \ 70 LOCK_SECTION_START("") \
diff --git a/include/asm-x86_64/semaphore.h b/include/asm-x86_64/semaphore.h
index a389aa6fe80f..064df08b9a0f 100644
--- a/include/asm-x86_64/semaphore.h
+++ b/include/asm-x86_64/semaphore.h
@@ -106,7 +106,7 @@ static inline void down(struct semaphore * sem)
106 106
107 __asm__ __volatile__( 107 __asm__ __volatile__(
108 "# atomic down operation\n\t" 108 "# atomic down operation\n\t"
109 LOCK "decl %0\n\t" /* --sem->count */ 109 LOCK_PREFIX "decl %0\n\t" /* --sem->count */
110 "js 2f\n" 110 "js 2f\n"
111 "1:\n" 111 "1:\n"
112 LOCK_SECTION_START("") 112 LOCK_SECTION_START("")
@@ -130,7 +130,7 @@ static inline int down_interruptible(struct semaphore * sem)
130 130
131 __asm__ __volatile__( 131 __asm__ __volatile__(
132 "# atomic interruptible down operation\n\t" 132 "# atomic interruptible down operation\n\t"
133 LOCK "decl %1\n\t" /* --sem->count */ 133 LOCK_PREFIX "decl %1\n\t" /* --sem->count */
134 "js 2f\n\t" 134 "js 2f\n\t"
135 "xorl %0,%0\n" 135 "xorl %0,%0\n"
136 "1:\n" 136 "1:\n"
@@ -154,7 +154,7 @@ static inline int down_trylock(struct semaphore * sem)
154 154
155 __asm__ __volatile__( 155 __asm__ __volatile__(
156 "# atomic interruptible down operation\n\t" 156 "# atomic interruptible down operation\n\t"
157 LOCK "decl %1\n\t" /* --sem->count */ 157 LOCK_PREFIX "decl %1\n\t" /* --sem->count */
158 "js 2f\n\t" 158 "js 2f\n\t"
159 "xorl %0,%0\n" 159 "xorl %0,%0\n"
160 "1:\n" 160 "1:\n"
@@ -178,7 +178,7 @@ static inline void up(struct semaphore * sem)
178{ 178{
179 __asm__ __volatile__( 179 __asm__ __volatile__(
180 "# atomic up operation\n\t" 180 "# atomic up operation\n\t"
181 LOCK "incl %0\n\t" /* ++sem->count */ 181 LOCK_PREFIX "incl %0\n\t" /* ++sem->count */
182 "jle 2f\n" 182 "jle 2f\n"
183 "1:\n" 183 "1:\n"
184 LOCK_SECTION_START("") 184 LOCK_SECTION_START("")
diff --git a/include/asm-x86_64/spinlock.h b/include/asm-x86_64/spinlock.h
index 5d8a5e3589ff..8d3421996f94 100644
--- a/include/asm-x86_64/spinlock.h
+++ b/include/asm-x86_64/spinlock.h
@@ -31,15 +31,19 @@
31 "jmp 1b\n" \ 31 "jmp 1b\n" \
32 LOCK_SECTION_END 32 LOCK_SECTION_END
33 33
34#define __raw_spin_lock_string_up \
35 "\n\tdecl %0"
36
34#define __raw_spin_unlock_string \ 37#define __raw_spin_unlock_string \
35 "movl $1,%0" \ 38 "movl $1,%0" \
36 :"=m" (lock->slock) : : "memory" 39 :"=m" (lock->slock) : : "memory"
37 40
38static inline void __raw_spin_lock(raw_spinlock_t *lock) 41static inline void __raw_spin_lock(raw_spinlock_t *lock)
39{ 42{
40 __asm__ __volatile__( 43 alternative_smp(
41 __raw_spin_lock_string 44 __raw_spin_lock_string,
42 :"=m" (lock->slock) : : "memory"); 45 __raw_spin_lock_string_up,
46 "=m" (lock->slock) : : "memory");
43} 47}
44 48
45#define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock) 49#define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock)
diff --git a/include/asm-x86_64/system.h b/include/asm-x86_64/system.h
index f48e0dad8b3d..68e559f3631c 100644
--- a/include/asm-x86_64/system.h
+++ b/include/asm-x86_64/system.h
@@ -3,15 +3,10 @@
3 3
4#include <linux/kernel.h> 4#include <linux/kernel.h>
5#include <asm/segment.h> 5#include <asm/segment.h>
6#include <asm/alternative.h>
6 7
7#ifdef __KERNEL__ 8#ifdef __KERNEL__
8 9
9#ifdef CONFIG_SMP
10#define LOCK_PREFIX "lock ; "
11#else
12#define LOCK_PREFIX ""
13#endif
14
15#define __STR(x) #x 10#define __STR(x) #x
16#define STR(x) __STR(x) 11#define STR(x) __STR(x)
17 12
@@ -34,7 +29,7 @@
34 "thread_return:\n\t" \ 29 "thread_return:\n\t" \
35 "movq %%gs:%P[pda_pcurrent],%%rsi\n\t" \ 30 "movq %%gs:%P[pda_pcurrent],%%rsi\n\t" \
36 "movq %P[thread_info](%%rsi),%%r8\n\t" \ 31 "movq %P[thread_info](%%rsi),%%r8\n\t" \
37 LOCK "btr %[tif_fork],%P[ti_flags](%%r8)\n\t" \ 32 LOCK_PREFIX "btr %[tif_fork],%P[ti_flags](%%r8)\n\t" \
38 "movq %%rax,%%rdi\n\t" \ 33 "movq %%rax,%%rdi\n\t" \
39 "jc ret_from_fork\n\t" \ 34 "jc ret_from_fork\n\t" \
40 RESTORE_CONTEXT \ 35 RESTORE_CONTEXT \
@@ -69,82 +64,6 @@ extern void load_gs_index(unsigned);
69 ".previous" \ 64 ".previous" \
70 : :"r" (value), "r" (0)) 65 : :"r" (value), "r" (0))
71 66
72#ifdef __KERNEL__
73struct alt_instr {
74 __u8 *instr; /* original instruction */
75 __u8 *replacement;
76 __u8 cpuid; /* cpuid bit set for replacement */
77 __u8 instrlen; /* length of original instruction */
78 __u8 replacementlen; /* length of new instruction, <= instrlen */
79 __u8 pad[5];
80};
81#endif
82
83/*
84 * Alternative instructions for different CPU types or capabilities.
85 *
86 * This allows to use optimized instructions even on generic binary
87 * kernels.
88 *
89 * length of oldinstr must be longer or equal the length of newinstr
90 * It can be padded with nops as needed.
91 *
92 * For non barrier like inlines please define new variants
93 * without volatile and memory clobber.
94 */
95#define alternative(oldinstr, newinstr, feature) \
96 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
97 ".section .altinstructions,\"a\"\n" \
98 " .align 8\n" \
99 " .quad 661b\n" /* label */ \
100 " .quad 663f\n" /* new instruction */ \
101 " .byte %c0\n" /* feature bit */ \
102 " .byte 662b-661b\n" /* sourcelen */ \
103 " .byte 664f-663f\n" /* replacementlen */ \
104 ".previous\n" \
105 ".section .altinstr_replacement,\"ax\"\n" \
106 "663:\n\t" newinstr "\n664:\n" /* replacement */ \
107 ".previous" :: "i" (feature) : "memory")
108
109/*
110 * Alternative inline assembly with input.
111 *
112 * Peculiarities:
113 * No memory clobber here.
114 * Argument numbers start with 1.
115 * Best is to use constraints that are fixed size (like (%1) ... "r")
116 * If you use variable sized constraints like "m" or "g" in the
117 * replacement make sure to pad to the worst case length.
118 */
119#define alternative_input(oldinstr, newinstr, feature, input...) \
120 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
121 ".section .altinstructions,\"a\"\n" \
122 " .align 8\n" \
123 " .quad 661b\n" /* label */ \
124 " .quad 663f\n" /* new instruction */ \
125 " .byte %c0\n" /* feature bit */ \
126 " .byte 662b-661b\n" /* sourcelen */ \
127 " .byte 664f-663f\n" /* replacementlen */ \
128 ".previous\n" \
129 ".section .altinstr_replacement,\"ax\"\n" \
130 "663:\n\t" newinstr "\n664:\n" /* replacement */ \
131 ".previous" :: "i" (feature), ##input)
132
133/* Like alternative_input, but with a single output argument */
134#define alternative_io(oldinstr, newinstr, feature, output, input...) \
135 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
136 ".section .altinstructions,\"a\"\n" \
137 " .align 8\n" \
138 " .quad 661b\n" /* label */ \
139 " .quad 663f\n" /* new instruction */ \
140 " .byte %c[feat]\n" /* feature bit */ \
141 " .byte 662b-661b\n" /* sourcelen */ \
142 " .byte 664f-663f\n" /* replacementlen */ \
143 ".previous\n" \
144 ".section .altinstr_replacement,\"ax\"\n" \
145 "663:\n\t" newinstr "\n664:\n" /* replacement */ \
146 ".previous" : output : [feat] "i" (feature), ##input)
147
148/* 67/*
149 * Clear and set 'TS' bit respectively 68 * Clear and set 'TS' bit respectively
150 */ 69 */
@@ -366,5 +285,6 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
366void cpu_idle_wait(void); 285void cpu_idle_wait(void);
367 286
368extern unsigned long arch_align_stack(unsigned long sp); 287extern unsigned long arch_align_stack(unsigned long sp);
288extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
369 289
370#endif 290#endif
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-21 06:53:07 -0500