diff options
author | H. Peter Anvin <hpa@linux.intel.com> | 2011-04-18 18:19:51 -0400 |
---|---|---|
committer | H. Peter Anvin <hpa@linux.intel.com> | 2011-04-18 19:40:21 -0400 |
commit | dc326fca2b640fc41aed7c015d0f456935a66255 (patch) | |
tree | 9181ef9c416e8fba822d75b53155c896b3900e72 /arch/x86/kernel/alternative.c | |
parent | b1e7734f024c9ce4393016a97c8d821e1f18d9b4 (diff) |
x86, cpu: Clean up and unify the NOP selection infrastructure
Clean up and unify the NOP selection infrastructure:
- Make the atomic 5-byte NOP a part of the selection system.
- Pick NOPs once during early boot and then be done with it.
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
Cc: Tejun Heo <tj@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Jason Baron <jbaron@redhat.com>
Link: http://lkml.kernel.org/r/1303166160-10315-3-git-send-email-hpa@linux.intel.com
Diffstat (limited to 'arch/x86/kernel/alternative.c')
-rw-r--r-- | arch/x86/kernel/alternative.c | 182 |
1 files changed, 100 insertions, 82 deletions
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 4a234677e213..846f61eb89c1 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c | |||
@@ -67,17 +67,30 @@ __setup("noreplace-paravirt", setup_noreplace_paravirt); | |||
67 | #define DPRINTK(fmt, args...) if (debug_alternative) \ | 67 | #define DPRINTK(fmt, args...) if (debug_alternative) \ |
68 | printk(KERN_DEBUG fmt, args) | 68 | printk(KERN_DEBUG fmt, args) |
69 | 69 | ||
70 | /* | ||
71 | * Each GENERIC_NOPX is of X bytes, and defined as an array of bytes | ||
72 | * that correspond to that nop. Getting from one nop to the next, we | ||
73 | * add to the array the offset that is equal to the sum of all sizes of | ||
74 | * nops preceding the one we are after. | ||
75 | * | ||
76 | * Note: The GENERIC_NOP5_ATOMIC is at the end, as it breaks the | ||
77 | * nice symmetry of sizes of the previous nops. | ||
78 | */ | ||
70 | #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64) | 79 | #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64) |
71 | /* Use inline assembly to define this because the nops are defined | 80 | static const unsigned char intelnops[] = |
72 | as inline assembly strings in the include files and we cannot | 81 | { |
73 | get them easily into strings. */ | 82 | GENERIC_NOP1, |
74 | asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nintelnops: " | 83 | GENERIC_NOP2, |
75 | GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6 | 84 | GENERIC_NOP3, |
76 | GENERIC_NOP7 GENERIC_NOP8 | 85 | GENERIC_NOP4, |
77 | "\t.previous"); | 86 | GENERIC_NOP5, |
78 | extern const unsigned char intelnops[]; | 87 | GENERIC_NOP6, |
79 | static const unsigned char *const __initconst_or_module | 88 | GENERIC_NOP7, |
80 | intel_nops[ASM_NOP_MAX+1] = { | 89 | GENERIC_NOP8, |
90 | GENERIC_NOP5_ATOMIC | ||
91 | }; | ||
92 | static const unsigned char * const intel_nops[ASM_NOP_MAX+2] = | ||
93 | { | ||
81 | NULL, | 94 | NULL, |
82 | intelnops, | 95 | intelnops, |
83 | intelnops + 1, | 96 | intelnops + 1, |
@@ -87,17 +100,25 @@ intel_nops[ASM_NOP_MAX+1] = { | |||
87 | intelnops + 1 + 2 + 3 + 4 + 5, | 100 | intelnops + 1 + 2 + 3 + 4 + 5, |
88 | intelnops + 1 + 2 + 3 + 4 + 5 + 6, | 101 | intelnops + 1 + 2 + 3 + 4 + 5 + 6, |
89 | intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7, | 102 | intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7, |
103 | intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, | ||
90 | }; | 104 | }; |
91 | #endif | 105 | #endif |
92 | 106 | ||
93 | #ifdef K8_NOP1 | 107 | #ifdef K8_NOP1 |
94 | asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk8nops: " | 108 | static const unsigned char k8nops[] = |
95 | K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6 | 109 | { |
96 | K8_NOP7 K8_NOP8 | 110 | K8_NOP1, |
97 | "\t.previous"); | 111 | K8_NOP2, |
98 | extern const unsigned char k8nops[]; | 112 | K8_NOP3, |
99 | static const unsigned char *const __initconst_or_module | 113 | K8_NOP4, |
100 | k8_nops[ASM_NOP_MAX+1] = { | 114 | K8_NOP5, |
115 | K8_NOP6, | ||
116 | K8_NOP7, | ||
117 | K8_NOP8, | ||
118 | K8_NOP5_ATOMIC | ||
119 | }; | ||
120 | static const unsigned char * const k8_nops[ASM_NOP_MAX+2] = | ||
121 | { | ||
101 | NULL, | 122 | NULL, |
102 | k8nops, | 123 | k8nops, |
103 | k8nops + 1, | 124 | k8nops + 1, |
@@ -107,17 +128,25 @@ k8_nops[ASM_NOP_MAX+1] = { | |||
107 | k8nops + 1 + 2 + 3 + 4 + 5, | 128 | k8nops + 1 + 2 + 3 + 4 + 5, |
108 | k8nops + 1 + 2 + 3 + 4 + 5 + 6, | 129 | k8nops + 1 + 2 + 3 + 4 + 5 + 6, |
109 | k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, | 130 | k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, |
131 | k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, | ||
110 | }; | 132 | }; |
111 | #endif | 133 | #endif |
112 | 134 | ||
113 | #if defined(K7_NOP1) && !defined(CONFIG_X86_64) | 135 | #if defined(K7_NOP1) && !defined(CONFIG_X86_64) |
114 | asm("\t" __stringify(__INITRODATA_OR_MODULE) "\nk7nops: " | 136 | static const unsigned char k7nops[] = |
115 | K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6 | 137 | { |
116 | K7_NOP7 K7_NOP8 | 138 | K7_NOP1, |
117 | "\t.previous"); | 139 | K7_NOP2, |
118 | extern const unsigned char k7nops[]; | 140 | K7_NOP3, |
119 | static const unsigned char *const __initconst_or_module | 141 | K7_NOP4, |
120 | k7_nops[ASM_NOP_MAX+1] = { | 142 | K7_NOP5, |
143 | K7_NOP6, | ||
144 | K7_NOP7, | ||
145 | K7_NOP8, | ||
146 | K7_NOP5_ATOMIC | ||
147 | }; | ||
148 | static const unsigned char * const k7_nops[ASM_NOP_MAX+2] = | ||
149 | { | ||
121 | NULL, | 150 | NULL, |
122 | k7nops, | 151 | k7nops, |
123 | k7nops + 1, | 152 | k7nops + 1, |
@@ -127,17 +156,25 @@ k7_nops[ASM_NOP_MAX+1] = { | |||
127 | k7nops + 1 + 2 + 3 + 4 + 5, | 156 | k7nops + 1 + 2 + 3 + 4 + 5, |
128 | k7nops + 1 + 2 + 3 + 4 + 5 + 6, | 157 | k7nops + 1 + 2 + 3 + 4 + 5 + 6, |
129 | k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, | 158 | k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, |
159 | k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, | ||
130 | }; | 160 | }; |
131 | #endif | 161 | #endif |
132 | 162 | ||
133 | #ifdef P6_NOP1 | 163 | #ifdef P6_NOP1 |
134 | asm("\t" __stringify(__INITRODATA_OR_MODULE) "\np6nops: " | 164 | static const unsigned char __initconst_or_module p6nops[] = |
135 | P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6 | 165 | { |
136 | P6_NOP7 P6_NOP8 | 166 | P6_NOP1, |
137 | "\t.previous"); | 167 | P6_NOP2, |
138 | extern const unsigned char p6nops[]; | 168 | P6_NOP3, |
139 | static const unsigned char *const __initconst_or_module | 169 | P6_NOP4, |
140 | p6_nops[ASM_NOP_MAX+1] = { | 170 | P6_NOP5, |
171 | P6_NOP6, | ||
172 | P6_NOP7, | ||
173 | P6_NOP8, | ||
174 | P6_NOP5_ATOMIC | ||
175 | }; | ||
176 | static const unsigned char * const p6_nops[ASM_NOP_MAX+2] = | ||
177 | { | ||
141 | NULL, | 178 | NULL, |
142 | p6nops, | 179 | p6nops, |
143 | p6nops + 1, | 180 | p6nops + 1, |
@@ -147,47 +184,53 @@ p6_nops[ASM_NOP_MAX+1] = { | |||
147 | p6nops + 1 + 2 + 3 + 4 + 5, | 184 | p6nops + 1 + 2 + 3 + 4 + 5, |
148 | p6nops + 1 + 2 + 3 + 4 + 5 + 6, | 185 | p6nops + 1 + 2 + 3 + 4 + 5 + 6, |
149 | p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, | 186 | p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, |
187 | p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, | ||
150 | }; | 188 | }; |
151 | #endif | 189 | #endif |
152 | 190 | ||
191 | /* Initialize these to a safe default */ | ||
153 | #ifdef CONFIG_X86_64 | 192 | #ifdef CONFIG_X86_64 |
193 | const unsigned char * const *ideal_nops = p6_nops; | ||
194 | #else | ||
195 | const unsigned char * const *ideal_nops = intel_nops; | ||
196 | #endif | ||
154 | 197 | ||
155 | extern char __vsyscall_0; | 198 | void __init arch_init_ideal_nops(void) |
156 | static const unsigned char *const *__init_or_module find_nop_table(void) | ||
157 | { | 199 | { |
158 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && | 200 | switch (boot_cpu_data.x86_vendor) { |
159 | boot_cpu_has(X86_FEATURE_NOPL)) | 201 | case X86_VENDOR_INTEL: |
160 | return p6_nops; | 202 | if (boot_cpu_has(X86_FEATURE_NOPL)) { |
161 | else | 203 | ideal_nops = p6_nops; |
162 | return k8_nops; | 204 | } else { |
163 | } | 205 | #ifdef CONFIG_X86_64 |
164 | 206 | ideal_nops = k8_nops; | |
165 | #else /* CONFIG_X86_64 */ | 207 | #else |
208 | ideal_nops = intel_nops; | ||
209 | #endif | ||
210 | } | ||
166 | 211 | ||
167 | static const unsigned char *const *__init_or_module find_nop_table(void) | 212 | default: |
168 | { | 213 | #ifdef CONFIG_X86_64 |
169 | if (boot_cpu_has(X86_FEATURE_K8)) | 214 | ideal_nops = k8_nops; |
170 | return k8_nops; | 215 | #else |
171 | else if (boot_cpu_has(X86_FEATURE_K7)) | 216 | if (boot_cpu_has(X86_FEATURE_K8)) |
172 | return k7_nops; | 217 | ideal_nops = k8_nops; |
173 | else if (boot_cpu_has(X86_FEATURE_NOPL)) | 218 | else if (boot_cpu_has(X86_FEATURE_K7)) |
174 | return p6_nops; | 219 | ideal_nops = k7_nops; |
175 | else | 220 | else |
176 | return intel_nops; | 221 | ideal_nops = intel_nops; |
222 | #endif | ||
223 | } | ||
177 | } | 224 | } |
178 | 225 | ||
179 | #endif /* CONFIG_X86_64 */ | ||
180 | |||
181 | /* Use this to add nops to a buffer, then text_poke the whole buffer. */ | 226 | /* Use this to add nops to a buffer, then text_poke the whole buffer. */ |
182 | static void __init_or_module add_nops(void *insns, unsigned int len) | 227 | static void __init_or_module add_nops(void *insns, unsigned int len) |
183 | { | 228 | { |
184 | const unsigned char *const *noptable = find_nop_table(); | ||
185 | |||
186 | while (len > 0) { | 229 | while (len > 0) { |
187 | unsigned int noplen = len; | 230 | unsigned int noplen = len; |
188 | if (noplen > ASM_NOP_MAX) | 231 | if (noplen > ASM_NOP_MAX) |
189 | noplen = ASM_NOP_MAX; | 232 | noplen = ASM_NOP_MAX; |
190 | memcpy(insns, noptable[noplen], noplen); | 233 | memcpy(insns, ideal_nops[noplen], noplen); |
191 | insns += noplen; | 234 | insns += noplen; |
192 | len -= noplen; | 235 | len -= noplen; |
193 | } | 236 | } |
@@ -195,6 +238,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len) | |||
195 | 238 | ||
196 | extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; | 239 | extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; |
197 | extern s32 __smp_locks[], __smp_locks_end[]; | 240 | extern s32 __smp_locks[], __smp_locks_end[]; |
241 | extern char __vsyscall_0; | ||
198 | void *text_poke_early(void *addr, const void *opcode, size_t len); | 242 | void *text_poke_early(void *addr, const void *opcode, size_t len); |
199 | 243 | ||
200 | /* Replace instructions with better alternatives for this CPU type. | 244 | /* Replace instructions with better alternatives for this CPU type. |
@@ -678,29 +722,3 @@ void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n) | |||
678 | wrote_text = 0; | 722 | wrote_text = 0; |
679 | __stop_machine(stop_machine_text_poke, (void *)&tpp, NULL); | 723 | __stop_machine(stop_machine_text_poke, (void *)&tpp, NULL); |
680 | } | 724 | } |
681 | |||
682 | #if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL) | ||
683 | |||
684 | #ifdef CONFIG_X86_64 | ||
685 | unsigned char ideal_nop5[5] = { 0x66, 0x66, 0x66, 0x66, 0x90 }; | ||
686 | #else | ||
687 | unsigned char ideal_nop5[5] = { 0x3e, 0x8d, 0x74, 0x26, 0x00 }; | ||
688 | #endif | ||
689 | |||
690 | void __init arch_init_ideal_nop5(void) | ||
691 | { | ||
692 | /* | ||
693 | * There is no good nop for all x86 archs. This selection | ||
694 | * algorithm should be unified with the one in find_nop_table(), | ||
695 | * but this should be good enough for now. | ||
696 | * | ||
697 | * For cases other than the ones below, use the safe (as in | ||
698 | * always functional) defaults above. | ||
699 | */ | ||
700 | #ifdef CONFIG_X86_64 | ||
701 | /* Don't use these on 32 bits due to broken virtualizers */ | ||
702 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) | ||
703 | memcpy(ideal_nop5, p6_nops[5], 5); | ||
704 | #endif | ||
705 | } | ||
706 | #endif | ||