aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/kernel')
-rw-r--r--arch/x86/kernel/acpi/boot.c2
-rw-r--r--arch/x86/kernel/cpu/bugs.c525
-rw-r--r--arch/x86/kernel/cpu/mcheck/mce_amd.c19
-rw-r--r--arch/x86/kernel/fpu/signal.c4
-rw-r--r--arch/x86/kernel/ftrace.c15
-rw-r--r--arch/x86/kernel/head32.c1
-rw-r--r--arch/x86/kernel/head64.c2
-rw-r--r--arch/x86/kernel/kprobes/opt.c2
-rw-r--r--arch/x86/kernel/process.c101
-rw-r--r--arch/x86/kernel/process.h39
-rw-r--r--arch/x86/kernel/process_32.c10
-rw-r--r--arch/x86/kernel/process_64.c10
-rw-r--r--arch/x86/kernel/setup.c17
-rw-r--r--arch/x86/kernel/tboot.c2
14 files changed, 527 insertions, 222 deletions
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index 92c76bf97ad8..06635fbca81c 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -1776,5 +1776,5 @@ void __init arch_reserve_mem_area(acpi_physical_address addr, size_t size)
1776 1776
1777u64 x86_default_get_root_pointer(void) 1777u64 x86_default_get_root_pointer(void)
1778{ 1778{
1779 return boot_params.hdr.acpi_rsdp_addr; 1779 return boot_params.acpi_rsdp_addr;
1780} 1780}
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index c37e66e493bf..500278f5308e 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -14,6 +14,7 @@
14#include <linux/module.h> 14#include <linux/module.h>
15#include <linux/nospec.h> 15#include <linux/nospec.h>
16#include <linux/prctl.h> 16#include <linux/prctl.h>
17#include <linux/sched/smt.h>
17 18
18#include <asm/spec-ctrl.h> 19#include <asm/spec-ctrl.h>
19#include <asm/cmdline.h> 20#include <asm/cmdline.h>
@@ -53,6 +54,13 @@ static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
53u64 __ro_after_init x86_amd_ls_cfg_base; 54u64 __ro_after_init x86_amd_ls_cfg_base;
54u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask; 55u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
55 56
57/* Control conditional STIPB in switch_to() */
58DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
59/* Control conditional IBPB in switch_mm() */
60DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
61/* Control unconditional IBPB in switch_mm() */
62DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
63
56void __init check_bugs(void) 64void __init check_bugs(void)
57{ 65{
58 identify_boot_cpu(); 66 identify_boot_cpu();
@@ -123,31 +131,6 @@ void __init check_bugs(void)
123#endif 131#endif
124} 132}
125 133
126/* The kernel command line selection */
127enum spectre_v2_mitigation_cmd {
128 SPECTRE_V2_CMD_NONE,
129 SPECTRE_V2_CMD_AUTO,
130 SPECTRE_V2_CMD_FORCE,
131 SPECTRE_V2_CMD_RETPOLINE,
132 SPECTRE_V2_CMD_RETPOLINE_GENERIC,
133 SPECTRE_V2_CMD_RETPOLINE_AMD,
134};
135
136static const char *spectre_v2_strings[] = {
137 [SPECTRE_V2_NONE] = "Vulnerable",
138 [SPECTRE_V2_RETPOLINE_MINIMAL] = "Vulnerable: Minimal generic ASM retpoline",
139 [SPECTRE_V2_RETPOLINE_MINIMAL_AMD] = "Vulnerable: Minimal AMD ASM retpoline",
140 [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline",
141 [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline",
142 [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS",
143};
144
145#undef pr_fmt
146#define pr_fmt(fmt) "Spectre V2 : " fmt
147
148static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
149 SPECTRE_V2_NONE;
150
151void 134void
152x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest) 135x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
153{ 136{
@@ -169,6 +152,10 @@ x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
169 static_cpu_has(X86_FEATURE_AMD_SSBD)) 152 static_cpu_has(X86_FEATURE_AMD_SSBD))
170 hostval |= ssbd_tif_to_spec_ctrl(ti->flags); 153 hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
171 154
155 /* Conditional STIBP enabled? */
156 if (static_branch_unlikely(&switch_to_cond_stibp))
157 hostval |= stibp_tif_to_spec_ctrl(ti->flags);
158
172 if (hostval != guestval) { 159 if (hostval != guestval) {
173 msrval = setguest ? guestval : hostval; 160 msrval = setguest ? guestval : hostval;
174 wrmsrl(MSR_IA32_SPEC_CTRL, msrval); 161 wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
@@ -202,7 +189,7 @@ x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
202 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) : 189 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
203 ssbd_spec_ctrl_to_tif(hostval); 190 ssbd_spec_ctrl_to_tif(hostval);
204 191
205 speculative_store_bypass_update(tif); 192 speculation_ctrl_update(tif);
206 } 193 }
207} 194}
208EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); 195EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
@@ -217,6 +204,15 @@ static void x86_amd_ssb_disable(void)
217 wrmsrl(MSR_AMD64_LS_CFG, msrval); 204 wrmsrl(MSR_AMD64_LS_CFG, msrval);
218} 205}
219 206
207#undef pr_fmt
208#define pr_fmt(fmt) "Spectre V2 : " fmt
209
210static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
211 SPECTRE_V2_NONE;
212
213static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init =
214 SPECTRE_V2_USER_NONE;
215
220#ifdef RETPOLINE 216#ifdef RETPOLINE
221static bool spectre_v2_bad_module; 217static bool spectre_v2_bad_module;
222 218
@@ -238,67 +234,217 @@ static inline const char *spectre_v2_module_string(void)
238static inline const char *spectre_v2_module_string(void) { return ""; } 234static inline const char *spectre_v2_module_string(void) { return ""; }
239#endif 235#endif
240 236
241static void __init spec2_print_if_insecure(const char *reason) 237static inline bool match_option(const char *arg, int arglen, const char *opt)
242{ 238{
243 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 239 int len = strlen(opt);
244 pr_info("%s selected on command line.\n", reason); 240
241 return len == arglen && !strncmp(arg, opt, len);
245} 242}
246 243
247static void __init spec2_print_if_secure(const char *reason) 244/* The kernel command line selection for spectre v2 */
245enum spectre_v2_mitigation_cmd {
246 SPECTRE_V2_CMD_NONE,
247 SPECTRE_V2_CMD_AUTO,
248 SPECTRE_V2_CMD_FORCE,
249 SPECTRE_V2_CMD_RETPOLINE,
250 SPECTRE_V2_CMD_RETPOLINE_GENERIC,
251 SPECTRE_V2_CMD_RETPOLINE_AMD,
252};
253
254enum spectre_v2_user_cmd {
255 SPECTRE_V2_USER_CMD_NONE,
256 SPECTRE_V2_USER_CMD_AUTO,
257 SPECTRE_V2_USER_CMD_FORCE,
258 SPECTRE_V2_USER_CMD_PRCTL,
259 SPECTRE_V2_USER_CMD_PRCTL_IBPB,
260 SPECTRE_V2_USER_CMD_SECCOMP,
261 SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
262};
263
264static const char * const spectre_v2_user_strings[] = {
265 [SPECTRE_V2_USER_NONE] = "User space: Vulnerable",
266 [SPECTRE_V2_USER_STRICT] = "User space: Mitigation: STIBP protection",
267 [SPECTRE_V2_USER_PRCTL] = "User space: Mitigation: STIBP via prctl",
268 [SPECTRE_V2_USER_SECCOMP] = "User space: Mitigation: STIBP via seccomp and prctl",
269};
270
271static const struct {
272 const char *option;
273 enum spectre_v2_user_cmd cmd;
274 bool secure;
275} v2_user_options[] __initdata = {
276 { "auto", SPECTRE_V2_USER_CMD_AUTO, false },
277 { "off", SPECTRE_V2_USER_CMD_NONE, false },
278 { "on", SPECTRE_V2_USER_CMD_FORCE, true },
279 { "prctl", SPECTRE_V2_USER_CMD_PRCTL, false },
280 { "prctl,ibpb", SPECTRE_V2_USER_CMD_PRCTL_IBPB, false },
281 { "seccomp", SPECTRE_V2_USER_CMD_SECCOMP, false },
282 { "seccomp,ibpb", SPECTRE_V2_USER_CMD_SECCOMP_IBPB, false },
283};
284
285static void __init spec_v2_user_print_cond(const char *reason, bool secure)
248{ 286{
249 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) 287 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
250 pr_info("%s selected on command line.\n", reason); 288 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
251} 289}
252 290
253static inline bool retp_compiler(void) 291static enum spectre_v2_user_cmd __init
292spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
254{ 293{
255 return __is_defined(RETPOLINE); 294 char arg[20];
295 int ret, i;
296
297 switch (v2_cmd) {
298 case SPECTRE_V2_CMD_NONE:
299 return SPECTRE_V2_USER_CMD_NONE;
300 case SPECTRE_V2_CMD_FORCE:
301 return SPECTRE_V2_USER_CMD_FORCE;
302 default:
303 break;
304 }
305
306 ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
307 arg, sizeof(arg));
308 if (ret < 0)
309 return SPECTRE_V2_USER_CMD_AUTO;
310
311 for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
312 if (match_option(arg, ret, v2_user_options[i].option)) {
313 spec_v2_user_print_cond(v2_user_options[i].option,
314 v2_user_options[i].secure);
315 return v2_user_options[i].cmd;
316 }
317 }
318
319 pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
320 return SPECTRE_V2_USER_CMD_AUTO;
256} 321}
257 322
258static inline bool match_option(const char *arg, int arglen, const char *opt) 323static void __init
324spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
259{ 325{
260 int len = strlen(opt); 326 enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
327 bool smt_possible = IS_ENABLED(CONFIG_SMP);
328 enum spectre_v2_user_cmd cmd;
261 329
262 return len == arglen && !strncmp(arg, opt, len); 330 if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
331 return;
332
333 if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
334 cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
335 smt_possible = false;
336
337 cmd = spectre_v2_parse_user_cmdline(v2_cmd);
338 switch (cmd) {
339 case SPECTRE_V2_USER_CMD_NONE:
340 goto set_mode;
341 case SPECTRE_V2_USER_CMD_FORCE:
342 mode = SPECTRE_V2_USER_STRICT;
343 break;
344 case SPECTRE_V2_USER_CMD_PRCTL:
345 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
346 mode = SPECTRE_V2_USER_PRCTL;
347 break;
348 case SPECTRE_V2_USER_CMD_AUTO:
349 case SPECTRE_V2_USER_CMD_SECCOMP:
350 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
351 if (IS_ENABLED(CONFIG_SECCOMP))
352 mode = SPECTRE_V2_USER_SECCOMP;
353 else
354 mode = SPECTRE_V2_USER_PRCTL;
355 break;
356 }
357
358 /* Initialize Indirect Branch Prediction Barrier */
359 if (boot_cpu_has(X86_FEATURE_IBPB)) {
360 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
361
362 switch (cmd) {
363 case SPECTRE_V2_USER_CMD_FORCE:
364 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
365 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
366 static_branch_enable(&switch_mm_always_ibpb);
367 break;
368 case SPECTRE_V2_USER_CMD_PRCTL:
369 case SPECTRE_V2_USER_CMD_AUTO:
370 case SPECTRE_V2_USER_CMD_SECCOMP:
371 static_branch_enable(&switch_mm_cond_ibpb);
372 break;
373 default:
374 break;
375 }
376
377 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
378 static_key_enabled(&switch_mm_always_ibpb) ?
379 "always-on" : "conditional");
380 }
381
382 /* If enhanced IBRS is enabled no STIPB required */
383 if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
384 return;
385
386 /*
387 * If SMT is not possible or STIBP is not available clear the STIPB
388 * mode.
389 */
390 if (!smt_possible || !boot_cpu_has(X86_FEATURE_STIBP))
391 mode = SPECTRE_V2_USER_NONE;
392set_mode:
393 spectre_v2_user = mode;
394 /* Only print the STIBP mode when SMT possible */
395 if (smt_possible)
396 pr_info("%s\n", spectre_v2_user_strings[mode]);
263} 397}
264 398
399static const char * const spectre_v2_strings[] = {
400 [SPECTRE_V2_NONE] = "Vulnerable",
401 [SPECTRE_V2_RETPOLINE_GENERIC] = "Mitigation: Full generic retpoline",
402 [SPECTRE_V2_RETPOLINE_AMD] = "Mitigation: Full AMD retpoline",
403 [SPECTRE_V2_IBRS_ENHANCED] = "Mitigation: Enhanced IBRS",
404};
405
265static const struct { 406static const struct {
266 const char *option; 407 const char *option;
267 enum spectre_v2_mitigation_cmd cmd; 408 enum spectre_v2_mitigation_cmd cmd;
268 bool secure; 409 bool secure;
269} mitigation_options[] = { 410} mitigation_options[] __initdata = {
270 { "off", SPECTRE_V2_CMD_NONE, false }, 411 { "off", SPECTRE_V2_CMD_NONE, false },
271 { "on", SPECTRE_V2_CMD_FORCE, true }, 412 { "on", SPECTRE_V2_CMD_FORCE, true },
272 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false }, 413 { "retpoline", SPECTRE_V2_CMD_RETPOLINE, false },
273 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false }, 414 { "retpoline,amd", SPECTRE_V2_CMD_RETPOLINE_AMD, false },
274 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false }, 415 { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
275 { "auto", SPECTRE_V2_CMD_AUTO, false }, 416 { "auto", SPECTRE_V2_CMD_AUTO, false },
276}; 417};
277 418
419static void __init spec_v2_print_cond(const char *reason, bool secure)
420{
421 if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
422 pr_info("%s selected on command line.\n", reason);
423}
424
278static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void) 425static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
279{ 426{
427 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
280 char arg[20]; 428 char arg[20];
281 int ret, i; 429 int ret, i;
282 enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
283 430
284 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2")) 431 if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
285 return SPECTRE_V2_CMD_NONE; 432 return SPECTRE_V2_CMD_NONE;
286 else {
287 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
288 if (ret < 0)
289 return SPECTRE_V2_CMD_AUTO;
290 433
291 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) { 434 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
292 if (!match_option(arg, ret, mitigation_options[i].option)) 435 if (ret < 0)
293 continue; 436 return SPECTRE_V2_CMD_AUTO;
294 cmd = mitigation_options[i].cmd;
295 break;
296 }
297 437
298 if (i >= ARRAY_SIZE(mitigation_options)) { 438 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
299 pr_err("unknown option (%s). Switching to AUTO select\n", arg); 439 if (!match_option(arg, ret, mitigation_options[i].option))
300 return SPECTRE_V2_CMD_AUTO; 440 continue;
301 } 441 cmd = mitigation_options[i].cmd;
442 break;
443 }
444
445 if (i >= ARRAY_SIZE(mitigation_options)) {
446 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
447 return SPECTRE_V2_CMD_AUTO;
302 } 448 }
303 449
304 if ((cmd == SPECTRE_V2_CMD_RETPOLINE || 450 if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
@@ -316,54 +462,11 @@ static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
316 return SPECTRE_V2_CMD_AUTO; 462 return SPECTRE_V2_CMD_AUTO;
317 } 463 }
318 464
319 if (mitigation_options[i].secure) 465 spec_v2_print_cond(mitigation_options[i].option,
320 spec2_print_if_secure(mitigation_options[i].option); 466 mitigation_options[i].secure);
321 else
322 spec2_print_if_insecure(mitigation_options[i].option);
323
324 return cmd; 467 return cmd;
325} 468}
326 469
327static bool stibp_needed(void)
328{
329 if (spectre_v2_enabled == SPECTRE_V2_NONE)
330 return false;
331
332 if (!boot_cpu_has(X86_FEATURE_STIBP))
333 return false;
334
335 return true;
336}
337
338static void update_stibp_msr(void *info)
339{
340 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
341}
342
343void arch_smt_update(void)
344{
345 u64 mask;
346
347 if (!stibp_needed())
348 return;
349
350 mutex_lock(&spec_ctrl_mutex);
351 mask = x86_spec_ctrl_base;
352 if (cpu_smt_control == CPU_SMT_ENABLED)
353 mask |= SPEC_CTRL_STIBP;
354 else
355 mask &= ~SPEC_CTRL_STIBP;
356
357 if (mask != x86_spec_ctrl_base) {
358 pr_info("Spectre v2 cross-process SMT mitigation: %s STIBP\n",
359 cpu_smt_control == CPU_SMT_ENABLED ?
360 "Enabling" : "Disabling");
361 x86_spec_ctrl_base = mask;
362 on_each_cpu(update_stibp_msr, NULL, 1);
363 }
364 mutex_unlock(&spec_ctrl_mutex);
365}
366
367static void __init spectre_v2_select_mitigation(void) 470static void __init spectre_v2_select_mitigation(void)
368{ 471{
369 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline(); 472 enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
@@ -417,14 +520,12 @@ retpoline_auto:
417 pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n"); 520 pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
418 goto retpoline_generic; 521 goto retpoline_generic;
419 } 522 }
420 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD : 523 mode = SPECTRE_V2_RETPOLINE_AMD;
421 SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
422 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD); 524 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
423 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 525 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
424 } else { 526 } else {
425 retpoline_generic: 527 retpoline_generic:
426 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC : 528 mode = SPECTRE_V2_RETPOLINE_GENERIC;
427 SPECTRE_V2_RETPOLINE_MINIMAL;
428 setup_force_cpu_cap(X86_FEATURE_RETPOLINE); 529 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
429 } 530 }
430 531
@@ -443,12 +544,6 @@ specv2_set_mode:
443 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW); 544 setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
444 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n"); 545 pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
445 546
446 /* Initialize Indirect Branch Prediction Barrier if supported */
447 if (boot_cpu_has(X86_FEATURE_IBPB)) {
448 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
449 pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n");
450 }
451
452 /* 547 /*
453 * Retpoline means the kernel is safe because it has no indirect 548 * Retpoline means the kernel is safe because it has no indirect
454 * branches. Enhanced IBRS protects firmware too, so, enable restricted 549 * branches. Enhanced IBRS protects firmware too, so, enable restricted
@@ -465,10 +560,67 @@ specv2_set_mode:
465 pr_info("Enabling Restricted Speculation for firmware calls\n"); 560 pr_info("Enabling Restricted Speculation for firmware calls\n");
466 } 561 }
467 562
563 /* Set up IBPB and STIBP depending on the general spectre V2 command */
564 spectre_v2_user_select_mitigation(cmd);
565
468 /* Enable STIBP if appropriate */ 566 /* Enable STIBP if appropriate */
469 arch_smt_update(); 567 arch_smt_update();
470} 568}
471 569
570static void update_stibp_msr(void * __unused)
571{
572 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
573}
574
575/* Update x86_spec_ctrl_base in case SMT state changed. */
576static void update_stibp_strict(void)
577{
578 u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
579
580 if (sched_smt_active())
581 mask |= SPEC_CTRL_STIBP;
582
583 if (mask == x86_spec_ctrl_base)
584 return;
585
586 pr_info("Update user space SMT mitigation: STIBP %s\n",
587 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
588 x86_spec_ctrl_base = mask;
589 on_each_cpu(update_stibp_msr, NULL, 1);
590}
591
592/* Update the static key controlling the evaluation of TIF_SPEC_IB */
593static void update_indir_branch_cond(void)
594{
595 if (sched_smt_active())
596 static_branch_enable(&switch_to_cond_stibp);
597 else
598 static_branch_disable(&switch_to_cond_stibp);
599}
600
601void arch_smt_update(void)
602{
603 /* Enhanced IBRS implies STIBP. No update required. */
604 if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
605 return;
606
607 mutex_lock(&spec_ctrl_mutex);
608
609 switch (spectre_v2_user) {
610 case SPECTRE_V2_USER_NONE:
611 break;
612 case SPECTRE_V2_USER_STRICT:
613 update_stibp_strict();
614 break;
615 case SPECTRE_V2_USER_PRCTL:
616 case SPECTRE_V2_USER_SECCOMP:
617 update_indir_branch_cond();
618 break;
619 }
620
621 mutex_unlock(&spec_ctrl_mutex);
622}
623
472#undef pr_fmt 624#undef pr_fmt
473#define pr_fmt(fmt) "Speculative Store Bypass: " fmt 625#define pr_fmt(fmt) "Speculative Store Bypass: " fmt
474 626
@@ -483,7 +635,7 @@ enum ssb_mitigation_cmd {
483 SPEC_STORE_BYPASS_CMD_SECCOMP, 635 SPEC_STORE_BYPASS_CMD_SECCOMP,
484}; 636};
485 637
486static const char *ssb_strings[] = { 638static const char * const ssb_strings[] = {
487 [SPEC_STORE_BYPASS_NONE] = "Vulnerable", 639 [SPEC_STORE_BYPASS_NONE] = "Vulnerable",
488 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled", 640 [SPEC_STORE_BYPASS_DISABLE] = "Mitigation: Speculative Store Bypass disabled",
489 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl", 641 [SPEC_STORE_BYPASS_PRCTL] = "Mitigation: Speculative Store Bypass disabled via prctl",
@@ -493,7 +645,7 @@ static const char *ssb_strings[] = {
493static const struct { 645static const struct {
494 const char *option; 646 const char *option;
495 enum ssb_mitigation_cmd cmd; 647 enum ssb_mitigation_cmd cmd;
496} ssb_mitigation_options[] = { 648} ssb_mitigation_options[] __initdata = {
497 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */ 649 { "auto", SPEC_STORE_BYPASS_CMD_AUTO }, /* Platform decides */
498 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */ 650 { "on", SPEC_STORE_BYPASS_CMD_ON }, /* Disable Speculative Store Bypass */
499 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */ 651 { "off", SPEC_STORE_BYPASS_CMD_NONE }, /* Don't touch Speculative Store Bypass */
@@ -604,10 +756,25 @@ static void ssb_select_mitigation(void)
604#undef pr_fmt 756#undef pr_fmt
605#define pr_fmt(fmt) "Speculation prctl: " fmt 757#define pr_fmt(fmt) "Speculation prctl: " fmt
606 758
607static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl) 759static void task_update_spec_tif(struct task_struct *tsk)
608{ 760{
609 bool update; 761 /* Force the update of the real TIF bits */
762 set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
610 763
764 /*
765 * Immediately update the speculation control MSRs for the current
766 * task, but for a non-current task delay setting the CPU
767 * mitigation until it is scheduled next.
768 *
769 * This can only happen for SECCOMP mitigation. For PRCTL it's
770 * always the current task.
771 */
772 if (tsk == current)
773 speculation_ctrl_update_current();
774}
775
776static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
777{
611 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL && 778 if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
612 ssb_mode != SPEC_STORE_BYPASS_SECCOMP) 779 ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
613 return -ENXIO; 780 return -ENXIO;
@@ -618,28 +785,56 @@ static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
618 if (task_spec_ssb_force_disable(task)) 785 if (task_spec_ssb_force_disable(task))
619 return -EPERM; 786 return -EPERM;
620 task_clear_spec_ssb_disable(task); 787 task_clear_spec_ssb_disable(task);
621 update = test_and_clear_tsk_thread_flag(task, TIF_SSBD); 788 task_update_spec_tif(task);
622 break; 789 break;
623 case PR_SPEC_DISABLE: 790 case PR_SPEC_DISABLE:
624 task_set_spec_ssb_disable(task); 791 task_set_spec_ssb_disable(task);
625 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD); 792 task_update_spec_tif(task);
626 break; 793 break;
627 case PR_SPEC_FORCE_DISABLE: 794 case PR_SPEC_FORCE_DISABLE:
628 task_set_spec_ssb_disable(task); 795 task_set_spec_ssb_disable(task);
629 task_set_spec_ssb_force_disable(task); 796 task_set_spec_ssb_force_disable(task);
630 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD); 797 task_update_spec_tif(task);
631 break; 798 break;
632 default: 799 default:
633 return -ERANGE; 800 return -ERANGE;
634 } 801 }
802 return 0;
803}
635 804
636 /* 805static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
637 * If being set on non-current task, delay setting the CPU 806{
638 * mitigation until it is next scheduled. 807 switch (ctrl) {
639 */ 808 case PR_SPEC_ENABLE:
640 if (task == current && update) 809 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
641 speculative_store_bypass_update_current(); 810 return 0;
642 811 /*
812 * Indirect branch speculation is always disabled in strict
813 * mode.
814 */
815 if (spectre_v2_user == SPECTRE_V2_USER_STRICT)
816 return -EPERM;
817 task_clear_spec_ib_disable(task);
818 task_update_spec_tif(task);
819 break;
820 case PR_SPEC_DISABLE:
821 case PR_SPEC_FORCE_DISABLE:
822 /*
823 * Indirect branch speculation is always allowed when
824 * mitigation is force disabled.
825 */
826 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
827 return -EPERM;
828 if (spectre_v2_user == SPECTRE_V2_USER_STRICT)
829 return 0;
830 task_set_spec_ib_disable(task);
831 if (ctrl == PR_SPEC_FORCE_DISABLE)
832 task_set_spec_ib_force_disable(task);
833 task_update_spec_tif(task);
834 break;
835 default:
836 return -ERANGE;
837 }
643 return 0; 838 return 0;
644} 839}
645 840
@@ -649,6 +844,8 @@ int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
649 switch (which) { 844 switch (which) {
650 case PR_SPEC_STORE_BYPASS: 845 case PR_SPEC_STORE_BYPASS:
651 return ssb_prctl_set(task, ctrl); 846 return ssb_prctl_set(task, ctrl);
847 case PR_SPEC_INDIRECT_BRANCH:
848 return ib_prctl_set(task, ctrl);
652 default: 849 default:
653 return -ENODEV; 850 return -ENODEV;
654 } 851 }
@@ -659,6 +856,8 @@ void arch_seccomp_spec_mitigate(struct task_struct *task)
659{ 856{
660 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) 857 if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
661 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); 858 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
859 if (spectre_v2_user == SPECTRE_V2_USER_SECCOMP)
860 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
662} 861}
663#endif 862#endif
664 863
@@ -681,11 +880,35 @@ static int ssb_prctl_get(struct task_struct *task)
681 } 880 }
682} 881}
683 882
883static int ib_prctl_get(struct task_struct *task)
884{
885 if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
886 return PR_SPEC_NOT_AFFECTED;
887
888 switch (spectre_v2_user) {
889 case SPECTRE_V2_USER_NONE:
890 return PR_SPEC_ENABLE;
891 case SPECTRE_V2_USER_PRCTL:
892 case SPECTRE_V2_USER_SECCOMP:
893 if (task_spec_ib_force_disable(task))
894 return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
895 if (task_spec_ib_disable(task))
896 return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
897 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
898 case SPECTRE_V2_USER_STRICT:
899 return PR_SPEC_DISABLE;
900 default:
901 return PR_SPEC_NOT_AFFECTED;
902 }
903}
904
684int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) 905int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
685{ 906{
686 switch (which) { 907 switch (which) {
687 case PR_SPEC_STORE_BYPASS: 908 case PR_SPEC_STORE_BYPASS:
688 return ssb_prctl_get(task); 909 return ssb_prctl_get(task);
910 case PR_SPEC_INDIRECT_BRANCH:
911 return ib_prctl_get(task);
689 default: 912 default:
690 return -ENODEV; 913 return -ENODEV;
691 } 914 }
@@ -823,7 +1046,7 @@ early_param("l1tf", l1tf_cmdline);
823#define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion" 1046#define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
824 1047
825#if IS_ENABLED(CONFIG_KVM_INTEL) 1048#if IS_ENABLED(CONFIG_KVM_INTEL)
826static const char *l1tf_vmx_states[] = { 1049static const char * const l1tf_vmx_states[] = {
827 [VMENTER_L1D_FLUSH_AUTO] = "auto", 1050 [VMENTER_L1D_FLUSH_AUTO] = "auto",
828 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable", 1051 [VMENTER_L1D_FLUSH_NEVER] = "vulnerable",
829 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes", 1052 [VMENTER_L1D_FLUSH_COND] = "conditional cache flushes",
@@ -839,13 +1062,14 @@ static ssize_t l1tf_show_state(char *buf)
839 1062
840 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED || 1063 if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
841 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER && 1064 (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
842 cpu_smt_control == CPU_SMT_ENABLED)) 1065 sched_smt_active())) {
843 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG, 1066 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
844 l1tf_vmx_states[l1tf_vmx_mitigation]); 1067 l1tf_vmx_states[l1tf_vmx_mitigation]);
1068 }
845 1069
846 return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG, 1070 return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
847 l1tf_vmx_states[l1tf_vmx_mitigation], 1071 l1tf_vmx_states[l1tf_vmx_mitigation],
848 cpu_smt_control == CPU_SMT_ENABLED ? "vulnerable" : "disabled"); 1072 sched_smt_active() ? "vulnerable" : "disabled");
849} 1073}
850#else 1074#else
851static ssize_t l1tf_show_state(char *buf) 1075static ssize_t l1tf_show_state(char *buf)
@@ -854,11 +1078,39 @@ static ssize_t l1tf_show_state(char *buf)
854} 1078}
855#endif 1079#endif
856 1080
1081static char *stibp_state(void)
1082{
1083 if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
1084 return "";
1085
1086 switch (spectre_v2_user) {
1087 case SPECTRE_V2_USER_NONE:
1088 return ", STIBP: disabled";
1089 case SPECTRE_V2_USER_STRICT:
1090 return ", STIBP: forced";
1091 case SPECTRE_V2_USER_PRCTL:
1092 case SPECTRE_V2_USER_SECCOMP:
1093 if (static_key_enabled(&switch_to_cond_stibp))
1094 return ", STIBP: conditional";
1095 }
1096 return "";
1097}
1098
1099static char *ibpb_state(void)
1100{
1101 if (boot_cpu_has(X86_FEATURE_IBPB)) {
1102 if (static_key_enabled(&switch_mm_always_ibpb))
1103 return ", IBPB: always-on";
1104 if (static_key_enabled(&switch_mm_cond_ibpb))
1105 return ", IBPB: conditional";
1106 return ", IBPB: disabled";
1107 }
1108 return "";
1109}
1110
857static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr, 1111static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
858 char *buf, unsigned int bug) 1112 char *buf, unsigned int bug)
859{ 1113{
860 int ret;
861
862 if (!boot_cpu_has_bug(bug)) 1114 if (!boot_cpu_has_bug(bug))
863 return sprintf(buf, "Not affected\n"); 1115 return sprintf(buf, "Not affected\n");
864 1116
@@ -876,13 +1128,12 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr
876 return sprintf(buf, "Mitigation: __user pointer sanitization\n"); 1128 return sprintf(buf, "Mitigation: __user pointer sanitization\n");
877 1129
878 case X86_BUG_SPECTRE_V2: 1130 case X86_BUG_SPECTRE_V2:
879 ret = sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled], 1131 return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
880 boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "", 1132 ibpb_state(),
881 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "", 1133 boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
882 (x86_spec_ctrl_base & SPEC_CTRL_STIBP) ? ", STIBP" : "", 1134 stibp_state(),
883 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "", 1135 boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
884 spectre_v2_module_string()); 1136 spectre_v2_module_string());
885 return ret;
886 1137
887 case X86_BUG_SPEC_STORE_BYPASS: 1138 case X86_BUG_SPEC_STORE_BYPASS:
888 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]); 1139 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index dd33c357548f..e12454e21b8a 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -56,7 +56,7 @@
56/* Threshold LVT offset is at MSR0xC0000410[15:12] */ 56/* Threshold LVT offset is at MSR0xC0000410[15:12] */
57#define SMCA_THR_LVT_OFF 0xF000 57#define SMCA_THR_LVT_OFF 0xF000
58 58
59static bool thresholding_en; 59static bool thresholding_irq_en;
60 60
61static const char * const th_names[] = { 61static const char * const th_names[] = {
62 "load_store", 62 "load_store",
@@ -534,9 +534,8 @@ prepare_threshold_block(unsigned int bank, unsigned int block, u32 addr,
534 534
535set_offset: 535set_offset:
536 offset = setup_APIC_mce_threshold(offset, new); 536 offset = setup_APIC_mce_threshold(offset, new);
537 537 if (offset == new)
538 if ((offset == new) && (mce_threshold_vector != amd_threshold_interrupt)) 538 thresholding_irq_en = true;
539 mce_threshold_vector = amd_threshold_interrupt;
540 539
541done: 540done:
542 mce_threshold_block_init(&b, offset); 541 mce_threshold_block_init(&b, offset);
@@ -1357,9 +1356,6 @@ int mce_threshold_remove_device(unsigned int cpu)
1357{ 1356{
1358 unsigned int bank; 1357 unsigned int bank;
1359 1358
1360 if (!thresholding_en)
1361 return 0;
1362
1363 for (bank = 0; bank < mca_cfg.banks; ++bank) { 1359 for (bank = 0; bank < mca_cfg.banks; ++bank) {
1364 if (!(per_cpu(bank_map, cpu) & (1 << bank))) 1360 if (!(per_cpu(bank_map, cpu) & (1 << bank)))
1365 continue; 1361 continue;
@@ -1377,9 +1373,6 @@ int mce_threshold_create_device(unsigned int cpu)
1377 struct threshold_bank **bp; 1373 struct threshold_bank **bp;
1378 int err = 0; 1374 int err = 0;
1379 1375
1380 if (!thresholding_en)
1381 return 0;
1382
1383 bp = per_cpu(threshold_banks, cpu); 1376 bp = per_cpu(threshold_banks, cpu);
1384 if (bp) 1377 if (bp)
1385 return 0; 1378 return 0;
@@ -1408,9 +1401,6 @@ static __init int threshold_init_device(void)
1408{ 1401{
1409 unsigned lcpu = 0; 1402 unsigned lcpu = 0;
1410 1403
1411 if (mce_threshold_vector == amd_threshold_interrupt)
1412 thresholding_en = true;
1413
1414 /* to hit CPUs online before the notifier is up */ 1404 /* to hit CPUs online before the notifier is up */
1415 for_each_online_cpu(lcpu) { 1405 for_each_online_cpu(lcpu) {
1416 int err = mce_threshold_create_device(lcpu); 1406 int err = mce_threshold_create_device(lcpu);
@@ -1419,6 +1409,9 @@ static __init int threshold_init_device(void)
1419 return err; 1409 return err;
1420 } 1410 }
1421 1411
1412 if (thresholding_irq_en)
1413 mce_threshold_vector = amd_threshold_interrupt;
1414
1422 return 0; 1415 return 0;
1423} 1416}
1424/* 1417/*
diff --git a/arch/x86/kernel/fpu/signal.c b/arch/x86/kernel/fpu/signal.c
index 61a949d84dfa..d99a8ee9e185 100644
--- a/arch/x86/kernel/fpu/signal.c
+++ b/arch/x86/kernel/fpu/signal.c
@@ -344,10 +344,10 @@ static int __fpu__restore_sig(void __user *buf, void __user *buf_fx, int size)
344 sanitize_restored_xstate(tsk, &env, xfeatures, fx_only); 344 sanitize_restored_xstate(tsk, &env, xfeatures, fx_only);
345 } 345 }
346 346
347 local_bh_disable();
347 fpu->initialized = 1; 348 fpu->initialized = 1;
348 preempt_disable();
349 fpu__restore(fpu); 349 fpu__restore(fpu);
350 preempt_enable(); 350 local_bh_enable();
351 351
352 return err; 352 return err;
353 } else { 353 } else {
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 01ebcb6f263e..7ee8067cbf45 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -994,7 +994,6 @@ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
994{ 994{
995 unsigned long old; 995 unsigned long old;
996 int faulted; 996 int faulted;
997 struct ftrace_graph_ent trace;
998 unsigned long return_hooker = (unsigned long) 997 unsigned long return_hooker = (unsigned long)
999 &return_to_handler; 998 &return_to_handler;
1000 999
@@ -1046,19 +1045,7 @@ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
1046 return; 1045 return;
1047 } 1046 }
1048 1047
1049 trace.func = self_addr; 1048 if (function_graph_enter(old, self_addr, frame_pointer, parent))
1050 trace.depth = current->curr_ret_stack + 1;
1051
1052 /* Only trace if the calling function expects to */
1053 if (!ftrace_graph_entry(&trace)) {
1054 *parent = old; 1049 *parent = old;
1055 return;
1056 }
1057
1058 if (ftrace_push_return_trace(old, self_addr, &trace.depth,
1059 frame_pointer, parent) == -EBUSY) {
1060 *parent = old;
1061 return;
1062 }
1063} 1050}
1064#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 1051#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
diff --git a/arch/x86/kernel/head32.c b/arch/x86/kernel/head32.c
index 76fa3b836598..ec6fefbfd3c0 100644
--- a/arch/x86/kernel/head32.c
+++ b/arch/x86/kernel/head32.c
@@ -37,7 +37,6 @@ asmlinkage __visible void __init i386_start_kernel(void)
37 cr4_init_shadow(); 37 cr4_init_shadow();
38 38
39 sanitize_boot_params(&boot_params); 39 sanitize_boot_params(&boot_params);
40 x86_verify_bootdata_version();
41 40
42 x86_early_init_platform_quirks(); 41 x86_early_init_platform_quirks();
43 42
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 7663a8eb602b..16b1cbd3a61e 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -457,8 +457,6 @@ void __init x86_64_start_reservations(char *real_mode_data)
457 if (!boot_params.hdr.version) 457 if (!boot_params.hdr.version)
458 copy_bootdata(__va(real_mode_data)); 458 copy_bootdata(__va(real_mode_data));
459 459
460 x86_verify_bootdata_version();
461
462 x86_early_init_platform_quirks(); 460 x86_early_init_platform_quirks();
463 461
464 switch (boot_params.hdr.hardware_subarch) { 462 switch (boot_params.hdr.hardware_subarch) {
diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c
index 40b16b270656..6adf6e6c2933 100644
--- a/arch/x86/kernel/kprobes/opt.c
+++ b/arch/x86/kernel/kprobes/opt.c
@@ -189,7 +189,7 @@ static int copy_optimized_instructions(u8 *dest, u8 *src, u8 *real)
189 int len = 0, ret; 189 int len = 0, ret;
190 190
191 while (len < RELATIVEJUMP_SIZE) { 191 while (len < RELATIVEJUMP_SIZE) {
192 ret = __copy_instruction(dest + len, src + len, real, &insn); 192 ret = __copy_instruction(dest + len, src + len, real + len, &insn);
193 if (!ret || !can_boost(&insn, src + len)) 193 if (!ret || !can_boost(&insn, src + len))
194 return -EINVAL; 194 return -EINVAL;
195 len += ret; 195 len += ret;
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index c93fcfdf1673..7d31192296a8 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -40,6 +40,8 @@
40#include <asm/prctl.h> 40#include <asm/prctl.h>
41#include <asm/spec-ctrl.h> 41#include <asm/spec-ctrl.h>
42 42
43#include "process.h"
44
43/* 45/*
44 * per-CPU TSS segments. Threads are completely 'soft' on Linux, 46 * per-CPU TSS segments. Threads are completely 'soft' on Linux,
45 * no more per-task TSS's. The TSS size is kept cacheline-aligned 47 * no more per-task TSS's. The TSS size is kept cacheline-aligned
@@ -252,11 +254,12 @@ void arch_setup_new_exec(void)
252 enable_cpuid(); 254 enable_cpuid();
253} 255}
254 256
255static inline void switch_to_bitmap(struct tss_struct *tss, 257static inline void switch_to_bitmap(struct thread_struct *prev,
256 struct thread_struct *prev,
257 struct thread_struct *next, 258 struct thread_struct *next,
258 unsigned long tifp, unsigned long tifn) 259 unsigned long tifp, unsigned long tifn)
259{ 260{
261 struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
262
260 if (tifn & _TIF_IO_BITMAP) { 263 if (tifn & _TIF_IO_BITMAP) {
261 /* 264 /*
262 * Copy the relevant range of the IO bitmap. 265 * Copy the relevant range of the IO bitmap.
@@ -395,32 +398,85 @@ static __always_inline void amd_set_ssb_virt_state(unsigned long tifn)
395 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, ssbd_tif_to_spec_ctrl(tifn)); 398 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, ssbd_tif_to_spec_ctrl(tifn));
396} 399}
397 400
398static __always_inline void intel_set_ssb_state(unsigned long tifn) 401/*
402 * Update the MSRs managing speculation control, during context switch.
403 *
404 * tifp: Previous task's thread flags
405 * tifn: Next task's thread flags
406 */
407static __always_inline void __speculation_ctrl_update(unsigned long tifp,
408 unsigned long tifn)
399{ 409{
400 u64 msr = x86_spec_ctrl_base | ssbd_tif_to_spec_ctrl(tifn); 410 unsigned long tif_diff = tifp ^ tifn;
411 u64 msr = x86_spec_ctrl_base;
412 bool updmsr = false;
413
414 /*
415 * If TIF_SSBD is different, select the proper mitigation
416 * method. Note that if SSBD mitigation is disabled or permanentely
417 * enabled this branch can't be taken because nothing can set
418 * TIF_SSBD.
419 */
420 if (tif_diff & _TIF_SSBD) {
421 if (static_cpu_has(X86_FEATURE_VIRT_SSBD)) {
422 amd_set_ssb_virt_state(tifn);
423 } else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD)) {
424 amd_set_core_ssb_state(tifn);
425 } else if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
426 static_cpu_has(X86_FEATURE_AMD_SSBD)) {
427 msr |= ssbd_tif_to_spec_ctrl(tifn);
428 updmsr = true;
429 }
430 }
431
432 /*
433 * Only evaluate TIF_SPEC_IB if conditional STIBP is enabled,
434 * otherwise avoid the MSR write.
435 */
436 if (IS_ENABLED(CONFIG_SMP) &&
437 static_branch_unlikely(&switch_to_cond_stibp)) {
438 updmsr |= !!(tif_diff & _TIF_SPEC_IB);
439 msr |= stibp_tif_to_spec_ctrl(tifn);
440 }
401 441
402 wrmsrl(MSR_IA32_SPEC_CTRL, msr); 442 if (updmsr)
443 wrmsrl(MSR_IA32_SPEC_CTRL, msr);
403} 444}
404 445
405static __always_inline void __speculative_store_bypass_update(unsigned long tifn) 446static unsigned long speculation_ctrl_update_tif(struct task_struct *tsk)
406{ 447{
407 if (static_cpu_has(X86_FEATURE_VIRT_SSBD)) 448 if (test_and_clear_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE)) {
408 amd_set_ssb_virt_state(tifn); 449 if (task_spec_ssb_disable(tsk))
409 else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD)) 450 set_tsk_thread_flag(tsk, TIF_SSBD);
410 amd_set_core_ssb_state(tifn); 451 else
411 else 452 clear_tsk_thread_flag(tsk, TIF_SSBD);
412 intel_set_ssb_state(tifn); 453
454 if (task_spec_ib_disable(tsk))
455 set_tsk_thread_flag(tsk, TIF_SPEC_IB);
456 else
457 clear_tsk_thread_flag(tsk, TIF_SPEC_IB);
458 }
459 /* Return the updated threadinfo flags*/
460 return task_thread_info(tsk)->flags;
413} 461}
414 462
415void speculative_store_bypass_update(unsigned long tif) 463void speculation_ctrl_update(unsigned long tif)
416{ 464{
465 /* Forced update. Make sure all relevant TIF flags are different */
417 preempt_disable(); 466 preempt_disable();
418 __speculative_store_bypass_update(tif); 467 __speculation_ctrl_update(~tif, tif);
419 preempt_enable(); 468 preempt_enable();
420} 469}
421 470
422void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, 471/* Called from seccomp/prctl update */
423 struct tss_struct *tss) 472void speculation_ctrl_update_current(void)
473{
474 preempt_disable();
475 speculation_ctrl_update(speculation_ctrl_update_tif(current));
476 preempt_enable();
477}
478
479void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p)
424{ 480{
425 struct thread_struct *prev, *next; 481 struct thread_struct *prev, *next;
426 unsigned long tifp, tifn; 482 unsigned long tifp, tifn;
@@ -430,7 +486,7 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
430 486
431 tifn = READ_ONCE(task_thread_info(next_p)->flags); 487 tifn = READ_ONCE(task_thread_info(next_p)->flags);
432 tifp = READ_ONCE(task_thread_info(prev_p)->flags); 488 tifp = READ_ONCE(task_thread_info(prev_p)->flags);
433 switch_to_bitmap(tss, prev, next, tifp, tifn); 489 switch_to_bitmap(prev, next, tifp, tifn);
434 490
435 propagate_user_return_notify(prev_p, next_p); 491 propagate_user_return_notify(prev_p, next_p);
436 492
@@ -451,8 +507,15 @@ void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p,
451 if ((tifp ^ tifn) & _TIF_NOCPUID) 507 if ((tifp ^ tifn) & _TIF_NOCPUID)
452 set_cpuid_faulting(!!(tifn & _TIF_NOCPUID)); 508 set_cpuid_faulting(!!(tifn & _TIF_NOCPUID));
453 509
454 if ((tifp ^ tifn) & _TIF_SSBD) 510 if (likely(!((tifp | tifn) & _TIF_SPEC_FORCE_UPDATE))) {
455 __speculative_store_bypass_update(tifn); 511 __speculation_ctrl_update(tifp, tifn);
512 } else {
513 speculation_ctrl_update_tif(prev_p);
514 tifn = speculation_ctrl_update_tif(next_p);
515
516 /* Enforce MSR update to ensure consistent state */
517 __speculation_ctrl_update(~tifn, tifn);
518 }
456} 519}
457 520
458/* 521/*
diff --git a/arch/x86/kernel/process.h b/arch/x86/kernel/process.h
new file mode 100644
index 000000000000..898e97cf6629
--- /dev/null
+++ b/arch/x86/kernel/process.h
@@ -0,0 +1,39 @@
1// SPDX-License-Identifier: GPL-2.0
2//
3// Code shared between 32 and 64 bit
4
5#include <asm/spec-ctrl.h>
6
7void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p);
8
9/*
10 * This needs to be inline to optimize for the common case where no extra
11 * work needs to be done.
12 */
13static inline void switch_to_extra(struct task_struct *prev,
14 struct task_struct *next)
15{
16 unsigned long next_tif = task_thread_info(next)->flags;
17 unsigned long prev_tif = task_thread_info(prev)->flags;
18
19 if (IS_ENABLED(CONFIG_SMP)) {
20 /*
21 * Avoid __switch_to_xtra() invocation when conditional
22 * STIPB is disabled and the only different bit is
23 * TIF_SPEC_IB. For CONFIG_SMP=n TIF_SPEC_IB is not
24 * in the TIF_WORK_CTXSW masks.
25 */
26 if (!static_branch_likely(&switch_to_cond_stibp)) {
27 prev_tif &= ~_TIF_SPEC_IB;
28 next_tif &= ~_TIF_SPEC_IB;
29 }
30 }
31
32 /*
33 * __switch_to_xtra() handles debug registers, i/o bitmaps,
34 * speculation mitigations etc.
35 */
36 if (unlikely(next_tif & _TIF_WORK_CTXSW_NEXT ||
37 prev_tif & _TIF_WORK_CTXSW_PREV))
38 __switch_to_xtra(prev, next);
39}
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 5046a3c9dec2..d3e593eb189f 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -59,6 +59,8 @@
59#include <asm/intel_rdt_sched.h> 59#include <asm/intel_rdt_sched.h>
60#include <asm/proto.h> 60#include <asm/proto.h>
61 61
62#include "process.h"
63
62void __show_regs(struct pt_regs *regs, enum show_regs_mode mode) 64void __show_regs(struct pt_regs *regs, enum show_regs_mode mode)
63{ 65{
64 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L; 66 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L;
@@ -232,7 +234,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
232 struct fpu *prev_fpu = &prev->fpu; 234 struct fpu *prev_fpu = &prev->fpu;
233 struct fpu *next_fpu = &next->fpu; 235 struct fpu *next_fpu = &next->fpu;
234 int cpu = smp_processor_id(); 236 int cpu = smp_processor_id();
235 struct tss_struct *tss = &per_cpu(cpu_tss_rw, cpu);
236 237
237 /* never put a printk in __switch_to... printk() calls wake_up*() indirectly */ 238 /* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
238 239
@@ -264,12 +265,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
264 if (get_kernel_rpl() && unlikely(prev->iopl != next->iopl)) 265 if (get_kernel_rpl() && unlikely(prev->iopl != next->iopl))
265 set_iopl_mask(next->iopl); 266 set_iopl_mask(next->iopl);
266 267
267 /* 268 switch_to_extra(prev_p, next_p);
268 * Now maybe handle debug registers and/or IO bitmaps
269 */
270 if (unlikely(task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV ||
271 task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT))
272 __switch_to_xtra(prev_p, next_p, tss);
273 269
274 /* 270 /*
275 * Leave lazy mode, flushing any hypercalls made here. 271 * Leave lazy mode, flushing any hypercalls made here.
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 0e0b4288a4b2..bbfbf017065c 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -60,6 +60,8 @@
60#include <asm/unistd_32_ia32.h> 60#include <asm/unistd_32_ia32.h>
61#endif 61#endif
62 62
63#include "process.h"
64
63/* Prints also some state that isn't saved in the pt_regs */ 65/* Prints also some state that isn't saved in the pt_regs */
64void __show_regs(struct pt_regs *regs, enum show_regs_mode mode) 66void __show_regs(struct pt_regs *regs, enum show_regs_mode mode)
65{ 67{
@@ -553,7 +555,6 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
553 struct fpu *prev_fpu = &prev->fpu; 555 struct fpu *prev_fpu = &prev->fpu;
554 struct fpu *next_fpu = &next->fpu; 556 struct fpu *next_fpu = &next->fpu;
555 int cpu = smp_processor_id(); 557 int cpu = smp_processor_id();
556 struct tss_struct *tss = &per_cpu(cpu_tss_rw, cpu);
557 558
558 WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) && 559 WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
559 this_cpu_read(irq_count) != -1); 560 this_cpu_read(irq_count) != -1);
@@ -617,12 +618,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
617 /* Reload sp0. */ 618 /* Reload sp0. */
618 update_task_stack(next_p); 619 update_task_stack(next_p);
619 620
620 /* 621 switch_to_extra(prev_p, next_p);
621 * Now maybe reload the debug registers and handle I/O bitmaps
622 */
623 if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
624 task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
625 __switch_to_xtra(prev_p, next_p, tss);
626 622
627#ifdef CONFIG_XEN_PV 623#ifdef CONFIG_XEN_PV
628 /* 624 /*
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index b74e7bfed6ab..d494b9bfe618 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -1280,23 +1280,6 @@ void __init setup_arch(char **cmdline_p)
1280 unwind_init(); 1280 unwind_init();
1281} 1281}
1282 1282
1283/*
1284 * From boot protocol 2.14 onwards we expect the bootloader to set the
1285 * version to "0x8000 | <used version>". In case we find a version >= 2.14
1286 * without the 0x8000 we assume the boot loader supports 2.13 only and
1287 * reset the version accordingly. The 0x8000 flag is removed in any case.
1288 */
1289void __init x86_verify_bootdata_version(void)
1290{
1291 if (boot_params.hdr.version & VERSION_WRITTEN)
1292 boot_params.hdr.version &= ~VERSION_WRITTEN;
1293 else if (boot_params.hdr.version >= 0x020e)
1294 boot_params.hdr.version = 0x020d;
1295
1296 if (boot_params.hdr.version < 0x020e)
1297 boot_params.hdr.acpi_rsdp_addr = 0;
1298}
1299
1300#ifdef CONFIG_X86_32 1283#ifdef CONFIG_X86_32
1301 1284
1302static struct resource video_ram_resource = { 1285static struct resource video_ram_resource = {
diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c
index a2486f444073..6e5ef8fb8a02 100644
--- a/arch/x86/kernel/tboot.c
+++ b/arch/x86/kernel/tboot.c
@@ -19,7 +19,7 @@
19 * 19 *
20 */ 20 */
21 21
22#include <linux/dma_remapping.h> 22#include <linux/intel-iommu.h>
23#include <linux/init_task.h> 23#include <linux/init_task.h>
24#include <linux/spinlock.h> 24#include <linux/spinlock.h>
25#include <linux/export.h> 25#include <linux/export.h>
generated by cgit v1.2.2 (git 2.25.0) at 2026-01-06 02:15:43 -0500