aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/include
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/include')
-rw-r--r--arch/x86/include/asm/alternative-asm.h43
-rw-r--r--arch/x86/include/asm/alternative.h65
-rw-r--r--arch/x86/include/asm/cpufeature.h22
-rw-r--r--arch/x86/include/asm/smap.h4
4 files changed, 100 insertions, 34 deletions
diff --git a/arch/x86/include/asm/alternative-asm.h b/arch/x86/include/asm/alternative-asm.h
index 372231c22a47..524bddce0b76 100644
--- a/arch/x86/include/asm/alternative-asm.h
+++ b/arch/x86/include/asm/alternative-asm.h
@@ -18,12 +18,53 @@
18 .endm 18 .endm
19#endif 19#endif
20 20
21.macro altinstruction_entry orig alt feature orig_len alt_len 21.macro altinstruction_entry orig alt feature orig_len alt_len pad_len
22 .long \orig - . 22 .long \orig - .
23 .long \alt - . 23 .long \alt - .
24 .word \feature 24 .word \feature
25 .byte \orig_len 25 .byte \orig_len
26 .byte \alt_len 26 .byte \alt_len
27 .byte \pad_len
28.endm
29
30.macro ALTERNATIVE oldinstr, newinstr, feature
31140:
32 \oldinstr
33141:
34 .skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
35142:
36
37 .pushsection .altinstructions,"a"
38 altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f,142b-141b
39 .popsection
40
41 .pushsection .altinstr_replacement,"ax"
42143:
43 \newinstr
44144:
45 .popsection
46.endm
47
48.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2
49140:
50 \oldinstr
51141:
52 .skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
53 .skip -(((145f-144f)-(144f-143f)-(141b-140b)) > 0) * ((145f-144f)-(144f-143f)-(141b-140b)),0x90
54142:
55
56 .pushsection .altinstructions,"a"
57 altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f,142b-141b
58 altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f,142b-141b
59 .popsection
60
61 .pushsection .altinstr_replacement,"ax"
62143:
63 \newinstr1
64144:
65 \newinstr2
66145:
67 .popsection
27.endm 68.endm
28 69
29#endif /* __ASSEMBLY__ */ 70#endif /* __ASSEMBLY__ */
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 473bdbee378a..5aef6a97d80e 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -48,8 +48,9 @@ struct alt_instr {
48 s32 repl_offset; /* offset to replacement instruction */ 48 s32 repl_offset; /* offset to replacement instruction */
49 u16 cpuid; /* cpuid bit set for replacement */ 49 u16 cpuid; /* cpuid bit set for replacement */
50 u8 instrlen; /* length of original instruction */ 50 u8 instrlen; /* length of original instruction */
51 u8 replacementlen; /* length of new instruction, <= instrlen */ 51 u8 replacementlen; /* length of new instruction */
52}; 52 u8 padlen; /* length of build-time padding */
53} __packed;
53 54
54extern void alternative_instructions(void); 55extern void alternative_instructions(void);
55extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); 56extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
@@ -76,50 +77,61 @@ static inline int alternatives_text_reserved(void *start, void *end)
76} 77}
77#endif /* CONFIG_SMP */ 78#endif /* CONFIG_SMP */
78 79
79#define OLDINSTR(oldinstr) "661:\n\t" oldinstr "\n662:\n" 80#define b_replacement(num) "664"#num
81#define e_replacement(num) "665"#num
80 82
81#define b_replacement(number) "663"#number 83#define alt_end_marker "663"
82#define e_replacement(number) "664"#number 84#define alt_slen "662b-661b"
85#define alt_pad_len alt_end_marker"b-662b"
86#define alt_total_slen alt_end_marker"b-661b"
87#define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f"
83 88
84#define alt_slen "662b-661b" 89#define __OLDINSTR(oldinstr, num) \
85#define alt_rlen(number) e_replacement(number)"f-"b_replacement(number)"f" 90 "661:\n\t" oldinstr "\n662:\n" \
91 ".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \
92 "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"
86 93
87#define ALTINSTR_ENTRY(feature, number) \ 94#define OLDINSTR(oldinstr, num) \
95 __OLDINSTR(oldinstr, num) \
96 alt_end_marker ":\n"
97
98/*
99 * Pad the second replacement alternative with additional NOPs if it is
100 * additionally longer than the first replacement alternative.
101 */
102#define OLDINSTR_2(oldinstr, num1, num2) \
103 __OLDINSTR(oldinstr, num1) \
104 ".skip -(((" alt_rlen(num2) ")-(" alt_rlen(num1) ")-(662b-661b)) > 0) * " \
105 "((" alt_rlen(num2) ")-(" alt_rlen(num1) ")-(662b-661b)),0x90\n" \
106 alt_end_marker ":\n"
107
108#define ALTINSTR_ENTRY(feature, num) \
88 " .long 661b - .\n" /* label */ \ 109 " .long 661b - .\n" /* label */ \
89 " .long " b_replacement(number)"f - .\n" /* new instruction */ \ 110 " .long " b_replacement(num)"f - .\n" /* new instruction */ \
90 " .word " __stringify(feature) "\n" /* feature bit */ \ 111 " .word " __stringify(feature) "\n" /* feature bit */ \
91 " .byte " alt_slen "\n" /* source len */ \ 112 " .byte " alt_total_slen "\n" /* source len */ \
92 " .byte " alt_rlen(number) "\n" /* replacement len */ 113 " .byte " alt_rlen(num) "\n" /* replacement len */ \
114 " .byte " alt_pad_len "\n" /* pad len */
93 115
94#define DISCARD_ENTRY(number) /* rlen <= slen */ \ 116#define ALTINSTR_REPLACEMENT(newinstr, feature, num) /* replacement */ \
95 " .byte 0xff + (" alt_rlen(number) ") - (" alt_slen ")\n" 117 b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t"
96
97#define ALTINSTR_REPLACEMENT(newinstr, feature, number) /* replacement */ \
98 b_replacement(number)":\n\t" newinstr "\n" e_replacement(number) ":\n\t"
99 118
100/* alternative assembly primitive: */ 119/* alternative assembly primitive: */
101#define ALTERNATIVE(oldinstr, newinstr, feature) \ 120#define ALTERNATIVE(oldinstr, newinstr, feature) \
102 OLDINSTR(oldinstr) \ 121 OLDINSTR(oldinstr, 1) \
103 ".pushsection .altinstructions,\"a\"\n" \ 122 ".pushsection .altinstructions,\"a\"\n" \
104 ALTINSTR_ENTRY(feature, 1) \ 123 ALTINSTR_ENTRY(feature, 1) \
105 ".popsection\n" \ 124 ".popsection\n" \
106 ".pushsection .discard,\"aw\",@progbits\n" \
107 DISCARD_ENTRY(1) \
108 ".popsection\n" \
109 ".pushsection .altinstr_replacement, \"ax\"\n" \ 125 ".pushsection .altinstr_replacement, \"ax\"\n" \
110 ALTINSTR_REPLACEMENT(newinstr, feature, 1) \ 126 ALTINSTR_REPLACEMENT(newinstr, feature, 1) \
111 ".popsection" 127 ".popsection"
112 128
113#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ 129#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
114 OLDINSTR(oldinstr) \ 130 OLDINSTR_2(oldinstr, 1, 2) \
115 ".pushsection .altinstructions,\"a\"\n" \ 131 ".pushsection .altinstructions,\"a\"\n" \
116 ALTINSTR_ENTRY(feature1, 1) \ 132 ALTINSTR_ENTRY(feature1, 1) \
117 ALTINSTR_ENTRY(feature2, 2) \ 133 ALTINSTR_ENTRY(feature2, 2) \
118 ".popsection\n" \ 134 ".popsection\n" \
119 ".pushsection .discard,\"aw\",@progbits\n" \
120 DISCARD_ENTRY(1) \
121 DISCARD_ENTRY(2) \
122 ".popsection\n" \
123 ".pushsection .altinstr_replacement, \"ax\"\n" \ 135 ".pushsection .altinstr_replacement, \"ax\"\n" \
124 ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \ 136 ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \
125 ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \ 137 ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \
@@ -146,6 +158,9 @@ static inline int alternatives_text_reserved(void *start, void *end)
146#define alternative(oldinstr, newinstr, feature) \ 158#define alternative(oldinstr, newinstr, feature) \
147 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory") 159 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
148 160
161#define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \
162 asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")
163
149/* 164/*
150 * Alternative inline assembly with input. 165 * Alternative inline assembly with input.
151 * 166 *
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 90a54851aedc..9b1df43dadbb 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -418,6 +418,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit)
418 " .word %P0\n" /* 1: do replace */ 418 " .word %P0\n" /* 1: do replace */
419 " .byte 2b - 1b\n" /* source len */ 419 " .byte 2b - 1b\n" /* source len */
420 " .byte 0\n" /* replacement len */ 420 " .byte 0\n" /* replacement len */
421 " .byte 0\n" /* pad len */
421 ".previous\n" 422 ".previous\n"
422 /* skipping size check since replacement size = 0 */ 423 /* skipping size check since replacement size = 0 */
423 : : "i" (X86_FEATURE_ALWAYS) : : t_warn); 424 : : "i" (X86_FEATURE_ALWAYS) : : t_warn);
@@ -432,6 +433,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit)
432 " .word %P0\n" /* feature bit */ 433 " .word %P0\n" /* feature bit */
433 " .byte 2b - 1b\n" /* source len */ 434 " .byte 2b - 1b\n" /* source len */
434 " .byte 0\n" /* replacement len */ 435 " .byte 0\n" /* replacement len */
436 " .byte 0\n" /* pad len */
435 ".previous\n" 437 ".previous\n"
436 /* skipping size check since replacement size = 0 */ 438 /* skipping size check since replacement size = 0 */
437 : : "i" (bit) : : t_no); 439 : : "i" (bit) : : t_no);
@@ -457,6 +459,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit)
457 " .word %P1\n" /* feature bit */ 459 " .word %P1\n" /* feature bit */
458 " .byte 2b - 1b\n" /* source len */ 460 " .byte 2b - 1b\n" /* source len */
459 " .byte 4f - 3f\n" /* replacement len */ 461 " .byte 4f - 3f\n" /* replacement len */
462 " .byte 0\n" /* pad len */
460 ".previous\n" 463 ".previous\n"
461 ".section .discard,\"aw\",@progbits\n" 464 ".section .discard,\"aw\",@progbits\n"
462 " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */ 465 " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
@@ -491,23 +494,28 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
491 */ 494 */
492 asm_volatile_goto("1: .byte 0xe9\n .long %l[t_dynamic] - 2f\n" 495 asm_volatile_goto("1: .byte 0xe9\n .long %l[t_dynamic] - 2f\n"
493 "2:\n" 496 "2:\n"
497 ".skip -(((5f-4f) - (2b-1b)) > 0) * "
498 "((5f-4f) - (2b-1b)),0x90\n"
499 "3:\n"
494 ".section .altinstructions,\"a\"\n" 500 ".section .altinstructions,\"a\"\n"
495 " .long 1b - .\n" /* src offset */ 501 " .long 1b - .\n" /* src offset */
496 " .long 3f - .\n" /* repl offset */ 502 " .long 4f - .\n" /* repl offset */
497 " .word %P1\n" /* always replace */ 503 " .word %P1\n" /* always replace */
498 " .byte 2b - 1b\n" /* src len */ 504 " .byte 3b - 1b\n" /* src len */
499 " .byte 4f - 3f\n" /* repl len */ 505 " .byte 5f - 4f\n" /* repl len */
506 " .byte 3b - 2b\n" /* pad len */
500 ".previous\n" 507 ".previous\n"
501 ".section .altinstr_replacement,\"ax\"\n" 508 ".section .altinstr_replacement,\"ax\"\n"
502 "3: .byte 0xe9\n .long %l[t_no] - 2b\n" 509 "4: .byte 0xe9\n .long %l[t_no] - 2b\n"
503 "4:\n" 510 "5:\n"
504 ".previous\n" 511 ".previous\n"
505 ".section .altinstructions,\"a\"\n" 512 ".section .altinstructions,\"a\"\n"
506 " .long 1b - .\n" /* src offset */ 513 " .long 1b - .\n" /* src offset */
507 " .long 0\n" /* no replacement */ 514 " .long 0\n" /* no replacement */
508 " .word %P0\n" /* feature bit */ 515 " .word %P0\n" /* feature bit */
509 " .byte 2b - 1b\n" /* src len */ 516 " .byte 3b - 1b\n" /* src len */
510 " .byte 0\n" /* repl len */ 517 " .byte 0\n" /* repl len */
518 " .byte 0\n" /* pad len */
511 ".previous\n" 519 ".previous\n"
512 : : "i" (bit), "i" (X86_FEATURE_ALWAYS) 520 : : "i" (bit), "i" (X86_FEATURE_ALWAYS)
513 : : t_dynamic, t_no); 521 : : t_dynamic, t_no);
@@ -527,6 +535,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
527 " .word %P2\n" /* always replace */ 535 " .word %P2\n" /* always replace */
528 " .byte 2b - 1b\n" /* source len */ 536 " .byte 2b - 1b\n" /* source len */
529 " .byte 4f - 3f\n" /* replacement len */ 537 " .byte 4f - 3f\n" /* replacement len */
538 " .byte 0\n" /* pad len */
530 ".previous\n" 539 ".previous\n"
531 ".section .discard,\"aw\",@progbits\n" 540 ".section .discard,\"aw\",@progbits\n"
532 " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */ 541 " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
@@ -541,6 +550,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
541 " .word %P1\n" /* feature bit */ 550 " .word %P1\n" /* feature bit */
542 " .byte 4b - 3b\n" /* src len */ 551 " .byte 4b - 3b\n" /* src len */
543 " .byte 6f - 5f\n" /* repl len */ 552 " .byte 6f - 5f\n" /* repl len */
553 " .byte 0\n" /* pad len */
544 ".previous\n" 554 ".previous\n"
545 ".section .discard,\"aw\",@progbits\n" 555 ".section .discard,\"aw\",@progbits\n"
546 " .byte 0xff + (6f-5f) - (4b-3b)\n" /* size check */ 556 " .byte 0xff + (6f-5f) - (4b-3b)\n" /* size check */
diff --git a/arch/x86/include/asm/smap.h b/arch/x86/include/asm/smap.h
index 8d3120f4e270..c56cb4f37be9 100644
--- a/arch/x86/include/asm/smap.h
+++ b/arch/x86/include/asm/smap.h
@@ -33,7 +33,7 @@
33 662: __ASM_CLAC ; \ 33 662: __ASM_CLAC ; \
34 .popsection ; \ 34 .popsection ; \
35 .pushsection .altinstructions, "a" ; \ 35 .pushsection .altinstructions, "a" ; \
36 altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3 ; \ 36 altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3, 0 ; \
37 .popsection 37 .popsection
38 38
39#define ASM_STAC \ 39#define ASM_STAC \
@@ -42,7 +42,7 @@
42 662: __ASM_STAC ; \ 42 662: __ASM_STAC ; \
43 .popsection ; \ 43 .popsection ; \
44 .pushsection .altinstructions, "a" ; \ 44 .pushsection .altinstructions, "a" ; \
45 altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3 ; \ 45 altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3, 0 ; \
46 .popsection 46 .popsection
47 47
48#else /* CONFIG_X86_SMAP */ 48#else /* CONFIG_X86_SMAP */
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-20 14:51:01 -0500