aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/x86/include/asm/alternative-asm.h43
-rw-r--r--arch/x86/include/asm/alternative.h65
-rw-r--r--arch/x86/include/asm/cpufeature.h22
-rw-r--r--arch/x86/include/asm/smap.h4
-rw-r--r--arch/x86/kernel/alternative.c6
-rw-r--r--arch/x86/kernel/entry_32.S2
-rw-r--r--arch/x86/lib/clear_page_64.S4
-rw-r--r--arch/x86/lib/copy_page_64.S2
-rw-r--r--arch/x86/lib/copy_user_64.S4
-rw-r--r--arch/x86/lib/memcpy_64.S4
-rw-r--r--arch/x86/lib/memmove_64.S2
-rw-r--r--arch/x86/lib/memset_64.S4
12 files changed, 114 insertions, 48 deletions
diff --git a/arch/x86/include/asm/alternative-asm.h b/arch/x86/include/asm/alternative-asm.h
index 372231c22a47..524bddce0b76 100644
--- a/arch/x86/include/asm/alternative-asm.h
+++ b/arch/x86/include/asm/alternative-asm.h
@@ -18,12 +18,53 @@
18 .endm 18 .endm
19#endif 19#endif
20 20
21.macro altinstruction_entry orig alt feature orig_len alt_len 21.macro altinstruction_entry orig alt feature orig_len alt_len pad_len
22 .long \orig - . 22 .long \orig - .
23 .long \alt - . 23 .long \alt - .
24 .word \feature 24 .word \feature
25 .byte \orig_len 25 .byte \orig_len
26 .byte \alt_len 26 .byte \alt_len
27 .byte \pad_len
28.endm
29
30.macro ALTERNATIVE oldinstr, newinstr, feature
31140:
32 \oldinstr
33141:
34 .skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
35142:
36
37 .pushsection .altinstructions,"a"
38 altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f,142b-141b
39 .popsection
40
41 .pushsection .altinstr_replacement,"ax"
42143:
43 \newinstr
44144:
45 .popsection
46.endm
47
48.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2
49140:
50 \oldinstr
51141:
52 .skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
53 .skip -(((145f-144f)-(144f-143f)-(141b-140b)) > 0) * ((145f-144f)-(144f-143f)-(141b-140b)),0x90
54142:
55
56 .pushsection .altinstructions,"a"
57 altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f,142b-141b
58 altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f,142b-141b
59 .popsection
60
61 .pushsection .altinstr_replacement,"ax"
62143:
63 \newinstr1
64144:
65 \newinstr2
66145:
67 .popsection
27.endm 68.endm
28 69
29#endif /* __ASSEMBLY__ */ 70#endif /* __ASSEMBLY__ */
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 473bdbee378a..5aef6a97d80e 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -48,8 +48,9 @@ struct alt_instr {
48 s32 repl_offset; /* offset to replacement instruction */ 48 s32 repl_offset; /* offset to replacement instruction */
49 u16 cpuid; /* cpuid bit set for replacement */ 49 u16 cpuid; /* cpuid bit set for replacement */
50 u8 instrlen; /* length of original instruction */ 50 u8 instrlen; /* length of original instruction */
51 u8 replacementlen; /* length of new instruction, <= instrlen */ 51 u8 replacementlen; /* length of new instruction */
52}; 52 u8 padlen; /* length of build-time padding */
53} __packed;
53 54
54extern void alternative_instructions(void); 55extern void alternative_instructions(void);
55extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); 56extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
@@ -76,50 +77,61 @@ static inline int alternatives_text_reserved(void *start, void *end)
76} 77}
77#endif /* CONFIG_SMP */ 78#endif /* CONFIG_SMP */
78 79
79#define OLDINSTR(oldinstr) "661:\n\t" oldinstr "\n662:\n" 80#define b_replacement(num) "664"#num
81#define e_replacement(num) "665"#num
80 82
81#define b_replacement(number) "663"#number 83#define alt_end_marker "663"
82#define e_replacement(number) "664"#number 84#define alt_slen "662b-661b"
85#define alt_pad_len alt_end_marker"b-662b"
86#define alt_total_slen alt_end_marker"b-661b"
87#define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f"
83 88
84#define alt_slen "662b-661b" 89#define __OLDINSTR(oldinstr, num) \
85#define alt_rlen(number) e_replacement(number)"f-"b_replacement(number)"f" 90 "661:\n\t" oldinstr "\n662:\n" \
91 ".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \
92 "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"
86 93
87#define ALTINSTR_ENTRY(feature, number) \ 94#define OLDINSTR(oldinstr, num) \
95 __OLDINSTR(oldinstr, num) \
96 alt_end_marker ":\n"
97
98/*
99 * Pad the second replacement alternative with additional NOPs if it is
100 * additionally longer than the first replacement alternative.
101 */
102#define OLDINSTR_2(oldinstr, num1, num2) \
103 __OLDINSTR(oldinstr, num1) \
104 ".skip -(((" alt_rlen(num2) ")-(" alt_rlen(num1) ")-(662b-661b)) > 0) * " \
105 "((" alt_rlen(num2) ")-(" alt_rlen(num1) ")-(662b-661b)),0x90\n" \
106 alt_end_marker ":\n"
107
108#define ALTINSTR_ENTRY(feature, num) \
88 " .long 661b - .\n" /* label */ \ 109 " .long 661b - .\n" /* label */ \
89 " .long " b_replacement(number)"f - .\n" /* new instruction */ \ 110 " .long " b_replacement(num)"f - .\n" /* new instruction */ \
90 " .word " __stringify(feature) "\n" /* feature bit */ \ 111 " .word " __stringify(feature) "\n" /* feature bit */ \
91 " .byte " alt_slen "\n" /* source len */ \ 112 " .byte " alt_total_slen "\n" /* source len */ \
92 " .byte " alt_rlen(number) "\n" /* replacement len */ 113 " .byte " alt_rlen(num) "\n" /* replacement len */ \
114 " .byte " alt_pad_len "\n" /* pad len */
93 115
94#define DISCARD_ENTRY(number) /* rlen <= slen */ \ 116#define ALTINSTR_REPLACEMENT(newinstr, feature, num) /* replacement */ \
95 " .byte 0xff + (" alt_rlen(number) ") - (" alt_slen ")\n" 117 b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t"
96
97#define ALTINSTR_REPLACEMENT(newinstr, feature, number) /* replacement */ \
98 b_replacement(number)":\n\t" newinstr "\n" e_replacement(number) ":\n\t"
99 118
100/* alternative assembly primitive: */ 119/* alternative assembly primitive: */
101#define ALTERNATIVE(oldinstr, newinstr, feature) \ 120#define ALTERNATIVE(oldinstr, newinstr, feature) \
102 OLDINSTR(oldinstr) \ 121 OLDINSTR(oldinstr, 1) \
103 ".pushsection .altinstructions,\"a\"\n" \ 122 ".pushsection .altinstructions,\"a\"\n" \
104 ALTINSTR_ENTRY(feature, 1) \ 123 ALTINSTR_ENTRY(feature, 1) \
105 ".popsection\n" \ 124 ".popsection\n" \
106 ".pushsection .discard,\"aw\",@progbits\n" \
107 DISCARD_ENTRY(1) \
108 ".popsection\n" \
109 ".pushsection .altinstr_replacement, \"ax\"\n" \ 125 ".pushsection .altinstr_replacement, \"ax\"\n" \
110 ALTINSTR_REPLACEMENT(newinstr, feature, 1) \ 126 ALTINSTR_REPLACEMENT(newinstr, feature, 1) \
111 ".popsection" 127 ".popsection"
112 128
113#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ 129#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
114 OLDINSTR(oldinstr) \ 130 OLDINSTR_2(oldinstr, 1, 2) \
115 ".pushsection .altinstructions,\"a\"\n" \ 131 ".pushsection .altinstructions,\"a\"\n" \
116 ALTINSTR_ENTRY(feature1, 1) \ 132 ALTINSTR_ENTRY(feature1, 1) \
117 ALTINSTR_ENTRY(feature2, 2) \ 133 ALTINSTR_ENTRY(feature2, 2) \
118 ".popsection\n" \ 134 ".popsection\n" \
119 ".pushsection .discard,\"aw\",@progbits\n" \
120 DISCARD_ENTRY(1) \
121 DISCARD_ENTRY(2) \
122 ".popsection\n" \
123 ".pushsection .altinstr_replacement, \"ax\"\n" \ 135 ".pushsection .altinstr_replacement, \"ax\"\n" \
124 ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \ 136 ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \
125 ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \ 137 ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \
@@ -146,6 +158,9 @@ static inline int alternatives_text_reserved(void *start, void *end)
146#define alternative(oldinstr, newinstr, feature) \ 158#define alternative(oldinstr, newinstr, feature) \
147 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory") 159 asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
148 160
161#define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \
162 asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")
163
149/* 164/*
150 * Alternative inline assembly with input. 165 * Alternative inline assembly with input.
151 * 166 *
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 90a54851aedc..9b1df43dadbb 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -418,6 +418,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit)
418 " .word %P0\n" /* 1: do replace */ 418 " .word %P0\n" /* 1: do replace */
419 " .byte 2b - 1b\n" /* source len */ 419 " .byte 2b - 1b\n" /* source len */
420 " .byte 0\n" /* replacement len */ 420 " .byte 0\n" /* replacement len */
421 " .byte 0\n" /* pad len */
421 ".previous\n" 422 ".previous\n"
422 /* skipping size check since replacement size = 0 */ 423 /* skipping size check since replacement size = 0 */
423 : : "i" (X86_FEATURE_ALWAYS) : : t_warn); 424 : : "i" (X86_FEATURE_ALWAYS) : : t_warn);
@@ -432,6 +433,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit)
432 " .word %P0\n" /* feature bit */ 433 " .word %P0\n" /* feature bit */
433 " .byte 2b - 1b\n" /* source len */ 434 " .byte 2b - 1b\n" /* source len */
434 " .byte 0\n" /* replacement len */ 435 " .byte 0\n" /* replacement len */
436 " .byte 0\n" /* pad len */
435 ".previous\n" 437 ".previous\n"
436 /* skipping size check since replacement size = 0 */ 438 /* skipping size check since replacement size = 0 */
437 : : "i" (bit) : : t_no); 439 : : "i" (bit) : : t_no);
@@ -457,6 +459,7 @@ static __always_inline __pure bool __static_cpu_has(u16 bit)
457 " .word %P1\n" /* feature bit */ 459 " .word %P1\n" /* feature bit */
458 " .byte 2b - 1b\n" /* source len */ 460 " .byte 2b - 1b\n" /* source len */
459 " .byte 4f - 3f\n" /* replacement len */ 461 " .byte 4f - 3f\n" /* replacement len */
462 " .byte 0\n" /* pad len */
460 ".previous\n" 463 ".previous\n"
461 ".section .discard,\"aw\",@progbits\n" 464 ".section .discard,\"aw\",@progbits\n"
462 " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */ 465 " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
@@ -491,23 +494,28 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
491 */ 494 */
492 asm_volatile_goto("1: .byte 0xe9\n .long %l[t_dynamic] - 2f\n" 495 asm_volatile_goto("1: .byte 0xe9\n .long %l[t_dynamic] - 2f\n"
493 "2:\n" 496 "2:\n"
497 ".skip -(((5f-4f) - (2b-1b)) > 0) * "
498 "((5f-4f) - (2b-1b)),0x90\n"
499 "3:\n"
494 ".section .altinstructions,\"a\"\n" 500 ".section .altinstructions,\"a\"\n"
495 " .long 1b - .\n" /* src offset */ 501 " .long 1b - .\n" /* src offset */
496 " .long 3f - .\n" /* repl offset */ 502 " .long 4f - .\n" /* repl offset */
497 " .word %P1\n" /* always replace */ 503 " .word %P1\n" /* always replace */
498 " .byte 2b - 1b\n" /* src len */ 504 " .byte 3b - 1b\n" /* src len */
499 " .byte 4f - 3f\n" /* repl len */ 505 " .byte 5f - 4f\n" /* repl len */
506 " .byte 3b - 2b\n" /* pad len */
500 ".previous\n" 507 ".previous\n"
501 ".section .altinstr_replacement,\"ax\"\n" 508 ".section .altinstr_replacement,\"ax\"\n"
502 "3: .byte 0xe9\n .long %l[t_no] - 2b\n" 509 "4: .byte 0xe9\n .long %l[t_no] - 2b\n"
503 "4:\n" 510 "5:\n"
504 ".previous\n" 511 ".previous\n"
505 ".section .altinstructions,\"a\"\n" 512 ".section .altinstructions,\"a\"\n"
506 " .long 1b - .\n" /* src offset */ 513 " .long 1b - .\n" /* src offset */
507 " .long 0\n" /* no replacement */ 514 " .long 0\n" /* no replacement */
508 " .word %P0\n" /* feature bit */ 515 " .word %P0\n" /* feature bit */
509 " .byte 2b - 1b\n" /* src len */ 516 " .byte 3b - 1b\n" /* src len */
510 " .byte 0\n" /* repl len */ 517 " .byte 0\n" /* repl len */
518 " .byte 0\n" /* pad len */
511 ".previous\n" 519 ".previous\n"
512 : : "i" (bit), "i" (X86_FEATURE_ALWAYS) 520 : : "i" (bit), "i" (X86_FEATURE_ALWAYS)
513 : : t_dynamic, t_no); 521 : : t_dynamic, t_no);
@@ -527,6 +535,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
527 " .word %P2\n" /* always replace */ 535 " .word %P2\n" /* always replace */
528 " .byte 2b - 1b\n" /* source len */ 536 " .byte 2b - 1b\n" /* source len */
529 " .byte 4f - 3f\n" /* replacement len */ 537 " .byte 4f - 3f\n" /* replacement len */
538 " .byte 0\n" /* pad len */
530 ".previous\n" 539 ".previous\n"
531 ".section .discard,\"aw\",@progbits\n" 540 ".section .discard,\"aw\",@progbits\n"
532 " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */ 541 " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
@@ -541,6 +550,7 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
541 " .word %P1\n" /* feature bit */ 550 " .word %P1\n" /* feature bit */
542 " .byte 4b - 3b\n" /* src len */ 551 " .byte 4b - 3b\n" /* src len */
543 " .byte 6f - 5f\n" /* repl len */ 552 " .byte 6f - 5f\n" /* repl len */
553 " .byte 0\n" /* pad len */
544 ".previous\n" 554 ".previous\n"
545 ".section .discard,\"aw\",@progbits\n" 555 ".section .discard,\"aw\",@progbits\n"
546 " .byte 0xff + (6f-5f) - (4b-3b)\n" /* size check */ 556 " .byte 0xff + (6f-5f) - (4b-3b)\n" /* size check */
diff --git a/arch/x86/include/asm/smap.h b/arch/x86/include/asm/smap.h
index 8d3120f4e270..c56cb4f37be9 100644
--- a/arch/x86/include/asm/smap.h
+++ b/arch/x86/include/asm/smap.h
@@ -33,7 +33,7 @@
33 662: __ASM_CLAC ; \ 33 662: __ASM_CLAC ; \
34 .popsection ; \ 34 .popsection ; \
35 .pushsection .altinstructions, "a" ; \ 35 .pushsection .altinstructions, "a" ; \
36 altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3 ; \ 36 altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3, 0 ; \
37 .popsection 37 .popsection
38 38
39#define ASM_STAC \ 39#define ASM_STAC \
@@ -42,7 +42,7 @@
42 662: __ASM_STAC ; \ 42 662: __ASM_STAC ; \
43 .popsection ; \ 43 .popsection ; \
44 .pushsection .altinstructions, "a" ; \ 44 .pushsection .altinstructions, "a" ; \
45 altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3 ; \ 45 altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3, 0 ; \
46 .popsection 46 .popsection
47 47
48#else /* CONFIG_X86_SMAP */ 48#else /* CONFIG_X86_SMAP */
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 1e86e85bcf58..c99b0f13a90e 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -270,7 +270,6 @@ void __init_or_module apply_alternatives(struct alt_instr *start,
270 for (a = start; a < end; a++) { 270 for (a = start; a < end; a++) {
271 instr = (u8 *)&a->instr_offset + a->instr_offset; 271 instr = (u8 *)&a->instr_offset + a->instr_offset;
272 replacement = (u8 *)&a->repl_offset + a->repl_offset; 272 replacement = (u8 *)&a->repl_offset + a->repl_offset;
273 BUG_ON(a->replacementlen > a->instrlen);
274 BUG_ON(a->instrlen > sizeof(insnbuf)); 273 BUG_ON(a->instrlen > sizeof(insnbuf));
275 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32); 274 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
276 if (!boot_cpu_has(a->cpuid)) 275 if (!boot_cpu_has(a->cpuid))
@@ -290,8 +289,9 @@ void __init_or_module apply_alternatives(struct alt_instr *start,
290 DPRINTK("Fix CALL offset: 0x%x", *(s32 *)(insnbuf + 1)); 289 DPRINTK("Fix CALL offset: 0x%x", *(s32 *)(insnbuf + 1));
291 } 290 }
292 291
293 add_nops(insnbuf + a->replacementlen, 292 if (a->instrlen > a->replacementlen)
294 a->instrlen - a->replacementlen); 293 add_nops(insnbuf + a->replacementlen,
294 a->instrlen - a->replacementlen);
295 295
296 text_poke_early(instr, insnbuf, a->instrlen); 296 text_poke_early(instr, insnbuf, a->instrlen);
297 } 297 }
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 000d4199b03e..d23c9a1d40e1 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -819,7 +819,7 @@ ENTRY(simd_coprocessor_error)
819661: pushl_cfi $do_general_protection 819661: pushl_cfi $do_general_protection
820662: 820662:
821.section .altinstructions,"a" 821.section .altinstructions,"a"
822 altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f 822 altinstruction_entry 661b, 663f, X86_FEATURE_XMM, 662b-661b, 664f-663f, 0
823.previous 823.previous
824.section .altinstr_replacement,"ax" 824.section .altinstr_replacement,"ax"
825663: pushl $do_simd_coprocessor_error 825663: pushl $do_simd_coprocessor_error
diff --git a/arch/x86/lib/clear_page_64.S b/arch/x86/lib/clear_page_64.S
index f2145cfa12a6..38e57faefd71 100644
--- a/arch/x86/lib/clear_page_64.S
+++ b/arch/x86/lib/clear_page_64.S
@@ -67,7 +67,7 @@ ENDPROC(clear_page)
67 .previous 67 .previous
68 .section .altinstructions,"a" 68 .section .altinstructions,"a"
69 altinstruction_entry clear_page,1b,X86_FEATURE_REP_GOOD,\ 69 altinstruction_entry clear_page,1b,X86_FEATURE_REP_GOOD,\
70 .Lclear_page_end-clear_page, 2b-1b 70 .Lclear_page_end-clear_page, 2b-1b, 0
71 altinstruction_entry clear_page,2b,X86_FEATURE_ERMS, \ 71 altinstruction_entry clear_page,2b,X86_FEATURE_ERMS, \
72 .Lclear_page_end-clear_page,3b-2b 72 .Lclear_page_end-clear_page,3b-2b, 0
73 .previous 73 .previous
diff --git a/arch/x86/lib/copy_page_64.S b/arch/x86/lib/copy_page_64.S
index 176cca67212b..f1ffdbb07755 100644
--- a/arch/x86/lib/copy_page_64.S
+++ b/arch/x86/lib/copy_page_64.S
@@ -106,5 +106,5 @@ ENDPROC(copy_page)
106 .previous 106 .previous
107 .section .altinstructions,"a" 107 .section .altinstructions,"a"
108 altinstruction_entry copy_page, 1b, X86_FEATURE_REP_GOOD, \ 108 altinstruction_entry copy_page, 1b, X86_FEATURE_REP_GOOD, \
109 .Lcopy_page_end-copy_page, 2b-1b 109 .Lcopy_page_end-copy_page, 2b-1b, 0
110 .previous 110 .previous
diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S
index 1530ec2c1b12..a9aedd6aa7f7 100644
--- a/arch/x86/lib/copy_user_64.S
+++ b/arch/x86/lib/copy_user_64.S
@@ -36,8 +36,8 @@
36 .previous 36 .previous
37 37
38 .section .altinstructions,"a" 38 .section .altinstructions,"a"
39 altinstruction_entry 0b,2b,\feature1,5,5 39 altinstruction_entry 0b,2b,\feature1,5,5,0
40 altinstruction_entry 0b,3b,\feature2,5,5 40 altinstruction_entry 0b,3b,\feature2,5,5,0
41 .previous 41 .previous
42 .endm 42 .endm
43 43
diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S
index 89b53c9968e7..bbfdacc01760 100644
--- a/arch/x86/lib/memcpy_64.S
+++ b/arch/x86/lib/memcpy_64.S
@@ -202,7 +202,7 @@ ENDPROC(__memcpy)
202 */ 202 */
203 .section .altinstructions, "a" 203 .section .altinstructions, "a"
204 altinstruction_entry __memcpy,.Lmemcpy_c,X86_FEATURE_REP_GOOD,\ 204 altinstruction_entry __memcpy,.Lmemcpy_c,X86_FEATURE_REP_GOOD,\
205 .Lmemcpy_e-.Lmemcpy_c,.Lmemcpy_e-.Lmemcpy_c 205 .Lmemcpy_e-.Lmemcpy_c,.Lmemcpy_e-.Lmemcpy_c,0
206 altinstruction_entry __memcpy,.Lmemcpy_c_e,X86_FEATURE_ERMS, \ 206 altinstruction_entry __memcpy,.Lmemcpy_c_e,X86_FEATURE_ERMS, \
207 .Lmemcpy_e_e-.Lmemcpy_c_e,.Lmemcpy_e_e-.Lmemcpy_c_e 207 .Lmemcpy_e_e-.Lmemcpy_c_e,.Lmemcpy_e_e-.Lmemcpy_c_e,0
208 .previous 208 .previous
diff --git a/arch/x86/lib/memmove_64.S b/arch/x86/lib/memmove_64.S
index 9c4b530575da..bbfa6b269ece 100644
--- a/arch/x86/lib/memmove_64.S
+++ b/arch/x86/lib/memmove_64.S
@@ -221,7 +221,7 @@ ENTRY(__memmove)
221 altinstruction_entry .Lmemmove_begin_forward, \ 221 altinstruction_entry .Lmemmove_begin_forward, \
222 .Lmemmove_begin_forward_efs,X86_FEATURE_ERMS, \ 222 .Lmemmove_begin_forward_efs,X86_FEATURE_ERMS, \
223 .Lmemmove_end_forward-.Lmemmove_begin_forward, \ 223 .Lmemmove_end_forward-.Lmemmove_begin_forward, \
224 .Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs 224 .Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs,0
225 .previous 225 .previous
226ENDPROC(__memmove) 226ENDPROC(__memmove)
227ENDPROC(memmove) 227ENDPROC(memmove)
diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S
index 6f44935c6a60..f6153c1cdddc 100644
--- a/arch/x86/lib/memset_64.S
+++ b/arch/x86/lib/memset_64.S
@@ -150,7 +150,7 @@ ENDPROC(__memset)
150 */ 150 */
151 .section .altinstructions,"a" 151 .section .altinstructions,"a"
152 altinstruction_entry __memset,.Lmemset_c,X86_FEATURE_REP_GOOD,\ 152 altinstruction_entry __memset,.Lmemset_c,X86_FEATURE_REP_GOOD,\
153 .Lfinal-__memset,.Lmemset_e-.Lmemset_c 153 .Lfinal-__memset,.Lmemset_e-.Lmemset_c,0
154 altinstruction_entry __memset,.Lmemset_c_e,X86_FEATURE_ERMS, \ 154 altinstruction_entry __memset,.Lmemset_c_e,X86_FEATURE_ERMS, \
155 .Lfinal-__memset,.Lmemset_e_e-.Lmemset_c_e 155 .Lfinal-__memset,.Lmemset_e_e-.Lmemset_c_e,0
156 .previous 156 .previous
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-11 07:50:37 -0500