diff options
| -rw-r--r-- | arch/x86/lib/memset_64.S | 61 |
1 files changed, 24 insertions, 37 deletions
diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S index f6153c1cdddc..93118fb23976 100644 --- a/arch/x86/lib/memset_64.S +++ b/arch/x86/lib/memset_64.S | |||
| @@ -5,19 +5,30 @@ | |||
| 5 | #include <asm/cpufeature.h> | 5 | #include <asm/cpufeature.h> |
| 6 | #include <asm/alternative-asm.h> | 6 | #include <asm/alternative-asm.h> |
| 7 | 7 | ||
| 8 | .weak memset | ||
| 9 | |||
| 8 | /* | 10 | /* |
| 9 | * ISO C memset - set a memory block to a byte value. This function uses fast | 11 | * ISO C memset - set a memory block to a byte value. This function uses fast |
| 10 | * string to get better performance than the original function. The code is | 12 | * string to get better performance than the original function. The code is |
| 11 | * simpler and shorter than the orignal function as well. | 13 | * simpler and shorter than the orignal function as well. |
| 12 | * | 14 | * |
| 13 | * rdi destination | 15 | * rdi destination |
| 14 | * rsi value (char) | 16 | * rsi value (char) |
| 15 | * rdx count (bytes) | 17 | * rdx count (bytes) |
| 16 | * | 18 | * |
| 17 | * rax original destination | 19 | * rax original destination |
| 18 | */ | 20 | */ |
| 19 | .section .altinstr_replacement, "ax", @progbits | 21 | ENTRY(memset) |
| 20 | .Lmemset_c: | 22 | ENTRY(__memset) |
| 23 | /* | ||
| 24 | * Some CPUs support enhanced REP MOVSB/STOSB feature. It is recommended | ||
| 25 | * to use it when possible. If not available, use fast string instructions. | ||
| 26 | * | ||
| 27 | * Otherwise, use original memset function. | ||
| 28 | */ | ||
| 29 | ALTERNATIVE_2 "jmp memset_orig", "", X86_FEATURE_REP_GOOD, \ | ||
| 30 | "jmp memset_erms", X86_FEATURE_ERMS | ||
| 31 | |||
| 21 | movq %rdi,%r9 | 32 | movq %rdi,%r9 |
| 22 | movq %rdx,%rcx | 33 | movq %rdx,%rcx |
| 23 | andl $7,%edx | 34 | andl $7,%edx |
| @@ -31,8 +42,8 @@ | |||
| 31 | rep stosb | 42 | rep stosb |
| 32 | movq %r9,%rax | 43 | movq %r9,%rax |
| 33 | ret | 44 | ret |
| 34 | .Lmemset_e: | 45 | ENDPROC(memset) |
| 35 | .previous | 46 | ENDPROC(__memset) |
| 36 | 47 | ||
| 37 | /* | 48 | /* |
| 38 | * ISO C memset - set a memory block to a byte value. This function uses | 49 | * ISO C memset - set a memory block to a byte value. This function uses |
| @@ -45,21 +56,16 @@ | |||
| 45 | * | 56 | * |
| 46 | * rax original destination | 57 | * rax original destination |
| 47 | */ | 58 | */ |
| 48 | .section .altinstr_replacement, "ax", @progbits | 59 | ENTRY(memset_erms) |
| 49 | .Lmemset_c_e: | ||
| 50 | movq %rdi,%r9 | 60 | movq %rdi,%r9 |
| 51 | movb %sil,%al | 61 | movb %sil,%al |
| 52 | movq %rdx,%rcx | 62 | movq %rdx,%rcx |
| 53 | rep stosb | 63 | rep stosb |
| 54 | movq %r9,%rax | 64 | movq %r9,%rax |
| 55 | ret | 65 | ret |
| 56 | .Lmemset_e_e: | 66 | ENDPROC(memset_erms) |
| 57 | .previous | ||
| 58 | |||
| 59 | .weak memset | ||
| 60 | 67 | ||
| 61 | ENTRY(memset) | 68 | ENTRY(memset_orig) |
| 62 | ENTRY(__memset) | ||
| 63 | CFI_STARTPROC | 69 | CFI_STARTPROC |
| 64 | movq %rdi,%r10 | 70 | movq %rdi,%r10 |
| 65 | 71 | ||
| @@ -134,23 +140,4 @@ ENTRY(__memset) | |||
| 134 | jmp .Lafter_bad_alignment | 140 | jmp .Lafter_bad_alignment |
| 135 | .Lfinal: | 141 | .Lfinal: |
| 136 | CFI_ENDPROC | 142 | CFI_ENDPROC |
| 137 | ENDPROC(memset) | 143 | ENDPROC(memset_orig) |
| 138 | ENDPROC(__memset) | ||
| 139 | |||
| 140 | /* Some CPUs support enhanced REP MOVSB/STOSB feature. | ||
| 141 | * It is recommended to use this when possible. | ||
| 142 | * | ||
| 143 | * If enhanced REP MOVSB/STOSB feature is not available, use fast string | ||
| 144 | * instructions. | ||
| 145 | * | ||
| 146 | * Otherwise, use original memset function. | ||
| 147 | * | ||
| 148 | * In .altinstructions section, ERMS feature is placed after REG_GOOD | ||
| 149 | * feature to implement the right patch order. | ||
| 150 | */ | ||
| 151 | .section .altinstructions,"a" | ||
| 152 | altinstruction_entry __memset,.Lmemset_c,X86_FEATURE_REP_GOOD,\ | ||
| 153 | .Lfinal-__memset,.Lmemset_e-.Lmemset_c,0 | ||
| 154 | altinstruction_entry __memset,.Lmemset_c_e,X86_FEATURE_ERMS, \ | ||
| 155 | .Lfinal-__memset,.Lmemset_e_e-.Lmemset_c_e,0 | ||
| 156 | .previous | ||