x86: Add new static_cpu_has() function using alternatives

For CPU-feature-specific code that touches performance-critical paths,
introduce a static patching version of [boot_]cpu_has().  This is run
at alternatives time and is therefore not appropriate for most
initialization code, but on the other hand initialization code is
generally not performance critical.

On gcc 4.5+ this uses the new "asm goto" feature.

Signed-off-by: H. Peter Anvin <hpa@zytor.com>
Cc: Avi Kivity <avi@redhat.com>
Cc: Suresh Siddha <suresh.b.siddha@intel.com>
LKML-Reference: <1273135546-29690-2-git-send-email-avi@redhat.com>

1 files changed, 57 insertions, 0 deletions

diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 0cd82d068613..9b11a5cc6662 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -175,6 +175,7 @@
 
 #if defined(__KERNEL__) && !defined(__ASSEMBLY__)
 
+#include <asm/asm.h>
 #include <linux/bitops.h>
 
 extern const char * const x86_cap_flags[NCAPINTS*32];
@@ -283,6 +284,62 @@ extern const char * const x86_power_flags[32];
 
 #endif /* CONFIG_X86_64 */
 
+/*
+ * Static testing of CPU features.  Used the same as boot_cpu_has().
+ * These are only valid after alternatives have run, but will statically
+ * patch the target code for additional performance.
+ *
+ */
+static __always_inline __pure bool __static_cpu_has(u8 bit)
+{
+#if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5)
+                asm goto("1: jmp %l[t_no]\n"
+                         "2:\n"
+                         ".section .altinstructions,\"a\"\n"
+                         _ASM_ALIGN "\n"
+                         _ASM_PTR "1b\n"
+                         _ASM_PTR "0\n"         /* no replacement */
+                         " .byte %P0\n"         /* feature bit */
+                         " .byte 2b - 1b\n"     /* source len */
+                         " .byte 0\n"           /* replacement len */
+                         " .byte 0xff + 0 - (2b-1b)\n"  /* padding */
+                         ".previous\n"
+                         : : "i" (bit) : : t_no);
+                return true;
+        t_no:
+                return false;
+#else
+                u8 flag;
+                /* Open-coded due to __stringify() in ALTERNATIVE() */
+                asm volatile("1: movb $0,%0\n"
+                             "2:\n"
+                             ".section .altinstructions,\"a\"\n"
+                             _ASM_ALIGN "\n"
+                             _ASM_PTR "1b\n"
+                             _ASM_PTR "3f\n"
+                             " .byte %P1\n"             /* feature bit */
+                             " .byte 2b - 1b\n"         /* source len */
+                             " .byte 4f - 3f\n"         /* replacement len */
+                             " .byte 0xff + (4f-3f) - (2b-1b)\n" /* padding */
+                             ".previous\n"
+                             ".section .altinstr_replacement,\"ax\"\n"
+                             "3: movb $1,%0\n"
+                             "4:\n"
+                             ".previous\n"
+                             : "=qm" (flag) : "i" (bit));
+                return flag;
+#endif
+}
+
+#define static_cpu_has(bit)                                     \
+(                                                               \
+        __builtin_constant_p(boot_cpu_has(bit)) ?               \
+                boot_cpu_has(bit) :                             \
+        (__builtin_constant_p(bit) && !((bit) & ~0xff)) ?       \
+                __static_cpu_has(bit) :                         \
+                boot_cpu_has(bit)                               \
+)
+
 #endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */
 
 #endif /* _ASM_X86_CPUFEATURE_H */