Merge branch 'x86/debug' into core/objtool, to pick up frame pointer fixes

Signed-off-by: Ingo Molnar <mingo@kernel.org>

129 files changed, 1841 insertions, 1172 deletions

diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt
index 9a53c929f017..0d27a9adabf8 100644
--- a/Documentation/kernel-parameters.txt
+++ b/Documentation/kernel-parameters.txt
@@ -666,7 +666,7 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
 
         clearcpuid=BITNUM [X86]
                         Disable CPUID feature X for the kernel. See
-                        arch/x86/include/asm/cpufeature.h for the valid bit
+                        arch/x86/include/asm/cpufeatures.h for the valid bit
                         numbers. Note the Linux specific bits are not necessarily
                         stable over kernel options, but the vendor specific
                         ones should be.
diff --git a/Documentation/x86/x86_64/boot-options.txt b/Documentation/x86/x86_64/boot-options.txt
index 68ed3114c363..0965a71f9942 100644
--- a/Documentation/x86/x86_64/boot-options.txt
+++ b/Documentation/x86/x86_64/boot-options.txt
@@ -60,6 +60,8 @@ Machine check
                 threshold to 1. Enabling this may make memory predictive failure
                 analysis less effective if the bios sets thresholds for memory
                 errors since we will not see details for all errors.
+   mce=recovery
+                Force-enable recoverable machine check code paths
 
    nomce (for compatibility with i386): same as mce=off
 
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 9b18ed97a8a2..68a2d1f0a683 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -350,16 +350,6 @@ config DEBUG_IMR_SELFTEST
 
           If unsure say N here.
 
-config X86_DEBUG_STATIC_CPU_HAS
-        bool "Debug alternatives"
-        depends on DEBUG_KERNEL
-        ---help---
-          This option causes additional code to be generated which
-          fails if static_cpu_has() is used before alternatives have
-          run.
-
-          If unsure, say N.
-
 config X86_DEBUG_FPU
         bool "Debug the x86 FPU code"
         depends on DEBUG_KERNEL
diff --git a/arch/x86/boot/cpuflags.h b/arch/x86/boot/cpuflags.h
index ea97697e51e4..4cb404fd45ce 100644
--- a/arch/x86/boot/cpuflags.h
+++ b/arch/x86/boot/cpuflags.h
@@ -1,7 +1,7 @@
 #ifndef BOOT_CPUFLAGS_H
 #define BOOT_CPUFLAGS_H
 
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/processor-flags.h>
 
 struct cpu_features {
diff --git a/arch/x86/boot/mkcpustr.c b/arch/x86/boot/mkcpustr.c
index 637097e66a62..f72498dc90d2 100644
--- a/arch/x86/boot/mkcpustr.c
+++ b/arch/x86/boot/mkcpustr.c
@@ -17,7 +17,7 @@
 
 #include "../include/asm/required-features.h"
 #include "../include/asm/disabled-features.h"
-#include "../include/asm/cpufeature.h"
+#include "../include/asm/cpufeatures.h"
 #include "../kernel/cpu/capflags.c"
 
 int main(void)
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 6bd2c6c95373..383a6f84a060 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -31,6 +31,7 @@
 
 #include <linux/linkage.h>
 #include <asm/inst.h>
+#include <asm/frame.h>
 
 /*
  * The following macros are used to move an (un)aligned 16 byte value to/from
@@ -1800,11 +1801,12 @@ ENDPROC(_key_expansion_256b)
  *                   unsigned int key_len)
  */
 ENTRY(aesni_set_key)
+        FRAME_BEGIN
 #ifndef __x86_64__
         pushl KEYP
-        movl 8(%esp), KEYP              # ctx
-        movl 12(%esp), UKEYP            # in_key
-        movl 16(%esp), %edx             # key_len
+        movl (FRAME_OFFSET+8)(%esp), KEYP       # ctx
+        movl (FRAME_OFFSET+12)(%esp), UKEYP     # in_key
+        movl (FRAME_OFFSET+16)(%esp), %edx      # key_len
 #endif
         movups (UKEYP), %xmm0           # user key (first 16 bytes)
         movaps %xmm0, (KEYP)
@@ -1905,6 +1907,7 @@ ENTRY(aesni_set_key)
 #ifndef __x86_64__
         popl KEYP
 #endif
+        FRAME_END
         ret
 ENDPROC(aesni_set_key)
 
@@ -1912,12 +1915,13 @@ ENDPROC(aesni_set_key)
  * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
  */
 ENTRY(aesni_enc)
+        FRAME_BEGIN
 #ifndef __x86_64__
         pushl KEYP
         pushl KLEN
-        movl 12(%esp), KEYP
-        movl 16(%esp), OUTP
-        movl 20(%esp), INP
+        movl (FRAME_OFFSET+12)(%esp), KEYP      # ctx
+        movl (FRAME_OFFSET+16)(%esp), OUTP      # dst
+        movl (FRAME_OFFSET+20)(%esp), INP       # src
 #endif
         movl 480(KEYP), KLEN            # key length
         movups (INP), STATE             # input
@@ -1927,6 +1931,7 @@ ENTRY(aesni_enc)
         popl KLEN
         popl KEYP
 #endif
+        FRAME_END
         ret
 ENDPROC(aesni_enc)
 
@@ -2101,12 +2106,13 @@ ENDPROC(_aesni_enc4)
  * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
  */
 ENTRY(aesni_dec)
+        FRAME_BEGIN
 #ifndef __x86_64__
         pushl KEYP
         pushl KLEN
-        movl 12(%esp), KEYP
-        movl 16(%esp), OUTP
-        movl 20(%esp), INP
+        movl (FRAME_OFFSET+12)(%esp), KEYP      # ctx
+        movl (FRAME_OFFSET+16)(%esp), OUTP      # dst
+        movl (FRAME_OFFSET+20)(%esp), INP       # src
 #endif
         mov 480(KEYP), KLEN             # key length
         add $240, KEYP
@@ -2117,6 +2123,7 @@ ENTRY(aesni_dec)
         popl KLEN
         popl KEYP
 #endif
+        FRAME_END
         ret
 ENDPROC(aesni_dec)
 
@@ -2292,14 +2299,15 @@ ENDPROC(_aesni_dec4)
  *                    size_t len)
  */
 ENTRY(aesni_ecb_enc)
+        FRAME_BEGIN
 #ifndef __x86_64__
         pushl LEN
         pushl KEYP
         pushl KLEN
-        movl 16(%esp), KEYP
-        movl 20(%esp), OUTP
-        movl 24(%esp), INP
-        movl 28(%esp), LEN
+        movl (FRAME_OFFSET+16)(%esp), KEYP      # ctx
+        movl (FRAME_OFFSET+20)(%esp), OUTP      # dst
+        movl (FRAME_OFFSET+24)(%esp), INP       # src
+        movl (FRAME_OFFSET+28)(%esp), LEN       # len
 #endif
         test LEN, LEN           # check length
         jz .Lecb_enc_ret
@@ -2342,6 +2350,7 @@ ENTRY(aesni_ecb_enc)
         popl KEYP
         popl LEN
 #endif
+        FRAME_END
         ret
 ENDPROC(aesni_ecb_enc)
 
@@ -2350,14 +2359,15 @@ ENDPROC(aesni_ecb_enc)
  *                    size_t len);
  */
 ENTRY(aesni_ecb_dec)
+        FRAME_BEGIN
 #ifndef __x86_64__
         pushl LEN
         pushl KEYP
         pushl KLEN
-        movl 16(%esp), KEYP
-        movl 20(%esp), OUTP
-        movl 24(%esp), INP
-        movl 28(%esp), LEN
+        movl (FRAME_OFFSET+16)(%esp), KEYP      # ctx
+        movl (FRAME_OFFSET+20)(%esp), OUTP      # dst
+        movl (FRAME_OFFSET+24)(%esp), INP       # src
+        movl (FRAME_OFFSET+28)(%esp), LEN       # len
 #endif
         test LEN, LEN
         jz .Lecb_dec_ret
@@ -2401,6 +2411,7 @@ ENTRY(aesni_ecb_dec)
         popl KEYP
         popl LEN
 #endif
+        FRAME_END
         ret
 ENDPROC(aesni_ecb_dec)
 
@@ -2409,16 +2420,17 @@ ENDPROC(aesni_ecb_dec)
  *                    size_t len, u8 *iv)
  */
 ENTRY(aesni_cbc_enc)
+        FRAME_BEGIN
 #ifndef __x86_64__
         pushl IVP
         pushl LEN
         pushl KEYP
         pushl KLEN
-        movl 20(%esp), KEYP
-        movl 24(%esp), OUTP
-        movl 28(%esp), INP
-        movl 32(%esp), LEN
-        movl 36(%esp), IVP
+        movl (FRAME_OFFSET+20)(%esp), KEYP      # ctx
+        movl (FRAME_OFFSET+24)(%esp), OUTP      # dst
+        movl (FRAME_OFFSET+28)(%esp), INP       # src
+        movl (FRAME_OFFSET+32)(%esp), LEN       # len
+        movl (FRAME_OFFSET+36)(%esp), IVP       # iv
 #endif
         cmp $16, LEN
         jb .Lcbc_enc_ret
@@ -2443,6 +2455,7 @@ ENTRY(aesni_cbc_enc)
         popl LEN
         popl IVP
 #endif
+        FRAME_END
         ret
 ENDPROC(aesni_cbc_enc)
 
@@ -2451,16 +2464,17 @@ ENDPROC(aesni_cbc_enc)
  *                    size_t len, u8 *iv)
  */
 ENTRY(aesni_cbc_dec)
+        FRAME_BEGIN
 #ifndef __x86_64__
         pushl IVP
         pushl LEN
         pushl KEYP
         pushl KLEN
-        movl 20(%esp), KEYP
-        movl 24(%esp), OUTP
-        movl 28(%esp), INP
-        movl 32(%esp), LEN
-        movl 36(%esp), IVP
+        movl (FRAME_OFFSET+20)(%esp), KEYP      # ctx
+        movl (FRAME_OFFSET+24)(%esp), OUTP      # dst
+        movl (FRAME_OFFSET+28)(%esp), INP       # src
+        movl (FRAME_OFFSET+32)(%esp), LEN       # len
+        movl (FRAME_OFFSET+36)(%esp), IVP       # iv
 #endif
         cmp $16, LEN
         jb .Lcbc_dec_just_ret
@@ -2534,13 +2548,16 @@ ENTRY(aesni_cbc_dec)
         popl LEN
         popl IVP
 #endif
+        FRAME_END
         ret
 ENDPROC(aesni_cbc_dec)
 
 #ifdef __x86_64__
+.pushsection .rodata
 .align 16
 .Lbswap_mask:
         .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+.popsection
 
 /*
  * _aesni_inc_init:     internal ABI
@@ -2598,6 +2615,7 @@ ENDPROC(_aesni_inc)
  *                    size_t len, u8 *iv)
  */
 ENTRY(aesni_ctr_enc)
+        FRAME_BEGIN
         cmp $16, LEN
         jb .Lctr_enc_just_ret
         mov 480(KEYP), KLEN
@@ -2651,6 +2669,7 @@ ENTRY(aesni_ctr_enc)
 .Lctr_enc_ret:
         movups IV, (IVP)
 .Lctr_enc_just_ret:
+        FRAME_END
         ret
 ENDPROC(aesni_ctr_enc)
 
@@ -2677,6 +2696,7 @@ ENDPROC(aesni_ctr_enc)
  *                       bool enc, u8 *iv)
  */
 ENTRY(aesni_xts_crypt8)
+        FRAME_BEGIN
         cmpb $0, %cl
         movl $0, %ecx
         movl $240, %r10d
@@ -2777,6 +2797,7 @@ ENTRY(aesni_xts_crypt8)
         pxor INC, STATE4
         movdqu STATE4, 0x70(OUTP)
 
+        FRAME_END
         ret
 ENDPROC(aesni_xts_crypt8)
 
diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
index ce71f9212409..aa9e8bd163f6 100644
--- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S
@@ -16,6 +16,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -726,6 +727,7 @@ __camellia_enc_blk16:
          *      %xmm0..%xmm15: 16 encrypted blocks, order swapped:
          *       7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
          */
+        FRAME_BEGIN
 
         leaq 8 * 16(%rax), %rcx;
 
@@ -780,6 +782,7 @@ __camellia_enc_blk16:
                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
                     %xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax));
 
+        FRAME_END
         ret;
 
 .align 8
@@ -812,6 +815,7 @@ __camellia_dec_blk16:
          *      %xmm0..%xmm15: 16 plaintext blocks, order swapped:
          *       7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
          */
+        FRAME_BEGIN
 
         leaq 8 * 16(%rax), %rcx;
 
@@ -865,6 +869,7 @@ __camellia_dec_blk16:
                     %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
                     %xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax));
 
+        FRAME_END
         ret;
 
 .align 8
@@ -890,6 +895,7 @@ ENTRY(camellia_ecb_enc_16way)
          *      %rsi: dst (16 blocks)
          *      %rdx: src (16 blocks)
          */
+         FRAME_BEGIN
 
         inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
                      %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
@@ -904,6 +910,7 @@ ENTRY(camellia_ecb_enc_16way)
                      %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
                      %xmm8, %rsi);
 
+        FRAME_END
         ret;
 ENDPROC(camellia_ecb_enc_16way)
 
@@ -913,6 +920,7 @@ ENTRY(camellia_ecb_dec_16way)
          *      %rsi: dst (16 blocks)
          *      %rdx: src (16 blocks)
          */
+         FRAME_BEGIN
 
         cmpl $16, key_length(CTX);
         movl $32, %r8d;
@@ -932,6 +940,7 @@ ENTRY(camellia_ecb_dec_16way)
                      %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
                      %xmm8, %rsi);
 
+        FRAME_END
         ret;
 ENDPROC(camellia_ecb_dec_16way)
 
@@ -941,6 +950,7 @@ ENTRY(camellia_cbc_dec_16way)
          *      %rsi: dst (16 blocks)
          *      %rdx: src (16 blocks)
          */
+        FRAME_BEGIN
 
         cmpl $16, key_length(CTX);
         movl $32, %r8d;
@@ -981,6 +991,7 @@ ENTRY(camellia_cbc_dec_16way)
                      %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
                      %xmm8, %rsi);
 
+        FRAME_END
         ret;
 ENDPROC(camellia_cbc_dec_16way)
 
@@ -997,6 +1008,7 @@ ENTRY(camellia_ctr_16way)
          *      %rdx: src (16 blocks)
          *      %rcx: iv (little endian, 128bit)
          */
+        FRAME_BEGIN
 
         subq $(16 * 16), %rsp;
         movq %rsp, %rax;
@@ -1092,6 +1104,7 @@ ENTRY(camellia_ctr_16way)
                      %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
                      %xmm8, %rsi);
 
+        FRAME_END
         ret;
 ENDPROC(camellia_ctr_16way)
 
@@ -1112,6 +1125,7 @@ camellia_xts_crypt_16way:
          *      %r8: index for input whitening key
          *      %r9: pointer to  __camellia_enc_blk16 or __camellia_dec_blk16
          */
+        FRAME_BEGIN
 
         subq $(16 * 16), %rsp;
         movq %rsp, %rax;
@@ -1234,6 +1248,7 @@ camellia_xts_crypt_16way:
                      %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9,
                      %xmm8, %rsi);
 
+        FRAME_END
         ret;
 ENDPROC(camellia_xts_crypt_16way)
 
diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
index 0e0b8863a34b..16186c18656d 100644
--- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
+++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S
@@ -11,6 +11,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 
 #define CAMELLIA_TABLE_BYTE_LEN 272
 
@@ -766,6 +767,7 @@ __camellia_enc_blk32:
          *      %ymm0..%ymm15: 32 encrypted blocks, order swapped:
          *       7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
          */
+        FRAME_BEGIN
 
         leaq 8 * 32(%rax), %rcx;
 
@@ -820,6 +822,7 @@ __camellia_enc_blk32:
                     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
                     %ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax));
 
+        FRAME_END
         ret;
 
 .align 8
@@ -852,6 +855,7 @@ __camellia_dec_blk32:
          *      %ymm0..%ymm15: 16 plaintext blocks, order swapped:
          *       7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8
          */
+        FRAME_BEGIN
 
         leaq 8 * 32(%rax), %rcx;
 
@@ -905,6 +909,7 @@ __camellia_dec_blk32:
                     %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14,
                     %ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax));
 
+        FRAME_END
         ret;
 
 .align 8
@@ -930,6 +935,7 @@ ENTRY(camellia_ecb_enc_32way)
          *      %rsi: dst (32 blocks)
          *      %rdx: src (32 blocks)
          */
+        FRAME_BEGIN
 
         vzeroupper;
 
@@ -948,6 +954,7 @@ ENTRY(camellia_ecb_enc_32way)
 
         vzeroupper;
 
+        FRAME_END
         ret;
 ENDPROC(camellia_ecb_enc_32way)
 
@@ -957,6 +964,7 @@ ENTRY(camellia_ecb_dec_32way)
          *      %rsi: dst (32 blocks)
          *      %rdx: src (32 blocks)
          */
+        FRAME_BEGIN
 
         vzeroupper;
 
@@ -980,6 +988,7 @@ ENTRY(camellia_ecb_dec_32way)
 
         vzeroupper;
 
+        FRAME_END
         ret;
 ENDPROC(camellia_ecb_dec_32way)
 
@@ -989,6 +998,7 @@ ENTRY(camellia_cbc_dec_32way)
          *      %rsi: dst (32 blocks)
          *      %rdx: src (32 blocks)
          */
+        FRAME_BEGIN
 
         vzeroupper;
 
@@ -1046,6 +1056,7 @@ ENTRY(camellia_cbc_dec_32way)
 
         vzeroupper;
 
+        FRAME_END
         ret;
 ENDPROC(camellia_cbc_dec_32way)
 
@@ -1070,6 +1081,7 @@ ENTRY(camellia_ctr_32way)
          *      %rdx: src (32 blocks)
          *      %rcx: iv (little endian, 128bit)
          */
+        FRAME_BEGIN
 
         vzeroupper;
 
@@ -1184,6 +1196,7 @@ ENTRY(camellia_ctr_32way)
 
         vzeroupper;
 
+        FRAME_END
         ret;
 ENDPROC(camellia_ctr_32way)
 
@@ -1216,6 +1229,7 @@ camellia_xts_crypt_32way:
          *      %r8: index for input whitening key
          *      %r9: pointer to  __camellia_enc_blk32 or __camellia_dec_blk32
          */
+        FRAME_BEGIN
 
         vzeroupper;
 
@@ -1349,6 +1363,7 @@ camellia_xts_crypt_32way:
 
         vzeroupper;
 
+        FRAME_END
         ret;
 ENDPROC(camellia_xts_crypt_32way)
 
diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
index c35fd5d6ecd2..14fa1966bf01 100644
--- a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
@@ -24,6 +24,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 
 .file "cast5-avx-x86_64-asm_64.S"
 
@@ -365,6 +366,7 @@ ENTRY(cast5_ecb_enc_16way)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         movq %rsi, %r11;
 
@@ -388,6 +390,7 @@ ENTRY(cast5_ecb_enc_16way)
         vmovdqu RR4, (6*4*4)(%r11);
         vmovdqu RL4, (7*4*4)(%r11);
 
+        FRAME_END
         ret;
 ENDPROC(cast5_ecb_enc_16way)
 
@@ -398,6 +401,7 @@ ENTRY(cast5_ecb_dec_16way)
          *      %rdx: src
          */
 
+        FRAME_BEGIN
         movq %rsi, %r11;
 
         vmovdqu (0*4*4)(%rdx), RL1;
@@ -420,6 +424,7 @@ ENTRY(cast5_ecb_dec_16way)
         vmovdqu RR4, (6*4*4)(%r11);
         vmovdqu RL4, (7*4*4)(%r11);
 
+        FRAME_END
         ret;
 ENDPROC(cast5_ecb_dec_16way)
 
@@ -429,6 +434,7 @@ ENTRY(cast5_cbc_dec_16way)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         pushq %r12;
 
@@ -469,6 +475,7 @@ ENTRY(cast5_cbc_dec_16way)
 
         popq %r12;
 
+        FRAME_END
         ret;
 ENDPROC(cast5_cbc_dec_16way)
 
@@ -479,6 +486,7 @@ ENTRY(cast5_ctr_16way)
          *      %rdx: src
          *      %rcx: iv (big endian, 64bit)
          */
+        FRAME_BEGIN
 
         pushq %r12;
 
@@ -542,5 +550,6 @@ ENTRY(cast5_ctr_16way)
 
         popq %r12;
 
+        FRAME_END
         ret;
 ENDPROC(cast5_ctr_16way)
diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
index e3531f833951..c419389889cd 100644
--- a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
@@ -24,6 +24,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "glue_helper-asm-avx.S"
 
 .file "cast6-avx-x86_64-asm_64.S"
@@ -349,6 +350,7 @@ ENTRY(cast6_ecb_enc_8way)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         movq %rsi, %r11;
 
@@ -358,6 +360,7 @@ ENTRY(cast6_ecb_enc_8way)
 
         store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+        FRAME_END
         ret;
 ENDPROC(cast6_ecb_enc_8way)
 
@@ -367,6 +370,7 @@ ENTRY(cast6_ecb_dec_8way)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         movq %rsi, %r11;
 
@@ -376,6 +380,7 @@ ENTRY(cast6_ecb_dec_8way)
 
         store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+        FRAME_END
         ret;
 ENDPROC(cast6_ecb_dec_8way)
 
@@ -385,6 +390,7 @@ ENTRY(cast6_cbc_dec_8way)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         pushq %r12;
 
@@ -399,6 +405,7 @@ ENTRY(cast6_cbc_dec_8way)
 
         popq %r12;
 
+        FRAME_END
         ret;
 ENDPROC(cast6_cbc_dec_8way)
 
@@ -409,6 +416,7 @@ ENTRY(cast6_ctr_8way)
          *      %rdx: src
          *      %rcx: iv (little endian, 128bit)
          */
+        FRAME_BEGIN
 
         pushq %r12;
 
@@ -424,6 +432,7 @@ ENTRY(cast6_ctr_8way)
 
         popq %r12;
 
+        FRAME_END
         ret;
 ENDPROC(cast6_ctr_8way)
 
@@ -434,6 +443,7 @@ ENTRY(cast6_xts_enc_8way)
          *      %rdx: src
          *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
          */
+        FRAME_BEGIN
 
         movq %rsi, %r11;
 
@@ -446,6 +456,7 @@ ENTRY(cast6_xts_enc_8way)
         /* dst <= regs xor IVs(in dst) */
         store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+        FRAME_END
         ret;
 ENDPROC(cast6_xts_enc_8way)
 
@@ -456,6 +467,7 @@ ENTRY(cast6_xts_dec_8way)
          *      %rdx: src
          *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
          */
+        FRAME_BEGIN
 
         movq %rsi, %r11;
 
@@ -468,5 +480,6 @@ ENTRY(cast6_xts_dec_8way)
         /* dst <= regs xor IVs(in dst) */
         store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+        FRAME_END
         ret;
 ENDPROC(cast6_xts_dec_8way)
diff --git a/arch/x86/crypto/crc32-pclmul_glue.c b/arch/x86/crypto/crc32-pclmul_glue.c
index 07d2c6c86a54..27226df3f7d8 100644
--- a/arch/x86/crypto/crc32-pclmul_glue.c
+++ b/arch/x86/crypto/crc32-pclmul_glue.c
@@ -33,7 +33,7 @@
 #include <linux/crc32.h>
 #include <crypto/internal/hash.h>
 
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/cpu_device_id.h>
 #include <asm/fpu/api.h>
 
diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c
index 0e9871693f24..0857b1a1de3b 100644
--- a/arch/x86/crypto/crc32c-intel_glue.c
+++ b/arch/x86/crypto/crc32c-intel_glue.c
@@ -30,7 +30,7 @@
 #include <linux/kernel.h>
 #include <crypto/internal/hash.h>
 
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/cpu_device_id.h>
 #include <asm/fpu/internal.h>
 
diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
index 4fe27e074194..dc05f010ca9b 100644
--- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
+++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S
@@ -170,8 +170,8 @@ continue_block:
         ## branch into array
         lea     jump_table(%rip), bufp
         movzxw  (bufp, %rax, 2), len
-        offset=crc_array-jump_table
-        lea     offset(bufp, len, 1), bufp
+        lea     crc_array(%rip), bufp
+        lea     (bufp, len, 1), bufp
         jmp     *bufp
 
         ################################################################
@@ -310,7 +310,9 @@ do_return:
         popq    %rdi
         popq    %rbx
         ret
+ENDPROC(crc_pcl)
 
+.section        .rodata, "a", %progbits
         ################################################################
         ## jump table        Table is 129 entries x 2 bytes each
         ################################################################
@@ -324,13 +326,11 @@ JMPTBL_ENTRY %i
         i=i+1
 .endr
 
-ENDPROC(crc_pcl)
 
         ################################################################
         ## PCLMULQDQ tables
         ## Table is 128 entries x 2 words (8 bytes) each
         ################################################################
-.section        .rodata, "a", %progbits
 .align 8
 K_table:
         .long 0x493c7d27, 0x00000001
diff --git a/arch/x86/crypto/crct10dif-pclmul_glue.c b/arch/x86/crypto/crct10dif-pclmul_glue.c
index a3fcfc97a311..cd4df9322501 100644
--- a/arch/x86/crypto/crct10dif-pclmul_glue.c
+++ b/arch/x86/crypto/crct10dif-pclmul_glue.c
@@ -30,7 +30,7 @@
 #include <linux/string.h>
 #include <linux/kernel.h>
 #include <asm/fpu/api.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/cpu_device_id.h>
 
 asmlinkage __u16 crc_t10dif_pcl(__u16 crc, const unsigned char *buf,
diff --git a/arch/x86/crypto/ghash-clmulni-intel_asm.S b/arch/x86/crypto/ghash-clmulni-intel_asm.S
index 5d1e0075ac24..eed55c8cca4f 100644
--- a/arch/x86/crypto/ghash-clmulni-intel_asm.S
+++ b/arch/x86/crypto/ghash-clmulni-intel_asm.S
@@ -18,6 +18,7 @@
 
 #include <linux/linkage.h>
 #include <asm/inst.h>
+#include <asm/frame.h>
 
 .data
 
@@ -94,6 +95,7 @@ ENDPROC(__clmul_gf128mul_ble)
 
 /* void clmul_ghash_mul(char *dst, const u128 *shash) */
 ENTRY(clmul_ghash_mul)
+        FRAME_BEGIN
         movups (%rdi), DATA
         movups (%rsi), SHASH
         movaps .Lbswap_mask, BSWAP
@@ -101,6 +103,7 @@ ENTRY(clmul_ghash_mul)
         call __clmul_gf128mul_ble
         PSHUFB_XMM BSWAP DATA
         movups DATA, (%rdi)
+        FRAME_END
         ret
 ENDPROC(clmul_ghash_mul)
 
@@ -109,6 +112,7 @@ ENDPROC(clmul_ghash_mul)
  *                         const u128 *shash);
  */
 ENTRY(clmul_ghash_update)
+        FRAME_BEGIN
         cmp $16, %rdx
         jb .Lupdate_just_ret    # check length
         movaps .Lbswap_mask, BSWAP
@@ -128,5 +132,6 @@ ENTRY(clmul_ghash_update)
         PSHUFB_XMM BSWAP DATA
         movups DATA, (%rdi)
 .Lupdate_just_ret:
+        FRAME_END
         ret
 ENDPROC(clmul_ghash_update)
diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
index 2f202f49872b..8be571808342 100644
--- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
@@ -24,6 +24,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "glue_helper-asm-avx.S"
 
 .file "serpent-avx-x86_64-asm_64.S"
@@ -681,6 +682,7 @@ ENTRY(serpent_ecb_enc_8way_avx)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
@@ -688,6 +690,7 @@ ENTRY(serpent_ecb_enc_8way_avx)
 
         store_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+        FRAME_END
         ret;
 ENDPROC(serpent_ecb_enc_8way_avx)
 
@@ -697,6 +700,7 @@ ENTRY(serpent_ecb_dec_8way_avx)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
@@ -704,6 +708,7 @@ ENTRY(serpent_ecb_dec_8way_avx)
 
         store_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
 
+        FRAME_END
         ret;
 ENDPROC(serpent_ecb_dec_8way_avx)
 
@@ -713,6 +718,7 @@ ENTRY(serpent_cbc_dec_8way_avx)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         load_8way(%rdx, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
@@ -720,6 +726,7 @@ ENTRY(serpent_cbc_dec_8way_avx)
 
         store_cbc_8way(%rdx, %rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
 
+        FRAME_END
         ret;
 ENDPROC(serpent_cbc_dec_8way_avx)
 
@@ -730,6 +737,7 @@ ENTRY(serpent_ctr_8way_avx)
          *      %rdx: src
          *      %rcx: iv (little endian, 128bit)
          */
+        FRAME_BEGIN
 
         load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2,
                       RD2, RK0, RK1, RK2);
@@ -738,6 +746,7 @@ ENTRY(serpent_ctr_8way_avx)
 
         store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+        FRAME_END
         ret;
 ENDPROC(serpent_ctr_8way_avx)
 
@@ -748,6 +757,7 @@ ENTRY(serpent_xts_enc_8way_avx)
          *      %rdx: src
          *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
          */
+        FRAME_BEGIN
 
         /* regs <= src, dst <= IVs, regs <= regs xor IVs */
         load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2,
@@ -758,6 +768,7 @@ ENTRY(serpent_xts_enc_8way_avx)
         /* dst <= regs xor IVs(in dst) */
         store_xts_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+        FRAME_END
         ret;
 ENDPROC(serpent_xts_enc_8way_avx)
 
@@ -768,6 +779,7 @@ ENTRY(serpent_xts_dec_8way_avx)
          *      %rdx: src
          *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
          */
+        FRAME_BEGIN
 
         /* regs <= src, dst <= IVs, regs <= regs xor IVs */
         load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2,
@@ -778,5 +790,6 @@ ENTRY(serpent_xts_dec_8way_avx)
         /* dst <= regs xor IVs(in dst) */
         store_xts_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2);
 
+        FRAME_END
         ret;
 ENDPROC(serpent_xts_dec_8way_avx)
diff --git a/arch/x86/crypto/serpent-avx2-asm_64.S b/arch/x86/crypto/serpent-avx2-asm_64.S
index b222085cccac..97c48add33ed 100644
--- a/arch/x86/crypto/serpent-avx2-asm_64.S
+++ b/arch/x86/crypto/serpent-avx2-asm_64.S
@@ -15,6 +15,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "glue_helper-asm-avx2.S"
 
 .file "serpent-avx2-asm_64.S"
@@ -673,6 +674,7 @@ ENTRY(serpent_ecb_enc_16way)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         vzeroupper;
 
@@ -684,6 +686,7 @@ ENTRY(serpent_ecb_enc_16way)
 
         vzeroupper;
 
+        FRAME_END
         ret;
 ENDPROC(serpent_ecb_enc_16way)
 
@@ -693,6 +696,7 @@ ENTRY(serpent_ecb_dec_16way)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         vzeroupper;
 
@@ -704,6 +708,7 @@ ENTRY(serpent_ecb_dec_16way)
 
         vzeroupper;
 
+        FRAME_END
         ret;
 ENDPROC(serpent_ecb_dec_16way)
 
@@ -713,6 +718,7 @@ ENTRY(serpent_cbc_dec_16way)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         vzeroupper;
 
@@ -725,6 +731,7 @@ ENTRY(serpent_cbc_dec_16way)
 
         vzeroupper;
 
+        FRAME_END
         ret;
 ENDPROC(serpent_cbc_dec_16way)
 
@@ -735,6 +742,7 @@ ENTRY(serpent_ctr_16way)
          *      %rdx: src (16 blocks)
          *      %rcx: iv (little endian, 128bit)
          */
+        FRAME_BEGIN
 
         vzeroupper;
 
@@ -748,6 +756,7 @@ ENTRY(serpent_ctr_16way)
 
         vzeroupper;
 
+        FRAME_END
         ret;
 ENDPROC(serpent_ctr_16way)
 
@@ -758,6 +767,7 @@ ENTRY(serpent_xts_enc_16way)
          *      %rdx: src (16 blocks)
          *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
          */
+        FRAME_BEGIN
 
         vzeroupper;
 
@@ -772,6 +782,7 @@ ENTRY(serpent_xts_enc_16way)
 
         vzeroupper;
 
+        FRAME_END
         ret;
 ENDPROC(serpent_xts_enc_16way)
 
@@ -782,6 +793,7 @@ ENTRY(serpent_xts_dec_16way)
          *      %rdx: src (16 blocks)
          *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
          */
+        FRAME_BEGIN
 
         vzeroupper;
 
@@ -796,5 +808,6 @@ ENTRY(serpent_xts_dec_16way)
 
         vzeroupper;
 
+        FRAME_END
         ret;
 ENDPROC(serpent_xts_dec_16way)
diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S
index 85c4e1cf7172..96df6a39d7e2 100644
--- a/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S
+++ b/arch/x86/crypto/sha-mb/sha1_mb_mgr_flush_avx2.S
@@ -52,6 +52,7 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "sha1_mb_mgr_datastruct.S"
 
 
@@ -86,16 +87,6 @@
 #define extra_blocks    %arg2
 #define p               %arg2
 
-
-# STACK_SPACE needs to be an odd multiple of 8
-_XMM_SAVE_SIZE  = 10*16
-_GPR_SAVE_SIZE  = 8*8
-_ALIGN_SIZE     = 8
-
-_XMM_SAVE       = 0
-_GPR_SAVE       = _XMM_SAVE + _XMM_SAVE_SIZE
-STACK_SPACE     = _GPR_SAVE + _GPR_SAVE_SIZE + _ALIGN_SIZE
-
 .macro LABEL prefix n
 \prefix\n\():
 .endm
@@ -113,16 +104,8 @@ offset = \_offset
 # JOB* sha1_mb_mgr_flush_avx2(MB_MGR *state)
 # arg 1 : rcx : state
 ENTRY(sha1_mb_mgr_flush_avx2)
-        mov     %rsp, %r10
-        sub     $STACK_SPACE, %rsp
-        and     $~31, %rsp
-        mov     %rbx, _GPR_SAVE(%rsp)
-        mov     %r10, _GPR_SAVE+8*1(%rsp) #save rsp
-        mov     %rbp, _GPR_SAVE+8*3(%rsp)
-        mov     %r12, _GPR_SAVE+8*4(%rsp)
-        mov     %r13, _GPR_SAVE+8*5(%rsp)
-        mov     %r14, _GPR_SAVE+8*6(%rsp)
-        mov     %r15, _GPR_SAVE+8*7(%rsp)
+        FRAME_BEGIN
+        push    %rbx
 
         # If bit (32+3) is set, then all lanes are empty
         mov     _unused_lanes(state), unused_lanes
@@ -230,16 +213,8 @@ len_is_0:
         mov     tmp2_w, offset(job_rax)
 
 return:
-
-        mov     _GPR_SAVE(%rsp), %rbx
-        mov     _GPR_SAVE+8*1(%rsp), %r10 #saved rsp
-        mov     _GPR_SAVE+8*3(%rsp), %rbp
-        mov     _GPR_SAVE+8*4(%rsp), %r12
-        mov     _GPR_SAVE+8*5(%rsp), %r13
-        mov     _GPR_SAVE+8*6(%rsp), %r14
-        mov     _GPR_SAVE+8*7(%rsp), %r15
-        mov     %r10, %rsp
-
+        pop     %rbx
+        FRAME_END
         ret
 
 return_null:
diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S
index 2ab9560b53c8..1435acf46f22 100644
--- a/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S
+++ b/arch/x86/crypto/sha-mb/sha1_mb_mgr_submit_avx2.S
@@ -53,6 +53,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "sha1_mb_mgr_datastruct.S"
 
 
@@ -86,33 +87,21 @@ job_rax         = %rax
 len             = %rax
 DWORD_len       = %eax
 
-lane            = %rbp
-tmp3            = %rbp
+lane            = %r12
+tmp3            = %r12
 
 tmp             = %r9
 DWORD_tmp       = %r9d
 
 lane_data       = %r10
 
-# STACK_SPACE needs to be an odd multiple of 8
-STACK_SPACE     = 8*8 + 16*10 + 8
-
 # JOB* submit_mb_mgr_submit_avx2(MB_MGR *state, job_sha1 *job)
 # arg 1 : rcx : state
 # arg 2 : rdx : job
 ENTRY(sha1_mb_mgr_submit_avx2)
-
-        mov     %rsp, %r10
-        sub     $STACK_SPACE, %rsp
-        and     $~31, %rsp
-
-        mov     %rbx, (%rsp)
-        mov     %r10, 8*2(%rsp) #save old rsp
-        mov     %rbp, 8*3(%rsp)
-        mov     %r12, 8*4(%rsp)
-        mov     %r13, 8*5(%rsp)
-        mov     %r14, 8*6(%rsp)
-        mov     %r15, 8*7(%rsp)
+        FRAME_BEGIN
+        push    %rbx
+        push    %r12
 
         mov     _unused_lanes(state), unused_lanes
         mov     unused_lanes, lane
@@ -203,16 +192,9 @@ len_is_0:
         movl    DWORD_tmp, _result_digest+1*16(job_rax)
 
 return:
-
-        mov     (%rsp), %rbx
-        mov     8*2(%rsp), %r10 #save old rsp
-        mov     8*3(%rsp), %rbp
-        mov     8*4(%rsp), %r12
-        mov     8*5(%rsp), %r13
-        mov     8*6(%rsp), %r14
-        mov     8*7(%rsp), %r15
-        mov     %r10, %rsp
-
+        pop     %r12
+        pop     %rbx
+        FRAME_END
         ret
 
 return_null:
diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
index 05058134c443..dc66273e610d 100644
--- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
@@ -24,6 +24,7 @@
  */
 
 #include <linux/linkage.h>
+#include <asm/frame.h>
 #include "glue_helper-asm-avx.S"
 
 .file "twofish-avx-x86_64-asm_64.S"
@@ -333,6 +334,7 @@ ENTRY(twofish_ecb_enc_8way)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         movq %rsi, %r11;
 
@@ -342,6 +344,7 @@ ENTRY(twofish_ecb_enc_8way)
 
         store_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
 
+        FRAME_END
         ret;
 ENDPROC(twofish_ecb_enc_8way)
 
@@ -351,6 +354,7 @@ ENTRY(twofish_ecb_dec_8way)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         movq %rsi, %r11;
 
@@ -360,6 +364,7 @@ ENTRY(twofish_ecb_dec_8way)
 
         store_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+        FRAME_END
         ret;
 ENDPROC(twofish_ecb_dec_8way)
 
@@ -369,6 +374,7 @@ ENTRY(twofish_cbc_dec_8way)
          *      %rsi: dst
          *      %rdx: src
          */
+        FRAME_BEGIN
 
         pushq %r12;
 
@@ -383,6 +389,7 @@ ENTRY(twofish_cbc_dec_8way)
 
         popq %r12;
 
+        FRAME_END
         ret;
 ENDPROC(twofish_cbc_dec_8way)
 
@@ -393,6 +400,7 @@ ENTRY(twofish_ctr_8way)
          *      %rdx: src
          *      %rcx: iv (little endian, 128bit)
          */
+        FRAME_BEGIN
 
         pushq %r12;
 
@@ -408,6 +416,7 @@ ENTRY(twofish_ctr_8way)
 
         popq %r12;
 
+        FRAME_END
         ret;
 ENDPROC(twofish_ctr_8way)
 
@@ -418,6 +427,7 @@ ENTRY(twofish_xts_enc_8way)
          *      %rdx: src
          *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
          */
+        FRAME_BEGIN
 
         movq %rsi, %r11;
 
@@ -430,6 +440,7 @@ ENTRY(twofish_xts_enc_8way)
         /* dst <= regs xor IVs(in dst) */
         store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2);
 
+        FRAME_END
         ret;
 ENDPROC(twofish_xts_enc_8way)
 
@@ -440,6 +451,7 @@ ENTRY(twofish_xts_dec_8way)
          *      %rdx: src
          *      %rcx: iv (t ⊕ αⁿ ∈ GF(2¹²⁸))
          */
+        FRAME_BEGIN
 
         movq %rsi, %r11;
 
@@ -452,5 +464,6 @@ ENTRY(twofish_xts_dec_8way)
         /* dst <= regs xor IVs(in dst) */
         store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2);
 
+        FRAME_END
         ret;
 ENDPROC(twofish_xts_dec_8way)
diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index e32206e09868..9a9e5884066c 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -201,37 +201,6 @@ For 32-bit we have the following conventions - kernel is built with
         .byte 0xf1
         .endm
 
-#else /* CONFIG_X86_64 */
-
-/*
- * For 32bit only simplified versions of SAVE_ALL/RESTORE_ALL. These
- * are different from the entry_32.S versions in not changing the segment
- * registers. So only suitable for in kernel use, not when transitioning
- * from or to user space. The resulting stack frame is not a standard
- * pt_regs frame. The main use case is calling C code from assembler
- * when all the registers need to be preserved.
- */
-
-        .macro SAVE_ALL
-        pushl %eax
-        pushl %ebp
-        pushl %edi
-        pushl %esi
-        pushl %edx
-        pushl %ecx
-        pushl %ebx
-        .endm
-
-        .macro RESTORE_ALL
-        popl %ebx
-        popl %ecx
-        popl %edx
-        popl %esi
-        popl %edi
-        popl %ebp
-        popl %eax
-        .endm
-
 #endif /* CONFIG_X86_64 */
 
 /*
diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index 03663740c866..1a000f59f04a 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -26,6 +26,7 @@
 #include <asm/traps.h>
 #include <asm/vdso.h>
 #include <asm/uaccess.h>
+#include <asm/cpufeature.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/syscalls.h>
@@ -268,6 +269,7 @@ static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
 /* Called with IRQs disabled. */
 __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
 {
+        struct thread_info *ti = pt_regs_to_thread_info(regs);
         u32 cached_flags;
 
         if (IS_ENABLED(CONFIG_PROVE_LOCKING) && WARN_ON(!irqs_disabled()))
@@ -275,12 +277,22 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
 
         lockdep_sys_exit();
 
-        cached_flags =
-                READ_ONCE(pt_regs_to_thread_info(regs)->flags);
+        cached_flags = READ_ONCE(ti->flags);
 
         if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
                 exit_to_usermode_loop(regs, cached_flags);
 
+#ifdef CONFIG_COMPAT
+        /*
+         * Compat syscalls set TS_COMPAT.  Make sure we clear it before
+         * returning to user mode.  We need to clear it *after* signal
+         * handling, because syscall restart has a fixup for compat
+         * syscalls.  The fixup is exercised by the ptrace_syscall_32
+         * selftest.
+         */
+        ti->status &= ~TS_COMPAT;
+#endif
+
         user_enter();
 }
 
@@ -332,17 +344,35 @@ __visible inline void syscall_return_slowpath(struct pt_regs *regs)
         if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS))
                 syscall_slow_exit_work(regs, cached_flags);
 
-#ifdef CONFIG_COMPAT
+        local_irq_disable();
+        prepare_exit_to_usermode(regs);
+}
+
+#ifdef CONFIG_X86_64
+__visible void do_syscall_64(struct pt_regs *regs)
+{
+        struct thread_info *ti = pt_regs_to_thread_info(regs);
+        unsigned long nr = regs->orig_ax;
+
+        local_irq_enable();
+
+        if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY)
+                nr = syscall_trace_enter(regs);
+
         /*
-         * Compat syscalls set TS_COMPAT.  Make sure we clear it before
-         * returning to user mode.
+         * NB: Native and x32 syscalls are dispatched from the same
+         * table.  The only functional difference is the x32 bit in
+         * regs->orig_ax, which changes the behavior of some syscalls.
          */
-        ti->status &= ~TS_COMPAT;
-#endif
+        if (likely((nr & __SYSCALL_MASK) < NR_syscalls)) {
+                regs->ax = sys_call_table[nr & __SYSCALL_MASK](
+                        regs->di, regs->si, regs->dx,
+                        regs->r10, regs->r8, regs->r9);
+        }
 
-        local_irq_disable();
-        prepare_exit_to_usermode(regs);
+        syscall_return_slowpath(regs);
 }
+#endif
 
 #if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
 /*
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 77d8c5112900..4c5228352744 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -40,7 +40,7 @@
 #include <asm/processor-flags.h>
 #include <asm/ftrace.h>
 #include <asm/irq_vectors.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/alternative-asm.h>
 #include <asm/asm.h>
 #include <asm/smap.h>
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 9d34d3cfceb6..70eadb0ea5fa 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -145,17 +145,11 @@ GLOBAL(entry_SYSCALL_64_after_swapgs)
         movq    %rsp, PER_CPU_VAR(rsp_scratch)
         movq    PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 
+        TRACE_IRQS_OFF
+
         /* Construct struct pt_regs on stack */
         pushq   $__USER_DS                      /* pt_regs->ss */
         pushq   PER_CPU_VAR(rsp_scratch)        /* pt_regs->sp */
-        /*
-         * Re-enable interrupts.
-         * We use 'rsp_scratch' as a scratch space, hence irq-off block above
-         * must execute atomically in the face of possible interrupt-driven
-         * task preemption. We must enable interrupts only after we're done
-         * with using rsp_scratch:
-         */
-        ENABLE_INTERRUPTS(CLBR_NONE)
         pushq   %r11                            /* pt_regs->flags */
         pushq   $__USER_CS                      /* pt_regs->cs */
         pushq   %rcx                            /* pt_regs->ip */
@@ -171,9 +165,21 @@ GLOBAL(entry_SYSCALL_64_after_swapgs)
         pushq   %r11                            /* pt_regs->r11 */
         sub     $(6*8), %rsp                    /* pt_regs->bp, bx, r12-15 not saved */
 
-        testl   $_TIF_WORK_SYSCALL_ENTRY, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
-        jnz     tracesys
+        /*
+         * If we need to do entry work or if we guess we'll need to do
+         * exit work, go straight to the slow path.
+         */
+        testl   $_TIF_WORK_SYSCALL_ENTRY|_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
+        jnz     entry_SYSCALL64_slow_path
+
 entry_SYSCALL_64_fastpath:
+        /*
+         * Easy case: enable interrupts and issue the syscall.  If the syscall
+         * needs pt_regs, we'll call a stub that disables interrupts again
+         * and jumps to the slow path.
+         */
+        TRACE_IRQS_ON
+        ENABLE_INTERRUPTS(CLBR_NONE)
 #if __SYSCALL_MASK == ~0
         cmpq    $__NR_syscall_max, %rax
 #else
@@ -182,103 +188,56 @@ entry_SYSCALL_64_fastpath:
 #endif
         ja      1f                              /* return -ENOSYS (already in pt_regs->ax) */
         movq    %r10, %rcx
+
+        /*
+         * This call instruction is handled specially in stub_ptregs_64.
+         * It might end up jumping to the slow path.  If it jumps, RAX
+         * and all argument registers are clobbered.
+         */
         call    *sys_call_table(, %rax, 8)
+.Lentry_SYSCALL_64_after_fastpath_call:
+
         movq    %rax, RAX(%rsp)
 1:
-/*
- * Syscall return path ending with SYSRET (fast path).
- * Has incompletely filled pt_regs.
- */
-        LOCKDEP_SYS_EXIT
-        /*
-         * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON,
-         * it is too small to ever cause noticeable irq latency.
-         */
-        DISABLE_INTERRUPTS(CLBR_NONE)
 
         /*
-         * We must check ti flags with interrupts (or at least preemption)
-         * off because we must *never* return to userspace without
-         * processing exit work that is enqueued if we're preempted here.
-         * In particular, returning to userspace with any of the one-shot
-         * flags (TIF_NOTIFY_RESUME, TIF_USER_RETURN_NOTIFY, etc) set is
-         * very bad.
+         * If we get here, then we know that pt_regs is clean for SYSRET64.
+         * If we see that no exit work is required (which we are required
+         * to check with IRQs off), then we can go straight to SYSRET64.
          */
+        DISABLE_INTERRUPTS(CLBR_NONE)
+        TRACE_IRQS_OFF
         testl   $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS)
-        jnz     int_ret_from_sys_call_irqs_off  /* Go to the slow path */
+        jnz     1f
 
-        RESTORE_C_REGS_EXCEPT_RCX_R11
+        LOCKDEP_SYS_EXIT
+        TRACE_IRQS_ON           /* user mode is traced as IRQs on */
         movq    RIP(%rsp), %rcx
         movq    EFLAGS(%rsp), %r11
+        RESTORE_C_REGS_EXCEPT_RCX_R11
         movq    RSP(%rsp), %rsp
-        /*
-         * 64-bit SYSRET restores rip from rcx,
-         * rflags from r11 (but RF and VM bits are forced to 0),
-         * cs and ss are loaded from MSRs.
-         * Restoration of rflags re-enables interrupts.
-         *
-         * NB: On AMD CPUs with the X86_BUG_SYSRET_SS_ATTRS bug, the ss
-         * descriptor is not reinitialized.  This means that we should
-         * avoid SYSRET with SS == NULL, which could happen if we schedule,
-         * exit the kernel, and re-enter using an interrupt vector.  (All
-         * interrupt entries on x86_64 set SS to NULL.)  We prevent that
-         * from happening by reloading SS in __switch_to.  (Actually
-         * detecting the failure in 64-bit userspace is tricky but can be
-         * done.)
-         */
         USERGS_SYSRET64
 
-GLOBAL(int_ret_from_sys_call_irqs_off)
+1:
+        /*
+         * The fast path looked good when we started, but something changed
+         * along the way and we need to switch to the slow path.  Calling
+         * raise(3) will trigger this, for example.  IRQs are off.
+         */
         TRACE_IRQS_ON
         ENABLE_INTERRUPTS(CLBR_NONE)
-        jmp int_ret_from_sys_call
-
-        /* Do syscall entry tracing */
-tracesys:
-        movq    %rsp, %rdi
-        movl    $AUDIT_ARCH_X86_64, %esi
-        call    syscall_trace_enter_phase1
-        test    %rax, %rax
-        jnz     tracesys_phase2                 /* if needed, run the slow path */
-        RESTORE_C_REGS_EXCEPT_RAX               /* else restore clobbered regs */
-        movq    ORIG_RAX(%rsp), %rax
-        jmp     entry_SYSCALL_64_fastpath       /* and return to the fast path */
-
-tracesys_phase2:
         SAVE_EXTRA_REGS
         movq    %rsp, %rdi
-        movl    $AUDIT_ARCH_X86_64, %esi
-        movq    %rax, %rdx
-        call    syscall_trace_enter_phase2
-
-        /*
-         * Reload registers from stack in case ptrace changed them.
-         * We don't reload %rax because syscall_trace_entry_phase2() returned
-         * the value it wants us to use in the table lookup.
-         */
-        RESTORE_C_REGS_EXCEPT_RAX
-        RESTORE_EXTRA_REGS
-#if __SYSCALL_MASK == ~0
-        cmpq    $__NR_syscall_max, %rax
-#else
-        andl    $__SYSCALL_MASK, %eax
-        cmpl    $__NR_syscall_max, %eax
-#endif
-        ja      1f                              /* return -ENOSYS (already in pt_regs->ax) */
-        movq    %r10, %rcx                      /* fixup for C */
-        call    *sys_call_table(, %rax, 8)
-        movq    %rax, RAX(%rsp)
-1:
-        /* Use IRET because user could have changed pt_regs->foo */
+        call    syscall_return_slowpath /* returns with IRQs disabled */
+        jmp     return_from_SYSCALL_64
 
-/*
- * Syscall return path ending with IRET.
- * Has correct iret frame.
- */
-GLOBAL(int_ret_from_sys_call)
+entry_SYSCALL64_slow_path:
+        /* IRQs are off. */
         SAVE_EXTRA_REGS
         movq    %rsp, %rdi
-        call    syscall_return_slowpath /* returns with IRQs disabled */
+        call    do_syscall_64           /* returns with IRQs disabled */
+
+return_from_SYSCALL_64:
         RESTORE_EXTRA_REGS
         TRACE_IRQS_IRETQ                /* we're about to change IF */
 
@@ -355,83 +314,45 @@ opportunistic_sysret_failed:
         jmp     restore_c_regs_and_iret
 END(entry_SYSCALL_64)
 
+ENTRY(stub_ptregs_64)
+        /*
+         * Syscalls marked as needing ptregs land here.
+         * If we are on the fast path, we need to save the extra regs,
+         * which we achieve by trying again on the slow path.  If we are on
+         * the slow path, the extra regs are already saved.
+         *
+         * RAX stores a pointer to the C function implementing the syscall.
+         * IRQs are on.
+         */
+        cmpq    $.Lentry_SYSCALL_64_after_fastpath_call, (%rsp)
+        jne     1f
 
-        .macro FORK_LIKE func
-ENTRY(stub_\func)
-        SAVE_EXTRA_REGS 8
-        jmp     sys_\func
-END(stub_\func)
-        .endm
-
-        FORK_LIKE  clone
-        FORK_LIKE  fork
-        FORK_LIKE  vfork
-
-ENTRY(stub_execve)
-        call    sys_execve
-return_from_execve:
-        testl   %eax, %eax
-        jz      1f
-        /* exec failed, can use fast SYSRET code path in this case */
-        ret
-1:
-        /* must use IRET code path (pt_regs->cs may have changed) */
-        addq    $8, %rsp
-        ZERO_EXTRA_REGS
-        movq    %rax, RAX(%rsp)
-        jmp     int_ret_from_sys_call
-END(stub_execve)
-/*
- * Remaining execve stubs are only 7 bytes long.
- * ENTRY() often aligns to 16 bytes, which in this case has no benefits.
- */
-        .align  8
-GLOBAL(stub_execveat)
-        call    sys_execveat
-        jmp     return_from_execve
-END(stub_execveat)
-
-#if defined(CONFIG_X86_X32_ABI)
-        .align  8
-GLOBAL(stub_x32_execve)
-        call    compat_sys_execve
-        jmp     return_from_execve
-END(stub_x32_execve)
-        .align  8
-GLOBAL(stub_x32_execveat)
-        call    compat_sys_execveat
-        jmp     return_from_execve
-END(stub_x32_execveat)
-#endif
-
-/*
- * sigreturn is special because it needs to restore all registers on return.
- * This cannot be done with SYSRET, so use the IRET return path instead.
- */
-ENTRY(stub_rt_sigreturn)
         /*
-         * SAVE_EXTRA_REGS result is not normally needed:
-         * sigreturn overwrites all pt_regs->GPREGS.
-         * But sigreturn can fail (!), and there is no easy way to detect that.
-         * To make sure RESTORE_EXTRA_REGS doesn't restore garbage on error,
-         * we SAVE_EXTRA_REGS here.
+         * Called from fast path -- disable IRQs again, pop return address
+         * and jump to slow path
          */
-        SAVE_EXTRA_REGS 8
-        call    sys_rt_sigreturn
-return_from_stub:
-        addq    $8, %rsp
-        RESTORE_EXTRA_REGS
-        movq    %rax, RAX(%rsp)
-        jmp     int_ret_from_sys_call
-END(stub_rt_sigreturn)
+        DISABLE_INTERRUPTS(CLBR_NONE)
+        TRACE_IRQS_OFF
+        popq    %rax
+        jmp     entry_SYSCALL64_slow_path
 
-#ifdef CONFIG_X86_X32_ABI
-ENTRY(stub_x32_rt_sigreturn)
-        SAVE_EXTRA_REGS 8
-        call    sys32_x32_rt_sigreturn
-        jmp     return_from_stub
-END(stub_x32_rt_sigreturn)
-#endif
+1:
+        /* Called from C */
+        jmp     *%rax                           /* called from C */
+END(stub_ptregs_64)
+
+.macro ptregs_stub func
+ENTRY(ptregs_\func)
+        leaq    \func(%rip), %rax
+        jmp     stub_ptregs_64
+END(ptregs_\func)
+.endm
+
+/* Instantiate ptregs_stub for each ptregs-using syscall */
+#define __SYSCALL_64_QUAL_(sym)
+#define __SYSCALL_64_QUAL_ptregs(sym) ptregs_stub sym
+#define __SYSCALL_64(nr, sym, qual) __SYSCALL_64_QUAL_##qual(sym)
+#include <asm/syscalls_64.h>
 
 /*
  * A newly forked process directly context switches into this address.
@@ -439,7 +360,6 @@ END(stub_x32_rt_sigreturn)
  * rdi: prev task we switched from
  */
 ENTRY(ret_from_fork)
-
         LOCK ; btr $TIF_FORK, TI_flags(%r8)
 
         pushq   $0x0002
@@ -447,28 +367,32 @@ ENTRY(ret_from_fork)
 
         call    schedule_tail                   /* rdi: 'prev' task parameter */
 
-        RESTORE_EXTRA_REGS
-
         testb   $3, CS(%rsp)                    /* from kernel_thread? */
+        jnz     1f
 
         /*
-         * By the time we get here, we have no idea whether our pt_regs,
-         * ti flags, and ti status came from the 64-bit SYSCALL fast path,
-         * the slow path, or one of the 32-bit compat paths.
-         * Use IRET code path to return, since it can safely handle
-         * all of the above.
+         * We came from kernel_thread.  This code path is quite twisted, and
+         * someone should clean it up.
+         *
+         * copy_thread_tls stashes the function pointer in RBX and the
+         * parameter to be passed in RBP.  The called function is permitted
+         * to call do_execve and thereby jump to user mode.
          */
-        jnz     int_ret_from_sys_call
+        movq    RBP(%rsp), %rdi
+        call    *RBX(%rsp)
+        movl    $0, RAX(%rsp)
 
         /*
-         * We came from kernel_thread
-         * nb: we depend on RESTORE_EXTRA_REGS above
+         * Fall through as though we're exiting a syscall.  This makes a
+         * twisted sort of sense if we just called do_execve.
          */
-        movq    %rbp, %rdi
-        call    *%rbx
-        movl    $0, RAX(%rsp)
-        RESTORE_EXTRA_REGS
-        jmp     int_ret_from_sys_call
+
+1:
+        movq    %rsp, %rdi
+        call    syscall_return_slowpath /* returns with IRQs disabled */
+        TRACE_IRQS_ON                   /* user mode is traced as IRQS on */
+        SWAPGS
+        jmp     restore_regs_and_iret
 END(ret_from_fork)
 
 /*
diff --git a/arch/x86/entry/syscall_32.c b/arch/x86/entry/syscall_32.c
index 9a6649857106..8f895ee13a1c 100644
--- a/arch/x86/entry/syscall_32.c
+++ b/arch/x86/entry/syscall_32.c
@@ -6,17 +6,11 @@
 #include <asm/asm-offsets.h>
 #include <asm/syscall.h>
 
-#ifdef CONFIG_IA32_EMULATION
-#define SYM(sym, compat) compat
-#else
-#define SYM(sym, compat) sym
-#endif
-
-#define __SYSCALL_I386(nr, sym, compat) extern asmlinkage long SYM(sym, compat)(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ;
+#define __SYSCALL_I386(nr, sym, qual) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ;
 #include <asm/syscalls_32.h>
 #undef __SYSCALL_I386
 
-#define __SYSCALL_I386(nr, sym, compat) [nr] = SYM(sym, compat),
+#define __SYSCALL_I386(nr, sym, qual) [nr] = sym,
 
 extern asmlinkage long sys_ni_syscall(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long);
 
diff --git a/arch/x86/entry/syscall_64.c b/arch/x86/entry/syscall_64.c
index 41283d22be7a..9dbc5abb6162 100644
--- a/arch/x86/entry/syscall_64.c
+++ b/arch/x86/entry/syscall_64.c
@@ -6,19 +6,14 @@
 #include <asm/asm-offsets.h>
 #include <asm/syscall.h>
 
-#define __SYSCALL_COMMON(nr, sym, compat) __SYSCALL_64(nr, sym, compat)
+#define __SYSCALL_64_QUAL_(sym) sym
+#define __SYSCALL_64_QUAL_ptregs(sym) ptregs_##sym
 
-#ifdef CONFIG_X86_X32_ABI
-# define __SYSCALL_X32(nr, sym, compat) __SYSCALL_64(nr, sym, compat)
-#else
-# define __SYSCALL_X32(nr, sym, compat) /* nothing */
-#endif
-
-#define __SYSCALL_64(nr, sym, compat) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ;
+#define __SYSCALL_64(nr, sym, qual) extern asmlinkage long __SYSCALL_64_QUAL_##qual(sym)(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long);
 #include <asm/syscalls_64.h>
 #undef __SYSCALL_64
 
-#define __SYSCALL_64(nr, sym, compat) [nr] = sym,
+#define __SYSCALL_64(nr, sym, qual) [nr] = __SYSCALL_64_QUAL_##qual(sym),
 
 extern long sys_ni_syscall(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long);
 
diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl
index dc1040a50bdc..2e5b565adacc 100644
--- a/arch/x86/entry/syscalls/syscall_64.tbl
+++ b/arch/x86/entry/syscalls/syscall_64.tbl
@@ -21,7 +21,7 @@
 12      common  brk                     sys_brk
 13      64      rt_sigaction            sys_rt_sigaction
 14      common  rt_sigprocmask          sys_rt_sigprocmask
-15      64      rt_sigreturn            stub_rt_sigreturn
+15      64      rt_sigreturn            sys_rt_sigreturn/ptregs
 16      64      ioctl                   sys_ioctl
 17      common  pread64                 sys_pread64
 18      common  pwrite64                sys_pwrite64
@@ -62,10 +62,10 @@
 53      common  socketpair              sys_socketpair
 54      64      setsockopt              sys_setsockopt
 55      64      getsockopt              sys_getsockopt
-56      common  clone                   stub_clone
-57      common  fork                    stub_fork
-58      common  vfork                   stub_vfork
-59      64      execve                  stub_execve
+56      common  clone                   sys_clone/ptregs
+57      common  fork                    sys_fork/ptregs
+58      common  vfork                   sys_vfork/ptregs
+59      64      execve                  sys_execve/ptregs
 60      common  exit                    sys_exit
 61      common  wait4                   sys_wait4
 62      common  kill                    sys_kill
@@ -178,7 +178,7 @@
 169     common  reboot                  sys_reboot
 170     common  sethostname             sys_sethostname
 171     common  setdomainname           sys_setdomainname
-172     common  iopl                    sys_iopl
+172     common  iopl                    sys_iopl/ptregs
 173     common  ioperm                  sys_ioperm
 174     64      create_module
 175     common  init_module             sys_init_module
@@ -328,7 +328,7 @@
 319     common  memfd_create            sys_memfd_create
 320     common  kexec_file_load         sys_kexec_file_load
 321     common  bpf                     sys_bpf
-322     64      execveat                stub_execveat
+322     64      execveat                sys_execveat/ptregs
 323     common  userfaultfd             sys_userfaultfd
 324     common  membarrier              sys_membarrier
 325     common  mlock2                  sys_mlock2
@@ -339,14 +339,14 @@
 # for native 64-bit operation.
 #
 512     x32     rt_sigaction            compat_sys_rt_sigaction
-513     x32     rt_sigreturn            stub_x32_rt_sigreturn
+513     x32     rt_sigreturn            sys32_x32_rt_sigreturn
 514     x32     ioctl                   compat_sys_ioctl
 515     x32     readv                   compat_sys_readv
 516     x32     writev                  compat_sys_writev
 517     x32     recvfrom                compat_sys_recvfrom
 518     x32     sendmsg                 compat_sys_sendmsg
 519     x32     recvmsg                 compat_sys_recvmsg
-520     x32     execve                  stub_x32_execve
+520     x32     execve                  compat_sys_execve/ptregs
 521     x32     ptrace                  compat_sys_ptrace
 522     x32     rt_sigpending           compat_sys_rt_sigpending
 523     x32     rt_sigtimedwait         compat_sys_rt_sigtimedwait
@@ -371,4 +371,4 @@
 542     x32     getsockopt              compat_sys_getsockopt
 543     x32     io_setup                compat_sys_io_setup
 544     x32     io_submit               compat_sys_io_submit
-545     x32     execveat                stub_x32_execveat
+545     x32     execveat                compat_sys_execveat/ptregs
diff --git a/arch/x86/entry/syscalls/syscalltbl.sh b/arch/x86/entry/syscalls/syscalltbl.sh
index 0e7f8ec071e7..cd3d3015d7df 100644
--- a/arch/x86/entry/syscalls/syscalltbl.sh
+++ b/arch/x86/entry/syscalls/syscalltbl.sh
@@ -3,13 +3,63 @@
 in="$1"
 out="$2"
 
+syscall_macro() {
+    abi="$1"
+    nr="$2"
+    entry="$3"
+
+    # Entry can be either just a function name or "function/qualifier"
+    real_entry="${entry%%/*}"
+    qualifier="${entry:${#real_entry}}"         # Strip the function name
+    qualifier="${qualifier:1}"                  # Strip the slash, if any
+
+    echo "__SYSCALL_${abi}($nr, $real_entry, $qualifier)"
+}
+
+emit() {
+    abi="$1"
+    nr="$2"
+    entry="$3"
+    compat="$4"
+
+    if [ "$abi" == "64" -a -n "$compat" ]; then
+        echo "a compat entry for a 64-bit syscall makes no sense" >&2
+        exit 1
+    fi
+
+    if [ -z "$compat" ]; then
+        if [ -n "$entry" ]; then
+            syscall_macro "$abi" "$nr" "$entry"
+        fi
+    else
+        echo "#ifdef CONFIG_X86_32"
+        if [ -n "$entry" ]; then
+            syscall_macro "$abi" "$nr" "$entry"
+        fi
+        echo "#else"
+        syscall_macro "$abi" "$nr" "$compat"
+        echo "#endif"
+    fi
+}
+
 grep '^[0-9]' "$in" | sort -n | (
     while read nr abi name entry compat; do
         abi=`echo "$abi" | tr '[a-z]' '[A-Z]'`
-        if [ -n "$compat" ]; then
-            echo "__SYSCALL_${abi}($nr, $entry, $compat)"
-        elif [ -n "$entry" ]; then
-            echo "__SYSCALL_${abi}($nr, $entry, $entry)"
+        if [ "$abi" == "COMMON" -o "$abi" == "64" ]; then
+            # COMMON is the same as 64, except that we don't expect X32
+            # programs to use it.  Our expectation has nothing to do with
+            # any generated code, so treat them the same.
+            emit 64 "$nr" "$entry" "$compat"
+        elif [ "$abi" == "X32" ]; then
+            # X32 is equivalent to 64 on an X32-compatible kernel.
+            echo "#ifdef CONFIG_X86_X32_ABI"
+            emit 64 "$nr" "$entry" "$compat"
+            echo "#endif"
+        elif [ "$abi" == "I386" ]; then
+            emit "$abi" "$nr" "$entry" "$compat"
+        else
+            echo "Unknown abi $abi" >&2
+            exit 1
         fi
     done
 ) > "$out"
diff --git a/arch/x86/entry/thunk_64.S b/arch/x86/entry/thunk_64.S
index efb2b932b748..98df1fa8825c 100644
--- a/arch/x86/entry/thunk_64.S
+++ b/arch/x86/entry/thunk_64.S
@@ -8,11 +8,14 @@
 #include <linux/linkage.h>
 #include "calling.h"
 #include <asm/asm.h>
+#include <asm/frame.h>
 
         /* rdi: arg1 ... normal C conventions. rax is saved/restored. */
         .macro THUNK name, func, put_ret_addr_in_rdi=0
         .globl \name
+        .type \name, @function
 \name:
+        FRAME_BEGIN
 
         /* this one pushes 9 elems, the next one would be %rIP */
         pushq %rdi
@@ -62,6 +65,7 @@ restore:
         popq %rdx
         popq %rsi
         popq %rdi
+        FRAME_END
         ret
         _ASM_NOKPROBE(restore)
 #endif
diff --git a/arch/x86/entry/vdso/vdso2c.h b/arch/x86/entry/vdso/vdso2c.h
index 0224987556ce..abe961c7c71c 100644
--- a/arch/x86/entry/vdso/vdso2c.h
+++ b/arch/x86/entry/vdso/vdso2c.h
@@ -150,16 +150,9 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len,
         }
         fprintf(outfile, "\n};\n\n");
 
-        fprintf(outfile, "static struct page *pages[%lu];\n\n",
-                mapping_size / 4096);
-
         fprintf(outfile, "const struct vdso_image %s = {\n", name);
         fprintf(outfile, "\t.data = raw_data,\n");
         fprintf(outfile, "\t.size = %lu,\n", mapping_size);
-        fprintf(outfile, "\t.text_mapping = {\n");
-        fprintf(outfile, "\t\t.name = \"[vdso]\",\n");
-        fprintf(outfile, "\t\t.pages = pages,\n");
-        fprintf(outfile, "\t},\n");
         if (alt_sec) {
                 fprintf(outfile, "\t.alt = %lu,\n",
                         (unsigned long)GET_LE(&alt_sec->sh_offset));
diff --git a/arch/x86/entry/vdso/vdso32-setup.c b/arch/x86/entry/vdso/vdso32-setup.c
index 08a317a9ae4b..7853b53959cd 100644
--- a/arch/x86/entry/vdso/vdso32-setup.c
+++ b/arch/x86/entry/vdso/vdso32-setup.c
@@ -11,7 +11,6 @@
 #include <linux/kernel.h>
 #include <linux/mm_types.h>
 
-#include <asm/cpufeature.h>
 #include <asm/processor.h>
 #include <asm/vdso.h>
 
diff --git a/arch/x86/entry/vdso/vdso32/system_call.S b/arch/x86/entry/vdso/vdso32/system_call.S
index 3a1d9297074b..0109ac6cb79c 100644
--- a/arch/x86/entry/vdso/vdso32/system_call.S
+++ b/arch/x86/entry/vdso/vdso32/system_call.S
@@ -3,7 +3,7 @@
 */
 
 #include <asm/dwarf2.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/alternative-asm.h>
 
 /*
diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c
index b8f69e264ac4..10f704584922 100644
--- a/arch/x86/entry/vdso/vma.c
+++ b/arch/x86/entry/vdso/vma.c
@@ -20,6 +20,7 @@
 #include <asm/page.h>
 #include <asm/hpet.h>
 #include <asm/desc.h>
+#include <asm/cpufeature.h>
 
 #if defined(CONFIG_X86_64)
 unsigned int __read_mostly vdso64_enabled = 1;
@@ -27,13 +28,7 @@ unsigned int __read_mostly vdso64_enabled = 1;
 
 void __init init_vdso_image(const struct vdso_image *image)
 {
-        int i;
-        int npages = (image->size) / PAGE_SIZE;
-
         BUG_ON(image->size % PAGE_SIZE != 0);
-        for (i = 0; i < npages; i++)
-                image->text_mapping.pages[i] =
-                        virt_to_page(image->data + i*PAGE_SIZE);
 
         apply_alternatives((struct alt_instr *)(image->data + image->alt),
                            (struct alt_instr *)(image->data + image->alt +
@@ -90,18 +85,87 @@ static unsigned long vdso_addr(unsigned long start, unsigned len)
 #endif
 }
 
+static int vdso_fault(const struct vm_special_mapping *sm,
+                      struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+        const struct vdso_image *image = vma->vm_mm->context.vdso_image;
+
+        if (!image || (vmf->pgoff << PAGE_SHIFT) >= image->size)
+                return VM_FAULT_SIGBUS;
+
+        vmf->page = virt_to_page(image->data + (vmf->pgoff << PAGE_SHIFT));
+        get_page(vmf->page);
+        return 0;
+}
+
+static const struct vm_special_mapping text_mapping = {
+        .name = "[vdso]",
+        .fault = vdso_fault,
+};
+
+static int vvar_fault(const struct vm_special_mapping *sm,
+                      struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+        const struct vdso_image *image = vma->vm_mm->context.vdso_image;
+        long sym_offset;
+        int ret = -EFAULT;
+
+        if (!image)
+                return VM_FAULT_SIGBUS;
+
+        sym_offset = (long)(vmf->pgoff << PAGE_SHIFT) +
+                image->sym_vvar_start;
+
+        /*
+         * Sanity check: a symbol offset of zero means that the page
+         * does not exist for this vdso image, not that the page is at
+         * offset zero relative to the text mapping.  This should be
+         * impossible here, because sym_offset should only be zero for
+         * the page past the end of the vvar mapping.
+         */
+        if (sym_offset == 0)
+                return VM_FAULT_SIGBUS;
+
+        if (sym_offset == image->sym_vvar_page) {
+                ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address,
+                                    __pa_symbol(&__vvar_page) >> PAGE_SHIFT);
+        } else if (sym_offset == image->sym_hpet_page) {
+#ifdef CONFIG_HPET_TIMER
+                if (hpet_address && vclock_was_used(VCLOCK_HPET)) {
+                        ret = vm_insert_pfn_prot(
+                                vma,
+                                (unsigned long)vmf->virtual_address,
+                                hpet_address >> PAGE_SHIFT,
+                                pgprot_noncached(PAGE_READONLY));
+                }
+#endif
+        } else if (sym_offset == image->sym_pvclock_page) {
+                struct pvclock_vsyscall_time_info *pvti =
+                        pvclock_pvti_cpu0_va();
+                if (pvti && vclock_was_used(VCLOCK_PVCLOCK)) {
+                        ret = vm_insert_pfn(
+                                vma,
+                                (unsigned long)vmf->virtual_address,
+                                __pa(pvti) >> PAGE_SHIFT);
+                }
+        }
+
+        if (ret == 0 || ret == -EBUSY)
+                return VM_FAULT_NOPAGE;
+
+        return VM_FAULT_SIGBUS;
+}
+
 static int map_vdso(const struct vdso_image *image, bool calculate_addr)
 {
         struct mm_struct *mm = current->mm;
         struct vm_area_struct *vma;
         unsigned long addr, text_start;
         int ret = 0;
-        static struct page *no_pages[] = {NULL};
-        static struct vm_special_mapping vvar_mapping = {
+        static const struct vm_special_mapping vvar_mapping = {
                 .name = "[vvar]",
-                .pages = no_pages,
+                .fault = vvar_fault,
         };
-        struct pvclock_vsyscall_time_info *pvti;
 
         if (calculate_addr) {
                 addr = vdso_addr(current->mm->start_stack,
@@ -121,6 +185,7 @@ static int map_vdso(const struct vdso_image *image, bool calculate_addr)
 
         text_start = addr - image->sym_vvar_start;
         current->mm->context.vdso = (void __user *)text_start;
+        current->mm->context.vdso_image = image;
 
         /*
          * MAYWRITE to allow gdb to COW and set breakpoints
@@ -130,7 +195,7 @@ static int map_vdso(const struct vdso_image *image, bool calculate_addr)
                                        image->size,
                                        VM_READ|VM_EXEC|
                                        VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
-                                       &image->text_mapping);
+                                       &text_mapping);
 
         if (IS_ERR(vma)) {
                 ret = PTR_ERR(vma);
@@ -140,7 +205,8 @@ static int map_vdso(const struct vdso_image *image, bool calculate_addr)
         vma = _install_special_mapping(mm,
                                        addr,
                                        -image->sym_vvar_start,
-                                       VM_READ|VM_MAYREAD,
+                                       VM_READ|VM_MAYREAD|VM_IO|VM_DONTDUMP|
+                                       VM_PFNMAP,
                                        &vvar_mapping);
 
         if (IS_ERR(vma)) {
@@ -148,41 +214,6 @@ static int map_vdso(const struct vdso_image *image, bool calculate_addr)
                 goto up_fail;
         }
 
-        if (image->sym_vvar_page)
-                ret = remap_pfn_range(vma,
-                                      text_start + image->sym_vvar_page,
-                                      __pa_symbol(&__vvar_page) >> PAGE_SHIFT,
-                                      PAGE_SIZE,
-                                      PAGE_READONLY);
-
-        if (ret)
-                goto up_fail;
-
-#ifdef CONFIG_HPET_TIMER
-        if (hpet_address && image->sym_hpet_page) {
-                ret = io_remap_pfn_range(vma,
-                        text_start + image->sym_hpet_page,
-                        hpet_address >> PAGE_SHIFT,
-                        PAGE_SIZE,
-                        pgprot_noncached(PAGE_READONLY));
-
-                if (ret)
-                        goto up_fail;
-        }
-#endif
-
-        pvti = pvclock_pvti_cpu0_va();
-        if (pvti && image->sym_pvclock_page) {
-                ret = remap_pfn_range(vma,
-                                      text_start + image->sym_pvclock_page,
-                                      __pa(pvti) >> PAGE_SHIFT,
-                                      PAGE_SIZE,
-                                      PAGE_READONLY);
-
-                if (ret)
-                        goto up_fail;
-        }
-
 up_fail:
         if (ret)
                 current->mm->context.vdso = NULL;
@@ -254,7 +285,7 @@ static void vgetcpu_cpu_init(void *arg)
 #ifdef CONFIG_NUMA
         node = cpu_to_node(cpu);
 #endif
-        if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
+        if (static_cpu_has(X86_FEATURE_RDTSCP))
                 write_rdtscp_aux((node << 12) | cpu);
 
         /*
diff --git a/arch/x86/entry/vsyscall/vsyscall_gtod.c b/arch/x86/entry/vsyscall/vsyscall_gtod.c
index 51e330416995..0fb3a104ac62 100644
--- a/arch/x86/entry/vsyscall/vsyscall_gtod.c
+++ b/arch/x86/entry/vsyscall/vsyscall_gtod.c
@@ -16,6 +16,8 @@
 #include <asm/vgtod.h>
 #include <asm/vvar.h>
 
+int vclocks_used __read_mostly;
+
 DEFINE_VVAR(struct vsyscall_gtod_data, vsyscall_gtod_data);
 
 void update_vsyscall_tz(void)
@@ -26,12 +28,17 @@ void update_vsyscall_tz(void)
 
 void update_vsyscall(struct timekeeper *tk)
 {
+        int vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode;
         struct vsyscall_gtod_data *vdata = &vsyscall_gtod_data;
 
+        /* Mark the new vclock used. */
+        BUILD_BUG_ON(VCLOCK_MAX >= 32);
+        WRITE_ONCE(vclocks_used, READ_ONCE(vclocks_used) | (1 << vclock_mode));
+
         gtod_write_begin(vdata);
 
         /* copy vsyscall data */
-        vdata->vclock_mode      = tk->tkr_mono.clock->archdata.vclock_mode;
+        vdata->vclock_mode      = vclock_mode;
         vdata->cycle_last       = tk->tkr_mono.cycle_last;
         vdata->mask             = tk->tkr_mono.mask;
         vdata->mult             = tk->tkr_mono.mult;
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 7bfc85bbb8ff..99afb665a004 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -152,12 +152,6 @@ static inline int alternatives_text_reserved(void *start, void *end)
         ".popsection"
 
 /*
- * This must be included *after* the definition of ALTERNATIVE due to
- * <asm/arch_hweight.h>
- */
-#include <asm/cpufeature.h>
-
-/*
  * Alternative instructions for different CPU types or capabilities.
  *
  * This allows to use optimized instructions even on generic binary
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index c80f6b6f3da2..0899cfc8dfe8 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -6,7 +6,6 @@
 
 #include <asm/alternative.h>
 #include <asm/cpufeature.h>
-#include <asm/processor.h>
 #include <asm/apicdef.h>
 #include <linux/atomic.h>
 #include <asm/fixmap.h>
diff --git a/arch/x86/include/asm/arch_hweight.h b/arch/x86/include/asm/arch_hweight.h
index 259a7c1ef709..02e799fa43d1 100644
--- a/arch/x86/include/asm/arch_hweight.h
+++ b/arch/x86/include/asm/arch_hweight.h
@@ -1,6 +1,8 @@
 #ifndef _ASM_X86_HWEIGHT_H
 #define _ASM_X86_HWEIGHT_H
 
+#include <asm/cpufeatures.h>
+
 #ifdef CONFIG_64BIT
 /* popcnt %edi, %eax -- redundant REX prefix for alignment */
 #define POPCNT32 ".byte 0xf3,0x40,0x0f,0xb8,0xc7"
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index cfe3b954d5e4..7766d1cf096e 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -91,7 +91,7 @@ set_bit(long nr, volatile unsigned long *addr)
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static inline void __set_bit(long nr, volatile unsigned long *addr)
+static __always_inline void __set_bit(long nr, volatile unsigned long *addr)
 {
         asm volatile("bts %1,%0" : ADDR : "Ir" (nr) : "memory");
 }
@@ -128,13 +128,13 @@ clear_bit(long nr, volatile unsigned long *addr)
  * clear_bit() is atomic and implies release semantics before the memory
  * operation. It can be used for an unlock.
  */
-static inline void clear_bit_unlock(long nr, volatile unsigned long *addr)
+static __always_inline void clear_bit_unlock(long nr, volatile unsigned long *addr)
 {
         barrier();
         clear_bit(nr, addr);
 }
 
-static inline void __clear_bit(long nr, volatile unsigned long *addr)
+static __always_inline void __clear_bit(long nr, volatile unsigned long *addr)
 {
         asm volatile("btr %1,%0" : ADDR : "Ir" (nr));
 }
@@ -151,7 +151,7 @@ static inline void __clear_bit(long nr, volatile unsigned long *addr)
  * No memory barrier is required here, because x86 cannot reorder stores past
  * older loads. Same principle as spin_unlock.
  */
-static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr)
+static __always_inline void __clear_bit_unlock(long nr, volatile unsigned long *addr)
 {
         barrier();
         __clear_bit(nr, addr);
@@ -166,7 +166,7 @@ static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr)
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static inline void __change_bit(long nr, volatile unsigned long *addr)
+static __always_inline void __change_bit(long nr, volatile unsigned long *addr)
 {
         asm volatile("btc %1,%0" : ADDR : "Ir" (nr));
 }
@@ -180,7 +180,7 @@ static inline void __change_bit(long nr, volatile unsigned long *addr)
  * Note that @nr may be almost arbitrarily large; this function is not
  * restricted to acting on a single-word quantity.
  */
-static inline void change_bit(long nr, volatile unsigned long *addr)
+static __always_inline void change_bit(long nr, volatile unsigned long *addr)
 {
         if (IS_IMMEDIATE(nr)) {
                 asm volatile(LOCK_PREFIX "xorb %1,%0"
@@ -201,7 +201,7 @@ static inline void change_bit(long nr, volatile unsigned long *addr)
  * This operation is atomic and cannot be reordered.
  * It also implies a memory barrier.
  */
-static inline int test_and_set_bit(long nr, volatile unsigned long *addr)
+static __always_inline int test_and_set_bit(long nr, volatile unsigned long *addr)
 {
         GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", "c");
 }
@@ -228,7 +228,7 @@ test_and_set_bit_lock(long nr, volatile unsigned long *addr)
  * If two examples of this operation race, one can appear to succeed
  * but actually fail.  You must protect multiple accesses with a lock.
  */
-static inline int __test_and_set_bit(long nr, volatile unsigned long *addr)
+static __always_inline int __test_and_set_bit(long nr, volatile unsigned long *addr)
 {
         int oldbit;
 
@@ -247,7 +247,7 @@ static inline int __test_and_set_bit(long nr, volatile unsigned long *addr)
  * This operation is atomic and cannot be reordered.
  * It also implies a memory barrier.
  */
-static inline int test_and_clear_bit(long nr, volatile unsigned long *addr)
+static __always_inline int test_and_clear_bit(long nr, volatile unsigned long *addr)
 {
         GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", "c");
 }
@@ -268,7 +268,7 @@ static inline int test_and_clear_bit(long nr, volatile unsigned long *addr)
  * accessed from a hypervisor on the same CPU if running in a VM: don't change
  * this without also updating arch/x86/kernel/kvm.c
  */
-static inline int __test_and_clear_bit(long nr, volatile unsigned long *addr)
+static __always_inline int __test_and_clear_bit(long nr, volatile unsigned long *addr)
 {
         int oldbit;
 
@@ -280,7 +280,7 @@ static inline int __test_and_clear_bit(long nr, volatile unsigned long *addr)
 }
 
 /* WARNING: non atomic and it can be reordered! */
-static inline int __test_and_change_bit(long nr, volatile unsigned long *addr)
+static __always_inline int __test_and_change_bit(long nr, volatile unsigned long *addr)
 {
         int oldbit;
 
@@ -300,7 +300,7 @@ static inline int __test_and_change_bit(long nr, volatile unsigned long *addr)
  * This operation is atomic and cannot be reordered.
  * It also implies a memory barrier.
  */
-static inline int test_and_change_bit(long nr, volatile unsigned long *addr)
+static __always_inline int test_and_change_bit(long nr, volatile unsigned long *addr)
 {
         GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", "c");
 }
@@ -311,7 +311,7 @@ static __always_inline int constant_test_bit(long nr, const volatile unsigned lo
                 (addr[nr >> _BITOPS_LONG_SHIFT])) != 0;
 }
 
-static inline int variable_test_bit(long nr, volatile const unsigned long *addr)
+static __always_inline int variable_test_bit(long nr, volatile const unsigned long *addr)
 {
         int oldbit;
 
@@ -343,7 +343,7 @@ static int test_bit(int nr, const volatile unsigned long *addr);
  *
  * Undefined if no bit exists, so code should check against 0 first.
  */
-static inline unsigned long __ffs(unsigned long word)
+static __always_inline unsigned long __ffs(unsigned long word)
 {
         asm("rep; bsf %1,%0"
                 : "=r" (word)
@@ -357,7 +357,7 @@ static inline unsigned long __ffs(unsigned long word)
  *
  * Undefined if no zero exists, so code should check against ~0UL first.
  */
-static inline unsigned long ffz(unsigned long word)
+static __always_inline unsigned long ffz(unsigned long word)
 {
         asm("rep; bsf %1,%0"
                 : "=r" (word)
@@ -371,7 +371,7 @@ static inline unsigned long ffz(unsigned long word)
  *
  * Undefined if no set bit exists, so code should check against 0 first.
  */
-static inline unsigned long __fls(unsigned long word)
+static __always_inline unsigned long __fls(unsigned long word)
 {
         asm("bsr %1,%0"
             : "=r" (word)
@@ -393,7 +393,7 @@ static inline unsigned long __fls(unsigned long word)
  * set bit if value is nonzero. The first (least significant) bit
  * is at position 1.
  */
-static inline int ffs(int x)
+static __always_inline int ffs(int x)
 {
         int r;
 
@@ -434,7 +434,7 @@ static inline int ffs(int x)
  * set bit if value is nonzero. The last (most significant) bit is
  * at position 32.
  */
-static inline int fls(int x)
+static __always_inline int fls(int x)
 {
         int r;
 
diff --git a/arch/x86/include/asm/clocksource.h b/arch/x86/include/asm/clocksource.h
index eda81dc0f4ae..d194266acb28 100644
--- a/arch/x86/include/asm/clocksource.h
+++ b/arch/x86/include/asm/clocksource.h
@@ -3,10 +3,11 @@
 #ifndef _ASM_X86_CLOCKSOURCE_H
 #define _ASM_X86_CLOCKSOURCE_H
 
-#define VCLOCK_NONE 0  /* No vDSO clock available.      */
-#define VCLOCK_TSC  1  /* vDSO should use vread_tsc.    */
-#define VCLOCK_HPET 2  /* vDSO should use vread_hpet.   */
-#define VCLOCK_PVCLOCK 3 /* vDSO should use vread_pvclock. */
+#define VCLOCK_NONE     0  /* No vDSO clock available.  */
+#define VCLOCK_TSC      1  /* vDSO should use vread_tsc.        */
+#define VCLOCK_HPET     2  /* vDSO should use vread_hpet.       */
+#define VCLOCK_PVCLOCK  3 /* vDSO should use vread_pvclock. */
+#define VCLOCK_MAX      3
 
 struct arch_clocksource_data {
         int vclock_mode;
diff --git a/arch/x86/include/asm/cmpxchg.h b/arch/x86/include/asm/cmpxchg.h
index ad19841eddfe..9733361fed6f 100644
--- a/arch/x86/include/asm/cmpxchg.h
+++ b/arch/x86/include/asm/cmpxchg.h
@@ -2,6 +2,7 @@
 #define ASM_X86_CMPXCHG_H
 
 #include <linux/compiler.h>
+#include <asm/cpufeatures.h>
 #include <asm/alternative.h> /* Provides LOCK_PREFIX */
 
 /*
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 7ad8c9464297..68e4e8258b84 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -1,288 +1,7 @@
-/*
- * Defines x86 CPU feature bits
- */
 #ifndef _ASM_X86_CPUFEATURE_H
 #define _ASM_X86_CPUFEATURE_H
 
-#ifndef _ASM_X86_REQUIRED_FEATURES_H
-#include <asm/required-features.h>
-#endif
-
-#ifndef _ASM_X86_DISABLED_FEATURES_H
-#include <asm/disabled-features.h>
-#endif
-
-#define NCAPINTS        16      /* N 32-bit words worth of info */
-#define NBUGINTS        1       /* N 32-bit bug flags */
-
-/*
- * Note: If the comment begins with a quoted string, that string is used
- * in /proc/cpuinfo instead of the macro name.  If the string is "",
- * this feature bit is not displayed in /proc/cpuinfo at all.
- */
-
-/* Intel-defined CPU features, CPUID level 0x00000001 (edx), word 0 */
-#define X86_FEATURE_FPU         ( 0*32+ 0) /* Onboard FPU */
-#define X86_FEATURE_VME         ( 0*32+ 1) /* Virtual Mode Extensions */
-#define X86_FEATURE_DE          ( 0*32+ 2) /* Debugging Extensions */
-#define X86_FEATURE_PSE         ( 0*32+ 3) /* Page Size Extensions */
-#define X86_FEATURE_TSC         ( 0*32+ 4) /* Time Stamp Counter */
-#define X86_FEATURE_MSR         ( 0*32+ 5) /* Model-Specific Registers */
-#define X86_FEATURE_PAE         ( 0*32+ 6) /* Physical Address Extensions */
-#define X86_FEATURE_MCE         ( 0*32+ 7) /* Machine Check Exception */
-#define X86_FEATURE_CX8         ( 0*32+ 8) /* CMPXCHG8 instruction */
-#define X86_FEATURE_APIC        ( 0*32+ 9) /* Onboard APIC */
-#define X86_FEATURE_SEP         ( 0*32+11) /* SYSENTER/SYSEXIT */
-#define X86_FEATURE_MTRR        ( 0*32+12) /* Memory Type Range Registers */
-#define X86_FEATURE_PGE         ( 0*32+13) /* Page Global Enable */
-#define X86_FEATURE_MCA         ( 0*32+14) /* Machine Check Architecture */
-#define X86_FEATURE_CMOV        ( 0*32+15) /* CMOV instructions */
-                                          /* (plus FCMOVcc, FCOMI with FPU) */
-#define X86_FEATURE_PAT         ( 0*32+16) /* Page Attribute Table */
-#define X86_FEATURE_PSE36       ( 0*32+17) /* 36-bit PSEs */
-#define X86_FEATURE_PN          ( 0*32+18) /* Processor serial number */
-#define X86_FEATURE_CLFLUSH     ( 0*32+19) /* CLFLUSH instruction */
-#define X86_FEATURE_DS          ( 0*32+21) /* "dts" Debug Store */
-#define X86_FEATURE_ACPI        ( 0*32+22) /* ACPI via MSR */
-#define X86_FEATURE_MMX         ( 0*32+23) /* Multimedia Extensions */
-#define X86_FEATURE_FXSR        ( 0*32+24) /* FXSAVE/FXRSTOR, CR4.OSFXSR */
-#define X86_FEATURE_XMM         ( 0*32+25) /* "sse" */
-#define X86_FEATURE_XMM2        ( 0*32+26) /* "sse2" */
-#define X86_FEATURE_SELFSNOOP   ( 0*32+27) /* "ss" CPU self snoop */
-#define X86_FEATURE_HT          ( 0*32+28) /* Hyper-Threading */
-#define X86_FEATURE_ACC         ( 0*32+29) /* "tm" Automatic clock control */
-#define X86_FEATURE_IA64        ( 0*32+30) /* IA-64 processor */
-#define X86_FEATURE_PBE         ( 0*32+31) /* Pending Break Enable */
-
-/* AMD-defined CPU features, CPUID level 0x80000001, word 1 */
-/* Don't duplicate feature flags which are redundant with Intel! */
-#define X86_FEATURE_SYSCALL     ( 1*32+11) /* SYSCALL/SYSRET */
-#define X86_FEATURE_MP          ( 1*32+19) /* MP Capable. */
-#define X86_FEATURE_NX          ( 1*32+20) /* Execute Disable */
-#define X86_FEATURE_MMXEXT      ( 1*32+22) /* AMD MMX extensions */
-#define X86_FEATURE_FXSR_OPT    ( 1*32+25) /* FXSAVE/FXRSTOR optimizations */
-#define X86_FEATURE_GBPAGES     ( 1*32+26) /* "pdpe1gb" GB pages */
-#define X86_FEATURE_RDTSCP      ( 1*32+27) /* RDTSCP */
-#define X86_FEATURE_LM          ( 1*32+29) /* Long Mode (x86-64) */
-#define X86_FEATURE_3DNOWEXT    ( 1*32+30) /* AMD 3DNow! extensions */
-#define X86_FEATURE_3DNOW       ( 1*32+31) /* 3DNow! */
-
-/* Transmeta-defined CPU features, CPUID level 0x80860001, word 2 */
-#define X86_FEATURE_RECOVERY    ( 2*32+ 0) /* CPU in recovery mode */
-#define X86_FEATURE_LONGRUN     ( 2*32+ 1) /* Longrun power control */
-#define X86_FEATURE_LRTI        ( 2*32+ 3) /* LongRun table interface */
-
-/* Other features, Linux-defined mapping, word 3 */
-/* This range is used for feature bits which conflict or are synthesized */
-#define X86_FEATURE_CXMMX       ( 3*32+ 0) /* Cyrix MMX extensions */
-#define X86_FEATURE_K6_MTRR     ( 3*32+ 1) /* AMD K6 nonstandard MTRRs */
-#define X86_FEATURE_CYRIX_ARR   ( 3*32+ 2) /* Cyrix ARRs (= MTRRs) */
-#define X86_FEATURE_CENTAUR_MCR ( 3*32+ 3) /* Centaur MCRs (= MTRRs) */
-/* cpu types for specific tunings: */
-#define X86_FEATURE_K8          ( 3*32+ 4) /* "" Opteron, Athlon64 */
-#define X86_FEATURE_K7          ( 3*32+ 5) /* "" Athlon */
-#define X86_FEATURE_P3          ( 3*32+ 6) /* "" P3 */
-#define X86_FEATURE_P4          ( 3*32+ 7) /* "" P4 */
-#define X86_FEATURE_CONSTANT_TSC ( 3*32+ 8) /* TSC ticks at a constant rate */
-#define X86_FEATURE_UP          ( 3*32+ 9) /* smp kernel running on up */
-/* free, was #define X86_FEATURE_FXSAVE_LEAK ( 3*32+10) * "" FXSAVE leaks FOP/FIP/FOP */
-#define X86_FEATURE_ARCH_PERFMON ( 3*32+11) /* Intel Architectural PerfMon */
-#define X86_FEATURE_PEBS        ( 3*32+12) /* Precise-Event Based Sampling */
-#define X86_FEATURE_BTS         ( 3*32+13) /* Branch Trace Store */
-#define X86_FEATURE_SYSCALL32   ( 3*32+14) /* "" syscall in ia32 userspace */
-#define X86_FEATURE_SYSENTER32  ( 3*32+15) /* "" sysenter in ia32 userspace */
-#define X86_FEATURE_REP_GOOD    ( 3*32+16) /* rep microcode works well */
-#define X86_FEATURE_MFENCE_RDTSC ( 3*32+17) /* "" Mfence synchronizes RDTSC */
-#define X86_FEATURE_LFENCE_RDTSC ( 3*32+18) /* "" Lfence synchronizes RDTSC */
-/* free, was #define X86_FEATURE_11AP   ( 3*32+19) * "" Bad local APIC aka 11AP */
-#define X86_FEATURE_NOPL        ( 3*32+20) /* The NOPL (0F 1F) instructions */
-#define X86_FEATURE_ALWAYS      ( 3*32+21) /* "" Always-present feature */
-#define X86_FEATURE_XTOPOLOGY   ( 3*32+22) /* cpu topology enum extensions */
-#define X86_FEATURE_TSC_RELIABLE ( 3*32+23) /* TSC is known to be reliable */
-#define X86_FEATURE_NONSTOP_TSC ( 3*32+24) /* TSC does not stop in C states */
-/* free, was #define X86_FEATURE_CLFLUSH_MONITOR ( 3*32+25) * "" clflush reqd with monitor */
-#define X86_FEATURE_EXTD_APICID ( 3*32+26) /* has extended APICID (8 bits) */
-#define X86_FEATURE_AMD_DCM     ( 3*32+27) /* multi-node processor */
-#define X86_FEATURE_APERFMPERF  ( 3*32+28) /* APERFMPERF */
-#define X86_FEATURE_EAGER_FPU   ( 3*32+29) /* "eagerfpu" Non lazy FPU restore */
-#define X86_FEATURE_NONSTOP_TSC_S3 ( 3*32+30) /* TSC doesn't stop in S3 state */
-
-/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
-#define X86_FEATURE_XMM3        ( 4*32+ 0) /* "pni" SSE-3 */
-#define X86_FEATURE_PCLMULQDQ   ( 4*32+ 1) /* PCLMULQDQ instruction */
-#define X86_FEATURE_DTES64      ( 4*32+ 2) /* 64-bit Debug Store */
-#define X86_FEATURE_MWAIT       ( 4*32+ 3) /* "monitor" Monitor/Mwait support */
-#define X86_FEATURE_DSCPL       ( 4*32+ 4) /* "ds_cpl" CPL Qual. Debug Store */
-#define X86_FEATURE_VMX         ( 4*32+ 5) /* Hardware virtualization */
-#define X86_FEATURE_SMX         ( 4*32+ 6) /* Safer mode */
-#define X86_FEATURE_EST         ( 4*32+ 7) /* Enhanced SpeedStep */
-#define X86_FEATURE_TM2         ( 4*32+ 8) /* Thermal Monitor 2 */
-#define X86_FEATURE_SSSE3       ( 4*32+ 9) /* Supplemental SSE-3 */
-#define X86_FEATURE_CID         ( 4*32+10) /* Context ID */
-#define X86_FEATURE_SDBG        ( 4*32+11) /* Silicon Debug */
-#define X86_FEATURE_FMA         ( 4*32+12) /* Fused multiply-add */
-#define X86_FEATURE_CX16        ( 4*32+13) /* CMPXCHG16B */
-#define X86_FEATURE_XTPR        ( 4*32+14) /* Send Task Priority Messages */
-#define X86_FEATURE_PDCM        ( 4*32+15) /* Performance Capabilities */
-#define X86_FEATURE_PCID        ( 4*32+17) /* Process Context Identifiers */
-#define X86_FEATURE_DCA         ( 4*32+18) /* Direct Cache Access */
-#define X86_FEATURE_XMM4_1      ( 4*32+19) /* "sse4_1" SSE-4.1 */
-#define X86_FEATURE_XMM4_2      ( 4*32+20) /* "sse4_2" SSE-4.2 */
-#define X86_FEATURE_X2APIC      ( 4*32+21) /* x2APIC */
-#define X86_FEATURE_MOVBE       ( 4*32+22) /* MOVBE instruction */
-#define X86_FEATURE_POPCNT      ( 4*32+23) /* POPCNT instruction */
-#define X86_FEATURE_TSC_DEADLINE_TIMER  ( 4*32+24) /* Tsc deadline timer */
-#define X86_FEATURE_AES         ( 4*32+25) /* AES instructions */
-#define X86_FEATURE_XSAVE       ( 4*32+26) /* XSAVE/XRSTOR/XSETBV/XGETBV */
-#define X86_FEATURE_OSXSAVE     ( 4*32+27) /* "" XSAVE enabled in the OS */
-#define X86_FEATURE_AVX         ( 4*32+28) /* Advanced Vector Extensions */
-#define X86_FEATURE_F16C        ( 4*32+29) /* 16-bit fp conversions */
-#define X86_FEATURE_RDRAND      ( 4*32+30) /* The RDRAND instruction */
-#define X86_FEATURE_HYPERVISOR  ( 4*32+31) /* Running on a hypervisor */
-
-/* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */
-#define X86_FEATURE_XSTORE      ( 5*32+ 2) /* "rng" RNG present (xstore) */
-#define X86_FEATURE_XSTORE_EN   ( 5*32+ 3) /* "rng_en" RNG enabled */
-#define X86_FEATURE_XCRYPT      ( 5*32+ 6) /* "ace" on-CPU crypto (xcrypt) */
-#define X86_FEATURE_XCRYPT_EN   ( 5*32+ 7) /* "ace_en" on-CPU crypto enabled */
-#define X86_FEATURE_ACE2        ( 5*32+ 8) /* Advanced Cryptography Engine v2 */
-#define X86_FEATURE_ACE2_EN     ( 5*32+ 9) /* ACE v2 enabled */
-#define X86_FEATURE_PHE         ( 5*32+10) /* PadLock Hash Engine */
-#define X86_FEATURE_PHE_EN      ( 5*32+11) /* PHE enabled */
-#define X86_FEATURE_PMM         ( 5*32+12) /* PadLock Montgomery Multiplier */
-#define X86_FEATURE_PMM_EN      ( 5*32+13) /* PMM enabled */
-
-/* More extended AMD flags: CPUID level 0x80000001, ecx, word 6 */
-#define X86_FEATURE_LAHF_LM     ( 6*32+ 0) /* LAHF/SAHF in long mode */
-#define X86_FEATURE_CMP_LEGACY  ( 6*32+ 1) /* If yes HyperThreading not valid */
-#define X86_FEATURE_SVM         ( 6*32+ 2) /* Secure virtual machine */
-#define X86_FEATURE_EXTAPIC     ( 6*32+ 3) /* Extended APIC space */
-#define X86_FEATURE_CR8_LEGACY  ( 6*32+ 4) /* CR8 in 32-bit mode */
-#define X86_FEATURE_ABM         ( 6*32+ 5) /* Advanced bit manipulation */
-#define X86_FEATURE_SSE4A       ( 6*32+ 6) /* SSE-4A */
-#define X86_FEATURE_MISALIGNSSE ( 6*32+ 7) /* Misaligned SSE mode */
-#define X86_FEATURE_3DNOWPREFETCH ( 6*32+ 8) /* 3DNow prefetch instructions */
-#define X86_FEATURE_OSVW        ( 6*32+ 9) /* OS Visible Workaround */
-#define X86_FEATURE_IBS         ( 6*32+10) /* Instruction Based Sampling */
-#define X86_FEATURE_XOP         ( 6*32+11) /* extended AVX instructions */
-#define X86_FEATURE_SKINIT      ( 6*32+12) /* SKINIT/STGI instructions */
-#define X86_FEATURE_WDT         ( 6*32+13) /* Watchdog timer */
-#define X86_FEATURE_LWP         ( 6*32+15) /* Light Weight Profiling */
-#define X86_FEATURE_FMA4        ( 6*32+16) /* 4 operands MAC instructions */
-#define X86_FEATURE_TCE         ( 6*32+17) /* translation cache extension */
-#define X86_FEATURE_NODEID_MSR  ( 6*32+19) /* NodeId MSR */
-#define X86_FEATURE_TBM         ( 6*32+21) /* trailing bit manipulations */
-#define X86_FEATURE_TOPOEXT     ( 6*32+22) /* topology extensions CPUID leafs */
-#define X86_FEATURE_PERFCTR_CORE ( 6*32+23) /* core performance counter extensions */
-#define X86_FEATURE_PERFCTR_NB  ( 6*32+24) /* NB performance counter extensions */
-#define X86_FEATURE_BPEXT       (6*32+26) /* data breakpoint extension */
-#define X86_FEATURE_PERFCTR_L2  ( 6*32+28) /* L2 performance counter extensions */
-#define X86_FEATURE_MWAITX      ( 6*32+29) /* MWAIT extension (MONITORX/MWAITX) */
-
-/*
- * Auxiliary flags: Linux defined - For features scattered in various
- * CPUID levels like 0x6, 0xA etc, word 7.
- *
- * Reuse free bits when adding new feature flags!
- */
-
-#define X86_FEATURE_CPB         ( 7*32+ 2) /* AMD Core Performance Boost */
-#define X86_FEATURE_EPB         ( 7*32+ 3) /* IA32_ENERGY_PERF_BIAS support */
-
-#define X86_FEATURE_HW_PSTATE   ( 7*32+ 8) /* AMD HW-PState */
-#define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */
-
-#define X86_FEATURE_INTEL_PT    ( 7*32+15) /* Intel Processor Trace */
-
-/* Virtualization flags: Linux defined, word 8 */
-#define X86_FEATURE_TPR_SHADOW  ( 8*32+ 0) /* Intel TPR Shadow */
-#define X86_FEATURE_VNMI        ( 8*32+ 1) /* Intel Virtual NMI */
-#define X86_FEATURE_FLEXPRIORITY ( 8*32+ 2) /* Intel FlexPriority */
-#define X86_FEATURE_EPT         ( 8*32+ 3) /* Intel Extended Page Table */
-#define X86_FEATURE_VPID        ( 8*32+ 4) /* Intel Virtual Processor ID */
-
-#define X86_FEATURE_VMMCALL     ( 8*32+15) /* Prefer vmmcall to vmcall */
-#define X86_FEATURE_XENPV       ( 8*32+16) /* "" Xen paravirtual guest */
-
-
-/* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */
-#define X86_FEATURE_FSGSBASE    ( 9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/
-#define X86_FEATURE_TSC_ADJUST  ( 9*32+ 1) /* TSC adjustment MSR 0x3b */
-#define X86_FEATURE_BMI1        ( 9*32+ 3) /* 1st group bit manipulation extensions */
-#define X86_FEATURE_HLE         ( 9*32+ 4) /* Hardware Lock Elision */
-#define X86_FEATURE_AVX2        ( 9*32+ 5) /* AVX2 instructions */
-#define X86_FEATURE_SMEP        ( 9*32+ 7) /* Supervisor Mode Execution Protection */
-#define X86_FEATURE_BMI2        ( 9*32+ 8) /* 2nd group bit manipulation extensions */
-#define X86_FEATURE_ERMS        ( 9*32+ 9) /* Enhanced REP MOVSB/STOSB */
-#define X86_FEATURE_INVPCID     ( 9*32+10) /* Invalidate Processor Context ID */
-#define X86_FEATURE_RTM         ( 9*32+11) /* Restricted Transactional Memory */
-#define X86_FEATURE_CQM         ( 9*32+12) /* Cache QoS Monitoring */
-#define X86_FEATURE_MPX         ( 9*32+14) /* Memory Protection Extension */
-#define X86_FEATURE_AVX512F     ( 9*32+16) /* AVX-512 Foundation */
-#define X86_FEATURE_RDSEED      ( 9*32+18) /* The RDSEED instruction */
-#define X86_FEATURE_ADX         ( 9*32+19) /* The ADCX and ADOX instructions */
-#define X86_FEATURE_SMAP        ( 9*32+20) /* Supervisor Mode Access Prevention */
-#define X86_FEATURE_PCOMMIT     ( 9*32+22) /* PCOMMIT instruction */
-#define X86_FEATURE_CLFLUSHOPT  ( 9*32+23) /* CLFLUSHOPT instruction */
-#define X86_FEATURE_CLWB        ( 9*32+24) /* CLWB instruction */
-#define X86_FEATURE_AVX512PF    ( 9*32+26) /* AVX-512 Prefetch */
-#define X86_FEATURE_AVX512ER    ( 9*32+27) /* AVX-512 Exponential and Reciprocal */
-#define X86_FEATURE_AVX512CD    ( 9*32+28) /* AVX-512 Conflict Detection */
-#define X86_FEATURE_SHA_NI      ( 9*32+29) /* SHA1/SHA256 Instruction Extensions */
-
-/* Extended state features, CPUID level 0x0000000d:1 (eax), word 10 */
-#define X86_FEATURE_XSAVEOPT    (10*32+ 0) /* XSAVEOPT */
-#define X86_FEATURE_XSAVEC      (10*32+ 1) /* XSAVEC */
-#define X86_FEATURE_XGETBV1     (10*32+ 2) /* XGETBV with ECX = 1 */
-#define X86_FEATURE_XSAVES      (10*32+ 3) /* XSAVES/XRSTORS */
-
-/* Intel-defined CPU QoS Sub-leaf, CPUID level 0x0000000F:0 (edx), word 11 */
-#define X86_FEATURE_CQM_LLC     (11*32+ 1) /* LLC QoS if 1 */
-
-/* Intel-defined CPU QoS Sub-leaf, CPUID level 0x0000000F:1 (edx), word 12 */
-#define X86_FEATURE_CQM_OCCUP_LLC (12*32+ 0) /* LLC occupancy monitoring if 1 */
-
-/* AMD-defined CPU features, CPUID level 0x80000008 (ebx), word 13 */
-#define X86_FEATURE_CLZERO      (13*32+0) /* CLZERO instruction */
-
-/* Thermal and Power Management Leaf, CPUID level 0x00000006 (eax), word 14 */
-#define X86_FEATURE_DTHERM      (14*32+ 0) /* Digital Thermal Sensor */
-#define X86_FEATURE_IDA         (14*32+ 1) /* Intel Dynamic Acceleration */
-#define X86_FEATURE_ARAT        (14*32+ 2) /* Always Running APIC Timer */
-#define X86_FEATURE_PLN         (14*32+ 4) /* Intel Power Limit Notification */
-#define X86_FEATURE_PTS         (14*32+ 6) /* Intel Package Thermal Status */
-#define X86_FEATURE_HWP         (14*32+ 7) /* Intel Hardware P-states */
-#define X86_FEATURE_HWP_NOTIFY  (14*32+ 8) /* HWP Notification */
-#define X86_FEATURE_HWP_ACT_WINDOW (14*32+ 9) /* HWP Activity Window */
-#define X86_FEATURE_HWP_EPP     (14*32+10) /* HWP Energy Perf. Preference */
-#define X86_FEATURE_HWP_PKG_REQ (14*32+11) /* HWP Package Level Request */
-
-/* AMD SVM Feature Identification, CPUID level 0x8000000a (edx), word 15 */
-#define X86_FEATURE_NPT         (15*32+ 0) /* Nested Page Table support */
-#define X86_FEATURE_LBRV        (15*32+ 1) /* LBR Virtualization support */
-#define X86_FEATURE_SVML        (15*32+ 2) /* "svm_lock" SVM locking MSR */
-#define X86_FEATURE_NRIPS       (15*32+ 3) /* "nrip_save" SVM next_rip save */
-#define X86_FEATURE_TSCRATEMSR  (15*32+ 4) /* "tsc_scale" TSC scaling support */
-#define X86_FEATURE_VMCBCLEAN   (15*32+ 5) /* "vmcb_clean" VMCB clean bits support */
-#define X86_FEATURE_FLUSHBYASID (15*32+ 6) /* flush-by-ASID support */
-#define X86_FEATURE_DECODEASSISTS (15*32+ 7) /* Decode Assists support */
-#define X86_FEATURE_PAUSEFILTER (15*32+10) /* filtered pause intercept */
-#define X86_FEATURE_PFTHRESHOLD (15*32+12) /* pause filter threshold */
-
-/*
- * BUG word(s)
- */
-#define X86_BUG(x)              (NCAPINTS*32 + (x))
-
-#define X86_BUG_F00F            X86_BUG(0) /* Intel F00F */
-#define X86_BUG_FDIV            X86_BUG(1) /* FPU FDIV */
-#define X86_BUG_COMA            X86_BUG(2) /* Cyrix 6x86 coma */
-#define X86_BUG_AMD_TLB_MMATCH  X86_BUG(3) /* "tlb_mmatch" AMD Erratum 383 */
-#define X86_BUG_AMD_APIC_C1E    X86_BUG(4) /* "apic_c1e" AMD Erratum 400 */
-#define X86_BUG_11AP            X86_BUG(5) /* Bad local APIC aka 11AP */
-#define X86_BUG_FXSAVE_LEAK     X86_BUG(6) /* FXSAVE leaks FOP/FIP/FOP */
-#define X86_BUG_CLFLUSH_MONITOR X86_BUG(7) /* AAI65, CLFLUSH required before MONITOR */
-#define X86_BUG_SYSRET_SS_ATTRS X86_BUG(8) /* SYSRET doesn't fix up SS attrs */
+#include <asm/processor.h>
 
 #if defined(__KERNEL__) && !defined(__ASSEMBLY__)
 
@@ -369,8 +88,7 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
  * is not relevant.
  */
 #define cpu_feature_enabled(bit)        \
-        (__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 :  \
-         cpu_has(&boot_cpu_data, bit))
+        (__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : static_cpu_has(bit))
 
 #define boot_cpu_has(bit)       cpu_has(&boot_cpu_data, bit)
 
@@ -406,106 +124,19 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 #define cpu_has_osxsave         boot_cpu_has(X86_FEATURE_OSXSAVE)
 #define cpu_has_hypervisor      boot_cpu_has(X86_FEATURE_HYPERVISOR)
 /*
- * Do not add any more of those clumsy macros - use static_cpu_has_safe() for
+ * Do not add any more of those clumsy macros - use static_cpu_has() for
  * fast paths and boot_cpu_has() otherwise!
  */
 
-#if __GNUC__ >= 4 && defined(CONFIG_X86_FAST_FEATURE_TESTS)
-extern void warn_pre_alternatives(void);
-extern bool __static_cpu_has_safe(u16 bit);
-
+#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_X86_FAST_FEATURE_TESTS)
 /*
  * Static testing of CPU features.  Used the same as boot_cpu_has().
- * These are only valid after alternatives have run, but will statically
- * patch the target code for additional performance.
+ * These will statically patch the target code for additional
+ * performance.
  */
-static __always_inline __pure bool __static_cpu_has(u16 bit)
-{
-#ifdef CC_HAVE_ASM_GOTO
-
-#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS
-
-                /*
-                 * Catch too early usage of this before alternatives
-                 * have run.
-                 */
-                asm_volatile_goto("1: jmp %l[t_warn]\n"
-                         "2:\n"
-                         ".section .altinstructions,\"a\"\n"
-                         " .long 1b - .\n"
-                         " .long 0\n"           /* no replacement */
-                         " .word %P0\n"         /* 1: do replace */
-                         " .byte 2b - 1b\n"     /* source len */
-                         " .byte 0\n"           /* replacement len */
-                         " .byte 0\n"           /* pad len */
-                         ".previous\n"
-                         /* skipping size check since replacement size = 0 */
-                         : : "i" (X86_FEATURE_ALWAYS) : : t_warn);
-
-#endif
-
-                asm_volatile_goto("1: jmp %l[t_no]\n"
-                         "2:\n"
-                         ".section .altinstructions,\"a\"\n"
-                         " .long 1b - .\n"
-                         " .long 0\n"           /* no replacement */
-                         " .word %P0\n"         /* feature bit */
-                         " .byte 2b - 1b\n"     /* source len */
-                         " .byte 0\n"           /* replacement len */
-                         " .byte 0\n"           /* pad len */
-                         ".previous\n"
-                         /* skipping size check since replacement size = 0 */
-                         : : "i" (bit) : : t_no);
-                return true;
-        t_no:
-                return false;
-
-#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS
-        t_warn:
-                warn_pre_alternatives();
-                return false;
-#endif
-
-#else /* CC_HAVE_ASM_GOTO */
-
-                u8 flag;
-                /* Open-coded due to __stringify() in ALTERNATIVE() */
-                asm volatile("1: movb $0,%0\n"
-                             "2:\n"
-                             ".section .altinstructions,\"a\"\n"
-                             " .long 1b - .\n"
-                             " .long 3f - .\n"
-                             " .word %P1\n"             /* feature bit */
-                             " .byte 2b - 1b\n"         /* source len */
-                             " .byte 4f - 3f\n"         /* replacement len */
-                             " .byte 0\n"               /* pad len */
-                             ".previous\n"
-                             ".section .discard,\"aw\",@progbits\n"
-                             " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
-                             ".previous\n"
-                             ".section .altinstr_replacement,\"ax\"\n"
-                             "3: movb $1,%0\n"
-                             "4:\n"
-                             ".previous\n"
-                             : "=qm" (flag) : "i" (bit));
-                return flag;
-
-#endif /* CC_HAVE_ASM_GOTO */
-}
-
-#define static_cpu_has(bit)                                     \
-(                                                               \
-        __builtin_constant_p(boot_cpu_has(bit)) ?               \
-                boot_cpu_has(bit) :                             \
-        __builtin_constant_p(bit) ?                             \
-                __static_cpu_has(bit) :                         \
-                boot_cpu_has(bit)                               \
-)
-
-static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
+static __always_inline __pure bool _static_cpu_has(u16 bit)
 {
-#ifdef CC_HAVE_ASM_GOTO
-                asm_volatile_goto("1: jmp %l[t_dynamic]\n"
+                asm_volatile_goto("1: jmp 6f\n"
                          "2:\n"
                          ".skip -(((5f-4f) - (2b-1b)) > 0) * "
                                  "((5f-4f) - (2b-1b)),0x90\n"
@@ -530,66 +161,34 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
                          " .byte 0\n"                   /* repl len */
                          " .byte 0\n"                   /* pad len */
                          ".previous\n"
-                         : : "i" (bit), "i" (X86_FEATURE_ALWAYS)
-                         : : t_dynamic, t_no);
+                         ".section .altinstr_aux,\"ax\"\n"
+                         "6:\n"
+                         " testb %[bitnum],%[cap_byte]\n"
+                         " jnz %l[t_yes]\n"
+                         " jmp %l[t_no]\n"
+                         ".previous\n"
+                         : : "i" (bit), "i" (X86_FEATURE_ALWAYS),
+                             [bitnum] "i" (1 << (bit & 7)),
+                             [cap_byte] "m" (((const char *)boot_cpu_data.x86_capability)[bit >> 3])
+                         : : t_yes, t_no);
+        t_yes:
                 return true;
         t_no:
                 return false;
-        t_dynamic:
-                return __static_cpu_has_safe(bit);
-#else
-                u8 flag;
-                /* Open-coded due to __stringify() in ALTERNATIVE() */
-                asm volatile("1: movb $2,%0\n"
-                             "2:\n"
-                             ".section .altinstructions,\"a\"\n"
-                             " .long 1b - .\n"          /* src offset */
-                             " .long 3f - .\n"          /* repl offset */
-                             " .word %P2\n"             /* always replace */
-                             " .byte 2b - 1b\n"         /* source len */
-                             " .byte 4f - 3f\n"         /* replacement len */
-                             " .byte 0\n"               /* pad len */
-                             ".previous\n"
-                             ".section .discard,\"aw\",@progbits\n"
-                             " .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */
-                             ".previous\n"
-                             ".section .altinstr_replacement,\"ax\"\n"
-                             "3: movb $0,%0\n"
-                             "4:\n"
-                             ".previous\n"
-                             ".section .altinstructions,\"a\"\n"
-                             " .long 1b - .\n"          /* src offset */
-                             " .long 5f - .\n"          /* repl offset */
-                             " .word %P1\n"             /* feature bit */
-                             " .byte 4b - 3b\n"         /* src len */
-                             " .byte 6f - 5f\n"         /* repl len */
-                             " .byte 0\n"               /* pad len */
-                             ".previous\n"
-                             ".section .discard,\"aw\",@progbits\n"
-                             " .byte 0xff + (6f-5f) - (4b-3b)\n" /* size check */
-                             ".previous\n"
-                             ".section .altinstr_replacement,\"ax\"\n"
-                             "5: movb $1,%0\n"
-                             "6:\n"
-                             ".previous\n"
-                             : "=qm" (flag)
-                             : "i" (bit), "i" (X86_FEATURE_ALWAYS));
-                return (flag == 2 ? __static_cpu_has_safe(bit) : flag);
-#endif /* CC_HAVE_ASM_GOTO */
 }
 
-#define static_cpu_has_safe(bit)                                \
+#define static_cpu_has(bit)                                     \
 (                                                               \
         __builtin_constant_p(boot_cpu_has(bit)) ?               \
                 boot_cpu_has(bit) :                             \
-                _static_cpu_has_safe(bit)                       \
+                _static_cpu_has(bit)                            \
 )
 #else
 /*
- * gcc 3.x is too stupid to do the static test; fall back to dynamic.
+ * Fall back to dynamic for gcc versions which don't support asm goto. Should be
+ * a minority now anyway.
  */
 #define static_cpu_has(bit)             boot_cpu_has(bit)
-#define static_cpu_has_safe(bit)        boot_cpu_has(bit)
 #endif
 
 #define cpu_has_bug(c, bit)             cpu_has(c, (bit))
@@ -597,7 +196,6 @@ static __always_inline __pure bool _static_cpu_has_safe(u16 bit)
 #define clear_cpu_bug(c, bit)           clear_cpu_cap(c, (bit))
 
 #define static_cpu_has_bug(bit)         static_cpu_has((bit))
-#define static_cpu_has_bug_safe(bit)    static_cpu_has_safe((bit))
 #define boot_cpu_has_bug(bit)           cpu_has_bug(&boot_cpu_data, (bit))
 
 #define MAX_CPU_FEATURES                (NCAPINTS * 32)
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
new file mode 100644
index 000000000000..6663fae71b12
--- /dev/null
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -0,0 +1,289 @@
+#ifndef _ASM_X86_CPUFEATURES_H
+#define _ASM_X86_CPUFEATURES_H
+
+#ifndef _ASM_X86_REQUIRED_FEATURES_H
+#include <asm/required-features.h>
+#endif
+
+#ifndef _ASM_X86_DISABLED_FEATURES_H
+#include <asm/disabled-features.h>
+#endif
+
+/*
+ * Defines x86 CPU feature bits
+ */
+#define NCAPINTS        16      /* N 32-bit words worth of info */
+#define NBUGINTS        1       /* N 32-bit bug flags */
+
+/*
+ * Note: If the comment begins with a quoted string, that string is used
+ * in /proc/cpuinfo instead of the macro name.  If the string is "",
+ * this feature bit is not displayed in /proc/cpuinfo at all.
+ */
+
+/* Intel-defined CPU features, CPUID level 0x00000001 (edx), word 0 */
+#define X86_FEATURE_FPU         ( 0*32+ 0) /* Onboard FPU */
+#define X86_FEATURE_VME         ( 0*32+ 1) /* Virtual Mode Extensions */
+#define X86_FEATURE_DE          ( 0*32+ 2) /* Debugging Extensions */
+#define X86_FEATURE_PSE         ( 0*32+ 3) /* Page Size Extensions */
+#define X86_FEATURE_TSC         ( 0*32+ 4) /* Time Stamp Counter */
+#define X86_FEATURE_MSR         ( 0*32+ 5) /* Model-Specific Registers */
+#define X86_FEATURE_PAE         ( 0*32+ 6) /* Physical Address Extensions */
+#define X86_FEATURE_MCE         ( 0*32+ 7) /* Machine Check Exception */
+#define X86_FEATURE_CX8         ( 0*32+ 8) /* CMPXCHG8 instruction */
+#define X86_FEATURE_APIC        ( 0*32+ 9) /* Onboard APIC */
+#define X86_FEATURE_SEP         ( 0*32+11) /* SYSENTER/SYSEXIT */
+#define X86_FEATURE_MTRR        ( 0*32+12) /* Memory Type Range Registers */
+#define X86_FEATURE_PGE         ( 0*32+13) /* Page Global Enable */
+#define X86_FEATURE_MCA         ( 0*32+14) /* Machine Check Architecture */
+#define X86_FEATURE_CMOV        ( 0*32+15) /* CMOV instructions */
+                                          /* (plus FCMOVcc, FCOMI with FPU) */
+#define X86_FEATURE_PAT         ( 0*32+16) /* Page Attribute Table */
+#define X86_FEATURE_PSE36       ( 0*32+17) /* 36-bit PSEs */
+#define X86_FEATURE_PN          ( 0*32+18) /* Processor serial number */
+#define X86_FEATURE_CLFLUSH     ( 0*32+19) /* CLFLUSH instruction */
+#define X86_FEATURE_DS          ( 0*32+21) /* "dts" Debug Store */
+#define X86_FEATURE_ACPI        ( 0*32+22) /* ACPI via MSR */
+#define X86_FEATURE_MMX         ( 0*32+23) /* Multimedia Extensions */
+#define X86_FEATURE_FXSR        ( 0*32+24) /* FXSAVE/FXRSTOR, CR4.OSFXSR */
+#define X86_FEATURE_XMM         ( 0*32+25) /* "sse" */
+#define X86_FEATURE_XMM2        ( 0*32+26) /* "sse2" */
+#define X86_FEATURE_SELFSNOOP   ( 0*32+27) /* "ss" CPU self snoop */
+#define X86_FEATURE_HT          ( 0*32+28) /* Hyper-Threading */
+#define X86_FEATURE_ACC         ( 0*32+29) /* "tm" Automatic clock control */
+#define X86_FEATURE_IA64        ( 0*32+30) /* IA-64 processor */
+#define X86_FEATURE_PBE         ( 0*32+31) /* Pending Break Enable */
+
+/* AMD-defined CPU features, CPUID level 0x80000001, word 1 */
+/* Don't duplicate feature flags which are redundant with Intel! */
+#define X86_FEATURE_SYSCALL     ( 1*32+11) /* SYSCALL/SYSRET */
+#define X86_FEATURE_MP          ( 1*32+19) /* MP Capable. */
+#define X86_FEATURE_NX          ( 1*32+20) /* Execute Disable */
+#define X86_FEATURE_MMXEXT      ( 1*32+22) /* AMD MMX extensions */
+#define X86_FEATURE_FXSR_OPT    ( 1*32+25) /* FXSAVE/FXRSTOR optimizations */
+#define X86_FEATURE_GBPAGES     ( 1*32+26) /* "pdpe1gb" GB pages */
+#define X86_FEATURE_RDTSCP      ( 1*32+27) /* RDTSCP */
+#define X86_FEATURE_LM          ( 1*32+29) /* Long Mode (x86-64) */
+#define X86_FEATURE_3DNOWEXT    ( 1*32+30) /* AMD 3DNow! extensions */
+#define X86_FEATURE_3DNOW       ( 1*32+31) /* 3DNow! */
+
+/* Transmeta-defined CPU features, CPUID level 0x80860001, word 2 */
+#define X86_FEATURE_RECOVERY    ( 2*32+ 0) /* CPU in recovery mode */
+#define X86_FEATURE_LONGRUN     ( 2*32+ 1) /* Longrun power control */
+#define X86_FEATURE_LRTI        ( 2*32+ 3) /* LongRun table interface */
+
+/* Other features, Linux-defined mapping, word 3 */
+/* This range is used for feature bits which conflict or are synthesized */
+#define X86_FEATURE_CXMMX       ( 3*32+ 0) /* Cyrix MMX extensions */
+#define X86_FEATURE_K6_MTRR     ( 3*32+ 1) /* AMD K6 nonstandard MTRRs */
+#define X86_FEATURE_CYRIX_ARR   ( 3*32+ 2) /* Cyrix ARRs (= MTRRs) */
+#define X86_FEATURE_CENTAUR_MCR ( 3*32+ 3) /* Centaur MCRs (= MTRRs) */
+/* cpu types for specific tunings: */
+#define X86_FEATURE_K8          ( 3*32+ 4) /* "" Opteron, Athlon64 */
+#define X86_FEATURE_K7          ( 3*32+ 5) /* "" Athlon */
+#define X86_FEATURE_P3          ( 3*32+ 6) /* "" P3 */
+#define X86_FEATURE_P4          ( 3*32+ 7) /* "" P4 */
+#define X86_FEATURE_CONSTANT_TSC ( 3*32+ 8) /* TSC ticks at a constant rate */
+#define X86_FEATURE_UP          ( 3*32+ 9) /* smp kernel running on up */
+/* free, was #define X86_FEATURE_FXSAVE_LEAK ( 3*32+10) * "" FXSAVE leaks FOP/FIP/FOP */
+#define X86_FEATURE_ARCH_PERFMON ( 3*32+11) /* Intel Architectural PerfMon */
+#define X86_FEATURE_PEBS        ( 3*32+12) /* Precise-Event Based Sampling */
+#define X86_FEATURE_BTS         ( 3*32+13) /* Branch Trace Store */
+#define X86_FEATURE_SYSCALL32   ( 3*32+14) /* "" syscall in ia32 userspace */
+#define X86_FEATURE_SYSENTER32  ( 3*32+15) /* "" sysenter in ia32 userspace */
+#define X86_FEATURE_REP_GOOD    ( 3*32+16) /* rep microcode works well */
+#define X86_FEATURE_MFENCE_RDTSC ( 3*32+17) /* "" Mfence synchronizes RDTSC */
+#define X86_FEATURE_LFENCE_RDTSC ( 3*32+18) /* "" Lfence synchronizes RDTSC */
+/* free, was #define X86_FEATURE_11AP   ( 3*32+19) * "" Bad local APIC aka 11AP */
+#define X86_FEATURE_NOPL        ( 3*32+20) /* The NOPL (0F 1F) instructions */
+#define X86_FEATURE_ALWAYS      ( 3*32+21) /* "" Always-present feature */
+#define X86_FEATURE_XTOPOLOGY   ( 3*32+22) /* cpu topology enum extensions */
+#define X86_FEATURE_TSC_RELIABLE ( 3*32+23) /* TSC is known to be reliable */
+#define X86_FEATURE_NONSTOP_TSC ( 3*32+24) /* TSC does not stop in C states */
+/* free, was #define X86_FEATURE_CLFLUSH_MONITOR ( 3*32+25) * "" clflush reqd with monitor */
+#define X86_FEATURE_EXTD_APICID ( 3*32+26) /* has extended APICID (8 bits) */
+#define X86_FEATURE_AMD_DCM     ( 3*32+27) /* multi-node processor */
+#define X86_FEATURE_APERFMPERF  ( 3*32+28) /* APERFMPERF */
+#define X86_FEATURE_EAGER_FPU   ( 3*32+29) /* "eagerfpu" Non lazy FPU restore */
+#define X86_FEATURE_NONSTOP_TSC_S3 ( 3*32+30) /* TSC doesn't stop in S3 state */
+#define X86_FEATURE_MCE_RECOVERY ( 3*32+31) /* cpu has recoverable machine checks */
+
+/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
+#define X86_FEATURE_XMM3        ( 4*32+ 0) /* "pni" SSE-3 */
+#define X86_FEATURE_PCLMULQDQ   ( 4*32+ 1) /* PCLMULQDQ instruction */
+#define X86_FEATURE_DTES64      ( 4*32+ 2) /* 64-bit Debug Store */
+#define X86_FEATURE_MWAIT       ( 4*32+ 3) /* "monitor" Monitor/Mwait support */
+#define X86_FEATURE_DSCPL       ( 4*32+ 4) /* "ds_cpl" CPL Qual. Debug Store */
+#define X86_FEATURE_VMX         ( 4*32+ 5) /* Hardware virtualization */
+#define X86_FEATURE_SMX         ( 4*32+ 6) /* Safer mode */
+#define X86_FEATURE_EST         ( 4*32+ 7) /* Enhanced SpeedStep */
+#define X86_FEATURE_TM2         ( 4*32+ 8) /* Thermal Monitor 2 */
+#define X86_FEATURE_SSSE3       ( 4*32+ 9) /* Supplemental SSE-3 */
+#define X86_FEATURE_CID         ( 4*32+10) /* Context ID */
+#define X86_FEATURE_SDBG        ( 4*32+11) /* Silicon Debug */
+#define X86_FEATURE_FMA         ( 4*32+12) /* Fused multiply-add */
+#define X86_FEATURE_CX16        ( 4*32+13) /* CMPXCHG16B */
+#define X86_FEATURE_XTPR        ( 4*32+14) /* Send Task Priority Messages */
+#define X86_FEATURE_PDCM        ( 4*32+15) /* Performance Capabilities */
+#define X86_FEATURE_PCID        ( 4*32+17) /* Process Context Identifiers */
+#define X86_FEATURE_DCA         ( 4*32+18) /* Direct Cache Access */
+#define X86_FEATURE_XMM4_1      ( 4*32+19) /* "sse4_1" SSE-4.1 */
+#define X86_FEATURE_XMM4_2      ( 4*32+20) /* "sse4_2" SSE-4.2 */
+#define X86_FEATURE_X2APIC      ( 4*32+21) /* x2APIC */
+#define X86_FEATURE_MOVBE       ( 4*32+22) /* MOVBE instruction */
+#define X86_FEATURE_POPCNT      ( 4*32+23) /* POPCNT instruction */
+#define X86_FEATURE_TSC_DEADLINE_TIMER  ( 4*32+24) /* Tsc deadline timer */
+#define X86_FEATURE_AES         ( 4*32+25) /* AES instructions */
+#define X86_FEATURE_XSAVE       ( 4*32+26) /* XSAVE/XRSTOR/XSETBV/XGETBV */
+#define X86_FEATURE_OSXSAVE     ( 4*32+27) /* "" XSAVE enabled in the OS */
+#define X86_FEATURE_AVX         ( 4*32+28) /* Advanced Vector Extensions */
+#define X86_FEATURE_F16C        ( 4*32+29) /* 16-bit fp conversions */
+#define X86_FEATURE_RDRAND      ( 4*32+30) /* The RDRAND instruction */
+#define X86_FEATURE_HYPERVISOR  ( 4*32+31) /* Running on a hypervisor */
+
+/* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */
+#define X86_FEATURE_XSTORE      ( 5*32+ 2) /* "rng" RNG present (xstore) */
+#define X86_FEATURE_XSTORE_EN   ( 5*32+ 3) /* "rng_en" RNG enabled */
+#define X86_FEATURE_XCRYPT      ( 5*32+ 6) /* "ace" on-CPU crypto (xcrypt) */
+#define X86_FEATURE_XCRYPT_EN   ( 5*32+ 7) /* "ace_en" on-CPU crypto enabled */
+#define X86_FEATURE_ACE2        ( 5*32+ 8) /* Advanced Cryptography Engine v2 */
+#define X86_FEATURE_ACE2_EN     ( 5*32+ 9) /* ACE v2 enabled */
+#define X86_FEATURE_PHE         ( 5*32+10) /* PadLock Hash Engine */
+#define X86_FEATURE_PHE_EN      ( 5*32+11) /* PHE enabled */
+#define X86_FEATURE_PMM         ( 5*32+12) /* PadLock Montgomery Multiplier */
+#define X86_FEATURE_PMM_EN      ( 5*32+13) /* PMM enabled */
+
+/* More extended AMD flags: CPUID level 0x80000001, ecx, word 6 */
+#define X86_FEATURE_LAHF_LM     ( 6*32+ 0) /* LAHF/SAHF in long mode */
+#define X86_FEATURE_CMP_LEGACY  ( 6*32+ 1) /* If yes HyperThreading not valid */
+#define X86_FEATURE_SVM         ( 6*32+ 2) /* Secure virtual machine */
+#define X86_FEATURE_EXTAPIC     ( 6*32+ 3) /* Extended APIC space */
+#define X86_FEATURE_CR8_LEGACY  ( 6*32+ 4) /* CR8 in 32-bit mode */
+#define X86_FEATURE_ABM         ( 6*32+ 5) /* Advanced bit manipulation */
+#define X86_FEATURE_SSE4A       ( 6*32+ 6) /* SSE-4A */
+#define X86_FEATURE_MISALIGNSSE ( 6*32+ 7) /* Misaligned SSE mode */
+#define X86_FEATURE_3DNOWPREFETCH ( 6*32+ 8) /* 3DNow prefetch instructions */
+#define X86_FEATURE_OSVW        ( 6*32+ 9) /* OS Visible Workaround */
+#define X86_FEATURE_IBS         ( 6*32+10) /* Instruction Based Sampling */
+#define X86_FEATURE_XOP         ( 6*32+11) /* extended AVX instructions */
+#define X86_FEATURE_SKINIT      ( 6*32+12) /* SKINIT/STGI instructions */
+#define X86_FEATURE_WDT         ( 6*32+13) /* Watchdog timer */
+#define X86_FEATURE_LWP         ( 6*32+15) /* Light Weight Profiling */
+#define X86_FEATURE_FMA4        ( 6*32+16) /* 4 operands MAC instructions */
+#define X86_FEATURE_TCE         ( 6*32+17) /* translation cache extension */
+#define X86_FEATURE_NODEID_MSR  ( 6*32+19) /* NodeId MSR */
+#define X86_FEATURE_TBM         ( 6*32+21) /* trailing bit manipulations */
+#define X86_FEATURE_TOPOEXT     ( 6*32+22) /* topology extensions CPUID leafs */
+#define X86_FEATURE_PERFCTR_CORE ( 6*32+23) /* core performance counter extensions */
+#define X86_FEATURE_PERFCTR_NB  ( 6*32+24) /* NB performance counter extensions */
+#define X86_FEATURE_BPEXT       (6*32+26) /* data breakpoint extension */
+#define X86_FEATURE_PERFCTR_L2  ( 6*32+28) /* L2 performance counter extensions */
+#define X86_FEATURE_MWAITX      ( 6*32+29) /* MWAIT extension (MONITORX/MWAITX) */
+
+/*
+ * Auxiliary flags: Linux defined - For features scattered in various
+ * CPUID levels like 0x6, 0xA etc, word 7.
+ *
+ * Reuse free bits when adding new feature flags!
+ */
+
+#define X86_FEATURE_CPB         ( 7*32+ 2) /* AMD Core Performance Boost */
+#define X86_FEATURE_EPB         ( 7*32+ 3) /* IA32_ENERGY_PERF_BIAS support */
+
+#define X86_FEATURE_HW_PSTATE   ( 7*32+ 8) /* AMD HW-PState */
+#define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */
+
+#define X86_FEATURE_INTEL_PT    ( 7*32+15) /* Intel Processor Trace */
+
+/* Virtualization flags: Linux defined, word 8 */
+#define X86_FEATURE_TPR_SHADOW  ( 8*32+ 0) /* Intel TPR Shadow */
+#define X86_FEATURE_VNMI        ( 8*32+ 1) /* Intel Virtual NMI */
+#define X86_FEATURE_FLEXPRIORITY ( 8*32+ 2) /* Intel FlexPriority */
+#define X86_FEATURE_EPT         ( 8*32+ 3) /* Intel Extended Page Table */
+#define X86_FEATURE_VPID        ( 8*32+ 4) /* Intel Virtual Processor ID */
+
+#define X86_FEATURE_VMMCALL     ( 8*32+15) /* Prefer vmmcall to vmcall */
+#define X86_FEATURE_XENPV       ( 8*32+16) /* "" Xen paravirtual guest */
+
+
+/* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */
+#define X86_FEATURE_FSGSBASE    ( 9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/
+#define X86_FEATURE_TSC_ADJUST  ( 9*32+ 1) /* TSC adjustment MSR 0x3b */
+#define X86_FEATURE_BMI1        ( 9*32+ 3) /* 1st group bit manipulation extensions */
+#define X86_FEATURE_HLE         ( 9*32+ 4) /* Hardware Lock Elision */
+#define X86_FEATURE_AVX2        ( 9*32+ 5) /* AVX2 instructions */
+#define X86_FEATURE_SMEP        ( 9*32+ 7) /* Supervisor Mode Execution Protection */
+#define X86_FEATURE_BMI2        ( 9*32+ 8) /* 2nd group bit manipulation extensions */
+#define X86_FEATURE_ERMS        ( 9*32+ 9) /* Enhanced REP MOVSB/STOSB */
+#define X86_FEATURE_INVPCID     ( 9*32+10) /* Invalidate Processor Context ID */
+#define X86_FEATURE_RTM         ( 9*32+11) /* Restricted Transactional Memory */
+#define X86_FEATURE_CQM         ( 9*32+12) /* Cache QoS Monitoring */
+#define X86_FEATURE_MPX         ( 9*32+14) /* Memory Protection Extension */
+#define X86_FEATURE_AVX512F     ( 9*32+16) /* AVX-512 Foundation */
+#define X86_FEATURE_RDSEED      ( 9*32+18) /* The RDSEED instruction */
+#define X86_FEATURE_ADX         ( 9*32+19) /* The ADCX and ADOX instructions */
+#define X86_FEATURE_SMAP        ( 9*32+20) /* Supervisor Mode Access Prevention */
+#define X86_FEATURE_PCOMMIT     ( 9*32+22) /* PCOMMIT instruction */
+#define X86_FEATURE_CLFLUSHOPT  ( 9*32+23) /* CLFLUSHOPT instruction */
+#define X86_FEATURE_CLWB        ( 9*32+24) /* CLWB instruction */
+#define X86_FEATURE_AVX512PF    ( 9*32+26) /* AVX-512 Prefetch */
+#define X86_FEATURE_AVX512ER    ( 9*32+27) /* AVX-512 Exponential and Reciprocal */
+#define X86_FEATURE_AVX512CD    ( 9*32+28) /* AVX-512 Conflict Detection */
+#define X86_FEATURE_SHA_NI      ( 9*32+29) /* SHA1/SHA256 Instruction Extensions */
+
+/* Extended state features, CPUID level 0x0000000d:1 (eax), word 10 */
+#define X86_FEATURE_XSAVEOPT    (10*32+ 0) /* XSAVEOPT */
+#define X86_FEATURE_XSAVEC      (10*32+ 1) /* XSAVEC */
+#define X86_FEATURE_XGETBV1     (10*32+ 2) /* XGETBV with ECX = 1 */
+#define X86_FEATURE_XSAVES      (10*32+ 3) /* XSAVES/XRSTORS */
+
+/* Intel-defined CPU QoS Sub-leaf, CPUID level 0x0000000F:0 (edx), word 11 */
+#define X86_FEATURE_CQM_LLC     (11*32+ 1) /* LLC QoS if 1 */
+
+/* Intel-defined CPU QoS Sub-leaf, CPUID level 0x0000000F:1 (edx), word 12 */
+#define X86_FEATURE_CQM_OCCUP_LLC (12*32+ 0) /* LLC occupancy monitoring if 1 */
+
+/* AMD-defined CPU features, CPUID level 0x80000008 (ebx), word 13 */
+#define X86_FEATURE_CLZERO      (13*32+0) /* CLZERO instruction */
+
+/* Thermal and Power Management Leaf, CPUID level 0x00000006 (eax), word 14 */
+#define X86_FEATURE_DTHERM      (14*32+ 0) /* Digital Thermal Sensor */
+#define X86_FEATURE_IDA         (14*32+ 1) /* Intel Dynamic Acceleration */
+#define X86_FEATURE_ARAT        (14*32+ 2) /* Always Running APIC Timer */
+#define X86_FEATURE_PLN         (14*32+ 4) /* Intel Power Limit Notification */
+#define X86_FEATURE_PTS         (14*32+ 6) /* Intel Package Thermal Status */
+#define X86_FEATURE_HWP         (14*32+ 7) /* Intel Hardware P-states */
+#define X86_FEATURE_HWP_NOTIFY  (14*32+ 8) /* HWP Notification */
+#define X86_FEATURE_HWP_ACT_WINDOW (14*32+ 9) /* HWP Activity Window */
+#define X86_FEATURE_HWP_EPP     (14*32+10) /* HWP Energy Perf. Preference */
+#define X86_FEATURE_HWP_PKG_REQ (14*32+11) /* HWP Package Level Request */
+
+/* AMD SVM Feature Identification, CPUID level 0x8000000a (edx), word 15 */
+#define X86_FEATURE_NPT         (15*32+ 0) /* Nested Page Table support */
+#define X86_FEATURE_LBRV        (15*32+ 1) /* LBR Virtualization support */
+#define X86_FEATURE_SVML        (15*32+ 2) /* "svm_lock" SVM locking MSR */
+#define X86_FEATURE_NRIPS       (15*32+ 3) /* "nrip_save" SVM next_rip save */
+#define X86_FEATURE_TSCRATEMSR  (15*32+ 4) /* "tsc_scale" TSC scaling support */
+#define X86_FEATURE_VMCBCLEAN   (15*32+ 5) /* "vmcb_clean" VMCB clean bits support */
+#define X86_FEATURE_FLUSHBYASID (15*32+ 6) /* flush-by-ASID support */
+#define X86_FEATURE_DECODEASSISTS (15*32+ 7) /* Decode Assists support */
+#define X86_FEATURE_PAUSEFILTER (15*32+10) /* filtered pause intercept */
+#define X86_FEATURE_PFTHRESHOLD (15*32+12) /* pause filter threshold */
+#define X86_FEATURE_AVIC        (15*32+13) /* Virtual Interrupt Controller */
+
+/*
+ * BUG word(s)
+ */
+#define X86_BUG(x)              (NCAPINTS*32 + (x))
+
+#define X86_BUG_F00F            X86_BUG(0) /* Intel F00F */
+#define X86_BUG_FDIV            X86_BUG(1) /* FPU FDIV */
+#define X86_BUG_COMA            X86_BUG(2) /* Cyrix 6x86 coma */
+#define X86_BUG_AMD_TLB_MMATCH  X86_BUG(3) /* "tlb_mmatch" AMD Erratum 383 */
+#define X86_BUG_AMD_APIC_C1E    X86_BUG(4) /* "apic_c1e" AMD Erratum 400 */
+#define X86_BUG_11AP            X86_BUG(5) /* Bad local APIC aka 11AP */
+#define X86_BUG_FXSAVE_LEAK     X86_BUG(6) /* FXSAVE leaks FOP/FIP/FOP */
+#define X86_BUG_CLFLUSH_MONITOR X86_BUG(7) /* AAI65, CLFLUSH required before MONITOR */
+#define X86_BUG_SYSRET_SS_ATTRS X86_BUG(8) /* SYSRET doesn't fix up SS attrs */
+
+#endif /* _ASM_X86_CPUFEATURES_H */
diff --git a/arch/x86/include/asm/desc_defs.h b/arch/x86/include/asm/desc_defs.h
index 278441f39856..eb5deb42484d 100644
--- a/arch/x86/include/asm/desc_defs.h
+++ b/arch/x86/include/asm/desc_defs.h
@@ -98,4 +98,27 @@ struct desc_ptr {
 
 #endif /* !__ASSEMBLY__ */
 
+/* Access rights as returned by LAR */
+#define AR_TYPE_RODATA          (0 * (1 << 9))
+#define AR_TYPE_RWDATA          (1 * (1 << 9))
+#define AR_TYPE_RODATA_EXPDOWN  (2 * (1 << 9))
+#define AR_TYPE_RWDATA_EXPDOWN  (3 * (1 << 9))
+#define AR_TYPE_XOCODE          (4 * (1 << 9))
+#define AR_TYPE_XRCODE          (5 * (1 << 9))
+#define AR_TYPE_XOCODE_CONF     (6 * (1 << 9))
+#define AR_TYPE_XRCODE_CONF     (7 * (1 << 9))
+#define AR_TYPE_MASK            (7 * (1 << 9))
+
+#define AR_DPL0                 (0 * (1 << 13))
+#define AR_DPL3                 (3 * (1 << 13))
+#define AR_DPL_MASK             (3 * (1 << 13))
+
+#define AR_A                    (1 << 8)   /* "Accessed" */
+#define AR_S                    (1 << 12)  /* If clear, "System" segment */
+#define AR_P                    (1 << 15)  /* "Present" */
+#define AR_AVL                  (1 << 20)  /* "AVaiLable" (no HW effect) */
+#define AR_L                    (1 << 21)  /* "Long mode" for code segments */
+#define AR_DB                   (1 << 22)  /* D/B, effect depends on type */
+#define AR_G                    (1 << 23)  /* "Granularity" (limit in pages) */
+
 #endif /* _ASM_X86_DESC_DEFS_H */
diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h
index 0fd440df63f1..c2e46eb96b6d 100644
--- a/arch/x86/include/asm/fpu/internal.h
+++ b/arch/x86/include/asm/fpu/internal.h
@@ -17,6 +17,7 @@
 #include <asm/user.h>
 #include <asm/fpu/api.h>
 #include <asm/fpu/xstate.h>
+#include <asm/cpufeature.h>
 
 /*
  * High level FPU state handling functions:
@@ -58,22 +59,22 @@ extern u64 fpu__get_supported_xfeatures_mask(void);
  */
 static __always_inline __pure bool use_eager_fpu(void)
 {
-        return static_cpu_has_safe(X86_FEATURE_EAGER_FPU);
+        return static_cpu_has(X86_FEATURE_EAGER_FPU);
 }
 
 static __always_inline __pure bool use_xsaveopt(void)
 {
-        return static_cpu_has_safe(X86_FEATURE_XSAVEOPT);
+        return static_cpu_has(X86_FEATURE_XSAVEOPT);
 }
 
 static __always_inline __pure bool use_xsave(void)
 {
-        return static_cpu_has_safe(X86_FEATURE_XSAVE);
+        return static_cpu_has(X86_FEATURE_XSAVE);
 }
 
 static __always_inline __pure bool use_fxsr(void)
 {
-        return static_cpu_has_safe(X86_FEATURE_FXSR);
+        return static_cpu_has(X86_FEATURE_FXSR);
 }
 
 /*
@@ -300,7 +301,7 @@ static inline void copy_xregs_to_kernel_booting(struct xregs_state *xstate)
 
         WARN_ON(system_state != SYSTEM_BOOTING);
 
-        if (static_cpu_has_safe(X86_FEATURE_XSAVES))
+        if (static_cpu_has(X86_FEATURE_XSAVES))
                 XSTATE_OP(XSAVES, xstate, lmask, hmask, err);
         else
                 XSTATE_OP(XSAVE, xstate, lmask, hmask, err);
@@ -322,7 +323,7 @@ static inline void copy_kernel_to_xregs_booting(struct xregs_state *xstate)
 
         WARN_ON(system_state != SYSTEM_BOOTING);
 
-        if (static_cpu_has_safe(X86_FEATURE_XSAVES))
+        if (static_cpu_has(X86_FEATURE_XSAVES))
                 XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
         else
                 XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
@@ -460,7 +461,7 @@ static inline void copy_kernel_to_fpregs(union fpregs_state *fpstate)
          * pending. Clear the x87 state here by setting it to fixed values.
          * "m" is a random variable that should be in L1.
          */
-        if (unlikely(static_cpu_has_bug_safe(X86_BUG_FXSAVE_LEAK))) {
+        if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK))) {
                 asm volatile(
                         "fnclex\n\t"
                         "emms\n\t"
diff --git a/arch/x86/include/asm/frame.h b/arch/x86/include/asm/frame.h
index 793179cf8e21..6e4d170726b7 100644
--- a/arch/x86/include/asm/frame.h
+++ b/arch/x86/include/asm/frame.h
@@ -1,23 +1,44 @@
-#ifdef __ASSEMBLY__
+#ifndef _ASM_X86_FRAME_H
+#define _ASM_X86_FRAME_H
 
 #include <asm/asm.h>
 
-/* The annotation hides the frame from the unwinder and makes it look
-   like a ordinary ebp save/restore. This avoids some special cases for
-   frame pointer later */
+/*
+ * These are stack frame creation macros.  They should be used by every
+ * callable non-leaf asm function to make kernel stack traces more reliable.
+ */
+
 #ifdef CONFIG_FRAME_POINTER
-        .macro FRAME
-        __ASM_SIZE(push,)       %__ASM_REG(bp)
-        __ASM_SIZE(mov)         %__ASM_REG(sp), %__ASM_REG(bp)
-        .endm
-        .macro ENDFRAME
-        __ASM_SIZE(pop,)        %__ASM_REG(bp)
-        .endm
-#else
-        .macro FRAME
-        .endm
-        .macro ENDFRAME
-        .endm
-#endif
-
-#endif  /*  __ASSEMBLY__  */
+
+#ifdef __ASSEMBLY__
+
+.macro FRAME_BEGIN
+        push %_ASM_BP
+        _ASM_MOV %_ASM_SP, %_ASM_BP
+.endm
+
+.macro FRAME_END
+        pop %_ASM_BP
+.endm
+
+#else /* !__ASSEMBLY__ */
+
+#define FRAME_BEGIN                             \
+        "push %" _ASM_BP "\n"                   \
+        _ASM_MOV "%" _ASM_SP ", %" _ASM_BP "\n"
+
+#define FRAME_END "pop %" _ASM_BP "\n"
+
+#endif /* __ASSEMBLY__ */
+
+#define FRAME_OFFSET __ASM_SEL(4, 8)
+
+#else /* !CONFIG_FRAME_POINTER */
+
+#define FRAME_BEGIN
+#define FRAME_END
+#define FRAME_OFFSET 0
+
+#endif /* CONFIG_FRAME_POINTER */
+
+#endif /* _ASM_X86_FRAME_H */
diff --git a/arch/x86/include/asm/irq_work.h b/arch/x86/include/asm/irq_work.h
index 78162f8e248b..d0afb05c84fc 100644
--- a/arch/x86/include/asm/irq_work.h
+++ b/arch/x86/include/asm/irq_work.h
@@ -1,7 +1,7 @@
 #ifndef _ASM_IRQ_WORK_H
 #define _ASM_IRQ_WORK_H
 
-#include <asm/processor.h>
+#include <asm/cpufeature.h>
 
 static inline bool arch_irq_work_has_interrupt(void)
 {
diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index 2ea4527e462f..18d2ba9c8e44 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -113,6 +113,7 @@ struct mca_config {
         bool ignore_ce;
         bool disabled;
         bool ser;
+        bool recovery;
         bool bios_cmci_threshold;
         u8 banks;
         s8 bootlog;
diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h
index 55234d5e7160..1ea0baef1175 100644
--- a/arch/x86/include/asm/mmu.h
+++ b/arch/x86/include/asm/mmu.h
@@ -19,7 +19,8 @@ typedef struct {
 #endif
 
         struct mutex lock;
-        void __user *vdso;
+        void __user *vdso;                      /* vdso base address */
+        const struct vdso_image *vdso_image;    /* vdso image in use */
 
         atomic_t perf_rdpmc_allowed;    /* nonzero if rdpmc is allowed */
 } mm_context_t;
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index b05402ef3b84..984ab75bf621 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -1,7 +1,12 @@
 #ifndef _ASM_X86_MSR_INDEX_H
 #define _ASM_X86_MSR_INDEX_H
 
-/* CPU model specific register (MSR) numbers */
+/*
+ * CPU model specific register (MSR) numbers.
+ *
+ * Do not add new entries to this file unless the definitions are shared
+ * between multiple compilation units.
+ */
 
 /* x86-64 specific MSRs */
 #define MSR_EFER                0xc0000080 /* extended feature register */
diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
index c70689b5e5aa..0deeb2d26df7 100644
--- a/arch/x86/include/asm/mwait.h
+++ b/arch/x86/include/asm/mwait.h
@@ -3,6 +3,8 @@
 
 #include <linux/sched.h>
 
+#include <asm/cpufeature.h>
+
 #define MWAIT_SUBSTATE_MASK             0xf
 #define MWAIT_CSTATE_MASK               0xf
 #define MWAIT_SUBSTATE_SIZE             4
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index f6192502149e..601f1b8f9961 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -13,6 +13,7 @@
 #include <linux/bug.h>
 #include <linux/types.h>
 #include <linux/cpumask.h>
+#include <asm/frame.h>
 
 static inline int paravirt_enabled(void)
 {
@@ -756,15 +757,19 @@ static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock,
  * call. The return value in rax/eax will not be saved, even for void
  * functions.
  */
+#define PV_THUNK_NAME(func) "__raw_callee_save_" #func
 #define PV_CALLEE_SAVE_REGS_THUNK(func)                                 \
         extern typeof(func) __raw_callee_save_##func;                   \
                                                                         \
         asm(".pushsection .text;"                                       \
-            ".globl __raw_callee_save_" #func " ; "                     \
-            "__raw_callee_save_" #func ": "                             \
+            ".globl " PV_THUNK_NAME(func) ";"                           \
+            ".type " PV_THUNK_NAME(func) ", @function;"                 \
+            PV_THUNK_NAME(func) ":"                                     \
+            FRAME_BEGIN                                                 \
             PV_SAVE_ALL_CALLER_REGS                                     \
             "call " #func ";"                                           \
             PV_RESTORE_ALL_CALLER_REGS                                  \
+            FRAME_END                                                   \
             "ret;"                                                      \
             ".popsection")
 
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 77db5616a473..e8c2326478c8 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -466,8 +466,9 @@ int paravirt_disable_iospace(void);
  * makes sure the incoming and outgoing types are always correct.
  */
 #ifdef CONFIG_X86_32
-#define PVOP_VCALL_ARGS                         \
-        unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx
+#define PVOP_VCALL_ARGS                                                 \
+        unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx;      \
+        register void *__sp asm("esp")
 #define PVOP_CALL_ARGS                  PVOP_VCALL_ARGS
 
 #define PVOP_CALL_ARG1(x)               "a" ((unsigned long)(x))
@@ -485,9 +486,10 @@ int paravirt_disable_iospace(void);
 #define VEXTRA_CLOBBERS
 #else  /* CONFIG_X86_64 */
 /* [re]ax isn't an arg, but the return val */
-#define PVOP_VCALL_ARGS                                 \
-        unsigned long __edi = __edi, __esi = __esi,     \
-                __edx = __edx, __ecx = __ecx, __eax = __eax
+#define PVOP_VCALL_ARGS                                         \
+        unsigned long __edi = __edi, __esi = __esi,             \
+                __edx = __edx, __ecx = __ecx, __eax = __eax;    \
+        register void *__sp asm("rsp")
 #define PVOP_CALL_ARGS          PVOP_VCALL_ARGS
 
 #define PVOP_CALL_ARG1(x)               "D" ((unsigned long)(x))
@@ -526,7 +528,7 @@ int paravirt_disable_iospace(void);
                         asm volatile(pre                                \
                                      paravirt_alt(PARAVIRT_CALL)        \
                                      post                               \
-                                     : call_clbr                        \
+                                     : call_clbr, "+r" (__sp)           \
                                      : paravirt_type(op),               \
                                        paravirt_clobber(clbr),          \
                                        ##__VA_ARGS__                    \
@@ -536,7 +538,7 @@ int paravirt_disable_iospace(void);
                         asm volatile(pre                                \
                                      paravirt_alt(PARAVIRT_CALL)        \
                                      post                               \
-                                     : call_clbr                        \
+                                     : call_clbr, "+r" (__sp)           \
                                      : paravirt_type(op),               \
                                        paravirt_clobber(clbr),          \
                                        ##__VA_ARGS__                    \
@@ -563,7 +565,7 @@ int paravirt_disable_iospace(void);
                 asm volatile(pre                                        \
                              paravirt_alt(PARAVIRT_CALL)                \
                              post                                       \
-                             : call_clbr                                \
+                             : call_clbr, "+r" (__sp)                   \
                              : paravirt_type(op),                       \
                                paravirt_clobber(clbr),                  \
                                ##__VA_ARGS__                            \
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 01bcde84d3e4..d397deb58146 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -94,10 +94,19 @@ static __always_inline bool should_resched(int preempt_offset)
 
 #ifdef CONFIG_PREEMPT
   extern asmlinkage void ___preempt_schedule(void);
-# define __preempt_schedule() asm ("call ___preempt_schedule")
+# define __preempt_schedule()                                   \
+({                                                              \
+        register void *__sp asm(_ASM_SP);                       \
+        asm volatile ("call ___preempt_schedule" : "+r"(__sp)); \
+})
+
   extern asmlinkage void preempt_schedule(void);
   extern asmlinkage void ___preempt_schedule_notrace(void);
-# define __preempt_schedule_notrace() asm ("call ___preempt_schedule_notrace")
+# define __preempt_schedule_notrace()                                   \
+({                                                                      \
+        register void *__sp asm(_ASM_SP);                               \
+        asm volatile ("call ___preempt_schedule_notrace" : "+r"(__sp)); \
+})
   extern asmlinkage void preempt_schedule_notrace(void);
 #endif
 
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 20c11d1aa4cc..ecb410310e70 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -13,7 +13,7 @@ struct vm86;
 #include <asm/types.h>
 #include <uapi/asm/sigcontext.h>
 #include <asm/current.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/page.h>
 #include <asm/pgtable_types.h>
 #include <asm/percpu.h>
@@ -24,7 +24,6 @@ struct vm86;
 #include <asm/fpu/types.h>
 
 #include <linux/personality.h>
-#include <linux/cpumask.h>
 #include <linux/cache.h>
 #include <linux/threads.h>
 #include <linux/math64.h>
diff --git a/arch/x86/include/asm/qspinlock_paravirt.h b/arch/x86/include/asm/qspinlock_paravirt.h
index 9f92c180ed2f..9d55f9b6e167 100644
--- a/arch/x86/include/asm/qspinlock_paravirt.h
+++ b/arch/x86/include/asm/qspinlock_paravirt.h
@@ -36,8 +36,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
  */
 asm    (".pushsection .text;"
         ".globl " PV_UNLOCK ";"
+        ".type " PV_UNLOCK ", @function;"
         ".align 4,0x90;"
         PV_UNLOCK ": "
+        FRAME_BEGIN
         "push  %rdx;"
         "mov   $0x1,%eax;"
         "xor   %edx,%edx;"
@@ -45,6 +47,7 @@ asm    (".pushsection .text;"
         "cmp   $0x1,%al;"
         "jne   .slowpath;"
         "pop   %rdx;"
+        FRAME_END
         "ret;"
         ".slowpath: "
         "push   %rsi;"
@@ -52,6 +55,7 @@ asm    (".pushsection .text;"
         "call " PV_UNLOCK_SLOWPATH ";"
         "pop    %rsi;"
         "pop    %rdx;"
+        FRAME_END
         "ret;"
         ".size " PV_UNLOCK ", .-" PV_UNLOCK ";"
         ".popsection");
diff --git a/arch/x86/include/asm/sighandling.h b/arch/x86/include/asm/sighandling.h
index 89db46752a8f..452c88b8ad06 100644
--- a/arch/x86/include/asm/sighandling.h
+++ b/arch/x86/include/asm/sighandling.h
@@ -13,7 +13,6 @@
                          X86_EFLAGS_CF | X86_EFLAGS_RF)
 
 void signal_fault(struct pt_regs *regs, void __user *frame, char *where);
-int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc);
 int setup_sigcontext(struct sigcontext __user *sc, void __user *fpstate,
                      struct pt_regs *regs, unsigned long mask);
 
diff --git a/arch/x86/include/asm/smap.h b/arch/x86/include/asm/smap.h
index ba665ebd17bb..db333300bd4b 100644
--- a/arch/x86/include/asm/smap.h
+++ b/arch/x86/include/asm/smap.h
@@ -15,7 +15,7 @@
 
 #include <linux/stringify.h>
 #include <asm/nops.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 
 /* "Raw" instruction opcodes */
 #define __ASM_CLAC      .byte 0x0f,0x01,0xca
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index dfcf0727623b..20a3de5cb3b0 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -16,7 +16,6 @@
 #endif
 #include <asm/thread_info.h>
 #include <asm/cpumask.h>
-#include <asm/cpufeature.h>
 
 extern int smp_num_siblings;
 extern unsigned int num_processors;
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index c7b551028740..c0778fcab06d 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -49,7 +49,7 @@
  */
 #ifndef __ASSEMBLY__
 struct task_struct;
-#include <asm/processor.h>
+#include <asm/cpufeature.h>
 #include <linux/atomic.h>
 
 struct thread_info {
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 6df2029405a3..0bb31cb8c73b 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -5,6 +5,7 @@
 #include <linux/sched.h>
 
 #include <asm/processor.h>
+#include <asm/cpufeature.h>
 #include <asm/special_insns.h>
 
 #ifdef CONFIG_PARAVIRT
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index a4a30e4b2d34..9bbb3b2d0372 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -179,10 +179,11 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
 ({                                                                      \
         int __ret_gu;                                                   \
         register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX);            \
+        register void *__sp asm(_ASM_SP);                               \
         __chk_user_ptr(ptr);                                            \
         might_fault();                                                  \
-        asm volatile("call __get_user_%P3"                              \
-                     : "=a" (__ret_gu), "=r" (__val_gu)                 \
+        asm volatile("call __get_user_%P4"                              \
+                     : "=a" (__ret_gu), "=r" (__val_gu), "+r" (__sp)    \
                      : "0" (ptr), "i" (sizeof(*(ptr))));                \
         (x) = (__force __typeof__(*(ptr))) __val_gu;                    \
         __builtin_expect(__ret_gu, 0);                                  \
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index b89c34c4019b..307698688fa1 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -8,7 +8,7 @@
 #include <linux/errno.h>
 #include <linux/lockdep.h>
 #include <asm/alternative.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/page.h>
 
 /*
diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h
index deabaf9759b6..43dc55be524e 100644
--- a/arch/x86/include/asm/vdso.h
+++ b/arch/x86/include/asm/vdso.h
@@ -13,9 +13,6 @@ struct vdso_image {
         void *data;
         unsigned long size;   /* Always a multiple of PAGE_SIZE */
 
-        /* text_mapping.pages is big enough for data/size page pointers */
-        struct vm_special_mapping text_mapping;
-
         unsigned long alt, alt_len;
 
         long sym_vvar_start;  /* Negative offset to the vvar area */
diff --git a/arch/x86/include/asm/vgtod.h b/arch/x86/include/asm/vgtod.h
index f556c4843aa1..e728699db774 100644
--- a/arch/x86/include/asm/vgtod.h
+++ b/arch/x86/include/asm/vgtod.h
@@ -37,6 +37,12 @@ struct vsyscall_gtod_data {
 };
 extern struct vsyscall_gtod_data vsyscall_gtod_data;
 
+extern int vclocks_used;
+static inline bool vclock_was_used(int vclock)
+{
+        return READ_ONCE(vclocks_used) & (1 << vclock);
+}
+
 static inline unsigned gtod_read_begin(const struct vsyscall_gtod_data *s)
 {
         unsigned ret;
diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h
index 3bcdcc84259d..a12a047184ee 100644
--- a/arch/x86/include/asm/xen/hypercall.h
+++ b/arch/x86/include/asm/xen/hypercall.h
@@ -110,9 +110,10 @@ extern struct { char _entry[32]; } hypercall_page[];
         register unsigned long __arg2 asm(__HYPERCALL_ARG2REG) = __arg2; \
         register unsigned long __arg3 asm(__HYPERCALL_ARG3REG) = __arg3; \
         register unsigned long __arg4 asm(__HYPERCALL_ARG4REG) = __arg4; \
-        register unsigned long __arg5 asm(__HYPERCALL_ARG5REG) = __arg5;
+        register unsigned long __arg5 asm(__HYPERCALL_ARG5REG) = __arg5; \
+        register void *__sp asm(_ASM_SP);
 
-#define __HYPERCALL_0PARAM      "=r" (__res)
+#define __HYPERCALL_0PARAM      "=r" (__res), "+r" (__sp)
 #define __HYPERCALL_1PARAM      __HYPERCALL_0PARAM, "+r" (__arg1)
 #define __HYPERCALL_2PARAM      __HYPERCALL_1PARAM, "+r" (__arg2)
 #define __HYPERCALL_3PARAM      __HYPERCALL_2PARAM, "+r" (__arg3)
diff --git a/arch/x86/include/uapi/asm/sigcontext.h b/arch/x86/include/uapi/asm/sigcontext.h
index d485232f1e9f..62d4111c1c54 100644
--- a/arch/x86/include/uapi/asm/sigcontext.h
+++ b/arch/x86/include/uapi/asm/sigcontext.h
@@ -256,7 +256,7 @@ struct sigcontext_64 {
         __u16                           cs;
         __u16                           gs;
         __u16                           fs;
-        __u16                           __pad0;
+        __u16                           ss;
         __u64                           err;
         __u64                           trapno;
         __u64                           oldmask;
@@ -341,9 +341,37 @@ struct sigcontext {
         __u64                           rip;
         __u64                           eflags;         /* RFLAGS */
         __u16                           cs;
+
+        /*
+         * Prior to 2.5.64 ("[PATCH] x86-64 updates for 2.5.64-bk3"),
+         * Linux saved and restored fs and gs in these slots.  This
+         * was counterproductive, as fsbase and gsbase were never
+         * saved, so arch_prctl was presumably unreliable.
+         *
+         * These slots should never be reused without extreme caution:
+         *
+         *  - Some DOSEMU versions stash fs and gs in these slots manually,
+         *    thus overwriting anything the kernel expects to be preserved
+         *    in these slots.
+         *
+         *  - If these slots are ever needed for any other purpose,
+         *    there is some risk that very old 64-bit binaries could get
+         *    confused.  I doubt that many such binaries still work,
+         *    though, since the same patch in 2.5.64 also removed the
+         *    64-bit set_thread_area syscall, so it appears that there
+         *    is no TLS API beyond modify_ldt that works in both pre-
+         *    and post-2.5.64 kernels.
+         *
+         * If the kernel ever adds explicit fs, gs, fsbase, and gsbase
+         * save/restore, it will most likely need to be opt-in and use
+         * different context slots.
+         */
         __u16                           gs;
         __u16                           fs;
-        __u16                           __pad0;
+        union {
+                __u16                   ss;     /* If UC_SIGCONTEXT_SS */
+                __u16                   __pad0; /* Alias name for old (!UC_SIGCONTEXT_SS) user-space */
+        };
         __u64                           err;
         __u64                           trapno;
         __u64                           oldmask;
diff --git a/arch/x86/include/uapi/asm/ucontext.h b/arch/x86/include/uapi/asm/ucontext.h
index b7c29c8017f2..e3d1ec90616e 100644
--- a/arch/x86/include/uapi/asm/ucontext.h
+++ b/arch/x86/include/uapi/asm/ucontext.h
@@ -1,11 +1,54 @@
 #ifndef _ASM_X86_UCONTEXT_H
 #define _ASM_X86_UCONTEXT_H
 
-#define UC_FP_XSTATE    0x1     /* indicates the presence of extended state
-                                 * information in the memory layout pointed
-                                 * by the fpstate pointer in the ucontext's
-                                 * sigcontext struct (uc_mcontext).
-                                 */
+/*
+ * Indicates the presence of extended state information in the memory
+ * layout pointed by the fpstate pointer in the ucontext's sigcontext
+ * struct (uc_mcontext).
+ */
+#define UC_FP_XSTATE    0x1
+
+#ifdef __x86_64__
+/*
+ * UC_SIGCONTEXT_SS will be set when delivering 64-bit or x32 signals on
+ * kernels that save SS in the sigcontext.  All kernels that set
+ * UC_SIGCONTEXT_SS will correctly restore at least the low 32 bits of esp
+ * regardless of SS (i.e. they implement espfix).
+ *
+ * Kernels that set UC_SIGCONTEXT_SS will also set UC_STRICT_RESTORE_SS
+ * when delivering a signal that came from 64-bit code.
+ *
+ * Sigreturn restores SS as follows:
+ *
+ * if (saved SS is valid || UC_STRICT_RESTORE_SS is set ||
+ *     saved CS is not 64-bit)
+ *         new SS = saved SS  (will fail IRET and signal if invalid)
+ * else
+ *         new SS = a flat 32-bit data segment
+ *
+ * This behavior serves three purposes:
+ *
+ * - Legacy programs that construct a 64-bit sigcontext from scratch
+ *   with zero or garbage in the SS slot (e.g. old CRIU) and call
+ *   sigreturn will still work.
+ *
+ * - Old DOSEMU versions sometimes catch a signal from a segmented
+ *   context, delete the old SS segment (with modify_ldt), and change
+ *   the saved CS to a 64-bit segment.  These DOSEMU versions expect
+ *   sigreturn to send them back to 64-bit mode without killing them,
+ *   despite the fact that the SS selector when the signal was raised is
+ *   no longer valid.  UC_STRICT_RESTORE_SS will be clear, so the kernel
+ *   will fix up SS for these DOSEMU versions.
+ *
+ * - Old and new programs that catch a signal and return without
+ *   modifying the saved context will end up in exactly the state they
+ *   started in, even if they were running in a segmented context when
+ *   the signal was raised..  Old kernels would lose track of the
+ *   previous SS value.
+ */
+#define UC_SIGCONTEXT_SS        0x2
+#define UC_STRICT_RESTORE_SS    0x4
+#endif
 
 #include <asm-generic/ucontext.h>
 
diff --git a/arch/x86/kernel/acpi/wakeup_64.S b/arch/x86/kernel/acpi/wakeup_64.S
index 8c35df468104..169963f471bb 100644
--- a/arch/x86/kernel/acpi/wakeup_64.S
+++ b/arch/x86/kernel/acpi/wakeup_64.S
@@ -5,6 +5,7 @@
 #include <asm/page_types.h>
 #include <asm/msr.h>
 #include <asm/asm-offsets.h>
+#include <asm/frame.h>
 
 # Copyright 2003 Pavel Machek <pavel@suse.cz>, distribute under GPLv2
 
@@ -39,6 +40,7 @@ bogus_64_magic:
         jmp     bogus_64_magic
 
 ENTRY(do_suspend_lowlevel)
+        FRAME_BEGIN
         subq    $8, %rsp
         xorl    %eax, %eax
         call    save_processor_state
@@ -109,6 +111,7 @@ ENTRY(do_suspend_lowlevel)
 
         xorl    %eax, %eax
         addq    $8, %rsp
+        FRAME_END
         jmp     restore_processor_state
 ENDPROC(do_suspend_lowlevel)
 
diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index c80c02c6ec49..ab5c2c685a3c 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -30,7 +30,7 @@ static unsigned int numachip1_get_apic_id(unsigned long x)
         unsigned long value;
         unsigned int id = (x >> 24) & 0xff;
 
-        if (static_cpu_has_safe(X86_FEATURE_NODEID_MSR)) {
+        if (static_cpu_has(X86_FEATURE_NODEID_MSR)) {
                 rdmsrl(MSR_FAM10H_NODE_ID, value);
                 id |= (value << 2) & 0xff00;
         }
@@ -178,7 +178,7 @@ static void fixup_cpu_id(struct cpuinfo_x86 *c, int node)
         this_cpu_write(cpu_llc_id, node);
 
         /* Account for nodes per socket in multi-core-module processors */
-        if (static_cpu_has_safe(X86_FEATURE_NODEID_MSR)) {
+        if (static_cpu_has(X86_FEATURE_NODEID_MSR)) {
                 rdmsrl(MSR_FAM10H_NODE_ID, val);
                 nodes = ((val >> 3) & 7) + 1;
         }
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index 6ce39025f467..fdeb0ce07c16 100644
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -7,7 +7,7 @@
 #include <linux/lguest.h>
 #include "../../../drivers/lguest/lg.h"
 
-#define __SYSCALL_I386(nr, sym, compat) [nr] = 1,
+#define __SYSCALL_I386(nr, sym, qual) [nr] = 1,
 static char syscalls[] = {
 #include <asm/syscalls_32.h>
 };
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index f2edafb5f24e..d875f97d4e0b 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -4,17 +4,11 @@
 
 #include <asm/ia32.h>
 
-#define __SYSCALL_64(nr, sym, compat) [nr] = 1,
-#define __SYSCALL_COMMON(nr, sym, compat) [nr] = 1,
-#ifdef CONFIG_X86_X32_ABI
-# define __SYSCALL_X32(nr, sym, compat) [nr] = 1,
-#else
-# define __SYSCALL_X32(nr, sym, compat) /* nothing */
-#endif
+#define __SYSCALL_64(nr, sym, qual) [nr] = 1,
 static char syscalls_64[] = {
 #include <asm/syscalls_64.h>
 };
-#define __SYSCALL_I386(nr, sym, compat) [nr] = 1,
+#define __SYSCALL_I386(nr, sym, qual) [nr] = 1,
 static char syscalls_ia32[] = {
 #include <asm/syscalls_32.h>
 };
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index 58031303e304..faa7b5204129 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -64,7 +64,7 @@ ifdef CONFIG_X86_FEATURE_NAMES
 quiet_cmd_mkcapflags = MKCAP   $@
       cmd_mkcapflags = $(CONFIG_SHELL) $(srctree)/$(src)/mkcapflags.sh $< $@
 
-cpufeature = $(src)/../../include/asm/cpufeature.h
+cpufeature = $(src)/../../include/asm/cpufeatures.h
 
 targets += capflags.c
 $(obj)/capflags.c: $(cpufeature) $(src)/mkcapflags.sh FORCE
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index a07956a08936..fe2f089f03d2 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -75,7 +75,10 @@ static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
  */
 
 extern __visible void vide(void);
-__asm__(".globl vide\n\t.align 4\nvide: ret");
+__asm__(".globl vide\n"
+        ".type vide, @function\n"
+        ".align 4\n"
+        "vide: ret\n");
 
 static void init_amd_k5(struct cpuinfo_x86 *c)
 {
diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c
index ae20be6e483c..6608c03c2126 100644
--- a/arch/x86/kernel/cpu/centaur.c
+++ b/arch/x86/kernel/cpu/centaur.c
@@ -1,7 +1,7 @@
 #include <linux/bitops.h>
 #include <linux/kernel.h>
 
-#include <asm/processor.h>
+#include <asm/cpufeature.h>
 #include <asm/e820.h>
 #include <asm/mtrr.h>
 #include <asm/msr.h>
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 37830de8f60a..079d83fc6488 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1475,20 +1475,6 @@ void cpu_init(void)
 }
 #endif
 
-#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS
-void warn_pre_alternatives(void)
-{
-        WARN(1, "You're using static_cpu_has before alternatives have run!\n");
-}
-EXPORT_SYMBOL_GPL(warn_pre_alternatives);
-#endif
-
-inline bool __static_cpu_has_safe(u16 bit)
-{
-        return boot_cpu_has(bit);
-}
-EXPORT_SYMBOL_GPL(__static_cpu_has_safe);
-
 static void bsp_resume(void)
 {
         if (this_cpu->c_bsp_resume)
diff --git a/arch/x86/kernel/cpu/cyrix.c b/arch/x86/kernel/cpu/cyrix.c
index aaf152e79637..15e47c1cd412 100644
--- a/arch/x86/kernel/cpu/cyrix.c
+++ b/arch/x86/kernel/cpu/cyrix.c
@@ -8,6 +8,7 @@
 #include <linux/timer.h>
 #include <asm/pci-direct.h>
 #include <asm/tsc.h>
+#include <asm/cpufeature.h>
 
 #include "cpu.h"
 
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 565648bc1a0a..9299e3bdfad6 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -8,7 +8,7 @@
 #include <linux/module.h>
 #include <linux/uaccess.h>
 
-#include <asm/processor.h>
+#include <asm/cpufeature.h>
 #include <asm/pgtable.h>
 #include <asm/msr.h>
 #include <asm/bugs.h>
diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index 0b6c52388cf4..341449c49f34 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -14,7 +14,7 @@
 #include <linux/sysfs.h>
 #include <linux/pci.h>
 
-#include <asm/processor.h>
+#include <asm/cpufeature.h>
 #include <asm/amd_nb.h>
 #include <asm/smp.h>
 
diff --git a/arch/x86/kernel/cpu/match.c b/arch/x86/kernel/cpu/match.c
index afa9f0d487ea..fbb5e90557a5 100644
--- a/arch/x86/kernel/cpu/match.c
+++ b/arch/x86/kernel/cpu/match.c
@@ -1,5 +1,5 @@
 #include <asm/cpu_device_id.h>
-#include <asm/processor.h>
+#include <asm/cpufeature.h>
 #include <linux/cpu.h>
 #include <linux/module.h>
 #include <linux/slab.h>
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index a006f4cd792b..b5b187c8cc07 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -1576,6 +1576,17 @@ static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
 
                 if (c->x86 == 6 && c->x86_model == 45)
                         quirk_no_way_out = quirk_sandybridge_ifu;
+                /*
+                 * MCG_CAP.MCG_SER_P is necessary but not sufficient to know
+                 * whether this processor will actually generate recoverable
+                 * machine checks. Check to see if this is an E7 model Xeon.
+                 * We can't do a model number check because E5 and E7 use the
+                 * same model number. E5 doesn't support recovery, E7 does.
+                 */
+                if (mca_cfg.recovery || (mca_cfg.ser &&
+                        !strncmp(c->x86_model_id,
+                                 "Intel(R) Xeon(R) CPU E7-", 24)))
+                        set_cpu_cap(c, X86_FEATURE_MCE_RECOVERY);
         }
         if (cfg->monarch_timeout < 0)
                 cfg->monarch_timeout = 0;
@@ -2028,6 +2039,8 @@ static int __init mcheck_enable(char *str)
                 cfg->bootlog = (str[0] == 'b');
         else if (!strcmp(str, "bios_cmci_threshold"))
                 cfg->bios_cmci_threshold = true;
+        else if (!strcmp(str, "recovery"))
+                cfg->recovery = true;
         else if (isdigit(str[0])) {
                 if (get_option(&str, &cfg->tolerant) == 2)
                         get_option(&str, &(cfg->monarch_timeout));
diff --git a/arch/x86/kernel/cpu/mkcapflags.sh b/arch/x86/kernel/cpu/mkcapflags.sh
index 3f20710a5b23..6988c74409a8 100644
--- a/arch/x86/kernel/cpu/mkcapflags.sh
+++ b/arch/x86/kernel/cpu/mkcapflags.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Generate the x86_cap/bug_flags[] arrays from include/asm/cpufeature.h
+# Generate the x86_cap/bug_flags[] arrays from include/asm/cpufeatures.h
 #
 
 IN=$1
@@ -49,8 +49,8 @@ dump_array()
 trap 'rm "$OUT"' EXIT
 
 (
-        echo "#ifndef _ASM_X86_CPUFEATURE_H"
-        echo "#include <asm/cpufeature.h>"
+        echo "#ifndef _ASM_X86_CPUFEATURES_H"
+        echo "#include <asm/cpufeatures.h>"
         echo "#endif"
         echo ""
 
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index 5c3d149ee91c..74f1d90f9c29 100644
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -47,7 +47,7 @@
 #include <linux/smp.h>
 #include <linux/syscore_ops.h>
 
-#include <asm/processor.h>
+#include <asm/cpufeature.h>
 #include <asm/e820.h>
 #include <asm/mtrr.h>
 #include <asm/msr.h>
diff --git a/arch/x86/kernel/cpu/transmeta.c b/arch/x86/kernel/cpu/transmeta.c
index 252da7aceca6..a19a663282b5 100644
--- a/arch/x86/kernel/cpu/transmeta.c
+++ b/arch/x86/kernel/cpu/transmeta.c
@@ -1,6 +1,6 @@
 #include <linux/kernel.h>
 #include <linux/mm.h>
-#include <asm/processor.h>
+#include <asm/cpufeature.h>
 #include <asm/msr.h>
 #include "cpu.h"
 
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c
index 569c1e4f96fe..b3c2a697820a 100644
--- a/arch/x86/kernel/e820.c
+++ b/arch/x86/kernel/e820.c
@@ -24,6 +24,7 @@
 #include <asm/e820.h>
 #include <asm/proto.h>
 #include <asm/setup.h>
+#include <asm/cpufeature.h>
 
 /*
  * The e820 map is the map that gets modified e.g. with command line parameters
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 29408d6d6626..04f9641e0cb6 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -697,9 +697,8 @@ static inline void tramp_free(void *tramp) { }
 #endif
 
 /* Defined as markers to the end of the ftrace default trampolines */
-extern void ftrace_caller_end(void);
 extern void ftrace_regs_caller_end(void);
-extern void ftrace_return(void);
+extern void ftrace_epilogue(void);
 extern void ftrace_caller_op_ptr(void);
 extern void ftrace_regs_caller_op_ptr(void);
 
@@ -746,7 +745,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
                 op_offset = (unsigned long)ftrace_regs_caller_op_ptr;
         } else {
                 start_offset = (unsigned long)ftrace_caller;
-                end_offset = (unsigned long)ftrace_caller_end;
+                end_offset = (unsigned long)ftrace_epilogue;
                 op_offset = (unsigned long)ftrace_caller_op_ptr;
         }
 
@@ -754,7 +753,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
 
         /*
          * Allocate enough size to store the ftrace_caller code,
-         * the jmp to ftrace_return, as well as the address of
+         * the jmp to ftrace_epilogue, as well as the address of
          * the ftrace_ops this trampoline is used for.
          */
         trampoline = alloc_tramp(size + MCOUNT_INSN_SIZE + sizeof(void *));
@@ -772,8 +771,8 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
 
         ip = (unsigned long)trampoline + size;
 
-        /* The trampoline ends with a jmp to ftrace_return */
-        jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_return);
+        /* The trampoline ends with a jmp to ftrace_epilogue */
+        jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_epilogue);
         memcpy(trampoline + size, jmp, MCOUNT_INSN_SIZE);
 
         /*
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 6bc9ae24b6d2..af1112980dd4 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -19,7 +19,7 @@
 #include <asm/setup.h>
 #include <asm/processor-flags.h>
 #include <asm/msr-index.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/percpu.h>
 #include <asm/nops.h>
 #include <asm/bootparam.h>
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index ffdc0e860390..2e974680f5ad 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -38,7 +38,6 @@
 #define pud_index(x)    (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
 
 L4_PAGE_OFFSET = pgd_index(__PAGE_OFFSET)
-L3_PAGE_OFFSET = pud_index(__PAGE_OFFSET)
 L4_START_KERNEL = pgd_index(__START_KERNEL_map)
 L3_START_KERNEL = pud_index(__START_KERNEL_map)
 
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c
index b8e6ff5cd5d0..be0ebbb6d1d1 100644
--- a/arch/x86/kernel/hpet.c
+++ b/arch/x86/kernel/hpet.c
@@ -12,6 +12,7 @@
 #include <linux/pm.h>
 #include <linux/io.h>
 
+#include <asm/cpufeature.h>
 #include <asm/irqdomain.h>
 #include <asm/fixmap.h>
 #include <asm/hpet.h>
diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index 1deffe6cc873..5b187dfbfc8b 100644
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -671,38 +671,37 @@ NOKPROBE_SYMBOL(kprobe_int3_handler);
  * When a retprobed function returns, this code saves registers and
  * calls trampoline_handler() runs, which calls the kretprobe's handler.
  */
-static void __used kretprobe_trampoline_holder(void)
-{
-        asm volatile (
-                        ".global kretprobe_trampoline\n"
-                        "kretprobe_trampoline: \n"
+asm(
+        ".global kretprobe_trampoline\n"
+        ".type kretprobe_trampoline, @function\n"
+        "kretprobe_trampoline:\n"
 #ifdef CONFIG_X86_64
-                        /* We don't bother saving the ss register */
-                        "       pushq %rsp\n"
-                        "       pushfq\n"
-                        SAVE_REGS_STRING
-                        "       movq %rsp, %rdi\n"
-                        "       call trampoline_handler\n"
-                        /* Replace saved sp with true return address. */
-                        "       movq %rax, 152(%rsp)\n"
-                        RESTORE_REGS_STRING
-                        "       popfq\n"
+        /* We don't bother saving the ss register */
+        "       pushq %rsp\n"
+        "       pushfq\n"
+        SAVE_REGS_STRING
+        "       movq %rsp, %rdi\n"
+        "       call trampoline_handler\n"
+        /* Replace saved sp with true return address. */
+        "       movq %rax, 152(%rsp)\n"
+        RESTORE_REGS_STRING
+        "       popfq\n"
 #else
-                        "       pushf\n"
-                        SAVE_REGS_STRING
-                        "       movl %esp, %eax\n"
-                        "       call trampoline_handler\n"
-                        /* Move flags to cs */
-                        "       movl 56(%esp), %edx\n"
-                        "       movl %edx, 52(%esp)\n"
-                        /* Replace saved flags with true return address. */
-                        "       movl %eax, 56(%esp)\n"
-                        RESTORE_REGS_STRING
-                        "       popf\n"
+        "       pushf\n"
+        SAVE_REGS_STRING
+        "       movl %esp, %eax\n"
+        "       call trampoline_handler\n"
+        /* Move flags to cs */
+        "       movl 56(%esp), %edx\n"
+        "       movl %edx, 52(%esp)\n"
+        /* Replace saved flags with true return address. */
+        "       movl %eax, 56(%esp)\n"
+        RESTORE_REGS_STRING
+        "       popf\n"
 #endif
-                        "       ret\n");
-}
-NOKPROBE_SYMBOL(kretprobe_trampoline_holder);
+        "       ret\n"
+        ".size kretprobe_trampoline, .-kretprobe_trampoline\n"
+);
 NOKPROBE_SYMBOL(kretprobe_trampoline);
 
 /*
diff --git a/arch/x86/kernel/mcount_64.S b/arch/x86/kernel/mcount_64.S
index 87e1762e2bca..ed48a9f465f8 100644
--- a/arch/x86/kernel/mcount_64.S
+++ b/arch/x86/kernel/mcount_64.S
@@ -168,12 +168,14 @@ GLOBAL(ftrace_call)
         restore_mcount_regs
 
         /*
-         * The copied trampoline must call ftrace_return as it
+         * The copied trampoline must call ftrace_epilogue as it
          * still may need to call the function graph tracer.
+         *
+         * The code up to this label is copied into trampolines so
+         * think twice before adding any new code or changing the
+         * layout here.
          */
-GLOBAL(ftrace_caller_end)
-
-GLOBAL(ftrace_return)
+GLOBAL(ftrace_epilogue)
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 GLOBAL(ftrace_graph_call)
@@ -244,14 +246,14 @@ GLOBAL(ftrace_regs_call)
         popfq
 
         /*
-         * As this jmp to ftrace_return can be a short jump
+         * As this jmp to ftrace_epilogue can be a short jump
          * it must not be copied into the trampoline.
          * The trampoline will add the code to jump
          * to the return.
          */
 GLOBAL(ftrace_regs_caller_end)
 
-        jmp ftrace_return
+        jmp ftrace_epilogue
 
 END(ftrace_regs_caller)
 
diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c
index 64f9616f93f1..7f3550acde1b 100644
--- a/arch/x86/kernel/msr.c
+++ b/arch/x86/kernel/msr.c
@@ -40,7 +40,7 @@
 #include <linux/uaccess.h>
 #include <linux/gfp.h>
 
-#include <asm/processor.h>
+#include <asm/cpufeature.h>
 #include <asm/msr.h>
 
 static struct class *msr_class;
diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c
index cb6282c3638f..548ddf7d6fd2 100644
--- a/arch/x86/kernel/signal.c
+++ b/arch/x86/kernel/signal.c
@@ -61,7 +61,38 @@
         regs->seg = GET_SEG(seg) | 3;                   \
 } while (0)
 
-int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc)
+#ifdef CONFIG_X86_64
+/*
+ * If regs->ss will cause an IRET fault, change it.  Otherwise leave it
+ * alone.  Using this generally makes no sense unless
+ * user_64bit_mode(regs) would return true.
+ */
+static void force_valid_ss(struct pt_regs *regs)
+{
+        u32 ar;
+        asm volatile ("lar %[old_ss], %[ar]\n\t"
+                      "jz 1f\n\t"               /* If invalid: */
+                      "xorl %[ar], %[ar]\n\t"   /* set ar = 0 */
+                      "1:"
+                      : [ar] "=r" (ar)
+                      : [old_ss] "rm" ((u16)regs->ss));
+
+        /*
+         * For a valid 64-bit user context, we need DPL 3, type
+         * read-write data or read-write exp-down data, and S and P
+         * set.  We can't use VERW because VERW doesn't check the
+         * P bit.
+         */
+        ar &= AR_DPL_MASK | AR_S | AR_P | AR_TYPE_MASK;
+        if (ar != (AR_DPL3 | AR_S | AR_P | AR_TYPE_RWDATA) &&
+            ar != (AR_DPL3 | AR_S | AR_P | AR_TYPE_RWDATA_EXPDOWN))
+                regs->ss = __USER_DS;
+}
+#endif
+
+static int restore_sigcontext(struct pt_regs *regs,
+                              struct sigcontext __user *sc,
+                              unsigned long uc_flags)
 {
         unsigned long buf_val;
         void __user *buf;
@@ -94,15 +125,18 @@ int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc)
                 COPY(r15);
 #endif /* CONFIG_X86_64 */
 
-#ifdef CONFIG_X86_32
                 COPY_SEG_CPL3(cs);
                 COPY_SEG_CPL3(ss);
-#else /* !CONFIG_X86_32 */
-                /* Kernel saves and restores only the CS segment register on signals,
-                 * which is the bare minimum needed to allow mixed 32/64-bit code.
-                 * App's signal handler can save/restore other segments if needed. */
-                COPY_SEG_CPL3(cs);
-#endif /* CONFIG_X86_32 */
+
+#ifdef CONFIG_X86_64
+                /*
+                 * Fix up SS if needed for the benefit of old DOSEMU and
+                 * CRIU.
+                 */
+                if (unlikely(!(uc_flags & UC_STRICT_RESTORE_SS) &&
+                             user_64bit_mode(regs)))
+                        force_valid_ss(regs);
+#endif
 
                 get_user_ex(tmpflags, &sc->flags);
                 regs->flags = (regs->flags & ~FIX_EFLAGS) | (tmpflags & FIX_EFLAGS);
@@ -165,6 +199,7 @@ int setup_sigcontext(struct sigcontext __user *sc, void __user *fpstate,
                 put_user_ex(regs->cs, &sc->cs);
                 put_user_ex(0, &sc->gs);
                 put_user_ex(0, &sc->fs);
+                put_user_ex(regs->ss, &sc->ss);
 #endif /* CONFIG_X86_32 */
 
                 put_user_ex(fpstate, &sc->fpstate);
@@ -403,6 +438,21 @@ static int __setup_rt_frame(int sig, struct ksignal *ksig,
         return 0;
 }
 #else /* !CONFIG_X86_32 */
+static unsigned long frame_uc_flags(struct pt_regs *regs)
+{
+        unsigned long flags;
+
+        if (cpu_has_xsave)
+                flags = UC_FP_XSTATE | UC_SIGCONTEXT_SS;
+        else
+                flags = UC_SIGCONTEXT_SS;
+
+        if (likely(user_64bit_mode(regs)))
+                flags |= UC_STRICT_RESTORE_SS;
+
+        return flags;
+}
+
 static int __setup_rt_frame(int sig, struct ksignal *ksig,
                             sigset_t *set, struct pt_regs *regs)
 {
@@ -422,10 +472,7 @@ static int __setup_rt_frame(int sig, struct ksignal *ksig,
 
         put_user_try {
                 /* Create the ucontext.  */
-                if (cpu_has_xsave)
-                        put_user_ex(UC_FP_XSTATE, &frame->uc.uc_flags);
-                else
-                        put_user_ex(0, &frame->uc.uc_flags);
+                put_user_ex(frame_uc_flags(regs), &frame->uc.uc_flags);
                 put_user_ex(0, &frame->uc.uc_link);
                 save_altstack_ex(&frame->uc.uc_stack, regs->sp);
 
@@ -459,10 +506,28 @@ static int __setup_rt_frame(int sig, struct ksignal *ksig,
 
         regs->sp = (unsigned long)frame;
 
-        /* Set up the CS register to run signal handlers in 64-bit mode,
-           even if the handler happens to be interrupting 32-bit code. */
+        /*
+         * Set up the CS and SS registers to run signal handlers in
+         * 64-bit mode, even if the handler happens to be interrupting
+         * 32-bit or 16-bit code.
+         *
+         * SS is subtle.  In 64-bit mode, we don't need any particular
+         * SS descriptor, but we do need SS to be valid.  It's possible
+         * that the old SS is entirely bogus -- this can happen if the
+         * signal we're trying to deliver is #GP or #SS caused by a bad
+         * SS value.  We also have a compatbility issue here: DOSEMU
+         * relies on the contents of the SS register indicating the
+         * SS value at the time of the signal, even though that code in
+         * DOSEMU predates sigreturn's ability to restore SS.  (DOSEMU
+         * avoids relying on sigreturn to restore SS; instead it uses
+         * a trampoline.)  So we do our best: if the old SS was valid,
+         * we keep it.  Otherwise we replace it.
+         */
         regs->cs = __USER_CS;
 
+        if (unlikely(regs->ss != __USER_DS))
+                force_valid_ss(regs);
+
         return 0;
 }
 #endif /* CONFIG_X86_32 */
@@ -489,10 +554,7 @@ static int x32_setup_rt_frame(struct ksignal *ksig,
 
         put_user_try {
                 /* Create the ucontext.  */
-                if (cpu_has_xsave)
-                        put_user_ex(UC_FP_XSTATE, &frame->uc.uc_flags);
-                else
-                        put_user_ex(0, &frame->uc.uc_flags);
+                put_user_ex(frame_uc_flags(regs), &frame->uc.uc_flags);
                 put_user_ex(0, &frame->uc.uc_link);
                 compat_save_altstack_ex(&frame->uc.uc_stack, regs->sp);
                 put_user_ex(0, &frame->uc.uc__pad0);
@@ -554,7 +616,11 @@ asmlinkage unsigned long sys_sigreturn(void)
 
         set_current_blocked(&set);
 
-        if (restore_sigcontext(regs, &frame->sc))
+        /*
+         * x86_32 has no uc_flags bits relevant to restore_sigcontext.
+         * Save a few cycles by skipping the __get_user.
+         */
+        if (restore_sigcontext(regs, &frame->sc, 0))
                 goto badframe;
         return regs->ax;
 
@@ -570,16 +636,19 @@ asmlinkage long sys_rt_sigreturn(void)
         struct pt_regs *regs = current_pt_regs();
         struct rt_sigframe __user *frame;
         sigset_t set;
+        unsigned long uc_flags;
 
         frame = (struct rt_sigframe __user *)(regs->sp - sizeof(long));
         if (!access_ok(VERIFY_READ, frame, sizeof(*frame)))
                 goto badframe;
         if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set)))
                 goto badframe;
+        if (__get_user(uc_flags, &frame->uc.uc_flags))
+                goto badframe;
 
         set_current_blocked(&set);
 
-        if (restore_sigcontext(regs, &frame->uc.uc_mcontext))
+        if (restore_sigcontext(regs, &frame->uc.uc_mcontext, uc_flags))
                 goto badframe;
 
         if (restore_altstack(&frame->uc.uc_stack))
@@ -692,12 +761,15 @@ handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 
 static inline unsigned long get_nr_restart_syscall(const struct pt_regs *regs)
 {
-#if defined(CONFIG_X86_32) || !defined(CONFIG_X86_64)
+#ifdef CONFIG_X86_64
+        if (is_ia32_task())
+                return __NR_ia32_restart_syscall;
+#endif
+#ifdef CONFIG_X86_X32_ABI
+        return __NR_restart_syscall | (regs->orig_ax & __X32_SYSCALL_BIT);
+#else
         return __NR_restart_syscall;
-#else /* !CONFIG_X86_32 && CONFIG_X86_64 */
-        return test_thread_flag(TIF_IA32) ? __NR_ia32_restart_syscall :
-                __NR_restart_syscall | (regs->orig_ax & __X32_SYSCALL_BIT);
-#endif /* CONFIG_X86_32 || !CONFIG_X86_64 */
+#endif
 }
 
 /*
@@ -763,6 +835,7 @@ asmlinkage long sys32_x32_rt_sigreturn(void)
         struct pt_regs *regs = current_pt_regs();
         struct rt_sigframe_x32 __user *frame;
         sigset_t set;
+        unsigned long uc_flags;
 
         frame = (struct rt_sigframe_x32 __user *)(regs->sp - 8);
 
@@ -770,10 +843,12 @@ asmlinkage long sys32_x32_rt_sigreturn(void)
                 goto badframe;
         if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set)))
                 goto badframe;
+        if (__get_user(uc_flags, &frame->uc.uc_flags))
+                goto badframe;
 
         set_current_blocked(&set);
 
-        if (restore_sigcontext(regs, &frame->uc.uc_mcontext))
+        if (restore_sigcontext(regs, &frame->uc.uc_mcontext, uc_flags))
                 goto badframe;
 
         if (compat_restore_altstack(&frame->uc.uc_stack))
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index ade185a46b1d..410e8e2700c5 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -83,30 +83,16 @@ gate_desc idt_table[NR_VECTORS] __page_aligned_bss;
 DECLARE_BITMAP(used_vectors, NR_VECTORS);
 EXPORT_SYMBOL_GPL(used_vectors);
 
-static inline void conditional_sti(struct pt_regs *regs)
+static inline void cond_local_irq_enable(struct pt_regs *regs)
 {
         if (regs->flags & X86_EFLAGS_IF)
                 local_irq_enable();
 }
 
-static inline void preempt_conditional_sti(struct pt_regs *regs)
-{
-        preempt_count_inc();
-        if (regs->flags & X86_EFLAGS_IF)
-                local_irq_enable();
-}
-
-static inline void conditional_cli(struct pt_regs *regs)
-{
-        if (regs->flags & X86_EFLAGS_IF)
-                local_irq_disable();
-}
-
-static inline void preempt_conditional_cli(struct pt_regs *regs)
+static inline void cond_local_irq_disable(struct pt_regs *regs)
 {
         if (regs->flags & X86_EFLAGS_IF)
                 local_irq_disable();
-        preempt_count_dec();
 }
 
 void ist_enter(struct pt_regs *regs)
@@ -286,7 +272,7 @@ static void do_error_trap(struct pt_regs *regs, long error_code, char *str,
 
         if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=
                         NOTIFY_STOP) {
-                conditional_sti(regs);
+                cond_local_irq_enable(regs);
                 do_trap(trapnr, signr, str, regs, error_code,
                         fill_trap_info(regs, signr, trapnr, &info));
         }
@@ -368,7 +354,7 @@ dotraplinkage void do_bounds(struct pt_regs *regs, long error_code)
         if (notify_die(DIE_TRAP, "bounds", regs, error_code,
                         X86_TRAP_BR, SIGSEGV) == NOTIFY_STOP)
                 return;
-        conditional_sti(regs);
+        cond_local_irq_enable(regs);
 
         if (!user_mode(regs))
                 die("bounds", regs, error_code);
@@ -443,7 +429,7 @@ do_general_protection(struct pt_regs *regs, long error_code)
         struct task_struct *tsk;
 
         RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
-        conditional_sti(regs);
+        cond_local_irq_enable(regs);
 
         if (v8086_mode(regs)) {
                 local_irq_enable();
@@ -517,9 +503,11 @@ dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
          * as we may switch to the interrupt stack.
          */
         debug_stack_usage_inc();
-        preempt_conditional_sti(regs);
+        preempt_disable();
+        cond_local_irq_enable(regs);
         do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);
-        preempt_conditional_cli(regs);
+        cond_local_irq_disable(regs);
+        preempt_enable_no_resched();
         debug_stack_usage_dec();
 exit:
         ist_exit(regs);
@@ -648,12 +636,14 @@ dotraplinkage void do_debug(struct pt_regs *regs, long error_code)
         debug_stack_usage_inc();
 
         /* It's safe to allow irq's after DR6 has been saved */
-        preempt_conditional_sti(regs);
+        preempt_disable();
+        cond_local_irq_enable(regs);
 
         if (v8086_mode(regs)) {
                 handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code,
                                         X86_TRAP_DB);
-                preempt_conditional_cli(regs);
+                cond_local_irq_disable(regs);
+                preempt_enable_no_resched();
                 debug_stack_usage_dec();
                 goto exit;
         }
@@ -673,7 +663,8 @@ dotraplinkage void do_debug(struct pt_regs *regs, long error_code)
         si_code = get_si_code(tsk->thread.debugreg6);
         if (tsk->thread.debugreg6 & (DR_STEP | DR_TRAP_BITS) || user_icebp)
                 send_sigtrap(tsk, regs, error_code, si_code);
-        preempt_conditional_cli(regs);
+        cond_local_irq_disable(regs);
+        preempt_enable_no_resched();
         debug_stack_usage_dec();
 
 exit:
@@ -696,7 +687,7 @@ static void math_error(struct pt_regs *regs, int error_code, int trapnr)
 
         if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, SIGFPE) == NOTIFY_STOP)
                 return;
-        conditional_sti(regs);
+        cond_local_irq_enable(regs);
 
         if (!user_mode(regs)) {
                 if (!fixup_exception(regs)) {
@@ -743,7 +734,7 @@ do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
 dotraplinkage void
 do_spurious_interrupt_bug(struct pt_regs *regs, long error_code)
 {
-        conditional_sti(regs);
+        cond_local_irq_enable(regs);
 }
 
 dotraplinkage void
@@ -756,7 +747,7 @@ do_device_not_available(struct pt_regs *regs, long error_code)
         if (read_cr0() & X86_CR0_EM) {
                 struct math_emu_info info = { };
 
-                conditional_sti(regs);
+                cond_local_irq_enable(regs);
 
                 info.regs = regs;
                 math_emulate(&info);
@@ -765,7 +756,7 @@ do_device_not_available(struct pt_regs *regs, long error_code)
 #endif
         fpu__restore(&current->thread.fpu); /* interrupts still off */
 #ifdef CONFIG_X86_32
-        conditional_sti(regs);
+        cond_local_irq_enable(regs);
 #endif
 }
 NOKPROBE_SYMBOL(do_device_not_available);
diff --git a/arch/x86/kernel/verify_cpu.S b/arch/x86/kernel/verify_cpu.S
index 07efb35ee4bc..014ea59aa153 100644
--- a/arch/x86/kernel/verify_cpu.S
+++ b/arch/x86/kernel/verify_cpu.S
@@ -30,7 +30,7 @@
  *      appropriately. Either display a message or halt.
  */
 
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/msr-index.h>
 
 verify_cpu:
diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c
index e574b8546518..3dce1ca0a653 100644
--- a/arch/x86/kernel/vm86_32.c
+++ b/arch/x86/kernel/vm86_32.c
@@ -362,7 +362,7 @@ static long do_sys_vm86(struct vm86plus_struct __user *user_vm86, bool plus)
         /* make room for real-mode segments */
         tsk->thread.sp0 += 16;
 
-        if (static_cpu_has_safe(X86_FEATURE_SEP))
+        if (static_cpu_has(X86_FEATURE_SEP))
                 tsk->thread.sysenter_cs = 0;
 
         load_sp0(tss, &tsk->thread);
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 74e4bf11f562..92dc211c11db 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -195,6 +195,17 @@ SECTIONS
         :init
 #endif
 
+        /*
+         * Section for code used exclusively before alternatives are run. All
+         * references to such code must be patched out by alternatives, normally
+         * by using X86_FEATURE_ALWAYS CPU feature bit.
+         *
+         * See static_cpu_has() for an example.
+         */
+        .altinstr_aux : AT(ADDR(.altinstr_aux) - LOAD_OFFSET) {
+                *(.altinstr_aux)
+        }
+
         INIT_DATA_SECTION(16)
 
         .x86_cpu_dev.init : AT(ADDR(.x86_cpu_dev.init) - LOAD_OFFSET) {
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index 1505587d06e9..80363ebfd0a6 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -309,23 +309,29 @@ static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
 
 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
 
-#define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
+#define FOP_FUNC(name) \
+        ".align " __stringify(FASTOP_SIZE) " \n\t" \
+        ".type " name ", @function \n\t" \
+        name ":\n\t"
+
 #define FOP_RET   "ret \n\t"
 
 #define FOP_START(op) \
         extern void em_##op(struct fastop *fake); \
         asm(".pushsection .text, \"ax\" \n\t" \
             ".global em_" #op " \n\t" \
-            FOP_ALIGN \
-            "em_" #op ": \n\t"
+            FOP_FUNC("em_" #op)
 
 #define FOP_END \
             ".popsection")
 
-#define FOPNOP() FOP_ALIGN FOP_RET
+#define FOPNOP() \
+        FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
+        FOP_RET
 
 #define FOP1E(op,  dst) \
-        FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
+        FOP_FUNC(#op "_" #dst) \
+        "10: " #op " %" #dst " \n\t" FOP_RET
 
 #define FOP1EEX(op,  dst) \
         FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
@@ -357,7 +363,8 @@ static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
         FOP_END
 
 #define FOP2E(op,  dst, src)       \
-        FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
+        FOP_FUNC(#op "_" #dst "_" #src) \
+        #op " %" #src ", %" #dst " \n\t" FOP_RET
 
 #define FASTOP2(op) \
         FOP_START(op) \
@@ -395,7 +402,8 @@ static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
         FOP_END
 
 #define FOP3E(op,  dst, src, src2) \
-        FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
+        FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
+        #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
 
 /* 3-operand, word-only, src2=cl */
 #define FASTOP3WCL(op) \
@@ -407,7 +415,12 @@ static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
         FOP_END
 
 /* Special case for SETcc - 1 instruction per cc */
-#define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
+#define FOP_SETCC(op) \
+        ".align 4 \n\t" \
+        ".type " #op ", @function \n\t" \
+        #op ": \n\t" \
+        #op " %al \n\t" \
+        FOP_RET
 
 asm(".global kvm_fastop_exception \n"
     "kvm_fastop_exception: xor %esi, %esi; ret");
@@ -956,7 +969,7 @@ static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
         return fastop(ctxt, em_bsr);
 }
 
-static u8 test_cc(unsigned int condition, unsigned long flags)
+static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
 {
         u8 rc;
         void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index e2951b6edbbc..e1535225bc1d 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -8356,6 +8356,7 @@ static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
 static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
 {
         u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
+        register void *__sp asm(_ASM_SP);
 
         /*
          * If external interrupt exists, IF bit is set in rflags/eflags on the
@@ -8388,8 +8389,9 @@ static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
                         "call *%[entry]\n\t"
                         :
 #ifdef CONFIG_X86_64
-                        [sp]"=&r"(tmp)
+                        [sp]"=&r"(tmp),
 #endif
+                        "+r"(__sp)
                         :
                         [entry]"r"(entry),
                         [ss]"i"(__KERNEL_DS),
diff --git a/arch/x86/lib/clear_page_64.S b/arch/x86/lib/clear_page_64.S
index a2fe51b00cce..65be7cfaf947 100644
--- a/arch/x86/lib/clear_page_64.S
+++ b/arch/x86/lib/clear_page_64.S
@@ -1,5 +1,5 @@
 #include <linux/linkage.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/alternative-asm.h>
 
 /*
diff --git a/arch/x86/lib/copy_page_64.S b/arch/x86/lib/copy_page_64.S
index 009f98216b7e..24ef1c2104d4 100644
--- a/arch/x86/lib/copy_page_64.S
+++ b/arch/x86/lib/copy_page_64.S
@@ -1,7 +1,7 @@
 /* Written 2003 by Andi Kleen, based on a kernel by Evandro Menezes */
 
 #include <linux/linkage.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/alternative-asm.h>
 
 /*
diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S
index 27f89c79a44b..2b0ef26da0bd 100644
--- a/arch/x86/lib/copy_user_64.S
+++ b/arch/x86/lib/copy_user_64.S
@@ -10,7 +10,7 @@
 #include <asm/current.h>
 #include <asm/asm-offsets.h>
 #include <asm/thread_info.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/alternative-asm.h>
 #include <asm/asm.h>
 #include <asm/smap.h>
diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S
index 16698bba87de..a0de849435ad 100644
--- a/arch/x86/lib/memcpy_64.S
+++ b/arch/x86/lib/memcpy_64.S
@@ -1,7 +1,7 @@
 /* Copyright 2002 Andi Kleen */
 
 #include <linux/linkage.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/alternative-asm.h>
 
 /*
diff --git a/arch/x86/lib/memmove_64.S b/arch/x86/lib/memmove_64.S
index ca2afdd6d98e..90ce01bee00c 100644
--- a/arch/x86/lib/memmove_64.S
+++ b/arch/x86/lib/memmove_64.S
@@ -6,7 +6,7 @@
  *      - Copyright 2011 Fenghua Yu <fenghua.yu@intel.com>
  */
 #include <linux/linkage.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/alternative-asm.h>
 
 #undef memmove
diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S
index 2661fad05827..c9c81227ea37 100644
--- a/arch/x86/lib/memset_64.S
+++ b/arch/x86/lib/memset_64.S
@@ -1,7 +1,7 @@
 /* Copyright 2002 Andi Kleen, SuSE Labs */
 
 #include <linux/linkage.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/alternative-asm.h>
 
 .weak memset
diff --git a/arch/x86/lib/rwsem.S b/arch/x86/lib/rwsem.S
index 40027db99140..be110efa0096 100644
--- a/arch/x86/lib/rwsem.S
+++ b/arch/x86/lib/rwsem.S
@@ -15,6 +15,7 @@
 
 #include <linux/linkage.h>
 #include <asm/alternative-asm.h>
+#include <asm/frame.h>
 
 #define __ASM_HALF_REG(reg)     __ASM_SEL(reg, e##reg)
 #define __ASM_HALF_SIZE(inst)   __ASM_SEL(inst##w, inst##l)
@@ -84,24 +85,29 @@
 
 /* Fix up special calling conventions */
 ENTRY(call_rwsem_down_read_failed)
+        FRAME_BEGIN
         save_common_regs
         __ASM_SIZE(push,) %__ASM_REG(dx)
         movq %rax,%rdi
         call rwsem_down_read_failed
         __ASM_SIZE(pop,) %__ASM_REG(dx)
         restore_common_regs
+        FRAME_END
         ret
 ENDPROC(call_rwsem_down_read_failed)
 
 ENTRY(call_rwsem_down_write_failed)
+        FRAME_BEGIN
         save_common_regs
         movq %rax,%rdi
         call rwsem_down_write_failed
         restore_common_regs
+        FRAME_END
         ret
 ENDPROC(call_rwsem_down_write_failed)
 
 ENTRY(call_rwsem_wake)
+        FRAME_BEGIN
         /* do nothing if still outstanding active readers */
         __ASM_HALF_SIZE(dec) %__ASM_HALF_REG(dx)
         jnz 1f
@@ -109,15 +115,18 @@ ENTRY(call_rwsem_wake)
         movq %rax,%rdi
         call rwsem_wake
         restore_common_regs
-1:      ret
+1:      FRAME_END
+        ret
 ENDPROC(call_rwsem_wake)
 
 ENTRY(call_rwsem_downgrade_wake)
+        FRAME_BEGIN
         save_common_regs
         __ASM_SIZE(push,) %__ASM_REG(dx)
         movq %rax,%rdi
         call rwsem_downgrade_wake
         __ASM_SIZE(pop,) %__ASM_REG(dx)
         restore_common_regs
+        FRAME_END
         ret
 ENDPROC(call_rwsem_downgrade_wake)
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index f4ae536b0914..04e2e7144bee 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -943,7 +943,7 @@ int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
                         return -EINVAL;
         }
 
-        *prot = __pgprot((pgprot_val(vma->vm_page_prot) & (~_PAGE_CACHE_MASK)) |
+        *prot = __pgprot((pgprot_val(*prot) & (~_PAGE_CACHE_MASK)) |
                          cachemode2protval(pcm));
 
         return 0;
@@ -959,7 +959,7 @@ int track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
 
         /* Set prot based on lookup */
         pcm = lookup_memtype(pfn_t_to_phys(pfn));
-        *prot = __pgprot((pgprot_val(vma->vm_page_prot) & (~_PAGE_CACHE_MASK)) |
+        *prot = __pgprot((pgprot_val(*prot) & (~_PAGE_CACHE_MASK)) |
                          cachemode2protval(pcm));
 
         return 0;
diff --git a/arch/x86/mm/setup_nx.c b/arch/x86/mm/setup_nx.c
index 92e2eacb3321..f65a33f505b6 100644
--- a/arch/x86/mm/setup_nx.c
+++ b/arch/x86/mm/setup_nx.c
@@ -4,6 +4,7 @@
 
 #include <asm/pgtable.h>
 #include <asm/proto.h>
+#include <asm/cpufeature.h>
 
 static int disable_nx;
 
diff --git a/arch/x86/net/bpf_jit.S b/arch/x86/net/bpf_jit.S
index 4093216b3791..f2a7faf4706e 100644
--- a/arch/x86/net/bpf_jit.S
+++ b/arch/x86/net/bpf_jit.S
@@ -8,6 +8,7 @@
  * of the License.
  */
 #include <linux/linkage.h>
+#include <asm/frame.h>
 
 /*
  * Calling convention :
@@ -22,15 +23,16 @@
         32 /* space for rbx,r13,r14,r15 */ + \
         8 /* space for skb_copy_bits */)
 
-sk_load_word:
-        .globl  sk_load_word
+#define FUNC(name) \
+        .globl name; \
+        .type name, @function; \
+        name:
 
+FUNC(sk_load_word)
         test    %esi,%esi
         js      bpf_slow_path_word_neg
 
-sk_load_word_positive_offset:
-        .globl  sk_load_word_positive_offset
-
+FUNC(sk_load_word_positive_offset)
         mov     %r9d,%eax               # hlen
         sub     %esi,%eax               # hlen - offset
         cmp     $3,%eax
@@ -39,15 +41,11 @@ sk_load_word_positive_offset:
         bswap   %eax                    /* ntohl() */
         ret
 
-sk_load_half:
-        .globl  sk_load_half
-
+FUNC(sk_load_half)
         test    %esi,%esi
         js      bpf_slow_path_half_neg
 
-sk_load_half_positive_offset:
-        .globl  sk_load_half_positive_offset
-
+FUNC(sk_load_half_positive_offset)
         mov     %r9d,%eax
         sub     %esi,%eax               #       hlen - offset
         cmp     $1,%eax
@@ -56,15 +54,11 @@ sk_load_half_positive_offset:
         rol     $8,%ax                  # ntohs()
         ret
 
-sk_load_byte:
-        .globl  sk_load_byte
-
+FUNC(sk_load_byte)
         test    %esi,%esi
         js      bpf_slow_path_byte_neg
 
-sk_load_byte_positive_offset:
-        .globl  sk_load_byte_positive_offset
-
+FUNC(sk_load_byte_positive_offset)
         cmp     %esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
         jle     bpf_slow_path_byte
         movzbl  (SKBDATA,%rsi),%eax
@@ -72,16 +66,18 @@ sk_load_byte_positive_offset:
 
 /* rsi contains offset and can be scratched */
 #define bpf_slow_path_common(LEN)               \
+        lea     -MAX_BPF_STACK + 32(%rbp), %rdx;\
+        FRAME_BEGIN;                            \
         mov     %rbx, %rdi; /* arg1 == skb */   \
         push    %r9;                            \
         push    SKBDATA;                        \
 /* rsi already has offset */                    \
         mov     $LEN,%ecx;      /* len */       \
-        lea     - MAX_BPF_STACK + 32(%rbp),%rdx;                        \
         call    skb_copy_bits;                  \
         test    %eax,%eax;                      \
         pop     SKBDATA;                        \
-        pop     %r9;
+        pop     %r9;                            \
+        FRAME_END
 
 
 bpf_slow_path_word:
@@ -106,6 +102,7 @@ bpf_slow_path_byte:
         ret
 
 #define sk_negative_common(SIZE)                                \
+        FRAME_BEGIN;                                            \
         mov     %rbx, %rdi; /* arg1 == skb */                   \
         push    %r9;                                            \
         push    SKBDATA;                                        \
@@ -115,13 +112,14 @@ bpf_slow_path_byte:
         test    %rax,%rax;                                      \
         pop     SKBDATA;                                        \
         pop     %r9;                                            \
+        FRAME_END;                                              \
         jz      bpf_error
 
 bpf_slow_path_word_neg:
         cmp     SKF_MAX_NEG_OFF, %esi   /* test range */
         jl      bpf_error       /* offset lower -> error  */
-sk_load_word_negative_offset:
-        .globl  sk_load_word_negative_offset
+
+FUNC(sk_load_word_negative_offset)
         sk_negative_common(4)
         mov     (%rax), %eax
         bswap   %eax
@@ -130,8 +128,8 @@ sk_load_word_negative_offset:
 bpf_slow_path_half_neg:
         cmp     SKF_MAX_NEG_OFF, %esi
         jl      bpf_error
-sk_load_half_negative_offset:
-        .globl  sk_load_half_negative_offset
+
+FUNC(sk_load_half_negative_offset)
         sk_negative_common(2)
         mov     (%rax),%ax
         rol     $8,%ax
@@ -141,8 +139,8 @@ sk_load_half_negative_offset:
 bpf_slow_path_byte_neg:
         cmp     SKF_MAX_NEG_OFF, %esi
         jl      bpf_error
-sk_load_byte_negative_offset:
-        .globl  sk_load_byte_negative_offset
+
+FUNC(sk_load_byte_negative_offset)
         sk_negative_common(1)
         movzbl  (%rax), %eax
         ret
diff --git a/arch/x86/oprofile/op_model_amd.c b/arch/x86/oprofile/op_model_amd.c
index 50d86c0e9ba4..660a83c8287b 100644
--- a/arch/x86/oprofile/op_model_amd.c
+++ b/arch/x86/oprofile/op_model_amd.c
@@ -24,7 +24,6 @@
 #include <asm/nmi.h>
 #include <asm/apic.h>
 #include <asm/processor.h>
-#include <asm/cpufeature.h>
 
 #include "op_x86_model.h"
 #include "op_counter.h"
diff --git a/arch/x86/platform/efi/efi_stub_64.S b/arch/x86/platform/efi/efi_stub_64.S
index 86d0f9e08dd9..0df2dcc18404 100644
--- a/arch/x86/platform/efi/efi_stub_64.S
+++ b/arch/x86/platform/efi/efi_stub_64.S
@@ -11,6 +11,7 @@
 #include <asm/msr.h>
 #include <asm/processor-flags.h>
 #include <asm/page_types.h>
+#include <asm/frame.h>
 
 #define SAVE_XMM                        \
         mov %rsp, %rax;                 \
@@ -74,6 +75,7 @@
         .endm
 
 ENTRY(efi_call)
+        FRAME_BEGIN
         SAVE_XMM
         mov (%rsp), %rax
         mov 8(%rax), %rax
@@ -88,6 +90,7 @@ ENTRY(efi_call)
         RESTORE_PGT
         addq $48, %rsp
         RESTORE_XMM
+        FRAME_END
         ret
 ENDPROC(efi_call)
 
diff --git a/arch/x86/power/hibernate_asm_64.S b/arch/x86/power/hibernate_asm_64.S
index e2386cb4e0c3..4400a43b9e28 100644
--- a/arch/x86/power/hibernate_asm_64.S
+++ b/arch/x86/power/hibernate_asm_64.S
@@ -21,8 +21,10 @@
 #include <asm/page_types.h>
 #include <asm/asm-offsets.h>
 #include <asm/processor-flags.h>
+#include <asm/frame.h>
 
 ENTRY(swsusp_arch_suspend)
+        FRAME_BEGIN
         movq    $saved_context, %rax
         movq    %rsp, pt_regs_sp(%rax)
         movq    %rbp, pt_regs_bp(%rax)
@@ -50,7 +52,9 @@ ENTRY(swsusp_arch_suspend)
         movq    %rax, restore_cr3(%rip)
 
         call swsusp_save
+        FRAME_END
         ret
+ENDPROC(swsusp_arch_suspend)
 
 ENTRY(restore_image)
         /* switch to temporary page tables */
@@ -107,6 +111,7 @@ ENTRY(core_restore_code)
          */
 
 ENTRY(restore_registers)
+        FRAME_BEGIN
         /* go back to the original page tables */
         movq    %rbx, %cr3
 
@@ -147,4 +152,6 @@ ENTRY(restore_registers)
         /* tell the hibernation core that we've just restored the memory */
         movq    %rax, in_suspend(%rip)
 
+        FRAME_END
         ret
+ENDPROC(restore_registers)
diff --git a/arch/x86/um/asm/barrier.h b/arch/x86/um/asm/barrier.h
index 174781a404ff..00c319048d52 100644
--- a/arch/x86/um/asm/barrier.h
+++ b/arch/x86/um/asm/barrier.h
@@ -3,7 +3,7 @@
 
 #include <asm/asm.h>
 #include <asm/segment.h>
-#include <asm/cpufeature.h>
+#include <asm/cpufeatures.h>
 #include <asm/cmpxchg.h>
 #include <asm/nops.h>
 
diff --git a/arch/x86/um/sys_call_table_32.c b/arch/x86/um/sys_call_table_32.c
index 439c0994b696..bfce503dffae 100644
--- a/arch/x86/um/sys_call_table_32.c
+++ b/arch/x86/um/sys_call_table_32.c
@@ -25,11 +25,11 @@
 
 #define old_mmap sys_old_mmap
 
-#define __SYSCALL_I386(nr, sym, compat) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ;
+#define __SYSCALL_I386(nr, sym, qual) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ;
 #include <asm/syscalls_32.h>
 
 #undef __SYSCALL_I386
-#define __SYSCALL_I386(nr, sym, compat) [ nr ] = sym,
+#define __SYSCALL_I386(nr, sym, qual) [ nr ] = sym,
 
 extern asmlinkage long sys_ni_syscall(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long);
 
diff --git a/arch/x86/um/sys_call_table_64.c b/arch/x86/um/sys_call_table_64.c
index b74ea6c2c0e7..f306413d3eb6 100644
--- a/arch/x86/um/sys_call_table_64.c
+++ b/arch/x86/um/sys_call_table_64.c
@@ -35,14 +35,11 @@
 #define stub_execveat sys_execveat
 #define stub_rt_sigreturn sys_rt_sigreturn
 
-#define __SYSCALL_COMMON(nr, sym, compat) __SYSCALL_64(nr, sym, compat)
-#define __SYSCALL_X32(nr, sym, compat) /* Not supported */
-
-#define __SYSCALL_64(nr, sym, compat) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ;
+#define __SYSCALL_64(nr, sym, qual) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ;
 #include <asm/syscalls_64.h>
 
 #undef __SYSCALL_64
-#define __SYSCALL_64(nr, sym, compat) [ nr ] = sym,
+#define __SYSCALL_64(nr, sym, qual) [ nr ] = sym,
 
 extern asmlinkage long sys_ni_syscall(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long);
 
diff --git a/arch/x86/um/user-offsets.c b/arch/x86/um/user-offsets.c
index ce7e3607a870..470564bbd08e 100644
--- a/arch/x86/um/user-offsets.c
+++ b/arch/x86/um/user-offsets.c
@@ -9,14 +9,12 @@
 #include <asm/types.h>
 
 #ifdef __i386__
-#define __SYSCALL_I386(nr, sym, compat) [nr] = 1,
+#define __SYSCALL_I386(nr, sym, qual) [nr] = 1,
 static char syscalls[] = {
 #include <asm/syscalls_32.h>
 };
 #else
-#define __SYSCALL_64(nr, sym, compat) [nr] = 1,
-#define __SYSCALL_COMMON(nr, sym, compat) [nr] = 1,
-#define __SYSCALL_X32(nr, sym, compat) /* Not supported */
+#define __SYSCALL_64(nr, sym, qual) [nr] = 1,
 static char syscalls[] = {
 #include <asm/syscalls_64.h>
 };
diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S
index 3e45aa000718..eff224df813f 100644
--- a/arch/x86/xen/xen-asm.S
+++ b/arch/x86/xen/xen-asm.S
@@ -14,6 +14,7 @@
 #include <asm/asm-offsets.h>
 #include <asm/percpu.h>
 #include <asm/processor-flags.h>
+#include <asm/frame.h>
 
 #include "xen-asm.h"
 
@@ -23,6 +24,7 @@
  * then enter the hypervisor to get them handled.
  */
 ENTRY(xen_irq_enable_direct)
+        FRAME_BEGIN
         /* Unmask events */
         movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
 
@@ -39,6 +41,7 @@ ENTRY(xen_irq_enable_direct)
 2:      call check_events
 1:
 ENDPATCH(xen_irq_enable_direct)
+        FRAME_END
         ret
         ENDPROC(xen_irq_enable_direct)
         RELOC(xen_irq_enable_direct, 2b+1)
@@ -82,6 +85,7 @@ ENDPATCH(xen_save_fl_direct)
  * enters the hypervisor to get them delivered if so.
  */
 ENTRY(xen_restore_fl_direct)
+        FRAME_BEGIN
 #ifdef CONFIG_X86_64
         testw $X86_EFLAGS_IF, %di
 #else
@@ -100,6 +104,7 @@ ENTRY(xen_restore_fl_direct)
 2:      call check_events
 1:
 ENDPATCH(xen_restore_fl_direct)
+        FRAME_END
         ret
         ENDPROC(xen_restore_fl_direct)
         RELOC(xen_restore_fl_direct, 2b+1)
@@ -109,7 +114,8 @@ ENDPATCH(xen_restore_fl_direct)
  * Force an event check by making a hypercall, but preserve regs
  * before making the call.
  */
-check_events:
+ENTRY(check_events)
+        FRAME_BEGIN
 #ifdef CONFIG_X86_32
         push %eax
         push %ecx
@@ -139,4 +145,6 @@ check_events:
         pop %rcx
         pop %rax
 #endif
+        FRAME_END
         ret
+ENDPROC(check_events)
diff --git a/arch/x86/xen/xen-asm_64.S b/arch/x86/xen/xen-asm_64.S
index cc8acc410ddb..c3df43141e70 100644
--- a/arch/x86/xen/xen-asm_64.S
+++ b/arch/x86/xen/xen-asm_64.S
@@ -26,6 +26,7 @@ ENTRY(xen_adjust_exception_frame)
         mov 8+0(%rsp), %rcx
         mov 8+8(%rsp), %r11
         ret $16
+ENDPROC(xen_adjust_exception_frame)
 
 hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
 /*
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index cd83d477e32d..3a4b39afc0ab 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -1431,7 +1431,7 @@ static int __init intel_pstate_init(void)
         if (!all_cpu_data)
                 return -ENOMEM;
 
-        if (static_cpu_has_safe(X86_FEATURE_HWP) && !no_hwp) {
+        if (static_cpu_has(X86_FEATURE_HWP) && !no_hwp) {
                 pr_info("intel_pstate: HWP enabled\n");
                 hwp_active++;
         }
diff --git a/drivers/watchdog/hpwdt.c b/drivers/watchdog/hpwdt.c
index 92443c319e59..8fc284cdce4e 100644
--- a/drivers/watchdog/hpwdt.c
+++ b/drivers/watchdog/hpwdt.c
@@ -37,6 +37,7 @@
 #include <asm/cacheflush.h>
 #endif /* CONFIG_HPWDT_NMI_DECODING */
 #include <asm/nmi.h>
+#include <asm/frame.h>
 
 #define HPWDT_VERSION                   "1.3.3"
 #define SECS_TO_TICKS(secs)             ((secs) * 1000 / 128)
@@ -353,10 +354,10 @@ static int detect_cru_service(void)
 
 asm(".text                      \n\t"
     ".align 4                   \n\t"
-    ".globl asminline_call      \n"
+    ".globl asminline_call      \n\t"
+    ".type asminline_call, @function \n\t"
     "asminline_call:            \n\t"
-    "pushq      %rbp            \n\t"
-    "movq       %rsp, %rbp      \n\t"
+    FRAME_BEGIN
     "pushq      %rax            \n\t"
     "pushq      %rbx            \n\t"
     "pushq      %rdx            \n\t"
@@ -386,7 +387,7 @@ asm(".text                      \n\t"
     "popq       %rdx            \n\t"
     "popq       %rbx            \n\t"
     "popq       %rax            \n\t"
-    "leave                      \n\t"
+    FRAME_END
     "ret                        \n\t"
     ".previous");
 
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 4545e2e2ad45..5699bbc23feb 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -931,7 +931,7 @@ static int check_async_write(struct inode *inode, unsigned long bio_flags)
         if (bio_flags & EXTENT_BIO_TREE_LOG)
                 return 0;
 #ifdef CONFIG_X86
-        if (static_cpu_has_safe(X86_FEATURE_XMM4_2))
+        if (static_cpu_has(X86_FEATURE_XMM4_2))
                 return 0;
 #endif
         return 1;
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 516e14944339..b1d4b8c7f7cd 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2138,6 +2138,8 @@ int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
 int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
 int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
                         unsigned long pfn);
+int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
+                        unsigned long pfn, pgprot_t pgprot);
 int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
                         pfn_t pfn);
 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 624b78b848b8..944b2b37313b 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -566,10 +566,26 @@ static inline void clear_tlb_flush_pending(struct mm_struct *mm)
 }
 #endif
 
-struct vm_special_mapping
-{
-        const char *name;
+struct vm_fault;
+
+struct vm_special_mapping {
+        const char *name;       /* The name, e.g. "[vdso]". */
+
+        /*
+         * If .fault is not provided, this points to a
+         * NULL-terminated array of pages that back the special mapping.
+         *
+         * This must not be NULL unless .fault is provided.
+         */
         struct page **pages;
+
+        /*
+         * If non-NULL, then this is called to resolve page faults
+         * on the special mapping.  If used, .pages is not checked.
+         */
+        int (*fault)(const struct vm_special_mapping *sm,
+                     struct vm_area_struct *vma,
+                     struct vm_fault *vmf);
 };
 
 enum tlb_flush_reason {
diff --git a/lib/atomic64_test.c b/lib/atomic64_test.c
index d62de8bf022d..123481814320 100644
--- a/lib/atomic64_test.c
+++ b/lib/atomic64_test.c
@@ -17,7 +17,7 @@
 #include <linux/atomic.h>
 
 #ifdef CONFIG_X86
-#include <asm/processor.h>      /* for boot_cpu_has below */
+#include <asm/cpufeature.h>     /* for boot_cpu_has below */
 #endif
 
 #define TEST(bit, op, c_op, val)                                \
diff --git a/mm/memory.c b/mm/memory.c
index 635451abc8f7..38090ca37a08 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1551,8 +1551,29 @@ out:
 int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
                         unsigned long pfn)
 {
+        return vm_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
+}
+EXPORT_SYMBOL(vm_insert_pfn);
+
+/**
+ * vm_insert_pfn_prot - insert single pfn into user vma with specified pgprot
+ * @vma: user vma to map to
+ * @addr: target user address of this page
+ * @pfn: source kernel pfn
+ * @pgprot: pgprot flags for the inserted page
+ *
+ * This is exactly like vm_insert_pfn, except that it allows drivers to
+ * to override pgprot on a per-page basis.
+ *
+ * This only makes sense for IO mappings, and it makes no sense for
+ * cow mappings.  In general, using multiple vmas is preferable;
+ * vm_insert_pfn_prot should only be used if using multiple VMAs is
+ * impractical.
+ */
+int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
+                        unsigned long pfn, pgprot_t pgprot)
+{
         int ret;
-        pgprot_t pgprot = vma->vm_page_prot;
         /*
          * Technically, architectures with pte_special can avoid all these
          * restrictions (same for remap_pfn_range).  However we would like
@@ -1574,7 +1595,7 @@ int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
 
         return ret;
 }
-EXPORT_SYMBOL(vm_insert_pfn);
+EXPORT_SYMBOL(vm_insert_pfn_prot);
 
 int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
                         pfn_t pfn)
diff --git a/mm/mmap.c b/mm/mmap.c
index 76d1ec29149b..90e3b869a8b9 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -3066,11 +3066,16 @@ static int special_mapping_fault(struct vm_area_struct *vma,
         pgoff_t pgoff;
         struct page **pages;
 
-        if (vma->vm_ops == &legacy_special_mapping_vmops)
+        if (vma->vm_ops == &legacy_special_mapping_vmops) {
                 pages = vma->vm_private_data;
-        else
-                pages = ((struct vm_special_mapping *)vma->vm_private_data)->
-                        pages;
+        } else {
+                struct vm_special_mapping *sm = vma->vm_private_data;
+
+                if (sm->fault)
+                        return sm->fault(sm, vma, vmf);
+
+                pages = sm->pages;
+        }
 
         for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
                 pgoff--;
diff --git a/tools/testing/selftests/x86/Makefile b/tools/testing/selftests/x86/Makefile
index d0c473f65850..d5ce7d7aae3e 100644
--- a/tools/testing/selftests/x86/Makefile
+++ b/tools/testing/selftests/x86/Makefile
@@ -4,15 +4,16 @@ include ../lib.mk
 
 .PHONY: all all_32 all_64 warn_32bit_failure clean
 
-TARGETS_C_BOTHBITS := single_step_syscall sysret_ss_attrs syscall_nt ptrace_syscall
-TARGETS_C_32BIT_ONLY := entry_from_vm86 syscall_arg_fault sigreturn test_syscall_vdso unwind_vdso \
+TARGETS_C_BOTHBITS := single_step_syscall sysret_ss_attrs syscall_nt ptrace_syscall \
+                        check_initial_reg_state sigreturn ldt_gdt
+TARGETS_C_32BIT_ONLY := entry_from_vm86 syscall_arg_fault test_syscall_vdso unwind_vdso \
                         test_FCMOV test_FCOMI test_FISTTP \
-                        ldt_gdt \
                         vdso_restorer
 
 TARGETS_C_32BIT_ALL := $(TARGETS_C_BOTHBITS) $(TARGETS_C_32BIT_ONLY)
+TARGETS_C_64BIT_ALL := $(TARGETS_C_BOTHBITS) $(TARGETS_C_64BIT_ONLY)
 BINARIES_32 := $(TARGETS_C_32BIT_ALL:%=%_32)
-BINARIES_64 := $(TARGETS_C_BOTHBITS:%=%_64)
+BINARIES_64 := $(TARGETS_C_64BIT_ALL:%=%_64)
 
 CFLAGS := -O2 -g -std=gnu99 -pthread -Wall
 
@@ -40,7 +41,7 @@ clean:
 $(TARGETS_C_32BIT_ALL:%=%_32): %_32: %.c
         $(CC) -m32 -o $@ $(CFLAGS) $(EXTRA_CFLAGS) $^ -lrt -ldl -lm
 
-$(TARGETS_C_BOTHBITS:%=%_64): %_64: %.c
+$(TARGETS_C_64BIT_ALL:%=%_64): %_64: %.c
         $(CC) -m64 -o $@ $(CFLAGS) $(EXTRA_CFLAGS) $^ -lrt -ldl
 
 # x86_64 users should be encouraged to install 32-bit libraries
@@ -65,3 +66,9 @@ endif
 sysret_ss_attrs_64: thunks.S
 ptrace_syscall_32: raw_syscall_helper_32.S
 test_syscall_vdso_32: thunks_32.S
+
+# check_initial_reg_state is special: it needs a custom entry, and it
+# needs to be static so that its interpreter doesn't destroy its initial
+# state.
+check_initial_reg_state_32: CFLAGS += -Wl,-ereal_start -static
+check_initial_reg_state_64: CFLAGS += -Wl,-ereal_start -static
diff --git a/tools/testing/selftests/x86/check_initial_reg_state.c b/tools/testing/selftests/x86/check_initial_reg_state.c
new file mode 100644
index 000000000000..6aaed9b85baf
--- /dev/null
+++ b/tools/testing/selftests/x86/check_initial_reg_state.c
@@ -0,0 +1,109 @@
+/*
+ * check_initial_reg_state.c - check that execve sets the correct state
+ * Copyright (c) 2014-2016 Andrew Lutomirski
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
+#define _GNU_SOURCE
+
+#include <stdio.h>
+
+unsigned long ax, bx, cx, dx, si, di, bp, sp, flags;
+unsigned long r8, r9, r10, r11, r12, r13, r14, r15;
+
+asm (
+        ".pushsection .text\n\t"
+        ".type real_start, @function\n\t"
+        ".global real_start\n\t"
+        "real_start:\n\t"
+#ifdef __x86_64__
+        "mov %rax, ax\n\t"
+        "mov %rbx, bx\n\t"
+        "mov %rcx, cx\n\t"
+        "mov %rdx, dx\n\t"
+        "mov %rsi, si\n\t"
+        "mov %rdi, di\n\t"
+        "mov %rbp, bp\n\t"
+        "mov %rsp, sp\n\t"
+        "mov %r8, r8\n\t"
+        "mov %r9, r9\n\t"
+        "mov %r10, r10\n\t"
+        "mov %r11, r11\n\t"
+        "mov %r12, r12\n\t"
+        "mov %r13, r13\n\t"
+        "mov %r14, r14\n\t"
+        "mov %r15, r15\n\t"
+        "pushfq\n\t"
+        "popq flags\n\t"
+#else
+        "mov %eax, ax\n\t"
+        "mov %ebx, bx\n\t"
+        "mov %ecx, cx\n\t"
+        "mov %edx, dx\n\t"
+        "mov %esi, si\n\t"
+        "mov %edi, di\n\t"
+        "mov %ebp, bp\n\t"
+        "mov %esp, sp\n\t"
+        "pushfl\n\t"
+        "popl flags\n\t"
+#endif
+        "jmp _start\n\t"
+        ".size real_start, . - real_start\n\t"
+        ".popsection");
+
+int main()
+{
+        int nerrs = 0;
+
+        if (sp == 0) {
+                printf("[FAIL]\tTest was built incorrectly\n");
+                return 1;
+        }
+
+        if (ax || bx || cx || dx || si || di || bp
+#ifdef __x86_64__
+            || r8 || r9 || r10 || r11 || r12 || r13 || r14 || r15
+#endif
+                ) {
+                printf("[FAIL]\tAll GPRs except SP should be 0\n");
+#define SHOW(x) printf("\t" #x " = 0x%lx\n", x);
+                SHOW(ax);
+                SHOW(bx);
+                SHOW(cx);
+                SHOW(dx);
+                SHOW(si);
+                SHOW(di);
+                SHOW(bp);
+                SHOW(sp);
+#ifdef __x86_64__
+                SHOW(r8);
+                SHOW(r9);
+                SHOW(r10);
+                SHOW(r11);
+                SHOW(r12);
+                SHOW(r13);
+                SHOW(r14);
+                SHOW(r15);
+#endif
+                nerrs++;
+        } else {
+                printf("[OK]\tAll GPRs except SP are 0\n");
+        }
+
+        if (flags != 0x202) {
+                printf("[FAIL]\tFLAGS is 0x%lx, but it should be 0x202\n", flags);
+                nerrs++;
+        } else {
+                printf("[OK]\tFLAGS is 0x202\n");
+        }
+
+        return nerrs ? 1 : 0;
+}
diff --git a/tools/testing/selftests/x86/ptrace_syscall.c b/tools/testing/selftests/x86/ptrace_syscall.c
index 5105b49cd8aa..421456784bc6 100644
--- a/tools/testing/selftests/x86/ptrace_syscall.c
+++ b/tools/testing/selftests/x86/ptrace_syscall.c
@@ -103,6 +103,17 @@ static void sethandler(int sig, void (*handler)(int, siginfo_t *, void *),
                 err(1, "sigaction");
 }
 
+static void setsigign(int sig, int flags)
+{
+        struct sigaction sa;
+        memset(&sa, 0, sizeof(sa));
+        sa.sa_sigaction = (void *)SIG_IGN;
+        sa.sa_flags = flags;
+        sigemptyset(&sa.sa_mask);
+        if (sigaction(sig, &sa, 0))
+                err(1, "sigaction");
+}
+
 static void clearhandler(int sig)
 {
         struct sigaction sa;
@@ -187,7 +198,7 @@ static void test_ptrace_syscall_restart(void)
 
         printf("[RUN]\tSYSEMU\n");
         if (ptrace(PTRACE_SYSEMU, chld, 0, 0) != 0)
-                err(1, "PTRACE_SYSCALL");
+                err(1, "PTRACE_SYSEMU");
         wait_trap(chld);
 
         if (ptrace(PTRACE_GETREGS, chld, 0, &regs) != 0)
@@ -218,7 +229,7 @@ static void test_ptrace_syscall_restart(void)
                 err(1, "PTRACE_SETREGS");
 
         if (ptrace(PTRACE_SYSEMU, chld, 0, 0) != 0)
-                err(1, "PTRACE_SYSCALL");
+                err(1, "PTRACE_SYSEMU");
         wait_trap(chld);
 
         if (ptrace(PTRACE_GETREGS, chld, 0, &regs) != 0)
@@ -250,7 +261,7 @@ static void test_ptrace_syscall_restart(void)
                 err(1, "PTRACE_SETREGS");
 
         if (ptrace(PTRACE_SYSEMU, chld, 0, 0) != 0)
-                err(1, "PTRACE_SYSCALL");
+                err(1, "PTRACE_SYSEMU");
         wait_trap(chld);
 
         if (ptrace(PTRACE_GETREGS, chld, 0, &regs) != 0)
@@ -277,6 +288,119 @@ static void test_ptrace_syscall_restart(void)
         }
 }
 
+static void test_restart_under_ptrace(void)
+{
+        printf("[RUN]\tkernel syscall restart under ptrace\n");
+        pid_t chld = fork();
+        if (chld < 0)
+                err(1, "fork");
+
+        if (chld == 0) {
+                if (ptrace(PTRACE_TRACEME, 0, 0, 0) != 0)
+                        err(1, "PTRACE_TRACEME");
+
+                printf("\tChild will take a nap until signaled\n");
+                setsigign(SIGUSR1, SA_RESTART);
+                raise(SIGSTOP);
+
+                syscall(SYS_pause, 0, 0, 0, 0, 0, 0);
+                _exit(0);
+        }
+
+        int status;
+
+        /* Wait for SIGSTOP. */
+        if (waitpid(chld, &status, 0) != chld || !WIFSTOPPED(status))
+                err(1, "waitpid");
+
+        struct user_regs_struct regs;
+
+        printf("[RUN]\tSYSCALL\n");
+        if (ptrace(PTRACE_SYSCALL, chld, 0, 0) != 0)
+                err(1, "PTRACE_SYSCALL");
+        wait_trap(chld);
+
+        /* We should be stopped at pause(2) entry. */
+
+        if (ptrace(PTRACE_GETREGS, chld, 0, &regs) != 0)
+                err(1, "PTRACE_GETREGS");
+
+        if (regs.user_syscall_nr != SYS_pause ||
+            regs.user_arg0 != 0 || regs.user_arg1 != 0 ||
+            regs.user_arg2 != 0 || regs.user_arg3 != 0 ||
+            regs.user_arg4 != 0 || regs.user_arg5 != 0) {
+                printf("[FAIL]\tInitial args are wrong (nr=%lu, args=%lu %lu %lu %lu %lu %lu)\n", (unsigned long)regs.user_syscall_nr, (unsigned long)regs.user_arg0, (unsigned long)regs.user_arg1, (unsigned long)regs.user_arg2, (unsigned long)regs.user_arg3, (unsigned long)regs.user_arg4, (unsigned long)regs.user_arg5);
+                nerrs++;
+        } else {
+                printf("[OK]\tInitial nr and args are correct\n");
+        }
+
+        /* Interrupt it. */
+        kill(chld, SIGUSR1);
+
+        /* Advance.  We should be stopped at exit. */
+        printf("[RUN]\tSYSCALL\n");
+        if (ptrace(PTRACE_SYSCALL, chld, 0, 0) != 0)
+                err(1, "PTRACE_SYSCALL");
+        wait_trap(chld);
+
+        if (ptrace(PTRACE_GETREGS, chld, 0, &regs) != 0)
+                err(1, "PTRACE_GETREGS");
+
+        if (regs.user_syscall_nr != SYS_pause ||
+            regs.user_arg0 != 0 || regs.user_arg1 != 0 ||
+            regs.user_arg2 != 0 || regs.user_arg3 != 0 ||
+            regs.user_arg4 != 0 || regs.user_arg5 != 0) {
+                printf("[FAIL]\tArgs after SIGUSR1 are wrong (nr=%lu, args=%lu %lu %lu %lu %lu %lu)\n", (unsigned long)regs.user_syscall_nr, (unsigned long)regs.user_arg0, (unsigned long)regs.user_arg1, (unsigned long)regs.user_arg2, (unsigned long)regs.user_arg3, (unsigned long)regs.user_arg4, (unsigned long)regs.user_arg5);
+                nerrs++;
+        } else {
+                printf("[OK]\tArgs after SIGUSR1 are correct (ax = %ld)\n",
+                       (long)regs.user_ax);
+        }
+
+        /* Poke the regs back in.  This must not break anything. */
+        if (ptrace(PTRACE_SETREGS, chld, 0, &regs) != 0)
+                err(1, "PTRACE_SETREGS");
+
+        /* Catch the (ignored) SIGUSR1. */
+        if (ptrace(PTRACE_CONT, chld, 0, 0) != 0)
+                err(1, "PTRACE_CONT");
+        if (waitpid(chld, &status, 0) != chld)
+                err(1, "waitpid");
+        if (!WIFSTOPPED(status)) {
+                printf("[FAIL]\tChild was stopped for SIGUSR1 (status = 0x%x)\n", status);
+                nerrs++;
+        } else {
+                printf("[OK]\tChild got SIGUSR1\n");
+        }
+
+        /* The next event should be pause(2) again. */
+        printf("[RUN]\tStep again\n");
+        if (ptrace(PTRACE_SYSCALL, chld, 0, 0) != 0)
+                err(1, "PTRACE_SYSCALL");
+        wait_trap(chld);
+
+        /* We should be stopped at pause(2) entry. */
+
+        if (ptrace(PTRACE_GETREGS, chld, 0, &regs) != 0)
+                err(1, "PTRACE_GETREGS");
+
+        if (regs.user_syscall_nr != SYS_pause ||
+            regs.user_arg0 != 0 || regs.user_arg1 != 0 ||
+            regs.user_arg2 != 0 || regs.user_arg3 != 0 ||
+            regs.user_arg4 != 0 || regs.user_arg5 != 0) {
+                printf("[FAIL]\tpause did not restart (nr=%lu, args=%lu %lu %lu %lu %lu %lu)\n", (unsigned long)regs.user_syscall_nr, (unsigned long)regs.user_arg0, (unsigned long)regs.user_arg1, (unsigned long)regs.user_arg2, (unsigned long)regs.user_arg3, (unsigned long)regs.user_arg4, (unsigned long)regs.user_arg5);
+                nerrs++;
+        } else {
+                printf("[OK]\tpause(2) restarted correctly\n");
+        }
+
+        /* Kill it. */
+        kill(chld, SIGKILL);
+        if (waitpid(chld, &status, 0) != chld)
+                err(1, "waitpid");
+}
+
 int main()
 {
         printf("[RUN]\tCheck int80 return regs\n");
@@ -290,5 +414,7 @@ int main()
 
         test_ptrace_syscall_restart();
 
+        test_restart_under_ptrace();
+
         return 0;
 }
diff --git a/tools/testing/selftests/x86/sigreturn.c b/tools/testing/selftests/x86/sigreturn.c
index b5aa1bab7416..8a577e7070c6 100644
--- a/tools/testing/selftests/x86/sigreturn.c
+++ b/tools/testing/selftests/x86/sigreturn.c
@@ -54,6 +54,37 @@
 #include <sys/ptrace.h>
 #include <sys/user.h>
 
+/* Pull in AR_xyz defines. */
+typedef unsigned int u32;
+typedef unsigned short u16;
+#include "../../../../arch/x86/include/asm/desc_defs.h"
+
+/*
+ * Copied from asm/ucontext.h, as asm/ucontext.h conflicts badly with the glibc
+ * headers.
+ */
+#ifdef __x86_64__
+/*
+ * UC_SIGCONTEXT_SS will be set when delivering 64-bit or x32 signals on
+ * kernels that save SS in the sigcontext.  All kernels that set
+ * UC_SIGCONTEXT_SS will correctly restore at least the low 32 bits of esp
+ * regardless of SS (i.e. they implement espfix).
+ *
+ * Kernels that set UC_SIGCONTEXT_SS will also set UC_STRICT_RESTORE_SS
+ * when delivering a signal that came from 64-bit code.
+ *
+ * Sigreturn restores SS as follows:
+ *
+ * if (saved SS is valid || UC_STRICT_RESTORE_SS is set ||
+ *     saved CS is not 64-bit)
+ *         new SS = saved SS  (will fail IRET and signal if invalid)
+ * else
+ *         new SS = a flat 32-bit data segment
+ */
+#define UC_SIGCONTEXT_SS       0x2
+#define UC_STRICT_RESTORE_SS   0x4
+#endif
+
 /*
  * In principle, this test can run on Linux emulation layers (e.g.
  * Illumos "LX branded zones").  Solaris-based kernels reserve LDT
@@ -267,6 +298,9 @@ static gregset_t initial_regs, requested_regs, resulting_regs;
 /* Instructions for the SIGUSR1 handler. */
 static volatile unsigned short sig_cs, sig_ss;
 static volatile sig_atomic_t sig_trapped, sig_err, sig_trapno;
+#ifdef __x86_64__
+static volatile sig_atomic_t sig_corrupt_final_ss;
+#endif
 
 /* Abstractions for some 32-bit vs 64-bit differences. */
 #ifdef __x86_64__
@@ -305,9 +339,105 @@ static greg_t *csptr(ucontext_t *ctx)
 }
 #endif
 
+/*
+ * Checks a given selector for its code bitness or returns -1 if it's not
+ * a usable code segment selector.
+ */
+int cs_bitness(unsigned short cs)
+{
+        uint32_t valid = 0, ar;
+        asm ("lar %[cs], %[ar]\n\t"
+             "jnz 1f\n\t"
+             "mov $1, %[valid]\n\t"
+             "1:"
+             : [ar] "=r" (ar), [valid] "+rm" (valid)
+             : [cs] "r" (cs));
+
+        if (!valid)
+                return -1;
+
+        bool db = (ar & (1 << 22));
+        bool l = (ar & (1 << 21));
+
+        if (!(ar & (1<<11)))
+            return -1;  /* Not code. */
+
+        if (l && !db)
+                return 64;
+        else if (!l && db)
+                return 32;
+        else if (!l && !db)
+                return 16;
+        else
+                return -1;      /* Unknown bitness. */
+}
+
+/*
+ * Checks a given selector for its code bitness or returns -1 if it's not
+ * a usable code segment selector.
+ */
+bool is_valid_ss(unsigned short cs)
+{
+        uint32_t valid = 0, ar;
+        asm ("lar %[cs], %[ar]\n\t"
+             "jnz 1f\n\t"
+             "mov $1, %[valid]\n\t"
+             "1:"
+             : [ar] "=r" (ar), [valid] "+rm" (valid)
+             : [cs] "r" (cs));
+
+        if (!valid)
+                return false;
+
+        if ((ar & AR_TYPE_MASK) != AR_TYPE_RWDATA &&
+            (ar & AR_TYPE_MASK) != AR_TYPE_RWDATA_EXPDOWN)
+                return false;
+
+        return (ar & AR_P);
+}
+
 /* Number of errors in the current test case. */
 static volatile sig_atomic_t nerrs;
 
+static void validate_signal_ss(int sig, ucontext_t *ctx)
+{
+#ifdef __x86_64__
+        bool was_64bit = (cs_bitness(*csptr(ctx)) == 64);
+
+        if (!(ctx->uc_flags & UC_SIGCONTEXT_SS)) {
+                printf("[FAIL]\tUC_SIGCONTEXT_SS was not set\n");
+                nerrs++;
+
+                /*
+                 * This happens on Linux 4.1.  The rest will fail, too, so
+                 * return now to reduce the noise.
+                 */
+                return;
+        }
+
+        /* UC_STRICT_RESTORE_SS is set iff we came from 64-bit mode. */
+        if (!!(ctx->uc_flags & UC_STRICT_RESTORE_SS) != was_64bit) {
+                printf("[FAIL]\tUC_STRICT_RESTORE_SS was wrong in signal %d\n",
+                       sig);
+                nerrs++;
+        }
+
+        if (is_valid_ss(*ssptr(ctx))) {
+                /*
+                 * DOSEMU was written before 64-bit sigcontext had SS, and
+                 * it tries to figure out the signal source SS by looking at
+                 * the physical register.  Make sure that keeps working.
+                 */
+                unsigned short hw_ss;
+                asm ("mov %%ss, %0" : "=rm" (hw_ss));
+                if (hw_ss != *ssptr(ctx)) {
+                        printf("[FAIL]\tHW SS didn't match saved SS\n");
+                        nerrs++;
+                }
+        }
+#endif
+}
+
 /*
  * SIGUSR1 handler.  Sets CS and SS as requested and points IP to the
  * int3 trampoline.  Sets SP to a large known value so that we can see
@@ -317,6 +447,8 @@ static void sigusr1(int sig, siginfo_t *info, void *ctx_void)
 {
         ucontext_t *ctx = (ucontext_t*)ctx_void;
 
+        validate_signal_ss(sig, ctx);
+
         memcpy(&initial_regs, &ctx->uc_mcontext.gregs, sizeof(gregset_t));
 
         *csptr(ctx) = sig_cs;
@@ -334,13 +466,16 @@ static void sigusr1(int sig, siginfo_t *info, void *ctx_void)
 }
 
 /*
- * Called after a successful sigreturn.  Restores our state so that
- * the original raise(SIGUSR1) returns.
+ * Called after a successful sigreturn (via int3) or from a failed
+ * sigreturn (directly by kernel).  Restores our state so that the
+ * original raise(SIGUSR1) returns.
  */
 static void sigtrap(int sig, siginfo_t *info, void *ctx_void)
 {
         ucontext_t *ctx = (ucontext_t*)ctx_void;
 
+        validate_signal_ss(sig, ctx);
+
         sig_err = ctx->uc_mcontext.gregs[REG_ERR];
         sig_trapno = ctx->uc_mcontext.gregs[REG_TRAPNO];
 
@@ -358,41 +493,62 @@ static void sigtrap(int sig, siginfo_t *info, void *ctx_void)
         memcpy(&resulting_regs, &ctx->uc_mcontext.gregs, sizeof(gregset_t));
         memcpy(&ctx->uc_mcontext.gregs, &initial_regs, sizeof(gregset_t));
 
+#ifdef __x86_64__
+        if (sig_corrupt_final_ss) {
+                if (ctx->uc_flags & UC_STRICT_RESTORE_SS) {
+                        printf("[FAIL]\tUC_STRICT_RESTORE_SS was set inappropriately\n");
+                        nerrs++;
+                } else {
+                        /*
+                         * DOSEMU transitions from 32-bit to 64-bit mode by
+                         * adjusting sigcontext, and it requires that this work
+                         * even if the saved SS is bogus.
+                         */
+                        printf("\tCorrupting SS on return to 64-bit mode\n");
+                        *ssptr(ctx) = 0;
+                }
+        }
+#endif
+
         sig_trapped = sig;
 }
 
-/*
- * Checks a given selector for its code bitness or returns -1 if it's not
- * a usable code segment selector.
- */
-int cs_bitness(unsigned short cs)
+#ifdef __x86_64__
+/* Tests recovery if !UC_STRICT_RESTORE_SS */
+static void sigusr2(int sig, siginfo_t *info, void *ctx_void)
 {
-        uint32_t valid = 0, ar;
-        asm ("lar %[cs], %[ar]\n\t"
-             "jnz 1f\n\t"
-             "mov $1, %[valid]\n\t"
-             "1:"
-             : [ar] "=r" (ar), [valid] "+rm" (valid)
-             : [cs] "r" (cs));
+        ucontext_t *ctx = (ucontext_t*)ctx_void;
 
-        if (!valid)
-                return -1;
+        if (!(ctx->uc_flags & UC_STRICT_RESTORE_SS)) {
+                printf("[FAIL]\traise(2) didn't set UC_STRICT_RESTORE_SS\n");
+                nerrs++;
+                return;  /* We can't do the rest. */
+        }
 
-        bool db = (ar & (1 << 22));
-        bool l = (ar & (1 << 21));
+        ctx->uc_flags &= ~UC_STRICT_RESTORE_SS;
+        *ssptr(ctx) = 0;
 
-        if (!(ar & (1<<11)))
-            return -1;  /* Not code. */
+        /* Return.  The kernel should recover without sending another signal. */
+}
 
-        if (l && !db)
-                return 64;
-        else if (!l && db)
-                return 32;
-        else if (!l && !db)
-                return 16;
-        else
-                return -1;      /* Unknown bitness. */
+static int test_nonstrict_ss(void)
+{
+        clearhandler(SIGUSR1);
+        clearhandler(SIGTRAP);
+        clearhandler(SIGSEGV);
+        clearhandler(SIGILL);
+        sethandler(SIGUSR2, sigusr2, 0);
+
+        nerrs = 0;
+
+        printf("[RUN]\tClear UC_STRICT_RESTORE_SS and corrupt SS\n");
+        raise(SIGUSR2);
+        if (!nerrs)
+                printf("[OK]\tIt worked\n");
+
+        return nerrs;
 }
+#endif
 
 /* Finds a usable code segment of the requested bitness. */
 int find_cs(int bitness)
@@ -576,6 +732,12 @@ static int test_bad_iret(int cs_bits, unsigned short ss, int force_cs)
                        errdesc, strsignal(sig_trapped));
                 return 0;
         } else {
+                /*
+                 * This also implicitly tests UC_STRICT_RESTORE_SS:
+                 * We check that these signals set UC_STRICT_RESTORE_SS and,
+                 * if UC_STRICT_RESTORE_SS doesn't cause strict behavior,
+                 * then we won't get SIGSEGV.
+                 */
                 printf("[FAIL]\tDid not get SIGSEGV\n");
                 return 1;
         }
@@ -632,6 +794,14 @@ int main()
                                                     GDT3(gdt_data16_idx));
         }
 
+#ifdef __x86_64__
+        /* Nasty ABI case: check SS corruption handling. */
+        sig_corrupt_final_ss = 1;
+        total_nerrs += test_valid_sigreturn(32, false, -1);
+        total_nerrs += test_valid_sigreturn(32, true, -1);
+        sig_corrupt_final_ss = 0;
+#endif
+
         /*
          * We're done testing valid sigreturn cases.  Now we test states
          * for which sigreturn itself will succeed but the subsequent
@@ -680,5 +850,9 @@ int main()
         if (gdt_npdata32_idx)
                 test_bad_iret(32, GDT3(gdt_npdata32_idx), -1);
 
+#ifdef __x86_64__
+        total_nerrs += test_nonstrict_ss();
+#endif
+
         return total_nerrs ? 1 : 0;
 }