37 files changed, 148 insertions, 281 deletions

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index e903c71f7e69..0952ecd60eca 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -26,6 +26,7 @@ config X86
         select HAVE_AOUT if X86_32
         select HAVE_UNSTABLE_SCHED_CLOCK
         select ARCH_SUPPORTS_NUMA_BALANCING
+        select ARCH_SUPPORTS_INT128 if X86_64
         select ARCH_WANTS_PROT_NUMA_PROT_NONE
         select HAVE_IDE
         select HAVE_OPROFILE
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 41250fb33985..57d021507120 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -31,6 +31,9 @@ ifeq ($(CONFIG_X86_32),y)
 
         KBUILD_CFLAGS += -msoft-float -mregparm=3 -freg-struct-return
 
+        # Don't autogenerate MMX or SSE instructions
+        KBUILD_CFLAGS += -mno-mmx -mno-sse
+
         # Never want PIC in a 32-bit kernel, prevent breakage with GCC built
         # with nonstandard options
         KBUILD_CFLAGS += -fno-pic
@@ -57,8 +60,11 @@ else
         KBUILD_AFLAGS += -m64
         KBUILD_CFLAGS += -m64
 
+        # Don't autogenerate MMX or SSE instructions
+        KBUILD_CFLAGS += -mno-mmx -mno-sse
+
         # Use -mpreferred-stack-boundary=3 if supported.
-        KBUILD_CFLAGS += $(call cc-option,-mno-sse -mpreferred-stack-boundary=3)
+        KBUILD_CFLAGS += $(call cc-option,-mpreferred-stack-boundary=3)
 
         # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu)
         cflags-$(CONFIG_MK8) += $(call cc-option,-march=k8)
diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
index dce69a256896..d9c11956fce0 100644
--- a/arch/x86/boot/Makefile
+++ b/arch/x86/boot/Makefile
@@ -53,18 +53,18 @@ $(obj)/cpustr.h: $(obj)/mkcpustr FORCE
 
 # How to compile the 16-bit code.  Note we always compile for -march=i386,
 # that way we can complain to the user if the CPU is insufficient.
-KBUILD_CFLAGS   := $(USERINCLUDE) -g -Os -D_SETUP -D__KERNEL__ \
+KBUILD_CFLAGS   := $(USERINCLUDE) -m32 -g -Os -D_SETUP -D__KERNEL__ \
                    -DDISABLE_BRANCH_PROFILING \
                    -Wall -Wstrict-prototypes \
                    -march=i386 -mregparm=3 \
                    -include $(srctree)/$(src)/code16gcc.h \
                    -fno-strict-aliasing -fomit-frame-pointer -fno-pic \
+                   -mno-mmx -mno-sse \
                    $(call cc-option, -ffreestanding) \
                    $(call cc-option, -fno-toplevel-reorder,\
-                        $(call cc-option, -fno-unit-at-a-time)) \
+                   $(call cc-option, -fno-unit-at-a-time)) \
                    $(call cc-option, -fno-stack-protector) \
                    $(call cc-option, -mpreferred-stack-boundary=2)
-KBUILD_CFLAGS   += $(call cc-option, -m32)
 KBUILD_AFLAGS   := $(KBUILD_CFLAGS) -D__ASSEMBLY__
 GCOV_PROFILE := n
 
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index dcd90df10ab4..c8a6792e7842 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -13,6 +13,7 @@ KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
 cflags-$(CONFIG_X86_32) := -march=i386
 cflags-$(CONFIG_X86_64) := -mcmodel=small
 KBUILD_CFLAGS += $(cflags-y)
+KBUILD_CFLAGS += -mno-mmx -mno-sse
 KBUILD_CFLAGS += $(call cc-option,-ffreestanding)
 KBUILD_CFLAGS += $(call cc-option,-fno-stack-protector)
 
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 7d6ba9db1be9..e0fc24db234a 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -3,8 +3,9 @@
 #
 
 avx_supported := $(call as-instr,vpxor %xmm0$(comma)%xmm0$(comma)%xmm0,yes,no)
+avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\
+                                $(comma)4)$(comma)%ymm2,yes,no)
 
-obj-$(CONFIG_CRYPTO_ABLK_HELPER_X86) += ablk_helper.o
 obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o
 
 obj-$(CONFIG_CRYPTO_AES_586) += aes-i586.o
diff --git a/arch/x86/crypto/ablk_helper.c b/arch/x86/crypto/ablk_helper.c
deleted file mode 100644
index 43282fe04a8b..000000000000
--- a/arch/x86/crypto/ablk_helper.c
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Shared async block cipher helpers
- *
- * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
- *
- * Based on aesni-intel_glue.c by:
- *  Copyright (C) 2008, Intel Corp.
- *    Author: Huang Ying <ying.huang@intel.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
- * USA
- *
- */
-
-#include <linux/kernel.h>
-#include <linux/crypto.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <crypto/algapi.h>
-#include <crypto/cryptd.h>
-#include <asm/i387.h>
-#include <asm/crypto/ablk_helper.h>
-
-int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
-                 unsigned int key_len)
-{
-        struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-        struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
-        int err;
-
-        crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
-        crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
-                                    & CRYPTO_TFM_REQ_MASK);
-        err = crypto_ablkcipher_setkey(child, key, key_len);
-        crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
-                                    & CRYPTO_TFM_RES_MASK);
-        return err;
-}
-EXPORT_SYMBOL_GPL(ablk_set_key);
-
-int __ablk_encrypt(struct ablkcipher_request *req)
-{
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
-        struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-        struct blkcipher_desc desc;
-
-        desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
-        desc.info = req->info;
-        desc.flags = 0;
-
-        return crypto_blkcipher_crt(desc.tfm)->encrypt(
-                &desc, req->dst, req->src, req->nbytes);
-}
-EXPORT_SYMBOL_GPL(__ablk_encrypt);
-
-int ablk_encrypt(struct ablkcipher_request *req)
-{
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
-        struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-
-        if (!irq_fpu_usable()) {
-                struct ablkcipher_request *cryptd_req =
-                        ablkcipher_request_ctx(req);
-
-                memcpy(cryptd_req, req, sizeof(*req));
-                ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
-
-                return crypto_ablkcipher_encrypt(cryptd_req);
-        } else {
-                return __ablk_encrypt(req);
-        }
-}
-EXPORT_SYMBOL_GPL(ablk_encrypt);
-
-int ablk_decrypt(struct ablkcipher_request *req)
-{
-        struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
-        struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-
-        if (!irq_fpu_usable()) {
-                struct ablkcipher_request *cryptd_req =
-                        ablkcipher_request_ctx(req);
-
-                memcpy(cryptd_req, req, sizeof(*req));
-                ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
-
-                return crypto_ablkcipher_decrypt(cryptd_req);
-        } else {
-                struct blkcipher_desc desc;
-
-                desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
-                desc.info = req->info;
-                desc.flags = 0;
-
-                return crypto_blkcipher_crt(desc.tfm)->decrypt(
-                        &desc, req->dst, req->src, req->nbytes);
-        }
-}
-EXPORT_SYMBOL_GPL(ablk_decrypt);
-
-void ablk_exit(struct crypto_tfm *tfm)
-{
-        struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm);
-
-        cryptd_free_ablkcipher(ctx->cryptd_tfm);
-}
-EXPORT_SYMBOL_GPL(ablk_exit);
-
-int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name)
-{
-        struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm);
-        struct cryptd_ablkcipher *cryptd_tfm;
-
-        cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0);
-        if (IS_ERR(cryptd_tfm))
-                return PTR_ERR(cryptd_tfm);
-
-        ctx->cryptd_tfm = cryptd_tfm;
-        tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
-                crypto_ablkcipher_reqsize(&cryptd_tfm->base);
-
-        return 0;
-}
-EXPORT_SYMBOL_GPL(ablk_init_common);
-
-int ablk_init(struct crypto_tfm *tfm)
-{
-        char drv_name[CRYPTO_MAX_ALG_NAME];
-
-        snprintf(drv_name, sizeof(drv_name), "__driver-%s",
-                                        crypto_tfm_alg_driver_name(tfm));
-
-        return ablk_init_common(tfm, drv_name);
-}
-EXPORT_SYMBOL_GPL(ablk_init);
-
-MODULE_LICENSE("GPL");
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index f80e668785c0..835488b745ee 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -34,7 +34,7 @@
 #include <asm/cpu_device_id.h>
 #include <asm/i387.h>
 #include <asm/crypto/aes.h>
-#include <asm/crypto/ablk_helper.h>
+#include <crypto/ablk_helper.h>
 #include <crypto/scatterwalk.h>
 #include <crypto/internal/aead.h>
 #include <linux/workqueue.h>
diff --git a/arch/x86/crypto/camellia_aesni_avx2_glue.c b/arch/x86/crypto/camellia_aesni_avx2_glue.c
index 414fe5d7946b..4209a76fcdaa 100644
--- a/arch/x86/crypto/camellia_aesni_avx2_glue.c
+++ b/arch/x86/crypto/camellia_aesni_avx2_glue.c
@@ -14,6 +14,7 @@
 #include <linux/types.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <crypto/ablk_helper.h>
 #include <crypto/algapi.h>
 #include <crypto/ctr.h>
 #include <crypto/lrw.h>
@@ -21,7 +22,6 @@
 #include <asm/xcr.h>
 #include <asm/xsave.h>
 #include <asm/crypto/camellia.h>
-#include <asm/crypto/ablk_helper.h>
 #include <asm/crypto/glue_helper.h>
 
 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
diff --git a/arch/x86/crypto/camellia_aesni_avx_glue.c b/arch/x86/crypto/camellia_aesni_avx_glue.c
index 37fd0c0a81ea..87a041a10f4a 100644
--- a/arch/x86/crypto/camellia_aesni_avx_glue.c
+++ b/arch/x86/crypto/camellia_aesni_avx_glue.c
@@ -14,6 +14,7 @@
 #include <linux/types.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <crypto/ablk_helper.h>
 #include <crypto/algapi.h>
 #include <crypto/ctr.h>
 #include <crypto/lrw.h>
@@ -21,7 +22,6 @@
 #include <asm/xcr.h>
 #include <asm/xsave.h>
 #include <asm/crypto/camellia.h>
-#include <asm/crypto/ablk_helper.h>
 #include <asm/crypto/glue_helper.h>
 
 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c
index c6631813dc11..e6a3700489b9 100644
--- a/arch/x86/crypto/cast5_avx_glue.c
+++ b/arch/x86/crypto/cast5_avx_glue.c
@@ -26,13 +26,13 @@
 #include <linux/types.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <crypto/ablk_helper.h>
 #include <crypto/algapi.h>
 #include <crypto/cast5.h>
 #include <crypto/cryptd.h>
 #include <crypto/ctr.h>
 #include <asm/xcr.h>
 #include <asm/xsave.h>
-#include <asm/crypto/ablk_helper.h>
 #include <asm/crypto/glue_helper.h>
 
 #define CAST5_PARALLEL_BLOCKS 16
diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c
index 8d0dfb86a559..09f3677393e4 100644
--- a/arch/x86/crypto/cast6_avx_glue.c
+++ b/arch/x86/crypto/cast6_avx_glue.c
@@ -28,6 +28,7 @@
 #include <linux/types.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <crypto/ablk_helper.h>
 #include <crypto/algapi.h>
 #include <crypto/cast6.h>
 #include <crypto/cryptd.h>
@@ -37,7 +38,6 @@
 #include <crypto/xts.h>
 #include <asm/xcr.h>
 #include <asm/xsave.h>
-#include <asm/crypto/ablk_helper.h>
 #include <asm/crypto/glue_helper.h>
 
 #define CAST6_PARALLEL_BLOCKS 8
diff --git a/arch/x86/crypto/serpent_avx2_glue.c b/arch/x86/crypto/serpent_avx2_glue.c
index 23aabc6c20a5..2fae489b1524 100644
--- a/arch/x86/crypto/serpent_avx2_glue.c
+++ b/arch/x86/crypto/serpent_avx2_glue.c
@@ -14,6 +14,7 @@
 #include <linux/types.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <crypto/ablk_helper.h>
 #include <crypto/algapi.h>
 #include <crypto/ctr.h>
 #include <crypto/lrw.h>
@@ -22,7 +23,6 @@
 #include <asm/xcr.h>
 #include <asm/xsave.h>
 #include <asm/crypto/serpent-avx.h>
-#include <asm/crypto/ablk_helper.h>
 #include <asm/crypto/glue_helper.h>
 
 #define SERPENT_AVX2_PARALLEL_BLOCKS 16
diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c
index 9ae83cf8d21e..ff4870870972 100644
--- a/arch/x86/crypto/serpent_avx_glue.c
+++ b/arch/x86/crypto/serpent_avx_glue.c
@@ -28,6 +28,7 @@
 #include <linux/types.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <crypto/ablk_helper.h>
 #include <crypto/algapi.h>
 #include <crypto/serpent.h>
 #include <crypto/cryptd.h>
@@ -38,7 +39,6 @@
 #include <asm/xcr.h>
 #include <asm/xsave.h>
 #include <asm/crypto/serpent-avx.h>
-#include <asm/crypto/ablk_helper.h>
 #include <asm/crypto/glue_helper.h>
 
 /* 8-way parallel cipher functions */
diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c
index 97a356ece24d..8c95f8637306 100644
--- a/arch/x86/crypto/serpent_sse2_glue.c
+++ b/arch/x86/crypto/serpent_sse2_glue.c
@@ -34,6 +34,7 @@
 #include <linux/types.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <crypto/ablk_helper.h>
 #include <crypto/algapi.h>
 #include <crypto/serpent.h>
 #include <crypto/cryptd.h>
@@ -42,7 +43,6 @@
 #include <crypto/lrw.h>
 #include <crypto/xts.h>
 #include <asm/crypto/serpent-sse2.h>
-#include <asm/crypto/ablk_helper.h>
 #include <asm/crypto/glue_helper.h>
 
 static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c
index 50226c4b86ed..f248546da1ca 100644
--- a/arch/x86/crypto/sha256_ssse3_glue.c
+++ b/arch/x86/crypto/sha256_ssse3_glue.c
@@ -281,7 +281,7 @@ static int __init sha256_ssse3_mod_init(void)
         /* allow AVX to override SSSE3, it's a little faster */
         if (avx_usable()) {
 #ifdef CONFIG_AS_AVX2
-                if (boot_cpu_has(X86_FEATURE_AVX2))
+                if (boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_BMI2))
                         sha256_transform_asm = sha256_transform_rorx;
                 else
 #endif
@@ -319,4 +319,4 @@ MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
 
 MODULE_ALIAS("sha256");
-MODULE_ALIAS("sha384");
+MODULE_ALIAS("sha224");
diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c
index a62ba541884e..4e3c665be129 100644
--- a/arch/x86/crypto/twofish_avx_glue.c
+++ b/arch/x86/crypto/twofish_avx_glue.c
@@ -28,6 +28,7 @@
 #include <linux/types.h>
 #include <linux/crypto.h>
 #include <linux/err.h>
+#include <crypto/ablk_helper.h>
 #include <crypto/algapi.h>
 #include <crypto/twofish.h>
 #include <crypto/cryptd.h>
@@ -39,7 +40,6 @@
 #include <asm/xcr.h>
 #include <asm/xsave.h>
 #include <asm/crypto/twofish.h>
-#include <asm/crypto/ablk_helper.h>
 #include <asm/crypto/glue_helper.h>
 #include <crypto/scatterwalk.h>
 #include <linux/workqueue.h>
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index da31c8b8a92d..b17f4f48ecd7 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -77,7 +77,7 @@ static inline void atomic_sub(int i, atomic_t *v)
  */
 static inline int atomic_sub_and_test(int i, atomic_t *v)
 {
-        GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, i, "%0", "e");
+        GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", "e");
 }
 
 /**
@@ -141,7 +141,7 @@ static inline int atomic_inc_and_test(atomic_t *v)
  */
 static inline int atomic_add_negative(int i, atomic_t *v)
 {
-        GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, i, "%0", "s");
+        GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", "s");
 }
 
 /**
diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h
index 3f065c985aee..46e9052bbd28 100644
--- a/arch/x86/include/asm/atomic64_64.h
+++ b/arch/x86/include/asm/atomic64_64.h
@@ -72,7 +72,7 @@ static inline void atomic64_sub(long i, atomic64_t *v)
  */
 static inline int atomic64_sub_and_test(long i, atomic64_t *v)
 {
-        GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, i, "%0", "e");
+        GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, "er", i, "%0", "e");
 }
 
 /**
@@ -138,7 +138,7 @@ static inline int atomic64_inc_and_test(atomic64_t *v)
  */
 static inline int atomic64_add_negative(long i, atomic64_t *v)
 {
-        GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, i, "%0", "s");
+        GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, "er", i, "%0", "s");
 }
 
 /**
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index 6d76d0935989..9fc1af74dc83 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -205,7 +205,7 @@ static inline void change_bit(long nr, volatile unsigned long *addr)
  */
 static inline int test_and_set_bit(long nr, volatile unsigned long *addr)
 {
-        GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, nr, "%0", "c");
+        GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", "c");
 }
 
 /**
@@ -251,7 +251,7 @@ static inline int __test_and_set_bit(long nr, volatile unsigned long *addr)
  */
 static inline int test_and_clear_bit(long nr, volatile unsigned long *addr)
 {
-        GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, nr, "%0", "c");
+        GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", "c");
 }
 
 /**
@@ -304,7 +304,7 @@ static inline int __test_and_change_bit(long nr, volatile unsigned long *addr)
  */
 static inline int test_and_change_bit(long nr, volatile unsigned long *addr)
 {
-        GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, nr, "%0", "c");
+        GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", "c");
 }
 
 static __always_inline int constant_test_bit(long nr, const volatile unsigned long *addr)
diff --git a/arch/x86/include/asm/crypto/ablk_helper.h b/arch/x86/include/asm/crypto/ablk_helper.h
deleted file mode 100644
index 4f93df50c23e..000000000000
--- a/arch/x86/include/asm/crypto/ablk_helper.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Shared async block cipher helpers
- */
-
-#ifndef _CRYPTO_ABLK_HELPER_H
-#define _CRYPTO_ABLK_HELPER_H
-
-#include <linux/crypto.h>
-#include <linux/kernel.h>
-#include <crypto/cryptd.h>
-
-struct async_helper_ctx {
-        struct cryptd_ablkcipher *cryptd_tfm;
-};
-
-extern int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
-                        unsigned int key_len);
-
-extern int __ablk_encrypt(struct ablkcipher_request *req);
-
-extern int ablk_encrypt(struct ablkcipher_request *req);
-
-extern int ablk_decrypt(struct ablkcipher_request *req);
-
-extern void ablk_exit(struct crypto_tfm *tfm);
-
-extern int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name);
-
-extern int ablk_init(struct crypto_tfm *tfm);
-
-#endif /* _CRYPTO_ABLK_HELPER_H */
diff --git a/arch/x86/include/asm/local.h b/arch/x86/include/asm/local.h
index 5b23e605e707..4ad6560847b1 100644
--- a/arch/x86/include/asm/local.h
+++ b/arch/x86/include/asm/local.h
@@ -52,7 +52,7 @@ static inline void local_sub(long i, local_t *l)
  */
 static inline int local_sub_and_test(long i, local_t *l)
 {
-        GEN_BINARY_RMWcc(_ASM_SUB, l->a.counter, i, "%0", "e");
+        GEN_BINARY_RMWcc(_ASM_SUB, l->a.counter, "er", i, "%0", "e");
 }
 
 /**
@@ -92,7 +92,7 @@ static inline int local_inc_and_test(local_t *l)
  */
 static inline int local_add_negative(long i, local_t *l)
 {
-        GEN_BINARY_RMWcc(_ASM_ADD, l->a.counter, i, "%0", "s");
+        GEN_BINARY_RMWcc(_ASM_ADD, l->a.counter, "er", i, "%0", "s");
 }
 
 /**
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 3d1999458709..bbc8b12fa443 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -452,9 +452,16 @@ static inline int pte_present(pte_t a)
 }
 
 #define pte_accessible pte_accessible
-static inline int pte_accessible(pte_t a)
+static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
 {
-        return pte_flags(a) & _PAGE_PRESENT;
+        if (pte_flags(a) & _PAGE_PRESENT)
+                return true;
+
+        if ((pte_flags(a) & (_PAGE_PROTNONE | _PAGE_NUMA)) &&
+                        mm_tlb_flush_pending(mm))
+                return true;
+
+        return false;
 }
 
 static inline int pte_hidden(pte_t pte)
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 8729723636fd..c8b051933b1b 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -8,6 +8,12 @@
 DECLARE_PER_CPU(int, __preempt_count);
 
 /*
+ * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
+ * that a decrement hitting 0 means we can and should reschedule.
+ */
+#define PREEMPT_ENABLED (0 + PREEMPT_NEED_RESCHED)
+
+/*
  * We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users
  * that think a non-zero value indicates we cannot preempt.
  */
@@ -74,6 +80,11 @@ static __always_inline void __preempt_count_sub(int val)
         __this_cpu_add_4(__preempt_count, -val);
 }
 
+/*
+ * Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule
+ * a decrement which hits zero means we have no preempt_count and should
+ * reschedule.
+ */
 static __always_inline bool __preempt_count_dec_and_test(void)
 {
         GEN_UNARY_RMWcc("decl", __preempt_count, __percpu_arg(0), "e");
diff --git a/arch/x86/include/asm/rmwcc.h b/arch/x86/include/asm/rmwcc.h
index 1ff990f1de8e..8f7866a5b9a4 100644
--- a/arch/x86/include/asm/rmwcc.h
+++ b/arch/x86/include/asm/rmwcc.h
@@ -16,8 +16,8 @@ cc_label:								\
 #define GEN_UNARY_RMWcc(op, var, arg0, cc)                              \
         __GEN_RMWcc(op " " arg0, var, cc)
 
-#define GEN_BINARY_RMWcc(op, var, val, arg0, cc)                        \
-        __GEN_RMWcc(op " %1, " arg0, var, cc, "er" (val))
+#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc)                  \
+        __GEN_RMWcc(op " %1, " arg0, var, cc, vcon (val))
 
 #else /* !CC_HAVE_ASM_GOTO */
 
@@ -33,8 +33,8 @@ do {									\
 #define GEN_UNARY_RMWcc(op, var, arg0, cc)                              \
         __GEN_RMWcc(op " " arg0, var, cc)
 
-#define GEN_BINARY_RMWcc(op, var, val, arg0, cc)                        \
-        __GEN_RMWcc(op " %2, " arg0, var, cc, "er" (val))
+#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc)                  \
+        __GEN_RMWcc(op " %2, " arg0, var, cc, vcon (val))
 
 #endif /* CC_HAVE_ASM_GOTO */
 
diff --git a/arch/x86/include/asm/simd.h b/arch/x86/include/asm/simd.h
new file mode 100644
index 000000000000..ee80b92f0096
--- /dev/null
+++ b/arch/x86/include/asm/simd.h
@@ -0,0 +1,11 @@
+
+#include <asm/i387.h>
+
+/*
+ * may_use_simd - whether it is allowable at this time to issue SIMD
+ *                instructions or access the SIMD register file
+ */
+static __must_check inline bool may_use_simd(void)
+{
+        return irq_fpu_usable();
+}
diff --git a/arch/x86/include/asm/trace/irq_vectors.h b/arch/x86/include/asm/trace/irq_vectors.h
index 2874df24e7a4..4cab890007a7 100644
--- a/arch/x86/include/asm/trace/irq_vectors.h
+++ b/arch/x86/include/asm/trace/irq_vectors.h
@@ -72,6 +72,17 @@ DEFINE_IRQ_VECTOR_EVENT(x86_platform_ipi);
 DEFINE_IRQ_VECTOR_EVENT(irq_work);
 
 /*
+ * We must dis-allow sampling irq_work_exit() because perf event sampling
+ * itself can cause irq_work, which would lead to an infinite loop;
+ *
+ *  1) irq_work_exit happens
+ *  2) generates perf sample
+ *  3) generates irq_work
+ *  4) goto 1
+ */
+TRACE_EVENT_PERF_PERM(irq_work_exit, is_sampling_event(p_event) ? -EPERM : 0);
+
+/*
  * call_function - called when entering/exiting a call function interrupt
  * vector handler
  */
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index dc1ec0dff939..ea04b342c026 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -387,7 +387,8 @@ static void init_intel(struct cpuinfo_x86 *c)
                         set_cpu_cap(c, X86_FEATURE_PEBS);
         }
 
-        if (c->x86 == 6 && c->x86_model == 29 && cpu_has_clflush)
+        if (c->x86 == 6 && cpu_has_clflush &&
+            (c->x86_model == 29 || c->x86_model == 46 || c->x86_model == 47))
                 set_cpu_cap(c, X86_FEATURE_CLFLUSH_MONITOR);
 
 #ifdef CONFIG_X86_64
diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index fd00bb29425d..c1a861829d81 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -262,11 +262,20 @@ struct cpu_hw_events {
         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \
                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
 
-#define EVENT_CONSTRAINT_END            \
-        EVENT_CONSTRAINT(0, 0, 0)
+/*
+ * We define the end marker as having a weight of -1
+ * to enable blacklisting of events using a counter bitmask
+ * of zero and thus a weight of zero.
+ * The end marker has a weight that cannot possibly be
+ * obtained from counting the bits in the bitmask.
+ */
+#define EVENT_CONSTRAINT_END { .weight = -1 }
 
+/*
+ * Check for end marker with weight == -1
+ */
 #define for_each_event_constraint(e, c) \
-        for ((e) = (c); (e)->weight; (e)++)
+        for ((e) = (c); (e)->weight != -1; (e)++)
 
 /*
  * Extra registers for specific events.
diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c
index da3c599584a3..c752cb43e52f 100644
--- a/arch/x86/kernel/reboot.c
+++ b/arch/x86/kernel/reboot.c
@@ -558,6 +558,17 @@ void native_machine_shutdown(void)
 {
         /* Stop the cpus and apics */
 #ifdef CONFIG_X86_IO_APIC
+        /*
+         * Disabling IO APIC before local APIC is a workaround for
+         * erratum AVR31 in "Intel Atom Processor C2000 Product Family
+         * Specification Update". In this situation, interrupts that target
+         * a Logical Processor whose Local APIC is either in the process of
+         * being hardware disabled or software disabled are neither delivered
+         * nor discarded. When this erratum occurs, the processor may hang.
+         *
+         * Even without the erratum, it still makes sense to quiet IO APIC
+         * before disabling Local APIC.
+         */
         disable_IO_APIC();
 #endif
 
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 5439117d5c4c..dec48bfaddb8 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -143,6 +143,8 @@ static inline int kvm_apic_id(struct kvm_lapic *apic)
         return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
 }
 
+#define KVM_X2APIC_CID_BITS 0
+
 static void recalculate_apic_map(struct kvm *kvm)
 {
         struct kvm_apic_map *new, *old = NULL;
@@ -180,7 +182,8 @@ static void recalculate_apic_map(struct kvm *kvm)
                 if (apic_x2apic_mode(apic)) {
                         new->ldr_bits = 32;
                         new->cid_shift = 16;
-                        new->cid_mask = new->lid_mask = 0xffff;
+                        new->cid_mask = (1 << KVM_X2APIC_CID_BITS) - 1;
+                        new->lid_mask = 0xffff;
                 } else if (kvm_apic_sw_enabled(apic) &&
                                 !new->cid_mask /* flat mode */ &&
                                 kvm_apic_get_reg(apic, APIC_DFR) == APIC_DFR_CLUSTER) {
@@ -841,7 +844,8 @@ static u32 apic_get_tmcct(struct kvm_lapic *apic)
         ASSERT(apic != NULL);
 
         /* if initial count is 0, current count should also be 0 */
-        if (kvm_apic_get_reg(apic, APIC_TMICT) == 0)
+        if (kvm_apic_get_reg(apic, APIC_TMICT) == 0 ||
+                apic->lapic_timer.period == 0)
                 return 0;
 
         remaining = hrtimer_get_remaining(&apic->lapic_timer.timer);
@@ -1691,7 +1695,6 @@ static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
 {
         u32 data;
-        void *vapic;
 
         if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
                 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
@@ -1699,9 +1702,8 @@ void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
                 return;
 
-        vapic = kmap_atomic(vcpu->arch.apic->vapic_page);
-        data = *(u32 *)(vapic + offset_in_page(vcpu->arch.apic->vapic_addr));
-        kunmap_atomic(vapic);
+        kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
+                                sizeof(u32));
 
         apic_set_tpr(vcpu->arch.apic, data & 0xff);
 }
@@ -1737,7 +1739,6 @@ void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
         u32 data, tpr;
         int max_irr, max_isr;
         struct kvm_lapic *apic = vcpu->arch.apic;
-        void *vapic;
 
         apic_sync_pv_eoi_to_guest(vcpu, apic);
 
@@ -1753,18 +1754,24 @@ void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
                 max_isr = 0;
         data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
 
-        vapic = kmap_atomic(vcpu->arch.apic->vapic_page);
-        *(u32 *)(vapic + offset_in_page(vcpu->arch.apic->vapic_addr)) = data;
-        kunmap_atomic(vapic);
+        kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
+                                sizeof(u32));
 }
 
-void kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
+int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
 {
-        vcpu->arch.apic->vapic_addr = vapic_addr;
-        if (vapic_addr)
+        if (vapic_addr) {
+                if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
+                                        &vcpu->arch.apic->vapic_cache,
+                                        vapic_addr, sizeof(u32)))
+                        return -EINVAL;
                 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
-        else
+        } else {
                 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
+        }
+
+        vcpu->arch.apic->vapic_addr = vapic_addr;
+        return 0;
 }
 
 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index c730ac9fe801..c8b0d0d2da5c 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -34,7 +34,7 @@ struct kvm_lapic {
          */
         void *regs;
         gpa_t vapic_addr;
-        struct page *vapic_page;
+        struct gfn_to_hva_cache vapic_cache;
         unsigned long pending_events;
         unsigned int sipi_vector;
 };
@@ -76,7 +76,7 @@ void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data);
 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset);
 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector);
 
-void kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr);
+int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr);
 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu);
 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu);
 
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 21ef1ba184ae..5d004da1e35d 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -3214,8 +3214,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
                 r = -EFAULT;
                 if (copy_from_user(&va, argp, sizeof va))
                         goto out;
-                r = 0;
-                kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
+                r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
                 break;
         }
         case KVM_X86_SETUP_MCE: {
@@ -5739,36 +5738,6 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu)
                         !kvm_event_needs_reinjection(vcpu);
 }
 
-static int vapic_enter(struct kvm_vcpu *vcpu)
-{
-        struct kvm_lapic *apic = vcpu->arch.apic;
-        struct page *page;
-
-        if (!apic || !apic->vapic_addr)
-                return 0;
-
-        page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
-        if (is_error_page(page))
-                return -EFAULT;
-
-        vcpu->arch.apic->vapic_page = page;
-        return 0;
-}
-
-static void vapic_exit(struct kvm_vcpu *vcpu)
-{
-        struct kvm_lapic *apic = vcpu->arch.apic;
-        int idx;
-
-        if (!apic || !apic->vapic_addr)
-                return;
-
-        idx = srcu_read_lock(&vcpu->kvm->srcu);
-        kvm_release_page_dirty(apic->vapic_page);
-        mark_page_dirty(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
-        srcu_read_unlock(&vcpu->kvm->srcu, idx);
-}
-
 static void update_cr8_intercept(struct kvm_vcpu *vcpu)
 {
         int max_irr, tpr;
@@ -6069,11 +6038,6 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
         struct kvm *kvm = vcpu->kvm;
 
         vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
-        r = vapic_enter(vcpu);
-        if (r) {
-                srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
-                return r;
-        }
 
         r = 1;
         while (r > 0) {
@@ -6132,8 +6096,6 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
 
         srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
 
-        vapic_exit(vcpu);
-
         return r;
 }
 
diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c
index dd74e46828c0..0596e8e0cc19 100644
--- a/arch/x86/mm/gup.c
+++ b/arch/x86/mm/gup.c
@@ -83,6 +83,12 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
                 pte_t pte = gup_get_pte(ptep);
                 struct page *page;
 
+                /* Similar to the PMD case, NUMA hinting must take slow path */
+                if (pte_numa(pte)) {
+                        pte_unmap(ptep);
+                        return 0;
+                }
+
                 if ((pte_flags(pte) & (mask | _PAGE_SPECIAL)) != mask) {
                         pte_unmap(ptep);
                         return 0;
@@ -167,6 +173,13 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
                 if (pmd_none(pmd) || pmd_trans_splitting(pmd))
                         return 0;
                 if (unlikely(pmd_large(pmd))) {
+                        /*
+                         * NUMA hinting faults need to be handled in the GUP
+                         * slowpath for accounting purposes and so that they
+                         * can be serialised against THP migration.
+                         */
+                        if (pmd_numa(pmd))
+                                return 0;
                         if (!gup_huge_pmd(pmd, addr, next, write, pages, nr))
                                 return 0;
                 } else {
diff --git a/arch/x86/platform/efi/early_printk.c b/arch/x86/platform/efi/early_printk.c
index 6599a0027b76..81b506d5befd 100644
--- a/arch/x86/platform/efi/early_printk.c
+++ b/arch/x86/platform/efi/early_printk.c
@@ -142,7 +142,7 @@ early_efi_write(struct console *con, const char *str, unsigned int num)
                         efi_y += font->height;
                 }
 
-                if (efi_y + font->height >= si->lfb_height) {
+                if (efi_y + font->height > si->lfb_height) {
                         u32 i;
 
                         efi_y -= font->height;
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index 92c02344a060..cceb813044ef 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -690,13 +690,6 @@ void __init efi_init(void)
 
         set_bit(EFI_MEMMAP, &x86_efi_facility);
 
-#ifdef CONFIG_X86_32
-        if (efi_is_native()) {
-                x86_platform.get_wallclock = efi_get_time;
-                x86_platform.set_wallclock = efi_set_rtc_mmss;
-        }
-#endif
-
 #if EFI_DEBUG
         print_efi_memmap();
 #endif
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 0f92173a12b6..efe4d7220397 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -1070,12 +1070,13 @@ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
         unsigned long status;
 
         bcp = &per_cpu(bau_control, cpu);
-        stat = bcp->statp;
-        stat->s_enters++;
 
         if (bcp->nobau)
                 return cpumask;
 
+        stat = bcp->statp;
+        stat->s_enters++;
+
         if (bcp->busy) {
                 descriptor_status =
                         read_lmmr(UVH_LB_BAU_SB_ACTIVATION_STATUS_0);
diff --git a/arch/x86/realmode/rm/Makefile b/arch/x86/realmode/rm/Makefile
index 88692871823f..9cac82588cbc 100644
--- a/arch/x86/realmode/rm/Makefile
+++ b/arch/x86/realmode/rm/Makefile
@@ -73,9 +73,10 @@ KBUILD_CFLAGS	:= $(LINUXINCLUDE) -m32 -g -Os -D_SETUP -D__KERNEL__ -D_WAKEUP \
                    -march=i386 -mregparm=3 \
                    -include $(srctree)/$(src)/../../boot/code16gcc.h \
                    -fno-strict-aliasing -fomit-frame-pointer -fno-pic \
+                   -mno-mmx -mno-sse \
                    $(call cc-option, -ffreestanding) \
                    $(call cc-option, -fno-toplevel-reorder,\
-                        $(call cc-option, -fno-unit-at-a-time)) \
+                   $(call cc-option, -fno-unit-at-a-time)) \
                    $(call cc-option, -fno-stack-protector) \
                    $(call cc-option, -mpreferred-stack-boundary=2)
 KBUILD_AFLAGS   := $(KBUILD_CFLAGS) -D__ASSEMBLY__